Split big sitemaps & generate proper index (closes #79, #106)

stefandoorn · Aug 24, 2020 · 24edb86 · 24edb86
1 parent 7851ecf
commit 24edb86
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 28 deletions.
diff --git a/src/Builder/SitemapIndexBuilder.php b/src/Builder/SitemapIndexBuilder.php
@@ -20,6 +20,9 @@ final class SitemapIndexBuilder implements SitemapIndexBuilderInterface
     /** @var array */
     private $indexProviders = [];
 
+    /** @var array */
+    private $paths = [];
+
     public function __construct(SitemapIndexFactoryInterface $sitemapIndexFactory)
     {
         $this->sitemapIndexFactory = $sitemapIndexFactory;
@@ -41,6 +44,18 @@ public function addIndexProvider(IndexUrlProviderInterface $provider): void
         $this->indexProviders[] = $provider;
     }
 
+    /**
+     * {@inheritdoc}
+     */
+    public function addPath(UrlProviderInterface $provider, string $path): void
+    {
+        if (!array_key_exists($provider->getName(), $this->paths)) {
+            $this->paths[$provider->getName()] = [];
+        }
+
+        $this->paths[$provider->getName()][] = $path;
+    }
+
     /**
      * {@inheritdoc}
      */
@@ -53,7 +68,7 @@ public function build(): SitemapInterface
         foreach ($this->indexProviders as $indexProvider) {
             /** @var UrlProviderInterface $provider */
             foreach ($this->providers as $provider) {
-                $indexProvider->addProvider($provider);
+                $indexProvider->addProvider($provider, $this->paths[$provider->getName()]);
             }
 
             $urls[] = $indexProvider->generate();

diff --git a/src/Command/GenerateSitemapCommand.php b/src/Command/GenerateSitemapCommand.php
@@ -56,43 +56,50 @@ public function __construct(
     protected function configure(): void
     {
         $this->addOption('channel', 'c', InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL, 'Channel codes to generate. If none supplied, all channels will generated.');
+        $this->addOption('limit', 'l', InputOption::VALUE_OPTIONAL, 'Limit amount of URLs per sitemap', 50000);
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
         foreach ($this->channels($input) as $channel) {
-            $this->executeChannel($channel, $output);
+            $this->executeChannel($channel, $input, $output);
         }
     }
 
-    private function executeChannel(ChannelInterface $channel, OutputInterface $output)
+    private function executeChannel(ChannelInterface $channel, InputInterface $input, OutputInterface $output)
     {
-        // TODO make sure providers are every time emptied (reset call or smth?)
         foreach ($this->sitemapBuilder->getProviders() as $provider) {
             $output->writeln(\sprintf('Start generating sitemap "%s" for channel "%s"', $provider->getName(), $channel->getCode()));
 
             $sitemap = $this->sitemapBuilder->build($provider, $channel); // TODO use provider instance, not the name
-            $xml = $this->sitemapRenderer->render($sitemap);
-            $path = $path = $this->path($channel, \sprintf('%s.xml', $provider->getName()));
+            $xml = $this->sitemapRenderer->render($sitemap, (int)$input->getOption('limit'));
+            foreach($xml as $index => $data) {
+                $path = $path = $this->path($channel, \sprintf('%s_%d.xml', $provider->getName(), $index));
 
-            $this->writer->write(
-                $path,
-                $xml
-            );
+                $this->writer->write(
+                    $path,
+                    $data
+                );
 
-            $output->writeln(\sprintf('Finished generating sitemap "%s" for channel "%s" at path "%s"', $provider->getName(), $channel->getCode(), $path));
+                $output->writeln(\sprintf('Finished generating sitemap "%s" (%d) for channel "%s" at path "%s"', $provider->getName(), $index, $channel->getCode(), $path));
+
+                $this->sitemapIndexBuilder->addPath($provider, $path);
+            }
         }
 
         $output->writeln(\sprintf('Start generating sitemap index for channel "%s"', $channel->getCode()));
 
         $sitemap = $this->sitemapIndexBuilder->build();
         $xml = $this->sitemapIndexRenderer->render($sitemap);
-        $path = $this->path($channel, 'sitemap_index.xml');
 
-        $this->writer->write(
-            $path,
-            $xml
-        );
+        foreach($xml as $index => $data) {
+            $path = $this->path($channel, 'sitemap_index.xml');
+
+            $this->writer->write(
+                $path,
+                $data
+            );
+        }
 
         $output->writeln(\sprintf('Finished generating sitemap index for channel "%s" at path "%s"', $channel->getCode(), $path));
     }

diff --git a/src/Controller/SitemapController.php b/src/Controller/SitemapController.php
@@ -22,9 +22,9 @@ public function __construct(
         parent::__construct($reader);
     }
 
-    public function showAction(string $name): Response
+    public function showAction(string $name, int $index): Response
     {
-        $path = \sprintf('%s/%s', $this->channelContext->getChannel()->getCode(), \sprintf('%s.xml', $name));
+        $path = \sprintf('%s/%s', $this->channelContext->getChannel()->getCode(), \sprintf('%s_%d.xml', $name, $index));
 
         return $this->createResponse($path);
     }

diff --git a/src/Provider/IndexUrlProvider.php b/src/Provider/IndexUrlProvider.php
@@ -21,6 +21,9 @@ final class IndexUrlProvider implements IndexUrlProviderInterface
     /** @var array */
     private $urls = [];
 
+    /** @var array */
+    private $paths = [];
+
     public function __construct(
         RouterInterface $router,
         IndexUrlFactoryInterface $sitemapIndexUrlFactory
@@ -29,17 +32,24 @@ public function __construct(
         $this->sitemapIndexUrlFactory = $sitemapIndexUrlFactory;
     }
 
-    public function addProvider(UrlProviderInterface $provider): void
+    public function addProvider(UrlProviderInterface $provider, array $paths = []): void
     {
         $this->providers[] = $provider;
+        $this->paths[$provider->getName()] = $paths;
     }
 
     public function generate(): iterable
     {
         foreach ($this->providers as $provider) {
-            $location = $this->router->generate('sylius_sitemap_' . $provider->getName());
+            $pathCount = count($this->paths[$provider->getName()]);
+
+            for ($i = 0; $i < $pathCount; $i++) {
+                $params = ['index' => $i];
+
+                $location = $this->router->generate('sylius_sitemap_'.$provider->getName(), $params);
 
-            $this->urls[] = $this->sitemapIndexUrlFactory->createNew($location);
+                $this->urls[] = $this->sitemapIndexUrlFactory->createNew($location);
+            }
         }
 
         return $this->urls;

diff --git a/src/Provider/IndexUrlProviderInterface.php b/src/Provider/IndexUrlProviderInterface.php
@@ -11,5 +11,5 @@ interface IndexUrlProviderInterface
      */
     public function generate(): iterable;
 
-    public function addProvider(UrlProviderInterface $provider): void;
+    public function addProvider(UrlProviderInterface $provider, array $paths = []): void;
 }
diff --git a/src/Renderer/SitemapRenderer.php b/src/Renderer/SitemapRenderer.php
@@ -19,8 +19,19 @@ public function __construct(RendererAdapterInterface $adapter)
     /**
      * {@inheritdoc}
      */
-    public function render(SitemapInterface $sitemap): string
+    public function render(SitemapInterface $sitemap, ?int $limit = null): iterable
     {
-        return $this->adapter->render($sitemap);
+        $urls = $sitemap->getUrls();
+        $total = count($urls);
+
+        if (null === $limit || $limit < 0) {
+            $limit = $total;
+        }
+
+        foreach(array_chunk($urls, $limit) as $slice) {
+            $sitemap->setUrls($slice);
+
+            yield $this->adapter->render($sitemap);
+        }
     }
 }
diff --git a/src/Renderer/SitemapRendererInterface.php b/src/Renderer/SitemapRendererInterface.php
@@ -8,5 +8,9 @@
 
 interface SitemapRendererInterface
 {
-    public function render(SitemapInterface $sitemap): string;
+
+    /**
+     * @return string[]
+     */
+    public function render(SitemapInterface $sitemap, ?int $limit = null): iterable;
 }
diff --git a/src/Routing/SitemapLoader.php b/src/Routing/SitemapLoader.php
@@ -34,7 +34,7 @@ public function __construct(SitemapBuilderInterface $sitemapBuilder)
     public function load($resource, $type = null): RouteCollection
     {
         $routes = new RouteCollection();
-        
+
         if (true === $this->loaded) {
             return $routes;
         }
@@ -50,12 +50,14 @@ public function load($resource, $type = null): RouteCollection
             $routes->add(
                 $name,
                 new Route(
-                    '/sitemap/' . $provider->getName() . '.xml',
+                    '/sitemap/' . $provider->getName() . '/{index}.xml',
                     [
                         '_controller' => 'sylius.controller.sitemap:showAction',
                         'name' => $provider->getName(),
                     ],
-                    [],
+                    [
+                        'index' => '\d+',
+                    ],
                     [],
                     '',
                     [],