Support paginated sitemaps (#344)

* Support paginated sitemaps I also refactored the publishedEntries querying to be more performant * Not paginated by default * update comment * 100 not 25 * 🍺 * Nit. * Improve config structure for consistency across rest of config. * Rename route param to `page`, and simplify loop logic. * Split large controller method into two (one for sitemap.xml, and one for paginated xml’s). * Nit. * Kick test suite. --------- Co-authored-by: Jesse Leite <[email protected]>
statamic · Sep 5, 2024 · 948d583 · 948d583
1 parent 740033f
commit 948d583
Show file tree

Hide file tree

Showing 8 changed files with 288 additions and 24 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,4 @@ composer.lock
 /vendor
 .phpunit.result.cache
 .phpunit.cache
+.idea/
diff --git a/composer.json b/composer.json
@@ -26,7 +26,8 @@
     },
     "require-dev": {
         "orchestra/testbench": "^8.0 || ^9.0",
-        "phpunit/phpunit": "^10.0"
+        "phpunit/phpunit": "^10.0",
+        "laravel/pint": "^1.17"
     },
     "config": {
         "allow-plugins": {

diff --git a/config/seo-pro.php b/config/seo-pro.php
@@ -22,6 +22,11 @@
         'enabled' => true,
         'url' => 'sitemap.xml',
         'expire' => 60,
+        'pagination' => [
+            'enabled' => false,
+            'url' => 'sitemap_{page}.xml',
+            'limit' => 100,
+        ],
     ],
 
     'humans' => [

diff --git a/resources/views/generated/sitemap_index.antlers.html b/resources/views/generated/sitemap_index.antlers.html
@@ -0,0 +1,8 @@
+{{ xml_header }}
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+{{ sitemaps }}
+    <sitemap>
+        <loc>{{ url }}</loc>
+    </sitemap>
+{{ /sitemaps }}
+</sitemapindex>
diff --git a/routes/web.php b/routes/web.php
@@ -2,5 +2,6 @@
 
 use Statamic\SeoPro\Http\Controllers;
 
-Route::get(config('statamic.seo-pro.sitemap.url'), [Controllers\SitemapController::class, 'show']);
+Route::get(config('statamic.seo-pro.sitemap.url'), [Controllers\SitemapController::class, 'index']);
+Route::get(config('statamic.seo-pro.sitemap.pagination.url'), [Controllers\SitemapController::class, 'show'])->name('statamic.seo-pro.sitemap.page.show');
 Route::get(config('statamic.seo-pro.humans.url'), [Controllers\HumansController::class, 'show']);
diff --git a/src/Http/Controllers/SitemapController.php b/src/Http/Controllers/SitemapController.php
@@ -9,16 +9,47 @@
 
 class SitemapController extends Controller
 {
-    public function show()
+    public function index()
     {
         abort_unless(config('statamic.seo-pro.sitemap.enabled'), 404);
 
         $cacheUntil = Carbon::now()->addMinutes(config('statamic.seo-pro.sitemap.expire'));
 
-        $content = Cache::remember(Sitemap::CACHE_KEY, $cacheUntil, function () {
+        if (config('statamic.seo-pro.sitemap.pagination.enabled', false)) {
+            $content = Cache::remember(Sitemap::CACHE_KEY.'_index', $cacheUntil, function () {
+               return view('seo-pro::sitemap_index', [
+                   'xml_header' => '<?xml version="1.0" encoding="UTF-8"?>',
+                   'sitemaps' => Sitemap::paginatedSitemaps(),
+               ])->render();
+           });
+        } else {
+            $content = Cache::remember(Sitemap::CACHE_KEY, $cacheUntil, function () {
+                return view('seo-pro::sitemap', [
+                    'xml_header' => '<?xml version="1.0" encoding="UTF-8"?>',
+                    'pages' => Sitemap::pages(),
+                ])->render();
+            });
+        }
+
+        return response($content)->header('Content-Type', 'text/xml');
+    }
+
+    public function show($page)
+    {
+        abort_unless(config('statamic.seo-pro.sitemap.enabled'), 404);
+        abort_unless(config('statamic.seo-pro.sitemap.pagination.enabled'), 404);
+        abort_unless(filter_var($page, FILTER_VALIDATE_INT), 404);
+
+        $cacheUntil = Carbon::now()->addMinutes(config('statamic.seo-pro.sitemap.expire'));
+
+        $cacheKey = Sitemap::CACHE_KEY.'_'.$page;
+
+        $content = Cache::remember($cacheKey, $cacheUntil, function () use ($page) {
+            abort_if(empty($pages = Sitemap::paginatedPages($page)), 404);
+
             return view('seo-pro::sitemap', [
-                'pages' => Sitemap::pages(),
                 'xml_header' => '<?xml version="1.0" encoding="UTF-8"?>',
+                'pages' => $pages,
             ])->render();
         });
 

diff --git a/src/Sitemap/Sitemap.php b/src/Sitemap/Sitemap.php
@@ -4,6 +4,7 @@
 
 use Statamic\Facades\Blink;
 use Statamic\Facades\Collection;
+use Statamic\Facades\Entry;
 use Statamic\Facades\Taxonomy;
 use Statamic\SeoPro\Cascade;
 use Statamic\SeoPro\GetsSectionDefaults;
@@ -31,6 +32,63 @@ public static function pages()
             ->toArray();
     }
 
+    public static function paginatedPages(int $page)
+    {
+        $sitemap = new static;
+
+        $perPage = config('statamic.seo-pro.sitemap.pagination.limit', 100);
+        $offset = ($page - 1) * $perPage;
+        $remaining = $perPage;
+
+        $pages = collect([]);
+
+        $entryCount = $sitemap->publishedEntriesCount() - 1;
+
+        if ($offset < $entryCount) {
+            $entries = $sitemap->publishedEntries()->skip($offset)->take($perPage);
+
+            if ($entries->count() < $remaining) {
+                $remaining -= $entries->count();
+            }
+
+            $pages = $pages->merge($entries);
+        }
+
+        if ($remaining > 0) {
+            $offset = max($offset - $entryCount, 0);
+
+            $pages = $pages->merge(
+                collect($sitemap->publishedTerms())
+                    ->merge($sitemap->publishedCollectionTerms())
+                    ->skip($offset)
+                    ->take($remaining)
+            );
+        }
+
+        if ($pages->isEmpty()) {
+            return [];
+        }
+
+        return $sitemap->getPages($pages)
+            ->values()
+            ->map
+            ->toArray();
+    }
+
+    public static function paginatedSitemaps()
+    {
+        $sitemap = new static;
+
+        // would be nice to make terms a count query rather than getting the count from the terms collection
+        $count = $sitemap->publishedEntriesCount() + $sitemap->publishedTerms()->count() + $sitemap->publishedCollectionTerms()->count();
+
+        $sitemapCount = ceil($count / config('statamic.seo-pro.sitemap.pagination.limit', 100));
+
+        return collect(range(1, $sitemapCount))
+            ->map(fn ($page) => ['url' => route('statamic.seo-pro.sitemap.page.show', ['page' => $page])])
+            ->all();
+    }
+
     protected function getPages($items)
     {
         return $items
@@ -54,20 +112,33 @@ protected function getPages($items)
             ->filter();
     }
 
-    protected function publishedEntries()
+    private function publishedEntriesQuery()
     {
-        return Collection::all()
-            ->flatMap(function ($collection) {
+        $collections = Collection::all()
+            ->map(function ($collection) {
                 return $collection->cascade('seo') !== false
-                    ? $collection->queryEntries()->get()
-                    : collect();
+                    ? $collection->handle()
+                    : false;
             })
-            ->filter(function ($entry) {
-                return $entry->status() === 'published';
-            })
-            ->reject(function ($entry) {
-                return is_null($entry->uri());
-            });
+            ->filter()
+            ->values()
+            ->all();
+
+        return Entry::query()
+            ->whereIn('collection', $collections)
+            ->whereNotNull('uri')
+            ->whereStatus('published')
+            ->orderBy('uri');
+    }
+
+    protected function publishedEntries()
+    {
+        return $this->publishedEntriesQuery()->lazy();
+    }
+
+    protected function publishedEntriesCount()
+    {
+        return $this->publishedEntriesQuery()->count();
     }
 
     protected function publishedTerms()
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,3 +5,4 @@ composer.lock @@
     /vendor
     .phpunit.result.cache
     .phpunit.cache
+    .idea/