Ability to paginate with feeds that provide a page number instead of the next page URL.

composer.json

@@ -2,7 +2,7 @@
     "name": "craftcms/feed-me",
     "description": "Import content from XML, RSS, CSV or JSON feeds into entries, categories, Craft Commerce products, and more.",
     "type": "craft-plugin",
-    "version": "4.3.6",
+    "version": "4.3.7",
     "keywords": [
         "craft",
         "cms",

src/base/DataType.php

@@ -41,10 +41,16 @@ public function setupPaginationUrl($array, $feed)
         if (!$feed->paginationNode) {
             return;
         }
-
         // Find the URL value in the feed
         $flatten = Hash::flatten($array, '/');
         $url = Hash::get($flatten, $feed->paginationNode);
+        $totalPages = Hash::get($flatten, $feed->paginationTotalNode);
+
+        //check if the pagination url provided is just a page number
+        $pagedUrl = $this->_generateNextPaginationFromPageNumber($feed->feedUrl, $url, $totalPages);
+        if($pagedUrl) {
+            $url = $pagedUrl;
+        }
 
         // if the feed provides a root relative URL, make it whole again based on the feed.
         if ($url && UrlHelper::isRootRelativeUrl($url)) {
@@ -55,4 +61,24 @@ public function setupPaginationUrl($array, $feed)
         $feed->paginationUrl = $url;
     }
 
+    /**
+     * Generates the next page url if the next page provided in the field is a page number
+     * instead of a url. In this case, the total pages field needs to be provided as well.
+     * @param $feedUrl
+     * @param $url
+     * @param $totalPages
+     */
+    private function _generateNextPaginationFromPageNumber($feedUrl, $url, $totalPages) {
+        if (is_numeric($url) && is_numeric($totalPages)) {
+            $nextPage = $url + 1;
+            if($nextPage > $totalPages) {
+                return null;
+            }
+
+            $feedUrl = UrlHelper::removeParam($feedUrl, "page");
+            return UrlHelper::urlWithParams($feedUrl, array("page" => $nextPage));
+        }
+        return null;
+    }
+
 }

src/controllers/FeedsController.php

@@ -400,6 +400,7 @@ private function _getModelFromPost()
         $feed->singleton = $request->getBodyParam('singleton', $feed->singleton);
         $feed->duplicateHandle = $request->getBodyParam('duplicateHandle', $feed->duplicateHandle);
         $feed->paginationNode = $request->getBodyParam('paginationNode', $feed->paginationNode);
+        $feed->paginationTotalNode = $request->getBodyParam('paginationTotalNode', $feed->paginationTotalNode);
         $feed->passkey = $request->getBodyParam('passkey', $feed->passkey);
         $feed->backup = (bool)$request->getBodyParam('backup', $feed->backup);
 

src/migrations/Install.php

@@ -41,6 +41,7 @@ protected function createTables()
             'singleton' => $this->boolean()->notNull()->defaultValue(false),
             'duplicateHandle' => $this->text(),
             'paginationNode' => $this->text(),
+            'paginationTotalNode' => $this->text(),
             'fieldMapping' => $this->text(),
             'fieldUnique' => $this->text(),
             'passkey' => $this->string()->notNull(),

src/migrations/m210313_164741_migration_for_total_page_number.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace craft\feedme\migrations;
+
+use Craft;
+use craft\db\Migration;
+
+/**
+ * m210313_164741_migration_for_total_page_number migration.
+ */
+class m210313_164741_migration_for_total_page_number extends Migration
+{
+    /**
+     * @inheritdoc
+     */
+    public function safeUp()
+    {
+        if (!$this->db->columnExists('{{%feedme_feeds}}', 'paginationTotalNode')) {
+            $this->addColumn('{{%feedme_feeds}}', 'paginationTotalNode', $this->text()->after('duplicateHandle'));
+        }
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function safeDown()
+    {
+        echo "m210313_164741_migration_for_total_page_number cannot be reverted.\n";
+        return false;
+    }
+}

src/models/FeedModel.php

@@ -83,6 +83,11 @@ class FeedModel extends Model
      */
     public $paginationNode;
 
+    /**
+     * @var
+     */
+    public $paginationTotalNode;
+
     /**
      * @var
      */
@@ -240,4 +245,4 @@ public function rules()
         ];
     }
 
-}
+}

src/services/Feeds.php

@@ -123,6 +123,7 @@ public function saveFeed(FeedModel $model, bool $runValidation = true): bool
         $record->singleton = (bool)$model->singleton;
         $record->duplicateHandle = $model->duplicateHandle;
         $record->paginationNode = $model->paginationNode;
+        $record->paginationTotalNode = $model->paginationTotalNode;
         $record->passkey = $model->passkey;
         $record->backup = $model->backup;
 
@@ -252,6 +253,7 @@ private function _getQuery()
                 'singleton',
                 'duplicateHandle',
                 'paginationNode',
+                'paginationTotalNode',
                 'fieldMapping',
                 'fieldUnique',
                 'passkey',

src/templates/feeds/_element.html

@@ -60,15 +60,25 @@
         {% endfor %}
 
         {{ forms.selectField({
-            label: "Pagination URL"|t('feed-me'),
-            instructions: 'If your feed is paginated, select the next page’s URL.'|t('feed-me'),
+            label: "Pagination URL or Page Number"|t('feed-me'),
+            instructions: 'If your feed is paginated, select the next page’s URL, or the current page number.'|t('feed-me'),
             id: 'paginationNode',
             name: 'paginationNode',
             value: feed.paginationNode,
             options: parsedFeedData,
             errors: feed.getErrors('paginationNode'),
         }) }}
 
+        {{ forms.selectField({
+            label: "Pagination Total Pages"|t('feed-me'),
+            instructions: 'If your feed is paginated using page numbers, select the field containing the total pages. This is required if your feed uses page numbers.'|t('feed-me'),
+            id: 'paginationTotalNode',
+            name: 'paginationTotalNode',
+            value: feed.paginationTotalNode,
+            options: parsedFeedData,
+            errors: feed.getErrors('paginationTotalNode'),
+        }) }}
+
     {% else %}
         <div class="feedme-fullpage fullpage-error">
             <img src="{{ baseAssetsUrl ~ '/img/icon-error.svg' }}">