Feature/acp 3507/acp 3746 persist webhooks and allow to run them later

.github/workflows/ci.yml

@@ -51,6 +51,11 @@ jobs:
             - name: Composer install
               run: composer install --optimize-autoloader
 
+            - name: Setup Database
+              run: |
+                  mkdir -p tests/_data
+                  touch tests/_data/app_webhook_db
+
             - name: Setup AppWebhook
               run: composer setup
 

codeception.yml

@@ -2,6 +2,7 @@ namespace: SprykerTest
 
 include:
     - tests/SprykerTest/Glue/AppWebhookBackendApi/
+    - tests/SprykerTest/Zed/AppWebhook/
 
 bootstrap: _data/Environment.php
 

composer.json

@@ -52,7 +52,7 @@
   "scripts": {
     "cs-check": "phpcs -p src/ tests/",
     "cs-fix": "phpcbf -p src/ tests/",
-    "setup": "tests/bin/console transfer:generate && tests/bin/console transfer:databuilder:generate && tests/bin/console dev:ide-auto-completion:zed:generate && tests/bin/console dev:ide-auto-completion:glue:generate",
+    "setup": "tests/bin/console app-webhook:setup && tests/bin/console transfer:generate && tests/bin/console transfer:databuilder:generate && tests/bin/console dev:ide-auto-completion:zed:generate && tests/bin/console dev:ide-auto-completion:glue:generate && tests/bin/console propel:install && vendor/bin/codecept build",
     "stan": "phpstan analyze src/Spryker/",
     "test": "codecept build && codecept run",
     "test-strict": "vendor/bin/infection --threads=max --min-msi=100 --min-covered-msi=100",

config/Shared/config_default.php

@@ -8,6 +8,61 @@
 use Spryker\Shared\ErrorHandler\ErrorHandlerConstants;
 use Spryker\Shared\Kernel\KernelConstants;
 use Spryker\Shared\Log\LogConstants;
+use Spryker\Shared\Propel\PropelConstants;
+use Spryker\Zed\PropelOrm\Business\Builder\ExtensionObjectBuilder;
+use Spryker\Zed\PropelOrm\Business\Builder\ExtensionQueryBuilder;
+use Spryker\Zed\PropelOrm\Business\Builder\ObjectBuilder;
+use Spryker\Zed\PropelOrm\Business\Builder\QueryBuilder;
+
+$connections = [
+    'mysql' => [
+        'adapter' => 'sqlite',
+        'dsn' => 'sqlite:tests/_data/app_webhook_db',
+        'user' => '',
+        'password' => '',
+        'settings' => [],
+    ],
+];
+
+$config[PropelConstants::PROPEL] = [
+    'database' => [
+        'connections' => [],
+    ],
+    'runtime' => [
+        'defaultConnection' => 'default',
+        'connections' => ['default', 'zed'],
+    ],
+    'generator' => [
+        'defaultConnection' => 'default',
+        'connections' => ['default', 'zed'],
+        'objectModel' => [
+            'defaultKeyType' => 'fieldName',
+            'builders' => [
+                // If you need full entity logging on Create/Update/Delete, then switch to
+                // Spryker\Zed\PropelOrm\Business\Builder\ObjectBuilderWithLogger instead.
+                'object' => ObjectBuilder::class,
+                'objectstub' => ExtensionObjectBuilder::class,
+                'query' => QueryBuilder::class,
+                'querystub' => ExtensionQueryBuilder::class,
+            ],
+        ],
+    ],
+    'paths' => [
+        'phpDir' => APPLICATION_ROOT_DIR,
+        'sqlDir' => APPLICATION_SOURCE_DIR . '/Orm/Propel/Sql/',
+        'migrationDir' => APPLICATION_SOURCE_DIR . '/Orm/Propel/Migration_SQLite/',
+        'schemaDir' => APPLICATION_SOURCE_DIR . '/Orm/Propel/Schema/',
+    ],
+];
+
+$config[PropelConstants::ZED_DB_ENGINE] = 'mysql';
+$config[PropelConstants::ZED_DB_HOST] = 'localhost';
+$config[PropelConstants::ZED_DB_PORT] = 1234;
+$config[PropelConstants::ZED_DB_USERNAME] = 'catface';
+$config[PropelConstants::ZED_DB_PASSWORD] = 'catface';
+
+$config[PropelConstants::PROPEL]['database']['connections']['default'] = $connections['mysql'];
+$config[PropelConstants::PROPEL]['database']['connections']['zed'] = $connections['mysql'];
 
 $config[KernelConstants::PROJECT_NAMESPACE] = 'Spryker';
 $config[KernelConstants::PROJECT_NAMESPACES] = ['Spryker'];

data/DE/logs/ZED/propel.log

@@ -0,0 +1,7 @@
+[2024-08-21T07:57:52.760961+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []
+[2024-08-21T08:01:33.888727+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []
+[2024-08-21T08:02:25.094834+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []
+[2024-08-21T08:03:20.923046+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []
+[2024-08-21T08:04:39.624414+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []
+[2024-08-21T08:05:22.702889+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []
+[2024-08-21T08:10:00.680570+00:00] defaultLogger.ERROR: SQLSTATE[23000]: Integrity constraint violation: 19 UNIQUE constraint failed: spy_webhook_inbox.identifier, spy_webhook_inbox.sequence_number [] []

data/cache/propel/generated-conf/loadDatabase.php

@@ -0,0 +1,25 @@
+<?php
+$serviceContainer = \Propel\Runtime\Propel::getServiceContainer();
+$serviceContainer->initDatabaseMapFromDumps(array (
+  'zed' => 
+  array (
+    'tablesByName' => 
+    array (
+      'spy_app_config' => '\\Orm\\Zed\\AppKernel\\Persistence\\Map\\SpyAppConfigTableMap',
+      'spy_locale' => '\\Orm\\Zed\\Locale\\Persistence\\Map\\SpyLocaleTableMap',
+      'spy_locale_store' => '\\Orm\\Zed\\Locale\\Persistence\\Map\\SpyLocaleStoreTableMap',
+      'spy_queue_process' => '\\Orm\\Zed\\Queue\\Persistence\\Map\\SpyQueueProcessTableMap',
+      'spy_store' => '\\Orm\\Zed\\Store\\Persistence\\Map\\SpyStoreTableMap',
+      'spy_webhook_inbox' => '\\Orm\\Zed\\AppWebhook\\Persistence\\Map\\SpyWebhookInboxTableMap',
+    ),
+    'tablesByPhpName' => 
+    array (
+      '\\SpyAppConfig' => '\\Orm\\Zed\\AppKernel\\Persistence\\Map\\SpyAppConfigTableMap',
+      '\\SpyLocale' => '\\Orm\\Zed\\Locale\\Persistence\\Map\\SpyLocaleTableMap',
+      '\\SpyLocaleStore' => '\\Orm\\Zed\\Locale\\Persistence\\Map\\SpyLocaleStoreTableMap',
+      '\\SpyQueueProcess' => '\\Orm\\Zed\\Queue\\Persistence\\Map\\SpyQueueProcessTableMap',
+      '\\SpyStore' => '\\Orm\\Zed\\Store\\Persistence\\Map\\SpyStoreTableMap',
+      '\\SpyWebhookInbox' => '\\Orm\\Zed\\AppWebhook\\Persistence\\Map\\SpyWebhookInboxTableMap',
+    ),
+  ),
+));

src/Spryker/Shared/AppWebhook/Transfer/app_webhook.transfer.xml

@@ -4,14 +4,23 @@
     <transfer name="WebhookRequest" strict="true">
         <property name="content" type="string"/>
         <property name="mode" type="string"/>
+        <property name="identifier" type="string" description="This is the default identifier set for internal processing."/>
+        <property name="isRetry" type="bool" description="When this WebhookRequest was tried to be processed before we set this to not persist it again in the database."/>
     </transfer>
 
     <transfer name="WebhookResponse" strict="true">
         <property name="isSuccessful" type="bool"/>
+        <property name="isHandled" type="bool" description="A Webhook can be accepted for later processing and setting this to false will keep it persisted. When true or null it will be removed from persistence."/>
+        <property name="identifier" type="string" description="The identifier will be set from outside and used to find previously unprocessed webhooks and execute them when possible."/>
+        <property name="sequenceNumber" type="int" description="This number will be set from outside. When there were already unprocessed webhooks for the same identifier this number will be increased."/>
         <property name="content" type="string"/>
         <property name="message" type="string"/>
     </transfer>
 
+    <transfer name="WebhookInboxCriteria" strict="true">
+        <property name="identifier" type="array" singular="identifier"/>
+    </transfer>
+
     <transfer name="GlueError">
         <property name="message" type="string"/>
     </transfer>

src/Spryker/Zed/AppWebhook/Business/AppWebhookBusinessFactory.php

@@ -8,18 +8,40 @@
 namespace Spryker\Zed\AppWebhook\Business;
 
 use Spryker\Zed\AppWebhook\AppWebhookDependencyProvider;
+use Spryker\Zed\AppWebhook\Business\Identifier\IdentifierBuilder;
+use Spryker\Zed\AppWebhook\Business\Identifier\IdentifierBuilderInterface;
 use Spryker\Zed\AppWebhook\Business\WebhookHandler\WebhookHandler;
+use Spryker\Zed\AppWebhook\Business\WebhookProcessor\WebhookProcessor;
 use Spryker\Zed\Kernel\Business\AbstractBusinessFactory;
 
+/**
+ * @method \Spryker\Zed\AppWebhook\Persistence\AppWebhookEntityManagerInterface getEntityManager()
+ * @method \Spryker\Zed\AppWebhook\Persistence\AppWebhookRepositoryInterface getRepository()
+ */
 class AppWebhookBusinessFactory extends AbstractBusinessFactory
 {
     public function createWebhookHandler(): WebhookHandler
     {
         return new WebhookHandler(
             $this->getAppWebhookHandlerPlugins(),
+            $this->getEntityManager(),
+            $this->createIdentifierBuilder(),
         );
     }
 
+    public function createWebhookProcessor(): WebhookProcessor
+    {
+        return new WebhookProcessor(
+            $this->createWebhookHandler(),
+            $this->getRepository(),
+        );
+    }
+
+    public function createIdentifierBuilder(): IdentifierBuilderInterface
+    {
+        return new IdentifierBuilder();
+    }
+
     /**
      * @return array<\Spryker\Zed\AppWebhook\Dependency\Plugin\WebhookHandlerPluginInterface>
      */

src/Spryker/Zed/AppWebhook/Business/AppWebhookFacade.php

@@ -7,12 +7,15 @@
 
 namespace Spryker\Zed\AppWebhook\Business;
 
+use Generated\Shared\Transfer\WebhookInboxCriteriaTransfer;
 use Generated\Shared\Transfer\WebhookRequestTransfer;
 use Generated\Shared\Transfer\WebhookResponseTransfer;
 use Spryker\Zed\Kernel\Business\AbstractFacade;
 
 /**
  * @method \Spryker\Zed\AppWebhook\Business\AppWebhookBusinessFactory getFactory()
+ * @method \Spryker\Zed\AppWebhook\Persistence\AppWebhookEntityManagerInterface getEntityManager()
+ * @method \Spryker\Zed\AppWebhook\Persistence\AppWebhookRepositoryInterface getRepository()
  */
 class AppWebhookFacade extends AbstractFacade implements AppWebhookFacadeInterface
 {
@@ -25,4 +28,14 @@ public function handleWebhook(WebhookRequestTransfer $webhookRequestTransfer, We
     {
         return $this->getFactory()->createWebhookHandler()->handleWebhook($webhookRequestTransfer, $webhookResponseTransfer);
     }
+
+    /**
+     * @api
+     *
+     * @inheritDoc
+     */
+    public function processUnprocessedWebhooks(WebhookInboxCriteriaTransfer $webhookInboxCriteriaTransfer): void
+    {
+        $this->getFactory()->createWebhookProcessor()->processUnprocessedWebhooks($webhookInboxCriteriaTransfer);
+    }
 }

src/Spryker/Zed/AppWebhook/Business/AppWebhookFacadeInterface.php

@@ -7,6 +7,7 @@
 
 namespace Spryker\Zed\AppWebhook\Business;
 
+use Generated\Shared\Transfer\WebhookInboxCriteriaTransfer;
 use Generated\Shared\Transfer\WebhookRequestTransfer;
 use Generated\Shared\Transfer\WebhookResponseTransfer;
 
@@ -24,4 +25,15 @@ interface AppWebhookFacadeInterface
      * @api
      */
     public function handleWebhook(WebhookRequestTransfer $webhookRequestTransfer, WebhookResponseTransfer $webhookResponseTransfer): WebhookResponseTransfer;
+
+    /**
+     * Specification:
+     * - Process unhandled webhook requests.
+     * - Loads all unprocessed webhooks from the inbox.
+     * - Converts entities to transfers.
+     * - Runs the WebhookHandler for all request transfers.
+     *
+     * @api
+     */
+    public function processUnprocessedWebhooks(WebhookInboxCriteriaTransfer $webhookInboxCriteriaTransfer): void;
 }

src/Spryker/Zed/AppWebhook/Business/Identifier/IdentifierBuilder.php

@@ -0,0 +1,18 @@
+<?php
+
+/**
+ * Copyright © 2016-present Spryker Systems GmbH. All rights reserved.
+ * Use of this software requires acceptance of the Evaluation License Agreement. See LICENSE file.
+ */
+
+namespace Spryker\Zed\AppWebhook\Business\Identifier;
+
+use Ramsey\Uuid\Uuid;
+
+class IdentifierBuilder implements IdentifierBuilderInterface
+{
+    public function getIdentifier(): string
+    {
+        return Uuid::uuid4()->toString();
+    }
+}

src/Spryker/Zed/AppWebhook/Business/Identifier/IdentifierBuilderInterface.php

@@ -0,0 +1,13 @@
+<?php
+
+/**
+ * Copyright © 2016-present Spryker Systems GmbH. All rights reserved.
+ * Use of this software requires acceptance of the Evaluation License Agreement. See LICENSE file.
+ */
+
+namespace Spryker\Zed\AppWebhook\Business\Identifier;
+
+interface IdentifierBuilderInterface
+{
+    public function getIdentifier(): string;
+}

src/Spryker/Zed/AppWebhook/Business/WebhookHandler/WebhookHandler.php

@@ -9,30 +9,62 @@
 
 use Generated\Shared\Transfer\WebhookRequestTransfer;
 use Generated\Shared\Transfer\WebhookResponseTransfer;
+use Spryker\Zed\AppWebhook\Business\Identifier\IdentifierBuilderInterface;
+use Spryker\Zed\AppWebhook\Persistence\AppWebhookEntityManagerInterface;
+use Throwable;
 
 class WebhookHandler
 {
     /**
      * @param array<\Spryker\Zed\AppWebhook\Dependency\Plugin\WebhookHandlerPluginInterface> $webhookHandlerPlugins
      */
-    public function __construct(protected array $webhookHandlerPlugins)
-    {
+    public function __construct(
+        protected array $webhookHandlerPlugins,
+        protected AppWebhookEntityManagerInterface $appWebhookEntityManager,
+        protected IdentifierBuilderInterface $identifierBuilder
+    ) {
     }
 
     public function handleWebhook(WebhookRequestTransfer $webhookRequestTransfer, WebhookResponseTransfer $webhookResponseTransfer): WebhookResponseTransfer
     {
-        foreach ($this->webhookHandlerPlugins as $webhookHandlerPlugin) {
-            if (!$webhookHandlerPlugin->canHandle($webhookRequestTransfer)) {
-                continue;
-            }
+        // get the default identifier which will be only used when the webhook is not a retry
+        $identifier = $this->identifierBuilder->getIdentifier();
 
-            $webhookResponseTransfer = $webhookHandlerPlugin->handleWebhook($webhookRequestTransfer, $webhookResponseTransfer);
+        // Only persist the webhook if it's not a retry.
+        if ($webhookRequestTransfer->getIsRetry() !== true) {
+            $webhookRequestTransfer->setIdentifier($identifier);
+            $this->appWebhookEntityManager->saveWebhookRequest($webhookRequestTransfer);
         }
 
-        if ($webhookResponseTransfer->getIsSuccessful() === null) {
+        try {
+            foreach ($this->webhookHandlerPlugins as $webhookHandlerPlugin) {
+                if (!$webhookHandlerPlugin->canHandle($webhookRequestTransfer)) {
+                    continue;
+                }
+
+                $webhookResponseTransfer = $webhookHandlerPlugin->handleWebhook($webhookRequestTransfer, $webhookResponseTransfer);
+            }
+
+            if ($webhookResponseTransfer->getIsSuccessful() === null) {
+                $webhookResponseTransfer
+                    ->setIsSuccessful(false)
+                    ->setMessage(sprintf('The webhook was not handled by any of the registered plugins. WebhookRequestTransfer: %s', json_encode($webhookRequestTransfer->toArray())));
+            }
+
+            if ($webhookResponseTransfer->getIsHandled() === false) {
+                $this->appWebhookEntityManager->updateWebhookRequest($webhookRequestTransfer, $webhookResponseTransfer);
+            }
+
+            // Using packages may not set or use the isHandled at all (null by default), so we need to check for null explicitly.
+            if ($webhookResponseTransfer->getIsSuccessful() === true && ($webhookResponseTransfer->getIsHandled() === true || $webhookResponseTransfer->getIsHandled() === null)) {
+                $this->appWebhookEntityManager->deleteWebhookRequest($webhookRequestTransfer);
+            }
+        } catch (Throwable $throwable) {
             $webhookResponseTransfer
                 ->setIsSuccessful(false)
-                ->setMessage(sprintf('The webhook was not handled by any of the registered plugins. WebhookRequestTransfer: %s', json_encode($webhookRequestTransfer->toArray())));
+                ->setMessage($throwable->getMessage());
+
+            $this->appWebhookEntityManager->updateWebhookRequest($webhookRequestTransfer, $webhookResponseTransfer);
         }
 
         return $webhookResponseTransfer;

src/Spryker/Zed/AppWebhook/Business/WebhookProcessor/WebhookProcessor.php

@@ -0,0 +1,30 @@
+<?php
+
+/**
+ * Copyright © 2016-present Spryker Systems GmbH. All rights reserved.
+ * Use of this software requires acceptance of the Evaluation License Agreement. See LICENSE file.
+ */
+
+namespace Spryker\Zed\AppWebhook\Business\WebhookProcessor;
+
+use Generated\Shared\Transfer\WebhookInboxCriteriaTransfer;
+use Generated\Shared\Transfer\WebhookResponseTransfer;
+use Spryker\Zed\AppWebhook\Business\WebhookHandler\WebhookHandler;
+use Spryker\Zed\AppWebhook\Persistence\AppWebhookRepositoryInterface;
+
+class WebhookProcessor
+{
+    public function __construct(protected WebhookHandler $webhookHandler, protected AppWebhookRepositoryInterface $appWebhookRepository)
+    {
+    }
+
+    public function processUnprocessedWebhooks(WebhookInboxCriteriaTransfer $webhookInboxCriteriaTransfer): void
+    {
+        $webhookRequestTransfers = $this->appWebhookRepository->getUnprocessedWebhookRequests($webhookInboxCriteriaTransfer);
+
+        foreach ($webhookRequestTransfers as $webhookRequestTransfer) {
+            $webhookRequestTransfer->setIsRetry(true);
+            $this->webhookHandler->handleWebhook($webhookRequestTransfer, new WebhookResponseTransfer());
+        }
+    }
+}