From 2172f099e0321929b81f211c12572c19ee3f3aad Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 24 Oct 2024 11:49:43 +0200 Subject: [PATCH 001/142] WIP Related: #4746 --- .../Classes/ContentRepository.php | 229 +++++------- .../Classes/EventStore/EventPersister.php | 2 - .../Factory/ContentRepositoryFactory.php | 54 ++- ...ntRepositoryServiceFactoryDependencies.php | 10 +- .../Factory/ProjectionFactoryDependencies.php | 1 - .../Projection/ProjectionEventHandler.php | 40 ++ .../Classes/Projection/ProjectionStates.php | 71 ++++ .../Classes/Subscription/Engine/Error.php | 20 + .../Subscription/Engine/ProcessedResult.php | 19 + .../Engine/SubscriptionEngine.php | 343 ++++++++++++++++++ .../Engine/SubscriptionEngineCriteria.php | 49 +++ .../EventStore/RunSubscriptionEventStore.php | 57 +++ .../RetryStrategy/ClockBasedRetryStrategy.php | 57 +++ .../RetryStrategy/NoRetryStrategy.php | 18 + .../RetryStrategy/RetryStrategy.php | 15 + .../Classes/Subscription/RunMode.php | 15 + .../Store/InMemorySubscriptionStore.php | 67 ++++ .../Store/SubscriptionCriteria.php | 67 ++++ .../Store/SubscriptionStoreInterface.php | 30 ++ .../Subscriber/EventHandlerInterface.php | 19 + .../Subscription/Subscriber/Subscriber.php | 23 ++ .../Subscription/Subscriber/Subscribers.php | 76 ++++ .../Classes/Subscription/Subscription.php | 103 ++++++ .../Subscription/SubscriptionError.php | 23 ++ .../Subscription/SubscriptionGroup.php | 28 ++ .../Subscription/SubscriptionGroups.php | 66 ++++ .../Classes/Subscription/SubscriptionId.php | 30 ++ .../Classes/Subscription/SubscriptionIds.php | 69 ++++ .../Subscription/SubscriptionStatus.php | 19 + .../Classes/Subscription/Subscriptions.php | 128 +++++++ .../Features/Bootstrap/CRTestSuiteTrait.php | 1 + .../Classes/Command/CrCommandController.php | 9 +- .../Classes/ContentRepositoryRegistry.php | 17 +- .../DoctrineSubscriptionStore.php | 227 ++++++++++++ .../SubscriptionStoreFactory.php | 27 ++ .../SubscriptionStoreFactoryInterface.php | 18 + .../Service/ProjectionReplayService.php | 81 +---- .../ProjectionReplayServiceFactory.php | 4 +- .../Configuration/Settings.yaml | 3 + 39 files changed, 1907 insertions(+), 228 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/RetryStrategy.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RunMode.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Store/InMemorySubscriptionStore.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscription.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php create mode 100644 Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 696b2de305d..46f62e2e733 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -20,24 +20,16 @@ use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\DecoratedEvent; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\EventStore\Events; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryFactory; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUp; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatuses; -use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryStatus; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\User\UserIdProviderInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; @@ -46,30 +38,21 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; -use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventMetadata; -use Neos\EventStore\Model\EventEnvelope; -use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Clock\ClockInterface; /** * Main Entry Point to the system. Encapsulates the full event-sourced Content Repository. * * Use this to: - * - set up the necessary database tables and contents via {@see ContentRepository::setUp()} - * - send commands to the system (to mutate state) via {@see ContentRepository::handle()} - * - access projection state (to read state) via {@see ContentRepository::projectionState()} - * - catch up projections via {@see ContentRepository::catchUpProjection()} + * - send commands to the system (to mutate state) via {@see self::handle()} + * - access the content graph read model + * - access 3rd party read models via {@see self::projectionState()} * * @api */ -final class ContentRepository +final readonly class ContentRepository { - /** - * @var array, ProjectionStateInterface> - */ - private array $projectionStateCache; - private CommandHandlingDependencies $commandHandlingDependencies; /** @@ -78,16 +61,14 @@ final class ContentRepository public function __construct( public readonly ContentRepositoryId $id, private readonly CommandBus $commandBus, - private readonly EventStoreInterface $eventStore, - private readonly ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, - private readonly EventNormalizer $eventNormalizer, private readonly EventPersister $eventPersister, private readonly NodeTypeManager $nodeTypeManager, private readonly InterDimensionalVariationGraph $variationGraph, private readonly ContentDimensionSourceInterface $contentDimensionSource, private readonly UserIdProviderInterface $userIdProvider, private readonly ClockInterface $clock, - private readonly ContentGraphReadModelInterface $contentGraphReadModel + private readonly ContentGraphReadModelInterface $contentGraphReadModel, + private readonly ProjectionStates $projectionStates, ) { $this->commandHandlingDependencies = new CommandHandlingDependencies($this, $this->contentGraphReadModel); } @@ -143,114 +124,100 @@ public function handle(CommandInterface $command): void */ public function projectionState(string $projectionStateClassName): ProjectionStateInterface { - if (!isset($this->projectionStateCache)) { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - if ($projection instanceof ContentGraphProjectionInterface) { - continue; - } - $projectionState = $projection->getState(); - $this->projectionStateCache[$projectionState::class] = $projectionState; - } - } - if (isset($this->projectionStateCache[$projectionStateClassName])) { - /** @var T $projectionState */ - $projectionState = $this->projectionStateCache[$projectionStateClassName]; - return $projectionState; - } - if (in_array(ContentGraphReadModelInterface::class, class_implements($projectionStateClassName), true)) { - throw new \InvalidArgumentException(sprintf('Accessing the internal content repository projection state via %s(%s) is not allowed. Please use the API on the content repository instead.', __FUNCTION__, $projectionStateClassName), 1729338679); - } - - throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository instance.', $projectionStateClassName), 1662033650); - } - - /** - * @param class-string> $projectionClassName - */ - public function catchUpProjection(string $projectionClassName, CatchUpOptions $options): void - { - $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); - - $catchUpHookFactory = $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection); - $catchUpHook = $catchUpHookFactory?->build($this); - - // TODO allow custom stream name per projection - $streamName = VirtualStreamName::all(); - $eventStream = $this->eventStore->load($streamName); - if ($options->maximumSequenceNumber !== null) { - $eventStream = $eventStream->withMaximumSequenceNumber($options->maximumSequenceNumber); - } - - $eventApplier = function (EventEnvelope $eventEnvelope) use ($projection, $catchUpHook, $options) { - $event = $this->eventNormalizer->denormalize($eventEnvelope->event); - if ($options->progressCallback !== null) { - ($options->progressCallback)($event, $eventEnvelope); - } - if (!$projection->canHandle($event)) { - return; - } - $catchUpHook?->onBeforeEvent($event, $eventEnvelope); - $projection->apply($event, $eventEnvelope); - if ($projection instanceof WithMarkStaleInterface) { - $projection->markStale(); - } - $catchUpHook?->onAfterEvent($event, $eventEnvelope); - }; - - $catchUp = CatchUp::create($eventApplier, $projection->getCheckpointStorage()); - - if ($catchUpHook !== null) { - $catchUpHook->onBeforeCatchUp(); - $catchUp = $catchUp->withOnBeforeBatchCompleted(fn() => $catchUpHook->onBeforeBatchCompleted()); + try { + return $this->projectionStates->get($projectionStateClassName); + } catch (\InvalidArgumentException $e) { + throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository instance: %s', $projectionStateClassName, $e->getMessage()), 1662033650, $e); } - $catchUp->run($eventStream); - $catchUpHook?->onAfterCatchUp(); } - public function catchupProjections(): void - { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - // FIXME optimise by only loading required events once and not per projection - // see https://github.com/neos/neos-development-collection/pull/4988/ - $this->catchUpProjection($projection::class, CatchUpOptions::create()); - } - } - - public function setUp(): void - { - $this->eventStore->setup(); - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projection->setUp(); - } - } - - public function status(): ContentRepositoryStatus - { - $projectionStatuses = ProjectionStatuses::createEmpty(); - foreach ($this->projectionsAndCatchUpHooks->projections as $projectionClassName => $projection) { - $projectionStatuses = $projectionStatuses->with($projectionClassName, $projection->status()); - } - return new ContentRepositoryStatus( - $this->eventStore->status(), - $projectionStatuses, - ); - } - - public function resetProjectionStates(): void - { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projection->reset(); - } - } - - /** - * @param class-string> $projectionClassName - */ - public function resetProjectionState(string $projectionClassName): void - { - $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); - $projection->reset(); - } +// /** +// * @param class-string> $projectionClassName +// */ +// public function catchUpProjection(string $projectionClassName, CatchUpOptions $options): void +// { +// $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); +// +// $catchUpHookFactory = $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection); +// $catchUpHook = $catchUpHookFactory?->build($this); +// +// // TODO allow custom stream name per projection +// $streamName = VirtualStreamName::all(); +// $eventStream = $this->eventStore->load($streamName); +// if ($options->maximumSequenceNumber !== null) { +// $eventStream = $eventStream->withMaximumSequenceNumber($options->maximumSequenceNumber); +// } +// +// $eventApplier = function (EventEnvelope $eventEnvelope) use ($projection, $catchUpHook, $options) { +// $event = $this->eventNormalizer->denormalize($eventEnvelope->event); +// if ($options->progressCallback !== null) { +// ($options->progressCallback)($event, $eventEnvelope); +// } +// if (!$projection->canHandle($event)) { +// return; +// } +// $catchUpHook?->onBeforeEvent($event, $eventEnvelope); +// $projection->apply($event, $eventEnvelope); +// if ($projection instanceof WithMarkStaleInterface) { +// $projection->markStale(); +// } +// $catchUpHook?->onAfterEvent($event, $eventEnvelope); +// }; +// +// $catchUp = CatchUp::create($eventApplier, $projection->getCheckpointStorage()); +// +// if ($catchUpHook !== null) { +// $catchUpHook->onBeforeCatchUp(); +// $catchUp = $catchUp->withOnBeforeBatchCompleted(fn() => $catchUpHook->onBeforeBatchCompleted()); +// } +// $catchUp->run($eventStream); +// $catchUpHook?->onAfterCatchUp(); +// } + +// public function catchupProjections(): void +// { +// foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { +// // FIXME optimise by only loading required events once and not per projection +// // see https://github.com/neos/neos-development-collection/pull/4988/ +// $this->catchUpProjection($projection::class, CatchUpOptions::create()); +// } +// } + +// public function setUp(): void +// { +// $this->eventStore->setup(); +// foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { +// $projection->setUp(); +// } +// } + +// public function status(): ContentRepositoryStatus +// { +// $projectionStatuses = ProjectionStatuses::createEmpty(); +// foreach ($this->projectionsAndCatchUpHooks->projections as $projectionClassName => $projection) { +// $projectionStatuses = $projectionStatuses->with($projectionClassName, $projection->status()); +// } +// return new ContentRepositoryStatus( +// $this->eventStore->status(), +// $projectionStatuses, +// ); +// } + +// public function resetProjectionStates(): void +// { +// foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { +// $projection->reset(); +// } +// } + +// /** +// * @param class-string> $projectionClassName +// */ +// public function resetProjectionState(string $projectionClassName): void +// { +// $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); +// $projection->reset(); +// } /** * @throws WorkspaceDoesNotExist if the workspace does not exist diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php b/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php index 7c53549dac8..6282bba9a36 100644 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php +++ b/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php @@ -42,7 +42,5 @@ public function publishEvents(ContentRepository $contentRepository, EventsToPubl $normalizedEvents, $eventsToPublish->expectedVersion ); - - $contentRepository->catchUpProjections(); } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index c5ba2c24772..44c597fda6a 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -28,9 +28,20 @@ use Neos\ContentRepository\Core\Feature\WorkspaceCommandHandler; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; +use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\User\UserIdProviderInterface; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\EventStore\RunSubscriptionEventStore; +use Neos\ContentRepository\Core\Subscription\RetryStrategy\NoRetryStrategy; +use Neos\ContentRepository\Core\Subscription\RunMode; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; +use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; +use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\EventStore\EventStoreInterface; use Psr\Clock\ClockInterface; use Symfony\Component\Serializer\Serializer; @@ -45,6 +56,10 @@ final class ContentRepositoryFactory private ProjectionFactoryDependencies $projectionFactoryDependencies; private ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks; + private EventStoreInterface $eventStore; + + private SubscriptionEngine $subscriptionEngine; + public function __construct( private readonly ContentRepositoryId $contentRepositoryId, EventStoreInterface $eventStore, @@ -54,16 +69,19 @@ public function __construct( ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, private readonly UserIdProviderInterface $userIdProvider, private readonly ClockInterface $clock, + SubscriptionStoreInterface $subscriptionStore, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); $interDimensionalVariationGraph = new InterDimensionalVariationGraph( $contentDimensionSource, $contentDimensionZookeeper ); + + + $eventNormalizer = new EventNormalizer(); $this->projectionFactoryDependencies = new ProjectionFactoryDependencies( $contentRepositoryId, - $eventStore, - new EventNormalizer(), + $eventNormalizer, $nodeTypeManager, $contentDimensionSource, $contentDimensionZookeeper, @@ -71,6 +89,21 @@ public function __construct( new PropertyConverter($propertySerializer) ); $this->projectionsAndCatchUpHooks = $projectionsAndCatchUpHooksFactory->build($this->projectionFactoryDependencies); + $subscribers = []; + foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { + $subscribers[] = new Subscriber( + SubscriptionId::fromString(substr(strrchr($projection::class, '\\'), 1)), + SubscriptionGroup::fromString('default'), + RunMode::FROM_BEGINNING, + new ProjectionEventHandler( + $projection, + $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection)?->build($this->getOrBuild()), + ), + ); + + } + $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy()); + $this->eventStore = new RunSubscriptionEventStore($eventStore, $this->subscriptionEngine); } // The following properties store "singleton" references of objects for this content repository @@ -87,19 +120,21 @@ public function __construct( public function getOrBuild(): ContentRepository { if (!$this->contentRepository) { + $projectionStates = []; + foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { + $projectionStates[] = $projection->getState(); + } $this->contentRepository = new ContentRepository( $this->contentRepositoryId, $this->buildCommandBus(), - $this->projectionFactoryDependencies->eventStore, - $this->projectionsAndCatchUpHooks, - $this->projectionFactoryDependencies->eventNormalizer, $this->buildEventPersister(), $this->projectionFactoryDependencies->nodeTypeManager, $this->projectionFactoryDependencies->interDimensionalVariationGraph, $this->projectionFactoryDependencies->contentDimensionSource, $this->userIdProvider, $this->clock, - $this->projectionsAndCatchUpHooks->contentGraphProjection->getState() + $this->projectionsAndCatchUpHooks->contentGraphProjection->getState(), + ProjectionStates::fromArray($projectionStates), ); } return $this->contentRepository; @@ -122,9 +157,10 @@ public function buildService( $serviceFactoryDependencies = ContentRepositoryServiceFactoryDependencies::create( $this->projectionFactoryDependencies, + $this->eventStore, $this->getOrBuild(), $this->buildEventPersister(), - $this->projectionsAndCatchUpHooks, + $this->subscriptionEngine, ); return $serviceFactory->build($serviceFactoryDependencies); } @@ -137,7 +173,7 @@ private function buildCommandBus(): CommandBus ), new WorkspaceCommandHandler( $this->buildEventPersister(), - $this->projectionFactoryDependencies->eventStore, + $this->eventStore, $this->projectionFactoryDependencies->eventNormalizer, ), new NodeAggregateCommandHandler( @@ -164,7 +200,7 @@ private function buildEventPersister(): EventPersister { if (!$this->eventPersister) { $this->eventPersister = new EventPersister( - $this->projectionFactoryDependencies->eventStore, + $this->eventStore, $this->projectionFactoryDependencies->eventNormalizer, ); } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index 08ea272181d..fe5faaacf70 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -24,6 +24,7 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; /** @@ -46,7 +47,7 @@ private function __construct( public ContentRepository $contentRepository, // we don't need CommandBus, because this is included in ContentRepository->handle() public EventPersister $eventPersister, - public ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, + public SubscriptionEngine $subscriptionEngine, ) { } @@ -55,13 +56,14 @@ private function __construct( */ public static function create( ProjectionFactoryDependencies $projectionFactoryDependencies, + EventStoreInterface $eventStore, ContentRepository $contentRepository, EventPersister $eventPersister, - ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, + SubscriptionEngine $subscriptionEngine, ): self { return new self( $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->eventStore, + $eventStore, $projectionFactoryDependencies->eventNormalizer, $projectionFactoryDependencies->nodeTypeManager, $projectionFactoryDependencies->contentDimensionSource, @@ -70,7 +72,7 @@ public static function create( $projectionFactoryDependencies->propertyConverter, $contentRepository, $eventPersister, - $projectionsAndCatchUpHooks, + $subscriptionEngine, ); } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php index 9bb2f0cc31f..e317f6726a5 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php @@ -30,7 +30,6 @@ { public function __construct( public ContentRepositoryId $contentRepositoryId, - public EventStoreInterface $eventStore, public EventNormalizer $eventNormalizer, public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php new file mode 100644 index 00000000000..9694e5f5275 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php @@ -0,0 +1,40 @@ +catchUpHook?->onBeforeCatchUp(); + } + + public function handle(EventInterface $event, EventEnvelope $eventEnvelope, Subscription $subscription): void + { + $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); + $this->projection->apply($event, $eventEnvelope); + $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); + } + + public function endBatch(): void + { + $this->catchUpHook?->onBeforeBatchCompleted(); + $this->catchUpHook?->onAfterCatchUp(); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php new file mode 100644 index 00000000000..b4eb01fee28 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php @@ -0,0 +1,71 @@ + + */ +final readonly class ProjectionStates implements \IteratorAggregate, \Countable +{ + /** + * @param array, ProjectionStateInterface> $statesByClassName + */ + private function __construct( + public array $statesByClassName, + ) { + } + + public static function createEmpty(): self + { + return new self([]); + } + + /** + * @param array $states + */ + public static function fromArray(array $states): self + { + $statesByClassName = []; + foreach ($states as $state) { + if (!$state instanceof ProjectionStateInterface) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionStateInterface::class, get_debug_type($state)), 1729687661); + } + if (array_key_exists($state::class, $statesByClassName)) { + throw new \InvalidArgumentException(sprintf('An instance of %s is already part of the set', $state::class), 1729687716); + } + $statesByClassName[$state::class] = $state; + } + return new self($statesByClassName); + } + + /** + * Retrieve a single state (aka read model) by its fully qualified PHP class name + * + * @template T of ProjectionStateInterface + * @param class-string $className + * @return T + * @throws \InvalidArgumentException if the specified state class is not registered + */ + public function get(string $className): ProjectionStateInterface + { + if (!array_key_exists($className, $this->statesByClassName)) { + throw new \InvalidArgumentException(sprintf('The state class "%s" does not exist.', $className), 1729687836); + } + return $this->statesByClassName[$className]; + } + + public function getIterator(): \Traversable + { + return new \ArrayIterator($this->statesByClassName); + } + + public function count(): int + { + return count($this->statesByClassName); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php new file mode 100644 index 00000000000..4ca5c434535 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php @@ -0,0 +1,20 @@ + $errors */ + public function __construct( + public readonly int $processedMessages, + public readonly bool $finished = false, + public readonly array $errors = [], + ) { + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php new file mode 100644 index 00000000000..705905bd68f --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -0,0 +1,343 @@ +ids, + groups: $criteria->groups, + status: [SubscriptionStatus::NEW], + ); + $this->runInternal($subscriptionCriteria, 'setup', $limit); + } + + + public function run( + SubscriptionEngineCriteria $criteria = null, + int $limit = null, + ): void { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + + $subscriptionCriteria = SubscriptionCriteria::create( + ids: $criteria->ids, + groups: $criteria->groups, + status: [SubscriptionStatus::ACTIVE], + ); + $this->runInternal($subscriptionCriteria, 'run', $limit); + } + + private function lockSubscriptions(Subscriptions $subscriptions): void + { + foreach ($subscriptions as $subscription) { + $sT = microtime(true); + while (!$this->subscriptionStore->acquireLock($subscription->id)) { + if (microtime(true) - $sT > 5) { + // TODO better exception handling + throw new \RuntimeException(sprintf('Failed to acquire lock for subscription "%s"', $subscription->id->value), 1721895494); + } + } + } + } + + private function releaseSubscriptions(Subscriptions $subscriptions): void + { + foreach ($subscriptions as $subscription) { + $this->subscriptionStore->releaseLock($subscription->id); + } + } + + private function runInternal(SubscriptionCriteria $criteria, string $process, int|null $limit): void + { + $this->logger?->info(sprintf('Subscription Engine: %s: Start.', $process)); + $this->discoverNewSubscriptions(); + $this->discoverDetachedSubscriptions($criteria); + $this->retrySubscriptions($criteria); + $subscriptions = $this->subscriptionStore->findByCriteria($criteria); + if ($subscriptions->isEmpty()) { + $this->logger?->info(sprintf('Subscription Engine: %s: No subscriptions to process, finishing', $process)); + return;// new ProcessedResult(0, true); + } + + $this->lockSubscriptions($subscriptions); + + $startSequenceNumber = $this->lowestSubscriptionPosition($subscriptions)->next(); + $this->logger?->debug( + sprintf( + 'Subscription Engine: %s: Event stream is processed from position %d.', + $process, + $startSequenceNumber->value, + ), + ); + + /** @var list $errors */ + $errors = []; + $messageCounter = 0; + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + $lastSequenceNumber = null; + $subscriptionsToRun = $subscriptions; + foreach ($eventStream as $eventEnvelope) { + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToRun as $subscription) { + if ($subscription->position->value > $eventEnvelope->sequenceNumber->value) { + $this->logger?->debug( + sprintf( + 'Subscription Engine: %s: Subscription "%s" is farther than the current position (%d > %d), skipped.', + $process, + $subscription->id->value, + $subscription->position->value, + $eventEnvelope->sequenceNumber->value, + ), + ); + continue; + } + $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription); + if (!$error) { + continue; + } + $errors[] = $error; + $subscriptionsToRun = $subscriptionsToRun->without($subscription->id); + } + $messageCounter++; + + $this->logger?->debug( + sprintf( + 'Subscription Engine: %s: Current event stream position: %s', + $process, + $eventEnvelope->sequenceNumber->value, + ), + ); + $lastSequenceNumber = $eventEnvelope->sequenceNumber; + if ($limit !== null && $messageCounter >= $limit) { + $this->logger?->info( + sprintf( + 'Subscription Engine: %s: Message limit (%d) reached, cancelled.', + $process, + $limit, + ), + ); + $this->releaseSubscriptions($subscriptions); + + return;// new ProcessedResult($messageCounter, false, $errors); + } + } + foreach ($subscriptions as $subscription) { + $newSubscriptionStatus = $subscription->runMode === RunMode::ONCE ? SubscriptionStatus::FINISHED : SubscriptionStatus::ACTIVE; + if ($subscription->status === $newSubscriptionStatus) { + continue; + } + $this->subscriptionStore->update($subscription->id, fn(Subscription $subscription) => $subscription->with( + status: $newSubscriptionStatus, + retryAttempt: 0, + )->withoutError()); + $this->logger?->info(sprintf( + 'Subscription Engine: %s: Subscription "%s" changed status from %s to %s.', + $process, + $subscription->id->value, + $subscription->status->name, + $newSubscriptionStatus->name + )); + } + $this->logger?->info( + sprintf( + 'Subscription Engine: %s: End of stream on position %d has been reached, finished.', + $process, + $lastSequenceNumber?->value ?? ($startSequenceNumber->value - 1), + ), + ); + $this->releaseSubscriptions($subscriptions); + + return;// new ProcessedResult($messageCounter, true, $errors); + } + + private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null + { + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->handler->handle($domainEvent, $eventEnvelope, $subscription); + } catch (\Throwable $e) { + $this->logger?->error( + sprintf( + 'Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', + $subscriber::class, + $subscription->id->value, + $eventEnvelope->event->type->value, + $eventEnvelope->sequenceNumber->value, + $e->getMessage(), + ), + ); + $this->subscriptionStore->update($subscription->id, static fn(Subscription $subscription) => $subscription->withError($e)); + return new Error( + $subscription->id, + $e->getMessage(), + $e, + ); + } + $this->logger?->debug( + sprintf( + 'Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', + $subscriber->handler::class, + $subscription->id->value, + $eventEnvelope->event->type->value, + $eventEnvelope->sequenceNumber->value, + ), + ); + $this->subscriptionStore->update($subscription->id, static fn(Subscription $subscription) => $subscription->with( + position: $eventEnvelope->sequenceNumber, + retryAttempt: 0, + )); + return null; + } + + private function discoverNewSubscriptions(): void + { + $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::noConstraints()); + foreach ($this->subscribers as $subscriber) { + if ($registeredSubscriptions->contain($subscriber->id)) { + continue; + } +// if ($subscriber->handler instanceof ProvidesSetup) { +// $subscriber->handler->setup(); +// } + $subscription = Subscription::create( + $subscriber->id, + $subscriber->group, + $subscriber->runMode, + ); + if ($subscriber->runMode === RunMode::FROM_NOW) { + $subscription = $subscription->with( + status: SubscriptionStatus::ACTIVE, + position: $this->lastSequenceNumber(), + ); + } + $this->subscriptionStore->add($subscription); + $this->logger?->info( + sprintf( + 'Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', + $subscriber->id->value, + ), + ); + } + } + + private function discoverDetachedSubscriptions(SubscriptionCriteria $criteria): void + { + $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create( + $criteria->ids, + $criteria->groups, + [SubscriptionStatus::ACTIVE, SubscriptionStatus::PAUSED, SubscriptionStatus::FINISHED], + )); + foreach ($registeredSubscriptions as $subscription) { + if ($this->subscribers->contain($subscription->id)) { + continue; + } + $this->subscriptionStore->update($subscription->id, fn(Subscription $subscription) => $subscription->with( + status: SubscriptionStatus::DETACHED, + )); + $this->logger?->info( + sprintf( + 'Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', + $subscription->id->value, + ), + ); + } + } + + private function retrySubscriptions(SubscriptionCriteria $criteria): void + { + $failedSubscriptions = $this->subscriptionStore->findByCriteria( + SubscriptionCriteria::create( + ids: $criteria->ids, + groups: $criteria->groups, + status: [SubscriptionStatus::ERROR], + ) + ); + foreach ($failedSubscriptions as $subscription) { + if ($subscription->error === null) { + continue; + } + $error = $subscription->error; + $retryable = in_array( + $error->previousStatus, + [SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE], + true, + ); + if (!$retryable) { + continue; + } + if (!$this->retryStrategy->shouldRetry($subscription)) { + continue; + } + $this->subscriptionStore->update($subscription->id, static fn(Subscription $subscription) => $subscription->with( + status: $error->previousStatus, + retryAttempt: $subscription->retryAttempt + 1, + )->withoutError()); + + $this->logger?->info( + sprintf( + 'Subscription Engine: Retry subscription "%s" (%d) and set back to %s.', + $subscription->id->value, + $subscription->retryAttempt + 1, + $error->previousStatus->name, + ), + ); + } + } + + + private function lastSequenceNumber(): SequenceNumber + { + foreach ($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1) as $eventEnvelope) { + return $eventEnvelope->sequenceNumber; + } + return SequenceNumber::fromInteger(0); + } + + private function lowestSubscriptionPosition(Subscriptions $subscriptions): SequenceNumber + { + $min = null; + foreach ($subscriptions as $subscription) { + if ($min !== null && $subscription->position->value >= $min->value) { + continue; + } + $min = $subscription->position; + } + return $min ?? SequenceNumber::fromInteger(0); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php new file mode 100644 index 00000000000..9eceb588f7d --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php @@ -0,0 +1,49 @@ +|null $ids + * @param SubscriptionGroups|list|null $groups + */ + public static function create( + SubscriptionIds|array $ids = null, + SubscriptionGroups|array $groups = null, + ): self { + if (is_array($ids)) { + $ids = SubscriptionIds::fromArray($ids); + } + if (is_array($groups)) { + $groups = SubscriptionGroups::fromArray($groups); + } + return new self( + $ids, + $groups, + ); + } + + public static function noConstraints(): self + { + return new self( + ids: null, + groups: null, + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php b/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php new file mode 100644 index 00000000000..6d08bb848c5 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php @@ -0,0 +1,57 @@ +eventStore->setup(); + } + + public function status(): Status + { + return $this->eventStore->status(); + } + + public function load(StreamName|VirtualStreamName $streamName, EventStreamFilter $filter = null): EventStreamInterface + { + return $this->eventStore->load($streamName, $filter); + } + + public function commit(StreamName $streamName, \Neos\EventStore\Model\Events|Event $events, ExpectedVersion $expectedVersion): CommitResult + { + $commitResult = $this->eventStore->commit($streamName, $events, $expectedVersion); + $this->subscriptionEngine->run($this->criteria ?? SubscriptionEngineCriteria::noConstraints()); + return $commitResult; + } + + public function deleteStream(StreamName $streamName): void + { + $this->eventStore->deleteStream($streamName); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php b/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php new file mode 100644 index 00000000000..674e50cacf7 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php @@ -0,0 +1,57 @@ +retryAttempt >= $this->maxAttempts) { + return false; + } + + $lastSavedAt = $subscription->lastSavedAt; + + if ($lastSavedAt === null) { + return false; + } + + $nextRetryDate = $this->calculateNextRetryDate($lastSavedAt, $subscription->retryAttempt); + + return $nextRetryDate <= $this->clock->now(); + } + + private function calculateNextRetryDate(\DateTimeImmutable $lastDate, int $attempt): \DateTimeImmutable + { + return $lastDate->modify(sprintf('+%d seconds', $this->calculateDelay($attempt))); + } + + private function calculateDelay(int $attempt): int + { + return (int)round($this->baseDelay * ($this->delayFactor ** $attempt)); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php b/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php new file mode 100644 index 00000000000..eccb35fb2f2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php @@ -0,0 +1,18 @@ +subscriptions = Subscriptions::none(); + } + + + public function findOneById(SubscriptionId $subscriptionId): ?Subscription + { + return $this->subscriptions->contain($subscriptionId) ? $this->subscriptions->get($subscriptionId) : null; + } + + public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions + { + return $this->subscriptions->filter(function (Subscription $subscription) use ($criteria) { + if ($criteria->ids !== null && !$criteria->ids->contain($subscription->id)) { + return false; + } + if ($criteria->groups !== null && !$criteria->groups->contain($subscription->group)) { + return false; + } + if ($criteria->status !== null && !in_array($subscription->status, $criteria->status, true)) { + return false; + } + return true; + }); + } + + public function acquireLock(SubscriptionId $subscriptionId): bool + { + // no locking for this implementation + return true; + } + + public function releaseLock(SubscriptionId $subscriptionId): void + { + // no locking for this implementation + } + + public function add(Subscription $subscription): void + { + $this->subscriptions = $this->subscriptions->withAdded($subscription); + } + + public function update(SubscriptionId $subscriptionId, \Closure $updater): void + { + $subscription = $this->subscriptions->get($subscriptionId); + $subscription = $updater($subscription); + $this->subscriptions = $this->subscriptions->withReplaced($subscriptionId, $subscription); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php new file mode 100644 index 00000000000..ccdc7659de2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php @@ -0,0 +1,67 @@ +|null $status + */ + private function __construct( + public readonly SubscriptionIds|null $ids, + public readonly SubscriptionGroups|null $groups, + public readonly array|null $status, + ) { + } + + /** + * @param SubscriptionIds|array|null $ids + * @param SubscriptionGroups|list|null $groups + * @param list|null $status + */ + public static function create( + SubscriptionIds|array $ids = null, + SubscriptionGroups|array $groups = null, + array $status = null, + ): self { + if (is_array($ids)) { + $ids = SubscriptionIds::fromArray($ids); + } + if (is_array($groups)) { + $groups = SubscriptionGroups::fromArray($groups); + } + return new self( + $ids, + $groups, + $status, + ); + } + + public static function noConstraints(): self + { + return new self( + ids: null, + groups: null, + status: null, + ); + } + + public static function withStatus(SubscriptionStatus $status): self + { + return new self( + ids: null, + groups: null, + status: [$status], + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php new file mode 100644 index 00000000000..f816e8e3fac --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -0,0 +1,30 @@ + + * @internal + */ +final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscribersById + */ + private function __construct( + private readonly array $subscribersById + ) { + } + + /** + * @param array $subscribers + */ + public static function fromArray(array $subscribers): self + { + $subscribersById = []; + foreach ($subscribers as $subscriber) { + if (!$subscriber instanceof Subscriber) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscriber::class, get_debug_type($subscriber)), 1721731490); + } + if (array_key_exists($subscriber->id->value, $subscribersById)) { + throw new \InvalidArgumentException(sprintf('Subscriber with id "%s" already part of this set', $subscriber->id->value), 1721731494); + } + $subscribersById[$subscriber->id->value] = $subscriber; + } + return new self($subscribersById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function get(SubscriptionId $id): Subscriber + { + if (!$this->contain($id)) { + throw new \InvalidArgumentException(sprintf('Subscriber with the subscription id "%s" not found.', $id->value), 1721731490); + } + return $this->subscribersById[$id->value]; + } + + public function contain(SubscriptionId $id): bool + { + return array_key_exists($id->value, $this->subscribersById); + } + + public function getIterator(): \Traversable + { + return yield from $this->subscribersById; + } + + public function count(): int + { + return count($this->subscribersById); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscribersById); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php new file mode 100644 index 00000000000..c54ef1486c3 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -0,0 +1,103 @@ +id, + $this->group, + $this->runMode, + $status ?? $this->status, + $position ?? $this->position, + $this->locked, + $this->error, + $retryAttempt ?? $this->retryAttempt, + $this->lastSavedAt, + ); + } + + public function withError(\Throwable|string $throwableOrMessage): self + { + if ($throwableOrMessage instanceof \Throwable) { + $error = SubscriptionError::fromThrowable($this->status, $throwableOrMessage); + } else { + $error = new SubscriptionError($throwableOrMessage, $this->status); + } + return new self( + $this->id, + $this->group, + $this->runMode, + SubscriptionStatus::ERROR, + $this->position, + $this->locked, + $error, + $this->retryAttempt, + $this->lastSavedAt, + ); + } +// +// public function doRetry(): void +// { +// if ($this->error === null) { +// throw new NoErrorToRetry(); +// } +// +// $this->retryAttempt++; +// $this->status = $this->error->previousStatus; +// $this->error = null; +// } + public function withoutError(): self + { + return new self( + $this->id, + $this->group, + $this->runMode, + $this->status, + $this->position, + $this->locked, + null, + $this->retryAttempt, + $this->lastSavedAt, + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php new file mode 100644 index 00000000000..e715d5266d3 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php @@ -0,0 +1,23 @@ +getMessage(), $status, $error->getTraceAsString()); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php new file mode 100644 index 00000000000..4c5b5466a17 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php @@ -0,0 +1,28 @@ +value === $this->value; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php new file mode 100644 index 00000000000..410379ff456 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php @@ -0,0 +1,66 @@ + + * @internal + */ +final class SubscriptionGroups implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $items + */ + private function __construct( + private readonly array $items + ) { + } + + /** + * @param list $items + */ + public static function fromArray(array $items): self + { + return new self(array_map(static fn ($item) => $item instanceof SubscriptionGroup ? $item : SubscriptionGroup::fromString($item), $items)); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + return yield from $this->items; + } + + public function count(): int + { + return count($this->items); + } + + public function contain(SubscriptionGroup $group): bool + { + foreach ($this->items as $item) { + if ($item->equals($group)) { + return true; + } + } + return false; + } + + /** + * @return array + */ + public function toStringArray(): array + { + return array_map(static fn (SubscriptionGroup $group) => $group->value, $this->items); + } + + public function jsonSerialize(): mixed + { + return array_values($this->items); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php new file mode 100644 index 00000000000..b1753d41885 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php @@ -0,0 +1,30 @@ +value === $this->value; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php new file mode 100644 index 00000000000..2d346a81886 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php @@ -0,0 +1,69 @@ + + * @internal + */ +final class SubscriptionIds implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $items + */ + private function __construct( + private readonly array $items + ) { + } + + /** + * @param array $items + */ + public static function fromArray(array $items): self + { + return new self(array_map(static fn ($item) => $item instanceof SubscriptionId ? $item : SubscriptionId::fromString($item), $items)); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + return yield from $this->items; + } + + public function count(): int + { + return count($this->items); + } + + public function contain(SubscriptionId $id): bool + { + foreach ($this->items as $item) { + if ($item->equals($id)) { + return true; + } + } + return false; + } + + /** + * @return array + */ + public function toStringArray(): array + { + return array_map(static fn (SubscriptionId $id) => $id->value, $this->items); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->items); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php new file mode 100644 index 00000000000..5b3dc856b9a --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -0,0 +1,19 @@ + + * @internal + */ +final class Subscriptions implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $items + */ + private function __construct( + private readonly array $items + ) { + } + + /** + * @param array $items + */ + public static function fromArray(array $items): self + { + foreach ($items as $item) { + if ($item instanceof Subscription) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscription::class, get_debug_type($item)), 1729679774); + } + } + return new self($items); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + return yield from $this->items; + } + + public function isEmpty(): bool + { + return $this->items === []; + } + + public function count(): int + { + return count($this->items); + } + + public function contain(SubscriptionId $subscriptionId): bool + { + foreach ($this->items as $item) { + if ($item->id->equals($subscriptionId)) { + return true; + } + } + return false; + } + + public function get(SubscriptionId $subscriptionId): Subscription + { + foreach ($this->items as $item) { + if ($item->id->equals($subscriptionId)) { + return $item; + } + } + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" not part of this set', $subscriptionId->value), 1723567808); + } + + public function without(SubscriptionId $subscriptionId): self + { + return $this->filter(static fn (Subscription $subscription) => !$subscription->id->equals($subscriptionId)); + } + + /** + * @param \Closure(Subscription): bool $callback + */ + public function filter(\Closure $callback): self + { + return self::fromArray(array_filter($this->items, $callback)); + } + + /** + * @template T + * @param \Closure(Subscription): T $callback + * @return array + */ + public function map(\Closure $callback): array + { + return array_map($callback, $this->items); + } + + public function withAdded(Subscription $subscription): self + { + if ($this->contain($subscription->id)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is already part of this set', $subscription->id->value), 1723568258); + } + return new self([...$this->items, $subscription]); + } + + public function withReplaced(SubscriptionId $subscriptionId, Subscription $subscription): self + { + if (!$this->contain($subscription->id)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is not part of this set', $subscription->id->value), 1723568412); + } + $newItems = []; + foreach ($this->items as $item) { + if ($item->id->equals($subscriptionId)) { + $newItems[] = $subscription; + } else { + $newItems[] = $item; + } + } + return new self($newItems); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->items); + } +} diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 0384232e778..a0ebf501883 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -162,6 +162,7 @@ public function iExpectTheGraphProjectionToConsistOfExactlyNodes(int $expectedNu public ContentGraphReadModelInterface|null $instance; public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { + // TODO find replacement – is that needed at all? $this->instance = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection->getState(); return new class implements ContentRepositoryServiceInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 459398dfeb0..63e6fa96ffc 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -128,7 +128,7 @@ public function projectionReplayCommand(string $projection, string $contentRepos $options = CatchUpOptions::create(); if (!$quiet) { $this->outputLine('Replaying events for projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); - $progressBar->start(max($until > 0 ? $until : $projectionService->highestSequenceNumber()->value, 1)); + $progressBar->start($until > 0 ? $until : null); $options = $options->with(progressCallback: fn () => $progressBar->advance()); } if ($until > 0) { @@ -187,14 +187,13 @@ public function projectionReplayAllCommand(string $contentRepository = 'default' if ($until > 0) { $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); } - $highestSequenceNumber = max($until > 0 ? $until : $projectionService->highestSequenceNumber()->value, 1); - $mainProgressBar->start($projectionService->numberOfProjections()); + $mainProgressBar->start(); $mainProgressCallback = null; if (!$quiet) { - $mainProgressCallback = static function (string $projectionAlias) use ($mainProgressBar, $progressBar, $highestSequenceNumber) { + $mainProgressCallback = static function (string $projectionAlias) use ($mainProgressBar, $progressBar, $until) { $mainProgressBar->advance(); $progressBar->setMessage($projectionAlias); - $progressBar->start($highestSequenceNumber); + $progressBar->start($until > 0 ? $until : null); $progressBar->setProgress(0); }; } diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 4b78f99f13d..f5b54605687 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -21,12 +21,14 @@ use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryIds; use Neos\ContentRepository\Core\SharedModel\User\UserIdProviderInterface; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; use Neos\ContentRepositoryRegistry\Exception\InvalidConfigurationException; use Neos\ContentRepositoryRegistry\Factory\Clock\ClockFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\ContentDimensionSource\ContentDimensionSourceFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\EventStore\EventStoreFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\NodeTypeManager\NodeTypeManagerFactoryInterface; +use Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\UserIdProvider\UserIdProviderFactoryInterface; use Neos\ContentRepositoryRegistry\SubgraphCachingInMemory\ContentSubgraphWithRuntimeCaches; use Neos\ContentRepositoryRegistry\SubgraphCachingInMemory\SubgraphCachePool; @@ -176,7 +178,8 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $this->buildPropertySerializer($contentRepositoryId, $contentRepositorySettings), $this->buildProjectionsFactory($contentRepositoryId, $contentRepositorySettings), $this->buildUserIdProvider($contentRepositoryId, $contentRepositorySettings), - $clock + $clock, + $this->buildSubscriptionStore($contentRepositoryId, $clock, $contentRepositorySettings), ); } catch (\Exception $exception) { throw InvalidConfigurationException::fromException($contentRepositoryId, $exception); @@ -313,4 +316,16 @@ private function buildClock(ContentRepositoryId $contentRepositoryIdentifier, ar } return $clockFactory->build($contentRepositoryIdentifier, $contentRepositorySettings['clock']['options'] ?? []); } + + /** @param array $contentRepositorySettings */ + private function buildSubscriptionStore(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $contentRepositorySettings): SubscriptionStoreInterface + { + isset($contentRepositorySettings['subscriptionStore']['factoryObjectName']) || throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have subscriptionStore.factoryObjectName configured.', $contentRepositoryId->value); + $subscriptionStoreFactory = $this->objectManager->get($contentRepositorySettings['subscriptionStore']['factoryObjectName']); + if (!$subscriptionStoreFactory instanceof SubscriptionStoreFactoryInterface) { + throw InvalidConfigurationException::fromMessage('subscriptionStore.factoryObjectName for content repository "%s" is not an instance of %s but %s.', $contentRepositoryId->value, SubscriptionStoreFactoryInterface::class, get_debug_type($subscriptionStoreFactory)); + } + return $subscriptionStoreFactory->build($contentRepositoryId, $clock, $contentRepositorySettings['subscriptionStore']['options'] ?? []); + } } + diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php new file mode 100644 index 00000000000..a5203af7205 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -0,0 +1,227 @@ +dbal->getSchemaManager()->createSchemaConfig(); + assert($schemaConfig !== null); + $schemaConfig->setDefaultTableOptions([ + 'charset' => 'utf8mb4' + ]); + $isSqlite = $this->dbal->getDatabasePlatform() instanceof SqlitePlatform; + $tableSchema = new Table($this->tableName, [ + (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('group_name', Type::getType(Types::STRING)))->setNotnull(true)->setLength(100)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('run_mode', Type::getType(Types::STRING)))->setNotnull(true)->setLength(16)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), + (new Column('locked', Type::getType(Types::BOOLEAN)))->setNotnull(true), + (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), + (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('error_trace', Type::getType(Types::TEXT)))->setNotnull(false), + (new Column('retry_attempt', Type::getType(Types::INTEGER)))->setNotnull(true), + (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), + ]); + $tableSchema->setPrimaryKey(['id']); + $tableSchema->addIndex(['group_name']); + $tableSchema->addIndex(['status']); + $schema = new Schema( + [$tableSchema], + [], + $schemaConfig, + ); + foreach (DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema) as $statement) { + $this->dbal->executeStatement($statement); + } + } + + public function findOneById(SubscriptionId $subscriptionId): ?Subscription + { + $row = $this->dbal->fetchAssociative('SELECT * FROM ' . $this->tableName . ' WHERE id = :subscriptionId', ['subscriptionId' => $subscriptionId->value]); + if ($row === false) { + return null; + } + return self::fromDatabase($row); + } + + public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions + { + $queryBuilder = $this->dbal->createQueryBuilder() + ->select('*') + ->from($this->tableName) + ->orderBy('id'); + if ($criteria->ids !== null) { + $queryBuilder->andWhere('id IN (:ids)') + ->setParameter( + 'ids', + $criteria->ids->toStringArray(), + Connection::PARAM_STR_ARRAY, + ); + } + if ($criteria->groups !== null) { + $queryBuilder->andWhere('group_name IN (:groups)') + ->setParameter( + 'groups', + $criteria->groups->toStringArray(), + Connection::PARAM_STR_ARRAY, + ); + } + if ($criteria->status !== null) { + $queryBuilder->andWhere('status IN (:status)') + ->setParameter( + 'status', + array_map(static fn (SubscriptionStatus $status) => $status->name, $criteria->status), + Connection::PARAM_STR_ARRAY, + ); + } + $result = $queryBuilder->executeQuery(); + assert($result instanceof Result); + $rows = $result->fetchAllAssociative(); + if ($rows === []) { + return Subscriptions::none(); + } + return Subscriptions::fromArray(array_map(self::fromDatabase(...), $rows)); + } + + public function acquireLock(SubscriptionId $subscriptionId): bool + { + $data = [ + 'locked' => 1, + 'last_saved_at' => $this->clock->now()->format('Y-m-d H:i:s'), + ]; + $acquired = $this->dbal->update($this->tableName, $data, [ + 'id' => $subscriptionId->value, + 'locked' => 0, + ]); + return $acquired >= 1; + } + + public function releaseLock(SubscriptionId $subscriptionId): void + { + $data = [ + 'locked' => 0, + 'last_saved_at' => $this->clock->now()->format('Y-m-d H:i:s'), + ]; + $this->dbal->update($this->tableName, $data, ['id' => $subscriptionId->value]); + } + + public function add(Subscription $subscription): void + { + $row = self::toDatabase($subscription); + $row['id'] = $subscription->id->value; + $row['locked'] = 0; + $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $this->dbal->insert( + $this->tableName, + $row, + ); + } + + public function update(SubscriptionId $subscriptionId, Closure $updater): void + { + $subscription = $this->findOneById($subscriptionId); + if ($subscription === null) { + throw new \InvalidArgumentException(sprintf('Failed to update subscription with id "%s" because it does not exist', $subscriptionId->value), 1721672347); + } + /** @var Subscription $subscription */ + $subscription = $updater($subscription); + $row = self::toDatabase($subscription); + $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $this->dbal->update( + $this->tableName, + $row, + [ + 'id' => $subscriptionId->value, + ] + ); + } + + /** + * @return array + */ + private static function toDatabase(Subscription $subscription): array + { + return [ + 'group_name' => $subscription->group->value, + 'run_mode' => $subscription->runMode->name, + 'status' => $subscription->status->name, + 'position' => $subscription->position->value, + 'error_message' => $subscription->error?->errorMessage, + 'error_previous_status' => $subscription->error?->previousStatus?->name, + 'error_trace' => $subscription->error?->errorTrace, + 'retry_attempt' => $subscription->retryAttempt, + ]; + } + + /** + * @param array $row + */ + private static function fromDatabase(array $row): Subscription + { + if (isset($row['error_message'])) { + assert(is_string($row['error_message'])); + assert(!isset($row['error_previous_status']) || is_string($row['error_previous_status'])); + assert(is_string($row['error_trace'])); + $subscriptionError = new SubscriptionError($row['error_message'], SubscriptionStatus::from($row['error_previous_status']), $row['error_trace']); + } else { + $subscriptionError = null; + } + assert(is_string($row['id'])); + assert(is_string($row['group_name'])); + assert(is_string($row['run_mode'])); + assert(is_string($row['status'])); + assert(is_int($row['position'])); + assert(is_int($row['locked'])); + assert(is_int($row['retry_attempt'])); + assert(is_string($row['last_saved_at'])); + $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); + assert($lastSavedAt instanceof DateTimeImmutable); + + return new Subscription( + SubscriptionId::fromString($row['id']), + SubscriptionGroup::fromString($row['group_name']), + RunMode::from($row['run_mode']), + SubscriptionStatus::from($row['status']), + SequenceNumber::fromInteger($row['position']), + (bool)$row['locked'], + $subscriptionError, + $row['retry_attempt'], + $lastSavedAt, + ); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php new file mode 100644 index 00000000000..ef1405837e3 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php @@ -0,0 +1,27 @@ + $options */ + public function build(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $options): SubscriptionStoreInterface + { + return new DoctrineSubscriptionStore(sprintf('cr_%s_subscriptions', $contentRepositoryId->value), $this->connection, $clock); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php new file mode 100644 index 00000000000..79e3516887c --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php @@ -0,0 +1,18 @@ + $options */ + public function build(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $options): SubscriptionStoreInterface; +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php index 00a946be01a..5643afb5fb2 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php @@ -9,6 +9,9 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\Projections; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -20,91 +23,25 @@ */ final class ProjectionReplayService implements ContentRepositoryServiceInterface { - public function __construct( - private readonly Projections $projections, - private readonly ContentRepository $contentRepository, - private readonly EventStoreInterface $eventStore, + private readonly SubscriptionEngine $subscriptionEngine, ) { } public function replayProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void { - $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); - $this->contentRepository->resetProjectionState($projectionClassName); - $this->contentRepository->catchUpProjection($projectionClassName, $options); + // TODO $this->subscriptionEngine->reset() + // TODO $this->subscriptionEngine->run() } public function replayAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - if ($progressCallback) { - $progressCallback($classNamesAndAlias['alias']); - } - $this->contentRepository->resetProjectionState($classNamesAndAlias['className']); - $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); - } + // TODO $this->subscriptionEngine->reset() + // TODO $this->subscriptionEngine->run() } public function resetAllProjections(): void { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - $this->contentRepository->resetProjectionState($classNamesAndAlias['className']); - } - } - - public function highestSequenceNumber(): SequenceNumber - { - foreach ($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1) as $eventEnvelope) { - return $eventEnvelope->sequenceNumber; - } - return SequenceNumber::none(); - } - - public function numberOfProjections(): int - { - return count($this->projections); - } - - /** - * @return class-string> - */ - private function resolveProjectionClassName(string $projectionAliasOrClassName): string - { - $lowerCaseProjectionName = strtolower($projectionAliasOrClassName); - $projectionClassNamesAndAliases = $this->projectionClassNamesAndAliases(); - foreach ($projectionClassNamesAndAliases as $classNamesAndAlias) { - if (strtolower($classNamesAndAlias['className']) === $lowerCaseProjectionName || strtolower($classNamesAndAlias['alias']) === $lowerCaseProjectionName) { - return $classNamesAndAlias['className']; - } - } - throw new \InvalidArgumentException(sprintf( - 'The projection "%s" is not registered for this Content Repository. The following projection aliases (or fully qualified class names) can be used: %s', - $projectionAliasOrClassName, - implode('', array_map(static fn (array $classNamesAndAlias) => sprintf(chr(10) . ' * %s (%s)', $classNamesAndAlias['alias'], $classNamesAndAlias['className']), $projectionClassNamesAndAliases)) - ), 1680519624); - } - - /** - * @return array>, alias: string}> - */ - private function projectionClassNamesAndAliases(): array - { - return array_map( - static fn (string $projectionClassName) => [ - 'className' => $projectionClassName, - 'alias' => self::projectionAlias($projectionClassName), - ], - $this->projections->getClassNames() - ); - } - - private static function projectionAlias(string $className): string - { - $alias = lcfirst(substr(strrchr($className, '\\') ?: '\\' . $className, 1)); - if (str_ends_with($alias, 'Projection')) { - $alias = substr($alias, 0, -10); - } - return $alias; + // TODO $this->subscriptionEngine->reset() } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php index 0f4bc5f7a05..3ee99bae132 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayServiceFactory.php @@ -22,9 +22,7 @@ final class ProjectionReplayServiceFactory implements ContentRepositoryServiceFa public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { return new ProjectionReplayService( - $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->subscriptionEngine, ); } } diff --git a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml index 83b207597eb..f76c430b6a8 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml @@ -37,6 +37,9 @@ Neos: clock: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory + propertyConverters: DateTimeNormalizer: className: Symfony\Component\Serializer\Normalizer\DateTimeNormalizer From 29ba9082ca8a98940040520d8b24a10ecb78fd8f Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 5 Nov 2024 14:15:46 +0100 Subject: [PATCH 002/142] WIP --- ...trineDbalContentGraphProjectionFactory.php | 5 +- .../src/HypergraphProjectionFactory.php | 5 +- .../RaceTrackerCatchUpHook.php | 11 +-- .../RaceTrackerCatchUpHookFactory.php | 6 +- .../Classes/ContentRepository.php | 6 -- .../Factory/ContentRepositoryFactory.php | 90 ++++++++++-------- ...ntRepositoryServiceFactoryDependencies.php | 2 +- .../ContentRepositorySubscriberFactories.php | 38 ++++++++ ...ntRepositorySubscriberFactoryInterface.php | 25 +++++ .../Factory/ProjectionSubscriberFactory.php | 32 +++++++ ....php => SubscriberFactoryDependencies.php} | 2 +- .../CatchUpHookFactories.php | 5 +- .../CatchUpHookFactoryDependencies.php | 9 +- .../CatchUpHookFactoryInterface.php | 4 +- .../CatchUpHookInterface.php | 25 ++--- .../DelegatingCatchUpHook.php | 16 +--- ...ContentGraphProjectionFactoryInterface.php | 12 +-- .../Projection/ProjectionEventHandler.php | 38 ++++++-- .../Projection/ProjectionFactoryInterface.php | 4 +- .../Classes/Projection/Projections.php | 3 + .../Engine/SubscriptionEngine.php | 8 +- .../Subscriber/EventHandlerInterface.php | 8 +- .../Classes/ContentRepositoryRegistry.php | 91 +++++++++++-------- .../DoctrineSubscriptionStore.php | 3 + .../Service/ProjectionReplayService.php | 13 +-- .../FlushSubgraphCachePoolCatchUpHook.php | 9 +- ...ushSubgraphCachePoolCatchUpHookFactory.php | 6 +- .../CatchUpHook/AssetUsageCatchUpHook.php | 11 +-- .../AssetUsageCatchUpHookFactory.php | 4 +- .../CatchUpHook/RouterCacheHook.php | 10 +- .../CatchUpHook/RouterCacheHookFactory.php | 6 +- .../DocumentUriPathProjectionFactory.php | 4 +- ...phProjectorCatchUpHookForCacheFlushing.php | 9 +- ...ctorCatchUpHookForCacheFlushingFactory.php | 4 +- .../ChangeProjectionFactory.php | 4 +- 35 files changed, 314 insertions(+), 214 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php create mode 100644 Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php create mode 100644 Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php rename Neos.ContentRepository.Core/Classes/Factory/{ProjectionFactoryDependencies.php => SubscriberFactoryDependencies.php} (96%) rename Neos.ContentRepository.Core/Classes/Projection/{ => CatchUpHook}/CatchUpHookFactories.php (93%) rename Neos.ContentRepository.Core/Classes/Projection/{ => CatchUpHook}/CatchUpHookFactoryDependencies.php (77%) rename Neos.ContentRepository.Core/Classes/Projection/{ => CatchUpHook}/CatchUpHookFactoryInterface.php (83%) rename Neos.ContentRepository.Core/Classes/Projection/{ => CatchUpHook}/CatchUpHookInterface.php (57%) rename Neos.ContentRepository.Core/Classes/Projection/{ => CatchUpHook}/DelegatingCatchUpHook.php (76%) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php index a5c4d6ae25e..0d69c3c6a22 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php @@ -8,7 +8,7 @@ use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\DimensionSpacePointsRepository; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\NodeFactory; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\ProjectionContentGraph; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; /** @@ -24,8 +24,7 @@ public function __construct( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): DoctrineDbalContentGraphProjection { $tableNames = ContentGraphTableNames::create( $projectionFactoryDependencies->contentRepositoryId diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php index 3d3c002c094..a70b0d5d148 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php @@ -7,7 +7,7 @@ use Doctrine\DBAL\Connection; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\HypergraphProjection; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Repository\NodeFactory; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -28,8 +28,7 @@ public static function graphProjectionTableNamePrefix( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): HypergraphProjection { $tableNamePrefix = self::graphProjectionTableNamePrefix( $projectionFactoryDependencies->contentRepositoryId diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php index 039156ef9da..1ce532ca5fd 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php @@ -17,7 +17,8 @@ use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntries; use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntryType; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Annotations as Flow; @@ -107,7 +108,7 @@ final class RaceTrackerCatchUpHook implements CatchUpHookInterface protected $configuration; private bool $inCriticalSection = false; - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { RedisInterleavingLogger::connect($this->configuration['redis']['host'], $this->configuration['redis']['port']); } @@ -126,7 +127,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event { } - public function onBeforeBatchCompleted(): void + public function onAfterCatchUp(): void { // we only want to track relevant lock release calls (i.e. if we were in the event processing loop before) if ($this->inCriticalSection) { @@ -134,8 +135,4 @@ public function onBeforeBatchCompleted(): void RedisInterleavingLogger::trace(TraceEntryType::LockWillBeReleasedIfItWasAcquiredBefore); } } - - public function onAfterCatchUp(): void - { - } } diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php index 389d7a324a5..829ed582f96 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php @@ -14,9 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 7840908a7d3..9e9b23ad267 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -23,9 +23,6 @@ use Neos\ContentRepository\Core\EventStore\InitiatingEventMetadata; use Neos\ContentRepository\Core\Factory\ContentRepositoryFactory; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUp; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; @@ -39,9 +36,6 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; -use Neos\EventStore\EventStoreInterface; -use Neos\EventStore\Model\EventEnvelope; -use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Clock\ClockInterface; /** diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 5d1529e1213..aa985d65a63 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -29,6 +29,10 @@ use Neos\ContentRepository\Core\Feature\WorkspaceCommandHandler; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -38,6 +42,7 @@ use Neos\ContentRepository\Core\Subscription\RetryStrategy\NoRetryStrategy; use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionEventHandlerInterface; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; @@ -53,11 +58,14 @@ */ final class ContentRepositoryFactory { - private ProjectionFactoryDependencies $projectionFactoryDependencies; + private SubscriberFactoryDependencies $subscriberFactoryDependencies; private EventStoreInterface $eventStore; - private SubscriptionEngine $subscriptionEngine; + private ProjectionStates $projectionStates; + + private ContentGraphProjectionInterface $contentGraphProjection; + public function __construct( private readonly ContentRepositoryId $contentRepositoryId, EventStoreInterface $eventStore, @@ -67,6 +75,9 @@ public function __construct( private readonly UserIdProviderInterface $userIdProvider, private readonly ClockInterface $clock, SubscriptionStoreInterface $subscriptionStore, + ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, + CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, + ContentRepositorySubscriberFactories $additionalSubscriberFactories, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); $interDimensionalVariationGraph = new InterDimensionalVariationGraph( @@ -74,7 +85,7 @@ public function __construct( $contentDimensionZookeeper ); $eventNormalizer = new EventNormalizer(); - $this->projectionFactoryDependencies = new ProjectionFactoryDependencies( + $this->subscriberFactoryDependencies = new SubscriberFactoryDependencies( $contentRepositoryId, $eventNormalizer, $nodeTypeManager, @@ -83,18 +94,26 @@ public function __construct( $interDimensionalVariationGraph, new PropertyConverter($propertySerializer) ); - $subscribers = []; - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $subscribers[] = new Subscriber( - SubscriptionId::fromString(substr(strrchr($projection::class, '\\'), 1)), - SubscriptionGroup::fromString('default'), - RunMode::FROM_BEGINNING, - new ProjectionEventHandler( - $projection, - $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection)?->build($this->getOrBuild()), - ), - ); + $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); + $contentGraphSubscriber = new Subscriber( + SubscriptionId::fromString('contentGraph'), + SubscriptionGroup::fromString('default'), + RunMode::FROM_BEGINNING, + ProjectionEventHandler::createWithCatchUpHook( + $this->contentGraphProjection, + $contentGraphCatchUpHookFactory->build(new CatchUpHookFactoryDependencies($this->contentRepositoryId, $this->contentGraphProjection->getState(), $nodeTypeManager, $contentDimensionSource, $interDimensionalVariationGraph)), + ), + ); + $subscribers = [$contentGraphSubscriber]; + $projectionStates = []; + foreach ($additionalSubscriberFactories as $subscriberFactory) { + $subscriber = $subscriberFactory->build($this->subscriberFactoryDependencies); + $subscribers[] = $subscriber; + if ($subscriber->handler instanceof ProjectionEventHandler) { + $projectionStates[] = $subscriber->handler->projection->getState(); + } } + $this->projectionStates = ProjectionStates::fromArray($projectionStates); $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy()); $this->eventStore = new RunSubscriptionEventStore($eventStore, $this->subscriptionEngine); } @@ -115,32 +134,32 @@ public function getOrBuild(): ContentRepository return $this->contentRepository; } - $contentGraphReadModel = $this->projectionsAndCatchUpHooks->contentGraphProjection->getState(); + $contentGraphReadModel = $this->contentGraphProjection->getState(); $commandHandlingDependencies = new CommandHandlingDependencies($contentGraphReadModel); // we dont need full recursion in rebase - e.g apply workspace commands - and thus we can use this set for simulation $commandBusForRebaseableCommands = new CommandBus( $commandHandlingDependencies, new NodeAggregateCommandHandler( - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, - $this->projectionFactoryDependencies->propertyConverter, + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionZookeeper, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->propertyConverter, ), new DimensionSpaceCommandHandler( - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->contentDimensionZookeeper, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, ), new NodeDuplicationCommandHandler( - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionZookeeper, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, ) ); $commandSimulatorFactory = new CommandSimulatorFactory( - $this->projectionsAndCatchUpHooks->contentGraphProjection, - $this->projectionFactoryDependencies->eventNormalizer, + $this->contentGraphProjection, + $this->subscriberFactoryDependencies->eventNormalizer, $commandBusForRebaseableCommands ); @@ -148,26 +167,21 @@ public function getOrBuild(): ContentRepository new WorkspaceCommandHandler( $commandSimulatorFactory, $this->eventStore, - $this->projectionFactoryDependencies->eventNormalizer, + $this->subscriberFactoryDependencies->eventNormalizer, ) ); - $projectionStates = []; - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projectionStates[] = $projection->getState(); - } - return $this->contentRepository = new ContentRepository( $this->contentRepositoryId, $publicCommandBus, $this->buildEventPersister(), - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, - $this->projectionFactoryDependencies->contentDimensionSource, + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->subscriberFactoryDependencies->contentDimensionSource, $this->userIdProvider, $this->clock, $contentGraphReadModel, - ProjectionStates::fromArray($projectionStates), + $this->projectionStates, ); } @@ -187,7 +201,7 @@ public function buildService( ): ContentRepositoryServiceInterface { $serviceFactoryDependencies = ContentRepositoryServiceFactoryDependencies::create( - $this->projectionFactoryDependencies, + $this->subscriberFactoryDependencies, $this->eventStore, $this->getOrBuild(), $this->buildEventPersister(), @@ -201,7 +215,7 @@ private function buildEventPersister(): EventPersister if (!$this->eventPersister) { $this->eventPersister = new EventPersister( $this->eventStore, - $this->projectionFactoryDependencies->eventNormalizer, + $this->subscriberFactoryDependencies->eventNormalizer, ); } return $this->eventPersister; diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index 0ac58b7a9d5..401f918a82a 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -54,7 +54,7 @@ private function __construct( * @internal */ public static function create( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, EventStoreInterface $eventStore, ContentRepository $contentRepository, EventPersister $eventPersister, diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php new file mode 100644 index 00000000000..bfec8810370 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php @@ -0,0 +1,38 @@ + + */ +final class ContentRepositorySubscriberFactories implements \IteratorAggregate +{ + private array $subscriberFactories; + + private function __construct(ContentRepositorySubscriberFactoryInterface ...$subscriberFactories) + { + $this->subscriberFactories = $subscriberFactories; + } + + public static function fromArray(array $subscriberFactories): self + { + return new self(...$subscriberFactories); + } + + public static function none(): self + { + return new self(); + } + + public function isEmpty(): bool + { + return $this->subscriberFactories === []; + } + + public function getIterator(): \Traversable + { + yield from $this->subscriberFactories; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php new file mode 100644 index 00000000000..1102a9342c5 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php @@ -0,0 +1,25 @@ +subscriptionId, + SubscriptionGroup::fromString('projections'), + RunMode::FROM_BEGINNING, + ProjectionEventHandler::create($this->projectionFactory->build($dependencies, $this->projectionFactoryOptions)), + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php similarity index 96% rename from Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php rename to Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php index e317f6726a5..8a6af1f94a4 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php @@ -26,7 +26,7 @@ /** * @api because it is used inside the ProjectionsFactory */ -final readonly class ProjectionFactoryDependencies +final readonly class SubscriberFactoryDependencies { public function __construct( public ContentRepositoryId $contentRepositoryId, diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php similarity index 93% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php index efa364124ba..ce8d8ac5673 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php @@ -2,7 +2,9 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; + +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** * @implements CatchUpHookFactoryInterface @@ -30,7 +32,6 @@ public static function create(): self /** * @param CatchUpHookFactoryInterface $catchUpHookFactory - * @return self */ public function with(CatchUpHookFactoryInterface $catchUpHookFactory): self { diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php similarity index 77% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php index 037e4164150..eb86feab742 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php @@ -12,15 +12,16 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface as T; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; /** - * @template-covariant T of ProjectionStateInterface + * @template-covariant T of T * * @api provides available dependencies for implementing a catch-up hook. */ @@ -28,11 +29,11 @@ { /** * @param ContentRepositoryId $contentRepositoryId the content repository the catchup was registered in - * @param ProjectionStateInterface&T $projectionState the state of the projection the catchup was registered to (Its only safe to access this projections state) + * @param T&T $projectionState the state of the projection the catchup was registered to (Its only safe to access this projections state) */ public function __construct( public ContentRepositoryId $contentRepositoryId, - public ProjectionStateInterface $projectionState, + public T $projectionState, public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, public InterDimensionalVariationGraph $variationGraph diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php similarity index 83% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php index 82fc7fea7b4..86e06d571cc 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php @@ -2,7 +2,9 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; + +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** * @template T of ProjectionStateInterface diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php similarity index 57% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index fbdfed6e8d5..e3f67b10c7c 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -2,9 +2,11 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\EventStore\EventInterface; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -18,12 +20,12 @@ interface CatchUpHookInterface { /** - * This hook is called at the beginning of {@see ProjectionInterface::catchUpProjection()}; - * BEFORE the Database Lock is acquired (by {@see CheckpointStorageInterface::acquireLock()}). + * This hook is called at the beginning of a catch-up run; + * BEFORE the Database Lock is acquired ({@see SubscriptionEngine::run()}). * * @return void */ - public function onBeforeCatchUp(): void; + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; /** * This hook is called for every event during the catchup process, **before** the projection @@ -38,20 +40,7 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; /** - * This hook is called directly before the database lock is RELEASED - * in {@see CheckpointStorageInterface::updateAndReleaseLock()}. - * - * It can happen that this method is called multiple times, even without - * having seen Events in the meantime. - * - * If there exist more events which need to be processed, the database lock - * is directly acquired again after it is released. - */ - public function onBeforeBatchCompleted(): void; - - /** - * This hook is called at the END of {@see ProjectionInterface::catchUpProjection()}, directly - * before exiting the method. + * This hook is called at the END of a catch-up run * * At this point, the Database Lock has already been released. */ diff --git a/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php similarity index 76% rename from Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php index 3edc73b751d..12a2e73c9b8 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\EventStore\EventInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -13,7 +14,7 @@ * * @internal */ -class DelegatingCatchUpHook implements CatchUpHookInterface +final class DelegatingCatchUpHook implements CatchUpHookInterface { /** * @var CatchUpHookInterface[] @@ -26,10 +27,10 @@ public function __construct( $this->catchUpHooks = $catchUpHooks; } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeCatchUp(); + $catchUpHook->onBeforeCatchUp($subscriptionStatus); } } @@ -47,13 +48,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event } } - public function onBeforeBatchCompleted(): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeBatchCompleted(); - } - } - public function onAfterCatchUp(): void { foreach ($this->catchUpHooks as $catchUpHook) { diff --git a/Neos.ContentRepository.Core/Classes/Projection/ContentGraph/ContentGraphProjectionFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ContentGraph/ContentGraphProjectionFactoryInterface.php index a385d454e76..a18914821f0 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ContentGraph/ContentGraphProjectionFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ContentGraph/ContentGraphProjectionFactoryInterface.php @@ -4,20 +4,14 @@ namespace Neos\ContentRepository\Core\Projection\ContentGraph; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; -use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; /** - * @extends ProjectionFactoryInterface * @api for creating a custom content repository graph projection implementation, **not for users of the CR** */ -interface ContentGraphProjectionFactoryInterface extends ProjectionFactoryInterface +interface ContentGraphProjectionFactoryInterface { - /** - * @param array $options - */ public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): ContentGraphProjectionInterface; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php index 9694e5f5275..580fb35dd82 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php @@ -5,36 +5,56 @@ namespace Neos\ContentRepository\Core\Projection; use Neos\ContentRepository\Core\EventStore\EventInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Subscription\Subscriber\EventHandlerInterface; -use Neos\ContentRepository\Core\Subscription\Subscription; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** - * @api + * @internal */ final readonly class ProjectionEventHandler implements EventHandlerInterface { - public function __construct( - private ProjectionInterface $projection, + /** + * @param ProjectionInterface $projection + */ + private function __construct( + public ProjectionInterface $projection, private CatchUpHookInterface|null $catchUpHook, ) { } - public function startBatch(): void + /** + * @param ProjectionInterface $projection + * @return self + */ + public static function create(ProjectionInterface $projection): self { - $this->catchUpHook?->onBeforeCatchUp(); + return new self($projection, null); } - public function handle(EventInterface $event, EventEnvelope $eventEnvelope, Subscription $subscription): void + /** + * @param ProjectionInterface $projection + */ + public static function createWithCatchUpHook(ProjectionInterface $projection, CatchUpHookInterface $catchUpHook): self + { + return new self($projection, $catchUpHook); + } + + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void + { + $this->catchUpHook?->onBeforeCatchUp($subscriptionStatus); + } + + public function handle(EventInterface $event, EventEnvelope $eventEnvelope): void { $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); $this->projection->apply($event, $eventEnvelope); $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); } - public function endBatch(): void + public function onAfterCatchUp(): void { - $this->catchUpHook?->onBeforeBatchCompleted(); $this->catchUpHook?->onAfterCatchUp(); } } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php index 6c0de0992d5..d2f03269b1d 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\Core\Projection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; /** * @template-covariant T of ProjectionInterface @@ -17,7 +17,7 @@ interface ProjectionFactoryInterface * @return T */ public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ProjectionInterface; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/Projections.php b/Neos.ContentRepository.Core/Classes/Projection/Projections.php index 766f4c6d1d2..a66d0ee9e3b 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/Projections.php +++ b/Neos.ContentRepository.Core/Classes/Projection/Projections.php @@ -55,6 +55,9 @@ public static function fromArray(array $projections): self return new self(...$projectionsByClassName); } + /** + * @return ProjectionInterface + */ public function get(SubscriptionId $id): ProjectionInterface { if (!$this->has($id)) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 705905bd68f..01a2989d884 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -96,6 +96,9 @@ private function runInternal(SubscriptionCriteria $criteria, string $process, in } $this->lockSubscriptions($subscriptions); + foreach ($subscriptions as $subscription) { + $this->subscribers->get($subscription->id)->handler->onBeforeCatchUp($subscription->status); + } $startSequenceNumber = $this->lowestSubscriptionPosition($subscriptions)->next(); $this->logger?->debug( @@ -182,6 +185,9 @@ private function runInternal(SubscriptionCriteria $criteria, string $process, in ), ); $this->releaseSubscriptions($subscriptions); + foreach ($subscriptions as $subscription) { + $this->subscribers->get($subscription->id)->handler->onAfterCatchUp(); + } return;// new ProcessedResult($messageCounter, true, $errors); } @@ -190,7 +196,7 @@ private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domai { $subscriber = $this->subscribers->get($subscription->id); try { - $subscriber->handler->handle($domainEvent, $eventEnvelope, $subscription); + $subscriber->handler->handle($domainEvent, $eventEnvelope); } catch (\Throwable $e) { $this->logger?->error( sprintf( diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php index d5cb6d9e000..a01cc15fc29 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription\Subscriber; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Subscription\Subscription; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -13,7 +13,7 @@ */ interface EventHandlerInterface { - public function startBatch(): void; - public function handle(EventInterface $event, EventEnvelope $eventEnvelope, Subscription $subscription): void; - public function endBatch(): void; + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; + public function handle(EventInterface $event, EventEnvelope $eventEnvelope): void; + public function onAfterCatchUp(): void; } diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index cdeaba74f93..7234e480d99 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -6,17 +6,21 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; +use Neos\ContentRepository\Core\Factory\AdditionalSubscribersFactory; use Neos\ContentRepository\Core\Factory\ContentRepositoryFactory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\Factory\ContentRepositorySubscriberFactories; +use Neos\ContentRepository\Core\Factory\ContentRepositorySubscribersFactoryInterface; +use Neos\ContentRepository\Core\Factory\ProjectionSubscriberFactory; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactories; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHooks; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryIds; use Neos\ContentRepository\Core\SharedModel\User\UserIdProviderInterface; @@ -176,10 +180,12 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $this->buildNodeTypeManager($contentRepositoryId, $contentRepositorySettings), $this->buildContentDimensionSource($contentRepositoryId, $contentRepositorySettings), $this->buildPropertySerializer($contentRepositoryId, $contentRepositorySettings), - $this->buildProjectionsFactory($contentRepositoryId, $contentRepositorySettings), $this->buildUserIdProvider($contentRepositoryId, $contentRepositorySettings), $clock, $this->buildSubscriptionStore($contentRepositoryId, $clock, $contentRepositorySettings), + $this->buildContentGraphProjectionFactory($contentRepositoryId, $contentRepositorySettings), + $this->buildContentGraphCatchUpHookFactory($contentRepositoryId, $contentRepositorySettings), + $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), ); } catch (\Exception $exception) { throw InvalidConfigurationException::fromException($contentRepositoryId, $exception); @@ -243,58 +249,65 @@ private function buildPropertySerializer(ContentRepositoryId $contentRepositoryI } /** @param array $contentRepositorySettings */ - private function buildProjectionsFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ProjectionsAndCatchUpHooksFactory + private function buildContentGraphProjectionFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentGraphProjectionFactoryInterface { - $projectionsAndCatchUpHooksFactory = new ProjectionsAndCatchUpHooksFactory(); - - // content graph projection: if (!isset($contentRepositorySettings['contentGraphProjection']['factoryObjectName'])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.factoryObjectName configured.', $contentRepositoryId->value); } - $projectionFactory = $this->objectManager->get($contentRepositorySettings['contentGraphProjection']['factoryObjectName']); - if (!$projectionFactory instanceof ContentGraphProjectionFactoryInterface) { - throw InvalidConfigurationException::fromMessage('Projection factory object name of contentGraphProjection (content repository "%s") is not an instance of %s but %s.', $contentRepositoryId->value, ContentGraphProjectionFactoryInterface::class, get_debug_type($projectionFactory)); + $contentGraphProjectionFactory = $this->objectManager->get($contentRepositorySettings['contentGraphProjection']['factoryObjectName']); + if (!$contentGraphProjectionFactory instanceof ContentGraphProjectionFactoryInterface) { + throw InvalidConfigurationException::fromMessage('Projection factory object name of contentGraphProjection (content repository "%s") is not an instance of %s but %s.', $contentRepositoryId->value, ContentGraphProjectionFactoryInterface::class, get_debug_type($contentGraphProjectionFactory)); } - $projectionId = SubscriptionId::fromString('contentGraph'); - $projectionsAndCatchUpHooksFactory->registerFactory($projectionId, $projectionFactory, $contentRepositorySettings['contentGraphProjection']['options'] ?? []); - - $this->registerCatchupHookForProjection($contentRepositorySettings['contentGraphProjection'], $projectionsAndCatchUpHooksFactory, $projectionId, $contentRepositoryId); + return $contentGraphProjectionFactory; + } - // additional projections: - (is_array($contentRepositorySettings['projections'] ?? [])) || throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); - foreach ($contentRepositorySettings['projections'] ?? [] as $id => $projectionOptions) { - if ($projectionOptions === null) { + private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactoryInterface + { + if (!isset($contentRepositorySettings['contentGraphProjection']['catchUpHooks'])) { + throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.catchUpHooks configured.', $contentRepositoryId->value); + } + $catchUpHookFactories = CatchUpHookFactories::create(); + foreach ($contentRepositorySettings['contentGraphProjection']['catchUpHooks'] as $catchUpHookName => $catchUpHookOptions) { + if ($catchUpHookOptions === null) { + // Allow catch up hooks to be disabled by setting their configuration to `null` continue; } - $projectionId = SubscriptionId::fromString($id); - (is_array($projectionOptions)) || throw InvalidConfigurationException::fromMessage('Projection "%s" (content repository "%s") must be configured as array got %s', $projectionId->value, $contentRepositoryId->value, get_debug_type($projectionOptions)); - $projectionFactory = isset($projectionOptions['factoryObjectName']) ? $this->objectManager->get($projectionOptions['factoryObjectName']) : null; - if (!$projectionFactory instanceof ProjectionFactoryInterface) { - throw InvalidConfigurationException::fromMessage('Projection factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s.', $projectionId->value, $contentRepositoryId->value, ProjectionFactoryInterface::class, get_debug_type($projectionFactory)); + $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); + if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { + throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for content graph CatchUpHook "%s" (content repository "%s") is not an instance of %s but %s', $catchUpHookName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); } - $projectionsAndCatchUpHooksFactory->registerFactory($projectionId, $projectionFactory, $projectionOptions['options'] ?? []); - - $this->registerCatchupHookForProjection($projectionOptions, $projectionsAndCatchUpHooksFactory, $projectionId, $contentRepositoryId); + $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); } - return $projectionsAndCatchUpHooksFactory; + return $catchUpHookFactories; } - /** - * @param ProjectionFactoryInterface> $projectionFactory - */ - private function registerCatchupHookForProjection(mixed $projectionOptions, ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, SubscriptionId $projectionId, ContentRepositoryId $contentRepositoryId): void + private function buildAdditionalSubscribersFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories { - foreach (($projectionOptions['catchUpHooks'] ?? []) as $catchUpHookOptions) { - if ($catchUpHookOptions === null) { + if (!is_array($contentRepositorySettings['projections'] ?? [])) { + throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); + } + /** @var array $projectionFactories */ + $projectionSubscriberFactories = []; + foreach (($contentRepositorySettings['projections'] ?? []) as $projectionName => $projectionOptions) { + // Allow projections to be disabled by setting their configuration to `null` + if ($projectionOptions === null) { continue; } - $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); - if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { - throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s', $projectionId->value, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); + if (!is_array($projectionOptions)) { + throw InvalidConfigurationException::fromMessage('Projection "%s" (content repository "%s") must be configured as array got %s', $projectionName, $contentRepositoryId->value, get_debug_type($projectionOptions)); + } + $projectionFactory = isset($projectionOptions['factoryObjectName']) ? $this->objectManager->get($projectionOptions['factoryObjectName']) : null; + if (!$projectionFactory instanceof ProjectionFactoryInterface) { + throw InvalidConfigurationException::fromMessage('Projection factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s.', $projectionName, $contentRepositoryId->value, ProjectionFactoryInterface::class, get_debug_type($projectionFactory)); } - $projectionsAndCatchUpHooksFactory->registerCatchUpHookFactory($projectionId, $catchUpHookFactory); + $projectionSubscriberFactories[$projectionName] = new ProjectionSubscriberFactory( + SubscriptionId::fromString($projectionName), + $projectionFactory, + $projectionOptions['options'] ?? [], + ); } + return ContentRepositorySubscriberFactories::fromArray($projectionSubscriberFactories); } /** @param array $contentRepositorySettings */ diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index a5203af7205..b7faee41150 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -27,6 +27,9 @@ use Neos\EventStore\Model\Event\SequenceNumber; use Psr\Clock\ClockInterface; +/** + * @internal + */ final class DoctrineSubscriptionStore implements SubscriptionStoreInterface { public function __construct( diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php index 5643afb5fb2..97c1773a88c 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php @@ -1,20 +1,11 @@ subgraphCachePool->reset(); } - public function onBeforeBatchCompleted(): void - { - } - public function onAfterCatchUp(): void { $this->subgraphCachePool->reset(); diff --git a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php index cebe03ebb43..f3017920113 100644 --- a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php @@ -4,9 +4,9 @@ namespace Neos\ContentRepositoryRegistry\SubgraphCachingInMemory; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php index e52b55bb99c..bf739401dc4 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php @@ -7,7 +7,6 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Feature\Common\EmbedsContentStreamId; use Neos\ContentRepository\Core\Feature\Common\EmbedsWorkspaceName; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Event\DimensionSpacePointWasMoved; use Neos\ContentRepository\Core\Feature\NodeCreation\Event\NodeAggregateWithNodeWasCreated; @@ -19,7 +18,7 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Dto\NodeIdsToPublishOrDiscard; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindDescendantNodesFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; @@ -28,6 +27,7 @@ use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Neos\AssetUsage\Service\AssetUsageIndexingService; @@ -43,7 +43,7 @@ public function __construct( ) { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -88,11 +88,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } - - public function onBeforeBatchCompleted(): void - { - } - public function onAfterCatchUp(): void { } diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php index 4188a10072f..71cf34fe2ea 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php @@ -14,8 +14,8 @@ * source code. */ -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\Neos\AssetUsage\Service\AssetUsageIndexingService; diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php index 1dc70d4fac7..b6bd864021b 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php @@ -10,8 +10,9 @@ use Neos\ContentRepository\Core\Feature\NodeMove\Event\NodeAggregateWasMoved; use Neos\ContentRepository\Core\Feature\NodeRemoval\Event\NodeAggregateWasRemoved; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Mvc\Routing\RouterCachingService; use Neos\Neos\FrontendRouting\Exception\NodeNotFoundException; @@ -32,7 +33,7 @@ public function __construct( ) { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { // Nothing to do here } @@ -59,11 +60,6 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } - public function onBeforeBatchCompleted(): void - { - // Nothing to do here - } - public function onAfterCatchUp(): void { // Nothing to do here diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php index cbdd469d930..b0c4faf2dd6 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php @@ -4,9 +4,9 @@ namespace Neos\Neos\FrontendRouting\CatchUpHook; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\Flow\Mvc\Routing\RouterCachingService; use Neos\Neos\FrontendRouting\Projection\DocumentUriPathFinder; diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php index 570e0f5485d..c70dfb2dbd3 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php @@ -5,7 +5,7 @@ namespace Neos\Neos\FrontendRouting\Projection; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -35,7 +35,7 @@ public static function projectionTableNamePrefix( public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): DocumentUriPathProjection { diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php index 4acf758411e..6f60eb626d2 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php @@ -34,13 +34,14 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPartiallyDiscarded; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeAggregate; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateIds; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -135,7 +136,7 @@ public function canHandle(EventInterface $event): bool ]); } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -244,10 +245,6 @@ private function scheduleCacheFlushJobForWorkspaceName( ); } - public function onBeforeBatchCompleted(): void - { - } - public function onAfterCatchUp(): void { foreach ($this->flushNodeAggregateRequestsOnAfterCatchUp as $request) { diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php index a988f1bcac6..35a07b11b70 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php @@ -14,8 +14,8 @@ * source code. */ -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php index 8b231897d78..244c1352426 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php @@ -15,7 +15,7 @@ namespace Neos\Neos\PendingChangesProjection; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; /** @@ -29,7 +29,7 @@ public function __construct( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ChangeProjection { return new ChangeProjection( From 207179b55de1983468a8ae6d1df9af624698eb10 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Wed, 6 Nov 2024 12:36:35 +0100 Subject: [PATCH 003/142] Tweak type comments --- .../Classes/Factory/ContentRepositoryFactory.php | 4 ++++ .../CatchUpHook/CatchUpHookFactoryDependencies.php | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index aa985d65a63..4561981c4ca 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -31,6 +31,7 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; @@ -66,6 +67,9 @@ final class ContentRepositoryFactory private ContentGraphProjectionInterface $contentGraphProjection; + /** + * @param CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory + */ public function __construct( private readonly ContentRepositoryId $contentRepositoryId, EventStoreInterface $eventStore, diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php index eb86feab742..5a3da2cf9ed 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php @@ -17,11 +17,11 @@ use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\ProjectionStateInterface as T; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; /** - * @template-covariant T of T + * @template-covariant T of ProjectionStateInterface * * @api provides available dependencies for implementing a catch-up hook. */ @@ -29,11 +29,11 @@ { /** * @param ContentRepositoryId $contentRepositoryId the content repository the catchup was registered in - * @param T&T $projectionState the state of the projection the catchup was registered to (Its only safe to access this projections state) + * @param ProjectionStateInterface&T $projectionState the state of the projection the catchup was registered to (Its only safe to access this projections state) */ public function __construct( public ContentRepositoryId $contentRepositoryId, - public T $projectionState, + public ProjectionStateInterface $projectionState, public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, public InterDimensionalVariationGraph $variationGraph From a12770628155b69627de45022f94ddb564869c39 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Wed, 13 Nov 2024 17:06:37 +0100 Subject: [PATCH 004/142] Wiring... --- .../RaceTrackerCatchUpHook.php | 2 +- .../Factory/ContentRepositoryFactory.php | 72 ++++++++++--------- .../ContentRepositorySubscriberFactories.php | 8 +++ ...tRepositorySubscribersFactoryInterface.php | 28 ++++++++ .../Factory/ProjectionSubscriberFactory.php | 8 +++ .../Classes/Service/ContentStreamPruner.php | 2 +- .../Engine/SubscriptionEngine.php | 2 +- .../Subscription/Subscriber/Subscribers.php | 5 ++ .../Classes/Subscription/SubscriptionIds.php | 2 +- .../Classes/Command/CrCommandController.php | 4 +- .../Classes/ContentRepositoryRegistry.php | 4 +- .../DoctrineSubscriptionStore.php | 2 +- .../Service/ProjectionReplayService.php | 4 +- phpstan-baseline.neon | 5 -- 14 files changed, 100 insertions(+), 48 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php index 1ce532ca5fd..39eaeb70b93 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php @@ -74,7 +74,7 @@ * * When {@see onBeforeEvent} is called, we know that we are inside applyEvent() in the diagram above, * thus we know the lock *HAS* been acquired. - * When {@see onBeforeBatchCompleted}is called, we know the lock will be released directly afterwards. + * When {@see onAfterCatchUp}is called, we know the lock will be released directly afterwards. * * We track these timings across processes in a single Redis Stream. Because Redis is single-threaded, * we can be sure that we observe the correct, total order of interleavings *across multiple processes* diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 4561981c4ca..c820e4af773 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -34,6 +34,7 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -43,7 +44,6 @@ use Neos\ContentRepository\Core\Subscription\RetryStrategy\NoRetryStrategy; use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; -use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionEventHandlerInterface; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; @@ -62,13 +62,15 @@ final class ContentRepositoryFactory private SubscriberFactoryDependencies $subscriberFactoryDependencies; private EventStoreInterface $eventStore; private SubscriptionEngine $subscriptionEngine; - - private ProjectionStates $projectionStates; - private ContentGraphProjectionInterface $contentGraphProjection; + private ProjectionStates $additionalProjectionStates; + + // The following properties store "singleton" references of objects for this content repository + private ?ContentRepository $contentRepositoryRuntimeCache = null; + private ?EventPersister $eventPersisterRuntimeCache = null; /** - * @param CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory + * @param CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory */ public function __construct( private readonly ContentRepositoryId $contentRepositoryId, @@ -80,8 +82,8 @@ public function __construct( private readonly ClockInterface $clock, SubscriptionStoreInterface $subscriptionStore, ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, - CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, - ContentRepositorySubscriberFactories $additionalSubscriberFactories, + private readonly CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, + private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); $interDimensionalVariationGraph = new InterDimensionalVariationGraph( @@ -98,34 +100,40 @@ public function __construct( $interDimensionalVariationGraph, new PropertyConverter($propertySerializer) ); + $subscribers = [$this->buildContentGraphSubscriber()]; + $additionalProjectionStates = []; + foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { + $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); + if ($subscriber->handler instanceof ProjectionEventHandler) { + $additionalProjectionStates[] = $subscriber->handler->projection->getState(); + } + $subscribers[] = $subscriber; + } + $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); - $contentGraphSubscriber = new Subscriber( + $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy()); + $this->eventStore = new RunSubscriptionEventStore($eventStore, $this->subscriptionEngine); + } + + private function buildContentGraphSubscriber(): Subscriber + { + return new Subscriber( SubscriptionId::fromString('contentGraph'), SubscriptionGroup::fromString('default'), RunMode::FROM_BEGINNING, ProjectionEventHandler::createWithCatchUpHook( $this->contentGraphProjection, - $contentGraphCatchUpHookFactory->build(new CatchUpHookFactoryDependencies($this->contentRepositoryId, $this->contentGraphProjection->getState(), $nodeTypeManager, $contentDimensionSource, $interDimensionalVariationGraph)), + $this->contentGraphCatchUpHookFactory->build(new CatchUpHookFactoryDependencies( + $this->contentRepositoryId, + $this->contentGraphProjection->getState(), + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionSource, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + )), ), ); - $subscribers = [$contentGraphSubscriber]; - $projectionStates = []; - foreach ($additionalSubscriberFactories as $subscriberFactory) { - $subscriber = $subscriberFactory->build($this->subscriberFactoryDependencies); - $subscribers[] = $subscriber; - if ($subscriber->handler instanceof ProjectionEventHandler) { - $projectionStates[] = $subscriber->handler->projection->getState(); - } - } - $this->projectionStates = ProjectionStates::fromArray($projectionStates); - $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy()); - $this->eventStore = new RunSubscriptionEventStore($eventStore, $this->subscriptionEngine); } - // The following properties store "singleton" references of objects for this content repository - private ?ContentRepository $contentRepository = null; - private ?EventPersister $eventPersister = null; - /** * Builds and returns the content repository. If it is already built, returns the same instance. * @@ -134,8 +142,8 @@ public function __construct( */ public function getOrBuild(): ContentRepository { - if ($this->contentRepository) { - return $this->contentRepository; + if ($this->contentRepositoryRuntimeCache) { + return $this->contentRepositoryRuntimeCache; } $contentGraphReadModel = $this->contentGraphProjection->getState(); @@ -175,7 +183,7 @@ public function getOrBuild(): ContentRepository ) ); - return $this->contentRepository = new ContentRepository( + return $this->contentRepositoryRuntimeCache = new ContentRepository( $this->contentRepositoryId, $publicCommandBus, $this->buildEventPersister(), @@ -185,7 +193,7 @@ public function getOrBuild(): ContentRepository $this->userIdProvider, $this->clock, $contentGraphReadModel, - $this->projectionStates, + $this->additionalProjectionStates, ); } @@ -216,12 +224,12 @@ public function buildService( private function buildEventPersister(): EventPersister { - if (!$this->eventPersister) { - $this->eventPersister = new EventPersister( + if (!$this->eventPersisterRuntimeCache) { + $this->eventPersisterRuntimeCache = new EventPersister( $this->eventStore, $this->subscriberFactoryDependencies->eventNormalizer, ); } - return $this->eventPersister; + return $this->eventPersisterRuntimeCache; } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php index bfec8810370..fa16ebb2594 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php @@ -6,9 +6,13 @@ /** * @implements \IteratorAggregate + * @internal */ final class ContentRepositorySubscriberFactories implements \IteratorAggregate { + /** + * @var array + */ private array $subscriberFactories; private function __construct(ContentRepositorySubscriberFactoryInterface ...$subscriberFactories) @@ -16,6 +20,10 @@ private function __construct(ContentRepositorySubscriberFactoryInterface ...$sub $this->subscriberFactories = $subscriberFactories; } + /** + * @param array $subscriberFactories + * @return self + */ public static function fromArray(array $subscriberFactories): self { return new self(...$subscriberFactories); diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php new file mode 100644 index 00000000000..90fa7021cf6 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php @@ -0,0 +1,28 @@ + $projectionFactory + * @param array $projectionFactoryOptions + */ public function __construct( private SubscriptionId $subscriptionId, private ProjectionFactoryInterface $projectionFactory, diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 5afbcddc590..b0658e8a565 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -162,7 +162,7 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta if ($danglingContentStreamsPresent) { try { - $this->contentRepository->catchUpProjections(); + //TODO $this->contentRepository->catchUpProjections(); } catch (\Throwable $e) { $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 01a2989d884..9c8df9308c8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -21,7 +21,7 @@ use Neos\ContentRepository\Core\Subscription\Subscriptions; /** - * @internal + * @api */ final class SubscriptionEngine { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php index 1648386faf2..b34c736d6e8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php @@ -43,6 +43,11 @@ public static function none(): self return self::fromArray([]); } + public function with(Subscriber $subscriber): self + { + return new self([...$this->subscribersById, $subscriber->id->value => $subscriber]); + } + public function get(SubscriptionId $id): Subscriber { if (!$this->contain($id)) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php index 2d346a81886..78ab33296dd 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @internal + * @api */ final class SubscriptionIds implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 63e6fa96ffc..1b2b1a7c506 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -44,7 +44,7 @@ public function setupCommand(string $contentRepository = 'default'): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $this->contentRepositoryRegistry->get($contentRepositoryId)->setUp(); + // TODO $this->contentRepositoryRegistry->get($contentRepositoryId); $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); } @@ -63,7 +63,7 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $status = $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); + $status = null;//TODO $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); $this->output('Event Store: '); $this->outputLine(match ($status->eventStoreStatus->type) { diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 7234e480d99..c3deef7e9f9 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -6,17 +6,14 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; -use Neos\ContentRepository\Core\Factory\AdditionalSubscribersFactory; use Neos\ContentRepository\Core\Factory\ContentRepositoryFactory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Factory\ContentRepositorySubscriberFactories; -use Neos\ContentRepository\Core\Factory\ContentRepositorySubscribersFactoryInterface; use Neos\ContentRepository\Core\Factory\ProjectionSubscriberFactory; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactories; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHooks; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; @@ -262,6 +259,7 @@ private function buildContentGraphProjectionFactory(ContentRepositoryId $content return $contentGraphProjectionFactory; } + /** @param array $contentRepositorySettings */ private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactoryInterface { if (!isset($contentRepositorySettings['contentGraphProjection']['catchUpHooks'])) { diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index b7faee41150..be7bfe97a18 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -28,7 +28,7 @@ use Psr\Clock\ClockInterface; /** - * @internal + * @api */ final class DoctrineSubscriptionStore implements SubscriptionStoreInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php index 97c1773a88c..1f80a776043 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php @@ -6,6 +6,7 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Projection\CatchUpOptions; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; /** * Content Repository service to perform Projection replays @@ -15,12 +16,13 @@ final class ProjectionReplayService implements ContentRepositoryServiceInterface { public function __construct( - //private readonly SubscriptionEngine $subscriptionEngine, + private readonly SubscriptionEngine $subscriptionEngine, ) { } public function replayProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void { + $this->subscriptionEngine->setup(); // TODO $this->subscriptionEngine->reset() // TODO $this->subscriptionEngine->run() } diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index ebbd2ba29a9..aab9e3e26d4 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -1,10 +1,5 @@ parameters: ignoreErrors: - - - message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Projection\\\\Projections\\:\\:getClassNames\" is called\\.$#" - count: 1 - path: Neos.ContentRepositoryRegistry/Classes/Service/ProjectionReplayService.php - - message: "#^Method Neos\\\\Neos\\\\Controller\\\\Backend\\\\MenuHelper\\:\\:buildModuleList\\(\\) return type has no value type specified in iterable type array\\.$#" count: 1 From 5b035c11121698dff3074a944e60c6dbb61861fc Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 14 Nov 2024 13:29:27 +0100 Subject: [PATCH 005/142] first (almost) working version --- .../src/ContentGraphTableNames.php | 5 - .../DoctrineDbalContentGraphProjection.php | 67 +-- .../Projection/HypergraphProjection.php | 67 +-- .../RaceTrackerCatchUpHook.php | 2 +- .../CommandHandler/CommandSimulator.php | 5 - .../Factory/ContentRepositoryFactory.php | 7 +- .../Infrastructure/DbalCheckpointStorage.php | 164 ------ .../Classes/Projection/CatchUp.php | 134 ----- .../Classes/Projection/CatchUpOptions.php | 62 --- .../Projection/CheckpointStorageInterface.php | 61 --- .../Projection/CheckpointStorageStatus.php | 55 -- .../CheckpointStorageStatusType.php | 12 - .../Projection/ProjectionEventHandler.php | 5 + .../Projection/ProjectionInterface.php | 8 +- .../Classes/Service/SubscriptionService.php | 26 + .../Service/SubscriptionServiceFactory.php | 24 + .../Classes/Subscription/Engine/Error.php | 11 +- .../Subscription/Engine/ProcessedResult.php | 2 +- .../Classes/Subscription/Engine/Result.php | 17 + .../Engine/SubscriptionEngine.php | 486 +++++++++--------- .../Engine/SubscriptionManager.php | 113 ++++ ...iptionEngineAlreadyProcessingException.php | 13 + .../Store/InMemorySubscriptionStore.php | 19 +- .../Store/SubscriptionCriteria.php | 38 +- .../Store/SubscriptionStoreInterface.php | 13 +- ...onStoreWithTransactionSupportInterface.php | 22 + .../Subscriber/EventHandlerInterface.php | 5 + .../Subscription/Subscriber/Subscribers.php | 4 +- .../Classes/Subscription/Subscription.php | 76 +-- .../Subscription/SubscriptionError.php | 4 +- .../Subscription/SubscriptionGroups.php | 47 +- .../Classes/Subscription/SubscriptionIds.php | 40 +- .../Subscription/SubscriptionStatusFilter.php | 64 +++ .../Classes/Subscription/Subscriptions.php | 89 ++-- .../Classes/Command/CrCommandController.php | 33 +- .../Classes/ContentRepositoryRegistry.php | 3 + .../DoctrineSubscriptionStore.php | 53 +- .../Classes/Service/ProjectionService.php | 2 +- .../Projection/DocumentUriPathProjection.php | 49 +- .../ChangeProjection.php | 48 +- 40 files changed, 748 insertions(+), 1207 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/CatchUp.php delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatus.php delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php create mode 100644 Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php create mode 100644 Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php index 787b5d24665..51f14b392cc 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php @@ -51,9 +51,4 @@ public function contentStream(): string { return $this->tableNamePrefix . '_contentstream'; } - - public function checkpoint(): string - { - return $this->tableNamePrefix . '_checkpoint'; - } } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 39186aa616f..4e998fde99a 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -61,10 +61,8 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeTags; @@ -75,7 +73,6 @@ use Neos\ContentRepository\Core\SharedModel\Node\NodeName; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; /** @@ -93,8 +90,6 @@ final class DoctrineDbalContentGraphProjection implements ContentGraphProjection public const RELATION_DEFAULT_OFFSET = 128; - private DbalCheckpointStorage $checkpointStorage; - public function __construct( private readonly Connection $dbal, private readonly ProjectionContentGraph $projectionContentGraph, @@ -102,11 +97,6 @@ public function __construct( private readonly DimensionSpacePointsRepository $dimensionSpacePointsRepository, private readonly ContentGraphReadModelInterface $contentGraphReadModel ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNames->checkpoint(), - self::class - ); } public function setUp(): void @@ -120,18 +110,10 @@ public function setUp(): void throw new \RuntimeException(sprintf('Failed to setup projection %s: %s', self::class, $e->getMessage()), 1716478255, $e); } } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -149,17 +131,9 @@ public function status(): ProjectionStatus return ProjectionStatus::ok(); } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); - } - - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; } public function getState(): ContentGraphReadModelInterface @@ -167,43 +141,6 @@ public function getState(): ContentGraphReadModelInterface return $this->contentGraphReadModel; } - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - ContentStreamWasClosed::class, - ContentStreamWasCreated::class, - ContentStreamWasForked::class, - ContentStreamWasRemoved::class, - ContentStreamWasReopened::class, - DimensionShineThroughWasAdded::class, - DimensionSpacePointWasMoved::class, - NodeAggregateNameWasChanged::class, - NodeAggregateTypeWasChanged::class, - NodeAggregateWasMoved::class, - NodeAggregateWasRemoved::class, - NodeAggregateWithNodeWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - NodePropertiesWereSet::class, - NodeReferencesWereSet::class, - NodeSpecializationVariantWasCreated::class, - RootNodeAggregateDimensionsWereUpdated::class, - RootNodeAggregateWithNodeWasCreated::class, - RootWorkspaceWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - WorkspaceBaseWorkspaceWasChanged::class, - WorkspaceRebaseFailed::class, - WorkspaceWasCreated::class, - WorkspaceWasDiscarded::class, - WorkspaceWasPartiallyDiscarded::class, - WorkspaceWasPartiallyPublished::class, - WorkspaceWasPublished::class, - WorkspaceWasRebased::class, - WorkspaceWasRemoved::class, - ]) || $event instanceof EmbedsContentStreamId; - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -238,7 +175,7 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void WorkspaceWasPublished::class => $this->whenWorkspaceWasPublished($event), WorkspaceWasRebased::class => $this->whenWorkspaceWasRebased($event), WorkspaceWasRemoved::class => $this->whenWorkspaceWasRemoved($event), - default => $event instanceof EmbedsContentStreamId || throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; if ( $event instanceof EmbedsContentStreamId diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 8c6ff9118b8..5961442b9e6 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -15,7 +15,6 @@ namespace Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection; use Doctrine\DBAL\Connection; -use Doctrine\DBAL\Schema\AbstractSchemaManager; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\ContentStreamForking; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeCreation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeModification; @@ -26,7 +25,6 @@ use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeVariation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\SubtreeTagging; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\SchemaBuilder\HypergraphSchemaBuilder; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked; use Neos\ContentRepository\Core\Feature\NodeCreation\Event\NodeAggregateWithNodeWasCreated; @@ -41,12 +39,10 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; -use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -66,7 +62,6 @@ final class HypergraphProjection implements ContentGraphProjectionInterface use NodeTypeChange; use NodeVariation; - private DbalCheckpointStorage $checkpointStorage; private ProjectionHypergraph $projectionHypergraph; public function __construct( @@ -75,11 +70,6 @@ public function __construct( private readonly ContentGraphReadModelInterface $contentGraphReadModel ) { $this->projectionHypergraph = new ProjectionHypergraph($this->dbal, $this->tableNamePrefix); - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } @@ -97,18 +87,10 @@ public function setUp(): void create index if not exists restriction_affected on ' . $this->tableNamePrefix . '_restrictionhyperrelation using gin (affectednodeaggregateids); '); - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->getDatabaseConnection()->connect(); } catch (\Throwable $e) { @@ -135,12 +117,9 @@ private function determineRequiredSqlStatements(): array return DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); } private function truncateDatabaseTables(): void @@ -151,39 +130,6 @@ private function truncateDatabaseTables(): void $this->dbal->executeQuery('TRUNCATE table ' . $this->tableNamePrefix . '_restrictionhyperrelation'); } - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - // ContentStreamForking - ContentStreamWasForked::class, - // NodeCreation - RootNodeAggregateWithNodeWasCreated::class, - NodeAggregateWithNodeWasCreated::class, - // SubtreeTagging - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - // NodeModification - NodePropertiesWereSet::class, - // NodeReferencing - NodeReferencesWereSet::class, - // NodeRemoval - NodeAggregateWasRemoved::class, - // NodeRenaming - NodeAggregateNameWasChanged::class, - // NodeTypeChange - NodeAggregateTypeWasChanged::class, - // NodeVariation - NodeSpecializationVariantWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - // TODO: not yet supported: - //ContentStreamWasRemoved::class, - //DimensionSpacePointWasMoved::class, - //DimensionShineThroughWasAdded::class, - //NodeAggregateWasMoved::class, - ]); - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -209,7 +155,7 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeSpecializationVariantWasCreated::class => $this->whenNodeSpecializationVariantWasCreated($event), NodeGeneralizationVariantWasCreated::class => $this->whenNodeGeneralizationVariantWasCreated($event), NodePeerVariantWasCreated::class => $this->whenNodePeerVariantWasCreated($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } @@ -228,11 +174,6 @@ public function inSimulation(\Closure $fn): mixed } } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): ContentGraphReadModelInterface { return $this->contentGraphReadModel; diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php index 39eaeb70b93..9f809c0b9ac 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php @@ -23,7 +23,7 @@ use Neos\Flow\Annotations as Flow; /** - * We had some race conditions in projections, where {@see \Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage} was not working properly. + * We had some race conditions in projections * We saw some non-deterministic, random errors when running the tests - unluckily only on Linux, not on OSX: * On OSX, forking a new subprocess in {@see SubprocessProjectionCatchUpTrigger} is *WAY* slower than in Linux; * and thus the race conditions which appears if two projector instances of the same class run concurrently diff --git a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php index 9becebc5a2d..f004613778f 100644 --- a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php +++ b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php @@ -135,11 +135,6 @@ private function handle(RebaseableCommand $rebaseableCommand): void foreach ($eventStream as $eventEnvelope) { $event = $this->eventNormalizer->denormalize($eventEnvelope->event); - - if (!$this->contentRepositoryProjection->canHandle($event)) { - continue; - } - $this->contentRepositoryProjection->apply($event, $eventEnvelope); } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index cb8baa235f9..744040ee0bf 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -49,6 +49,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\EventStore\EventStoreInterface; use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Serializer\Serializer; /** @@ -87,6 +88,7 @@ public function __construct( private readonly CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, + private LoggerInterface|null $logger = null, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); $interDimensionalVariationGraph = new InterDimensionalVariationGraph( @@ -103,7 +105,7 @@ public function __construct( $interDimensionalVariationGraph, new PropertyConverter($propertySerializer) ); - $subscribers = [$this->buildContentGraphSubscriber()]; + $subscribers = []; $additionalProjectionStates = []; foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); @@ -114,7 +116,8 @@ public function __construct( } $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); - $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy()); + $subscribers[] = $this->buildContentGraphSubscriber(); + $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy(), $this->logger); $this->eventStore = new RunSubscriptionEventStore($eventStore, $this->subscriptionEngine); } diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php deleted file mode 100644 index 8551d5febb8..00000000000 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php +++ /dev/null @@ -1,164 +0,0 @@ -connection->getDatabasePlatform(); - if (!($platform instanceof MySQLPlatform || $platform instanceof PostgreSqlPlatform)) { - throw new \InvalidArgumentException(sprintf('The %s only supports the platforms %s and %s currently. Given: %s', $this::class, MySQLPlatform::class, PostgreSQLPlatform::class, get_debug_type($platform)), 1660556004); - } - if (strlen($this->subscriberId) > 255) { - throw new \InvalidArgumentException('The subscriberId must not exceed 255 characters', 1705673456); - } - $this->platform = $platform; - } - - public function setUp(): void - { - foreach ($this->determineRequiredSqlStatements() as $statement) { - $this->connection->executeStatement($statement); - } - try { - $this->connection->insert($this->tableName, ['subscriberid' => $this->subscriberId, 'appliedsequencenumber' => 0]); - } catch (UniqueConstraintViolationException $e) { - // table and row already exists, ignore - } - } - - public function status(): CheckpointStorageStatus - { - try { - $this->connection->connect(); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to connect to database for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - try { - $requiredSqlStatements = $this->determineRequiredSqlStatements(); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to compare database schema for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - if ($requiredSqlStatements !== []) { - return CheckpointStorageStatus::setupRequired(sprintf('The following SQL statement%s required for subscriber "%s": %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', $this->subscriberId, implode(chr(10), $requiredSqlStatements))); - } - try { - $appliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->tableName . ' WHERE subscriberid = :subscriberId', ['subscriberId' => $this->subscriberId]); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to determine initial applied sequence number for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - if ($appliedSequenceNumber === false) { - return CheckpointStorageStatus::setupRequired(sprintf('Initial initial applied sequence number not set for subscriber "%s"', $this->subscriberId)); - } - return CheckpointStorageStatus::ok(); - } - - public function acquireLock(): SequenceNumber - { - if ($this->connection->isTransactionActive()) { - throw new \RuntimeException(sprintf('Failed to acquire checkpoint lock for subscriber "%s" because a transaction is active already', $this->subscriberId), 1652268416); - } - $this->connection->beginTransaction(); - try { - $highestAppliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->connection->quoteIdentifier($this->tableName) . ' WHERE subscriberid = :subscriberId ' . $this->platform->getForUpdateSQL() . ' NOWAIT', [ - 'subscriberId' => $this->subscriberId - ]); - } catch (DBALException $exception) { - $this->connection->rollBack(); - if ($exception instanceof LockWaitTimeoutException || ($exception instanceof DBALDriverException && ($exception->getCode() === 3572 || $exception->getCode() === 7))) { - throw new \RuntimeException(sprintf('Failed to acquire checkpoint lock for subscriber "%s" because it is acquired already', $this->subscriberId), 1652279016); - } - throw new \RuntimeException($exception->getMessage(), 1544207778, $exception); - } - if (!is_numeric($highestAppliedSequenceNumber)) { - $this->connection->rollBack(); - throw new \RuntimeException(sprintf('Failed to fetch highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279139); - } - $this->lockedSequenceNumber = SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); - return $this->lockedSequenceNumber; - } - - public function updateAndReleaseLock(SequenceNumber $sequenceNumber): void - { - if ($this->lockedSequenceNumber === null) { - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because the lock has not been acquired successfully before', $this->subscriberId), 1660556344); - } - if (!$this->connection->isTransactionActive()) { - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because no transaction is active', $this->subscriberId), 1652279314); - } - if ($this->connection->isRollbackOnly()) { - // TODO as described in https://github.com/neos/neos-development-collection/issues/4970 we are in a bad state and cannot commit after a nested transaction was rolled back. - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because the transaction has been marked for rollback only. See https://github.com/neos/neos-development-collection/issues/4970', $this->subscriberId), 1711964313); - } - try { - if (!$this->lockedSequenceNumber->equals($sequenceNumber)) { - $this->connection->update($this->tableName, ['appliedsequencenumber' => $sequenceNumber->value], ['subscriberid' => $this->subscriberId]); - } - $this->connection->commit(); - } catch (DBALException $exception) { - $this->connection->rollBack(); - throw new \RuntimeException(sprintf('Failed to update and commit highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279375, $exception); - } finally { - $this->lockedSequenceNumber = null; - } - } - - public function getHighestAppliedSequenceNumber(): SequenceNumber - { - $highestAppliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->connection->quoteIdentifier($this->tableName) . ' WHERE subscriberid = :subscriberId ', [ - 'subscriberId' => $this->subscriberId - ]); - if (!is_numeric($highestAppliedSequenceNumber)) { - throw new \RuntimeException(sprintf('Failed to fetch highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279427); - } - return SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); - } - - // -------------- - - /** - * @return array - */ - private function determineRequiredSqlStatements(): array - { - $schemaManager = $this->connection->createSchemaManager(); - $tableSchema = new Table( - $this->tableName, - [ - (new Column('subscriberid', Type::getType(Types::STRING)))->setLength(255), - (new Column('appliedsequencenumber', Type::getType(Types::INTEGER))) - ] - ); - $tableSchema->setPrimaryKey(['subscriberid']); - $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$tableSchema]); - return DbalSchemaDiff::determineRequiredSqlStatements($this->connection, $schema); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php deleted file mode 100644 index 35cd26467a9..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php +++ /dev/null @@ -1,134 +0,0 @@ -batchSize < 1) { - throw new \InvalidArgumentException(sprintf('batch size must be a positive integer, given: %d', $this->batchSize), 1705672467); - } - } - - /** - * @param \Closure(EventEnvelope): void $eventHandler The callback that is invoked for every {@see EventEnvelope} that is processed - * @param CheckpointStorageInterface $checkpointStorage The checkpoint storage that saves the last processed {@see SequenceNumber} - */ - public static function create(\Closure $eventHandler, CheckpointStorageInterface $checkpointStorage): self - { - return new self($eventHandler, $checkpointStorage, 1, null); - } - - /** - * After how many events should the (database) transaction be committed? - * - * @param int $batchSize Number of events to process before the checkpoint is written - */ - public function withBatchSize(int $batchSize): self - { - if ($batchSize === $this->batchSize) { - return $this; - } - return new self($this->eventHandler, $this->checkpointStorage, $batchSize, $this->onBeforeBatchCompletedHook); - } - - /** - * This hook is called directly before the sequence number is persisted back in CheckpointStorage. - * Use this to trigger any operation which need to happen BEFORE the sequence number update is made - * visible to the outside. - * - * Overrides all previously registered onBeforeBatchCompleted hooks. - * - * @param \Closure(): void $callback the hook being called before the batch is completed - */ - public function withOnBeforeBatchCompleted(\Closure $callback): self - { - return new self($this->eventHandler, $this->checkpointStorage, $this->batchSize, $callback); - } - - /** - * Iterate over the $eventStream, invoke the specified event handler closure for every {@see EventEnvelope} and update - * the last processed sequence number in the {@see CheckpointStorageInterface} - * - * @param EventStreamInterface $eventStream The event stream to process - * @return SequenceNumber The last processed {@see SequenceNumber} - * @throws \Throwable Exceptions that are thrown during callback handling are re-thrown - */ - public function run(EventStreamInterface $eventStream): SequenceNumber - { - $highestAppliedSequenceNumber = $this->checkpointStorage->acquireLock(); - $iteration = 0; - try { - foreach ($eventStream->withMinimumSequenceNumber($highestAppliedSequenceNumber->next()) as $eventEnvelope) { - if ($eventEnvelope->sequenceNumber->value <= $highestAppliedSequenceNumber->value) { - continue; - } - try { - ($this->eventHandler)($eventEnvelope); - } catch (\Exception $e) { - throw new \RuntimeException(sprintf('Exception while catching up to sequence number %d: %s', $eventEnvelope->sequenceNumber->value, $e->getMessage()), 1710707311, $e); - } - $iteration++; - if ($this->batchSize === 1 || $iteration % $this->batchSize === 0) { - if ($this->onBeforeBatchCompletedHook) { - ($this->onBeforeBatchCompletedHook)(); - } - $this->checkpointStorage->updateAndReleaseLock($eventEnvelope->sequenceNumber); - $highestAppliedSequenceNumber = $this->checkpointStorage->acquireLock(); - } else { - $highestAppliedSequenceNumber = $eventEnvelope->sequenceNumber; - } - } - } finally { - try { - if ($this->onBeforeBatchCompletedHook) { - ($this->onBeforeBatchCompletedHook)(); - } - } finally { - $this->checkpointStorage->updateAndReleaseLock($highestAppliedSequenceNumber); - } - } - return $highestAppliedSequenceNumber; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php deleted file mode 100644 index 8df856faccc..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpOptions.php +++ /dev/null @@ -1,62 +0,0 @@ -maximumSequenceNumber, - $progressCallback ?? $this->progressCallback, - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php deleted file mode 100644 index 43dc37f8ab7..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php +++ /dev/null @@ -1,61 +0,0 @@ -type, $details); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php deleted file mode 100644 index 3e138d4348d..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php +++ /dev/null @@ -1,12 +0,0 @@ -projection->setUp(); + } + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { $this->catchUpHook?->onBeforeCatchUp($subscriptionStatus); diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 22c65ced9f7..7e195ff4644 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -21,7 +21,7 @@ interface ProjectionInterface { /** - * Set up the projection state (create databases, call {@see CheckpointStorageInterface::setUp()}). + * Set up the projection state (create/update required database tables, ...). */ public function setUp(): void; @@ -30,12 +30,8 @@ public function setUp(): void; */ public function status(): ProjectionStatus; - public function canHandle(EventInterface $event): bool; - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; - public function getCheckpointStorage(): CheckpointStorageInterface; - /** * NOTE: The ProjectionStateInterface returned must be ALWAYS THE SAME INSTANCE. * @@ -46,5 +42,5 @@ public function getCheckpointStorage(): CheckpointStorageInterface; */ public function getState(): ProjectionStateInterface; - public function reset(): void; + public function resetState(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php b/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php new file mode 100644 index 00000000000..f1713c9fd08 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php @@ -0,0 +1,26 @@ +eventStore->setup(); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php b/Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php new file mode 100644 index 00000000000..3fe17594800 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php @@ -0,0 +1,24 @@ + + * @api + */ +class SubscriptionServiceFactory implements ContentRepositoryServiceFactoryInterface +{ + public function build( + ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies + ): SubscriptionService { + return new SubscriptionService( + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->subscriptionEngine, + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php index 4ca5c434535..699dac846f3 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php @@ -11,10 +11,19 @@ */ final class Error { - public function __construct( + private function __construct( public readonly SubscriptionId $subscriptionId, public readonly string $message, public readonly \Throwable $throwable, ) { } + + public static function fromSubscriptionIdAndException(SubscriptionId $subscriptionId, \Throwable $exception): self + { + return new self( + $subscriptionId, + $exception->getMessage(), + $exception, + ); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php index 5b276868010..1c02bbbca32 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -11,7 +11,7 @@ final class ProcessedResult { /** @param list $errors */ public function __construct( - public readonly int $processedMessages, + public readonly int $numberOfProcessedEvents, public readonly bool $finished = false, public readonly array $errors = [], ) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php new file mode 100644 index 00000000000..f1b2be9b586 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php @@ -0,0 +1,17 @@ + $errors */ + public function __construct( + public readonly array $errors = [], + ) { + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 9c8df9308c8..818c74a3505 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -6,6 +6,8 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; +use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; @@ -25,6 +27,9 @@ */ final class SubscriptionEngine { + private bool $processing = false; + private readonly SubscriptionManager $subscriptionManager; + public function __construct( private readonly EventStoreInterface $eventStore, private readonly SubscriptionStoreInterface $subscriptionStore, @@ -33,299 +38,278 @@ public function __construct( private readonly RetryStrategy $retryStrategy, private readonly LoggerInterface|null $logger = null, ) { + $this->subscriptionManager = new SubscriptionManager($this->subscriptionStore); } - public function setup( - SubscriptionEngineCriteria $criteria = null, - int $limit = null, - ): void { + public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $skipBooting = false): Result + { $criteria ??= SubscriptionEngineCriteria::noConstraints(); - $subscriptionCriteria = SubscriptionCriteria::create( - ids: $criteria->ids, - groups: $criteria->groups, - status: [SubscriptionStatus::NEW], - ); - $this->runInternal($subscriptionCriteria, 'setup', $limit); - } + $this->logger?->info('Subscription Engine: Start to setup.'); - public function run( - SubscriptionEngineCriteria $criteria = null, - int $limit = null, - ): void { - $criteria ??= SubscriptionEngineCriteria::noConstraints(); - - $subscriptionCriteria = SubscriptionCriteria::create( - ids: $criteria->ids, - groups: $criteria->groups, - status: [SubscriptionStatus::ACTIVE], + $this->subscriptionStore->setup(); + $this->discoverNewSubscriptions(); + $this->retrySubscriptions($criteria); + return $this->subscriptionManager->findForUpdate( + SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::NEW), + function (Subscriptions $subscriptions) use ($skipBooting) { + if ($subscriptions->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); + return new Result(); + } + $lastSequenceNumber = $this->lastSequenceNumber(); + $errors = []; + foreach ($subscriptions as $subscription) { + $error = $this->setupSubscription($subscription, $lastSequenceNumber, $skipBooting); + if ($error !== null) { + $errors[] = $error; + } + } + return new Result($errors); + } ); - $this->runInternal($subscriptionCriteria, 'run', $limit); } - private function lockSubscriptions(Subscriptions $subscriptions): void + public function boot(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult { - foreach ($subscriptions as $subscription) { - $sT = microtime(true); - while (!$this->subscriptionStore->acquireLock($subscription->id)) { - if (microtime(true) - $sT > 5) { - // TODO better exception handling - throw new \RuntimeException(sprintf('Failed to acquire lock for subscription "%s"', $subscription->id->value), 1721895494); - } - } - } + return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::BOOTING)); } - private function releaseSubscriptions(Subscriptions $subscriptions): void + public function run(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult { - foreach ($subscriptions as $subscription) { - $this->subscriptionStore->releaseLock($subscription->id); - } + return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE)); } - private function runInternal(SubscriptionCriteria $criteria, string $process, int|null $limit): void + public function teardown(SubscriptionEngineCriteria|null $criteria = null): Result { - $this->logger?->info(sprintf('Subscription Engine: %s: Start.', $process)); - $this->discoverNewSubscriptions(); - $this->discoverDetachedSubscriptions($criteria); - $this->retrySubscriptions($criteria); - $subscriptions = $this->subscriptionStore->findByCriteria($criteria); - if ($subscriptions->isEmpty()) { - $this->logger?->info(sprintf('Subscription Engine: %s: No subscriptions to process, finishing', $process)); - return;// new ProcessedResult(0, true); - } - - $this->lockSubscriptions($subscriptions); - foreach ($subscriptions as $subscription) { - $this->subscribers->get($subscription->id)->handler->onBeforeCatchUp($subscription->status); - } - - $startSequenceNumber = $this->lowestSubscriptionPosition($subscriptions)->next(); - $this->logger?->debug( - sprintf( - 'Subscription Engine: %s: Event stream is processed from position %d.', - $process, - $startSequenceNumber->value, - ), - ); + // TODO implement + } - /** @var list $errors */ - $errors = []; - $messageCounter = 0; - $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); - $lastSequenceNumber = null; - $subscriptionsToRun = $subscriptions; - foreach ($eventStream as $eventEnvelope) { - $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - foreach ($subscriptionsToRun as $subscription) { - if ($subscription->position->value > $eventEnvelope->sequenceNumber->value) { - $this->logger?->debug( - sprintf( - 'Subscription Engine: %s: Subscription "%s" is farther than the current position (%d > %d), skipped.', - $process, - $subscription->id->value, - $subscription->position->value, - $eventEnvelope->sequenceNumber->value, - ), - ); - continue; - } - $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription); - if (!$error) { - continue; - } - $errors[] = $error; - $subscriptionsToRun = $subscriptionsToRun->without($subscription->id); - } - $messageCounter++; - - $this->logger?->debug( - sprintf( - 'Subscription Engine: %s: Current event stream position: %s', - $process, - $eventEnvelope->sequenceNumber->value, - ), - ); - $lastSequenceNumber = $eventEnvelope->sequenceNumber; - if ($limit !== null && $messageCounter >= $limit) { - $this->logger?->info( - sprintf( - 'Subscription Engine: %s: Message limit (%d) reached, cancelled.', - $process, - $limit, - ), - ); - $this->releaseSubscriptions($subscriptions); + public function remove(SubscriptionEngineCriteria|null $criteria = null): Result + { + // TODO implement + } - return;// new ProcessedResult($messageCounter, false, $errors); - } - } - foreach ($subscriptions as $subscription) { - $newSubscriptionStatus = $subscription->runMode === RunMode::ONCE ? SubscriptionStatus::FINISHED : SubscriptionStatus::ACTIVE; - if ($subscription->status === $newSubscriptionStatus) { - continue; - } - $this->subscriptionStore->update($subscription->id, fn(Subscription $subscription) => $subscription->with( - status: $newSubscriptionStatus, - retryAttempt: 0, - )->withoutError()); - $this->logger?->info(sprintf( - 'Subscription Engine: %s: Subscription "%s" changed status from %s to %s.', - $process, - $subscription->id->value, - $subscription->status->name, - $newSubscriptionStatus->name - )); - } - $this->logger?->info( - sprintf( - 'Subscription Engine: %s: End of stream on position %d has been reached, finished.', - $process, - $lastSequenceNumber?->value ?? ($startSequenceNumber->value - 1), - ), - ); - $this->releaseSubscriptions($subscriptions); - foreach ($subscriptions as $subscription) { - $this->subscribers->get($subscription->id)->handler->onAfterCatchUp(); - } + public function reactivate(SubscriptionEngineCriteria|null $criteria = null): Result + { + // TODO implement + } - return;// new ProcessedResult($messageCounter, true, $errors); + public function pause(SubscriptionEngineCriteria|null $criteria = null): Result + { + // TODO implement } private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null { $subscriber = $this->subscribers->get($subscription->id); - try { +// try { $subscriber->handler->handle($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - $this->logger?->error( - sprintf( - 'Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', - $subscriber::class, - $subscription->id->value, - $eventEnvelope->event->type->value, - $eventEnvelope->sequenceNumber->value, - $e->getMessage(), - ), - ); - $this->subscriptionStore->update($subscription->id, static fn(Subscription $subscription) => $subscription->withError($e)); - return new Error( - $subscription->id, - $e->getMessage(), - $e, - ); - } - $this->logger?->debug( - sprintf( - 'Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', - $subscriber->handler::class, - $subscription->id->value, - $eventEnvelope->event->type->value, - $eventEnvelope->sequenceNumber->value, - ), - ); - $this->subscriptionStore->update($subscription->id, static fn(Subscription $subscription) => $subscription->with( +// } catch (\Throwable $e) { +// $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); +// $subscription->fail($e); +// $this->subscriptionManager->update($subscription); +// return Error::fromSubscriptionIdAndException($subscription->id, $e); +// } + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', $subscriber->handler::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + $subscription->set( position: $eventEnvelope->sequenceNumber, - retryAttempt: 0, - )); + retryAttempt: 0 + ); return null; } + /** + * Find all subscribers that don't have a corresponding subscription. + * For each match a subscription is added + * + * Note: newly discovered subscriptions are not ACTIVE by default, instead they have to be initialized via {@see self::setup()} explicitly + */ private function discoverNewSubscriptions(): void { - $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::noConstraints()); - foreach ($this->subscribers as $subscriber) { - if ($registeredSubscriptions->contain($subscriber->id)) { - continue; - } -// if ($subscriber->handler instanceof ProvidesSetup) { -// $subscriber->handler->setup(); -// } - $subscription = Subscription::create( - $subscriber->id, - $subscriber->group, - $subscriber->runMode, - ); - if ($subscriber->runMode === RunMode::FROM_NOW) { - $subscription = $subscription->with( - status: SubscriptionStatus::ACTIVE, - position: $this->lastSequenceNumber(), - ); + $this->subscriptionManager->findForUpdate( + SubscriptionCriteria::noConstraints(), + function (Subscriptions $subscriptions) { + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; + } + $subscription = Subscription::createFromSubscriber($subscriber); + $this->subscriptionManager->add($subscription); + $this->logger?->info(sprintf('Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', $subscriber->id->value)); + } } - $this->subscriptionStore->add($subscription); - $this->logger?->info( - sprintf( - 'Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', - $subscriber->id->value, - ), - ); - } + ); } - private function discoverDetachedSubscriptions(SubscriptionCriteria $criteria): void + private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $criteria): void { $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create( $criteria->ids, $criteria->groups, - [SubscriptionStatus::ACTIVE, SubscriptionStatus::PAUSED, SubscriptionStatus::FINISHED], + SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE, SubscriptionStatus::PAUSED, SubscriptionStatus::FINISHED]), )); foreach ($registeredSubscriptions as $subscription) { if ($this->subscribers->contain($subscription->id)) { continue; } - $this->subscriptionStore->update($subscription->id, fn(Subscription $subscription) => $subscription->with( + $subscription->set( status: SubscriptionStatus::DETACHED, - )); - $this->logger?->info( - sprintf( - 'Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', - $subscription->id->value, - ), ); + $this->subscriptionManager->update($subscription); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); } } - private function retrySubscriptions(SubscriptionCriteria $criteria): void + + /** + * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler + * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned + * + * @param bool $skipBooting + */ + private function setupSubscription(Subscription $subscription, SequenceNumber $lastSequenceNumber, bool $skipBooting): ?Error { - $failedSubscriptions = $this->subscriptionStore->findByCriteria( - SubscriptionCriteria::create( - ids: $criteria->ids, - groups: $criteria->groups, - status: [SubscriptionStatus::ERROR], - ) - ); - foreach ($failedSubscriptions as $subscription) { - if ($subscription->error === null) { - continue; - } - $error = $subscription->error; - $retryable = in_array( - $error->previousStatus, - [SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE], - true, + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->handler->setup(); + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); + $subscription->fail($e); + $this->subscriptionManager->update($subscription); + return Error::fromSubscriptionIdAndException($subscription->id, $e); + } + if ($subscription->runMode === RunMode::FROM_NOW) { + $subscription->set( + status: SubscriptionStatus::ACTIVE, + position: $lastSequenceNumber, ); - if (!$retryable) { - continue; - } - if (!$this->retryStrategy->shouldRetry($subscription)) { - continue; - } - $this->subscriptionStore->update($subscription->id, static fn(Subscription $subscription) => $subscription->with( - status: $error->previousStatus, - retryAttempt: $subscription->retryAttempt + 1, - )->withoutError()); - - $this->logger?->info( - sprintf( - 'Subscription Engine: Retry subscription "%s" (%d) and set back to %s.', - $subscription->id->value, - $subscription->retryAttempt + 1, - $error->previousStatus->name, - ), + } else { + $subscription->set( + status: $skipBooting ? SubscriptionStatus::ACTIVE : SubscriptionStatus::BOOTING ); } + $this->subscriptionManager->update($subscription); + $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the setup method has been executed, set to %s.', $subscriber::class, $subscription->id->value, $subscription->status->value)); + return null; + } + + private function retrySubscriptions(SubscriptionEngineCriteria $criteria): void + { + $this->subscriptionManager->findForUpdate( + SubscriptionCriteria::create($criteria->ids, $criteria->groups, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR])), + fn (Subscriptions $subscriptions) => $subscriptions->map($this->retrySubscription(...)), + ); } + private function retrySubscription(Subscription $subscription): void + { + if ($subscription->error === null) { + return; + } + $retryable = in_array( + $subscription->error->previousStatus, + [SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE], + true, + ); + if (!$retryable) { + return; + } + if (!$this->retryStrategy->shouldRetry($subscription)) { + return; + } + $subscription->set( + status: $subscription->error->previousStatus, + retryAttempt: $subscription->retryAttempt + 1, + ); + $subscription->error = null; + $this->subscriptionManager->update($subscription); + + $this->logger?->info(sprintf('Subscription Engine: Retry subscription "%s" (%d) and set back to %s.', $subscription->id->value, $subscription->retryAttempt, $subscription->status->value)); + } + + private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatus $subscriptionStatus): ProcessedResult + { + $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); + + $this->discoverNewSubscriptions(); + $this->discoverDetachedSubscriptions($criteria); + $this->retrySubscriptions($criteria); + + return $this->subscriptionManager->findForUpdate( + SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), + function (Subscriptions $subscriptions) use ($subscriptionStatus) { + if ($subscriptions->isEmpty()) { + $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); + + return new ProcessedResult(0, true); + } + $startSequenceNumber = $subscriptions->lowestPosition(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + + /** @var list $errors */ + $errors = []; + $numberOfProcessedEvents = 0; + try { + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + $sequenceNumber = $eventEnvelope->sequenceNumber; + foreach ($subscriptions as $subscription) { + if ($subscription->status !== $subscriptionStatus) { + continue; + } + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; + } + $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription); + if (!$error) { + continue; + } + $errors[] = $error; + } + $numberOfProcessedEvents++; + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } + } finally { + foreach ($subscriptions as $subscription) { + $this->subscriptionManager->update($subscription); + } + } + foreach ($subscriptions as $subscription) { + if ($subscription->status !== $subscriptionStatus) { + continue; + } + + if ($subscription->runMode === RunMode::ONCE) { + $subscription->set( + status: SubscriptionStatus::FINISHED, + ); + $this->subscriptionManager->update($subscription); + + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" run only once and has been set to finished.', $subscription->id->value)); + continue; + } + if ($subscription->status !== SubscriptionStatus::ACTIVE) { + $subscription->set( + status: SubscriptionStatus::ACTIVE, + ); + $this->subscriptionManager->update($subscription); + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); + } + } + + $this->logger?->info('Subscription Engine: Finish catch up.'); + + return new ProcessedResult( + $numberOfProcessedEvents, + true, + $errors, + ); + } + ); + } private function lastSequenceNumber(): SequenceNumber { @@ -335,15 +319,21 @@ private function lastSequenceNumber(): SequenceNumber return SequenceNumber::fromInteger(0); } - private function lowestSubscriptionPosition(Subscriptions $subscriptions): SequenceNumber + /** + * @template T + * @param \Closure(): T $closure + * @return T + */ + private function processExclusively(\Closure $closure): mixed { - $min = null; - foreach ($subscriptions as $subscription) { - if ($min !== null && $subscription->position->value >= $min->value) { - continue; - } - $min = $subscription->position; + if ($this->processing) { + throw new SubscriptionEngineAlreadyProcessingException(); + } + $this->processing = true; + try { + return $closure(); + } finally { + $this->processing = false; } - return $min ?? SequenceNumber::fromInteger(0); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php new file mode 100644 index 00000000000..03d487c0254 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -0,0 +1,113 @@ + */ + private \SplObjectStorage $forAdd; + + /** @var \SplObjectStorage */ + private \SplObjectStorage $forUpdate; + + /** @var \SplObjectStorage */ + private \SplObjectStorage $forRemove; + + public function __construct( + private readonly SubscriptionStoreInterface $subscriptionStore, + ) { + $this->forAdd = new \SplObjectStorage(); + $this->forUpdate = new \SplObjectStorage(); + $this->forRemove = new \SplObjectStorage(); + } + + /** + * @template T + * @param \Closure(Subscriptions):T $closure + * @return T + */ + public function findForUpdate(SubscriptionCriteria $criteria, \Closure $closure): mixed + { + if ($this->subscriptionStore instanceof SubscriptionStoreWithTransactionSupportInterface) { + return $this->subscriptionStore->transactional( + /** @return T */ + function () use ($closure, $criteria): mixed { + try { + return $closure($this->subscriptionStore->findByCriteria($criteria)); + } finally { + $this->flush(); + } + }, + ); + } + try { + return $closure($this->subscriptionStore->findByCriteria($criteria)); + } finally { + $this->flush(); + } + } + + public function find(SubscriptionCriteria $criteria): Subscriptions + { + return $this->subscriptionStore->findByCriteria($criteria); + } + + public function add(Subscription $subscription): void + { + $this->forAdd->attach($subscription); + } + + public function update(Subscription $subscription): void + { + $this->forUpdate->attach($subscription); + } + + public function remove(Subscription $subscription): void + { + $this->forRemove->attach($subscription); + } + + private function flush(): void + { + foreach ($this->forAdd as $subscription) { + if ($this->forRemove->contains($subscription)) { + continue; + } + + $this->subscriptionStore->add($subscription); + } + + foreach ($this->forUpdate as $subscription) { + if ($this->forAdd->contains($subscription)) { + continue; + } + + if ($this->forRemove->contains($subscription)) { + continue; + } + + $this->subscriptionStore->update($subscription); + } + + foreach ($this->forRemove as $subscription) { + if ($this->forAdd->contains($subscription)) { + continue; + } + + $this->subscriptionStore->remove($subscription); + } + + $this->forAdd = new \SplObjectStorage(); + $this->forUpdate = new \SplObjectStorage(); + $this->forRemove = new \SplObjectStorage(); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php new file mode 100644 index 00000000000..3bf565d6b4c --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php @@ -0,0 +1,13 @@ +subscriptions = $this->subscriptions->withAdded($subscription); + $this->subscriptions = $this->subscriptions->with($subscription); } - public function update(SubscriptionId $subscriptionId, \Closure $updater): void + public function update(Subscription $subscription): void { - $subscription = $this->subscriptions->get($subscriptionId); - $subscription = $updater($subscription); - $this->subscriptions = $this->subscriptions->withReplaced($subscriptionId, $subscription); + $this->subscriptions = $this->subscriptions->with($subscription); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php index ccdc7659de2..0985dafe9c6 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php @@ -4,35 +4,34 @@ namespace Neos\ContentRepository\Core\Subscription\Store; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionGroups; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionIds; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; /** * @internal */ -final class SubscriptionCriteria +final readonly class SubscriptionCriteria { - /** - * @param list|null $status - */ private function __construct( - public readonly SubscriptionIds|null $ids, - public readonly SubscriptionGroups|null $groups, - public readonly array|null $status, + public SubscriptionIds|null $ids, + public SubscriptionGroups|null $groups, + public SubscriptionStatusFilter $status, ) { } /** * @param SubscriptionIds|array|null $ids * @param SubscriptionGroups|list|null $groups - * @param list|null $status + * @param SubscriptionStatusFilter|null $status */ public static function create( SubscriptionIds|array $ids = null, SubscriptionGroups|array $groups = null, - array $status = null, + SubscriptionStatusFilter $status = null, ): self { if (is_array($ids)) { $ids = SubscriptionIds::fromArray($ids); @@ -43,25 +42,30 @@ public static function create( return new self( $ids, $groups, - $status, + $status ?? SubscriptionStatusFilter::any(), ); } - public static function noConstraints(): self - { + public static function forEngineCriteriaAndStatus( + SubscriptionEngineCriteria $criteria, + SubscriptionStatusFilter|SubscriptionStatus $status, + ): self { + if ($status instanceof SubscriptionStatus) { + $status = SubscriptionStatusFilter::fromArray([$status]); + } return new self( - ids: null, - groups: null, - status: null, + $criteria->ids, + $criteria->groups, + $status, ); } - public static function withStatus(SubscriptionStatus $status): self + public static function noConstraints(): self { return new self( ids: null, groups: null, - status: [$status], + status: SubscriptionStatusFilter::any(), ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index f816e8e3fac..6a0c4da5bbe 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -9,22 +9,17 @@ use Neos\ContentRepository\Core\Subscription\Subscriptions; /** - * @internal + * @api */ interface SubscriptionStoreInterface { + public function setup(): void; + public function findOneById(SubscriptionId $subscriptionId): ?Subscription; public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions; - public function acquireLock(SubscriptionId $subscriptionId): bool; - - public function releaseLock(SubscriptionId $subscriptionId): void; - public function add(Subscription $subscription): void; - /** - * @param \Closure(Subscription): Subscription $updater - */ - public function update(SubscriptionId $subscriptionId, \Closure $updater): void; + public function update(Subscription $subscription): void; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php new file mode 100644 index 00000000000..57c5deaa45d --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php @@ -0,0 +1,22 @@ +id->value, $subscribersById)) { - throw new \InvalidArgumentException(sprintf('Subscriber with id "%s" already part of this set', $subscriber->id->value), 1721731494); + throw new \InvalidArgumentException(sprintf('Subscriber with id "%s" is already part of this set', $subscriber->id->value), 1721731494); } $subscribersById[$subscriber->id->value] = $subscriber; } @@ -63,7 +63,7 @@ public function contain(SubscriptionId $id): bool public function getIterator(): \Traversable { - return yield from $this->subscribersById; + yield from array_values($this->subscribersById); } public function count(): int diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index c54ef1486c3..2d8dcba7b99 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -4,9 +4,12 @@ namespace Neos\ContentRepository\Core\Subscription; +use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\EventStore\Model\Event\SequenceNumber; /** + * Note: This class is mutable by design! + * * @internal */ final class Subscription @@ -15,89 +18,48 @@ public function __construct( public readonly SubscriptionId $id, public readonly SubscriptionGroup $group, public readonly RunMode $runMode, - public readonly SubscriptionStatus $status, - public readonly SequenceNumber $position, - public readonly bool $locked = false, - public readonly SubscriptionError|null $error = null, - public readonly int $retryAttempt = 0, + public SubscriptionStatus $status, + public SequenceNumber $position, + public SubscriptionError|null $error = null, + public int $retryAttempt = 0, public readonly \DateTimeImmutable|null $lastSavedAt = null, ) { } - public static function create( - SubscriptionId $id, - SubscriptionGroup $group, - RunMode $runMode, - ): self { + public static function createFromSubscriber(Subscriber $subscriber): self + { return new self( - $id, - $group, - $runMode, + $subscriber->id, + $subscriber->group, + $subscriber->runMode, SubscriptionStatus::NEW, SequenceNumber::fromInteger(0), ); } - public function with( + public function set( SubscriptionStatus $status = null, SequenceNumber $position = null, int $retryAttempt = null, ): self { + $this->status = $status ?? $this->status; + $this->position = $position ?? $this->position; + $this->retryAttempt = $retryAttempt ?? $this->retryAttempt; return new self( $this->id, $this->group, $this->runMode, $status ?? $this->status, $position ?? $this->position, - $this->locked, $this->error, $retryAttempt ?? $this->retryAttempt, $this->lastSavedAt, ); } - public function withError(\Throwable|string $throwableOrMessage): self - { - if ($throwableOrMessage instanceof \Throwable) { - $error = SubscriptionError::fromThrowable($this->status, $throwableOrMessage); - } else { - $error = new SubscriptionError($throwableOrMessage, $this->status); - } - return new self( - $this->id, - $this->group, - $this->runMode, - SubscriptionStatus::ERROR, - $this->position, - $this->locked, - $error, - $this->retryAttempt, - $this->lastSavedAt, - ); - } -// -// public function doRetry(): void -// { -// if ($this->error === null) { -// throw new NoErrorToRetry(); -// } -// -// $this->retryAttempt++; -// $this->status = $this->error->previousStatus; -// $this->error = null; -// } - public function withoutError(): self + public function fail(\Throwable $exception): void { - return new self( - $this->id, - $this->group, - $this->runMode, - $this->status, - $this->position, - $this->locked, - null, - $this->retryAttempt, - $this->lastSavedAt, - ); + $this->error = SubscriptionError::fromPreviousStatusAndException($this->status, $exception); + $this->status = SubscriptionStatus::ERROR; } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php index e715d5266d3..df42ca5e965 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php @@ -16,8 +16,8 @@ public function __construct( ) { } - public static function fromThrowable(SubscriptionStatus $status, \Throwable $error): self + public static function fromPreviousStatusAndException(SubscriptionStatus $previousStatus, \Throwable $error): self { - return new self($error->getMessage(), $status, $error->getTraceAsString()); + return new self($error->getMessage(), $previousStatus, $error->getTraceAsString()); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php index 410379ff456..ca98b63b387 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php @@ -11,56 +11,67 @@ final class SubscriptionGroups implements \IteratorAggregate, \Countable, \JsonSerializable { /** - * @param array $items + * @param array $groupsByValue */ private function __construct( - private readonly array $items + private readonly array $groupsByValue ) { } /** - * @param list $items + * @param list $groups */ - public static function fromArray(array $items): self + public static function fromArray(array $groups): self { - return new self(array_map(static fn ($item) => $item instanceof SubscriptionGroup ? $item : SubscriptionGroup::fromString($item), $items)); + $groupsByValue = []; + foreach ($groups as $group) { + if (is_string($group)) { + $group = SubscriptionGroup::fromString($group); + } + if (!$group instanceof SubscriptionGroup) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionGroup::class, get_debug_type($group)), 1731580587); + } + if (array_key_exists($group->value, $groupsByValue)) { + throw new \InvalidArgumentException(sprintf('Group "%s" is already part of this set', $group->value), 1731580633); + } + $groupsByValue[$group->value] = $group; + } + return new self($groupsByValue); } public static function none(): self { - return self::fromArray([]); + return new self([]); } public function getIterator(): \Traversable { - return yield from $this->items; + yield from array_values($this->groupsByValue); } public function count(): int { - return count($this->items); + return count($this->groupsByValue); } public function contain(SubscriptionGroup $group): bool { - foreach ($this->items as $item) { - if ($item->equals($group)) { - return true; - } - } - return false; + return array_key_exists($group->value, $this->groupsByValue); } /** - * @return array + * @return list */ public function toStringArray(): array { - return array_map(static fn (SubscriptionGroup $group) => $group->value, $this->items); + return array_values(array_map(static fn (SubscriptionGroup $group) => $group->value, $this->groupsByValue)); } - public function jsonSerialize(): mixed + /** + * @return iterable + */ + public function jsonSerialize(): iterable { - return array_values($this->items); + return array_values($this->groupsByValue); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php index 78ab33296dd..7b672d887ec 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php @@ -11,19 +11,32 @@ final class SubscriptionIds implements \IteratorAggregate, \Countable, \JsonSerializable { /** - * @param array $items + * @param array $subscriptionIdsById */ private function __construct( - private readonly array $items + private readonly array $subscriptionIdsById ) { } /** - * @param array $items + * @param array $ids */ - public static function fromArray(array $items): self + public static function fromArray(array $ids): self { - return new self(array_map(static fn ($item) => $item instanceof SubscriptionId ? $item : SubscriptionId::fromString($item), $items)); + $subscriptionIdsById = []; + foreach ($ids as $id) { + if (is_string($id)) { + $id = SubscriptionId::fromString($id); + } + if (!$id instanceof SubscriptionId) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionId::class, get_debug_type($id)), 1731580820); + } + if (array_key_exists($id->value, $subscriptionIdsById)) { + throw new \InvalidArgumentException(sprintf('Subscription id "%s" is already part of this set', $id->value), 1731580838); + } + $subscriptionIdsById[$id->value] = $id; + } + return new self($subscriptionIdsById); } public static function none(): self @@ -33,30 +46,25 @@ public static function none(): self public function getIterator(): \Traversable { - return yield from $this->items; + yield from array_values($this->subscriptionIdsById); } public function count(): int { - return count($this->items); + return count($this->subscriptionIdsById); } public function contain(SubscriptionId $id): bool { - foreach ($this->items as $item) { - if ($item->equals($id)) { - return true; - } - } - return false; + return array_key_exists($id->value, $this->subscriptionIdsById); } /** - * @return array + * @return list */ public function toStringArray(): array { - return array_map(static fn (SubscriptionId $id) => $id->value, $this->items); + return array_values(array_map(static fn (SubscriptionId $id) => $id->value, $this->subscriptionIdsById)); } /** @@ -64,6 +72,6 @@ public function toStringArray(): array */ public function jsonSerialize(): iterable { - return array_values($this->items); + return array_values($this->subscriptionIdsById); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php new file mode 100644 index 00000000000..61c100a3aae --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php @@ -0,0 +1,64 @@ + + * @internal + */ +final class SubscriptionStatusFilter implements \IteratorAggregate +{ + /** + * @param array $statusByValue + */ + private function __construct( + private readonly array $statusByValue, + ) { + } + + /** + * @param array $status + */ + public static function fromArray(array $status): self + { + $statusByValue = []; + foreach ($status as $singleStatus) { + if (is_string($singleStatus)) { + $singleStatus = SubscriptionStatus::from($singleStatus); + } + if (!$singleStatus instanceof SubscriptionStatus) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionStatus::class, get_debug_type($singleStatus)), 1731580994); + } + if (array_key_exists($singleStatus->value, $statusByValue)) { + throw new \InvalidArgumentException(sprintf('Status "%s" is already part of this set', $singleStatus->value), 1731581002); + } + $statusByValue[$singleStatus->value] = $singleStatus; + } + return new self($statusByValue); + } + + public static function any(): self + { + return new self([]); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->statusByValue); + } + + public function isEmpty(): bool + { + return $this->statusByValue === []; + } + + /** + * @return list + */ + public function toStringArray(): array + { + return array_values(array_map(static fn (SubscriptionStatus $id) => $id->value, $this->statusByValue)); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php index 9b157aa014a..df6f6e5e03c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -4,6 +4,8 @@ namespace Neos\ContentRepository\Core\Subscription; +use Neos\EventStore\Model\Event\SequenceNumber; + /** * @implements \IteratorAggregate * @internal @@ -11,24 +13,29 @@ final class Subscriptions implements \IteratorAggregate, \Countable, \JsonSerializable { /** - * @param array $items + * @param array $subscriptionsById */ private function __construct( - private readonly array $items + private readonly array $subscriptionsById ) { } /** - * @param array $items + * @param array $subscriptions */ - public static function fromArray(array $items): self + public static function fromArray(array $subscriptions): self { - foreach ($items as $item) { - if ($item instanceof Subscription) { - throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscription::class, get_debug_type($item)), 1729679774); + $subscriptionsById = []; + foreach ($subscriptions as $subscription) { + if (!$subscription instanceof Subscription) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscription::class, get_debug_type($subscription)), 1729679774); + } + if (array_key_exists($subscription->id->value, $subscriptionsById)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is contained multiple times in this set', $subscription->id->value), 1731580354); } + $subscriptionsById[$subscription->id->value] = $subscription; } - return new self($items); + return new self($subscriptionsById); } public static function none(): self @@ -38,42 +45,37 @@ public static function none(): self public function getIterator(): \Traversable { - return yield from $this->items; + yield from $this->subscriptionsById; } public function isEmpty(): bool { - return $this->items === []; + return $this->subscriptionsById === []; } public function count(): int { - return count($this->items); + return count($this->subscriptionsById); } public function contain(SubscriptionId $subscriptionId): bool { - foreach ($this->items as $item) { - if ($item->id->equals($subscriptionId)) { - return true; - } - } - return false; + return array_key_exists($subscriptionId->value, $this->subscriptionsById); } public function get(SubscriptionId $subscriptionId): Subscription { - foreach ($this->items as $item) { - if ($item->id->equals($subscriptionId)) { - return $item; - } + if (!$this->contain($subscriptionId)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" not part of this set', $subscriptionId->value), 1723567808); } - throw new \InvalidArgumentException(sprintf('Subscription with id "%s" not part of this set', $subscriptionId->value), 1723567808); + return $this->subscriptionsById[$subscriptionId->value]; } public function without(SubscriptionId $subscriptionId): self { - return $this->filter(static fn (Subscription $subscription) => !$subscription->id->equals($subscriptionId)); + $subscriptionsById = $this->subscriptionsById; + unset($subscriptionsById[$subscriptionId->value]); + return new self($subscriptionsById); } /** @@ -81,7 +83,7 @@ public function without(SubscriptionId $subscriptionId): self */ public function filter(\Closure $callback): self { - return self::fromArray(array_filter($this->items, $callback)); + return self::fromArray(array_filter($this->subscriptionsById, $callback)); } /** @@ -91,31 +93,12 @@ public function filter(\Closure $callback): self */ public function map(\Closure $callback): array { - return array_map($callback, $this->items); + return array_map($callback, $this->subscriptionsById); } - public function withAdded(Subscription $subscription): self + public function with(Subscription $subscription): self { - if ($this->contain($subscription->id)) { - throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is already part of this set', $subscription->id->value), 1723568258); - } - return new self([...$this->items, $subscription]); - } - - public function withReplaced(SubscriptionId $subscriptionId, Subscription $subscription): self - { - if (!$this->contain($subscription->id)) { - throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is not part of this set', $subscription->id->value), 1723568412); - } - $newItems = []; - foreach ($this->items as $item) { - if ($item->id->equals($subscriptionId)) { - $newItems[] = $subscription; - } else { - $newItems[] = $item; - } - } - return new self($newItems); + return new self([...$this->subscriptionsById, $subscription->id->value => $subscription]); } /** @@ -123,6 +106,18 @@ public function withReplaced(SubscriptionId $subscriptionId, Subscription $subsc */ public function jsonSerialize(): iterable { - return array_values($this->items); + return array_values($this->subscriptionsById); + } + + public function lowestPosition(): SequenceNumber + { + $min = null; + foreach ($this->subscriptionsById as $subscription) { + if ($min !== null && $subscription->position->value >= $min->value) { + continue; + } + $min = $subscription->position; + } + return $min ?? SequenceNumber::fromInteger(0); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 64ab9b40899..6c5031826aa 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -6,8 +6,10 @@ use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ProjectionStatusType; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; +use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\EventStore\Model\Event\SequenceNumber; @@ -24,7 +26,7 @@ final class CrCommandController extends CommandController public function __construct( private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly ProjectionServiceFactory $projectionServiceFactory, + private readonly ProjectionServiceFactory $projectionServiceFactory, ) { parent::__construct(); } @@ -44,11 +46,38 @@ public function __construct( public function setupCommand(string $contentRepository = 'default'): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + $subscriptionService->setupEventStore(); + $subscriptionService->subscriptionEngine->setup(); - // TODO $this->contentRepositoryRegistry->get($contentRepositoryId); $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); } + public function subscriptionsBootCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + $subscriptionService->subscriptionEngine->boot(); + } + + public function subscriptionsCatchUpCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + $subscriptionService->subscriptionEngine->run(); + } + + public function subscriptionsResetCommand(string $contentRepository = 'default', bool $force = false): void + { + if (!$force && !$this->output->askConfirmation('Are you sure? (y/n) ', false)) { + $this->outputLine('Cancelled'); + $this->quit(); + } + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + //$subscriptionService->subscriptionEngine->reset(); + } + /** * Determine and output the status of the event store and all registered projections for a given Content Repository * diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 9014cc8ae3b..684039ed124 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -42,6 +42,7 @@ use Neos\Utility\Arrays; use Neos\Utility\PositionalArraySorter; use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Serializer\Normalizer\DenormalizerInterface; use Symfony\Component\Serializer\Normalizer\NormalizerInterface; use Symfony\Component\Serializer\Serializer; @@ -64,6 +65,7 @@ public function __construct( private readonly array $settings, private readonly ObjectManagerInterface $objectManager, private readonly SubgraphCachePool $subgraphCachePool, + private readonly LoggerInterface $logger, ) { } @@ -187,6 +189,7 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $this->buildContentGraphCatchUpHookFactory($contentRepositoryId, $contentRepositorySettings), $this->buildCommandHooksFactory($contentRepositoryId, $contentRepositorySettings), $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), + $this->logger, ); } catch (\Exception $exception) { throw InvalidConfigurationException::fromException($contentRepositoryId, $exception); diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index be7bfe97a18..6893c95f1cc 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -4,7 +4,6 @@ namespace Neos\ContentRepositoryRegistry\Factory\SubscriptionStore; -use Closure; use DateTimeImmutable; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Platforms\SqlitePlatform; @@ -17,7 +16,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreWithTransactionSupportInterface; use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; @@ -27,10 +26,7 @@ use Neos\EventStore\Model\Event\SequenceNumber; use Psr\Clock\ClockInterface; -/** - * @api - */ -final class DoctrineSubscriptionStore implements SubscriptionStoreInterface +final class DoctrineSubscriptionStore implements SubscriptionStoreWithTransactionSupportInterface { public function __construct( private string $tableName, @@ -52,7 +48,6 @@ public function setup(): void (new Column('group_name', Type::getType(Types::STRING)))->setNotnull(true)->setLength(100)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('run_mode', Type::getType(Types::STRING)))->setNotnull(true)->setLength(16)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), - (new Column('locked', Type::getType(Types::BOOLEAN)))->setNotnull(true), (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), @@ -104,11 +99,11 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions Connection::PARAM_STR_ARRAY, ); } - if ($criteria->status !== null) { + if (!$criteria->status->isEmpty()) { $queryBuilder->andWhere('status IN (:status)') ->setParameter( 'status', - array_map(static fn (SubscriptionStatus $status) => $status->name, $criteria->status), + $criteria->status->toStringArray(), Connection::PARAM_STR_ARRAY, ); } @@ -121,33 +116,10 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions return Subscriptions::fromArray(array_map(self::fromDatabase(...), $rows)); } - public function acquireLock(SubscriptionId $subscriptionId): bool - { - $data = [ - 'locked' => 1, - 'last_saved_at' => $this->clock->now()->format('Y-m-d H:i:s'), - ]; - $acquired = $this->dbal->update($this->tableName, $data, [ - 'id' => $subscriptionId->value, - 'locked' => 0, - ]); - return $acquired >= 1; - } - - public function releaseLock(SubscriptionId $subscriptionId): void - { - $data = [ - 'locked' => 0, - 'last_saved_at' => $this->clock->now()->format('Y-m-d H:i:s'), - ]; - $this->dbal->update($this->tableName, $data, ['id' => $subscriptionId->value]); - } - public function add(Subscription $subscription): void { $row = self::toDatabase($subscription); $row['id'] = $subscription->id->value; - $row['locked'] = 0; $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); $this->dbal->insert( $this->tableName, @@ -155,21 +127,15 @@ public function add(Subscription $subscription): void ); } - public function update(SubscriptionId $subscriptionId, Closure $updater): void + public function update(Subscription $subscription): void { - $subscription = $this->findOneById($subscriptionId); - if ($subscription === null) { - throw new \InvalidArgumentException(sprintf('Failed to update subscription with id "%s" because it does not exist', $subscriptionId->value), 1721672347); - } - /** @var Subscription $subscription */ - $subscription = $updater($subscription); $row = self::toDatabase($subscription); $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); $this->dbal->update( $this->tableName, $row, [ - 'id' => $subscriptionId->value, + 'id' => $subscription->id->value, ] ); } @@ -209,7 +175,6 @@ private static function fromDatabase(array $row): Subscription assert(is_string($row['run_mode'])); assert(is_string($row['status'])); assert(is_int($row['position'])); - assert(is_int($row['locked'])); assert(is_int($row['retry_attempt'])); assert(is_string($row['last_saved_at'])); $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); @@ -221,10 +186,14 @@ private static function fromDatabase(array $row): Subscription RunMode::from($row['run_mode']), SubscriptionStatus::from($row['status']), SequenceNumber::fromInteger($row['position']), - (bool)$row['locked'], $subscriptionError, $row['retry_attempt'], $lastSavedAt, ); } + + public function transactional(\Closure $closure): mixed + { + return $this->dbal->transactional($closure); + } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php index 922c622c6e5..2203768f8dc 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php @@ -13,7 +13,7 @@ * * @internal */ -final class ProjectionReplayService implements ContentRepositoryServiceInterface +final class ProjectionService implements ContentRepositoryServiceInterface { public function __construct( private readonly SubscriptionEngine $subscriptionEngine, diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index de859059ae0..016814020bc 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -24,16 +24,13 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; use Neos\Neos\Domain\Model\SiteNodeName; use Neos\Neos\FrontendRouting\Exception\NodeNotFoundException; @@ -47,7 +44,6 @@ final class DocumentUriPathProjection implements ProjectionInterface, WithMarkSt 'shortcutTarget' => Types::JSON, ]; - private DbalCheckpointStorage $checkpointStorage; private ?DocumentUriPathFinder $stateAccessor = null; /** @@ -60,11 +56,6 @@ public function __construct( private readonly Connection $dbal, private readonly string $tableNamePrefix, ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } public function setUp(): void @@ -72,18 +63,10 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -113,11 +96,9 @@ private function determineRequiredSqlStatements(): array } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); $this->stateAccessor = null; } @@ -130,27 +111,6 @@ private function truncateDatabaseTables(): void } } - - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - RootNodeAggregateWithNodeWasCreated::class, - RootNodeAggregateDimensionsWereUpdated::class, - NodeAggregateWithNodeWasCreated::class, - NodeAggregateTypeWasChanged::class, - NodePeerVariantWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodeSpecializationVariantWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - NodeAggregateWasRemoved::class, - NodePropertiesWereSet::class, - NodeAggregateWasMoved::class, - DimensionSpacePointWasMoved::class, - DimensionShineThroughWasAdded::class, - ]); - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -168,15 +128,10 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeAggregateWasMoved::class => $this->whenNodeAggregateWasMoved($event), DimensionSpacePointWasMoved::class => $this->whenDimensionSpacePointWasMoved($event), DimensionShineThroughWasAdded::class => $this->whenDimensionShineThroughWasAdded($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): DocumentUriPathFinder { if (!$this->stateAccessor) { diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index 4f91f39cea2..ead06f7b588 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -37,15 +37,12 @@ use Neos\ContentRepository\Core\Feature\NodeVariation\Event\NodeSpecializationVariantWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; /** @@ -61,17 +58,10 @@ class ChangeProjection implements ProjectionInterface */ private ?ChangeFinder $changeFinder = null; - private DbalCheckpointStorage $checkpointStorage; - public function __construct( private readonly Connection $dbal, private readonly string $tableNamePrefix, ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } /** @@ -83,18 +73,10 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -146,30 +128,9 @@ private function determineRequiredSqlStatements(): array return $statements; } - public function reset(): void + public function resetState(): void { $this->dbal->exec('TRUNCATE ' . $this->tableNamePrefix); - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); - } - - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - NodeAggregateWasMoved::class, - NodePropertiesWereSet::class, - NodeReferencesWereSet::class, - NodeAggregateWithNodeWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - NodeAggregateWasRemoved::class, - DimensionSpacePointWasMoved::class, - NodeGeneralizationVariantWasCreated::class, - NodeSpecializationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - NodeAggregateTypeWasChanged::class, - NodeAggregateNameWasChanged::class, - ]); } public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void @@ -188,15 +149,10 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodePeerVariantWasCreated::class => $this->whenNodePeerVariantWasCreated($event), NodeAggregateTypeWasChanged::class => $this->whenNodeAggregateTypeWasChanged($event), NodeAggregateNameWasChanged::class => $this->whenNodeAggregateNameWasChanged($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): ChangeFinder { if (!$this->changeFinder) { From aca0f9bca6bc4d026cd578529a5127f74dd334a6 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 14 Nov 2024 13:36:59 +0100 Subject: [PATCH 006/142] Fix CatchUpHooks --- .../CatchUpHook/CatchUpHookInterface.php | 7 ++----- .../Subscription/Engine/SubscriptionEngine.php | 18 +++++++++++------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index e3f67b10c7c..8d6f225f233 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -21,9 +21,7 @@ interface CatchUpHookInterface { /** * This hook is called at the beginning of a catch-up run; - * BEFORE the Database Lock is acquired ({@see SubscriptionEngine::run()}). - * - * @return void + * AFTER the Database Lock is acquired ({@see SubscriptionEngine::run()}). */ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; @@ -41,8 +39,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event /** * This hook is called at the END of a catch-up run - * - * At this point, the Database Lock has already been released. + * BEFORE the Database Lock is released ({@see SubscriptionEngine::run()}). */ public function onAfterCatchUp(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 818c74a3505..d2031917bbc 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -103,14 +103,14 @@ public function pause(SubscriptionEngineCriteria|null $criteria = null): Result private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null { $subscriber = $this->subscribers->get($subscription->id); -// try { + try { $subscriber->handler->handle($domainEvent, $eventEnvelope); -// } catch (\Throwable $e) { -// $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); -// $subscription->fail($e); -// $this->subscriptionManager->update($subscription); -// return Error::fromSubscriptionIdAndException($subscription->id, $e); -// } + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + $subscription->fail($e); + $this->subscriptionManager->update($subscription); + return Error::fromSubscriptionIdAndException($subscription->id, $e); + } $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', $subscriber->handler::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); $subscription->set( position: $eventEnvelope->sequenceNumber, @@ -244,6 +244,9 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus) { return new ProcessedResult(0, true); } + foreach ($subscriptions as $subscription) { + $this->subscribers->get($subscription->id)->handler->onBeforeCatchUp($subscription->status); + } $startSequenceNumber = $subscriptions->lowestPosition(); $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); @@ -278,6 +281,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus) { } } foreach ($subscriptions as $subscription) { + $this->subscribers->get($subscription->id)->handler->onAfterCatchUp(); if ($subscription->status !== $subscriptionStatus) { continue; } From 7fea53eb2d7a8b25a771a19b3e8776bf1aed1236 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 14 Nov 2024 14:04:43 +0100 Subject: [PATCH 007/142] Replace ProjectionService --- .../Classes/Command/CrCommandController.php | 116 ------------------ .../Processors/ProjectionCatchupProcessor.php | 9 +- .../Processors/ProjectionResetProcessor.php | 8 +- .../Classes/Service/ProjectionService.php | 40 ------ .../Service/ProjectionServiceFactory.php | 26 ---- .../Domain/Service/SiteImportService.php | 13 +- .../Domain/Service/SitePruningService.php | 4 +- 7 files changed, 17 insertions(+), 199 deletions(-) delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 6c5031826aa..8887a1d02d7 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -3,19 +3,13 @@ namespace Neos\ContentRepositoryRegistry\Command; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ProjectionStatusType; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; -use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Cli\CommandController; -use Neos\Neos\Domain\Service\WorkspaceService; use stdClass; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\ConsoleOutput; @@ -26,7 +20,6 @@ final class CrCommandController extends CommandController public function __construct( private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly ProjectionServiceFactory $projectionServiceFactory, ) { parent::__construct(); } @@ -125,113 +118,4 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->quit(1); } } - - /** - * Replays the specified projection of a Content Repository by resetting its state and performing a full catchup. - * - * @param string $projection Full Qualified Class Name or alias of the projection to replay (e.g. "contentStream") - * @param string $contentRepository Identifier of the Content Repository instance to operate on - * @param bool $force Replay the projection without confirmation. This may take some time! - * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - * @param int $until Until which sequence number should projections be replayed? useful for debugging - */ - public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false, int $until = 0): void - { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - $progressBar = new ProgressBar($this->output->getOutput()); - $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:16s%/%estimated:-16s% %memory:6s%'); - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); - $this->quit(1); - } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the projection "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $projection, $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - - $options = CatchUpOptions::create(); - if (!$quiet) { - $this->outputLine('Replaying events for projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); - $progressBar->start($until > 0 ? $until : null); - $options = $options->with(progressCallback: fn () => $progressBar->advance()); - } - if ($until > 0) { - $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); - } - $projectionService->replayProjection($projection, $options); - if (!$quiet) { - $progressBar->finish(); - $this->outputLine(); - $this->outputLine('Done.'); - } - } - - /** - * Replays all projections of the specified Content Repository by resetting their states and performing a full catchup - * - * @param string $contentRepository Identifier of the Content Repository instance to operate on - * @param bool $force Replay the projection without confirmation. This may take some time! - * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - * @param int $until Until which sequence number should projections be replayed? useful for debugging - */ - public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false, int $until = 0): void - { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - $mainSection = ($this->output->getOutput() instanceof ConsoleOutput) ? $this->output->getOutput()->section() : $this->output->getOutput(); - $mainProgressBar = new ProgressBar($mainSection); - $mainProgressBar->setBarCharacter('█'); - $mainProgressBar->setEmptyBarCharacter('░'); - $mainProgressBar->setProgressCharacter('█'); - $mainProgressBar->setFormat('debug'); - - $subSection = ($this->output->getOutput() instanceof ConsoleOutput) ? $this->output->getOutput()->section() : $this->output->getOutput(); - $progressBar = new ProgressBar($subSection); - $progressBar->setFormat(' %message% - %current%/%max% [%bar%] %percent:3s%% %elapsed:16s%/%estimated:-16s% %memory:6s%'); - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); - $this->quit(1); - } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - if (!$quiet) { - $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); - } - $options = CatchUpOptions::create(); - if (!$quiet) { - $options = $options->with(progressCallback: fn () => $progressBar->advance()); - } - if ($until > 0) { - $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); - } - $mainProgressBar->start(); - $mainProgressCallback = null; - if (!$quiet) { - $mainProgressCallback = static function (string $projectionAlias) use ($mainProgressBar, $progressBar, $until) { - $mainProgressBar->advance(); - $progressBar->setMessage($projectionAlias); - $progressBar->start($until > 0 ? $until : null); - $progressBar->setProgress(0); - }; - } - $projectionService->replayAllProjections($options, $mainProgressCallback); - if (!$quiet) { - $mainProgressBar->finish(); - $progressBar->finish(); - $this->outputLine('Done.'); - } - } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php index 69587bb4806..25b9650b322 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php @@ -3,23 +3,22 @@ namespace Neos\ContentRepositoryRegistry\Processors; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; +use Neos\ContentRepository\Core\Service\SubscriptionService; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepositoryRegistry\Service\ProjectionService; /** * @internal */ -final class ProjectionCatchupProcessor implements ProcessorInterface +final readonly class ProjectionCatchupProcessor implements ProcessorInterface { public function __construct( - private readonly ProjectionService $projectionservice, + private SubscriptionService $subscriptionService, ) { } public function run(ProcessingContext $context): void { - $this->projectionservice->catchupAllProjections(CatchUpOptions::create()); + $this->subscriptionService->subscriptionEngine->run(); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php index 7a5a1f9013f..50509d60367 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php @@ -3,22 +3,22 @@ namespace Neos\ContentRepositoryRegistry\Processors; +use Neos\ContentRepository\Core\Service\SubscriptionService; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepositoryRegistry\Service\ProjectionService; /** * @internal */ -final class ProjectionResetProcessor implements ProcessorInterface +final readonly class ProjectionResetProcessor implements ProcessorInterface { public function __construct( - private readonly ProjectionService $projectionService, + private SubscriptionService $subscriptionService, ) { } public function run(ProcessingContext $context): void { - $this->projectionService->resetAllProjections(); + // TODO implement $this->subscriptionService->reset(); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php deleted file mode 100644 index 2203768f8dc..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php +++ /dev/null @@ -1,40 +0,0 @@ -subscriptionEngine->setup(); - // TODO $this->subscriptionEngine->reset() - // TODO $this->subscriptionEngine->run() - } - - public function replayAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void - { - // TODO $this->subscriptionEngine->reset() - // TODO $this->subscriptionEngine->run() - } - - public function resetAllProjections(): void - { - // TODO $this->subscriptionEngine->reset() - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php deleted file mode 100644 index 4e9ed04ccad..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php +++ /dev/null @@ -1,26 +0,0 @@ - - * @internal - */ -#[Flow\Scope("singleton")] -final class ProjectionServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ProjectionService( - $serviceFactoryDependencies->subscriptionEngine, - ); - } -} diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 741424a02e2..494b4779c8f 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -18,6 +18,7 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; @@ -28,7 +29,6 @@ use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -78,7 +78,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())), + 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory())), ]); foreach ($processors as $processorLabel => $processor) { @@ -89,10 +89,11 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string private function requireContentRepositoryToBeSetup(ContentRepository $contentRepository): void { - $status = $contentRepository->status(); - if (!$status->isOk()) { - throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepository->id->value)); - } +// TODO reimplement +// $status = $contentRepository->status(); +// if (!$status->isOk()) { +// throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepository->id->value)); +// } } private function requireDataBaseSchemaToBeSetup(): void diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 71ed7c18763..0decf5d556d 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -17,6 +17,7 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; +use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; @@ -25,7 +26,6 @@ use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Processors\ProjectionResetProcessor; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; @@ -75,7 +75,7 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP 'Reset all projections' => new ProjectionResetProcessor( $this->contentRepositoryRegistry->buildService( $contentRepositoryId, - new ProjectionServiceFactory() + new SubscriptionServiceFactory() ) ) ]); From ec08d6407fac15ca223e9cb269760cbe773d1f42 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 14 Nov 2024 18:35:46 +0100 Subject: [PATCH 008/142] remove `EventPersister` and much more :) --- .../Classes/ContentRepository.php | 19 ++++--- .../Classes/EventStore/EventNormalizer.php | 6 +++ .../Classes/EventStore/EventPersister.php | 52 ------------------- .../Factory/ContentRepositoryFactory.php | 18 ++----- ...ntRepositoryServiceFactoryDependencies.php | 4 -- .../CatchUpHook/CatchUpHookInterface.php | 4 +- .../Projection/WithMarkStaleInterface.php | 5 +- .../Classes/Service/ContentStreamPruner.php | 11 ++-- .../Service/ContentStreamPrunerFactory.php | 3 +- .../Classes/Subscription/Engine/Errors.php | 48 +++++++++++++++++ .../Subscription/Engine/ProcessedResult.php | 25 ++++++--- .../Classes/Subscription/Engine/Result.php | 19 +++++-- .../Engine/SubscriptionEngine.php | 31 ++++++----- .../Engine/SubscriptionManager.php | 28 ++++------ .../EventStore/RunSubscriptionEventStore.php | 5 +- .../Classes/Subscription/RunMode.php | 2 +- .../Store/InMemorySubscriptionStore.php | 18 +++++-- .../Store/SubscriptionCriteria.php | 2 +- .../Store/SubscriptionStoreInterface.php | 11 +++- ...onStoreWithTransactionSupportInterface.php | 22 -------- .../Subscription/Subscriber/Subscriber.php | 2 +- .../Subscription/Subscriber/Subscribers.php | 2 +- .../Classes/Subscription/Subscription.php | 12 ++++- .../Subscription/SubscriptionError.php | 2 +- .../Subscription/SubscriptionGroup.php | 2 +- .../Subscription/SubscriptionGroups.php | 2 +- .../Classes/Subscription/SubscriptionId.php | 2 +- .../Subscription/SubscriptionStatus.php | 2 +- .../Subscription/SubscriptionStatusFilter.php | 7 ++- .../Classes/Subscription/Subscriptions.php | 2 +- .../src/StructureAdjustmentService.php | 30 ++++++++--- .../src/StructureAdjustmentServiceFactory.php | 4 +- ...ricCommandExecutionAndEventPublication.php | 38 ++++++++------ .../Classes/Command/CrCommandController.php | 2 +- .../Classes/ContentRepositoryRegistry.php | 3 +- .../DoctrineSubscriptionStore.php | 26 ++++++---- .../Processors/ProjectionCatchupProcessor.php | 2 +- 37 files changed, 262 insertions(+), 211 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index e0bd3d05740..789f0ef6d04 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -19,7 +19,7 @@ use Neos\ContentRepository\Core\CommandHandler\CommandInterface; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\EventStore\InitiatingEventMetadata; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; @@ -36,7 +36,10 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Exception\ConcurrencyException; +use Neos\EventStore\Model\Events; use Psr\Clock\ClockInterface; /** @@ -57,7 +60,9 @@ public function __construct( public readonly ContentRepositoryId $id, private readonly CommandBus $commandBus, - private readonly EventPersister $eventPersister, + private readonly EventStoreInterface $eventStore, + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, private readonly NodeTypeManager $nodeTypeManager, private readonly InterDimensionalVariationGraph $variationGraph, private readonly ContentDimensionSourceInterface $contentDimensionSource, @@ -83,8 +88,8 @@ public function handle(CommandInterface $command): void // simple case if ($toPublish instanceof EventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); - $this->eventPersister->publishWithoutCatchup($eventsToPublish); - // TODO how to solve this with a decoupled subscription engine? $this->catchupProjections(); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish), $eventsToPublish->expectedVersion); + $this->subscriptionEngine->catchUpActive(); return; } @@ -93,7 +98,7 @@ public function handle(CommandInterface $command): void foreach ($toPublish as $yieldedEventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($yieldedEventsToPublish); try { - $this->eventPersister->publishWithoutCatchup($eventsToPublish); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish), $eventsToPublish->expectedVersion); } catch (ConcurrencyException $concurrencyException) { // we pass the exception into the generator (->throw), so it could be try-caught and reacted upon: // @@ -105,7 +110,7 @@ public function handle(CommandInterface $command): void // } $yieldedErrorStrategy = $toPublish->throw($concurrencyException); if ($yieldedErrorStrategy instanceof EventsToPublish) { - $this->eventPersister->publishWithoutCatchup($yieldedErrorStrategy); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy), $yieldedErrorStrategy->expectedVersion); } throw $concurrencyException; } @@ -113,7 +118,7 @@ public function handle(CommandInterface $command): void } finally { // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. - // TODO how to solve with the decoupled subscription engine? $this->catchupProjections(); + $this->subscriptionEngine->catchUpActive(); } } diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php index 52f23d63910..0e6226f4873 100644 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php +++ b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php @@ -46,6 +46,7 @@ use Neos\EventStore\Model\Event\EventData; use Neos\EventStore\Model\Event\EventId; use Neos\EventStore\Model\Event\EventType; +use Neos\EventStore\Model\Events; /** * Central authority to convert Content Repository domain events to Event Store EventData and EventType, vice versa. @@ -147,6 +148,11 @@ public function normalize(EventInterface|DecoratedEvent $event): Event ); } + public function normalizeEvents(EventsToPublish $eventsToPublish): Events + { + return Events::fromArray($eventsToPublish->events->map($this->normalize(...))); + } + public function denormalize(Event $event): EventInterface { $eventClassName = $this->getEventClassName($event); diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php b/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php deleted file mode 100644 index 65ff452eaa7..00000000000 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php +++ /dev/null @@ -1,52 +0,0 @@ -publishWithoutCatchup($eventsToPublish); - // TODO $contentRepository->catchUpProjections(); - } - - /** - * TODO Will be refactored via https://github.com/neos/neos-development-collection/pull/5321 - * @throws ConcurrencyException in case the expectedVersion does not match - */ - public function publishWithoutCatchup(EventsToPublish $eventsToPublish): CommitResult - { - $normalizedEvents = Events::fromArray( - $eventsToPublish->events->map($this->eventNormalizer->normalize(...)) - ); - return $this->eventStore->commit( - $eventsToPublish->streamName, - $normalizedEvents, - $eventsToPublish->expectedVersion - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 744040ee0bf..e3bde71144a 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -22,7 +22,6 @@ use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\DimensionSpaceCommandHandler; use Neos\ContentRepository\Core\Feature\NodeAggregateCommandHandler; use Neos\ContentRepository\Core\Feature\NodeDuplication\NodeDuplicationCommandHandler; @@ -70,7 +69,6 @@ final class ContentRepositoryFactory // The following properties store "singleton" references of objects for this content repository private ?ContentRepository $contentRepositoryRuntimeCache = null; - private ?EventPersister $eventPersisterRuntimeCache = null; /** * @param CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory @@ -202,7 +200,9 @@ public function getOrBuild(): ContentRepository $this->contentRepositoryRuntimeCache = new ContentRepository( $this->contentRepositoryId, $publicCommandBus, - $this->buildEventPersister(), + $this->eventStore, + $this->subscriberFactoryDependencies->eventNormalizer, + $this->subscriptionEngine, $this->subscriberFactoryDependencies->nodeTypeManager, $this->subscriberFactoryDependencies->interDimensionalVariationGraph, $this->subscriberFactoryDependencies->contentDimensionSource, @@ -235,20 +235,8 @@ public function buildService( $this->subscriberFactoryDependencies, $this->eventStore, $this->getOrBuild(), - $this->buildEventPersister(), $this->subscriptionEngine, ); return $serviceFactory->build($serviceFactoryDependencies); } - - private function buildEventPersister(): EventPersister - { - if (!$this->eventPersisterRuntimeCache) { - $this->eventPersisterRuntimeCache = new EventPersister( - $this->eventStore, - $this->subscriberFactoryDependencies->eventNormalizer, - ); - } - return $this->eventPersisterRuntimeCache; - } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index 401f918a82a..d3172398e7c 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -19,7 +19,6 @@ use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -45,7 +44,6 @@ private function __construct( public PropertyConverter $propertyConverter, public ContentRepository $contentRepository, // we don't need CommandBus, because this is included in ContentRepository->handle() - public EventPersister $eventPersister, public SubscriptionEngine $subscriptionEngine, ) { } @@ -57,7 +55,6 @@ public static function create( SubscriberFactoryDependencies $projectionFactoryDependencies, EventStoreInterface $eventStore, ContentRepository $contentRepository, - EventPersister $eventPersister, SubscriptionEngine $subscriptionEngine, ): self { return new self( @@ -70,7 +67,6 @@ public static function create( $projectionFactoryDependencies->interDimensionalVariationGraph, $projectionFactoryDependencies->propertyConverter, $contentRepository, - $eventPersister, $subscriptionEngine, ); } diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index 8d6f225f233..6dd4301f2a7 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -21,7 +21,7 @@ interface CatchUpHookInterface { /** * This hook is called at the beginning of a catch-up run; - * AFTER the Database Lock is acquired ({@see SubscriptionEngine::run()}). + * AFTER the Database Lock is acquired ({@see SubscriptionEngine::catchUpActive()}). */ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; @@ -39,7 +39,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event /** * This hook is called at the END of a catch-up run - * BEFORE the Database Lock is released ({@see SubscriptionEngine::run()}). + * BEFORE the Database Lock is released ({@see SubscriptionEngine::catchUpActive()}). */ public function onAfterCatchUp(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php b/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php index baf93b2d1b5..30034de7358 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php @@ -4,8 +4,7 @@ namespace Neos\ContentRepository\Core\Projection; -use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; /** * Additional marker interface to add to a {@see ProjectionInterface}. @@ -19,7 +18,7 @@ interface WithMarkStaleInterface { /** * Triggered during catching up after applying events - * {@see ContentRepository::catchUpProjection()} + * {@see SubscriptionEngine::catchUpActive()} * * Can be f.e. used to flush caches inside the Projection State. * diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index e0329cf2463..eac4e78f9af 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -23,6 +23,7 @@ use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventType; use Neos\EventStore\Model\Event\EventTypes; @@ -40,7 +41,8 @@ class ContentStreamPruner implements ContentRepositoryServiceInterface { public function __construct( private readonly EventStoreInterface $eventStore, - private readonly EventNormalizer $eventNormalizer + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, ) { } @@ -160,10 +162,9 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta } if ($danglingContentStreamsPresent) { - try { - //TODO $this->contentRepository->catchUpProjections(); - } catch (\Throwable $e) { - $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); + $result = $this->subscriptionEngine->catchUpActive(); + if ($result->hasErrors()) { + $outputFn('Catchup after removing unused content streams led to errors. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.'); } } else { $outputFn('Okay. No pruneable streams in the event stream'); diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php index 1ddb91c1a5d..ecdb4dc2107 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php @@ -17,7 +17,8 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor { return new ContentStreamPruner( $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventNormalizer + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php new file mode 100644 index 00000000000..cc8ca6a638e --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -0,0 +1,48 @@ + + */ +final readonly class Errors implements \IteratorAggregate, \Countable +{ + /** + * array + */ + private array $errors; + + /** + * @param array $errors + */ + private function __construct( + Error ...$errors + ) { + $this->errors = $errors; + } + + /** + * @param array $errors + */ + public static function fromArray(array $errors): self + { + return new self(...$errors); + } + + public function getIterator(): \Traversable + { + yield from $this->errors; + } + + public function isEmpty(): bool + { + return $this->errors === []; + } + + public function count(): int + { + return count($this->errors); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php index 1c02bbbca32..e1a84e5c38c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -5,15 +5,28 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; /** - * @internal + * @api */ -final class ProcessedResult +final readonly class ProcessedResult { - /** @param list $errors */ - public function __construct( + private function __construct( public readonly int $numberOfProcessedEvents, - public readonly bool $finished = false, - public readonly array $errors = [], + public readonly Errors|null $errors, ) { } + + public static function success(int $numberOfProcessedEvents): self + { + return new self($numberOfProcessedEvents, null); + } + + public static function failed(int $numberOfProcessedEvents, Errors $errors): self + { + return new self($numberOfProcessedEvents, $errors); + } + + public function hasErrors(): bool + { + return $this->errors !== null; + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php index f1b2be9b586..62aaf947553 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php @@ -5,13 +5,22 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; /** - * @internal + * @api */ -final class Result +final readonly class Result { - /** @param list $errors */ - public function __construct( - public readonly array $errors = [], + private function __construct( + public readonly Errors|null $errors, ) { } + + public static function success(): self + { + return new self(null); + } + + public static function failed(Errors $errors): self + { + return new self($errors); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index d2031917bbc..e91cf09fb0d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -55,7 +55,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $sk function (Subscriptions $subscriptions) use ($skipBooting) { if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); - return new Result(); + return Result::success(); } $lastSequenceNumber = $this->lastSequenceNumber(); $errors = []; @@ -65,7 +65,7 @@ function (Subscriptions $subscriptions) use ($skipBooting) { $errors[] = $error; } } - return new Result($errors); + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } ); } @@ -75,29 +75,38 @@ public function boot(SubscriptionEngineCriteria|null $criteria = null): Processe return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::BOOTING)); } - public function run(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult + public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult { return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE)); } public function teardown(SubscriptionEngineCriteria|null $criteria = null): Result { - // TODO implement + // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L470) + return Result::success(); } public function remove(SubscriptionEngineCriteria|null $criteria = null): Result { - // TODO implement + // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L562) + return Result::success(); } public function reactivate(SubscriptionEngineCriteria|null $criteria = null): Result { - // TODO implement + // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L648) + return Result::success(); } public function pause(SubscriptionEngineCriteria|null $criteria = null): Result { - // TODO implement + // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L712) + return Result::success(); + } + + public function subscriptions(SubscriptionCriteria|null $criteria = null): Subscriptions + { + return $this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()); } private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null @@ -242,7 +251,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus) { if ($subscriptions->isEmpty()) { $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); - return new ProcessedResult(0, true); + return ProcessedResult::success(0); } foreach ($subscriptions as $subscription) { $this->subscribers->get($subscription->id)->handler->onBeforeCatchUp($subscription->status); @@ -306,11 +315,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus) { $this->logger?->info('Subscription Engine: Finish catch up.'); - return new ProcessedResult( - $numberOfProcessedEvents, - true, - $errors, - ); + return ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); } ); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php index 03d487c0254..9bbb74e1832 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -6,7 +6,6 @@ use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreWithTransactionSupportInterface; use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\Subscriptions; @@ -37,23 +36,16 @@ public function __construct( */ public function findForUpdate(SubscriptionCriteria $criteria, \Closure $closure): mixed { - if ($this->subscriptionStore instanceof SubscriptionStoreWithTransactionSupportInterface) { - return $this->subscriptionStore->transactional( - /** @return T */ - function () use ($closure, $criteria): mixed { - try { - return $closure($this->subscriptionStore->findByCriteria($criteria)); - } finally { - $this->flush(); - } - }, - ); - } - try { - return $closure($this->subscriptionStore->findByCriteria($criteria)); - } finally { - $this->flush(); - } + return $this->subscriptionStore->transactional( + /** @return T */ + function () use ($closure, $criteria): mixed { + try { + return $closure($this->subscriptionStore->findByCriteria($criteria)); + } finally { + $this->flush(); + } + }, + ); } public function find(SubscriptionCriteria $criteria): Subscriptions diff --git a/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php b/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php index 6d08bb848c5..fe69b74ab2b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php @@ -46,7 +46,10 @@ public function load(StreamName|VirtualStreamName $streamName, EventStreamFilter public function commit(StreamName $streamName, \Neos\EventStore\Model\Events|Event $events, ExpectedVersion $expectedVersion): CommitResult { $commitResult = $this->eventStore->commit($streamName, $events, $expectedVersion); - $this->subscriptionEngine->run($this->criteria ?? SubscriptionEngineCriteria::noConstraints()); + $result = $this->subscriptionEngine->catchUpActive($this->criteria ?? SubscriptionEngineCriteria::noConstraints()); + if ($result->errors !== []) { + throw new \RuntimeException('ASDA'); + } return $commitResult; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/RunMode.php b/Neos.ContentRepository.Core/Classes/Subscription/RunMode.php index a59db48528f..49a2583a27c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/RunMode.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/RunMode.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @internal + * @api */ enum RunMode : string { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/InMemorySubscriptionStore.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/InMemorySubscriptionStore.php index a24ad4e2141..36b35ce3b73 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/InMemorySubscriptionStore.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/InMemorySubscriptionStore.php @@ -20,10 +20,9 @@ public function __construct() $this->subscriptions = Subscriptions::none(); } - - public function findOneById(SubscriptionId $subscriptionId): ?Subscription + public function setup(): void { - return $this->subscriptions->contain($subscriptionId) ? $this->subscriptions->get($subscriptionId) : null; + // no setup required } public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions @@ -35,7 +34,7 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions if ($criteria->groups !== null && !$criteria->groups->contain($subscription->group)) { return false; } - if ($criteria->status !== null && !in_array($subscription->status, $criteria->status, true)) { + if (!$criteria->status->matches($subscription->status)) { return false; } return true; @@ -51,4 +50,15 @@ public function update(Subscription $subscription): void { $this->subscriptions = $this->subscriptions->with($subscription); } + + public function remove(Subscription $subscription): void + { + $this->subscriptions = $this->subscriptions->without($subscription->id); + } + + public function transactional(\Closure $closure): mixed + { + // In memory store does not support transaction boundaries + return $closure(); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php index 0985dafe9c6..b33c656e86f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php @@ -12,7 +12,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; /** - * @internal + * @api */ final readonly class SubscriptionCriteria { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 6a0c4da5bbe..d1a1e3d041d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -15,11 +15,18 @@ interface SubscriptionStoreInterface { public function setup(): void; - public function findOneById(SubscriptionId $subscriptionId): ?Subscription; - public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions; public function add(Subscription $subscription): void; public function update(Subscription $subscription): void; + + public function remove(Subscription $subscription): void; + + /** + * @template T + * @param \Closure():T $closure + * @return T + */ + public function transactional(\Closure $closure): mixed; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php deleted file mode 100644 index 57c5deaa45d..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreWithTransactionSupportInterface.php +++ /dev/null @@ -1,22 +0,0 @@ - - * @internal + * @api */ final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 2d8dcba7b99..b480f1ac1c5 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -4,13 +4,14 @@ namespace Neos\ContentRepository\Core\Subscription; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\EventStore\Model\Event\SequenceNumber; /** * Note: This class is mutable by design! * - * @internal + * @api */ final class Subscription { @@ -26,6 +27,9 @@ public function __construct( ) { } + /** + * @internal Only the {@see SubscriptionEngine} is supposed to instantiate subscriptions + */ public static function createFromSubscriber(Subscriber $subscriber): self { return new self( @@ -37,6 +41,9 @@ public static function createFromSubscriber(Subscriber $subscriber): self ); } + /** + * @internal Only the {@see SubscriptionEngine} is supposed to mutate subscriptions + */ public function set( SubscriptionStatus $status = null, SequenceNumber $position = null, @@ -57,6 +64,9 @@ public function set( ); } + /** + * @internal Only the {@see SubscriptionEngine} is supposed to mutate subscriptions + */ public function fail(\Throwable $exception): void { $this->error = SubscriptionError::fromPreviousStatusAndException($this->status, $exception); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php index df42ca5e965..5c53135dc66 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @internal + * @api */ final class SubscriptionError { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php index 4c5b5466a17..3f0eb880cf4 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @internal + * @api */ final class SubscriptionGroup { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php index ca98b63b387..69f9c32ed69 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @internal + * @api */ final class SubscriptionGroups implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php index 52b394c9fe4..777814c275e 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @internal + * @api */ final class SubscriptionId { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php index 5b3dc856b9a..9ccb916cb70 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @internal + * @api */ enum SubscriptionStatus : string { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php index 61c100a3aae..5753bd6c266 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @internal + * @api */ final class SubscriptionStatusFilter implements \IteratorAggregate { @@ -61,4 +61,9 @@ public function toStringArray(): array { return array_values(array_map(static fn (SubscriptionStatus $id) => $id->value, $this->statusByValue)); } + + public function matches(SubscriptionStatus $status): bool + { + return array_key_exists($status->value, $this->statusByValue); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php index df6f6e5e03c..dc50137b32a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -8,7 +8,7 @@ /** * @implements \IteratorAggregate - * @internal + * @api */ final class Subscriptions implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php index 753525af34f..bf18eaea3b2 100644 --- a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php +++ b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php @@ -6,7 +6,7 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; @@ -14,12 +14,15 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\StructureAdjustment\Adjustment\DimensionAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\DisallowedChildNodeAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\PropertyAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\StructureAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\TetheredNodeAdjustments; use Neos\ContentRepository\StructureAdjustment\Adjustment\UnknownNodeTypeAdjustment; +use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\Events; class StructureAdjustmentService implements ContentRepositoryServiceInterface { @@ -38,8 +41,10 @@ class StructureAdjustmentService implements ContentRepositoryServiceInterface private readonly ContentGraphInterface $liveContentGraph; public function __construct( - private readonly ContentRepository $contentRepository, - private readonly EventPersister $eventPersister, + ContentRepository $contentRepository, + private readonly EventStoreInterface $eventStore, + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, NodeTypeManager $nodeTypeManager, InterDimensionalVariationGraph $interDimensionalVariationGraph, PropertyConverter $propertyConverter, @@ -98,11 +103,20 @@ public function findAdjustmentsForNodeType(NodeTypeName $nodeTypeName): \Generat public function fixError(StructureAdjustment $adjustment): void { - if ($adjustment->remediation) { - $remediation = $adjustment->remediation; - $eventsToPublish = $remediation(); - assert($eventsToPublish instanceof EventsToPublish); - $this->eventPersister->publishEvents($this->contentRepository, $eventsToPublish); + if (!$adjustment->remediation) { + return; } + $remediation = $adjustment->remediation; + $eventsToPublish = $remediation(); + assert($eventsToPublish instanceof EventsToPublish); + $normalizedEvents = Events::fromArray( + $eventsToPublish->events->map($this->eventNormalizer->normalize(...)) + ); + $this->eventStore->commit( + $eventsToPublish->streamName, + $normalizedEvents, + $eventsToPublish->expectedVersion + ); + $this->subscriptionEngine->catchUpActive(); } } diff --git a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php index b9e75abeaff..534fe279028 100644 --- a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php +++ b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php @@ -16,7 +16,9 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor { return new StructureAdjustmentService( $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventPersister, + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, $serviceFactoryDependencies->nodeTypeManager, $serviceFactoryDependencies->interDimensionalVariationGraph, $serviceFactoryDependencies->propertyConverter, diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php index 8ca963361d9..0d80f9ba11a 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php @@ -19,9 +19,9 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; -use Neos\ContentRepository\Core\EventStore\Events; -use Neos\ContentRepository\Core\EventStore\EventsToPublish; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\Common\RebasableToOtherWorkspaceInterface; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Command\AddDimensionShineThrough; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Command\MoveDimensionSpacePoint; @@ -56,10 +56,12 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Exception\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventEnvelope; +use Neos\EventStore\Model\Events; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; use Neos\Utility\Arrays; @@ -286,19 +288,23 @@ protected function publishEvent(string $eventType, StreamName $streamName, array Event\EventData::fromString(json_encode($eventPayload)), Event\EventMetadata::fromArray([]) ); - /** @var EventPersister $eventPersister */ - $eventPersister = (new \ReflectionClass($this->currentContentRepository))->getProperty('eventPersister') - ->getValue($this->currentContentRepository); - /** @var EventNormalizer $eventNormalizer */ - $eventNormalizer = (new \ReflectionClass($eventPersister))->getProperty('eventNormalizer') - ->getValue($eventPersister); - $event = $eventNormalizer->denormalize($artificiallyConstructedEvent); - - $eventPersister->publishEvents($this->currentContentRepository, new EventsToPublish( - $streamName, - Events::with($event), - ExpectedVersion::ANY() - )); + + // HACK can be replaced, once https://github.com/neos/neos-development-collection/pull/5341 is merged + $eventStoreAndSubscriptionEngine = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getContentRepositoryService($eventStoreAndSubscriptionEngine); + $eventStoreAndSubscriptionEngine->eventStore->commit($streamName, Events::with($artificiallyConstructedEvent), ExpectedVersion::ANY()); + $eventStoreAndSubscriptionEngine->subscriptionEngine->catchUpActive(); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 8887a1d02d7..2662093bcbf 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -57,7 +57,7 @@ public function subscriptionsCatchUpCommand(string $contentRepository = 'default { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $subscriptionService->subscriptionEngine->run(); + $subscriptionService->subscriptionEngine->catchUpActive(); } public function subscriptionsResetCommand(string $contentRepository = 'default', bool $force = false): void diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 684039ed124..154e6b52b11 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -17,6 +17,7 @@ use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactories; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; @@ -268,7 +269,7 @@ private function buildContentGraphProjectionFactory(ContentRepositoryId $content /** * @param array $contentRepositorySettings - * @return CatchUpHookFactoryInterface + * @return CatchUpHookFactoryInterface */ private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactoryInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 6893c95f1cc..525f0f6fbc7 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -16,7 +16,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreWithTransactionSupportInterface; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; @@ -26,7 +26,7 @@ use Neos\EventStore\Model\Event\SequenceNumber; use Psr\Clock\ClockInterface; -final class DoctrineSubscriptionStore implements SubscriptionStoreWithTransactionSupportInterface +final class DoctrineSubscriptionStore implements SubscriptionStoreInterface { public function __construct( private string $tableName, @@ -68,21 +68,15 @@ public function setup(): void } } - public function findOneById(SubscriptionId $subscriptionId): ?Subscription - { - $row = $this->dbal->fetchAssociative('SELECT * FROM ' . $this->tableName . ' WHERE id = :subscriptionId', ['subscriptionId' => $subscriptionId->value]); - if ($row === false) { - return null; - } - return self::fromDatabase($row); - } - public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions { $queryBuilder = $this->dbal->createQueryBuilder() ->select('*') ->from($this->tableName) ->orderBy('id'); + if (!$this->dbal->getDatabasePlatform() instanceof SQLitePlatform) { + $queryBuilder->forUpdate(); + } if ($criteria->ids !== null) { $queryBuilder->andWhere('id IN (:ids)') ->setParameter( @@ -140,6 +134,16 @@ public function update(Subscription $subscription): void ); } + public function remove(Subscription $subscription): void + { + $this->dbal->delete( + $this->tableName, + [ + 'id' => $subscription->id->value, + ] + ); + } + /** * @return array */ diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php index 25b9650b322..71bada85b93 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php @@ -19,6 +19,6 @@ public function __construct( public function run(ProcessingContext $context): void { - $this->subscriptionService->subscriptionEngine->run(); + $this->subscriptionService->subscriptionEngine->catchUpActive(); } } From 9fef068cd10fa81308262ea187726077f0f2a285 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 14 Nov 2024 20:35:19 +0100 Subject: [PATCH 009/142] Remove `RunSubscriptionEventStore` --- .../Factory/ContentRepositoryFactory.php | 9 +-- .../EventStore/RunSubscriptionEventStore.php | 60 ------------------- 2 files changed, 3 insertions(+), 66 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index e3bde71144a..e2b49f095c1 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -38,7 +38,6 @@ use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\User\UserIdProviderInterface; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; -use Neos\ContentRepository\Core\Subscription\EventStore\RunSubscriptionEventStore; use Neos\ContentRepository\Core\Subscription\RetryStrategy\NoRetryStrategy; use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; @@ -59,7 +58,6 @@ final class ContentRepositoryFactory { private SubscriberFactoryDependencies $subscriberFactoryDependencies; - private EventStoreInterface $eventStore; private SubscriptionEngine $subscriptionEngine; private ContentGraphProjectionInterface $contentGraphProjection; private ProjectionStates $additionalProjectionStates; @@ -75,7 +73,7 @@ final class ContentRepositoryFactory */ public function __construct( private readonly ContentRepositoryId $contentRepositoryId, - EventStoreInterface $eventStore, + private readonly EventStoreInterface $eventStore, NodeTypeManager $nodeTypeManager, ContentDimensionSourceInterface $contentDimensionSource, Serializer $propertySerializer, @@ -86,7 +84,7 @@ public function __construct( private readonly CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, - private LoggerInterface|null $logger = null, + LoggerInterface|null $logger = null, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); $interDimensionalVariationGraph = new InterDimensionalVariationGraph( @@ -115,8 +113,7 @@ public function __construct( $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); $subscribers[] = $this->buildContentGraphSubscriber(); - $this->subscriptionEngine = new SubscriptionEngine($eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy(), $this->logger); - $this->eventStore = new RunSubscriptionEventStore($eventStore, $this->subscriptionEngine); + $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy(), $logger); } private function buildContentGraphSubscriber(): Subscriber diff --git a/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php b/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php deleted file mode 100644 index fe69b74ab2b..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/EventStore/RunSubscriptionEventStore.php +++ /dev/null @@ -1,60 +0,0 @@ -eventStore->setup(); - } - - public function status(): Status - { - return $this->eventStore->status(); - } - - public function load(StreamName|VirtualStreamName $streamName, EventStreamFilter $filter = null): EventStreamInterface - { - return $this->eventStore->load($streamName, $filter); - } - - public function commit(StreamName $streamName, \Neos\EventStore\Model\Events|Event $events, ExpectedVersion $expectedVersion): CommitResult - { - $commitResult = $this->eventStore->commit($streamName, $events, $expectedVersion); - $result = $this->subscriptionEngine->catchUpActive($this->criteria ?? SubscriptionEngineCriteria::noConstraints()); - if ($result->errors !== []) { - throw new \RuntimeException('ASDA'); - } - return $commitResult; - } - - public function deleteStream(StreamName $streamName): void - { - $this->eventStore->deleteStream($streamName); - } -} From 46ad9d1768b786adbb84501db5af85e7a20d24e6 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Thu, 14 Nov 2024 20:35:29 +0100 Subject: [PATCH 010/142] Improve error handling (WIP) --- .../Classes/ContentRepository.php | 10 ++++++++-- .../Classes/Subscription/Engine/Errors.php | 8 +++----- .../Classes/Subscription/Engine/SubscriptionEngine.php | 4 +--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 789f0ef6d04..296282295cf 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -89,7 +89,10 @@ public function handle(CommandInterface $command): void if ($toPublish instanceof EventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish), $eventsToPublish->expectedVersion); - $this->subscriptionEngine->catchUpActive(); + $catchUpResult = $this->subscriptionEngine->catchUpActive(); + if ($catchUpResult->hasErrors()) { + throw new \RuntimeException('Catchup led to errors.. todo', 1731612294); + } return; } @@ -118,7 +121,10 @@ public function handle(CommandInterface $command): void } finally { // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. - $this->subscriptionEngine->catchUpActive(); + $catchUpResult = $this->subscriptionEngine->catchUpActive(); + if ($catchUpResult->hasErrors()) { + throw new \RuntimeException('Catchup led to errors.. todo', 1731612294); + } } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index cc8ca6a638e..1291b70ff4d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -21,6 +21,9 @@ private function __construct( Error ...$errors ) { $this->errors = $errors; + if ($this->errors === []) { + throw new \InvalidArgumentException('Errors must not be empty.', 1731612542); + } } /** @@ -36,11 +39,6 @@ public function getIterator(): \Traversable yield from $this->errors; } - public function isEmpty(): bool - { - return $this->errors === []; - } - public function count(): int { return count($this->errors); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index e91cf09fb0d..547d880f17d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -312,10 +312,8 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus) { $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); } } - $this->logger?->info('Subscription Engine: Finish catch up.'); - - return ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); + return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); } ); } From 88cc6003f1673b35d16cff11e8f057f77e15d119 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Sun, 17 Nov 2024 13:47:37 +0100 Subject: [PATCH 011/142] Fix `ContentRepositoryFactory` constructor --- .../Classes/Factory/ContentRepositoryFactory.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index f81dde8718e..4ac9e61936c 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -77,7 +77,6 @@ public function __construct( NodeTypeManager $nodeTypeManager, ContentDimensionSourceInterface $contentDimensionSource, Serializer $propertySerializer, - ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, private readonly AuthProviderFactoryInterface $authProviderFactory, private readonly ClockInterface $clock, SubscriptionStoreInterface $subscriptionStore, From e3f85ac95590ff6ba41b4b55d883dd94b7e564be Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Sun, 17 Nov 2024 13:52:53 +0100 Subject: [PATCH 012/142] Improve error handling during `SubscriptionEngine::setup()` --- .../Engine/SubscriptionEngine.php | 30 ++++++++----------- .../Classes/Command/CrCommandController.php | 13 ++++++-- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 547d880f17d..5b6a13e07e5 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -50,24 +50,20 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $sk $this->subscriptionStore->setup(); $this->discoverNewSubscriptions(); $this->retrySubscriptions($criteria); - return $this->subscriptionManager->findForUpdate( - SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::NEW), - function (Subscriptions $subscriptions) use ($skipBooting) { - if ($subscriptions->isEmpty()) { - $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); - return Result::success(); - } - $lastSequenceNumber = $this->lastSequenceNumber(); - $errors = []; - foreach ($subscriptions as $subscription) { - $error = $this->setupSubscription($subscription, $lastSequenceNumber, $skipBooting); - if ($error !== null) { - $errors[] = $error; - } - } - return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); + $lastSequenceNumber = $this->lastSequenceNumber(); + $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::NEW)); + if ($subscriptions->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); + return Result::success(); + } + $errors = []; + foreach ($subscriptions as $subscription) { + $error = $this->setupSubscription($subscription, $lastSequenceNumber, $skipBooting); + if ($error !== null) { + $errors[] = $error; } - ); + } + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } public function boot(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 2662093bcbf..25472065b3d 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -41,9 +41,16 @@ public function setupCommand(string $contentRepository = 'default'): void $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); $subscriptionService->setupEventStore(); - $subscriptionService->subscriptionEngine->setup(); - - $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); + $setupResult = $subscriptionService->subscriptionEngine->setup(); + if ($setupResult->errors === null) { + $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); + return; + } + $this->outputLine('Setup of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $setupResult->errors->count() === 1 ? '' : 's']); + foreach ($setupResult->errors as $error) { + $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); + } + $this->quit(1); } public function subscriptionsBootCommand(string $contentRepository = 'default'): void From c43a2eab8d61c0356bb77f0ae7183adb4d5f88a5 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 10:22:42 +0100 Subject: [PATCH 013/142] Allow to reset subscriptions and implement progress callback (for progress bars) --- .../Engine/SubscriptionEngine.php | 65 ++++++++++++++++--- .../Engine/SubscriptionManager.php | 2 +- .../Classes/Command/CrCommandController.php | 38 +++++++++-- .../Processors/ProjectionResetProcessor.php | 2 +- 4 files changed, 93 insertions(+), 14 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 5b6a13e07e5..047e7cfa972 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -6,6 +6,7 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; +use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; @@ -63,17 +64,39 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $sk $errors[] = $error; } } + $this->subscriptionManager->flush(); return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } - public function boot(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult + public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { - return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::BOOTING)); + return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::BOOTING, $progressCallback)); } - public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null): ProcessedResult + public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { - return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE)); + return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE, $progressCallback)); + } + + public function reset(SubscriptionEngineCriteria|null $criteria = null, bool $skipBooting = false): Result + { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + + $this->logger?->info('Subscription Engine: Start to reset.'); + $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::any())); + if ($subscriptions->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions to reset.'); + return Result::success(); + } + $errors = []; + foreach ($subscriptions as $subscription) { + $error = $this->resetSubscription($subscription, $skipBooting); + if ($error !== null) { + $errors[] = $error; + } + } + $this->subscriptionManager->flush(); + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } public function teardown(SubscriptionEngineCriteria|null $criteria = null): Result @@ -170,8 +193,6 @@ private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $crite /** * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned - * - * @param bool $skipBooting */ private function setupSubscription(Subscription $subscription, SequenceNumber $lastSequenceNumber, bool $skipBooting): ?Error { @@ -199,6 +220,31 @@ private function setupSubscription(Subscription $subscription, SequenceNumber $l return null; } + /** + * TODO + */ + private function resetSubscription(Subscription $subscription, bool $skipBooting): ?Error + { + $subscriber = $this->subscribers->get($subscription->id); + if (!$subscriber->handler instanceof ProjectionEventHandler) { + $this->logger?->info(sprintf('Subscription Engine: Subscriber handler "%s" for "%s" is no instance of %s, skipping reset', $subscriber->handler::class, $subscription->id->value, ProjectionEventHandler::class)); + return null; + } + try { + $subscriber->handler->projection->resetState(); + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber handler "%s" for "%s" has an error in the resetState method: %s', $subscriber->handler::class, $subscription->id->value, $e->getMessage())); + return Error::fromSubscriptionIdAndException($subscription->id, $e); + } + $subscription->set( + status: $skipBooting ? SubscriptionStatus::ACTIVE : SubscriptionStatus::BOOTING, + position: SequenceNumber::none(), + ); + $this->subscriptionManager->update($subscription); + $this->logger?->debug(sprintf('Subscription Engine: For Subscriber handler "%s" for "%s" the resetState method has been executed.', $subscriber->handler::class, $subscription->id->value)); + return null; + } + private function retrySubscriptions(SubscriptionEngineCriteria $criteria): void { $this->subscriptionManager->findForUpdate( @@ -233,7 +279,7 @@ private function retrySubscription(Subscription $subscription): void $this->logger?->info(sprintf('Subscription Engine: Retry subscription "%s" (%d) and set back to %s.', $subscription->id->value, $subscription->retryAttempt, $subscription->status->value)); } - private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatus $subscriptionStatus): ProcessedResult + private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatus $subscriptionStatus, \Closure $progressClosure = null): ProcessedResult { $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); @@ -243,7 +289,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs return $this->subscriptionManager->findForUpdate( SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), - function (Subscriptions $subscriptions) use ($subscriptionStatus) { + function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosure) { if ($subscriptions->isEmpty()) { $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); @@ -261,6 +307,9 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus) { try { $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); foreach ($eventStream as $eventEnvelope) { + if ($progressClosure !== null) { + $progressClosure($eventEnvelope); + } $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); $sequenceNumber = $eventEnvelope->sequenceNumber; foreach ($subscriptions as $subscription) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php index 9bbb74e1832..d174c66e462 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -68,7 +68,7 @@ public function remove(Subscription $subscription): void $this->forRemove->attach($subscription); } - private function flush(): void + public function flush(): void { foreach ($this->forAdd as $subscription) { if ($this->forRemove->contains($subscription)) { diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 25472065b3d..7c2606c5063 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -53,14 +53,35 @@ public function setupCommand(string $contentRepository = 'default'): void $this->quit(1); } - public function subscriptionsBootCommand(string $contentRepository = 'default'): void + public function subscriptionsBootCommand(string $contentRepository = 'default', bool $quiet = false): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $subscriptionService->subscriptionEngine->boot(); + if (!$quiet) { + $this->outputLine('Booting new subscriptions'); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $bootResult = $subscriptionService->subscriptionEngine->boot(progressCallback: fn () => $this->output->progressAdvance()); + $this->output->progressFinish(); + $this->outputLine(); + if ($bootResult->errors === null) { + $this->outputLine('Done'); + return; + } + } else { + $bootResult = $subscriptionService->subscriptionEngine->boot(); + } + if ($bootResult->errors !== null) { + $this->outputLine('Booting of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $bootResult->errors->count() === 1 ? '' : 's']); + foreach ($bootResult->errors as $error) { + $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); + } + $this->quit(1); + } } - public function subscriptionsCatchUpCommand(string $contentRepository = 'default'): void + public function subscriptionsCatchUpCommand(string $contentRepository = 'default', bool $quiet = false): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); @@ -75,7 +96,16 @@ public function subscriptionsResetCommand(string $contentRepository = 'default', } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - //$subscriptionService->subscriptionEngine->reset(); + $resetResult = $subscriptionService->subscriptionEngine->reset(); + if ($resetResult->errors === null) { + $this->outputLine('Content Repository "%s" was reset', [$contentRepositoryId->value]); + return; + } + $this->outputLine('Reset of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $resetResult->errors->count() === 1 ? '' : 's']); + foreach ($resetResult->errors as $error) { + $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); + } + $this->quit(1); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php index 50509d60367..1dfdd7d4208 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php @@ -19,6 +19,6 @@ public function __construct( public function run(ProcessingContext $context): void { - // TODO implement $this->subscriptionService->reset(); + $this->subscriptionService->subscriptionEngine->reset(); } } From 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 11:19:00 +0100 Subject: [PATCH 014/142] Re-implement `cr:status` CLI command --- .../CatchUpHook/CatchUpHookFactories.php | 2 +- .../Classes/Projection/ProjectionStatuses.php | 41 --------- .../Classes/Service/SubscriptionService.php | 6 ++ .../ContentRepositoryStatus.php | 45 ---------- .../Classes/Subscription/Engine/Errors.php | 3 +- .../Engine/SubscriptionEngine.php | 17 +++- ...iptionEngineAlreadyProcessingException.php | 6 ++ .../Classes/Subscription/Subscription.php | 2 +- .../SubscriptionAndProjectionStatus.php | 33 +++++++ .../SubscriptionAndProjectionStatuses.php | 38 +++++++++ .../Classes/Command/CrCommandController.php | 85 +++++++++++++++---- 11 files changed, 171 insertions(+), 107 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php delete mode 100644 Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php index ce8d8ac5673..e383b32299f 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php @@ -8,7 +8,7 @@ /** * @implements CatchUpHookFactoryInterface - * @internal + * @api */ final class CatchUpHookFactories implements CatchUpHookFactoryInterface { diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php deleted file mode 100644 index cc146b1a76e..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php +++ /dev/null @@ -1,41 +0,0 @@ - - */ -final readonly class ProjectionStatuses implements \IteratorAggregate -{ - /** - * @param array>, ProjectionStatus> $statuses - */ - private function __construct( - public array $statuses, - ) { - } - - public static function createEmpty(): self - { - return new self([]); - } - - /** - * @param class-string> $projectionClassName - */ - public function with(string $projectionClassName, ProjectionStatus $projectionStatus): self - { - $statuses = $this->statuses; - $statuses[$projectionClassName] = $projectionStatus; - return new self($statuses); - } - - - public function getIterator(): \Traversable - { - yield from $this->statuses; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php b/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php index f1713c9fd08..321df1534ff 100644 --- a/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php +++ b/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php @@ -7,6 +7,7 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\EventStore\Status; /** * @api @@ -23,4 +24,9 @@ public function setupEventStore(): void { $this->eventStore->setup(); } + + public function eventStoreStatus(): Status + { + return $this->eventStore->status(); + } } diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php deleted file mode 100644 index c7de2e5bb28..00000000000 --- a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php +++ /dev/null @@ -1,45 +0,0 @@ -eventStoreStatus->type !== EventStoreStatusType::OK) { - return false; - } - foreach ($this->projectionStatuses as $projectionStatus) { - if ($projectionStatus->type !== ProjectionStatusType::OK) { - return false; - } - } - return true; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index 1291b70ff4d..bb15dae2360 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -6,11 +6,12 @@ /** * @implements \IteratorAggregate + * @api */ final readonly class Errors implements \IteratorAggregate, \Countable { /** - * array + * @var array */ private array $errors; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 047e7cfa972..d3d0617d5d5 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -8,6 +8,8 @@ use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; +use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; @@ -123,9 +125,20 @@ public function pause(SubscriptionEngineCriteria|null $criteria = null): Result return Result::success(); } - public function subscriptions(SubscriptionCriteria|null $criteria = null): Subscriptions + public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null): SubscriptionAndProjectionStatuses { - return $this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()); + $statuses = []; + foreach ($this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()) as $subscription) { + $subscriber = $this->subscribers->get($subscription->id); + $statuses[] = SubscriptionAndProjectionStatus::create( + subscriptionId: $subscription->id, + subscriptionStatus: $subscription->status, + subscriptionPosition: $subscription->position, + subscriptionError: $subscription->error, + projectionStatus: $subscriber->handler instanceof ProjectionEventHandler ? $subscriber->handler->projection->status() : null, + ); + } + return SubscriptionAndProjectionStatuses::fromArray($statuses); } private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php index 3bf565d6b4c..9631724adfb 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php @@ -4,8 +4,14 @@ namespace Neos\ContentRepository\Core\Subscription\Exception; +/** + * @api + */ final class SubscriptionEngineAlreadyProcessingException extends \RuntimeException { + /** + * @internal + */ public function __construct() { parent::__construct('Subscription engine is already processing'); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index b480f1ac1c5..5cda857fce0 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -11,7 +11,7 @@ /** * Note: This class is mutable by design! * - * @api + * @internal */ final class Subscription { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php new file mode 100644 index 00000000000..56fca0ac88a --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php @@ -0,0 +1,33 @@ + + */ +final readonly class SubscriptionAndProjectionStatuses implements \IteratorAggregate +{ + /** + * @var array $statuses + */ + private array $statuses; + + private function __construct( + SubscriptionAndProjectionStatus ...$statuses, + ) { + $this->statuses = $statuses; + } + + public static function fromArray(array $statuses): self + { + return new self(...$statuses); + } + + public function getIterator(): \Traversable + { + yield from $this->statuses; + } + + public function isEmpty(): bool + { + return $this->statuses === []; + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 7c2606c5063..5905aff04f0 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -6,6 +6,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionStatusType; use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\StatusType; @@ -123,35 +124,87 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $status = new stdClass();//TODO $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + $eventStoreStatus = $subscriptionService->eventStoreStatus(); + $hasErrors = false; + $setupRequired = false; + $bootingRequired = false; + $resetRequired = false; $this->output('Event Store: '); - $this->outputLine(match ($status->eventStoreStatus->type) { + $this->outputLine(match ($eventStoreStatus->type) { StatusType::OK => 'OK', StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - if ($verbose && $status->eventStoreStatus->details !== '') { - $this->outputFormatted($status->eventStoreStatus->details, [], 2); + $hasErrors |= $eventStoreStatus->type === StatusType::ERROR; + if ($verbose && $eventStoreStatus->details !== '') { + $this->outputFormatted($eventStoreStatus->details, [], 2); } $this->outputLine(); - foreach ($status->projectionStatuses as $projectionName => $projectionStatus) { - $this->output('Projection "%s": ', [$projectionName]); - $this->outputLine(match ($projectionStatus->type) { - ProjectionStatusType::OK => 'OK', - ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', - ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', - ProjectionStatusType::ERROR => 'ERROR', + $this->outputLine('Subscriptions:'); + $subscriptionStatuses = $subscriptionService->subscriptionEngine->subscriptionStatuses(); + if ($subscriptionStatuses->isEmpty()) { + $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); + $this->quit(1); + } + foreach ($subscriptionStatuses as $status) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Subscription: ', [$status->subscriptionId->value]); + $this->output(match ($status->subscriptionStatus) { + SubscriptionStatus::NEW => 'NEW', + SubscriptionStatus::BOOTING => 'BOOTING', + SubscriptionStatus::ACTIVE => 'ACTIVE', + SubscriptionStatus::PAUSED => 'PAUSED', + SubscriptionStatus::FINISHED => 'FINISHED', + SubscriptionStatus::DETACHED => 'DETACHED', + SubscriptionStatus::ERROR => 'ERROR', }); - if ($verbose && ($projectionStatus->type !== ProjectionStatusType::OK || $projectionStatus->details)) { - $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $bootingRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; + if ($verbose && $status->subscriptionError !== null) { + $lines = explode(chr(10), $status->subscriptionError->errorMessage ?: 'No details available.'); foreach ($lines as $line) { - $this->outputLine(' ' . $line); + $this->outputLine(' %s', [$line]); } - $this->outputLine(); + } + if ($status->projectionStatus !== null) { + $this->output(' Projection: '); + $this->outputLine(match ($status->projectionStatus->type) { + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', + ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', + ProjectionStatusType::ERROR => 'ERROR', + }); + $hasErrors |= $status->projectionStatus->type === ProjectionStatusType::ERROR; + $setupRequired |= $status->projectionStatus->type === ProjectionStatusType::SETUP_REQUIRED; + $resetRequired |= $status->projectionStatus->type === ProjectionStatusType::REPLAY_REQUIRED; + if ($verbose && ($status->projectionStatus->type !== ProjectionStatusType::OK || $status->projectionStatus->details)) { + $lines = explode(chr(10), $status->projectionStatus->details ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' ' . $line); + } + $this->outputLine(); + } + } + } + if ($verbose) { + $this->outputLine(); + if ($setupRequired) { + $this->outputLine('Setup required, please run ./flow cr:setup'); + } + if ($bootingRequired) { + $this->outputLine('Some subscriptions need to be booted, please run ./flow cr:subscriptionsboot'); + } + if ($resetRequired) { + $this->outputLine('Some subscriptions need to be replayed, please run ./flow cr:subscriptionsreset'); + } + if ($hasErrors) { + $this->outputLine('Some subscriptions/projections have failed'); } } - if (!$status->isOk()) { + if ($hasErrors) { $this->quit(1); } } From 3744fd5b4603394aab36efec0c8dda96f31e9d68 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 11:29:47 +0100 Subject: [PATCH 015/142] Fix `test_parallel` cr settings type annotation tweaks --- .../Configuration/Settings.yaml | 2 ++ .../Classes/Subscription/Engine/Errors.php | 3 --- .../Classes/Subscription/SubscriptionAndProjectionStatuses.php | 3 +++ .../Classes/ContentRepositoryRegistry.php | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml index d38e376fd54..9e6a03fbb4a 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml @@ -31,6 +31,8 @@ Neos: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\AuthProvider\StaticAuthProviderFactory clock: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory propertyConverters: {} contentGraphProjection: factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index bb15dae2360..0664f52deb7 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -15,9 +15,6 @@ */ private array $errors; - /** - * @param array $errors - */ private function __construct( Error ...$errors ) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php index c2b2babe263..79c4b74e661 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php @@ -21,6 +21,9 @@ private function __construct( $this->statuses = $statuses; } + /** + * @param array $statuses + */ public static function fromArray(array $statuses): self { return new self(...$statuses); diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 6d032333fb8..6e6287f502d 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -267,9 +267,8 @@ private function buildContentGraphProjectionFactory(ContentRepositoryId $content /** * @param array $contentRepositorySettings - * @return CatchUpHookFactoryInterface */ - private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactoryInterface + private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactories { if (!isset($contentRepositorySettings['contentGraphProjection']['catchUpHooks'])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.catchUpHooks configured.', $contentRepositoryId->value); From c0fbfe1fc819f92c28e85e913eb3d03d0fd2bcc3 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 11:47:31 +0100 Subject: [PATCH 016/142] Fix `test_parallel` cr settings 2/2 --- .../Configuration/Settings.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml index 9e6a03fbb4a..44bf52aa6de 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Settings.yaml @@ -36,6 +36,7 @@ Neos: propertyConverters: {} contentGraphProjection: factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory + catchUpHooks: {} Flow: object: From f5ff7d6a24c02eb2250d992a65f5712920fab126 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 13:06:31 +0100 Subject: [PATCH 017/142] Tweak subscription engine setup/reset from tests --- .../CRBehavioralTestsSubjectProvider.php | 8 +- .../Parallel/AbstractParallelTestCase.php | 10 ++- .../Classes/ContentRepository.php | 88 ------------------- .../Behavior/CRRegistrySubjectProvider.php | 8 +- 4 files changed, 20 insertions(+), 94 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php index 19eb183ff16..791aecab3b9 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php @@ -18,6 +18,7 @@ use Behat\Gherkin\Node\TableNode; use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Helpers\GherkinTableNodeBasedContentDimensionSource; use Neos\EventStore\EventStoreInterface; @@ -169,8 +170,10 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository * Catch Up process and the testcase reset. */ $contentRepository = $this->createContentRepository($contentRepositoryId); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $contentRepository->setUp(); + $subscriptionService->setupEventStore(); + $subscriptionService->subscriptionEngine->setup(); self::$alreadySetUpContentRepositories[] = $contentRepository->id; } /** @var EventStoreInterface $eventStore */ @@ -179,7 +182,8 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); - $contentRepository->resetProjectionStates(); + $subscriptionService->subscriptionEngine->reset(); + $subscriptionService->subscriptionEngine->boot(); return $contentRepository; } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php index 569609ee5ce..d4646e8165e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php @@ -16,6 +16,8 @@ use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\SubscriptionService; +use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Core\Bootstrap; @@ -69,15 +71,19 @@ final protected function setUpContentRepository( ContentRepositoryId $contentRepositoryId ): ContentRepository { $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - $contentRepository->setUp(); + /** @var SubscriptionService $subscriptionService */ + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + $subscriptionService->setupEventStore(); + $subscriptionService->subscriptionEngine->setup(); $connection = $this->objectManager->get(Connection::class); // reset events and projections $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); $connection->executeStatement('TRUNCATE ' . $eventTableName); - $contentRepository->resetProjectionStates(); + $subscriptionService->subscriptionEngine->reset(); + $subscriptionService->subscriptionEngine->boot(); return $contentRepository; } diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 7c13c0a28aa..43534f3100d 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -151,94 +151,6 @@ public function projectionState(string $projectionStateClassName): ProjectionSta } } -// /** -// * @param class-string> $projectionClassName -// */ -// public function catchUpProjection(string $projectionClassName, CatchUpOptions $options): void -// { -// $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); -// -// $catchUpHookFactory = $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection); -// $catchUpHook = $catchUpHookFactory?->build($this); -// -// // TODO allow custom stream name per projection -// $streamName = VirtualStreamName::all(); -// $eventStream = $this->eventStore->load($streamName); -// if ($options->maximumSequenceNumber !== null) { -// $eventStream = $eventStream->withMaximumSequenceNumber($options->maximumSequenceNumber); -// } -// -// $eventApplier = function (EventEnvelope $eventEnvelope) use ($projection, $catchUpHook, $options) { -// $event = $this->eventNormalizer->denormalize($eventEnvelope->event); -// if ($options->progressCallback !== null) { -// ($options->progressCallback)($event, $eventEnvelope); -// } -// if (!$projection->canHandle($event)) { -// return; -// } -// $catchUpHook?->onBeforeEvent($event, $eventEnvelope); -// $projection->apply($event, $eventEnvelope); -// if ($projection instanceof WithMarkStaleInterface) { -// $projection->markStale(); -// } -// $catchUpHook?->onAfterEvent($event, $eventEnvelope); -// }; -// -// $catchUp = CatchUp::create($eventApplier, $projection->getCheckpointStorage()); -// -// if ($catchUpHook !== null) { -// $catchUpHook->onBeforeCatchUp(); -// $catchUp = $catchUp->withOnBeforeBatchCompleted(fn() => $catchUpHook->onBeforeBatchCompleted()); -// } -// $catchUp->run($eventStream); -// $catchUpHook?->onAfterCatchUp(); -// } - -// public function catchupProjections(): void -// { -// foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { -// // FIXME optimise by only loading required events once and not per projection -// // see https://github.com/neos/neos-development-collection/pull/4988/ -// $this->catchUpProjection($projection::class, CatchUpOptions::create()); -// } -// } - -// public function setUp(): void -// { -// $this->eventStore->setup(); -// foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { -// $projection->setUp(); -// } -// } - -// public function status(): ContentRepositoryStatus -// { -// $projectionStatuses = ProjectionStatuses::createEmpty(); -// foreach ($this->projectionsAndCatchUpHooks->projections as $projectionClassName => $projection) { -// $projectionStatuses = $projectionStatuses->with($projectionClassName, $projection->status()); -// } -// return new ContentRepositoryStatus( -// $this->eventStore->status(), -// $projectionStatuses, -// ); -// } - -// public function resetProjectionStates(): void -// { -// foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { -// $projection->reset(); -// } -// } - -// /** -// * @param class-string> $projectionClassName -// */ -// public function resetProjectionState(string $projectionClassName): void -// { -// $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); -// $projection->reset(); -// } - /** * @throws WorkspaceDoesNotExist if the workspace does not exist * @throws AccessDenied if no read access is granted to the workspace ({@see AuthProviderInterface}) diff --git a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php index c026f755119..51216bbbd7a 100644 --- a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php +++ b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php @@ -17,6 +17,7 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; @@ -55,6 +56,7 @@ protected function setUpCRRegistry(): void public function iInitializeContentRepository(string $contentRepositoryId): void { $contentRepository = $this->getContentRepository(ContentRepositoryId::fromString($contentRepositoryId)); + $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepository->id, new SubscriptionServiceFactory()); /** @var EventStoreInterface $eventStore */ $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); /** @var Connection $databaseConnection */ @@ -63,10 +65,12 @@ public function iInitializeContentRepository(string $contentRepositoryId): void $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $contentRepository->setUp(); + $subscriptionService->setupEventStore(); + $subscriptionService->subscriptionEngine->setup(); self::$alreadySetUpContentRepositories[] = $contentRepository->id; } - $contentRepository->resetProjectionStates(); + $subscriptionService->subscriptionEngine->reset(); + $subscriptionService->subscriptionEngine->boot(); } /** From f8a8b5b773d8a6cd8e96526e5ef6685206273e23 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 13:22:03 +0100 Subject: [PATCH 018/142] Fix behat tests ? ...probably not --- .../Classes/Factory/ContentRepositoryFactory.php | 1 + ...ContentRepositoryServiceFactoryDependencies.php | 4 ++++ .../Features/Bootstrap/CRTestSuiteTrait.php | 2 +- .../Bootstrap/ContentRepositorySecurityTrait.php | 14 +++++++------- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 4ac9e61936c..bf80a6e6b91 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -233,6 +233,7 @@ public function buildService( $this->subscriberFactoryDependencies, $this->eventStore, $this->getOrBuild(), + $this->contentGraphProjection->getState(), $this->subscriptionEngine, ); return $serviceFactory->build($serviceFactoryDependencies); diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index d3172398e7c..756440449f3 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -21,6 +21,7 @@ use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; @@ -43,6 +44,7 @@ private function __construct( public InterDimensionalVariationGraph $interDimensionalVariationGraph, public PropertyConverter $propertyConverter, public ContentRepository $contentRepository, + public ContentGraphReadModelInterface $contentGraphReadModel, // we don't need CommandBus, because this is included in ContentRepository->handle() public SubscriptionEngine $subscriptionEngine, ) { @@ -55,6 +57,7 @@ public static function create( SubscriberFactoryDependencies $projectionFactoryDependencies, EventStoreInterface $eventStore, ContentRepository $contentRepository, + ContentGraphReadModelInterface $contentGraphReadModel, SubscriptionEngine $subscriptionEngine, ): self { return new self( @@ -67,6 +70,7 @@ public static function create( $projectionFactoryDependencies->interDimensionalVariationGraph, $projectionFactoryDependencies->propertyConverter, $contentRepository, + $contentGraphReadModel, $subscriptionEngine, ); } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index a7347cadcc7..10583ef8e3c 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -148,7 +148,7 @@ public function iExpectTheGraphProjectionToConsistOfExactlyNodes(int $expectedNu public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { // TODO find replacement – is that needed at all? - $this->instance = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection->getState(); + $this->instance = $serviceFactoryDependencies->contentGraphReadModel; return new class implements ContentRepositoryServiceInterface { }; diff --git a/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php b/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php index 86fc2bb454e..e5737155da6 100644 --- a/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php +++ b/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php @@ -17,7 +17,7 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Helpers\TestingAuthProvider; @@ -67,19 +67,19 @@ private function enableContentRepositorySecurity(): void return; } $contentRepositoryAuthProviderFactory = $this->getObject(ContentRepositoryAuthProviderFactory::class); - $contentGraphProjection = $this->getContentRepositoryService(new class implements ContentRepositoryServiceFactoryInterface { + $contentGraphReadModel = $this->getContentRepositoryService(new class implements ContentRepositoryServiceFactoryInterface { public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - $contentGraphProjection = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection; - return new class ($contentGraphProjection) implements ContentRepositoryServiceInterface { + $contentGraphReadModel = $serviceFactoryDependencies->contentGraphReadModel; + return new class ($contentGraphReadModel) implements ContentRepositoryServiceInterface { public function __construct( - public ContentGraphProjectionInterface $contentGraphProjection, + public ContentGraphReadModelInterface $contentGraphReadModel, ) { } }; } - })->contentGraphProjection; - $contentRepositoryAuthProvider = $contentRepositoryAuthProviderFactory->build($this->currentContentRepository->id, $contentGraphProjection->getState()); + })->contentGraphReadModel; + $contentRepositoryAuthProvider = $contentRepositoryAuthProviderFactory->build($this->currentContentRepository->id, $contentGraphReadModel); TestingAuthProvider::replaceAuthProvider($contentRepositoryAuthProvider); $this->crSecurity_contentRepositorySecurityEnabled = true; From 1562435ab9319ca7b31b9dbf98cfd5a4ea42227a Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 13:56:47 +0100 Subject: [PATCH 019/142] Disable `EventExportProcessor` test to see wether other tests succeed --- ...ntExportProcessor.feature => EventExportProcessor.feature.bkp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Neos.ContentRepository.Export/Tests/Behavior/Features/{EventExportProcessor.feature => EventExportProcessor.feature.bkp} (100%) diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature.bkp similarity index 100% rename from Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature rename to Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature.bkp From 2902dc25ceec9af61dd189f1fc73146c958025f2 Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Mon, 18 Nov 2024 15:24:41 +0100 Subject: [PATCH 020/142] Revert "Disable `EventExportProcessor` test" This reverts commit 1562435ab9319ca7b31b9dbf98cfd5a4ea42227a. --- ...ntExportProcessor.feature.bkp => EventExportProcessor.feature} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Neos.ContentRepository.Export/Tests/Behavior/Features/{EventExportProcessor.feature.bkp => EventExportProcessor.feature} (100%) diff --git a/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature.bkp b/Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature similarity index 100% rename from Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature.bkp rename to Neos.ContentRepository.Export/Tests/Behavior/Features/EventExportProcessor.feature From 0fe05b560c0f2a33bc81c9d27168ca41b9fb9a2e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:06:09 +0100 Subject: [PATCH 021/142] TASK: Fix phpstan --- .../Classes/Factory/ProjectionSubscriberFactory.php | 3 ++- .../Classes/Projection/ProjectionStates.php | 4 +++- .../Classes/ContentRepositoryRegistry.php | 8 ++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php index 574d4a7d312..00398b362c1 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -6,6 +6,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; @@ -18,7 +19,7 @@ final readonly class ProjectionSubscriberFactory implements ContentRepositorySubscriberFactoryInterface { /** - * @param ProjectionFactoryInterface $projectionFactory + * @param ProjectionFactoryInterface> $projectionFactory * @param array $projectionFactoryOptions */ public function __construct( diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php index b4eb01fee28..ba04da0e7a6 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php @@ -56,7 +56,9 @@ public function get(string $className): ProjectionStateInterface if (!array_key_exists($className, $this->statesByClassName)) { throw new \InvalidArgumentException(sprintf('The state class "%s" does not exist.', $className), 1729687836); } - return $this->statesByClassName[$className]; + /** @var T $state */ + $state = $this->statesByClassName[$className]; + return $state; } public function getIterator(): \Traversable diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 6e6287f502d..fc75a43629b 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -267,8 +267,9 @@ private function buildContentGraphProjectionFactory(ContentRepositoryId $content /** * @param array $contentRepositorySettings + * @return CatchUpHookFactoryInterface */ - private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactories + private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactoryInterface { if (!isset($contentRepositorySettings['contentGraphProjection']['catchUpHooks'])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.catchUpHooks configured.', $contentRepositoryId->value); @@ -285,9 +286,11 @@ private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $conten } $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); } + /** @var CatchUpHookFactoryInterface $catchUpHookFactories */ return $catchUpHookFactories; } + /** @param array $contentRepositorySettings */ private function buildCommandHooksFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CommandHooksFactory { $commandHooksSettings = $contentRepositorySettings['commandHooks'] ?? []; @@ -309,12 +312,13 @@ private function buildCommandHooksFactory(ContentRepositoryId $contentRepository return new CommandHooksFactory(...$commandHookFactories); } + /** @param array $contentRepositorySettings */ private function buildAdditionalSubscribersFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories { if (!is_array($contentRepositorySettings['projections'] ?? [])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); } - /** @var array $projectionFactories */ + /** @var array $projectionSubscriberFactories */ $projectionSubscriberFactories = []; foreach (($contentRepositorySettings['projections'] ?? []) as $projectionName => $projectionOptions) { // Allow projections to be disabled by setting their configuration to `null` From 8967eada6c18e9aad2d4e218e586ab598399d376 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:22:30 +0100 Subject: [PATCH 022/142] TASK: Add sanity check assertion after setting node properties a couple of times --- .../WorkspacePublicationDuringWritingTest.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php index d96a9adddf0..92e22a60d68 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php @@ -173,6 +173,11 @@ public function whileANodesArWrittenOnLive(): void $this->log('writing finished'); Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::forLive())->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('nody-mc-nodeface')); + Assert::assertNotNull($node); + Assert::assertSame($node->getProperty('title'), 'changed-title-50'); } /** From 2b5d33817b91b8d65d4ef4b774c18dcb3403b49e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:37:55 +0100 Subject: [PATCH 023/142] TASK: Introduce dedicated `contentRepositoryLogger` --- .../Classes/ContentRepository.php | 2 +- .../Classes/ContentRepositoryRegistry.php | 4 +++- .../Configuration/Objects.yaml | 9 +++++++++ .../Configuration/Settings.yaml | 15 +++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 43534f3100d..09ee8642405 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -55,7 +55,7 @@ * * @api */ -final readonly class ContentRepository +final class ContentRepository { /** * @internal use the {@see ContentRepositoryFactory::getOrBuild()} to instantiate diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index fc75a43629b..5ffe1c220b7 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -57,6 +57,9 @@ final class ContentRepositoryRegistry */ private array $factoryInstances = []; + #[Flow\Inject(name: 'Neos.ContentRepositoryRegistry:Logger', lazy: false)] + protected LoggerInterface $logger; + /** * @param array $settings */ @@ -64,7 +67,6 @@ public function __construct( private readonly array $settings, private readonly ObjectManagerInterface $objectManager, private readonly SubgraphCachePool $subgraphCachePool, - private readonly LoggerInterface $logger, ) { } diff --git a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml index df582d41eba..f41e1e795aa 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml @@ -14,3 +14,12 @@ Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory: value: 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory' 2: object: 'Doctrine\DBAL\Connection' + +'Neos.ContentRepositoryRegistry:Logger': + className: Psr\Log\LoggerInterface + scope: singleton + factoryObjectName: Neos\Flow\Log\PsrLoggerFactoryInterface + factoryMethodName: get + arguments: + 1: + value: contentRepositoryLogger diff --git a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml index d8b23715729..d78307056a8 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml @@ -12,6 +12,21 @@ Neos: ignoredClasses: Neos\\ContentRepository\\SharedModel\\NodeType\\NodeTypeManager: true + log: + psr3: + 'Neos\Flow\Log\PsrLoggerFactory': + contentRepositoryLogger: + default: + class: Neos\Flow\Log\Backend\FileBackend + options: + # todo context aware? FLOW_APPLICATION_CONTEXT .. but that contains / + logFileURL: '%FLOW_PATH_DATA%Logs/ContentRepository.log' + createParentDirectories: true + severityThreshold: '%LOG_INFO%' + maximumLogFileSize: 10485760 + logFilesToKeep: 1 + logMessageOrigin: false + ContentRepositoryRegistry: contentRepositories: default: From 2d3a13611717f6d5ce4ef69edda97e5e66066d7f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:40:37 +0100 Subject: [PATCH 024/142] Fix parallel tests and publish events on correct stream --- Neos.ContentRepository.Core/Classes/ContentRepository.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 09ee8642405..7effaadba9a 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -121,7 +121,7 @@ public function handle(CommandInterface $command): void // } $yieldedErrorStrategy = $toPublish->throw($concurrencyException); if ($yieldedErrorStrategy instanceof EventsToPublish) { - $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy), $yieldedErrorStrategy->expectedVersion); + $this->eventStore->commit($yieldedErrorStrategy->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy), $yieldedErrorStrategy->expectedVersion); } throw $concurrencyException; } From 46dc510d4b213426bca40bec2393dde956b2e474 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:45:48 +0100 Subject: [PATCH 025/142] TASK: Simplify `normalizeEvents` --- Neos.ContentRepository.Core/Classes/ContentRepository.php | 6 +++--- .../Classes/EventStore/EventNormalizer.php | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 7effaadba9a..ec73f97a9a9 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -96,7 +96,7 @@ public function handle(CommandInterface $command): void // simple case if ($toPublish instanceof EventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); - $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish), $eventsToPublish->expectedVersion); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); $catchUpResult = $this->subscriptionEngine->catchUpActive(); if ($catchUpResult->hasErrors()) { throw new \RuntimeException('Catchup led to errors.. todo', 1731612294); @@ -109,7 +109,7 @@ public function handle(CommandInterface $command): void foreach ($toPublish as $yieldedEventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($yieldedEventsToPublish); try { - $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish), $eventsToPublish->expectedVersion); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); } catch (ConcurrencyException $concurrencyException) { // we pass the exception into the generator (->throw), so it could be try-caught and reacted upon: // @@ -121,7 +121,7 @@ public function handle(CommandInterface $command): void // } $yieldedErrorStrategy = $toPublish->throw($concurrencyException); if ($yieldedErrorStrategy instanceof EventsToPublish) { - $this->eventStore->commit($yieldedErrorStrategy->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy), $yieldedErrorStrategy->expectedVersion); + $this->eventStore->commit($yieldedErrorStrategy->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy->events), $yieldedErrorStrategy->expectedVersion); } throw $concurrencyException; } diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php index 0e6226f4873..14943d68c97 100644 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php +++ b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php @@ -5,6 +5,7 @@ namespace Neos\ContentRepository\Core\EventStore; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\EventStore\Events as DomainEvents; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasClosed; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasReopened; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; @@ -148,9 +149,9 @@ public function normalize(EventInterface|DecoratedEvent $event): Event ); } - public function normalizeEvents(EventsToPublish $eventsToPublish): Events + public function normalizeEvents(DomainEvents $events): Events { - return Events::fromArray($eventsToPublish->events->map($this->normalize(...))); + return Events::fromArray($events->map($this->normalize(...))); } public function denormalize(Event $event): EventInterface From de92d79487c8ef30e5f7dfd5578c29224b122e0e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:30:21 +0100 Subject: [PATCH 026/142] TASK: Test that locking and concurrent writing works under heavy load Attempting to write to the content repository from two threads previously would cause likely an exception in Neos 9.0: > Failed to acquire checkpoint lock for subscriber "Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjection" because it is acquired already Now `forUpdate` is used in `DoctrineSubscriptionStore::findByCriteria` which solves this. and if commented out throws as expected: > An exception occurred while executing a query: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction. --- .../ParallelWritingInWorkspacesTest.php | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php new file mode 100644 index 00000000000..ff40fdca1bc --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php @@ -0,0 +1,217 @@ +log('------ process started ------'); + FakeContentDimensionSourceFactory::setWithoutDimensions(); + FakeNodeTypeManagerFactory::setConfiguration([ + 'Neos.ContentRepository:Root' => [], + 'Neos.ContentRepository.Testing:Content' => [], + 'Neos.ContentRepository.Testing:Document' => [ + 'properties' => [ + 'title' => [ + 'type' => 'string' + ] + ], + 'childNodes' => [ + 'tethered-a' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-b' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-c' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-d' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-e' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ] + ] + ] + ]); + + $setupLockResource = fopen(self::SETUP_LOCK_PATH, 'w+'); + + $exclusiveNonBlockingLockResult = flock($setupLockResource, LOCK_EX | LOCK_NB); + if ($exclusiveNonBlockingLockResult === false) { + $this->log('waiting for setup'); + if (!flock($setupLockResource, LOCK_SH)) { + throw new \RuntimeException('failed to acquire blocking shared lock'); + } + $this->contentRepository = $this->contentRepositoryRegistry + ->get(ContentRepositoryId::fromString('test_parallel')); + $this->log('wait for setup finished'); + return; + } + + $this->log('setup started'); + $contentRepository = $this->setUpContentRepository(ContentRepositoryId::fromString('test_parallel')); + + $origin = OriginDimensionSpacePoint::createWithoutDimensions(); + $contentRepository->handle(CreateRootWorkspace::create( + WorkspaceName::forLive(), + ContentStreamId::fromString('live-cs-id') + )); + $contentRepository->handle(CreateRootNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + NodeTypeName::fromString(NodeTypeName::ROOT_NODE_TYPE_NAME) + )); + $contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('nody-mc-nodeface'), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + $origin, + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title-original' + ]) + )); + $contentRepository->handle(CreateWorkspace::create( + WorkspaceName::fromString('user-test'), + WorkspaceName::forLive(), + ContentStreamId::fromString('user-cs-id') + )); + + $this->contentRepository = $contentRepository; + + if (!flock($setupLockResource, LOCK_UN)) { + throw new \RuntimeException('failed to release setup lock'); + } + + $this->log('setup finished'); + } + + /** + * @test + * @group parallel + */ + public function whileANodesArWrittenOnLive(): void + { + $this->log('1. writing started'); + + touch(self::WRITING_IS_RUNNING_FLAG_PATH); + + try { + for ($i = 0; $i <= 100; $i++) { + $this->contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('nody-mc-nodeface-' . $i), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + OriginDimensionSpacePoint::createWithoutDimensions(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title' + ]) + )); + } + } finally { + unlink(self::WRITING_IS_RUNNING_FLAG_PATH); + } + + $this->log('1. writing finished'); + Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::forLive())->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('nody-mc-nodeface-100')); + Assert::assertNotNull($node); + } + + /** + * @test + * @group parallel + */ + public function thenConcurrentPublishLeadsToException(): void + { + if (!is_file(self::WRITING_IS_RUNNING_FLAG_PATH)) { + $this->log('waiting for 2. writing'); + + $this->awaitFile(self::WRITING_IS_RUNNING_FLAG_PATH); + // If write is the process that does the (slowish) setup, and then waits for the rebase to start, + // We give the CR some time to close the content stream + // TODO find another way than to randomly wait!!! + // The problem is, if we dont sleep it happens often that the modification works only then the rebase is startet _really_ + // Doing the modification several times in hope that the second one fails will likely just stop the rebase thread as it cannot close + usleep(10000); + } + + $this->log('2. writing started'); + + for ($i = 0; $i <= 100; $i++) { + $this->contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::fromString('user-test'), + NodeAggregateId::fromString('user-nody-mc-nodeface-' . $i), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + OriginDimensionSpacePoint::createWithoutDimensions(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title' + ]) + )); + } + + $this->log('2. writing finished'); + + Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::fromString('user-test'))->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('user-nody-mc-nodeface-100')); + Assert::assertNotNull($node); + } +} From f58cefa97c76ee4125534a9564b54874bc88b4e7 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:47:00 +0100 Subject: [PATCH 027/142] TASK: Improve exception thrown if subscriber failed --- .../Classes/ContentRepository.php | 8 ++---- .../Classes/Service/ContentStreamPruner.php | 2 +- .../Subscription/Engine/ProcessedResult.php | 25 ++++++++++++++++--- .../Engine/SubscriptionManager.php | 2 +- .../Classes/Command/CrCommandController.php | 4 +-- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index ec73f97a9a9..95d29db1dd3 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -98,9 +98,7 @@ public function handle(CommandInterface $command): void $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); $catchUpResult = $this->subscriptionEngine->catchUpActive(); - if ($catchUpResult->hasErrors()) { - throw new \RuntimeException('Catchup led to errors.. todo', 1731612294); - } + $catchUpResult->throwOnFailure(); return; } @@ -130,9 +128,7 @@ public function handle(CommandInterface $command): void // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. $catchUpResult = $this->subscriptionEngine->catchUpActive(); - if ($catchUpResult->hasErrors()) { - throw new \RuntimeException('Catchup led to errors.. todo', 1731612294); - } + $catchUpResult->throwOnFailure(); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index eac4e78f9af..a71f8a4d7d2 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -163,7 +163,7 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta if ($danglingContentStreamsPresent) { $result = $this->subscriptionEngine->catchUpActive(); - if ($result->hasErrors()) { + if ($result->hasFailed()) { $outputFn('Catchup after removing unused content streams led to errors. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.'); } } else { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php index e1a84e5c38c..824b5e87572 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -10,8 +10,8 @@ final readonly class ProcessedResult { private function __construct( - public readonly int $numberOfProcessedEvents, - public readonly Errors|null $errors, + public int $numberOfProcessedEvents, + public Errors|null $errors, ) { } @@ -25,8 +25,27 @@ public static function failed(int $numberOfProcessedEvents, Errors $errors): sel return new self($numberOfProcessedEvents, $errors); } - public function hasErrors(): bool + /** @phpstan-assert-if-true !null $this->errors */ + public function hasFailed(): bool { return $this->errors !== null; } + + public function throwOnFailure(): void + { + /** @var Error[] $errors */ + $errors = iterator_to_array($this->errors ?? []); + if ($errors === []) { + return; + } + $firstError = array_shift($errors); + + $additionalFailedSubscribers = array_map(fn (Error $error) => $error->subscriptionId->value, $errors); + + $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); + $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s.%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); + + // todo custom exception! + throw new \RuntimeException($exceptionMessage, 1732132930, $firstError->throwable); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php index d174c66e462..e61e077dc1e 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -37,7 +37,7 @@ public function __construct( public function findForUpdate(SubscriptionCriteria $criteria, \Closure $closure): mixed { return $this->subscriptionStore->transactional( - /** @return T */ + /** @return T */ function () use ($closure, $criteria): mixed { try { return $closure($this->subscriptionStore->findByCriteria($criteria)); diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 5905aff04f0..fa4747c913a 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -66,14 +66,14 @@ public function subscriptionsBootCommand(string $contentRepository = 'default', $bootResult = $subscriptionService->subscriptionEngine->boot(progressCallback: fn () => $this->output->progressAdvance()); $this->output->progressFinish(); $this->outputLine(); - if ($bootResult->errors === null) { + if ($bootResult->hasFailed() === false) { $this->outputLine('Done'); return; } } else { $bootResult = $subscriptionService->subscriptionEngine->boot(); } - if ($bootResult->errors !== null) { + if ($bootResult->hasFailed()) { $this->outputLine('Booting of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $bootResult->errors->count() === 1 ? '' : 's']); foreach ($bootResult->errors as $error) { $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); From e15ae79326ebfc31998b5628c7eb0c1748a37ac3 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:55:28 +0100 Subject: [PATCH 028/142] TASK: Fix of-by-one error in catchup Without advancing via `->next()` we will attempt to handle one event twice, and then run into the log: Which makes sense as the subscriber _was_ on that sequence number before and we are interested in the next events. > subscription "contentGraph" is farther than the current position (5053 >= 5053) Also simplifies `lowestPosition` --- .../Engine/SubscriptionEngine.php | 2 +- .../Classes/Subscription/Subscriptions.php | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index d3d0617d5d5..1bfcd8ec891 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -311,7 +311,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu foreach ($subscriptions as $subscription) { $this->subscribers->get($subscription->id)->handler->onBeforeCatchUp($subscription->status); } - $startSequenceNumber = $subscriptions->lowestPosition(); + $startSequenceNumber = $subscriptions->lowestPosition()?->next() ?? SequenceNumber::none(); $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); /** @var list $errors */ diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php index dc50137b32a..03a09abe0f1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -109,15 +109,18 @@ public function jsonSerialize(): iterable return array_values($this->subscriptionsById); } - public function lowestPosition(): SequenceNumber + public function lowestPosition(): SequenceNumber|null { - $min = null; - foreach ($this->subscriptionsById as $subscription) { - if ($min !== null && $subscription->position->value >= $min->value) { - continue; - } - $min = $subscription->position; + if ($this->subscriptionsById === []) { + return null; } - return $min ?? SequenceNumber::fromInteger(0); + return SequenceNumber::fromInteger( + min( + array_map( + fn (Subscription $subscription) => $subscription->position->value, + $this->subscriptionsById + ) + ) + ); } } From c7df8201ade4d95f77a82b776b072d8875757f31 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:59:56 +0100 Subject: [PATCH 029/142] TASK: Improve logging and simplify information on debug --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 1bfcd8ec891..9c1a8af8ff7 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -152,7 +152,7 @@ private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domai $this->subscriptionManager->update($subscription); return Error::fromSubscriptionIdAndException($subscription->id, $e); } - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', $subscriber->handler::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber->handler::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); $subscription->set( position: $eventEnvelope->sequenceNumber, retryAttempt: 0 @@ -320,11 +320,14 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu try { $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } if ($progressClosure !== null) { $progressClosure($eventEnvelope); } $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - $sequenceNumber = $eventEnvelope->sequenceNumber; foreach ($subscriptions as $subscription) { if ($subscription->status !== $subscriptionStatus) { continue; @@ -340,7 +343,6 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu $errors[] = $error; } $numberOfProcessedEvents++; - $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); } } finally { foreach ($subscriptions as $subscription) { @@ -370,7 +372,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); } } - $this->logger?->info('Subscription Engine: Finish catch up.'); + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); } ); From 0bd129a0769a18c2d9115d5e707fdc9367822355 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:17:45 +0100 Subject: [PATCH 030/142] TASK: Remove obsolete ContentRepositorySubscribersFactoryInterface --- ...tRepositorySubscribersFactoryInterface.php | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php deleted file mode 100644 index 90fa7021cf6..00000000000 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscribersFactoryInterface.php +++ /dev/null @@ -1,28 +0,0 @@ - Date: Wed, 20 Nov 2024 22:26:47 +0100 Subject: [PATCH 031/142] TASK: Remove generic `EventHandlerInterface` for now, everything is a projection (can be later reintroduced) --- .../Factory/ContentRepositoryFactory.php | 4 +-- .../ContentRepositorySubscriberFactories.php | 8 +++--- ...ntRepositorySubscriberFactoryInterface.php | 25 ------------------- .../Factory/ProjectionSubscriberFactory.php | 12 ++++++++- .../Projection/ProjectionEventHandler.php | 8 +----- .../Engine/SubscriptionEngine.php | 9 ++----- .../Subscriber/EventHandlerInterface.php | 24 ------------------ .../Subscription/Subscriber/Subscriber.php | 4 ++- .../Classes/ContentRepositoryRegistry.php | 4 +-- 9 files changed, 25 insertions(+), 73 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index bf80a6e6b91..4fffd90fec6 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -83,7 +83,7 @@ public function __construct( ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, private readonly CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, - private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, + private readonly ContentRepositorySubscriberFactories $additionalProjectionsFactories, LoggerInterface|null $logger = null, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); @@ -103,7 +103,7 @@ public function __construct( ); $subscribers = []; $additionalProjectionStates = []; - foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { + foreach ($this->additionalProjectionsFactories as $additionalSubscriberFactory) { $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); if ($subscriber->handler instanceof ProjectionEventHandler) { $additionalProjectionStates[] = $subscriber->handler->projection->getState(); diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php index fa16ebb2594..82a797e4745 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php @@ -5,23 +5,23 @@ namespace Neos\ContentRepository\Core\Factory; /** - * @implements \IteratorAggregate + * @implements \IteratorAggregate * @internal */ final class ContentRepositorySubscriberFactories implements \IteratorAggregate { /** - * @var array + * @var array */ private array $subscriberFactories; - private function __construct(ContentRepositorySubscriberFactoryInterface ...$subscriberFactories) + private function __construct(ProjectionSubscriberFactory ...$subscriberFactories) { $this->subscriberFactories = $subscriberFactories; } /** - * @param array $subscriberFactories + * @param array $subscriberFactories * @return self */ public static function fromArray(array $subscriberFactories): self diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php deleted file mode 100644 index 1102a9342c5..00000000000 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactoryInterface.php +++ /dev/null @@ -1,25 +0,0 @@ -> $projectionFactory diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php index a903620edfd..9ec8c924c73 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php @@ -6,14 +6,13 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; -use Neos\ContentRepository\Core\Subscription\Subscriber\EventHandlerInterface; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** * @internal */ -final readonly class ProjectionEventHandler implements EventHandlerInterface +final readonly class ProjectionEventHandler { /** * @param ProjectionInterface $projection @@ -41,11 +40,6 @@ public static function createWithCatchUpHook(ProjectionInterface $projection, Ca return new self($projection, $catchUpHook); } - public function setup(): void - { - $this->projection->setUp(); - } - public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { $this->catchUpHook?->onBeforeCatchUp($subscriptionStatus); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 9c1a8af8ff7..3c878f6111d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -6,7 +6,6 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; @@ -135,7 +134,7 @@ public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null) subscriptionStatus: $subscription->status, subscriptionPosition: $subscription->position, subscriptionError: $subscription->error, - projectionStatus: $subscriber->handler instanceof ProjectionEventHandler ? $subscriber->handler->projection->status() : null, + projectionStatus: $subscriber->handler->projection->status(), ); } return SubscriptionAndProjectionStatuses::fromArray($statuses); @@ -211,7 +210,7 @@ private function setupSubscription(Subscription $subscription, SequenceNumber $l { $subscriber = $this->subscribers->get($subscription->id); try { - $subscriber->handler->setup(); + $subscriber->handler->projection->setUp(); } catch (\Throwable $e) { $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); $subscription->fail($e); @@ -239,10 +238,6 @@ private function setupSubscription(Subscription $subscription, SequenceNumber $l private function resetSubscription(Subscription $subscription, bool $skipBooting): ?Error { $subscriber = $this->subscribers->get($subscription->id); - if (!$subscriber->handler instanceof ProjectionEventHandler) { - $this->logger?->info(sprintf('Subscription Engine: Subscriber handler "%s" for "%s" is no instance of %s, skipping reset', $subscriber->handler::class, $subscription->id->value, ProjectionEventHandler::class)); - return null; - } try { $subscriber->handler->projection->resetState(); } catch (\Throwable $e) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php deleted file mode 100644 index 17c062b989c..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/EventHandlerInterface.php +++ /dev/null @@ -1,24 +0,0 @@ -buildContentGraphProjectionFactory($contentRepositoryId, $contentRepositorySettings), $this->buildContentGraphCatchUpHookFactory($contentRepositoryId, $contentRepositorySettings), $this->buildCommandHooksFactory($contentRepositoryId, $contentRepositorySettings), - $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), + $this->buildAdditionalProjectionsFactories($contentRepositoryId, $contentRepositorySettings), $this->logger, ); } catch (\Exception $exception) { @@ -315,7 +315,7 @@ private function buildCommandHooksFactory(ContentRepositoryId $contentRepository } /** @param array $contentRepositorySettings */ - private function buildAdditionalSubscribersFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories + private function buildAdditionalProjectionsFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories { if (!is_array($contentRepositorySettings['projections'] ?? [])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); From 441b03589fbd4673045557956472a50558876326 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:38:17 +0100 Subject: [PATCH 032/142] TASK: Remove other generic subscription concepts not required for projections - retry strategy - runmode - pausing, finishing - InMemorySubscriptionStore --- .../Factory/ContentRepositoryFactory.php | 5 +- .../Factory/ProjectionSubscriberFactory.php | 2 - .../Engine/SubscriptionEngine.php | 67 ++----------------- .../RetryStrategy/ClockBasedRetryStrategy.php | 57 ---------------- .../RetryStrategy/NoRetryStrategy.php | 18 ----- .../RetryStrategy/RetryStrategy.php | 15 ----- .../Classes/Subscription/RunMode.php | 15 ----- .../Store/InMemorySubscriptionStore.php | 64 ------------------ .../Store/SubscriptionStoreInterface.php | 1 - .../Subscription/Subscriber/Subscriber.php | 3 - .../Classes/Subscription/Subscription.php | 3 - .../Subscription/SubscriptionStatus.php | 2 - .../Classes/Command/CrCommandController.php | 2 - .../DoctrineSubscriptionStore.php | 4 -- 14 files changed, 7 insertions(+), 251 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/RetryStrategy.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/RunMode.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Store/InMemorySubscriptionStore.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 4fffd90fec6..9273cc8b64d 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -37,8 +37,6 @@ use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; -use Neos\ContentRepository\Core\Subscription\RetryStrategy\NoRetryStrategy; -use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; @@ -113,7 +111,7 @@ public function __construct( $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); $subscribers[] = $this->buildContentGraphSubscriber(); - $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, new NoRetryStrategy(), $logger); + $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, $logger); } private function buildContentGraphSubscriber(): Subscriber @@ -121,7 +119,6 @@ private function buildContentGraphSubscriber(): Subscriber return new Subscriber( SubscriptionId::fromString('contentGraph'), SubscriptionGroup::fromString('default'), - RunMode::FROM_BEGINNING, ProjectionEventHandler::createWithCatchUpHook( $this->contentGraphProjection, $this->contentGraphCatchUpHookFactory->build(CatchUpHookFactoryDependencies::create( diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php index 6c16135df3f..34e5a733385 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -18,7 +18,6 @@ use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -44,7 +43,6 @@ public function build(SubscriberFactoryDependencies $dependencies): Subscriber return new Subscriber( $this->subscriptionId, SubscriptionGroup::fromString('projections'), - RunMode::FROM_BEGINNING, ProjectionEventHandler::create($this->projectionFactory->build($dependencies, $this->projectionFactoryOptions)), ); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 3c878f6111d..1942f1a0bcb 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -15,8 +15,6 @@ use Neos\EventStore\Model\EventEnvelope; use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Log\LoggerInterface; -use Neos\ContentRepository\Core\Subscription\RetryStrategy\RetryStrategy; -use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; @@ -37,7 +35,6 @@ public function __construct( private readonly SubscriptionStoreInterface $subscriptionStore, private readonly Subscribers $subscribers, private readonly EventNormalizer $eventNormalizer, - private readonly RetryStrategy $retryStrategy, private readonly LoggerInterface|null $logger = null, ) { $this->subscriptionManager = new SubscriptionManager($this->subscriptionStore); @@ -52,7 +49,6 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $sk $this->subscriptionStore->setup(); $this->discoverNewSubscriptions(); $this->retrySubscriptions($criteria); - $lastSequenceNumber = $this->lastSequenceNumber(); $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::NEW)); if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); @@ -60,7 +56,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $sk } $errors = []; foreach ($subscriptions as $subscription) { - $error = $this->setupSubscription($subscription, $lastSequenceNumber, $skipBooting); + $error = $this->setupSubscription($subscription, $skipBooting); if ($error !== null) { $errors[] = $error; } @@ -100,30 +96,6 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null, bool $sk return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } - public function teardown(SubscriptionEngineCriteria|null $criteria = null): Result - { - // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L470) - return Result::success(); - } - - public function remove(SubscriptionEngineCriteria|null $criteria = null): Result - { - // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L562) - return Result::success(); - } - - public function reactivate(SubscriptionEngineCriteria|null $criteria = null): Result - { - // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L648) - return Result::success(); - } - - public function pause(SubscriptionEngineCriteria|null $criteria = null): Result - { - // TODO implement (see https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/Engine/DefaultSubscriptionEngine.php#L712) - return Result::success(); - } - public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null): SubscriptionAndProjectionStatuses { $statuses = []; @@ -187,7 +159,7 @@ private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $crite $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create( $criteria->ids, $criteria->groups, - SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE, SubscriptionStatus::PAUSED, SubscriptionStatus::FINISHED]), + SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), )); foreach ($registeredSubscriptions as $subscription) { if ($this->subscribers->contain($subscription->id)) { @@ -206,7 +178,7 @@ private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $crite * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned */ - private function setupSubscription(Subscription $subscription, SequenceNumber $lastSequenceNumber, bool $skipBooting): ?Error + private function setupSubscription(Subscription $subscription, bool $skipBooting): ?Error { $subscriber = $this->subscribers->get($subscription->id); try { @@ -217,16 +189,9 @@ private function setupSubscription(Subscription $subscription, SequenceNumber $l $this->subscriptionManager->update($subscription); return Error::fromSubscriptionIdAndException($subscription->id, $e); } - if ($subscription->runMode === RunMode::FROM_NOW) { - $subscription->set( - status: SubscriptionStatus::ACTIVE, - position: $lastSequenceNumber, - ); - } else { - $subscription->set( - status: $skipBooting ? SubscriptionStatus::ACTIVE : SubscriptionStatus::BOOTING - ); - } + $subscription->set( + status: $skipBooting ? SubscriptionStatus::ACTIVE : SubscriptionStatus::BOOTING + ); $this->subscriptionManager->update($subscription); $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the setup method has been executed, set to %s.', $subscriber::class, $subscription->id->value, $subscription->status->value)); return null; @@ -274,9 +239,6 @@ private function retrySubscription(Subscription $subscription): void if (!$retryable) { return; } - if (!$this->retryStrategy->shouldRetry($subscription)) { - return; - } $subscription->set( status: $subscription->error->previousStatus, retryAttempt: $subscription->retryAttempt + 1, @@ -350,15 +312,6 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu continue; } - if ($subscription->runMode === RunMode::ONCE) { - $subscription->set( - status: SubscriptionStatus::FINISHED, - ); - $this->subscriptionManager->update($subscription); - - $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" run only once and has been set to finished.', $subscription->id->value)); - continue; - } if ($subscription->status !== SubscriptionStatus::ACTIVE) { $subscription->set( status: SubscriptionStatus::ACTIVE, @@ -373,14 +326,6 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu ); } - private function lastSequenceNumber(): SequenceNumber - { - foreach ($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1) as $eventEnvelope) { - return $eventEnvelope->sequenceNumber; - } - return SequenceNumber::fromInteger(0); - } - /** * @template T * @param \Closure(): T $closure diff --git a/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php b/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php deleted file mode 100644 index 674e50cacf7..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/ClockBasedRetryStrategy.php +++ /dev/null @@ -1,57 +0,0 @@ -retryAttempt >= $this->maxAttempts) { - return false; - } - - $lastSavedAt = $subscription->lastSavedAt; - - if ($lastSavedAt === null) { - return false; - } - - $nextRetryDate = $this->calculateNextRetryDate($lastSavedAt, $subscription->retryAttempt); - - return $nextRetryDate <= $this->clock->now(); - } - - private function calculateNextRetryDate(\DateTimeImmutable $lastDate, int $attempt): \DateTimeImmutable - { - return $lastDate->modify(sprintf('+%d seconds', $this->calculateDelay($attempt))); - } - - private function calculateDelay(int $attempt): int - { - return (int)round($this->baseDelay * ($this->delayFactor ** $attempt)); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php b/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php deleted file mode 100644 index eccb35fb2f2..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/RetryStrategy/NoRetryStrategy.php +++ /dev/null @@ -1,18 +0,0 @@ -subscriptions = Subscriptions::none(); - } - - public function setup(): void - { - // no setup required - } - - public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions - { - return $this->subscriptions->filter(function (Subscription $subscription) use ($criteria) { - if ($criteria->ids !== null && !$criteria->ids->contain($subscription->id)) { - return false; - } - if ($criteria->groups !== null && !$criteria->groups->contain($subscription->group)) { - return false; - } - if (!$criteria->status->matches($subscription->status)) { - return false; - } - return true; - }); - } - - public function add(Subscription $subscription): void - { - $this->subscriptions = $this->subscriptions->with($subscription); - } - - public function update(Subscription $subscription): void - { - $this->subscriptions = $this->subscriptions->with($subscription); - } - - public function remove(Subscription $subscription): void - { - $this->subscriptions = $this->subscriptions->without($subscription->id); - } - - public function transactional(\Closure $closure): mixed - { - // In memory store does not support transaction boundaries - return $closure(); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index d1a1e3d041d..568294ced6c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -5,7 +5,6 @@ namespace Neos\ContentRepository\Core\Subscription\Store; use Neos\ContentRepository\Core\Subscription\Subscription; -use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\Subscriptions; /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php index e158f7cc0c0..6ff8369e9af 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php @@ -4,9 +4,7 @@ namespace Neos\ContentRepository\Core\Subscription\Subscriber; -use Neos\ContentRepository\Core\Projection\EventHandlerInterface; use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; -use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -18,7 +16,6 @@ final class Subscriber public function __construct( public readonly SubscriptionId $id, public readonly SubscriptionGroup $group, - public readonly RunMode $runMode, public readonly ProjectionEventHandler $handler, ) { } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 5cda857fce0..0d1e909a49e 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -18,7 +18,6 @@ final class Subscription public function __construct( public readonly SubscriptionId $id, public readonly SubscriptionGroup $group, - public readonly RunMode $runMode, public SubscriptionStatus $status, public SequenceNumber $position, public SubscriptionError|null $error = null, @@ -35,7 +34,6 @@ public static function createFromSubscriber(Subscriber $subscriber): self return new self( $subscriber->id, $subscriber->group, - $subscriber->runMode, SubscriptionStatus::NEW, SequenceNumber::fromInteger(0), ); @@ -55,7 +53,6 @@ public function set( return new self( $this->id, $this->group, - $this->runMode, $status ?? $this->status, $position ?? $this->position, $this->error, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php index 9ccb916cb70..f1b29c7e572 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -12,8 +12,6 @@ enum SubscriptionStatus : string case NEW = 'NEW'; case BOOTING = 'BOOTING'; case ACTIVE = 'ACTIVE'; - case PAUSED = 'PAUSED'; - case FINISHED = 'FINISHED'; case DETACHED = 'DETACHED'; case ERROR = 'ERROR'; } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index fa4747c913a..0e8ee668a87 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -155,8 +155,6 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo SubscriptionStatus::NEW => 'NEW', SubscriptionStatus::BOOTING => 'BOOTING', SubscriptionStatus::ACTIVE => 'ACTIVE', - SubscriptionStatus::PAUSED => 'PAUSED', - SubscriptionStatus::FINISHED => 'FINISHED', SubscriptionStatus::DETACHED => 'DETACHED', SubscriptionStatus::ERROR => 'ERROR', }); diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 525f0f6fbc7..cb60dba506e 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -46,7 +46,6 @@ public function setup(): void $tableSchema = new Table($this->tableName, [ (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('group_name', Type::getType(Types::STRING)))->setNotnull(true)->setLength(100)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), - (new Column('run_mode', Type::getType(Types::STRING)))->setNotnull(true)->setLength(16)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), @@ -151,7 +150,6 @@ private static function toDatabase(Subscription $subscription): array { return [ 'group_name' => $subscription->group->value, - 'run_mode' => $subscription->runMode->name, 'status' => $subscription->status->name, 'position' => $subscription->position->value, 'error_message' => $subscription->error?->errorMessage, @@ -176,7 +174,6 @@ private static function fromDatabase(array $row): Subscription } assert(is_string($row['id'])); assert(is_string($row['group_name'])); - assert(is_string($row['run_mode'])); assert(is_string($row['status'])); assert(is_int($row['position'])); assert(is_int($row['retry_attempt'])); @@ -187,7 +184,6 @@ private static function fromDatabase(array $row): Subscription return new Subscription( SubscriptionId::fromString($row['id']), SubscriptionGroup::fromString($row['group_name']), - RunMode::from($row['run_mode']), SubscriptionStatus::from($row['status']), SequenceNumber::fromInteger($row['position']), $subscriptionError, From 9c6ba75e3b25666d356c6503599452f3653c542b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:27:28 +0100 Subject: [PATCH 033/142] TASK: Remove subscription groups and filtering except for status and ids Followup for remove other generic subscription concepts not required for projections --- .../Factory/ContentRepositoryFactory.php | 2 - .../Factory/ProjectionSubscriberFactory.php | 2 - .../Engine/SubscriptionEngine.php | 3 +- .../Engine/SubscriptionEngineCriteria.php | 17 +--- .../Store/SubscriptionCriteria.php | 12 +-- .../Subscription/Subscriber/Subscriber.php | 2 - .../Classes/Subscription/Subscription.php | 3 - .../Subscription/SubscriptionGroup.php | 28 ------- .../Subscription/SubscriptionGroups.php | 77 ------------------- .../Subscription/SubscriptionStatusFilter.php | 5 -- .../DoctrineSubscriptionStore.php | 15 ---- 11 files changed, 6 insertions(+), 160 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 9273cc8b64d..a2030f9cafb 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -40,7 +40,6 @@ use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; -use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\Factory\AuthProvider\AuthProviderFactoryInterface; use Neos\EventStore\EventStoreInterface; @@ -118,7 +117,6 @@ private function buildContentGraphSubscriber(): Subscriber { return new Subscriber( SubscriptionId::fromString('contentGraph'), - SubscriptionGroup::fromString('default'), ProjectionEventHandler::createWithCatchUpHook( $this->contentGraphProjection, $this->contentGraphCatchUpHookFactory->build(CatchUpHookFactoryDependencies::create( diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php index 34e5a733385..14d22fb6529 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -19,7 +19,6 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; -use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; use Neos\ContentRepository\Core\Subscription\SubscriptionId; /** @@ -42,7 +41,6 @@ public function build(SubscriberFactoryDependencies $dependencies): Subscriber { return new Subscriber( $this->subscriptionId, - SubscriptionGroup::fromString('projections'), ProjectionEventHandler::create($this->projectionFactory->build($dependencies, $this->projectionFactoryOptions)), ); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 1942f1a0bcb..77b24af3abd 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -158,7 +158,6 @@ private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $crite { $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create( $criteria->ids, - $criteria->groups, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), )); foreach ($registeredSubscriptions as $subscription) { @@ -221,7 +220,7 @@ private function resetSubscription(Subscription $subscription, bool $skipBooting private function retrySubscriptions(SubscriptionEngineCriteria $criteria): void { $this->subscriptionManager->findForUpdate( - SubscriptionCriteria::create($criteria->ids, $criteria->groups, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR])), + SubscriptionCriteria::create($criteria->ids, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR])), fn (Subscriptions $subscriptions) => $subscriptions->map($this->retrySubscription(...)), ); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php index 9eceb588f7d..3cb3946aa17 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php @@ -4,7 +4,6 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; -use Neos\ContentRepository\Core\Subscription\SubscriptionGroups; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionIds; @@ -14,36 +13,28 @@ final class SubscriptionEngineCriteria { private function __construct( - public readonly SubscriptionIds|null $ids, - public readonly SubscriptionGroups|null $groups, + public readonly SubscriptionIds|null $ids ) { } /** * @param SubscriptionIds|array|null $ids - * @param SubscriptionGroups|list|null $groups */ public static function create( - SubscriptionIds|array $ids = null, - SubscriptionGroups|array $groups = null, + SubscriptionIds|array $ids = null ): self { if (is_array($ids)) { $ids = SubscriptionIds::fromArray($ids); } - if (is_array($groups)) { - $groups = SubscriptionGroups::fromArray($groups); - } return new self( - $ids, - $groups, + $ids ); } public static function noConstraints(): self { return new self( - ids: null, - groups: null, + ids: null ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php index b33c656e86f..8eca4a8ed79 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php @@ -5,10 +5,9 @@ namespace Neos\ContentRepository\Core\Subscription\Store; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionGroups; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionIds; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; /** @@ -18,30 +17,23 @@ { private function __construct( public SubscriptionIds|null $ids, - public SubscriptionGroups|null $groups, public SubscriptionStatusFilter $status, ) { } /** * @param SubscriptionIds|array|null $ids - * @param SubscriptionGroups|list|null $groups * @param SubscriptionStatusFilter|null $status */ public static function create( SubscriptionIds|array $ids = null, - SubscriptionGroups|array $groups = null, SubscriptionStatusFilter $status = null, ): self { if (is_array($ids)) { $ids = SubscriptionIds::fromArray($ids); } - if (is_array($groups)) { - $groups = SubscriptionGroups::fromArray($groups); - } return new self( $ids, - $groups, $status ?? SubscriptionStatusFilter::any(), ); } @@ -55,7 +47,6 @@ public static function forEngineCriteriaAndStatus( } return new self( $criteria->ids, - $criteria->groups, $status, ); } @@ -64,7 +55,6 @@ public static function noConstraints(): self { return new self( ids: null, - groups: null, status: SubscriptionStatusFilter::any(), ); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php index 6ff8369e9af..01b9f4a85f2 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php @@ -5,7 +5,6 @@ namespace Neos\ContentRepository\Core\Subscription\Subscriber; use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; -use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; use Neos\ContentRepository\Core\Subscription\SubscriptionId; /** @@ -15,7 +14,6 @@ final class Subscriber { public function __construct( public readonly SubscriptionId $id, - public readonly SubscriptionGroup $group, public readonly ProjectionEventHandler $handler, ) { } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 0d1e909a49e..bcf37fdaa11 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -17,7 +17,6 @@ final class Subscription { public function __construct( public readonly SubscriptionId $id, - public readonly SubscriptionGroup $group, public SubscriptionStatus $status, public SequenceNumber $position, public SubscriptionError|null $error = null, @@ -33,7 +32,6 @@ public static function createFromSubscriber(Subscriber $subscriber): self { return new self( $subscriber->id, - $subscriber->group, SubscriptionStatus::NEW, SequenceNumber::fromInteger(0), ); @@ -52,7 +50,6 @@ public function set( $this->retryAttempt = $retryAttempt ?? $this->retryAttempt; return new self( $this->id, - $this->group, $status ?? $this->status, $position ?? $this->position, $this->error, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php deleted file mode 100644 index 3f0eb880cf4..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroup.php +++ /dev/null @@ -1,28 +0,0 @@ -value === $this->value; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php deleted file mode 100644 index 69f9c32ed69..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionGroups.php +++ /dev/null @@ -1,77 +0,0 @@ - - * @api - */ -final class SubscriptionGroups implements \IteratorAggregate, \Countable, \JsonSerializable -{ - /** - * @param array $groupsByValue - */ - private function __construct( - private readonly array $groupsByValue - ) { - } - - /** - * @param list $groups - */ - public static function fromArray(array $groups): self - { - $groupsByValue = []; - foreach ($groups as $group) { - if (is_string($group)) { - $group = SubscriptionGroup::fromString($group); - } - if (!$group instanceof SubscriptionGroup) { - throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionGroup::class, get_debug_type($group)), 1731580587); - } - if (array_key_exists($group->value, $groupsByValue)) { - throw new \InvalidArgumentException(sprintf('Group "%s" is already part of this set', $group->value), 1731580633); - } - $groupsByValue[$group->value] = $group; - } - return new self($groupsByValue); - } - - public static function none(): self - { - return new self([]); - } - - public function getIterator(): \Traversable - { - yield from array_values($this->groupsByValue); - } - - public function count(): int - { - return count($this->groupsByValue); - } - - public function contain(SubscriptionGroup $group): bool - { - return array_key_exists($group->value, $this->groupsByValue); - } - - /** - * @return list - */ - public function toStringArray(): array - { - return array_values(array_map(static fn (SubscriptionGroup $group) => $group->value, $this->groupsByValue)); - } - - /** - * @return iterable - */ - public function jsonSerialize(): iterable - { - return array_values($this->groupsByValue); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php index 5753bd6c266..adc3fa4e51b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php @@ -61,9 +61,4 @@ public function toStringArray(): array { return array_values(array_map(static fn (SubscriptionStatus $id) => $id->value, $this->statusByValue)); } - - public function matches(SubscriptionStatus $status): bool - { - return array_key_exists($status->value, $this->statusByValue); - } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index cb60dba506e..11cc0a92bb2 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -14,12 +14,10 @@ use Doctrine\DBAL\Types\Type; use Doctrine\DBAL\Types\Types; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Subscription\RunMode; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\SubscriptionError; -use Neos\ContentRepository\Core\Subscription\SubscriptionGroup; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\Subscriptions; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -45,7 +43,6 @@ public function setup(): void $isSqlite = $this->dbal->getDatabasePlatform() instanceof SqlitePlatform; $tableSchema = new Table($this->tableName, [ (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), - (new Column('group_name', Type::getType(Types::STRING)))->setNotnull(true)->setLength(100)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), @@ -55,7 +52,6 @@ public function setup(): void (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), ]); $tableSchema->setPrimaryKey(['id']); - $tableSchema->addIndex(['group_name']); $tableSchema->addIndex(['status']); $schema = new Schema( [$tableSchema], @@ -84,14 +80,6 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions Connection::PARAM_STR_ARRAY, ); } - if ($criteria->groups !== null) { - $queryBuilder->andWhere('group_name IN (:groups)') - ->setParameter( - 'groups', - $criteria->groups->toStringArray(), - Connection::PARAM_STR_ARRAY, - ); - } if (!$criteria->status->isEmpty()) { $queryBuilder->andWhere('status IN (:status)') ->setParameter( @@ -149,7 +137,6 @@ public function remove(Subscription $subscription): void private static function toDatabase(Subscription $subscription): array { return [ - 'group_name' => $subscription->group->value, 'status' => $subscription->status->name, 'position' => $subscription->position->value, 'error_message' => $subscription->error?->errorMessage, @@ -173,7 +160,6 @@ private static function fromDatabase(array $row): Subscription $subscriptionError = null; } assert(is_string($row['id'])); - assert(is_string($row['group_name'])); assert(is_string($row['status'])); assert(is_int($row['position'])); assert(is_int($row['retry_attempt'])); @@ -183,7 +169,6 @@ private static function fromDatabase(array $row): Subscription return new Subscription( SubscriptionId::fromString($row['id']), - SubscriptionGroup::fromString($row['group_name']), SubscriptionStatus::from($row['status']), SequenceNumber::fromInteger($row['position']), $subscriptionError, From 0ac8751e843a5a88103bbac5a5615238374a083d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 23:08:05 +0100 Subject: [PATCH 034/142] TASK: Remove `$skipBooting` because its an odd signature and unused --- .../Subscription/Engine/SubscriptionEngine.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 77b24af3abd..7641f24a07b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -40,7 +40,7 @@ public function __construct( $this->subscriptionManager = new SubscriptionManager($this->subscriptionStore); } - public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $skipBooting = false): Result + public function setup(SubscriptionEngineCriteria|null $criteria = null): Result { $criteria ??= SubscriptionEngineCriteria::noConstraints(); @@ -56,7 +56,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null, bool $sk } $errors = []; foreach ($subscriptions as $subscription) { - $error = $this->setupSubscription($subscription, $skipBooting); + $error = $this->setupSubscription($subscription); if ($error !== null) { $errors[] = $error; } @@ -75,7 +75,7 @@ public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE, $progressCallback)); } - public function reset(SubscriptionEngineCriteria|null $criteria = null, bool $skipBooting = false): Result + public function reset(SubscriptionEngineCriteria|null $criteria = null): Result { $criteria ??= SubscriptionEngineCriteria::noConstraints(); @@ -87,7 +87,7 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null, bool $sk } $errors = []; foreach ($subscriptions as $subscription) { - $error = $this->resetSubscription($subscription, $skipBooting); + $error = $this->resetSubscription($subscription); if ($error !== null) { $errors[] = $error; } @@ -177,7 +177,7 @@ private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $crite * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned */ - private function setupSubscription(Subscription $subscription, bool $skipBooting): ?Error + private function setupSubscription(Subscription $subscription): ?Error { $subscriber = $this->subscribers->get($subscription->id); try { @@ -189,7 +189,7 @@ private function setupSubscription(Subscription $subscription, bool $skipBooting return Error::fromSubscriptionIdAndException($subscription->id, $e); } $subscription->set( - status: $skipBooting ? SubscriptionStatus::ACTIVE : SubscriptionStatus::BOOTING + status: SubscriptionStatus::BOOTING ); $this->subscriptionManager->update($subscription); $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the setup method has been executed, set to %s.', $subscriber::class, $subscription->id->value, $subscription->status->value)); @@ -199,7 +199,7 @@ private function setupSubscription(Subscription $subscription, bool $skipBooting /** * TODO */ - private function resetSubscription(Subscription $subscription, bool $skipBooting): ?Error + private function resetSubscription(Subscription $subscription): ?Error { $subscriber = $this->subscribers->get($subscription->id); try { @@ -209,7 +209,7 @@ private function resetSubscription(Subscription $subscription, bool $skipBooting return Error::fromSubscriptionIdAndException($subscription->id, $e); } $subscription->set( - status: $skipBooting ? SubscriptionStatus::ACTIVE : SubscriptionStatus::BOOTING, + status: SubscriptionStatus::BOOTING, position: SequenceNumber::none(), ); $this->subscriptionManager->update($subscription); From 9be33089b9c3399694600ac7b02443c2139a09d4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 23:11:11 +0100 Subject: [PATCH 035/142] TASK: Remove removal of subscriptions Detached subscriptions cannot be removed for now --- .../Engine/SubscriptionManager.php | 26 ------------------- .../Store/SubscriptionStoreInterface.php | 2 -- .../DoctrineSubscriptionStore.php | 10 ------- 3 files changed, 38 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php index e61e077dc1e..817065843f0 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -18,15 +18,11 @@ final class SubscriptionManager /** @var \SplObjectStorage */ private \SplObjectStorage $forUpdate; - /** @var \SplObjectStorage */ - private \SplObjectStorage $forRemove; - public function __construct( private readonly SubscriptionStoreInterface $subscriptionStore, ) { $this->forAdd = new \SplObjectStorage(); $this->forUpdate = new \SplObjectStorage(); - $this->forRemove = new \SplObjectStorage(); } /** @@ -63,18 +59,9 @@ public function update(Subscription $subscription): void $this->forUpdate->attach($subscription); } - public function remove(Subscription $subscription): void - { - $this->forRemove->attach($subscription); - } - public function flush(): void { foreach ($this->forAdd as $subscription) { - if ($this->forRemove->contains($subscription)) { - continue; - } - $this->subscriptionStore->add($subscription); } @@ -83,23 +70,10 @@ public function flush(): void continue; } - if ($this->forRemove->contains($subscription)) { - continue; - } - $this->subscriptionStore->update($subscription); } - foreach ($this->forRemove as $subscription) { - if ($this->forAdd->contains($subscription)) { - continue; - } - - $this->subscriptionStore->remove($subscription); - } - $this->forAdd = new \SplObjectStorage(); $this->forUpdate = new \SplObjectStorage(); - $this->forRemove = new \SplObjectStorage(); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 568294ced6c..83aab8a185c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -20,8 +20,6 @@ public function add(Subscription $subscription): void; public function update(Subscription $subscription): void; - public function remove(Subscription $subscription): void; - /** * @template T * @param \Closure():T $closure diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 11cc0a92bb2..a0a2ecb72e0 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -121,16 +121,6 @@ public function update(Subscription $subscription): void ); } - public function remove(Subscription $subscription): void - { - $this->dbal->delete( - $this->tableName, - [ - 'id' => $subscription->id->value, - ] - ); - } - /** * @return array */ From 53790b9c7e47c772f39ac2258f9fb41ce3ecd61f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 20 Nov 2024 23:12:38 +0100 Subject: [PATCH 036/142] TASK: Remove sqlite support for `DoctrineSubscriptionStore` as db locking is untested there --- .../SubscriptionStore/DoctrineSubscriptionStore.php | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index a0a2ecb72e0..6841156f8d7 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -6,7 +6,6 @@ use DateTimeImmutable; use Doctrine\DBAL\Connection; -use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\Column; use Doctrine\DBAL\Schema\Schema; @@ -40,13 +39,12 @@ public function setup(): void $schemaConfig->setDefaultTableOptions([ 'charset' => 'utf8mb4' ]); - $isSqlite = $this->dbal->getDatabasePlatform() instanceof SqlitePlatform; $tableSchema = new Table($this->tableName, [ - (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), - (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), - (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', $isSqlite ? null : 'ascii_general_ci'), + (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), (new Column('error_trace', Type::getType(Types::TEXT)))->setNotnull(false), (new Column('retry_attempt', Type::getType(Types::INTEGER)))->setNotnull(true), (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), @@ -69,9 +67,7 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions ->select('*') ->from($this->tableName) ->orderBy('id'); - if (!$this->dbal->getDatabasePlatform() instanceof SQLitePlatform) { - $queryBuilder->forUpdate(); - } + $queryBuilder->forUpdate(); if ($criteria->ids !== null) { $queryBuilder->andWhere('id IN (:ids)') ->setParameter( From 481f17327cf1cb0cb24de49d082e76c381334f04 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:32:41 +0100 Subject: [PATCH 037/142] TASK: Inline `ProjectionEventHandler` to `Subscriber` and make it a `SubscriberForProjection` Simplification to make the projection use case first level See https://github.com/neos/neos-development-collection/pull/5375#issuecomment-2491375655 --- .../Factory/ContentRepositoryFactory.php | 29 +++++++--------- .../Factory/ProjectionSubscriberFactory.php | 10 +++--- .../Engine/SubscriptionEngine.php | 18 +++++----- .../Subscriber/ProjectionSubscriber.php} | 33 ++++++------------- .../Subscription/Subscriber/Subscriber.php | 20 ----------- .../Subscription/Subscriber/Subscribers.php | 14 ++++---- .../Classes/Subscription/Subscription.php | 4 +-- 7 files changed, 45 insertions(+), 83 deletions(-) rename Neos.ContentRepository.Core/Classes/{Projection/ProjectionEventHandler.php => Subscription/Subscriber/ProjectionSubscriber.php} (55%) delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index a2030f9cafb..add804950ec 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -33,12 +33,11 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; -use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; -use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; +use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionSubscriber; use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\Factory\AuthProvider\AuthProviderFactoryInterface; @@ -102,9 +101,7 @@ public function __construct( $additionalProjectionStates = []; foreach ($this->additionalProjectionsFactories as $additionalSubscriberFactory) { $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); - if ($subscriber->handler instanceof ProjectionEventHandler) { - $additionalProjectionStates[] = $subscriber->handler->projection->getState(); - } + $additionalProjectionStates[] = $subscriber->projection->getState(); $subscribers[] = $subscriber; } $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); @@ -113,20 +110,18 @@ public function __construct( $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, $logger); } - private function buildContentGraphSubscriber(): Subscriber + private function buildContentGraphSubscriber(): ProjectionSubscriber { - return new Subscriber( + return new ProjectionSubscriber( SubscriptionId::fromString('contentGraph'), - ProjectionEventHandler::createWithCatchUpHook( - $this->contentGraphProjection, - $this->contentGraphCatchUpHookFactory->build(CatchUpHookFactoryDependencies::create( - $this->contentRepositoryId, - $this->contentGraphProjection->getState(), - $this->subscriberFactoryDependencies->nodeTypeManager, - $this->subscriberFactoryDependencies->contentDimensionSource, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, - )), - ), + $this->contentGraphProjection, + $this->contentGraphCatchUpHookFactory->build(CatchUpHookFactoryDependencies::create( + $this->contentRepositoryId, + $this->contentGraphProjection->getState(), + $this->subscriberFactoryDependencies->nodeTypeManager, + $this->subscriberFactoryDependencies->contentDimensionSource, + $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + )), ); } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php index 14d22fb6529..f763dfb9328 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -14,11 +14,10 @@ namespace Neos\ContentRepository\Core\Factory; -use Neos\ContentRepository\Core\Projection\ProjectionEventHandler; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; +use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionSubscriber; use Neos\ContentRepository\Core\Subscription\SubscriptionId; /** @@ -37,11 +36,12 @@ public function __construct( ) { } - public function build(SubscriberFactoryDependencies $dependencies): Subscriber + public function build(SubscriberFactoryDependencies $dependencies): ProjectionSubscriber { - return new Subscriber( + return new ProjectionSubscriber( $this->subscriptionId, - ProjectionEventHandler::create($this->projectionFactory->build($dependencies, $this->projectionFactoryOptions)), + $this->projectionFactory->build($dependencies, $this->projectionFactoryOptions), + null ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 7641f24a07b..e2171e51f1e 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -106,7 +106,7 @@ public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null) subscriptionStatus: $subscription->status, subscriptionPosition: $subscription->position, subscriptionError: $subscription->error, - projectionStatus: $subscriber->handler->projection->status(), + projectionStatus: $subscriber->projection->status(), ); } return SubscriptionAndProjectionStatuses::fromArray($statuses); @@ -116,14 +116,14 @@ private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domai { $subscriber = $this->subscribers->get($subscription->id); try { - $subscriber->handler->handle($domainEvent, $eventEnvelope); + $subscriber->handle($domainEvent, $eventEnvelope); } catch (\Throwable $e) { $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); $subscription->fail($e); $this->subscriptionManager->update($subscription); return Error::fromSubscriptionIdAndException($subscription->id, $e); } - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber->handler::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); $subscription->set( position: $eventEnvelope->sequenceNumber, retryAttempt: 0 @@ -181,7 +181,7 @@ private function setupSubscription(Subscription $subscription): ?Error { $subscriber = $this->subscribers->get($subscription->id); try { - $subscriber->handler->projection->setUp(); + $subscriber->projection->setUp(); } catch (\Throwable $e) { $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); $subscription->fail($e); @@ -203,9 +203,9 @@ private function resetSubscription(Subscription $subscription): ?Error { $subscriber = $this->subscribers->get($subscription->id); try { - $subscriber->handler->projection->resetState(); + $subscriber->projection->resetState(); } catch (\Throwable $e) { - $this->logger?->error(sprintf('Subscription Engine: Subscriber handler "%s" for "%s" has an error in the resetState method: %s', $subscriber->handler::class, $subscription->id->value, $e->getMessage())); + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); return Error::fromSubscriptionIdAndException($subscription->id, $e); } $subscription->set( @@ -213,7 +213,7 @@ private function resetSubscription(Subscription $subscription): ?Error position: SequenceNumber::none(), ); $this->subscriptionManager->update($subscription); - $this->logger?->debug(sprintf('Subscription Engine: For Subscriber handler "%s" for "%s" the resetState method has been executed.', $subscriber->handler::class, $subscription->id->value)); + $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the resetState method has been executed.', $subscriber::class, $subscription->id->value)); return null; } @@ -265,7 +265,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu return ProcessedResult::success(0); } foreach ($subscriptions as $subscription) { - $this->subscribers->get($subscription->id)->handler->onBeforeCatchUp($subscription->status); + $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); } $startSequenceNumber = $subscriptions->lowestPosition()?->next() ?? SequenceNumber::none(); $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); @@ -306,7 +306,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu } } foreach ($subscriptions as $subscription) { - $this->subscribers->get($subscription->id)->handler->onAfterCatchUp(); + $this->subscribers->get($subscription->id)->onAfterCatchUp(); if ($subscription->status !== $subscriptionStatus) { continue; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php similarity index 55% rename from Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php rename to Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php index 9ec8c924c73..03d80cce95a 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionEventHandler.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php @@ -2,44 +2,31 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Subscription\Subscriber; use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** - * @internal + * @api */ -final readonly class ProjectionEventHandler +final class ProjectionSubscriber { /** * @param ProjectionInterface $projection */ - private function __construct( - public ProjectionInterface $projection, - private CatchUpHookInterface|null $catchUpHook, + public function __construct( + public readonly SubscriptionId $id, + public readonly ProjectionInterface $projection, + private readonly ?CatchUpHookInterface $catchUpHook ) { } - /** - * @param ProjectionInterface $projection - * @return self - */ - public static function create(ProjectionInterface $projection): self - { - return new self($projection, null); - } - - /** - * @param ProjectionInterface $projection - */ - public static function createWithCatchUpHook(ProjectionInterface $projection, CatchUpHookInterface $catchUpHook): self - { - return new self($projection, $catchUpHook); - } - public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { $this->catchUpHook?->onBeforeCatchUp($subscriptionStatus); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php deleted file mode 100644 index 01b9f4a85f2..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscriber.php +++ /dev/null @@ -1,20 +0,0 @@ - + * @implements \IteratorAggregate * @api */ final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable { /** - * @param array $subscribersById + * @param array $subscribersById */ private function __construct( private readonly array $subscribersById @@ -21,14 +21,14 @@ private function __construct( } /** - * @param array $subscribers + * @param array $subscribers */ public static function fromArray(array $subscribers): self { $subscribersById = []; foreach ($subscribers as $subscriber) { - if (!$subscriber instanceof Subscriber) { - throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscriber::class, get_debug_type($subscriber)), 1721731490); + if (!$subscriber instanceof ProjectionSubscriber) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionSubscriber::class, get_debug_type($subscriber)), 1721731490); } if (array_key_exists($subscriber->id->value, $subscribersById)) { throw new \InvalidArgumentException(sprintf('Subscriber with id "%s" is already part of this set', $subscriber->id->value), 1721731494); @@ -43,12 +43,12 @@ public static function none(): self return self::fromArray([]); } - public function with(Subscriber $subscriber): self + public function with(ProjectionSubscriber $subscriber): self { return new self([...$this->subscribersById, $subscriber->id->value => $subscriber]); } - public function get(SubscriptionId $id): Subscriber + public function get(SubscriptionId $id): ProjectionSubscriber { if (!$this->contain($id)) { throw new \InvalidArgumentException(sprintf('Subscriber with the subscription id "%s" not found.', $id->value), 1721731490); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index bcf37fdaa11..ed3ffc13cad 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; -use Neos\ContentRepository\Core\Subscription\Subscriber\Subscriber; +use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionSubscriber; use Neos\EventStore\Model\Event\SequenceNumber; /** @@ -28,7 +28,7 @@ public function __construct( /** * @internal Only the {@see SubscriptionEngine} is supposed to instantiate subscriptions */ - public static function createFromSubscriber(Subscriber $subscriber): self + public static function createFromSubscriber(ProjectionSubscriber $subscriber): self { return new self( $subscriber->id, From b9569c572059ba70a32fd805aef268f9cd3801c7 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:37:31 +0100 Subject: [PATCH 038/142] TASK: Introduce tests for subscription booting, active and error state --- .../Configuration/Testing/Settings.yaml | 24 + .../Functional/SubscriptionEngineTest.php | 480 ++++++++++++++++++ .../Classes/Subscription/Engine/Errors.php | 9 +- .../Subscription/Engine/ProcessedResult.php | 4 +- .../Classes/Fakes/FakeProjectionFactory.php | 26 + 5 files changed, 540 insertions(+), 3 deletions(-) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php create mode 100644 Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index 16f67463bde..9e6b4e2cb8a 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -38,6 +38,30 @@ Neos: factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory catchUpHooks: {} + t_subscription: + eventStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory + nodeTypeManager: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory + contentDimensionSource: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory + authProvider: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeAuthProviderFactory + clock: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory + propertyConverters: {} + contentGraphProjection: + factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory + catchUpHooks: {} + projections: + 'Vendor.Package:FakeProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + # TODO Test catchUpHooks: + # 'Neos.Neos:FlushRouteCache': + # factoryObjectName: Neos\Neos\FrontendRouting\CatchUpHook\RouterCacheHookFactory + Flow: object: includeClasses: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php new file mode 100644 index 00000000000..f9b815624d9 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -0,0 +1,480 @@ +objectManager = Bootstrap::$staticObjectManager; + $contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); + + $this->truncateTables( + $this->getObject(Connection::class), + $contentRepositoryId + ); + + $this->fakeProjectionState = $this->getMockBuilder(ProjectionStateInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $this->fakeProjection->method('getState')->willReturn($this->fakeProjectionState); + + FakeProjectionFactory::setProjection( + $this->fakeProjection + ); + FakeNodeTypeManagerFactory::setConfiguration([]); + FakeContentDimensionSourceFactory::setWithoutDimensions(); + + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); + + $this->contentRepository = $this->getObject(ContentRepositoryRegistry::class)->get( + $contentRepositoryId + ); + + $this->subscriptionService = $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + + $subscriptionEngineAndEventStoreAccessor = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, $subscriptionEngineAndEventStoreAccessor); + $this->eventStore = $subscriptionEngineAndEventStoreAccessor->eventStore; + $this->subscriptionEngine = $subscriptionEngineAndEventStoreAccessor->subscriptionEngine; + } + + /** @test */ + public function statusOnEmptyDatabase() + { + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + + $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + + $expected = SubscriptionAndProjectionStatuses::fromArray([ + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired(''), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + } + + /** @test */ + public function setupOnEmptyDatabase() + { + $this->subscriptionService->setupEventStore(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + + $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); + $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + + $expected = SubscriptionAndProjectionStatuses::fromArray([ + $contentGraphStatus = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + $fakeProjectionStatus = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + + self::assertEquals($contentGraphStatus, $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString('contentGraph')]))->first()); + self::assertEquals($fakeProjectionStatus, $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString('Vendor.Package:FakeProjection')]))->first()); + } + + /** @test */ + public function setupProjectionsAndCatchup() + { + $this->subscriptionService->setupEventStore(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::none()), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + + // commit an event: + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + + // subsequent catchup setup'd does not change the position + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::none()), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + } + + /** @test */ + public function existingEventStoreEventsAreCaughtUpOnBoot() + { + $this->eventStore->setup(); + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + self::assertEquals( + self::expectedStatusesAtPosition(SubscriptionStatus::BOOTING, SequenceNumber::none()), + $this->subscriptionService->subscriptionEngine->subscriptionStatuses() + ); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->subscriptionService->subscriptionEngine->boot(); + + self::assertEquals( + self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)), + $this->subscriptionService->subscriptionEngine->subscriptionStatuses() + ); + + // catchup is a noop because there are no unhandled events + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + } + + /** @test */ + public function projectionWithError() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::none()), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + + // commit an event: + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + + // catchup active tries to apply the commited event + $this->fakeProjection->expects(self::exactly(3))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( + new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), + new WillThrowException(new \Error('Something really wrong.')), + new WillThrowException(new \InvalidArgumentException('Dead.')), + ); + // TIME 1 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // TIME 2 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + self::assertEquals($result->errors->first()->message, 'Something really wrong.'); + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); + + // TIME 3 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + self::assertEquals($result->errors->first()->message, 'Dead.'); + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); + + // succeeding calls, nothing to do. + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + // still dead + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); + } + + /** @test */ + public function fixProjectionWithError() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event: + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + + // catchup active tries to apply the commited event + $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( + new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), + null // okay again + ); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + // TIME 1 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // todo BOOT and SETUP should not attempt to retry?! + // setup does not change anything + // $result = $this->subscriptionService->subscriptionEngine->setup(); + // self::assertNull($result->errors); + // boot neither + // $result = $this->subscriptionService->subscriptionEngine->boot(); + // self::assertNull($result->errors); + // still the same state + // self::assertEquals( + // $expectedFailure, + // $this->subscriptionStatus('Vendor.Package:FakeProjection') + // ); + + $this->fakeProjection->expects(self::once())->method('resetState'); + + $result = $this->subscriptionService->subscriptionEngine->reset(); + self::assertNull($result->errors); + + // expect the subscriptionError to be reset to null + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $originalException = new \RuntimeException('This projection is kaputt.'), + ); + + $handleException = null; + try { + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root'), ContentStreamId::fromString('root-cs'))); + } catch (\RuntimeException $exception) { + $handleException = $exception; + } + self::assertNotNull($handleException); + self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); + self::assertSame($originalException, $handleException->getPrevious()); + + // workspace is created + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + + $handleException = null; + try { + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); + } catch (\RuntimeException $exception) { + $handleException = $exception; + } + self::assertNotNull($handleException); + + // workspace two is created. The fake projection is still dead and FAILS on the FIRST event, but the content graph gets only the new event: + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + } + + private function truncateTables(Connection $connection, ContentRepositoryId $contentRepositoryId): void + { + $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); + foreach ($connection->getSchemaManager()->listTableNames() as $tableNames) { + if (!str_starts_with($tableNames, sprintf('cr_%s_', $contentRepositoryId->value))) { + // speedup deletion, only delete current cr + continue; + } + // todo use TRUNCATE to speed up + $sql = 'DROP TABLE ' . $tableNames; + $connection->prepare($sql)->executeStatement(); + } + $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); + } + + private function subscriptionStatus(string $subscriptionId): SubscriptionAndProjectionStatus + { + return $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + } + + private function expectOkayStatus($subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void + { + $actual = $this->subscriptionStatus($subscriptionId); + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString($subscriptionId), + subscriptionStatus: $status, + subscriptionPosition: $sequenceNumber, + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + $actual + ); + } + + // todo replace with expectOkayStatus + public static function expectedStatusesAtPosition(SubscriptionStatus $status, SequenceNumber $sequenceNumber): SubscriptionAndProjectionStatuses + { + return SubscriptionAndProjectionStatuses::fromArray([ + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: $status, + subscriptionPosition: $sequenceNumber, + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: $status, + subscriptionPosition: $sequenceNumber, + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + ]); + } + + /** + * @template T of object + * @param class-string $className + * + * @return T + */ + private function getObject(string $className): object + { + return $this->objectManager->get($className); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index 0664f52deb7..715a087168e 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -11,7 +11,7 @@ final readonly class Errors implements \IteratorAggregate, \Countable { /** - * @var array + * @var non-empty-array */ private array $errors; @@ -41,4 +41,11 @@ public function count(): int { return count($this->errors); } + + public function first(): Error + { + foreach ($this->errors as $error) { + return $error; + } + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php index 824b5e87572..61f1a07e840 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -42,8 +42,8 @@ public function throwOnFailure(): void $additionalFailedSubscribers = array_map(fn (Error $error) => $error->subscriptionId->value, $errors); - $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); - $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s.%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); + $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' | And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); + $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); // todo custom exception! throw new \RuntimeException($exceptionMessage, 1732132930, $firstError->throwable); diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php new file mode 100644 index 00000000000..e2bc347e545 --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php @@ -0,0 +1,26 @@ + Date: Thu, 21 Nov 2024 19:51:08 +0100 Subject: [PATCH 039/142] TASK: Allow to replace setting of cr registry via `injectSettings` --- .../Classes/ContentRepositoryRegistry.php | 39 +++++++++++-------- .../Configuration/Objects.yaml | 6 --- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 5ffe1c220b7..1f073833b21 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -49,7 +49,7 @@ /** * @api */ -#[Flow\Scope("singleton")] +#[Flow\Scope('singleton')] final class ContentRepositoryRegistry { /** @@ -57,17 +57,24 @@ final class ContentRepositoryRegistry */ private array $factoryInstances = []; + private array $settings; + #[Flow\Inject(name: 'Neos.ContentRepositoryRegistry:Logger', lazy: false)] protected LoggerInterface $logger; + #[Flow\Inject()] + protected ObjectManagerInterface $objectManager; + + #[Flow\Inject()] + protected SubgraphCachePool $subgraphCachePool; + /** + * @internal for flow wiring and test cases only * @param array $settings */ - public function __construct( - private readonly array $settings, - private readonly ObjectManagerInterface $objectManager, - private readonly SubgraphCachePool $subgraphCachePool, - ) { + public function injectSettings(array $settings): void + { + $this->settings = $settings; } /** @@ -102,16 +109,6 @@ public function getContentRepositoryIds(): ContentRepositoryIds return ContentRepositoryIds::fromArray($contentRepositoryIds); } - /** - * @internal for test cases only - */ - public function resetFactoryInstance(ContentRepositoryId $contentRepositoryId): void - { - if (array_key_exists($contentRepositoryId->value, $this->factoryInstances)) { - unset($this->factoryInstances[$contentRepositoryId->value]); - } - } - public function subgraphForNode(Node $node): ContentSubgraphInterface { $contentRepository = $this->get($node->contentRepositoryId); @@ -140,6 +137,16 @@ public function buildService(ContentRepositoryId $contentRepositoryId, ContentRe return $this->getFactory($contentRepositoryId)->buildService($contentRepositoryServiceFactory); } + /** + * @internal for test cases only + */ + public function resetFactoryInstance(ContentRepositoryId $contentRepositoryId): void + { + if (array_key_exists($contentRepositoryId->value, $this->factoryInstances)) { + unset($this->factoryInstances[$contentRepositoryId->value]); + } + } + /** * @throws ContentRepositoryNotFoundException | InvalidConfigurationException */ diff --git a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml index f41e1e795aa..70297ff0415 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml @@ -1,9 +1,3 @@ -Neos\ContentRepositoryRegistry\ContentRepositoryRegistry: - arguments: - 1: - setting: Neos.ContentRepositoryRegistry - -# !!! UGLY WORKAROUNDS, because we cannot wire non-Flow class constructor arguments here. # This adds a soft-dependency to the neos/contentgraph-doctrinedbaladapter package Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory: From 715ec2e817b5d39e21e95dedd69a9bff39711674 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:41:45 +0100 Subject: [PATCH 040/142] TASK: Subscription engine test new and detached status --- .../Configuration/Testing/Settings.yaml | 2 + .../Functional/SubscriptionEngineTest.php | 131 +++++++++++++++++- 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index 9e6b4e2cb8a..faae2b1c543 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -58,6 +58,8 @@ Neos: projections: 'Vendor.Package:FakeProjection': factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: default # TODO Test catchUpHooks: # 'Neos.Neos:FlushRouteCache': # factoryObjectName: Neos\Neos\FrontendRouting\CatchUpHook\RouterCacheHookFactory diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index f9b815624d9..0f635c8e02e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -38,6 +38,7 @@ use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStream\ExpectedVersion; +use Neos\Flow\Configuration\ConfigurationManager; use Neos\Flow\Core\Bootstrap; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\MockObject\MockObject; @@ -75,13 +76,21 @@ public function setUp(): void $this->fakeProjection->method('getState')->willReturn($this->fakeProjectionState); FakeProjectionFactory::setProjection( + 'default', $this->fakeProjection ); FakeNodeTypeManagerFactory::setConfiguration([]); FakeContentDimensionSourceFactory::setWithoutDimensions(); $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + $this->setupContentRepositoryDependencies($contentRepositoryId); + } + + public function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId) + { $this->contentRepository = $this->getObject(ContentRepositoryRegistry::class)->get( $contentRepositoryId ); @@ -105,6 +114,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor $this->subscriptionEngine = $subscriptionEngineAndEventStoreAccessor->subscriptionEngine; } + /** @test */ public function statusOnEmptyDatabase() { @@ -411,6 +421,125 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); } + /** @test */ + public function projectionIsDetachedIfConfigurationIsRemoved() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + // todo status is stale??, should be DETACHED + // $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $this->fakeProjection->expects(self::never())->method('apply'); + // catchup or anything that finds detached subscribers + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: null // no calculate-able at this point! + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } + + /** @test */ + public function newProjectionIsFoundConfigurationIsAdded() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + $newFakeProjection->expects(self::exactly(3))->method('status')->willReturnOnConsecutiveCalls( + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::ok(), + ProjectionStatus::ok(), + ); + + FakeProjectionFactory::setProjection( + 'newFake', + $newFakeProjection + ); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:NewFakeProjection'] = [ + 'factoryObjectName' => FakeProjectionFactory::class, + 'options' => [ + 'instanceId' => 'newFake' + ] + ]; + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + // todo status doesnt find this projection yet? + self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + + // do something that finds new subscriptions + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Set me up') + ), + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // setup this projection + $newFakeProjection->expects(self::once())->method('setUp'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } + private function truncateTables(Connection $connection, ContentRepositoryId $contentRepositoryId): void { $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); @@ -426,7 +555,7 @@ private function truncateTables(Connection $connection, ContentRepositoryId $con $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); } - private function subscriptionStatus(string $subscriptionId): SubscriptionAndProjectionStatus + private function subscriptionStatus(string $subscriptionId): ?SubscriptionAndProjectionStatus { return $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } From 512e3c46fa8dd879dbe5446ed78506af2b064cc1 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:45:17 +0100 Subject: [PATCH 041/142] TASK: Speedup tests by using truncate --- .../Functional/SubscriptionEngineTest.php | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index 0f635c8e02e..1c08113352d 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -66,9 +66,10 @@ public function setUp(): void $this->objectManager = Bootstrap::$staticObjectManager; $contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); - $this->truncateTables( + $this->resetDatabase( $this->getObject(Connection::class), - $contentRepositoryId + $contentRepositoryId, + true ); $this->fakeProjectionState = $this->getMockBuilder(ProjectionStateInterface::class)->disableAutoReturnValueGeneration()->getMock(); @@ -118,6 +119,13 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor /** @test */ public function statusOnEmptyDatabase() { + // fully drop the tables so that status has to recover if the subscriptions table is not there + $this->resetDatabase( + $this->getObject(Connection::class), + $this->contentRepository->id, + true + ); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); @@ -540,16 +548,20 @@ public function newProjectionIsFoundConfigurationIsAdded() $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } - private function truncateTables(Connection $connection, ContentRepositoryId $contentRepositoryId): void + private function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void { $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); - foreach ($connection->getSchemaManager()->listTableNames() as $tableNames) { + foreach ($connection->createSchemaManager()->listTableNames() as $tableNames) { if (!str_starts_with($tableNames, sprintf('cr_%s_', $contentRepositoryId->value))) { // speedup deletion, only delete current cr continue; } - // todo use TRUNCATE to speed up - $sql = 'DROP TABLE ' . $tableNames; + if ($keepSchema) { + // truncate is faster + $sql = 'TRUNCATE TABLE ' . $tableNames; + } else { + $sql = 'DROP TABLE ' . $tableNames; + } $connection->prepare($sql)->executeStatement(); } $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); From 408ceb22b5ca19f9de0d069ac0f534a22a06e397 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:18:52 +0100 Subject: [PATCH 042/142] TASK: Subscription engine test filtering by subscription id --- .../Configuration/Testing/Settings.yaml | 4 + .../Functional/SubscriptionEngineTest.php | 192 ++++++++++++++---- .../SubscriptionAndProjectionStatuses.php | 8 + .../Classes/Fakes/FakeProjectionFactory.php | 8 +- 4 files changed, 166 insertions(+), 46 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index faae2b1c543..779034a0d2e 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -60,6 +60,10 @@ Neos: factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory options: instanceId: default + 'Vendor.Package:SecondFakeProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: second # TODO Test catchUpHooks: # 'Neos.Neos:FlushRouteCache': # factoryObjectName: Neos\Neos\FrontendRouting\CatchUpHook\RouterCacheHookFactory diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index 1c08113352d..210698c7c95 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -24,6 +24,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; @@ -59,7 +60,7 @@ final class SubscriptionEngineTest extends TestCase // we don't use Flows functi private ProjectionInterface&MockObject $fakeProjection; - private ProjectionStateInterface&MockObject $fakeProjectionState; + private ProjectionInterface&MockObject $secondFakeProjection; public function setUp(): void { @@ -69,17 +70,25 @@ public function setUp(): void $this->resetDatabase( $this->getObject(Connection::class), $contentRepositoryId, - true + keepSchema: true ); - $this->fakeProjectionState = $this->getMockBuilder(ProjectionStateInterface::class)->disableAutoReturnValueGeneration()->getMock(); $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); - $this->fakeProjection->method('getState')->willReturn($this->fakeProjectionState); + $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); FakeProjectionFactory::setProjection( 'default', $this->fakeProjection ); + + $this->secondFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->getMock(); + $this->secondFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + + FakeProjectionFactory::setProjection( + 'second', + $this->secondFakeProjection + ); + FakeNodeTypeManagerFactory::setConfiguration([]); FakeContentDimensionSourceFactory::setWithoutDimensions(); @@ -123,7 +132,7 @@ public function statusOnEmptyDatabase() $this->resetDatabase( $this->getObject(Connection::class), $this->contentRepository->id, - true + keepSchema: false ); $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); @@ -145,6 +154,13 @@ public function statusOnEmptyDatabase() subscriptionError: null, projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), ]); self::assertEquals($expected, $actualStatuses); @@ -158,30 +174,39 @@ public function setupOnEmptyDatabase() $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionService->subscriptionEngine->setup(); + $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); $expected = SubscriptionAndProjectionStatuses::fromArray([ - $contentGraphStatus = SubscriptionAndProjectionStatus::create( + SubscriptionAndProjectionStatus::create( subscriptionId: SubscriptionId::fromString('contentGraph'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, projectionStatus: ProjectionStatus::ok(), ), - $fakeProjectionStatus = SubscriptionAndProjectionStatus::create( + SubscriptionAndProjectionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, projectionStatus: ProjectionStatus::ok(), ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), ]); self::assertEquals($expected, $actualStatuses); - self::assertEquals($contentGraphStatus, $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString('contentGraph')]))->first()); - self::assertEquals($fakeProjectionStatus, $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString('Vendor.Package:FakeProjection')]))->first()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); } /** @test */ @@ -195,7 +220,10 @@ public function setupProjectionsAndCatchup() $result = $this->subscriptionService->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::none()), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); // commit an event: $this->eventStore->commit( @@ -211,13 +239,18 @@ public function setupProjectionsAndCatchup() // subsequent catchup setup'd does not change the position $result = $this->subscriptionService->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); - self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::none()), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); // catchup active does apply the commited event $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(1), $result); - self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); } /** @test */ @@ -238,18 +271,14 @@ public function existingEventStoreEventsAreCaughtUpOnBoot() $this->subscriptionService->subscriptionEngine->setup(); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - self::assertEquals( - self::expectedStatusesAtPosition(SubscriptionStatus::BOOTING, SequenceNumber::none()), - $this->subscriptionService->subscriptionEngine->subscriptionStatuses() - ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); $this->subscriptionService->subscriptionEngine->boot(); - self::assertEquals( - self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)), - $this->subscriptionService->subscriptionEngine->subscriptionStatuses() - ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); // catchup is a noop because there are no unhandled events $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); @@ -265,7 +294,8 @@ public function projectionWithError() $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $result = $this->subscriptionService->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); - self::assertEquals(self::expectedStatusesAtPosition(SubscriptionStatus::ACTIVE, SequenceNumber::none()), $this->subscriptionService->subscriptionEngine->subscriptionStatuses()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); // commit an event: $this->eventStore->commit( @@ -548,6 +578,105 @@ public function newProjectionIsFoundConfigurationIsAdded() $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } + /** @test */ + public function filteringSetup() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->secondFakeProjection->expects(self::never())->method('setUp'); + $this->secondFakeProjection->expects(self::never())->method('apply'); + $this->secondFakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Set me up')); + + $this->subscriptionService->setupEventStore(); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionService->subscriptionEngine->setup($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Set me up') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } + + /** @test */ + public function filteringCatchUpBoot() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->secondFakeProjection->expects(self::once())->method('setUp'); + $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionEngine->boot($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } + + /** @test */ + public function filteringCatchUpActive() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->secondFakeProjection->expects(self::once())->method('setUp'); + $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event: + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + + $this->fakeProjection->expects(self::once())->method('apply'); + $this->secondFakeProjection->expects(self::never())->method('apply'); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->catchUpActive($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } + private function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void { $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); @@ -587,27 +716,6 @@ private function expectOkayStatus($subscriptionId, SubscriptionStatus $status, S ); } - // todo replace with expectOkayStatus - public static function expectedStatusesAtPosition(SubscriptionStatus $status, SequenceNumber $sequenceNumber): SubscriptionAndProjectionStatuses - { - return SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('contentGraph'), - subscriptionStatus: $status, - subscriptionPosition: $sequenceNumber, - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: $status, - subscriptionPosition: $sequenceNumber, - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - ]); - } - /** * @template T of object * @param class-string $className diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php index 79c4b74e661..4fe2fb5e3bb 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php @@ -29,6 +29,14 @@ public static function fromArray(array $statuses): self return new self(...$statuses); } + public function first(): ?SubscriptionAndProjectionStatus + { + foreach ($this->statuses as $status) { + return $status; + } + return null; + } + public function getIterator(): \Traversable { yield from $this->statuses; diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php index e2bc347e545..ea70c2ef79c 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php @@ -10,17 +10,17 @@ class FakeProjectionFactory implements ProjectionFactoryInterface { - private static ProjectionInterface $projection; + private static array $projections; public function build( SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ProjectionInterface { - return static::$projection ?? throw new \RuntimeException('No projection defined for Fake.'); + return static::$projections[$options['instanceId']] ?? throw new \RuntimeException('No projection defined for Fake.'); } - public static function setProjection(ProjectionInterface $projection): void + public static function setProjection(string $instanceId, ProjectionInterface $projection): void { - self::$projection = $projection; + self::$projections[$instanceId] = $projection; } } From b71165c571eea7d23976c6131bfd1aa911c358e9 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:58:53 +0100 Subject: [PATCH 043/142] TASK: Subscription engine test `filteringReset` and extract commitExampleContentStreamEvent --- .../Functional/SubscriptionEngineTest.php | 115 +++++++++--------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index 210698c7c95..089cbf66ff7 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -225,16 +225,8 @@ public function setupProjectionsAndCatchup() $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - // commit an event: - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ExpectedVersion::NO_STREAM() - ); + // commit an event + $this->commitExampleContentStreamEvent(); // subsequent catchup setup'd does not change the position $result = $this->subscriptionService->subscriptionEngine->boot(); @@ -257,15 +249,8 @@ public function setupProjectionsAndCatchup() public function existingEventStoreEventsAreCaughtUpOnBoot() { $this->eventStore->setup(); - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ExpectedVersion::NO_STREAM() - ); + // commit an event + $this->commitExampleContentStreamEvent(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionService->subscriptionEngine->setup(); @@ -297,16 +282,8 @@ public function projectionWithError() $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - // commit an event: - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ExpectedVersion::NO_STREAM() - ); + // commit an event + $this->commitExampleContentStreamEvent(); // catchup active tries to apply the commited event $this->fakeProjection->expects(self::exactly(3))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( @@ -358,16 +335,8 @@ public function fixProjectionWithError() $this->subscriptionService->subscriptionEngine->setup(); $this->subscriptionService->subscriptionEngine->boot(); - // commit an event: - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ExpectedVersion::NO_STREAM() - ); + // commit an event + $this->commitExampleContentStreamEvent(); // catchup active tries to apply the commited event $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( @@ -473,15 +442,8 @@ public function projectionIsDetachedIfConfigurationIsRemoved() $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ExpectedVersion::NO_STREAM() - ); + // commit an event + $this->commitExampleContentStreamEvent(); $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); @@ -656,15 +618,7 @@ public function filteringCatchUpActive() $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); // commit an event: - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ExpectedVersion::NO_STREAM() - ); + $this->commitExampleContentStreamEvent(); $this->fakeProjection->expects(self::once())->method('apply'); $this->secondFakeProjection->expects(self::never())->method('apply'); @@ -677,6 +631,40 @@ public function filteringCatchUpActive() $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } + /** @test */ + public function filteringReset() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->secondFakeProjection->expects(self::once())->method('setUp'); + $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->secondFakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->reset($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + private function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void { $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); @@ -701,6 +689,19 @@ private function subscriptionStatus(string $subscriptionId): ?SubscriptionAndPro return $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } + private function commitExampleContentStreamEvent(): void + { + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ExpectedVersion::NO_STREAM() + ); + } + private function expectOkayStatus($subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void { $actual = $this->subscriptionStatus($subscriptionId); From c8f8c6af1c750c66feda828c37549201aeb7c638 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:47:19 +0100 Subject: [PATCH 044/142] TASK: introduce `DebugEventProjection` for testing to assert each event is only handled once and further introspection --- .../TestSuite/DebugEventProjection.php | 94 +++++++++++++++++++ .../TestSuite/DebugEventProjectionState.php | 33 +++++++ .../Configuration/Testing/Settings.yaml | 5 + .../Functional/DebugEventProjectionTest.php | 57 +++++++++++ .../Functional/SubscriptionEngineTest.php | 44 ++++----- .../ParallelWritingInWorkspacesTest.php | 13 +++ .../WorkspacePublicationDuringWritingTest.php | 13 +++ .../WorkspaceWritingDuringRebaseTest.php | 13 +++ 8 files changed, 251 insertions(+), 21 deletions(-) create mode 100644 Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php create mode 100644 Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php new file mode 100644 index 00000000000..11314225e3d --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -0,0 +1,94 @@ +state = new DebugEventProjectionState($this->tableNamePrefix, $this->dbal); + } + + public function setUp(): void + { + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->dbal->executeStatement($statement); + } + } + + public function status(): ProjectionStatus + { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + private function determineRequiredSqlStatements(): array + { + $schemaManager = $this->dbal->createSchemaManager(); + + $table = new Table($this->tableNamePrefix, [ + (new Column('sequencenumber', Type::getType(Types::INTEGER))), + (new Column('stream', Type::getType(Types::STRING))), + (new Column('type', Type::getType(Types::STRING))) + ]); + + $table->setPrimaryKey([ + 'sequencenumber' + ]); + + $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$table]); + $statements = DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); + + return $statements; + } + + public function resetState(): void + { + $this->dbal->exec('TRUNCATE ' . $this->tableNamePrefix); + } + + public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void + { + $this->dbal->insert($this->tableNamePrefix, [ + 'sequencenumber' => $eventEnvelope->sequenceNumber->value, + 'stream' => $eventEnvelope->streamName->value, + 'type' => $eventEnvelope->event->type->value, + ]); + } + + public function getState(): ProjectionStateInterface + { + return $this->state; + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php new file mode 100644 index 00000000000..02dde323ad5 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php @@ -0,0 +1,33 @@ + + */ + public function findAppliedSequenceNumbers(): iterable + { + return array_map( + fn ($value) => SequenceNumber::fromInteger((int)$value['sequenceNumber']), + $this->dbal->fetchAllAssociative("SELECT sequenceNumber from {$this->tableNamePrefix}") + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index 779034a0d2e..fced33c3691 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -37,6 +37,11 @@ Neos: contentGraphProjection: factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory catchUpHooks: {} + projections: + 'Neos.Testing:DebugProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: debug t_subscription: eventStore: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php new file mode 100644 index 00000000000..3c05b3ef58f --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php @@ -0,0 +1,57 @@ +get(Connection::class) + ); + + $debugProjection->setUp(); + + $this->expectException(\Doctrine\DBAL\Exception\UniqueConstraintViolationException::class); + + $fakeEventEnvelope = new EventEnvelope( + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + ), + ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + Event\Version::first(), + SequenceNumber::fromInteger(1), + new \DateTimeImmutable() + ); + + $debugProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope + ); + + $debugProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index 089cbf66ff7..3958732a982 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -5,6 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional; use Doctrine\DBAL\Connection; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; @@ -60,7 +61,7 @@ final class SubscriptionEngineTest extends TestCase // we don't use Flows functi private ProjectionInterface&MockObject $fakeProjection; - private ProjectionInterface&MockObject $secondFakeProjection; + private DebugEventProjection $secondFakeProjection; public function setUp(): void { @@ -81,8 +82,10 @@ public function setUp(): void $this->fakeProjection ); - $this->secondFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->getMock(); - $this->secondFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + $this->secondFakeProjection = new DebugEventProjection( + 'cr_t_subscription_debug_projection', + $this->getObject(Connection::class) + ); FakeProjectionFactory::setProjection( 'second', @@ -174,7 +177,6 @@ public function setupOnEmptyDatabase() $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionService->subscriptionEngine->setup(); - $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); @@ -207,6 +209,10 @@ public function setupOnEmptyDatabase() $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); } /** @test */ @@ -220,7 +226,6 @@ public function setupProjectionsAndCatchup() $result = $this->subscriptionService->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -546,10 +551,6 @@ public function filteringSetup() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); - $this->secondFakeProjection->expects(self::never())->method('setUp'); - $this->secondFakeProjection->expects(self::never())->method('apply'); - $this->secondFakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Set me up')); - $this->subscriptionService->setupEventStore(); $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); @@ -565,7 +566,7 @@ public function filteringSetup() subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Set me up') + projectionStatus: ProjectionStatus::ok() ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -577,9 +578,6 @@ public function filteringCatchUpBoot() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->secondFakeProjection->expects(self::once())->method('setUp'); - $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); $result = $this->subscriptionService->subscriptionEngine->setup(); @@ -604,9 +602,6 @@ public function filteringCatchUpActive() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->secondFakeProjection->expects(self::once())->method('setUp'); - $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); $result = $this->subscriptionService->subscriptionEngine->setup(); @@ -621,12 +616,15 @@ public function filteringCatchUpActive() $this->commitExampleContentStreamEvent(); $this->fakeProjection->expects(self::once())->method('apply'); - $this->secondFakeProjection->expects(self::never())->method('apply'); $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); $result = $this->subscriptionEngine->catchUpActive($filter); self::assertNull($result->errors); + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } @@ -637,20 +635,19 @@ public function filteringReset() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->secondFakeProjection->expects(self::once())->method('setUp'); - $this->secondFakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); $result = $this->subscriptionService->subscriptionEngine->setup(); self::assertNull($result->errors); + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); // commit an event: $this->commitExampleContentStreamEvent(); $this->fakeProjection->expects(self::once())->method('apply'); $this->fakeProjection->expects(self::once())->method('resetState'); - $this->secondFakeProjection->expects(self::once())->method('apply'); $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); @@ -663,6 +660,11 @@ public function filteringReset() $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); } private function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php index ff40fdca1bc..58d0ce6acc9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\ParallelWritingInWorkspaces; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -31,6 +33,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -56,6 +59,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php index ef544241cf2..3c62dcc5149 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\WorkspacePublicationDuringWriting; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -34,6 +36,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\EventStore\Exception\ConcurrencyException; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -51,6 +54,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php index f4a37360ed1..566a86c9bc9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\WorkspaceWritingDuringRebase; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -35,6 +37,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\EventStore\Exception\ConcurrencyException; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -53,6 +56,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], From 91046e6a403410374a61ea3d818cb267026a51a6 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 22 Nov 2024 20:32:05 +0100 Subject: [PATCH 045/142] TASK: introduce test that projection is rollback'd in case of error --- .../TestSuite/DebugEventProjection.php | 15 +++++++ .../Functional/SubscriptionEngineTest.php | 45 +++++++++++++++++++ .../Classes/Fakes/FakeProjectionFactory.php | 5 ++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 11314225e3d..1c4cf701f90 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -30,6 +30,8 @@ final class DebugEventProjection implements ProjectionInterface { private DebugEventProjectionState $state; + private \Exception|null $exceptionToThrowAfterApply; + public function __construct( private string $tableNamePrefix, private Connection $dbal @@ -85,10 +87,23 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void 'stream' => $eventEnvelope->streamName->value, 'type' => $eventEnvelope->event->type->value, ]); + if ($this->exceptionToThrowAfterApply) { + throw $this->exceptionToThrowAfterApply; + } } public function getState(): ProjectionStateInterface { return $this->state; } + + public function sabotageAfterApply(\Exception $exceptionToThrowAfterApply): void + { + $this->exceptionToThrowAfterApply = $exceptionToThrowAfterApply; + } + + public function killSaboteur(): void + { + $this->exceptionToThrowAfterApply = null; + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index 3958732a982..3b80ba7f224 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -393,6 +393,51 @@ public function fixProjectionWithError() $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); } + /** @test */ + public function projectionIsRolledBackAfterError() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + + $this->secondFakeProjection->sabotageAfterApply($exception = new \RuntimeException('This projection is kaputt.')); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->secondFakeProjection->killSaboteur(); + + // todo find way to retry projection? catchup force? + } + /** @test */ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() { diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php index ea70c2ef79c..62366fa7691 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php @@ -8,7 +8,10 @@ use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; -class FakeProjectionFactory implements ProjectionFactoryInterface +/** + * @internal helper to configure custom projection mocks for testing + */ +final class FakeProjectionFactory implements ProjectionFactoryInterface { private static array $projections; From f624ac639776025804d5ebb4f24a1c9075297892 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:11:57 +0100 Subject: [PATCH 046/142] TASK: test catchup hooks on failure --- .../TestSuite/DebugEventProjection.php | 2 +- .../Configuration/Testing/Settings.yaml | 6 +- .../Functional/SubscriptionEngineTest.php | 57 +++++++++++++++++++ .../Classes/Fakes/FakeCatchUpHookFactory.php | 28 +++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 1c4cf701f90..f450c3fab56 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -30,7 +30,7 @@ final class DebugEventProjection implements ProjectionInterface { private DebugEventProjectionState $state; - private \Exception|null $exceptionToThrowAfterApply; + private \Exception|null $exceptionToThrowAfterApply = null; public function __construct( private string $tableNamePrefix, diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index fced33c3691..b0472856b5e 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -69,9 +69,9 @@ Neos: factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory options: instanceId: second - # TODO Test catchUpHooks: - # 'Neos.Neos:FlushRouteCache': - # factoryObjectName: Neos\Neos\FrontendRouting\CatchUpHook\RouterCacheHookFactory + catchUpHooks: + 'Vendor.Package:FakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory Flow: object: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index 3b80ba7f224..c13cd2c754b 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -13,6 +13,7 @@ use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Command\CreateRootWorkspace; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; @@ -32,6 +33,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; +use Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; @@ -63,6 +65,8 @@ final class SubscriptionEngineTest extends TestCase // we don't use Flows functi private DebugEventProjection $secondFakeProjection; + private CatchUpHookInterface&MockObject $catchupHookForFakeProjection; + public function setUp(): void { $this->objectManager = Bootstrap::$staticObjectManager; @@ -92,6 +96,13 @@ public function setUp(): void $this->secondFakeProjection ); + $this->catchupHookForFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + + FakeCatchUpHookFactory::setCatchupHook( + $this->secondFakeProjection->getState(), + $this->catchupHookForFakeProjection + ); + FakeNodeTypeManagerFactory::setConfiguration([]); FakeContentDimensionSourceFactory::setWithoutDimensions(); @@ -421,6 +432,7 @@ public function projectionIsRolledBackAfterError() ); $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + // todo check that exception message is in the result self::assertTrue($result->hasFailed()); self::assertEquals( @@ -438,6 +450,51 @@ public function projectionIsRolledBackAfterError() // todo find way to retry projection? catchup force? } + /** @test todo test also what happens if onAfterCatchup fails */ + public function projectionIsRolledBackAfterCatchupError() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->subscriptionEngine->catchUpActive(); + + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + // todo check that exception message is in the result + self::assertTrue($result->hasFailed()); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + /** @test */ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() { diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php new file mode 100644 index 00000000000..23e7b1e9241 --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php @@ -0,0 +1,28 @@ +projectionState)] ?? throw new \RuntimeException('No catchup hook defined for Fake.'); + } + + public static function setCatchupHook(ProjectionStateInterface $projectionState, CatchUpHookInterface $catchUpHook): void + { + self::$catchupHooks[spl_object_hash($projectionState)] = $catchUpHook; + } +} From 0197017548cde49f3524d1ce79a48f3d1f46d621 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 12:15:21 +0100 Subject: [PATCH 047/142] TASK: Introduce test to assert that projection keeps events previously applied in the same batch that did not cause errors --- .../TestSuite/DebugEventProjection.php | 26 +++--- .../Functional/DebugEventProjectionTest.php | 84 +++++++++++++++---- .../Functional/SubscriptionEngineTest.php | 66 +++++++++++++-- 3 files changed, 140 insertions(+), 36 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index f450c3fab56..fa1ae446d43 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -30,7 +30,7 @@ final class DebugEventProjection implements ProjectionInterface { private DebugEventProjectionState $state; - private \Exception|null $exceptionToThrowAfterApply = null; + private \Closure|null $saboteur = null; public function __construct( private string $tableNamePrefix, @@ -82,13 +82,17 @@ public function resetState(): void public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { - $this->dbal->insert($this->tableNamePrefix, [ - 'sequencenumber' => $eventEnvelope->sequenceNumber->value, - 'stream' => $eventEnvelope->streamName->value, - 'type' => $eventEnvelope->event->type->value, - ]); - if ($this->exceptionToThrowAfterApply) { - throw $this->exceptionToThrowAfterApply; + try { + $this->dbal->insert($this->tableNamePrefix, [ + 'sequencenumber' => $eventEnvelope->sequenceNumber->value, + 'stream' => $eventEnvelope->streamName->value, + 'type' => $eventEnvelope->event->type->value, + ]); + } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $exception) { + throw new \RuntimeException(sprintf('Must not happen! Debug projection detected duplicate event %s of type %s', $eventEnvelope->sequenceNumber->value, $eventEnvelope->event->type->value), 1732360282, $exception); + } + if ($this->saboteur) { + ($this->saboteur)($eventEnvelope); } } @@ -97,13 +101,13 @@ public function getState(): ProjectionStateInterface return $this->state; } - public function sabotageAfterApply(\Exception $exceptionToThrowAfterApply): void + public function injectSaboteur(\Closure $saboteur): void { - $this->exceptionToThrowAfterApply = $exceptionToThrowAfterApply; + $this->saboteur = $saboteur; } public function killSaboteur(): void { - $this->exceptionToThrowAfterApply = null; + $this->saboteur = null; } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php index 3c05b3ef58f..baebcc9e91c 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php @@ -20,38 +20,88 @@ */ class DebugEventProjectionTest extends TestCase { - /** @test */ - public function fakeProjectionRejectsDuplicateEvents() + private DebugEventProjection $debugEventProjection; + + public function setUp(): void { - $debugProjection = new DebugEventProjection( + $this->debugEventProjection = new DebugEventProjection( 'test_debug_projection', Bootstrap::$staticObjectManager->get(Connection::class) ); - $debugProjection->setUp(); + $this->debugEventProjection->setUp(); + } - $this->expectException(\Doctrine\DBAL\Exception\UniqueConstraintViolationException::class); + public function tearDown(): void + { + $this->debugEventProjection->resetState(); + } - $fakeEventEnvelope = new EventEnvelope( - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) - ), - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), - Event\Version::first(), - SequenceNumber::fromInteger(1), - new \DateTimeImmutable() + /** @test */ + public function fakeProjectionRejectsDuplicateEvents() + { + $fakeEventEnvelope = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(1) ); - $debugProjection->apply( + $this->debugEventProjection->apply( $this->getMockBuilder(EventInterface::class)->getMock(), $fakeEventEnvelope ); - $debugProjection->apply( + $this->expectExceptionMessage('Must not happen! Debug projection detected duplicate event 1 of type ContentStreamWasCreated'); + + $this->debugEventProjection->apply( $this->getMockBuilder(EventInterface::class)->getMock(), $fakeEventEnvelope ); } + + /** @test */ + public function fakeProjectionWithSaboteur() + { + $fakeEventEnvelope1 = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(1) + ); + + $fakeEventEnvelope2 = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(2) + ); + + $this->debugEventProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw new \RuntimeException('sabotage!!!') + : null + ); + + // catchup + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope1 + ); + + $this->expectExceptionMessage('sabotage!!!'); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope2 + ); + } + + private function createExampleEventEnvelopeForPosition(SequenceNumber $sequenceNumber): EventEnvelope + { + $cs = ContentStreamId::create(); + return new EventEnvelope( + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) + ), + ContentStreamEventStreamName::fromContentStreamId($cs)->getEventStreamName(), + Event\Version::first(), + $sequenceNumber, + new \DateTimeImmutable() + ); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index c13cd2c754b..d804f6a8bad 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -41,6 +41,7 @@ use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\EventStore\Model\EventEnvelope; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\Flow\Configuration\ConfigurationManager; use Neos\Flow\Core\Bootstrap; @@ -409,15 +410,16 @@ public function projectionIsRolledBackAfterError() { $this->subscriptionService->setupEventStore(); $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); $this->subscriptionService->subscriptionEngine->setup(); $this->subscriptionService->subscriptionEngine->boot(); // commit an event $this->commitExampleContentStreamEvent(); - $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $exception = new \RuntimeException('This projection is kaputt.'); - $this->secondFakeProjection->sabotageAfterApply($exception = new \RuntimeException('This projection is kaputt.')); + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); $expectedFailure = SubscriptionAndProjectionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), @@ -432,8 +434,7 @@ public function projectionIsRolledBackAfterError() ); $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - // todo check that exception message is in the result - self::assertTrue($result->hasFailed()); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); self::assertEquals( $expectedFailure, @@ -450,6 +451,56 @@ public function projectionIsRolledBackAfterError() // todo find way to retry projection? catchup force? } + /** @test */ + public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('Event 2 is kaputt.'); + + // fail at the second event + $this->secondFakeProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw $exception + : null + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // the first successful event is applied and committet: + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + /** @test todo test also what happens if onAfterCatchup fails */ public function projectionIsRolledBackAfterCatchupError() { @@ -481,8 +532,7 @@ public function projectionIsRolledBackAfterCatchupError() $this->subscriptionEngine->catchUpActive(); $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - // todo check that exception message is in the result - self::assertTrue($result->hasFailed()); + self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); self::assertEquals( $expectedFailure, @@ -796,11 +846,11 @@ private function subscriptionStatus(string $subscriptionId): ?SubscriptionAndPro private function commitExampleContentStreamEvent(): void { $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId(ContentStreamId::fromString('cs-id'))->getEventStreamName(), + ContentStreamEventStreamName::fromContentStreamId($cs = ContentStreamId::create())->getEventStreamName(), new Event( Event\EventId::create(), Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => 'cs-id'])) + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) ), ExpectedVersion::NO_STREAM() ); From 3b6ca2601075d35ef848461584d4c70876470527 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 12:41:52 +0100 Subject: [PATCH 048/142] TASK: Improve catchup rollback test --- .../Configuration/Testing/Settings.yaml | 6 +++--- .../Tests/Functional/SubscriptionEngineTest.php | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index b0472856b5e..7f2fdbc0cc4 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -69,9 +69,9 @@ Neos: factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory options: instanceId: second - catchUpHooks: - 'Vendor.Package:FakeCatchupHook': - factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory + catchUpHooks: + 'Vendor.Package:FakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory Flow: object: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php index d804f6a8bad..16f4cdb3c53 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php @@ -513,9 +513,12 @@ public function projectionIsRolledBackAfterCatchupError() // commit an event $this->commitExampleContentStreamEvent(); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); + // TODO pass the error subscription status to onAfterCatchUp, so that in case of an error it can be prevented that mails f.x. will be sent? + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); $expectedFailure = SubscriptionAndProjectionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), @@ -529,9 +532,7 @@ public function projectionIsRolledBackAfterCatchupError() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $this->subscriptionEngine->catchUpActive(); - - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); self::assertEquals( From 794be116e03dad4c0591bd5da0772bdeea60d147 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:29:23 +0100 Subject: [PATCH 049/142] TASK: Split up mighty `SubscriptionEngineTest` --- .../AbstractSubscriptionEngineTestCase.php | 189 ++++ .../Subscription/CatchUpHookErrorTest.php | 61 ++ .../DebugEventProjectionTest.php | 4 +- .../Subscription/ProjectionErrorTest.php | 278 ++++++ .../SubscriptionActiveStatusTest.php | 84 ++ .../SubscriptionBootingStatusTest.php | 66 ++ .../SubscriptionDetachedStatusTest.php | 60 ++ .../SubscriptionGetStatusTest.php | 57 ++ .../SubscriptionNewStatusTest.php | 89 ++ .../Subscription/SubscriptionResetTest.php | 52 + .../Subscription/SubscriptionSetupTest.php | 89 ++ .../Functional/SubscriptionEngineTest.php | 885 ------------------ 12 files changed, 1027 insertions(+), 887 deletions(-) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php rename Neos.ContentRepository.BehavioralTests/Tests/Functional/{ => Subscription}/DebugEventProjectionTest.php (97%) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php delete mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php new file mode 100644 index 00000000000..85d1e577e77 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -0,0 +1,189 @@ +resetDatabase( + $this->getObject(Connection::class), + $contentRepositoryId, + keepSchema: true + ); + + $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + + FakeProjectionFactory::setProjection( + 'default', + $this->fakeProjection + ); + + $this->secondFakeProjection = new DebugEventProjection( + 'cr_t_subscription_debug_projection', + $this->getObject(Connection::class) + ); + + FakeProjectionFactory::setProjection( + 'second', + $this->secondFakeProjection + ); + + $this->catchupHookForFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + + FakeCatchUpHookFactory::setCatchupHook( + $this->secondFakeProjection->getState(), + $this->catchupHookForFakeProjection + ); + + FakeNodeTypeManagerFactory::setConfiguration([]); + FakeContentDimensionSourceFactory::setWithoutDimensions(); + + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + + $this->setupContentRepositoryDependencies($contentRepositoryId); + } + + final protected function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId) + { + $this->contentRepository = $this->getObject(ContentRepositoryRegistry::class)->get( + $contentRepositoryId + ); + + $this->subscriptionService = $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + + $subscriptionEngineAndEventStoreAccessor = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, $subscriptionEngineAndEventStoreAccessor); + $this->eventStore = $subscriptionEngineAndEventStoreAccessor->eventStore; + $this->subscriptionEngine = $subscriptionEngineAndEventStoreAccessor->subscriptionEngine; + } + + final protected function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void + { + $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); + foreach ($connection->createSchemaManager()->listTableNames() as $tableNames) { + if (!str_starts_with($tableNames, sprintf('cr_%s_', $contentRepositoryId->value))) { + // speedup deletion, only delete current cr + continue; + } + if ($keepSchema) { + // truncate is faster + $sql = 'TRUNCATE TABLE ' . $tableNames; + } else { + $sql = 'DROP TABLE ' . $tableNames; + } + $connection->prepare($sql)->executeStatement(); + } + $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); + } + + final protected function subscriptionStatus(string $subscriptionId): ?SubscriptionAndProjectionStatus + { + return $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + } + + final protected function commitExampleContentStreamEvent(): void + { + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId($cs = ContentStreamId::create())->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) + ), + ExpectedVersion::NO_STREAM() + ); + } + + final protected function expectOkayStatus($subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void + { + $actual = $this->subscriptionStatus($subscriptionId); + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString($subscriptionId), + subscriptionStatus: $status, + subscriptionPosition: $sequenceNumber, + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + $actual + ); + } + + /** + * @template T of object + * @param class-string $className + * + * @return T + */ + final protected function getObject(string $className): object + { + return Bootstrap::$staticObjectManager->get($className); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php new file mode 100644 index 00000000000..a4ec44bbaff --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -0,0 +1,61 @@ +subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + // TODO pass the error subscription status to onAfterCatchUp, so that in case of an error it can be prevented that mails f.x. will be sent? + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php similarity index 97% rename from Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php rename to Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php index baebcc9e91c..e80be3c98ea 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/DebugEventProjectionTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Neos\ContentRepository\BehavioralTests\Tests\Functional; +namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; @@ -18,7 +18,7 @@ /** * Just a test to check our mock is working {@see DebugEventProjection} */ -class DebugEventProjectionTest extends TestCase +final class DebugEventProjectionTest extends TestCase { private DebugEventProjection $debugEventProjection; diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php new file mode 100644 index 00000000000..2fa2c8fef48 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -0,0 +1,278 @@ +subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active tries to apply the commited event + $this->fakeProjection->expects(self::exactly(3))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( + new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), + new WillThrowException(new \Error('Something really wrong.')), + new WillThrowException(new \InvalidArgumentException('Dead.')), + ); + // TIME 1 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // TIME 2 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + self::assertEquals($result->errors->first()->message, 'Something really wrong.'); + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); + + // TIME 3 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + self::assertEquals($result->errors->first()->message, 'Dead.'); + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); + + // succeeding calls, nothing to do. + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + // still dead + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); + } + + /** @test */ + public function fixProjectionWithError() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active tries to apply the commited event + $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( + new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), + null // okay again + ); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + // TIME 1 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // todo BOOT and SETUP should not attempt to retry?! + // setup does not change anything + // $result = $this->subscriptionService->subscriptionEngine->setup(); + // self::assertNull($result->errors); + // boot neither + // $result = $this->subscriptionService->subscriptionEngine->boot(); + // self::assertNull($result->errors); + // still the same state + // self::assertEquals( + // $expectedFailure, + // $this->subscriptionStatus('Vendor.Package:FakeProjection') + // ); + + $this->fakeProjection->expects(self::once())->method('resetState'); + + $result = $this->subscriptionService->subscriptionEngine->reset(); + self::assertNull($result->errors); + + // expect the subscriptionError to be reset to null + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function projectionIsRolledBackAfterError() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->secondFakeProjection->killSaboteur(); + + // todo find way to retry projection? catchup force? + } + + /** @test */ + public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('Event 2 is kaputt.'); + + // fail at the second event + $this->secondFakeProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw $exception + : null + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // the first successful event is applied and committet: + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $originalException = new \RuntimeException('This projection is kaputt.'), + ); + + $handleException = null; + try { + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root'), ContentStreamId::fromString('root-cs'))); + } catch (\RuntimeException $exception) { + $handleException = $exception; + } + self::assertNotNull($handleException); + self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); + self::assertSame($originalException, $handleException->getPrevious()); + + // workspace is created + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + + $handleException = null; + try { + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); + } catch (\RuntimeException $exception) { + $handleException = $exception; + } + self::assertNotNull($handleException); + + // workspace two is created. The fake projection is still dead and FAILS on the FIRST event, but the content graph gets only the new event: + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php new file mode 100644 index 00000000000..ad5c4b99656 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -0,0 +1,84 @@ +subscriptionService->setupEventStore(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // subsequent catchup setup'd does not change the position + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function filteringCatchUpActive() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->catchUpActive($filter); + self::assertNull($result->errors); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php new file mode 100644 index 00000000000..01b15d9137c --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php @@ -0,0 +1,66 @@ +eventStore->setup(); + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->subscriptionService->subscriptionEngine->boot(); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // catchup is a noop because there are no unhandled events + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + } + + /** @test */ + public function filteringCatchUpBoot() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionEngine->boot($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php new file mode 100644 index 00000000000..391d7d45f03 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -0,0 +1,60 @@ +fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + // todo status is stale??, should be DETACHED + // $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $this->fakeProjection->expects(self::never())->method('apply'); + // catchup or anything that finds detached subscribers + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: null // no calculate-able at this point! + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php new file mode 100644 index 00000000000..72e355d6226 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -0,0 +1,57 @@ +resetDatabase( + $this->getObject(Connection::class), + $this->contentRepository->id, + keepSchema: false + ); + + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + + $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + + $expected = SubscriptionAndProjectionStatuses::fromArray([ + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired(''), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php new file mode 100644 index 00000000000..d57e13b69c4 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -0,0 +1,89 @@ +fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + $newFakeProjection->expects(self::exactly(3))->method('status')->willReturnOnConsecutiveCalls( + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::ok(), + ProjectionStatus::ok(), + ); + + FakeProjectionFactory::setProjection( + 'newFake', + $newFakeProjection + ); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:NewFakeProjection'] = [ + 'factoryObjectName' => FakeProjectionFactory::class, + 'options' => [ + 'instanceId' => 'newFake' + ] + ]; + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + // todo status doesnt find this projection yet? + self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + + // do something that finds new subscriptions + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Set me up') + ), + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // setup this projection + $newFakeProjection->expects(self::once())->method('setUp'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php new file mode 100644 index 00000000000..ea1a741ad57 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php @@ -0,0 +1,52 @@ +fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->reset($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php new file mode 100644 index 00000000000..60472c66b24 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -0,0 +1,89 @@ +subscriptionService->setupEventStore(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionService->subscriptionEngine->setup(); + + $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); + $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + + $expected = SubscriptionAndProjectionStatuses::fromArray([ + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function filteringSetup() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionService->subscriptionEngine->setup($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok() + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php deleted file mode 100644 index 16f4cdb3c53..00000000000 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/SubscriptionEngineTest.php +++ /dev/null @@ -1,885 +0,0 @@ -objectManager = Bootstrap::$staticObjectManager; - $contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); - - $this->resetDatabase( - $this->getObject(Connection::class), - $contentRepositoryId, - keepSchema: true - ); - - $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); - $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); - - FakeProjectionFactory::setProjection( - 'default', - $this->fakeProjection - ); - - $this->secondFakeProjection = new DebugEventProjection( - 'cr_t_subscription_debug_projection', - $this->getObject(Connection::class) - ); - - FakeProjectionFactory::setProjection( - 'second', - $this->secondFakeProjection - ); - - $this->catchupHookForFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); - - FakeCatchUpHookFactory::setCatchupHook( - $this->secondFakeProjection->getState(), - $this->catchupHookForFakeProjection - ); - - FakeNodeTypeManagerFactory::setConfiguration([]); - FakeContentDimensionSourceFactory::setWithoutDimensions(); - - $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); - $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); - $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); - - $this->setupContentRepositoryDependencies($contentRepositoryId); - } - - public function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId) - { - $this->contentRepository = $this->getObject(ContentRepositoryRegistry::class)->get( - $contentRepositoryId - ); - - $this->subscriptionService = $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - - $subscriptionEngineAndEventStoreAccessor = new class implements ContentRepositoryServiceFactoryInterface { - public EventStoreInterface|null $eventStore; - public SubscriptionEngine|null $subscriptionEngine; - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - $this->eventStore = $serviceFactoryDependencies->eventStore; - $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; - return new class implements ContentRepositoryServiceInterface - { - }; - } - }; - $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, $subscriptionEngineAndEventStoreAccessor); - $this->eventStore = $subscriptionEngineAndEventStoreAccessor->eventStore; - $this->subscriptionEngine = $subscriptionEngineAndEventStoreAccessor->subscriptionEngine; - } - - - /** @test */ - public function statusOnEmptyDatabase() - { - // fully drop the tables so that status has to recover if the subscriptions table is not there - $this->resetDatabase( - $this->getObject(Connection::class), - $this->contentRepository->id, - keepSchema: false - ); - - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); - - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); - - $expected = SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('contentGraph'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired(''), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - ]); - - self::assertEquals($expected, $actualStatuses); - } - - /** @test */ - public function setupOnEmptyDatabase() - { - $this->subscriptionService->setupEventStore(); - - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); - - $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); - - $expected = SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('contentGraph'), - subscriptionStatus: SubscriptionStatus::BOOTING, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::BOOTING, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::BOOTING, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - ]); - - self::assertEquals($expected, $actualStatuses); - - $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - - /** @test */ - public function setupProjectionsAndCatchup() - { - $this->subscriptionService->setupEventStore(); - - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); - - $result = $this->subscriptionService->subscriptionEngine->boot(); - self::assertEquals(ProcessedResult::success(0), $result); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - // commit an event - $this->commitExampleContentStreamEvent(); - - // subsequent catchup setup'd does not change the position - $result = $this->subscriptionService->subscriptionEngine->boot(); - self::assertEquals(ProcessedResult::success(0), $result); - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - // catchup active does apply the commited event - $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::success(1), $result); - - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - } - - /** @test */ - public function existingEventStoreEventsAreCaughtUpOnBoot() - { - $this->eventStore->setup(); - // commit an event - $this->commitExampleContentStreamEvent(); - - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->subscriptionService->subscriptionEngine->boot(); - - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - - // catchup is a noop because there are no unhandled events - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::success(0), $result); - } - - /** @test */ - public function projectionWithError() - { - $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $result = $this->subscriptionService->subscriptionEngine->boot(); - self::assertEquals(ProcessedResult::success(0), $result); - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - // commit an event - $this->commitExampleContentStreamEvent(); - - // catchup active tries to apply the commited event - $this->fakeProjection->expects(self::exactly(3))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( - new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), - new WillThrowException(new \Error('Something really wrong.')), - new WillThrowException(new \InvalidArgumentException('Dead.')), - ); - // TIME 1 - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); - - self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), - ), - $this->subscriptionStatus('Vendor.Package:FakeProjection') - ); - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - - // TIME 2 - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - self::assertEquals($result->errors->first()->message, 'Something really wrong.'); - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); - - // TIME 3 - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - self::assertEquals($result->errors->first()->message, 'Dead.'); - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); - - // succeeding calls, nothing to do. - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::success(0), $result); - // still dead - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); - } - - /** @test */ - public function fixProjectionWithError() - { - $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); - - // commit an event - $this->commitExampleContentStreamEvent(); - - // catchup active tries to apply the commited event - $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( - new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), - null // okay again - ); - - $expectedFailure = SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), - ); - - // TIME 1 - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:FakeProjection') - ); - - // todo BOOT and SETUP should not attempt to retry?! - // setup does not change anything - // $result = $this->subscriptionService->subscriptionEngine->setup(); - // self::assertNull($result->errors); - // boot neither - // $result = $this->subscriptionService->subscriptionEngine->boot(); - // self::assertNull($result->errors); - // still the same state - // self::assertEquals( - // $expectedFailure, - // $this->subscriptionStatus('Vendor.Package:FakeProjection') - // ); - - $this->fakeProjection->expects(self::once())->method('resetState'); - - $result = $this->subscriptionService->subscriptionEngine->reset(); - self::assertNull($result->errors); - - // expect the subscriptionError to be reset to null - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - $result = $this->subscriptionService->subscriptionEngine->boot(); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - } - - /** @test */ - public function projectionIsRolledBackAfterError() - { - $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); - - // commit an event - $this->commitExampleContentStreamEvent(); - - $exception = new \RuntimeException('This projection is kaputt.'); - - $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); - - $expectedFailure = SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // should be empty as we need an exact once delivery - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $this->secondFakeProjection->killSaboteur(); - - // todo find way to retry projection? catchup force? - } - - /** @test */ - public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() - { - $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::exactly(2))->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); - - // commit two events - $this->commitExampleContentStreamEvent(); - $this->commitExampleContentStreamEvent(); - - $exception = new \RuntimeException('Event 2 is kaputt.'); - - // fail at the second event - $this->secondFakeProjection->injectSaboteur( - fn (EventEnvelope $eventEnvelope) => - $eventEnvelope->sequenceNumber->value === 2 - ? throw $exception - : null - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - - $expectedFailure = SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::fromInteger(1), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), - ); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // the first successful event is applied and committet: - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - - /** @test todo test also what happens if onAfterCatchup fails */ - public function projectionIsRolledBackAfterCatchupError() - { - $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); - - // commit an event - $this->commitExampleContentStreamEvent(); - - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( - $exception = new \RuntimeException('This catchup hook is kaputt.') - ); - // TODO pass the error subscription status to onAfterCatchUp, so that in case of an error it can be prevented that mails f.x. will be sent? - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); - - $expectedFailure = SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionEngine->catchUpActive(); - self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // should be empty as we need an exact once delivery - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - - /** @test */ - public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() - { - $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); - - $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( - $originalException = new \RuntimeException('This projection is kaputt.'), - ); - - $handleException = null; - try { - $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root'), ContentStreamId::fromString('root-cs'))); - } catch (\RuntimeException $exception) { - $handleException = $exception; - } - self::assertNotNull($handleException); - self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); - self::assertSame($originalException, $handleException->getPrevious()); - - // workspace is created - self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); - - $handleException = null; - try { - $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); - } catch (\RuntimeException $exception) { - $handleException = $exception; - } - self::assertNotNull($handleException); - - // workspace two is created. The fake projection is still dead and FAILS on the FIRST event, but the content graph gets only the new event: - self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); - $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); - } - - /** @test */ - public function projectionIsDetachedIfConfigurationIsRemoved() - { - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->subscriptionService->setupEventStore(); - $this->subscriptionService->subscriptionEngine->setup(); - - $result = $this->subscriptionService->subscriptionEngine->boot(); - self::assertEquals(ProcessedResult::success(0), $result); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - // commit an event - $this->commitExampleContentStreamEvent(); - - $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); - unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); - $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); - $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); - $this->setupContentRepositoryDependencies($this->contentRepository->id); - - // todo status is stale??, should be DETACHED - // $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - $this->fakeProjection->expects(self::never())->method('apply'); - // catchup or anything that finds detached subscribers - $result = $this->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::success(1), $result); - - self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::DETACHED, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: null // no calculate-able at this point! - ), - $this->subscriptionStatus('Vendor.Package:FakeProjection') - ); - } - - /** @test */ - public function newProjectionIsFoundConfigurationIsAdded() - { - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->subscriptionService->setupEventStore(); - $this->subscriptionService->subscriptionEngine->setup(); - - $result = $this->subscriptionService->subscriptionEngine->boot(); - self::assertEquals(ProcessedResult::success(0), $result); - - self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); - $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); - $newFakeProjection->expects(self::exactly(3))->method('status')->willReturnOnConsecutiveCalls( - ProjectionStatus::setupRequired('Set me up'), - ProjectionStatus::ok(), - ProjectionStatus::ok(), - ); - - FakeProjectionFactory::setProjection( - 'newFake', - $newFakeProjection - ); - - $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); - $mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:NewFakeProjection'] = [ - 'factoryObjectName' => FakeProjectionFactory::class, - 'options' => [ - 'instanceId' => 'newFake' - ] - ]; - $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); - $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); - $this->setupContentRepositoryDependencies($this->contentRepository->id); - - // todo status doesnt find this projection yet? - self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); - - // do something that finds new subscriptions - $result = $this->subscriptionEngine->catchUpActive(); - self::assertNull($result->errors); - - self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Set me up') - ), - $this->subscriptionStatus('Vendor.Package:NewFakeProjection') - ); - - // setup this projection - $newFakeProjection->expects(self::once())->method('setUp'); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); - $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - } - - /** @test */ - public function filteringSetup() - { - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->subscriptionService->setupEventStore(); - - $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); - - $result = $this->subscriptionService->subscriptionEngine->setup($filter); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok() - ), - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - } - - /** @test */ - public function filteringCatchUpBoot() - { - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->subscriptionService->setupEventStore(); - - $result = $this->subscriptionService->subscriptionEngine->setup(); - self::assertNull($result->errors); - - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); - - $result = $this->subscriptionEngine->boot($filter); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - } - - /** @test */ - public function filteringCatchUpActive() - { - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->subscriptionService->setupEventStore(); - - $result = $this->subscriptionService->subscriptionEngine->setup(); - self::assertNull($result->errors); - $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - - // commit an event: - $this->commitExampleContentStreamEvent(); - - $this->fakeProjection->expects(self::once())->method('apply'); - - $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); - $result = $this->subscriptionEngine->catchUpActive($filter); - self::assertNull($result->errors); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - } - - /** @test */ - public function filteringReset() - { - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - - $this->subscriptionService->setupEventStore(); - - $result = $this->subscriptionService->subscriptionEngine->setup(); - self::assertNull($result->errors); - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // commit an event: - $this->commitExampleContentStreamEvent(); - - $this->fakeProjection->expects(self::once())->method('apply'); - $this->fakeProjection->expects(self::once())->method('resetState'); - $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - - $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); - $result = $this->subscriptionEngine->reset($filter); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - - private function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void - { - $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); - foreach ($connection->createSchemaManager()->listTableNames() as $tableNames) { - if (!str_starts_with($tableNames, sprintf('cr_%s_', $contentRepositoryId->value))) { - // speedup deletion, only delete current cr - continue; - } - if ($keepSchema) { - // truncate is faster - $sql = 'TRUNCATE TABLE ' . $tableNames; - } else { - $sql = 'DROP TABLE ' . $tableNames; - } - $connection->prepare($sql)->executeStatement(); - } - $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); - } - - private function subscriptionStatus(string $subscriptionId): ?SubscriptionAndProjectionStatus - { - return $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); - } - - private function commitExampleContentStreamEvent(): void - { - $this->eventStore->commit( - ContentStreamEventStreamName::fromContentStreamId($cs = ContentStreamId::create())->getEventStreamName(), - new Event( - Event\EventId::create(), - Event\EventType::fromString('ContentStreamWasCreated'), - Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) - ), - ExpectedVersion::NO_STREAM() - ); - } - - private function expectOkayStatus($subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void - { - $actual = $this->subscriptionStatus($subscriptionId); - self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString($subscriptionId), - subscriptionStatus: $status, - subscriptionPosition: $sequenceNumber, - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - $actual - ); - } - - /** - * @template T of object - * @param class-string $className - * - * @return T - */ - private function getObject(string $className): object - { - return $this->objectManager->get($className); - } -} From 752d434d043b6908091784dbe131dd23cb348d4a Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:32:18 +0100 Subject: [PATCH 050/142] TASK: Simplify AbstractSubscriptionEngineTestCase by moving out special logic --- .../Subscription/AbstractSubscriptionEngineTestCase.php | 3 --- .../Subscription/SubscriptionDetachedStatusTest.php | 7 +++++++ .../Functional/Subscription/SubscriptionNewStatusTest.php | 7 +++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 85d1e577e77..ec1284009c2 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -33,7 +33,6 @@ use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStream\ExpectedVersion; -use Neos\Flow\Configuration\ConfigurationManager; use Neos\Flow\Core\Bootstrap; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -93,8 +92,6 @@ public function setUp(): void FakeContentDimensionSourceFactory::setWithoutDimensions(); $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); - $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); - $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); $this->setupContentRepositoryDependencies($contentRepositoryId); } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 391d7d45f03..11d67592ff5 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -15,6 +15,13 @@ final class SubscriptionDetachedStatusTest extends AbstractSubscriptionEngineTestCase { + /** @after */ + public function resetContentRepositoryRegistry(): void + { + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + } + /** @test */ public function projectionIsDetachedIfConfigurationIsRemoved() { diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index d57e13b69c4..f428d764054 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -18,6 +18,13 @@ final class SubscriptionNewStatusTest extends AbstractSubscriptionEngineTestCase { + /** @after */ + public function resetContentRepositoryRegistry(): void + { + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + } + /** @test */ public function newProjectionIsFoundConfigurationIsAdded() { From c4da7fecfd0c8209861a972d15eb708a3bb03e51 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 16:15:38 +0100 Subject: [PATCH 051/142] TASK: Test error behaviour for onBeforeCatchUp and onAfterCatchUp --- .../Subscription/CatchUpHookErrorTest.php | 144 +++++++++++++++++- 1 file changed, 142 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index a4ec44bbaff..98520214fe9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -6,6 +6,7 @@ use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -14,8 +15,8 @@ final class CatchUpHookErrorTest extends AbstractSubscriptionEngineTestCase { - /** @test todo test also what happens if onAfterCatchup fails and also test catchup hooks in general */ - public function projectionIsRolledBackAfterCatchupError() + /** @test */ + public function error_onBeforeEvent_projectionIsNotRun() { $this->subscriptionService->setupEventStore(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -27,6 +28,55 @@ public function projectionIsRolledBackAfterCatchupError() $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + // Todo test that onBeforeEvent|onAfterEvent are in the same transaction and that a rollback will also revert their state + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // must be still empty because apply was never called + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterEvent_projectionIsRolledBack() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); @@ -58,4 +108,94 @@ public function projectionIsRolledBackAfterCatchupError() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } + + /** @test */ + public function error_onBeforeCatchUp_abortsCatchup() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willThrowException( + new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedFailure = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $expectedFailure = $e; + } + self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" could not invoke onBeforeCatchUp: This catchup hook is kaputt.'); + + // still the initial status + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // must be still empty because apply was never called + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterCatchUp_abortsCatchupAndRollBack() + { + $this->subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + new \RuntimeException('This catchup hook is kaputt.') + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedFailure = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $expectedFailure = $e; + } + self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" could not invoke onAfterCatchUp: This catchup hook is kaputt.'); + + // still the initial status + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // must be empty because full rollback + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } } From c8cb0a27fc336620f2a68173bff992010a106bb0 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 16:23:57 +0100 Subject: [PATCH 052/142] TASK: Add test for happy catchup hooks --- .../Subscription/CatchUpHookTest.php | 50 +++++++++++++++++++ .../Subscription/Exception/CatchUpFailed.php | 12 +++++ 2 files changed, 62 insertions(+) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php new file mode 100644 index 00000000000..71fb6e8ca7c --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -0,0 +1,50 @@ +subscriptionService->setupEventStore(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionService->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $expectNoHandledEvents = fn () => self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectOneHandledEvent = fn () => self::assertEquals( + [ + SequenceNumber::fromInteger(1) + ], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); + + $expectNoHandledEvents(); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $expectOneHandledEvent(); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php new file mode 100644 index 00000000000..6ac8878dc00 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php @@ -0,0 +1,12 @@ + Date: Sat, 23 Nov 2024 16:39:08 +0100 Subject: [PATCH 053/142] TASK: Adjust subscription test exceptions to do no retry No retry is simpler at first and its unlikely that a projection will fix itself. --- .../Subscription/ProjectionErrorTest.php | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 2fa2c8fef48..3f6031a2062 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -38,48 +38,45 @@ public function projectionWithError() $this->commitExampleContentStreamEvent(); // catchup active tries to apply the commited event - $this->fakeProjection->expects(self::exactly(3))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( - new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), - new WillThrowException(new \Error('Something really wrong.')), - new WillThrowException(new \InvalidArgumentException('Dead.')), + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This projection is kaputt.') + ); + $expectedStatusForFailedProjection = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + projectionStatus: ProjectionStatus::ok(), ); - // TIME 1 + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), - ), + $expectedStatusForFailedProjection, $this->subscriptionStatus('Vendor.Package:FakeProjection') ); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - // TIME 2 - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - self::assertEquals($result->errors->first()->message, 'Something really wrong.'); - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); - - // TIME 3 - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - self::assertEquals($result->errors->first()->message, 'Dead.'); - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); + // todo test retry if reimplemented: https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/RetryStrategy/RetryStrategy.php + // // CatchUp 2 with retry + // $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + // self::assertTrue($result->hasFailed()); + // self::assertEquals($result->errors->first()->message, 'Something really wrong.'); + // self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); - // succeeding calls, nothing to do. + // no retry, nothing to do. $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); - // still dead - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Dead.'); + self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'This projection is kaputt.'); + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); } /** @test */ - public function fixProjectionWithError() + public function fixFailedProjection() { $this->subscriptionService->setupEventStore(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -104,7 +101,6 @@ public function fixProjectionWithError() projectionStatus: ProjectionStatus::ok(), ); - // TIME 1 $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); self::assertTrue($result->hasFailed()); @@ -245,7 +241,7 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( $this->subscriptionService->subscriptionEngine->setup(); $this->subscriptionService->subscriptionEngine->boot(); - $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $originalException = new \RuntimeException('This projection is kaputt.'), ); @@ -259,20 +255,25 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); self::assertSame($originalException, $handleException->getPrevious()); - // workspace is created - self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); + // workspace is created. The fake projection failed on the first event, but other projections succeed: $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); - $handleException = null; - try { - $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); - } catch (\RuntimeException $exception) { - $handleException = $exception; - } - self::assertNotNull($handleException); + // to exception thrown here because the failed projection is not retried and now in error state + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); - // workspace two is created. The fake projection is still dead and FAILS on the FIRST event, but the content graph gets only the new event: - self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); + // workspace two is created. $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2), SequenceNumber::fromInteger(3), SequenceNumber::fromInteger(4)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); } } From a32bfb28e4ee77d04c32cd7c8d0e118f092ba24f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 16:59:25 +0100 Subject: [PATCH 054/142] TASK: Remove `retry_attempt` from subscriptions https://github.com/neos/neos-development-collection/pull/5321#issuecomment-2492882147 > Anyways, I think with the removed retry strategy we should just get rid of the automatic retry altogether right now. It's quite unlikely that a retry suddenly works without other changes. So I'd be fully OK if it was only possible to manually retry failed subscriptions for 9.0 --- .../Engine/SubscriptionEngine.php | 36 +------------------ .../Classes/Subscription/Subscription.php | 6 +--- .../DoctrineSubscriptionStore.php | 4 --- 3 files changed, 2 insertions(+), 44 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index e2171e51f1e..6746bdf5147 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -48,7 +48,6 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result $this->subscriptionStore->setup(); $this->discoverNewSubscriptions(); - $this->retrySubscriptions($criteria); $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::NEW)); if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); @@ -125,8 +124,7 @@ private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domai } $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); $subscription->set( - position: $eventEnvelope->sequenceNumber, - retryAttempt: 0 + position: $eventEnvelope->sequenceNumber ); return null; } @@ -217,44 +215,12 @@ private function resetSubscription(Subscription $subscription): ?Error return null; } - private function retrySubscriptions(SubscriptionEngineCriteria $criteria): void - { - $this->subscriptionManager->findForUpdate( - SubscriptionCriteria::create($criteria->ids, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR])), - fn (Subscriptions $subscriptions) => $subscriptions->map($this->retrySubscription(...)), - ); - } - - private function retrySubscription(Subscription $subscription): void - { - if ($subscription->error === null) { - return; - } - $retryable = in_array( - $subscription->error->previousStatus, - [SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE], - true, - ); - if (!$retryable) { - return; - } - $subscription->set( - status: $subscription->error->previousStatus, - retryAttempt: $subscription->retryAttempt + 1, - ); - $subscription->error = null; - $this->subscriptionManager->update($subscription); - - $this->logger?->info(sprintf('Subscription Engine: Retry subscription "%s" (%d) and set back to %s.', $subscription->id->value, $subscription->retryAttempt, $subscription->status->value)); - } - private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatus $subscriptionStatus, \Closure $progressClosure = null): ProcessedResult { $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); $this->discoverNewSubscriptions(); $this->discoverDetachedSubscriptions($criteria); - $this->retrySubscriptions($criteria); return $this->subscriptionManager->findForUpdate( SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index ed3ffc13cad..52aca33c25f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -20,7 +20,6 @@ public function __construct( public SubscriptionStatus $status, public SequenceNumber $position, public SubscriptionError|null $error = null, - public int $retryAttempt = 0, public readonly \DateTimeImmutable|null $lastSavedAt = null, ) { } @@ -42,18 +41,15 @@ public static function createFromSubscriber(ProjectionSubscriber $subscriber): s */ public function set( SubscriptionStatus $status = null, - SequenceNumber $position = null, - int $retryAttempt = null, + SequenceNumber $position = null ): self { $this->status = $status ?? $this->status; $this->position = $position ?? $this->position; - $this->retryAttempt = $retryAttempt ?? $this->retryAttempt; return new self( $this->id, $status ?? $this->status, $position ?? $this->position, $this->error, - $retryAttempt ?? $this->retryAttempt, $this->lastSavedAt, ); } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 6841156f8d7..1d933b3aecf 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -46,7 +46,6 @@ public function setup(): void (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), (new Column('error_trace', Type::getType(Types::TEXT)))->setNotnull(false), - (new Column('retry_attempt', Type::getType(Types::INTEGER)))->setNotnull(true), (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), ]); $tableSchema->setPrimaryKey(['id']); @@ -128,7 +127,6 @@ private static function toDatabase(Subscription $subscription): array 'error_message' => $subscription->error?->errorMessage, 'error_previous_status' => $subscription->error?->previousStatus?->name, 'error_trace' => $subscription->error?->errorTrace, - 'retry_attempt' => $subscription->retryAttempt, ]; } @@ -148,7 +146,6 @@ private static function fromDatabase(array $row): Subscription assert(is_string($row['id'])); assert(is_string($row['status'])); assert(is_int($row['position'])); - assert(is_int($row['retry_attempt'])); assert(is_string($row['last_saved_at'])); $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); assert($lastSavedAt instanceof DateTimeImmutable); @@ -158,7 +155,6 @@ private static function fromDatabase(array $row): Subscription SubscriptionStatus::from($row['status']), SequenceNumber::fromInteger($row['position']), $subscriptionError, - $row['retry_attempt'], $lastSavedAt, ); } From 836c34788efeaafee68775aa89636b1155d549ad Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 17:21:39 +0100 Subject: [PATCH 055/142] TASK: Rename factory back to `$additionalSubscriberFactories` > there's a mismatch between variable name "additionalProjectionsFactory" and type "ContentRepositorySubsciberFactory" > the distinction between projection and subscription makes sense even if the only supported type of subscription target projections (for now) https://github.com/neos/neos-development-collection/pull/5375#pullrequestreview-2453314204 --- .../Classes/Factory/ContentRepositoryFactory.php | 4 ++-- .../Classes/ContentRepositoryRegistry.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index add804950ec..f914f0436e9 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -79,7 +79,7 @@ public function __construct( ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, private readonly CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, - private readonly ContentRepositorySubscriberFactories $additionalProjectionsFactories, + private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, LoggerInterface|null $logger = null, ) { $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); @@ -99,7 +99,7 @@ public function __construct( ); $subscribers = []; $additionalProjectionStates = []; - foreach ($this->additionalProjectionsFactories as $additionalSubscriberFactory) { + foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); $additionalProjectionStates[] = $subscriber->projection->getState(); $subscribers[] = $subscriber; diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index fceb98322ca..5ffe1c220b7 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -189,7 +189,7 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $this->buildContentGraphProjectionFactory($contentRepositoryId, $contentRepositorySettings), $this->buildContentGraphCatchUpHookFactory($contentRepositoryId, $contentRepositorySettings), $this->buildCommandHooksFactory($contentRepositoryId, $contentRepositorySettings), - $this->buildAdditionalProjectionsFactories($contentRepositoryId, $contentRepositorySettings), + $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), $this->logger, ); } catch (\Exception $exception) { @@ -315,7 +315,7 @@ private function buildCommandHooksFactory(ContentRepositoryId $contentRepository } /** @param array $contentRepositorySettings */ - private function buildAdditionalProjectionsFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories + private function buildAdditionalSubscribersFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories { if (!is_array($contentRepositorySettings['projections'] ?? [])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); From 830b5cc15176ff6d3083a48878a3bfb23819dd2c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 19:36:19 +0100 Subject: [PATCH 056/142] BUGFIX: `discoverDetachedSubscriptions` did not persist changes which lead to it being invoked on aktive > Subscriber "Vendor.Package:FakeProjection" could not invoke onBeforeCatchUp: Subscriber with the subscription id "Vendor.Package:FakeProjection" not found. --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 6746bdf5147..0defedaeff8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -168,6 +168,8 @@ private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $crite $this->subscriptionManager->update($subscription); $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); } + // todo use transaction here for discovery as well!!! + $this->subscriptionManager->flush(); } From 6f43825c3909893c1a907764b02a69506bf87767 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 19:34:11 +0100 Subject: [PATCH 057/142] TASK: Fix phpstan --- .../Classes/TestSuite/DebugEventProjection.php | 4 ++++ .../Classes/Subscription/Engine/Errors.php | 4 ++-- .../Classes/Subscription/Exception/CatchUpFailed.php | 1 + .../Classes/Fakes/FakeCatchUpHookFactory.php | 4 ++++ .../Classes/Fakes/FakeProjectionFactory.php | 8 ++++++++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index fa1ae446d43..018ebb6eeba 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -23,6 +23,7 @@ * * TODO check also that order of inserted sequence numbers is correct and no holes * + * @implements ProjectionInterface * @internal * @Flow\Proxy(false) */ @@ -55,6 +56,9 @@ public function status(): ProjectionStatus return ProjectionStatus::ok(); } + /** + * @return array + */ private function determineRequiredSqlStatements(): array { $schemaManager = $this->dbal->createSchemaManager(); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index 715a087168e..bf7dd3668b9 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -18,10 +18,10 @@ private function __construct( Error ...$errors ) { - $this->errors = $errors; - if ($this->errors === []) { + if ($errors === []) { throw new \InvalidArgumentException('Errors must not be empty.', 1731612542); } + $this->errors = $errors; } /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php index 6ac8878dc00..c18aaf9c3ba 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php @@ -6,6 +6,7 @@ /** * Only thrown if there is no way to recover the started catchup. The transaction will be rolled back. + * @api */ final class CatchUpFailed extends \RuntimeException { diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php index 23e7b1e9241..a20650ae93e 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php @@ -10,10 +10,14 @@ use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** + * @implements CatchUpHookFactoryInterface * @internal helper to configure custom catchup hook mocks for testing */ final class FakeCatchUpHookFactory implements CatchUpHookFactoryInterface { + /** + * @var array + */ private static array $catchupHooks; public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php index 62366fa7691..d0e21e2ec5a 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php @@ -7,12 +7,17 @@ use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** + * @implements ProjectionFactoryInterface>> * @internal helper to configure custom projection mocks for testing */ final class FakeProjectionFactory implements ProjectionFactoryInterface { + /** + * @var array> + */ private static array $projections; public function build( @@ -22,6 +27,9 @@ public function build( return static::$projections[$options['instanceId']] ?? throw new \RuntimeException('No projection defined for Fake.'); } + /** + * @param ProjectionInterface $projection + */ public static function setProjection(string $instanceId, ProjectionInterface $projection): void { self::$projections[$instanceId] = $projection; From 60d4f8ca431a580f48f3c2df4c7e551e8586b35e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 19:33:26 +0100 Subject: [PATCH 058/142] BUGFIX: Reintroduce catchup hooks for all projections Also allows the `Vendor.Package:FakeCatchupHook` to be picked up for testing --- .../Factory/ContentRepositoryFactory.php | 6 ++--- .../Factory/ProjectionSubscriberFactory.php | 17 ++++++++++-- .../CatchUpHook/CatchUpHookFactories.php | 5 ++++ .../Classes/ContentRepositoryRegistry.php | 27 ++++++++++++------- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index f914f0436e9..ea4593a6a70 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -65,7 +65,7 @@ final class ContentRepositoryFactory private ?ContentRepository $contentRepositoryRuntimeCache = null; /** - * @param CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory + * @param CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory */ public function __construct( private readonly ContentRepositoryId $contentRepositoryId, @@ -77,7 +77,7 @@ public function __construct( private readonly ClockInterface $clock, SubscriptionStoreInterface $subscriptionStore, ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, - private readonly CatchUpHookFactoryInterface $contentGraphCatchUpHookFactory, + private readonly CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, LoggerInterface|null $logger = null, @@ -115,7 +115,7 @@ private function buildContentGraphSubscriber(): ProjectionSubscriber return new ProjectionSubscriber( SubscriptionId::fromString('contentGraph'), $this->contentGraphProjection, - $this->contentGraphCatchUpHookFactory->build(CatchUpHookFactoryDependencies::create( + $this->contentGraphCatchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( $this->contentRepositoryId, $this->contentGraphProjection->getState(), $this->subscriberFactoryDependencies->nodeTypeManager, diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php index f763dfb9328..1970b085565 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -14,6 +14,8 @@ namespace Neos\ContentRepository\Core\Factory; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; @@ -27,21 +29,32 @@ { /** * @param ProjectionFactoryInterface> $projectionFactory + * @param CatchUpHookFactoryInterface|null $catchUpHookFactory * @param array $projectionFactoryOptions */ public function __construct( private SubscriptionId $subscriptionId, private ProjectionFactoryInterface $projectionFactory, + private ?CatchUpHookFactoryInterface $catchUpHookFactory, private array $projectionFactoryOptions, ) { } public function build(SubscriberFactoryDependencies $dependencies): ProjectionSubscriber { + $projection = $this->projectionFactory->build($dependencies, $this->projectionFactoryOptions); + $catchUpHook = $this->catchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( + $dependencies->contentRepositoryId, + $projection->getState(), + $dependencies->nodeTypeManager, + $dependencies->contentDimensionSource, + $dependencies->interDimensionalVariationGraph, + )); + return new ProjectionSubscriber( $this->subscriptionId, - $this->projectionFactory->build($dependencies, $this->projectionFactoryOptions), - null + $projection, + $catchUpHook, ); } } diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php index e383b32299f..22f2621e2a2 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php @@ -51,6 +51,11 @@ private function has(string $catchUpHookFactoryClassName): bool return array_key_exists($catchUpHookFactoryClassName, $this->catchUpHookFactories); } + public function isEmpty(): bool + { + return $this->catchUpHookFactories === []; + } + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface { $catchUpHooks = array_map(static fn(CatchUpHookFactoryInterface $catchUpHookFactory) => $catchUpHookFactory->build($dependencies), $this->catchUpHookFactories); diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 1f073833b21..4fdec55cd73 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -21,6 +21,7 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryIds; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; @@ -57,6 +58,9 @@ final class ContentRepositoryRegistry */ private array $factoryInstances = []; + /** + * @var array + */ private array $settings; #[Flow\Inject(name: 'Neos.ContentRepositoryRegistry:Logger', lazy: false)] @@ -183,6 +187,8 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content unset($contentRepositorySettings['preset']); } try { + /** @var CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory */ + $contentGraphCatchUpHookFactory = $this->buildCatchUpHookFactory($contentRepositoryId, 'contentGraph', $contentRepositorySettings['contentGraphProjection']); $clock = $this->buildClock($contentRepositoryId, $contentRepositorySettings); return new ContentRepositoryFactory( $contentRepositoryId, @@ -194,7 +200,7 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $clock, $this->buildSubscriptionStore($contentRepositoryId, $clock, $contentRepositorySettings), $this->buildContentGraphProjectionFactory($contentRepositoryId, $contentRepositorySettings), - $this->buildContentGraphCatchUpHookFactory($contentRepositoryId, $contentRepositorySettings), + $contentGraphCatchUpHookFactory, $this->buildCommandHooksFactory($contentRepositoryId, $contentRepositorySettings), $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), $this->logger, @@ -275,27 +281,29 @@ private function buildContentGraphProjectionFactory(ContentRepositoryId $content } /** - * @param array $contentRepositorySettings - * @return CatchUpHookFactoryInterface + * @param array $projectionOptions + * @return CatchUpHookFactoryInterface|null */ - private function buildContentGraphCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): CatchUpHookFactoryInterface + private function buildCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, string $projectionName, array $projectionOptions): ?CatchUpHookFactoryInterface { - if (!isset($contentRepositorySettings['contentGraphProjection']['catchUpHooks'])) { - throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.catchUpHooks configured.', $contentRepositoryId->value); + if (!isset($projectionOptions['catchUpHooks'])) { + return null; } $catchUpHookFactories = CatchUpHookFactories::create(); - foreach ($contentRepositorySettings['contentGraphProjection']['catchUpHooks'] as $catchUpHookName => $catchUpHookOptions) { + foreach ($projectionOptions['catchUpHooks'] as $catchUpHookName => $catchUpHookOptions) { if ($catchUpHookOptions === null) { // Allow catch up hooks to be disabled by setting their configuration to `null` continue; } $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { - throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for content graph CatchUpHook "%s" (content repository "%s") is not an instance of %s but %s', $catchUpHookName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); + throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for hook "%s" in projection "%s" (content repository "%s") is not an instance of %s but %s', $catchUpHookName, $projectionName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); } $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); } - /** @var CatchUpHookFactoryInterface $catchUpHookFactories */ + if ($catchUpHookFactories->isEmpty()) { + return null; + } return $catchUpHookFactories; } @@ -344,6 +352,7 @@ private function buildAdditionalSubscribersFactories(ContentRepositoryId $conten $projectionSubscriberFactories[$projectionName] = new ProjectionSubscriberFactory( SubscriptionId::fromString($projectionName), $projectionFactory, + $this->buildCatchUpHookFactory($contentRepositoryId, $projectionName, $projectionOptions), $projectionOptions['options'] ?? [], ); } From 5ef4ab1a39d885f5dc7a176c469975285da3a620 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:20:36 +0100 Subject: [PATCH 059/142] TASK: Adjust to doctrine deprecations in DoctrineSubscriptionStore --- .../DoctrineSubscriptionStore.php | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 1d933b3aecf..bfa0bf4e041 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -5,6 +5,7 @@ namespace Neos\ContentRepositoryRegistry\Factory\SubscriptionStore; use DateTimeImmutable; +use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\Column; @@ -34,7 +35,7 @@ public function __construct( public function setup(): void { - $schemaConfig = $this->dbal->getSchemaManager()->createSchemaConfig(); + $schemaConfig = $this->dbal->createSchemaManager()->createSchemaConfig(); assert($schemaConfig !== null); $schemaConfig->setDefaultTableOptions([ 'charset' => 'utf8mb4' @@ -72,7 +73,7 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions ->setParameter( 'ids', $criteria->ids->toStringArray(), - Connection::PARAM_STR_ARRAY, + ArrayParameterType::STRING, ); } if (!$criteria->status->isEmpty()) { @@ -80,7 +81,7 @@ public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions ->setParameter( 'status', $criteria->status->toStringArray(), - Connection::PARAM_STR_ARRAY, + ArrayParameterType::STRING, ); } $result = $queryBuilder->executeQuery(); @@ -136,17 +137,10 @@ private static function toDatabase(Subscription $subscription): array private static function fromDatabase(array $row): Subscription { if (isset($row['error_message'])) { - assert(is_string($row['error_message'])); - assert(!isset($row['error_previous_status']) || is_string($row['error_previous_status'])); - assert(is_string($row['error_trace'])); $subscriptionError = new SubscriptionError($row['error_message'], SubscriptionStatus::from($row['error_previous_status']), $row['error_trace']); } else { $subscriptionError = null; } - assert(is_string($row['id'])); - assert(is_string($row['status'])); - assert(is_int($row['position'])); - assert(is_string($row['last_saved_at'])); $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); assert($lastSavedAt instanceof DateTimeImmutable); From c7cb75b93b33aff3413992e4040e5bdb2f2b66ff Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:44:35 +0100 Subject: [PATCH 060/142] TASK: Minor code adjustments --- .../Factory/ContentRepositorySubscriberFactories.php | 2 +- .../Classes/Factory/ProjectionSubscriberFactory.php | 2 +- .../Classes/Subscription/Engine/Result.php | 2 +- .../Classes/Subscription/Engine/SubscriptionEngine.php | 4 ++-- .../Classes/Subscription/Engine/SubscriptionManager.php | 7 +------ .../Subscription/Subscriber/ProjectionSubscriber.php | 2 +- .../Classes/Subscription/Subscriber/Subscribers.php | 2 +- .../Classes/Subscription/Subscription.php | 9 +-------- .../Behavior/Features/Bootstrap/CRTestSuiteTrait.php | 3 --- .../SubscriptionStore/DoctrineSubscriptionStore.php | 5 ++++- 10 files changed, 13 insertions(+), 25 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php index 82a797e4745..399573c23a8 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @internal + * @internal only API for custom content repository integrations */ final class ContentRepositorySubscriberFactories implements \IteratorAggregate { diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php index 1970b085565..203936237e3 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -23,7 +23,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionId; /** - * @internal + * @internal only API for custom content repository integrations */ final readonly class ProjectionSubscriberFactory { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php index 62aaf947553..dc8da2ef08b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php @@ -10,7 +10,7 @@ final readonly class Result { private function __construct( - public readonly Errors|null $errors, + public Errors|null $errors, ) { } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 0defedaeff8..0e45ed0382c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -137,7 +137,7 @@ private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domai */ private function discoverNewSubscriptions(): void { - $this->subscriptionManager->findForUpdate( + $this->subscriptionManager->findForAndUpdate( SubscriptionCriteria::noConstraints(), function (Subscriptions $subscriptions) { foreach ($this->subscribers as $subscriber) { @@ -224,7 +224,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->discoverNewSubscriptions(); $this->discoverDetachedSubscriptions($criteria); - return $this->subscriptionManager->findForUpdate( + return $this->subscriptionManager->findForAndUpdate( SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosure) { if ($subscriptions->isEmpty()) { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php index 817065843f0..4fd33df2388 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -30,7 +30,7 @@ public function __construct( * @param \Closure(Subscriptions):T $closure * @return T */ - public function findForUpdate(SubscriptionCriteria $criteria, \Closure $closure): mixed + public function findForAndUpdate(SubscriptionCriteria $criteria, \Closure $closure): mixed { return $this->subscriptionStore->transactional( /** @return T */ @@ -44,11 +44,6 @@ function () use ($closure, $criteria): mixed { ); } - public function find(SubscriptionCriteria $criteria): Subscriptions - { - return $this->subscriptionStore->findByCriteria($criteria); - } - public function add(Subscription $subscription): void { $this->forAdd->attach($subscription); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php index 03d80cce95a..117b52265bc 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php @@ -13,7 +13,7 @@ use Neos\EventStore\Model\EventEnvelope; /** - * @api + * @internal */ final class ProjectionSubscriber { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php index 3cb3529ca3e..ba40fbddb1a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php @@ -8,7 +8,7 @@ /** * @implements \IteratorAggregate - * @api + * @internal */ final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 52aca33c25f..16b35fedbbd 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -42,16 +42,9 @@ public static function createFromSubscriber(ProjectionSubscriber $subscriber): s public function set( SubscriptionStatus $status = null, SequenceNumber $position = null - ): self { + ): void { $this->status = $status ?? $this->status; $this->position = $position ?? $this->position; - return new self( - $this->id, - $status ?? $this->status, - $position ?? $this->position, - $this->error, - $this->lastSavedAt, - ); } /** diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 33e318943e1..f65d0de69fc 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -22,7 +22,6 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\NodeModification\Dto\PropertyValuesToWrite; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindSubtreeFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\NodeType\NodeTypeCriteria; @@ -33,7 +32,6 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\ContentStreamClosing; -use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCopying; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCreation; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeModification; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeMove; @@ -146,7 +144,6 @@ public function iExpectTheGraphProjectionToConsistOfExactlyNodes(int $expectedNu public ContentGraphReadModelInterface|null $instance; public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - // TODO find replacement – is that needed at all? $this->instance = $serviceFactoryDependencies->contentGraphReadModel; return new class implements ContentRepositoryServiceInterface { diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index bfa0bf4e041..214fd7fcd19 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -23,7 +23,11 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Psr\Clock\ClockInterface; +use Neos\Flow\Annotations as Flow; +/** + * @Flow\Proxy(false) + */ final class DoctrineSubscriptionStore implements SubscriptionStoreInterface { public function __construct( @@ -36,7 +40,6 @@ public function __construct( public function setup(): void { $schemaConfig = $this->dbal->createSchemaManager()->createSchemaConfig(); - assert($schemaConfig !== null); $schemaConfig->setDefaultTableOptions([ 'charset' => 'utf8mb4' ]); From a80639b2260cdd8291c2f0b900609b3d10504d14 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:45:04 +0100 Subject: [PATCH 061/142] TASK: Ensure that the content graph projection is not part of the generic `ProjectionStates` --- .../Classes/ContentRepository.php | 6 +---- .../Classes/Projection/ProjectionStates.php | 25 ++++++++----------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 95d29db1dd3..460cc969898 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -140,11 +140,7 @@ public function handle(CommandInterface $command): void */ public function projectionState(string $projectionStateClassName): ProjectionStateInterface { - try { - return $this->projectionStates->get($projectionStateClassName); - } catch (\InvalidArgumentException $e) { - throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository instance: %s', $projectionStateClassName, $e->getMessage()), 1662033650, $e); - } + return $this->projectionStates->get($projectionStateClassName); } /** diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php index ba04da0e7a6..81b6d82a0eb 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php @@ -4,19 +4,20 @@ namespace Neos\ContentRepository\Core\Projection; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; + /** * Collection of all states (aka read models) of all projections for a Content Repository * * @internal - * @implements \IteratorAggregate */ -final readonly class ProjectionStates implements \IteratorAggregate, \Countable +final readonly class ProjectionStates { /** * @param array, ProjectionStateInterface> $statesByClassName */ private function __construct( - public array $statesByClassName, + private array $statesByClassName, ) { } @@ -35,6 +36,9 @@ public static function fromArray(array $states): self if (!$state instanceof ProjectionStateInterface) { throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionStateInterface::class, get_debug_type($state)), 1729687661); } + if ($state instanceof ContentGraphReadModelInterface) { + throw new \InvalidArgumentException(sprintf('The content graph state (%s) must not be part of the additional projection states', ContentGraphReadModelInterface::class), 1732390657); + } if (array_key_exists($state::class, $statesByClassName)) { throw new \InvalidArgumentException(sprintf('An instance of %s is already part of the set', $state::class), 1729687716); } @@ -53,21 +57,14 @@ public static function fromArray(array $states): self */ public function get(string $className): ProjectionStateInterface { + if ($className === ContentGraphReadModelInterface::class) { + throw new \InvalidArgumentException(sprintf('Accessing the content repository projection state (%s) via is not allowed. Please use the API on the content repository instead.', ContentGraphReadModelInterface::class), 1732390824); + } if (!array_key_exists($className, $this->statesByClassName)) { - throw new \InvalidArgumentException(sprintf('The state class "%s" does not exist.', $className), 1729687836); + throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository.', $className), 1662033650); } /** @var T $state */ $state = $this->statesByClassName[$className]; return $state; } - - public function getIterator(): \Traversable - { - return new \ArrayIterator($this->statesByClassName); - } - - public function count(): int - { - return count($this->statesByClassName); - } } From e7acfaa50b778e57685075cea81b272dbec39f0b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:53:00 +0100 Subject: [PATCH 062/142] BUGFIX: ProjectionErrorTest::fixFailedProjection reset error on reset --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 5 +---- .../Classes/Subscription/Subscription.php | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 0e45ed0382c..c4e2181ea99 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -208,10 +208,7 @@ private function resetSubscription(Subscription $subscription): ?Error $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); return Error::fromSubscriptionIdAndException($subscription->id, $e); } - $subscription->set( - status: SubscriptionStatus::BOOTING, - position: SequenceNumber::none(), - ); + $subscription->reset(); $this->subscriptionManager->update($subscription); $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the resetState method has been executed.', $subscriber::class, $subscription->id->value)); return null; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 16b35fedbbd..556a29ab89d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -47,6 +47,13 @@ public function set( $this->position = $position ?? $this->position; } + public function reset(): void + { + $this->status = SubscriptionStatus::BOOTING; + $this->position = SequenceNumber::none(); + $this->error = null; + } + /** * @internal Only the {@see SubscriptionEngine} is supposed to mutate subscriptions */ From fdeec75529880a91d0386acd92e1787ea97ce403 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:56:39 +0100 Subject: [PATCH 063/142] TASK: Assertions that setup and boot do not retry failed projections Readded assertions as retry was removed for now ... and should probably NOT do anything if setup is called! --- .../Subscription/ProjectionErrorTest.php | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 3f6031a2062..6565dda492a 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -12,6 +12,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\Error; use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; +use Neos\ContentRepository\Core\Subscription\Engine\Result; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -109,18 +110,20 @@ public function fixFailedProjection() $this->subscriptionStatus('Vendor.Package:FakeProjection') ); - // todo BOOT and SETUP should not attempt to retry?! // setup does not change anything - // $result = $this->subscriptionService->subscriptionEngine->setup(); - // self::assertNull($result->errors); + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertEquals(Result::success(), $result); + // nor catchup active + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); // boot neither - // $result = $this->subscriptionService->subscriptionEngine->boot(); - // self::assertNull($result->errors); + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); // still the same state - // self::assertEquals( - // $expectedFailure, - // $this->subscriptionStatus('Vendor.Package:FakeProjection') - // ); + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); $this->fakeProjection->expects(self::once())->method('resetState'); From 31913f53c1ce4c1400ef5580021be3020f4e9864 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:59:00 +0100 Subject: [PATCH 064/142] BUGFIX: SubscriptionDetachedStatusTest::projectionIsDetachedIfConfigurationIsRemoved > InvalidArgumentException: Subscriber with the subscription id "Vendor.Package:FakeProjection" not found. The subscriber is detached, so the state is not calculate-able at this point! --- .../Subscription/SubscriptionDetachedStatusTest.php | 2 +- .../Classes/Subscription/Engine/SubscriptionEngine.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 11d67592ff5..55c1e8825d9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -59,7 +59,7 @@ public function projectionIsDetachedIfConfigurationIsRemoved() subscriptionStatus: SubscriptionStatus::DETACHED, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: null // no calculate-able at this point! + projectionStatus: null // not calculate-able at this point! ), $this->subscriptionStatus('Vendor.Package:FakeProjection') ); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index c4e2181ea99..2cd1eda4cbc 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -99,13 +99,13 @@ public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null) { $statuses = []; foreach ($this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()) as $subscription) { - $subscriber = $this->subscribers->get($subscription->id); + $subscriber = $this->subscribers->contain($subscription->id) ? $this->subscribers->get($subscription->id) : null; $statuses[] = SubscriptionAndProjectionStatus::create( subscriptionId: $subscription->id, subscriptionStatus: $subscription->status, subscriptionPosition: $subscription->position, subscriptionError: $subscription->error, - projectionStatus: $subscriber->projection->status(), + projectionStatus: $subscriber?->projection->status(), ); } return SubscriptionAndProjectionStatuses::fromArray($statuses); From c5ea7577e08f3aa07f8192b897860ae244400e32 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:02:41 +0100 Subject: [PATCH 065/142] TASK: Throw `CatchUpFailed` exception in case onBeforeCatchUp or onAfterCatchUp failes as we consider it a critical developer error For `onBeforeCatchUp` we could probably wrap a savepoint and roll it back and skip also the projection, but errors in `onAfterCatchUp` would then analog also need to rollback only the one projection where it was registered. This is not possible and too complex. see \Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription\CatchUpHookErrorTest::error_onBeforeCatchUp_abortsCatchup --- .../Subscription/CatchUpHookErrorTest.php | 4 ++-- .../Engine/SubscriptionEngine.php | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 98520214fe9..63b1bbc6806 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -144,7 +144,7 @@ public function error_onBeforeCatchUp_abortsCatchup() } self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); - self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" could not invoke onBeforeCatchUp: This catchup hook is kaputt.'); + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onBeforeCatchUp: This catchup hook is kaputt.'); // still the initial status $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -188,7 +188,7 @@ public function error_onAfterCatchUp_abortsCatchupAndRollBack() } self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); - self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" could not invoke onAfterCatchUp: This catchup hook is kaputt.'); + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onAfterCatchUp: This catchup hook is kaputt.'); // still the initial status $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 2cd1eda4cbc..fb25a55d2f8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -6,6 +6,7 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; @@ -230,7 +231,14 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu return ProcessedResult::success(0); } foreach ($subscriptions as $subscription) { - $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); + try { + $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); + } } $startSequenceNumber = $subscriptions->lowestPosition()?->next() ?? SequenceNumber::none(); $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); @@ -271,7 +279,14 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu } } foreach ($subscriptions as $subscription) { - $this->subscribers->get($subscription->id)->onAfterCatchUp(); + try { + $this->subscribers->get($subscription->id)->onAfterCatchUp(); + } catch (\Throwable $e) { + // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); + } if ($subscription->status !== $subscriptionStatus) { continue; } From 794eaf22e516a4dd2ac56acae23bf4f45c4eb3c0 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:11:07 +0100 Subject: [PATCH 066/142] TASK: Use save points to rollback projections during transaction on failure https://neos-project.slack.com/archives/C04PYL8H3/p1732318989845619 We dont want to rollback the main transaction, as other projections still need to be processed, the previously working events need to be applied, and we want to set the ERROR state of the projection --- .../Subscription/Engine/SubscriptionEngine.php | 3 +++ .../Store/SubscriptionStoreInterface.php | 6 ++++++ .../DoctrineSubscriptionStore.php | 15 +++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index fb25a55d2f8..5582056ab90 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -265,10 +265,13 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); continue; } + $this->subscriptionStore->createSavepoint(); $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription); if (!$error) { + $this->subscriptionStore->releaseSavepoint(); continue; } + $this->subscriptionStore->rollbackSavepoint(); $errors[] = $error; } $numberOfProcessedEvents++; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 83aab8a185c..4b6a827f8e6 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -26,4 +26,10 @@ public function update(Subscription $subscription): void; * @return T */ public function transactional(\Closure $closure): mixed; + + public function createSavepoint(): void; + + public function releaseSavepoint(): void; + + public function rollbackSavepoint(): void; } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 214fd7fcd19..234f1b44838 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -160,4 +160,19 @@ public function transactional(\Closure $closure): mixed { return $this->dbal->transactional($closure); } + + public function createSavepoint(): void + { + $this->dbal->createSavepoint('SUBSCRIBER'); + } + + public function releaseSavepoint(): void + { + $this->dbal->releaseSavepoint('SUBSCRIBER'); + } + + public function rollbackSavepoint(): void + { + $this->dbal->rollbackSavepoint('SUBSCRIBER'); + } } From 7852f61456d20969e818a43c94f544765e353406 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:22:23 +0100 Subject: [PATCH 067/142] TASK: Handle `TableNotFoundException` gracefully in `subscriptionStatuses` --- .../SubscriptionGetStatusTest.php | 43 +++++-------------- .../SubscriptionNewStatusTest.php | 39 +++++++++++++++++ .../Engine/SubscriptionEngine.php | 9 +++- .../SubscriptionAndProjectionStatuses.php | 5 +++ 4 files changed, 63 insertions(+), 33 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index 72e355d6226..84cc079a451 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -5,12 +5,6 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; -use Neos\ContentRepository\Core\Subscription\SubscriptionId; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; -use Neos\EventStore\Model\Event\SequenceNumber; final class SubscriptionGetStatusTest extends AbstractSubscriptionEngineTestCase { @@ -24,34 +18,19 @@ public function statusOnEmptyDatabase() keepSchema: false ); - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + $this->fakeProjection->expects(self::never())->method('status'); $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + self::assertTrue($actualStatuses->isEmpty()); - $expected = SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('contentGraph'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired(''), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - ]); - - self::assertEquals($expected, $actualStatuses); + self::assertNull( + $this->subscriptionStatus('contentGraph') + ); + self::assertNull( + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + self::assertNull( + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index f428d764054..db27649e83c 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -8,7 +8,9 @@ use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; @@ -93,4 +95,41 @@ public function newProjectionIsFoundConfigurationIsAdded() self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } + + /** @test */ + public function newProjectionsAreFoundViaStatus() + { + // only setup content graph so that the other projections are NEW, but still found + $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + + $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + + $expected = SubscriptionAndProjectionStatuses::fromArray([ + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 5582056ab90..4614244f947 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -4,6 +4,7 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; +use Doctrine\DBAL\Exception\TableNotFoundException; use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; @@ -99,7 +100,13 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null): SubscriptionAndProjectionStatuses { $statuses = []; - foreach ($this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()) as $subscription) { + try { + $subscriptions = $this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()); + } catch (TableNotFoundException) { + // the schema is not setup - thus there are no subscribers + return SubscriptionAndProjectionStatuses::createEmpty(); + } + foreach ($subscriptions as $subscription) { $subscriber = $this->subscribers->contain($subscription->id) ? $this->subscribers->get($subscription->id) : null; $statuses[] = SubscriptionAndProjectionStatus::create( subscriptionId: $subscription->id, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php index 4fe2fb5e3bb..47c237813b5 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php @@ -21,6 +21,11 @@ private function __construct( $this->statuses = $statuses; } + public static function createEmpty(): self + { + return new self(); + } + /** * @param array $statuses */ From d5827666b076619eece1ac20ce290e1e49fd7db0 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:22:23 +0100 Subject: [PATCH 068/142] TASK: Move back to `subscriptionStatuses` test. We need to make sure the tables are dropped --- .../AbstractSubscriptionEngineTestCase.php | 2 +- .../SubscriptionGetStatusTest.php | 47 ++++++++++++++++++- .../SubscriptionNewStatusTest.php | 37 --------------- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index ec1284009c2..1602b362781 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -72,7 +72,7 @@ public function setUp(): void ); $this->secondFakeProjection = new DebugEventProjection( - 'cr_t_subscription_debug_projection', + sprintf('cr_%s_debug_projection', $contentRepositoryId->value), $this->getObject(Connection::class) ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index 84cc079a451..594c7264e92 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -5,6 +5,13 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Doctrine\DBAL\Connection; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; +use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; +use Neos\EventStore\Model\Event\SequenceNumber; final class SubscriptionGetStatusTest extends AbstractSubscriptionEngineTestCase { @@ -18,8 +25,6 @@ public function statusOnEmptyDatabase() keepSchema: false ); - $this->fakeProjection->expects(self::never())->method('status'); - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); self::assertTrue($actualStatuses->isEmpty()); @@ -32,5 +37,43 @@ public function statusOnEmptyDatabase() self::assertNull( $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); + + // + // setup and fetch status + // + + // only setup content graph so that the other projections are NEW, but still found + $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + + $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + + $expected = SubscriptionAndProjectionStatuses::fromArray([ + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok(), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), + ), + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), + ), + ]); + + self::assertEquals($expected, $actualStatuses); } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index db27649e83c..38e4a477d0c 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -95,41 +95,4 @@ public function newProjectionIsFoundConfigurationIsAdded() self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } - - /** @test */ - public function newProjectionsAreFoundViaStatus() - { - // only setup content graph so that the other projections are NEW, but still found - $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); - - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); - - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); - - $expected = SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('contentGraph'), - subscriptionStatus: SubscriptionStatus::BOOTING, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), - ), - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), - ), - ]); - - self::assertEquals($expected, $actualStatuses); - } } From 5dfa592a6d8d2b20847056b2da717d5cd98a7a09 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:43:16 +0100 Subject: [PATCH 069/142] TASK: Inline `discoverDetachedSubscriptions` To reduce additional sql query and lock, and do it in the main transaction --- .../SubscriptionDetachedStatusTest.php | 40 +++++++++++++++---- .../Engine/SubscriptionEngine.php | 33 +++++---------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 55c1e8825d9..8528517da00 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -45,22 +45,46 @@ public function projectionIsDetachedIfConfigurationIsRemoved() $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); $this->setupContentRepositoryDependencies($this->contentRepository->id); - // todo status is stale??, should be DETACHED + // todo status is stale??, should be DETACHED, and also cr:setup should marke detached projections?!! // $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->fakeProjection->expects(self::never())->method('apply'); // catchup or anything that finds detached subscribers $result = $this->subscriptionEngine->catchUpActive(); + // todo result should reflect that there was an detachment? Throw error in CR? self::assertEquals(ProcessedResult::success(1), $result); + $expectedDetachedState = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: null // not calculate-able at this point! + ); + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // other projections are not interrupted + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // succeeding catchup's do not handle the detached one: + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // still detached self::assertEquals( - SubscriptionAndProjectionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::DETACHED, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - projectionStatus: null // not calculate-able at this point! - ), + $expectedDetachedState, $this->subscriptionStatus('Vendor.Package:FakeProjection') ); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 4614244f947..724f2404288 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -160,27 +160,6 @@ function (Subscriptions $subscriptions) { ); } - private function discoverDetachedSubscriptions(SubscriptionEngineCriteria $criteria): void - { - $registeredSubscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create( - $criteria->ids, - SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), - )); - foreach ($registeredSubscriptions as $subscription) { - if ($this->subscribers->contain($subscription->id)) { - continue; - } - $subscription->set( - status: SubscriptionStatus::DETACHED, - ); - $this->subscriptionManager->update($subscription); - $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); - } - // todo use transaction here for discovery as well!!! - $this->subscriptionManager->flush(); - } - - /** * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned @@ -227,11 +206,21 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); $this->discoverNewSubscriptions(); - $this->discoverDetachedSubscriptions($criteria); return $this->subscriptionManager->findForAndUpdate( SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosure) { + foreach ($subscriptions as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $subscription->set( + status: SubscriptionStatus::DETACHED, + ); + $this->subscriptionManager->update($subscription); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptions = $subscriptions->without($subscription->id); + } + } if ($subscriptions->isEmpty()) { $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); From d5715c7a7c6f5b0b4f75ff5e85b48f8ee82665fb Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:46:31 +0100 Subject: [PATCH 070/142] TASK: Do not discover new subscriptions during catchup We do not expect any changes during runtime. Setup and status should handle this case. --- .../Functional/Subscription/SubscriptionNewStatusTest.php | 5 ++--- .../Classes/Subscription/Engine/SubscriptionEngine.php | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index 38e4a477d0c..6e46704b0fb 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -10,7 +10,6 @@ use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; @@ -69,8 +68,8 @@ public function newProjectionIsFoundConfigurationIsAdded() // todo status doesnt find this projection yet? self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); - // do something that finds new subscriptions - $result = $this->subscriptionEngine->catchUpActive(); + // do something that finds new subscriptions, trigger a setup on a specific projection: + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); self::assertNull($result->errors); self::assertEquals( diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 724f2404288..72d5684a5f2 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -205,8 +205,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs { $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); - $this->discoverNewSubscriptions(); - return $this->subscriptionManager->findForAndUpdate( SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosure) { From 37a4e47b1642a2a6276eb7b8081fbf9901db9c94 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 10:20:44 +0100 Subject: [PATCH 071/142] TASK: Introduce further tests to assert behaviour for catchup and setup and failing test for setupIsInvokedForPreviouslyActiveSubscribers --- .../TestSuite/DebugEventProjection.php | 20 +++- .../SubscriptionActiveStatusTest.php | 33 +++++++ .../Subscription/SubscriptionSetupTest.php | 94 +++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 018ebb6eeba..7f71796e96d 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -33,6 +33,11 @@ final class DebugEventProjection implements ProjectionInterface private \Closure|null $saboteur = null; + /** + * @var array + */ + private array $additionalColumnsForSchema = []; + public function __construct( private string $tableNamePrefix, private Connection $dbal @@ -66,7 +71,8 @@ private function determineRequiredSqlStatements(): array $table = new Table($this->tableNamePrefix, [ (new Column('sequencenumber', Type::getType(Types::INTEGER))), (new Column('stream', Type::getType(Types::STRING))), - (new Column('type', Type::getType(Types::STRING))) + (new Column('type', Type::getType(Types::STRING))), + ...$this->additionalColumnsForSchema ]); $table->setPrimaryKey([ @@ -81,7 +87,7 @@ private function determineRequiredSqlStatements(): array public function resetState(): void { - $this->dbal->exec('TRUNCATE ' . $this->tableNamePrefix); + $this->dbal->executeStatement('TRUNCATE ' . $this->tableNamePrefix); } public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void @@ -114,4 +120,14 @@ public function killSaboteur(): void { $this->saboteur = null; } + + public function schemaNeedsAdditionalColumn(string $name): void + { + $this->additionalColumnsForSchema[$name] = (new Column($name, Type::getType(Types::STRING)))->setNotnull(false); + } + + public function dropTables(): void + { + $this->dbal->executeStatement('DROP TABLE ' . $this->tableNamePrefix); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php index ad5c4b99656..2585852b998 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -81,4 +81,37 @@ public function filteringCatchUpActive() $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); } + + /** @test */ + public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() + { + $this->subscriptionService->setupEventStore(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionService->subscriptionEngine->setup(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // empty catchup must keep the sequence numbers of the projections okay + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 60472c66b24..410a5fa85c8 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -8,6 +8,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; @@ -86,4 +87,97 @@ public function filteringSetup() $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); } + + /** @test */ + public function setupIsInvokedForPreviouslyActiveSubscribers() + { + // Usecase: Setup a content repository and then when the subscribers are active, update to a new patch which requires a setup + + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + + $this->subscriptionService->setupEventStore(); + // setup subscription tables + $result = $this->subscriptionService->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + self::assertNull($result->errors); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // initial setup for FakeProjection + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // regular work + + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // then an update is fetched, the status changes: + + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function failingSetupWillMarkProjectionAsErrored() + { + $this->fakeProjection->expects(self::once())->method('setUp')->willThrowException( + $exception = new \RuntimeException('Projection could not be setup') + ); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Needs setup')); + + $this->subscriptionService->setupEventStore(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertSame('Projection could not be setup', $result->errors?->first()->message); + + $expectedFailure = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::NEW, $exception), + projectionStatus: ProjectionStatus::setupRequired('Needs setup'), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } } From a8f246be1bbffa5259cd135b27bdfa4cb9446ba9 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:06:47 +0100 Subject: [PATCH 072/142] BUGFIX: Setup must re-setup active projections for migrations --- .../SubscriptionNewStatusTest.php | 4 +- .../Subscription/SubscriptionSetupTest.php | 38 +++++++++++++++++++ .../Engine/SubscriptionEngine.php | 16 ++++++-- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index 6e46704b0fb..fc8e61beddc 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -27,9 +27,9 @@ public function resetContentRepositoryRegistry(): void } /** @test */ - public function newProjectionIsFoundConfigurationIsAdded() + public function newProjectionIsFoundWhenConfigurationIsAdded() { - $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->subscriptionService->setupEventStore(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 410a5fa85c8..1483e90851f 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -88,6 +88,44 @@ public function filteringSetup() ); } + /** @test */ + public function setupIsInvokedForBootingSubscribers() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + + $this->subscriptionService->setupEventStore(); + + // initial setup for FakeProjection + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // then an update is fetched, the status changes: + + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } + /** @test */ public function setupIsInvokedForPreviouslyActiveSubscribers() { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 72d5684a5f2..f5399d8b29c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -50,9 +50,13 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result $this->subscriptionStore->setup(); $this->discoverNewSubscriptions(); - $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::NEW)); + $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ + SubscriptionStatus::NEW, + SubscriptionStatus::BOOTING, + SubscriptionStatus::ACTIVE, + ]))); if ($subscriptions->isEmpty()) { - $this->logger?->info('Subscription Engine: No subscriptions to setup, finish setup.'); + $this->logger?->info('Subscription Engine: No subscriptions found.'); // todo not happy? Because there must be at least the content graph?!! return Result::success(); } $errors = []; @@ -170,16 +174,22 @@ private function setupSubscription(Subscription $subscription): ?Error try { $subscriber->projection->setUp(); } catch (\Throwable $e) { + // todo wrap in savepoint to ensure error do not mess up the projection? $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); $subscription->fail($e); $this->subscriptionManager->update($subscription); return Error::fromSubscriptionIdAndException($subscription->id, $e); } + + if ($subscription->status === SubscriptionStatus::ACTIVE) { + $this->logger?->debug(sprintf('Subscription Engine: Active subscriber "%s" for "%s" has been re-setup.', $subscriber::class, $subscription->id->value)); + return null; + } + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->status->name)); $subscription->set( status: SubscriptionStatus::BOOTING ); $this->subscriptionManager->update($subscription); - $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the setup method has been executed, set to %s.', $subscriber::class, $subscription->id->value, $subscription->status->value)); return null; } From 73e10975cb872a2013a7acf43edd8f498bee8e0e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:08:08 +0100 Subject: [PATCH 073/142] BUGFIX: Setup should reattach detached projections if possible, and mark detached ones as detached. --- .../SubscriptionDetachedStatusTest.php | 78 ++++++++++++++++++- .../Engine/SubscriptionEngine.php | 11 +++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 8528517da00..2a45ec60c4e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -23,7 +23,7 @@ public function resetContentRepositoryRegistry(): void } /** @test */ - public function projectionIsDetachedIfConfigurationIsRemoved() + public function projectionIsDetachedOnCatchupActive() { $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); @@ -49,7 +49,7 @@ public function projectionIsDetachedIfConfigurationIsRemoved() // $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->fakeProjection->expects(self::never())->method('apply'); - // catchup or anything that finds detached subscribers + // catchup to mark detached subscribers $result = $this->subscriptionEngine->catchUpActive(); // todo result should reflect that there was an detachment? Throw error in CR? self::assertEquals(ProcessedResult::success(1), $result); @@ -88,4 +88,78 @@ public function projectionIsDetachedIfConfigurationIsRemoved() $this->subscriptionStatus('Vendor.Package:FakeProjection') ); } + + /** @test */ + public function projectionIsDetachedOnSetupAndReattachedIfPossible() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->subscriptionService->setupEventStore(); + $this->subscriptionService->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // "uninstall" the projection + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings = $originalSettings; + unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:FakeProjection']); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + $this->fakeProjection->expects(self::never())->method('apply'); + // setup to find detached subscribers + $result = $this->subscriptionEngine->setup(); + // todo result should reflect that there was an detachment? + self::assertNull($result->errors); + + $expectedDetachedState = SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + projectionStatus: null // not calculate-able at this point! + ); + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // another setup does not reattach, because there is no subscriber + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // "reinstall" the projection + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + self::assertEquals( + SubscriptionAndProjectionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + projectionStatus: ProjectionStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // setup does re-attach as the projection is found again + $this->subscriptionEngine->setup(); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index f5399d8b29c..4d0b9c9bd5a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -54,6 +54,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE, + SubscriptionStatus::DETACHED, ]))); if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions found.'); // todo not happy? Because there must be at least the content graph?!! @@ -170,6 +171,16 @@ function (Subscriptions $subscriptions) { */ private function setupSubscription(Subscription $subscription): ?Error { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot set up + $subscription->set( + status: SubscriptionStatus::DETACHED, + ); + $this->subscriptionManager->update($subscription); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + return null; + } + $subscriber = $this->subscribers->get($subscription->id); try { $subscriber->projection->setUp(); From 6726d738d2fcdd9a361b6985431af61f17f5c3aa Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 11:21:44 +0100 Subject: [PATCH 074/142] FEATURE: Setup marks failed projections to be booted again --- .../Subscription/ProjectionErrorTest.php | 60 +++++++++++++++---- .../Engine/SubscriptionEngine.php | 16 ++++- .../Classes/Subscription/Subscription.php | 4 +- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 6565dda492a..86550aae570 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -12,7 +12,6 @@ use Neos\ContentRepository\Core\Subscription\Engine\Error; use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; -use Neos\ContentRepository\Core\Subscription\Engine\Result; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -110,10 +109,7 @@ public function fixFailedProjection() $this->subscriptionStatus('Vendor.Package:FakeProjection') ); - // setup does not change anything - $result = $this->subscriptionService->subscriptionEngine->setup(); - self::assertEquals(Result::success(), $result); - // nor catchup active + // catchup active does not change anything $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); // boot neither @@ -143,10 +139,12 @@ public function fixFailedProjection() public function projectionIsRolledBackAfterError() { $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertNull($result->errors); // commit an event $this->commitExampleContentStreamEvent(); @@ -180,16 +178,33 @@ public function projectionIsRolledBackAfterError() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); + // + // fix projection and catchup + // + $this->secondFakeProjection->killSaboteur(); - // todo find way to retry projection? catchup force? + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + // subscriptionError is reset + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // catchup after fix + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertNull($result->errors); + + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); } /** @test */ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() { $this->subscriptionService->setupEventStore(); - $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionService->subscriptionEngine->setup(); $this->subscriptionService->subscriptionEngine->boot(); @@ -233,6 +248,31 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() [SequenceNumber::fromInteger(1)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + $result = $this->subscriptionService->subscriptionEngine->setup(); + self::assertNull($result->errors); + + // subscriptionError is reset, but the position is preserved + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // catchup after fix + $result = $this->subscriptionService->subscriptionEngine->boot(); + self::assertNull($result->errors); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); } /** @test */ diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 4d0b9c9bd5a..0ad1265502c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -55,6 +55,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE, SubscriptionStatus::DETACHED, + SubscriptionStatus::ERROR, ]))); if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions found.'); // todo not happy? Because there must be at least the content graph?!! @@ -196,6 +197,15 @@ private function setupSubscription(Subscription $subscription): ?Error $this->logger?->debug(sprintf('Subscription Engine: Active subscriber "%s" for "%s" has been re-setup.', $subscriber::class, $subscription->id->value)); return null; } + if ($subscription->status === SubscriptionStatus::ERROR) { + $this->logger?->debug(sprintf('Subscription Engine: Failed subscriber "%s" for "%s" has been re-setup, set to %s. Previous error: %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->error?->errorMessage)); + $subscription->set( + status: SubscriptionStatus::BOOTING + ); + $subscription->unsetError(); + $this->subscriptionManager->update($subscription); + return null; + } $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->status->name)); $subscription->set( status: SubscriptionStatus::BOOTING @@ -216,7 +226,11 @@ private function resetSubscription(Subscription $subscription): ?Error $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); return Error::fromSubscriptionIdAndException($subscription->id, $e); } - $subscription->reset(); + $subscription->set( + status: SubscriptionStatus::BOOTING, + position: SequenceNumber::none() + ); + $subscription->unsetError(); $this->subscriptionManager->update($subscription); $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the resetState method has been executed.', $subscriber::class, $subscription->id->value)); return null; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 556a29ab89d..7a6c22ab82f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -47,10 +47,8 @@ public function set( $this->position = $position ?? $this->position; } - public function reset(): void + public function unsetError(): void { - $this->status = SubscriptionStatus::BOOTING; - $this->position = SequenceNumber::none(); $this->error = null; } From f46077ec892a3aa0869d91a50ba4423a44983d99 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 17:17:36 +0100 Subject: [PATCH 075/142] FEATURE: Introduce `ContentRepositoryMaintainer` and restore cr:projection commands --- .../CRBehavioralTestsSubjectProvider.php | 20 +- .../AbstractSubscriptionEngineTestCase.php | 8 +- .../Subscription/CatchUpHookErrorTest.php | 24 +- .../Subscription/CatchUpHookTest.php | 6 +- .../Subscription/ProjectionErrorTest.php | 58 ++-- .../SubscriptionActiveStatusTest.php | 24 +- .../SubscriptionBootingStatusTest.php | 10 +- .../SubscriptionDetachedStatusTest.php | 12 +- .../SubscriptionGetStatusTest.php | 4 +- .../SubscriptionNewStatusTest.php | 6 +- .../Subscription/SubscriptionResetTest.php | 4 +- .../Subscription/SubscriptionSetupTest.php | 32 +- .../Parallel/AbstractParallelTestCase.php | 21 +- .../Service/ContentRepositoryMaintainer.php | 146 +++++++++ ...=> ContentRepositoryMaintainerFactory.php} | 13 +- .../Classes/Service/SubscriptionService.php | 32 -- .../Engine/SubscriptionEngine.php | 4 +- .../SubscriptionAndProjectionStatuses.php | 13 + .../Features/Bootstrap/CRTestSuiteTrait.php | 7 +- .../Behavior/CRRegistrySubjectProvider.php | 29 +- .../Classes/Command/CrCommandController.php | 306 ++++++++++-------- ...ssor.php => ProjectionReplayProcessor.php} | 8 +- .../Processors/ProjectionResetProcessor.php | 24 -- .../ContentRepositoryPruningProcessor.php | 11 +- .../Domain/Service/SiteImportService.php | 26 +- .../Domain/Service/SitePruningService.php | 14 +- .../Features/FrontendRouting/Basic.feature | 2 +- 27 files changed, 497 insertions(+), 367 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php rename Neos.ContentRepository.Core/Classes/Service/{SubscriptionServiceFactory.php => ContentRepositoryMaintainerFactory.php} (50%) delete mode 100644 Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php rename Neos.ContentRepositoryRegistry/Classes/Processors/{ProjectionCatchupProcessor.php => ProjectionReplayProcessor.php} (53%) delete mode 100644 Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php index 3fb475d02bd..2fb4fe8ac9b 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php @@ -18,12 +18,14 @@ use Behat\Gherkin\Node\TableNode; use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Helpers\GherkinTableNodeBasedContentDimensionSource; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; use Neos\EventStore\EventStoreInterface; +use PHPUnit\Framework\Assert; use Symfony\Component\Yaml\Yaml; /** @@ -179,20 +181,26 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository * Catch Up process and the testcase reset. */ $contentRepository = $this->createContentRepository($contentRepositoryId); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $subscriptionService->setupEventStore(); - $subscriptionService->subscriptionEngine->setup(); + $result = $contentRepositoryMaintainer->setUp(); + Assert::assertNull($result); self::$alreadySetUpContentRepositories[] = $contentRepository->id; } + // todo we TRUNCATE here and do not want to use $contentRepositoryMaintainer->prune(); here as it would not reset the autoincrement sequence number making some assertions impossible /** @var EventStoreInterface $eventStore */ $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); /** @var Connection $databaseConnection */ $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); - $subscriptionService->subscriptionEngine->reset(); - $subscriptionService->subscriptionEngine->boot(); + + /** @var SubscriptionEngine $subscriptionEngine */ + $subscriptionEngine = (new \ReflectionClass($contentRepositoryMaintainer))->getProperty('subscriptionEngine')->getValue($contentRepositoryMaintainer); + $result = $subscriptionEngine->reset(); + Assert::assertNull($result->errors); + $result = $subscriptionEngine->boot(); + Assert::assertNull($result->errors); return $contentRepository; } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 1602b362781..c4e9506aee9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -15,8 +15,6 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; -use Neos\ContentRepository\Core\Service\SubscriptionService; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; @@ -41,8 +39,6 @@ abstract class AbstractSubscriptionEngineTestCase extends TestCase // we don't u { protected ContentRepository $contentRepository; - protected SubscriptionService $subscriptionService; - protected SubscriptionEngine $subscriptionEngine; protected EventStoreInterface $eventStore; @@ -102,8 +98,6 @@ final protected function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId ); - $this->subscriptionService = $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $subscriptionEngineAndEventStoreAccessor = new class implements ContentRepositoryServiceFactoryInterface { public EventStoreInterface|null $eventStore; public SubscriptionEngine|null $subscriptionEngine; @@ -142,7 +136,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository final protected function subscriptionStatus(string $subscriptionId): ?SubscriptionAndProjectionStatus { - return $this->subscriptionService->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + return $this->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } final protected function commitExampleContentStreamEvent(): void diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 63b1bbc6806..9948144bbad 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -18,11 +18,11 @@ final class CatchUpHookErrorTest extends AbstractSubscriptionEngineTestCase /** @test */ public function error_onBeforeEvent_projectionIsNotRun() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); // commit an event $this->commitExampleContentStreamEvent(); @@ -66,11 +66,11 @@ public function error_onBeforeEvent_projectionIsNotRun() /** @test */ public function error_onAfterEvent_projectionIsRolledBack() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); // commit an event $this->commitExampleContentStreamEvent(); @@ -112,11 +112,11 @@ public function error_onAfterEvent_projectionIsRolledBack() /** @test */ public function error_onBeforeCatchUp_abortsCatchup() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::never())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -158,11 +158,11 @@ public function error_onBeforeCatchUp_abortsCatchup() /** @test */ public function error_onAfterCatchUp_abortsCatchupAndRollBack() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php index 71fb6e8ca7c..56b9d632597 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -13,11 +13,11 @@ final class CatchUpHookTest extends AbstractSubscriptionEngineTestCase /** @test */ public function catchUpHooksAreExecutedAndCanAccessTheCorrectProjectionsState() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); // commit an event $this->commitExampleContentStreamEvent(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 86550aae570..8f82f1ef9e9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -25,11 +25,11 @@ final class ProjectionErrorTest extends AbstractSubscriptionEngineTestCase /** @test */ public function projectionWithError() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionEngine->setup(); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -49,7 +49,7 @@ public function projectionWithError() projectionStatus: ProjectionStatus::ok(), ); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); self::assertEquals( @@ -60,13 +60,13 @@ public function projectionWithError() // todo test retry if reimplemented: https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/RetryStrategy/RetryStrategy.php // // CatchUp 2 with retry - // $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + // $result = $this->subscriptionEngine->catchUpActive(); // self::assertTrue($result->hasFailed()); // self::assertEquals($result->errors->first()->message, 'Something really wrong.'); // self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); // no retry, nothing to do. - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'This projection is kaputt.'); self::assertEquals( @@ -78,11 +78,11 @@ public function projectionWithError() /** @test */ public function fixFailedProjection() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); // commit an event $this->commitExampleContentStreamEvent(); @@ -101,7 +101,7 @@ public function fixFailedProjection() projectionStatus: ProjectionStatus::ok(), ); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertTrue($result->hasFailed()); self::assertEquals( @@ -110,10 +110,10 @@ public function fixFailedProjection() ); // catchup active does not change anything - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); // boot neither - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); // still the same state self::assertEquals( @@ -123,13 +123,13 @@ public function fixFailedProjection() $this->fakeProjection->expects(self::once())->method('resetState'); - $result = $this->subscriptionService->subscriptionEngine->reset(); + $result = $this->subscriptionEngine->reset(); self::assertNull($result->errors); // expect the subscriptionError to be reset to null $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); @@ -138,12 +138,12 @@ public function fixFailedProjection() /** @test */ public function projectionIsRolledBackAfterError() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); // commit an event @@ -165,7 +165,7 @@ public function projectionIsRolledBackAfterError() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); self::assertEquals( @@ -184,14 +184,14 @@ public function projectionIsRolledBackAfterError() $this->secondFakeProjection->killSaboteur(); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); // subscriptionError is reset $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); // catchup after fix - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); self::assertEquals( @@ -203,11 +203,11 @@ public function projectionIsRolledBackAfterError() /** @test */ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::exactly(2))->method('apply'); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); // commit two events $this->commitExampleContentStreamEvent(); @@ -227,7 +227,7 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertTrue($result->hasFailed()); $expectedFailure = SubscriptionAndProjectionStatus::create( @@ -255,7 +255,7 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() $this->secondFakeProjection->killSaboteur(); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); // subscriptionError is reset, but the position is preserved @@ -266,7 +266,7 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() ); // catchup after fix - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); self::assertEquals( @@ -278,11 +278,11 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() /** @test */ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->subscriptionEngine->setup(); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $originalException = new \RuntimeException('This projection is kaputt.'), diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php index 2585852b998..421d9a4c212 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -17,12 +17,12 @@ final class SubscriptionActiveStatusTest extends AbstractSubscriptionEngineTestC /** @test */ public function setupProjectionsAndCatchup() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionEngine->setup(); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -33,7 +33,7 @@ public function setupProjectionsAndCatchup() $this->commitExampleContentStreamEvent(); // subsequent catchup setup'd does not change the position - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -41,7 +41,7 @@ public function setupProjectionsAndCatchup() // catchup active does apply the commited event $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(1), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); @@ -55,9 +55,9 @@ public function filteringCatchUpActive() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); @@ -85,13 +85,13 @@ public function filteringCatchUpActive() /** @test */ public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionEngine->setup(); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -101,13 +101,13 @@ public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() // catchup active does apply the commited event $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(1), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); // empty catchup must keep the sequence numbers of the projections okay - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php index 01b15d9137c..a89370bd57e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php @@ -23,20 +23,20 @@ public function existingEventStoreEventsAreCaughtUpOnBoot() $this->commitExampleContentStreamEvent(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionEngine->setup(); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->subscriptionService->subscriptionEngine->boot(); + $this->subscriptionEngine->boot(); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); // catchup is a noop because there are no unhandled events - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); } @@ -46,9 +46,9 @@ public function filteringCatchUpBoot() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 2a45ec60c4e..855b68d473b 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -28,10 +28,10 @@ public function projectionIsDetachedOnCatchupActive() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); - $this->subscriptionService->subscriptionEngine->setup(); + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -96,12 +96,12 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() $this->fakeProjection->expects(self::once())->method('apply'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); - $this->subscriptionService->subscriptionEngine->setup(); + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); $this->commitExampleContentStreamEvent(); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(1), $result); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index 594c7264e92..000944385f3 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -25,7 +25,7 @@ public function statusOnEmptyDatabase() keepSchema: false ); - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); self::assertTrue($actualStatuses->isEmpty()); self::assertNull( @@ -48,7 +48,7 @@ public function statusOnEmptyDatabase() $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); $expected = SubscriptionAndProjectionStatuses::fromArray([ SubscriptionAndProjectionStatus::create( diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index fc8e61beddc..7cc7c2c1670 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -32,10 +32,10 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); - $this->subscriptionService->subscriptionEngine->setup(); + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php index ea1a741ad57..cbebb621ceb 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php @@ -18,9 +18,9 @@ public function filteringReset() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 1483e90851f..d3b3e6d5ada 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -18,13 +18,13 @@ final class SubscriptionSetupTest extends AbstractSubscriptionEngineTestCase /** @test */ public function setupOnEmptyDatabase() { - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionService->subscriptionEngine->setup(); + $this->subscriptionEngine->setup(); $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); - $actualStatuses = $this->subscriptionService->subscriptionEngine->subscriptionStatuses(); + $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); $expected = SubscriptionAndProjectionStatuses::fromArray([ SubscriptionAndProjectionStatus::create( @@ -67,11 +67,11 @@ public function filteringSetup() $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); - $result = $this->subscriptionService->subscriptionEngine->setup($filter); + $result = $this->subscriptionEngine->setup($filter); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); @@ -97,11 +97,11 @@ public function setupIsInvokedForBootingSubscribers() // hard reset, so that the tests actually need sql migrations $this->secondFakeProjection->dropTables(); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); // initial setup for FakeProjection - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); @@ -120,7 +120,7 @@ public function setupIsInvokedForBootingSubscribers() $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); @@ -138,9 +138,9 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() // hard reset, so that the tests actually need sql migrations $this->secondFakeProjection->dropTables(); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); // setup subscription tables - $result = $this->subscriptionService->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); self::assertNull($result->errors); self::assertEquals( @@ -156,17 +156,17 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() // initial setup for FakeProjection - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $result = $this->subscriptionService->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); // regular work $this->commitExampleContentStreamEvent(); - $result = $this->subscriptionService->subscriptionEngine->catchUpActive(); + $result = $this->subscriptionEngine->catchUpActive(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); @@ -186,7 +186,7 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); @@ -200,9 +200,9 @@ public function failingSetupWillMarkProjectionAsErrored() ); $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Needs setup')); - $this->subscriptionService->setupEventStore(); + $this->eventStore->setup(); - $result = $this->subscriptionService->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); self::assertSame('Projection could not be setup', $result->errors?->first()->message); $expectedFailure = SubscriptionAndProjectionStatus::create( diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php index d4646e8165e..efa68008845 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php @@ -14,10 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel; -use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\Service\SubscriptionService; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Core\Bootstrap; @@ -71,19 +70,11 @@ final protected function setUpContentRepository( ContentRepositoryId $contentRepositoryId ): ContentRepository { $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - /** @var SubscriptionService $subscriptionService */ - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $subscriptionService->setupEventStore(); - $subscriptionService->subscriptionEngine->setup(); - - $connection = $this->objectManager->get(Connection::class); - + /** @var ContentRepositoryMaintainer $contentRepositoryMaintainer */ + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $contentRepositoryMaintainer->setUp(); // reset events and projections - $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); - $connection->executeStatement('TRUNCATE ' . $eventTableName); - - $subscriptionService->subscriptionEngine->reset(); - $subscriptionService->subscriptionEngine->boot(); + $contentRepositoryMaintainer->prune(); return $contentRepository; } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php new file mode 100644 index 00000000000..1db74979dae --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -0,0 +1,146 @@ +eventStore->setup(); + $eventStoreIsEmpty = iterator_count($this->eventStore->load(VirtualStreamName::all())->limit(1)) === 0; + $setupResult = $this->subscriptionEngine->setup(); + if ($setupResult->errors !== null) { + return self::createErrorForReason('setup', $setupResult->errors); + } + if ($eventStoreIsEmpty) { + // todo reintroduce skipBooting flag, and also notify if the flag is not set, e.g. because there are events + $bootResult = $this->subscriptionEngine->boot(); + if ($bootResult->errors !== null) { + return self::createErrorForReason('initial catchup', $bootResult->errors); + } + } + return null; + } + + public function eventStoreStatus(): EventStoreStatus + { + return $this->eventStore->status(); + } + + public function subscriptionStatuses(): SubscriptionAndProjectionStatuses + { + return $this->subscriptionEngine->subscriptionStatuses(); + } + + public function replayProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + { + $resetResult = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([$subscriptionId])); + if ($resetResult->errors !== null) { + return self::createErrorForReason('reset', $resetResult->errors); + } + $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), $progressCallback); + if ($bootResult->errors !== null) { + return self::createErrorForReason('catchup', $bootResult->errors); + } + return null; + } + + public function replayAllProjections(\Closure|null $progressCallback = null): Error|null + { + $resetResult = $this->subscriptionEngine->reset(); + if ($resetResult->errors !== null) { + return self::createErrorForReason('reset', $resetResult->errors); + } + $bootResult = $this->subscriptionEngine->boot(progressCallback: $progressCallback); + if ($bootResult->errors !== null) { + return self::createErrorForReason('catchup', $bootResult->errors); + } + return null; + } + + /** + * Catchup one specific projection. + * + * The explicit catchup is required for new projections in the booting state. + * + * We don't offer an API to catch up all projections catchupAllProjection as we would have to distinct between booting or catchup if its active already. + * + * This method is only needed in rare cases for debugging or after installing a new projection or fixing its errors. + */ + public function catchupProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + { + $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); + if ($bootResult->errors !== null) { + return self::createErrorForReason('catchup', $bootResult->errors); + } + if ($bootResult->numberOfProcessedEvents > 0) { + // the projection was bootet + return null; + } + // todo the projection was active, and we might still want to catch it up ... find reason for this? And combine boot and catchup? + $catchupResult = $this->subscriptionEngine->catchUpActive(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); + if ($catchupResult->errors !== null) { + return self::createErrorForReason('catchup', $catchupResult->errors); + } + return null; + } + + /** + * WARNING: Removes all events from the content repository and resets the projections + * This operation cannot be undone. + */ + public function prune(): Error|null + { + // todo move pruneAllWorkspacesAndContentStreamsFromEventStream here. + $this->contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); + $resetResult = $this->subscriptionEngine->reset(); + if ($resetResult->errors !== null) { + return self::createErrorForReason('reset', $resetResult->errors); + } + // todo reintroduce skipBooting flag to reset + $bootResult = $this->subscriptionEngine->boot(); + if ($bootResult->errors !== null) { + return self::createErrorForReason('booting', $bootResult->errors); + } + return null; + } + + private static function createErrorForReason(string $method, Errors $errors): Error + { + // todo log throwable via flow???, but we are here in the CORE ... + $message = []; + $message[] = sprintf('%s produced the following error%s', $method, $errors->count() === 1 ? '' : 's'); + foreach ($errors as $error) { + $message[] = sprintf(' Subscription "%s": %s', $error->subscriptionId->value, $error->message); + } + return new Error(join("\n", $message)); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php similarity index 50% rename from Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php rename to Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php index 3fe17594800..e4ddcf4e76d 100644 --- a/Neos.ContentRepository.Core/Classes/Service/SubscriptionServiceFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php @@ -8,17 +8,22 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; /** - * @implements ContentRepositoryServiceFactoryInterface + * @implements ContentRepositoryServiceFactoryInterface * @api */ -class SubscriptionServiceFactory implements ContentRepositoryServiceFactoryInterface +class ContentRepositoryMaintainerFactory implements ContentRepositoryServiceFactoryInterface { public function build( ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies - ): SubscriptionService { - return new SubscriptionService( + ): ContentRepositoryMaintainer { + return new ContentRepositoryMaintainer( $serviceFactoryDependencies->eventStore, $serviceFactoryDependencies->subscriptionEngine, + new ContentStreamPruner( + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, + ) ); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php b/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php deleted file mode 100644 index 321df1534ff..00000000000 --- a/Neos.ContentRepository.Core/Classes/Service/SubscriptionService.php +++ /dev/null @@ -1,32 +0,0 @@ -eventStore->setup(); - } - - public function eventStoreStatus(): Status - { - return $this->eventStore->status(); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 0ad1265502c..5d8bb9ac086 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -5,8 +5,10 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; use Doctrine\DBAL\Exception\TableNotFoundException; +use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; @@ -25,7 +27,7 @@ use Neos\ContentRepository\Core\Subscription\Subscriptions; /** - * @api + * @internal public API is the {@see ContentRepository::handle()} and the {@see ContentRepositoryMaintainer} */ final class SubscriptionEngine { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php index 47c237813b5..1c3351ed666 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php @@ -51,4 +51,17 @@ public function isEmpty(): bool { return $this->statuses === []; } + + public function isOk(): bool + { + foreach ($this->statuses as $status) { + if ($status->subscriptionStatus === SubscriptionStatus::ERROR) { + return false; + } + if ($status->projectionStatus?->type !== ProjectionStatusType::OK) { + return false; + } + } + return true; + } } diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index f65d0de69fc..777d5a9c1f3 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -27,10 +27,12 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\NodeType\NodeTypeCriteria; use Neos\ContentRepository\Core\Projection\ContentGraph\Subtree; use Neos\ContentRepository\Core\Projection\ContentGraph\VisibilityConstraints; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\ContentStreamClosing; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCreation; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeModification; @@ -254,8 +256,9 @@ abstract protected function getContentRepositoryService( */ public function iReplayTheProjection(string $projectionName): void { - $this->currentContentRepository->resetProjectionState($projectionName); - $this->currentContentRepository->catchUpProjection($projectionName, CatchUpOptions::create()); + $contentRepositoryMaintainer = $this->getContentRepositoryService(new ContentRepositoryMaintainerFactory()); + $result = $contentRepositoryMaintainer->replayProjection(SubscriptionId::fromString($projectionName)); + Assert::assertNull($result); } protected function deserializeProperties(array $properties): PropertyValuesToWrite diff --git a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php index 51216bbbd7a..e4eeaedd33b 100644 --- a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php +++ b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php @@ -13,15 +13,14 @@ * source code. */ -use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; -use Neos\EventStore\EventStoreInterface; +use PHPUnit\Framework\Assert; /** * The CR registry subject provider trait for behavioral tests @@ -53,24 +52,18 @@ protected function setUpCRRegistry(): void /** * @Given /^I initialize content repository "([^"]*)"$/ */ - public function iInitializeContentRepository(string $contentRepositoryId): void + public function iInitializeContentRepository(string $rawContentRepositoryId): void { - $contentRepository = $this->getContentRepository(ContentRepositoryId::fromString($contentRepositoryId)); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepository->id, new SubscriptionServiceFactory()); - /** @var EventStoreInterface $eventStore */ - $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); - /** @var Connection $databaseConnection */ - $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); - $eventTableName = sprintf('cr_%s_events', $contentRepositoryId); - $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); + $contentRepositoryId = ContentRepositoryId::fromString($rawContentRepositoryId); - if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $subscriptionService->setupEventStore(); - $subscriptionService->subscriptionEngine->setup(); - self::$alreadySetUpContentRepositories[] = $contentRepository->id; + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + if (!in_array($contentRepositoryId, self::$alreadySetUpContentRepositories)) { + $result = $contentRepositoryMaintainer->setUp(); + Assert::assertNull($result); + self::$alreadySetUpContentRepositories[] = $contentRepositoryId; } - $subscriptionService->subscriptionEngine->reset(); - $subscriptionService->subscriptionEngine->boot(); + $result = $contentRepositoryMaintainer->prune(); + Assert::assertNull($result); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 0e8ee668a87..d7a03eed6bb 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -4,26 +4,19 @@ namespace Neos\ContentRepositoryRegistry\Command; use Neos\ContentRepository\Core\Projection\ProjectionStatusType; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\StatusType; +use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; -use stdClass; -use Symfony\Component\Console\Helper\ProgressBar; -use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\Output; final class CrCommandController extends CommandController { - - public function __construct( - private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - ) { - parent::__construct(); - } + #[Flow\Inject()] + protected ContentRepositoryRegistry $contentRepositoryRegistry; /** * Sets up and checks required dependencies for a Content Repository instance @@ -40,73 +33,14 @@ public function __construct( public function setupCommand(string $contentRepository = 'default'): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $subscriptionService->setupEventStore(); - $setupResult = $subscriptionService->subscriptionEngine->setup(); - if ($setupResult->errors === null) { - $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); - return; - } - $this->outputLine('Setup of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $setupResult->errors->count() === 1 ? '' : 's']); - foreach ($setupResult->errors as $error) { - $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); - } - $this->quit(1); - } + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - public function subscriptionsBootCommand(string $contentRepository = 'default', bool $quiet = false): void - { - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - if (!$quiet) { - $this->outputLine('Booting new subscriptions'); - // render memory consumption and time remaining - $this->output->getProgressBar()->setFormat('debug'); - $this->output->progressStart(); - $bootResult = $subscriptionService->subscriptionEngine->boot(progressCallback: fn () => $this->output->progressAdvance()); - $this->output->progressFinish(); - $this->outputLine(); - if ($bootResult->hasFailed() === false) { - $this->outputLine('Done'); - return; - } - } else { - $bootResult = $subscriptionService->subscriptionEngine->boot(); - } - if ($bootResult->hasFailed()) { - $this->outputLine('Booting of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $bootResult->errors->count() === 1 ? '' : 's']); - foreach ($bootResult->errors as $error) { - $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); - } + $result = $contentRepositoryMaintainer->setUp(); + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); $this->quit(1); } - } - - public function subscriptionsCatchUpCommand(string $contentRepository = 'default', bool $quiet = false): void - { - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $subscriptionService->subscriptionEngine->catchUpActive(); - } - - public function subscriptionsResetCommand(string $contentRepository = 'default', bool $force = false): void - { - if (!$force && !$this->output->askConfirmation('Are you sure? (y/n) ', false)) { - $this->outputLine('Cancelled'); - $this->quit(); - } - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $resetResult = $subscriptionService->subscriptionEngine->reset(); - if ($resetResult->errors === null) { - $this->outputLine('Content Repository "%s" was reset', [$contentRepositoryId->value]); - return; - } - $this->outputLine('Reset of Content Repository "%s" produced the following error%s', [$contentRepositoryId->value, $resetResult->errors->count() === 1 ? '' : 's']); - foreach ($resetResult->errors as $error) { - $this->outputLine('Subscription "%s": %s', [$error->subscriptionId->value, $error->message]); - } - $this->quit(1); + $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); } /** @@ -124,86 +58,184 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $subscriptionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory()); - $eventStoreStatus = $subscriptionService->eventStoreStatus(); - $hasErrors = false; - $setupRequired = false; - $bootingRequired = false; - $resetRequired = false; + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $eventStoreStatus = $contentRepositoryMaintainer->eventStoreStatus(); $this->output('Event Store: '); $this->outputLine(match ($eventStoreStatus->type) { StatusType::OK => 'OK', StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - $hasErrors |= $eventStoreStatus->type === StatusType::ERROR; if ($verbose && $eventStoreStatus->details !== '') { $this->outputFormatted($eventStoreStatus->details, [], 2); } $this->outputLine(); - $this->outputLine('Subscriptions:'); - $subscriptionStatuses = $subscriptionService->subscriptionEngine->subscriptionStatuses(); - if ($subscriptionStatuses->isEmpty()) { - $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); - $this->quit(1); - } - foreach ($subscriptionStatuses as $status) { - $this->outputLine(' %s:', [$status->subscriptionId->value]); - $this->output(' Subscription: ', [$status->subscriptionId->value]); - $this->output(match ($status->subscriptionStatus) { - SubscriptionStatus::NEW => 'NEW', - SubscriptionStatus::BOOTING => 'BOOTING', - SubscriptionStatus::ACTIVE => 'ACTIVE', - SubscriptionStatus::DETACHED => 'DETACHED', - SubscriptionStatus::ERROR => 'ERROR', + + $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); + foreach ($subscriptionStatuses as $subscriptionStatus) { + // todo reimplement 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 + $this->output('Projection "%s": ', [$subscriptionStatus->subscriptionId->value]); + $projectionStatus = $subscriptionStatus->projectionStatus; + if ($projectionStatus === null) { + $this->outputLine('No status available.'); // todo this means detached? + continue; + } + $this->outputLine(match ($projectionStatus->type) { + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', + ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', + ProjectionStatusType::ERROR => 'ERROR', }); - $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); - $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; - $bootingRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; - if ($verbose && $status->subscriptionError !== null) { - $lines = explode(chr(10), $status->subscriptionError->errorMessage ?: 'No details available.'); + if ($verbose && ($projectionStatus->type !== ProjectionStatusType::OK || $projectionStatus->details)) { + $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); foreach ($lines as $line) { - $this->outputLine(' %s', [$line]); - } - } - if ($status->projectionStatus !== null) { - $this->output(' Projection: '); - $this->outputLine(match ($status->projectionStatus->type) { - ProjectionStatusType::OK => 'OK', - ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', - ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', - ProjectionStatusType::ERROR => 'ERROR', - }); - $hasErrors |= $status->projectionStatus->type === ProjectionStatusType::ERROR; - $setupRequired |= $status->projectionStatus->type === ProjectionStatusType::SETUP_REQUIRED; - $resetRequired |= $status->projectionStatus->type === ProjectionStatusType::REPLAY_REQUIRED; - if ($verbose && ($status->projectionStatus->type !== ProjectionStatusType::OK || $status->projectionStatus->details)) { - $lines = explode(chr(10), $status->projectionStatus->details ?: 'No details available.'); - foreach ($lines as $line) { - $this->outputLine(' ' . $line); - } - $this->outputLine(); + $this->outputLine(' ' . $line); } + $this->outputLine(); } } - if ($verbose) { + if ($eventStoreStatus->type !== StatusType::OK || !$subscriptionStatuses->isOk()) { + $this->quit(1); + } + } + + /** + * Replays the specified projection of a Content Repository by resetting its state and performing a full catchup. + * + * @param string $projection Identifier of the projection to replay like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $force Replay the projection without confirmation. This may take some time! + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false): void + { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the projection "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $projection, $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replayProjection(SubscriptionId::fromString($projection), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); $this->outputLine(); - if ($setupRequired) { - $this->outputLine('Setup required, please run ./flow cr:setup'); - } - if ($bootingRequired) { - $this->outputLine('Some subscriptions need to be booted, please run ./flow cr:subscriptionsboot'); - } - if ($resetRequired) { - $this->outputLine('Some subscriptions need to be replayed, please run ./flow cr:subscriptionsreset'); - } - if ($hasErrors) { - $this->outputLine('Some subscriptions/projections have failed'); - } } - if ($hasErrors) { + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Replays all projections of the specified Content Repository by resetting their states and performing a full catchup + * + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $force Replay the projection without confirmation. This may take some time! + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void + { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); + // render memory consumption and time remaining + // todo maybe reintroduce pretty output: https://github.com/neos/neos-development-collection/pull/5010 but without using highestSequenceNumber + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replayAllProjections(progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Catchup one specific projection. + * + * The explicit catchup is required for new projections in the booting state, after installing a new projection or fixing its errors. + * + * @param string $projection Identifier of the projection to catchup like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function projectionCatchupCommand(string $projection, string $contentRepository = 'default', bool $quiet = false): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Catchup projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->catchupProjection(SubscriptionId::fromString($projection), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); } } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php similarity index 53% rename from Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php rename to Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php index 71bada85b93..b97a37eb27d 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php @@ -3,22 +3,22 @@ namespace Neos\ContentRepositoryRegistry\Processors; -use Neos\ContentRepository\Core\Service\SubscriptionService; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; /** * @internal */ -final readonly class ProjectionCatchupProcessor implements ProcessorInterface +final readonly class ProjectionReplayProcessor implements ProcessorInterface { public function __construct( - private SubscriptionService $subscriptionService, + private ContentRepositoryMaintainer $contentRepositoryMaintainer, ) { } public function run(ProcessingContext $context): void { - $this->subscriptionService->subscriptionEngine->catchUpActive(); + $this->contentRepositoryMaintainer->replayAllProjections(); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php deleted file mode 100644 index 1dfdd7d4208..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php +++ /dev/null @@ -1,24 +0,0 @@ -subscriptionService->subscriptionEngine->reset(); - } -} diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php index 0b94195c9d1..9cb7cf60b24 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -14,22 +14,25 @@ namespace Neos\Neos\Domain\Pruning; -use Neos\ContentRepository\Core\Service\ContentStreamPruner; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; /** - * Pruning processor that removes all events from the given cr + * Pruning processor that removes all events from the given cr and resets the projections */ final readonly class ContentRepositoryPruningProcessor implements ProcessorInterface { public function __construct( - private ContentStreamPruner $contentStreamPruner, + private ContentRepositoryMaintainer $contentRepositoryMaintainer, ) { } public function run(ProcessingContext $context): void { - $this->contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); + $result = $this->contentRepositoryMaintainer->prune(); + if ($result !== null) { + throw new \RuntimeException($result->getMessage(), 1732461335); + } } } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 494b4779c8f..e0489cc6440 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -17,8 +17,8 @@ use Doctrine\DBAL\Exception as DBALException; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; @@ -28,7 +28,8 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; +use Neos\ContentRepositoryRegistry\Processors\ProjectionReplayProcessor; +use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -67,8 +68,10 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string } $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $this->requireDataBaseSchemaToBeSetup(); - $this->requireContentRepositoryToBeSetup($contentRepository); + $this->requireContentRepositoryToBeSetup($contentRepositoryMaintainer, $contentRepositoryId); $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); $context = new ProcessingContext($filesystem, $onMessage); @@ -78,7 +81,8 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new SubscriptionServiceFactory())), + // todo we do a replay here even though it will redo the live workspace creation. But otherwise the catchup hooks are not skipped because it seems like a regular catchup + 'Catchup all projections' => new ProjectionReplayProcessor($contentRepositoryMaintainer), ]); foreach ($processors as $processorLabel => $processor) { @@ -87,13 +91,13 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string } } - private function requireContentRepositoryToBeSetup(ContentRepository $contentRepository): void + private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $contentRepositoryMaintainer, ContentRepositoryId $contentRepositoryId): void { -// TODO reimplement -// $status = $contentRepository->status(); -// if (!$status->isOk()) { -// throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepository->id->value)); -// } + $eventStoreStatus = $contentRepositoryMaintainer->eventStoreStatus(); + $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); + if ($eventStoreStatus->type !== StatusType::OK || !$subscriptionStatuses->isOk()) { + throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepositoryId->value)); + } } private function requireDataBaseSchemaToBeSetup(): void diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 968ec5d9427..18ab6f1d78f 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -16,8 +16,7 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; -use Neos\ContentRepository\Core\Service\SubscriptionServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; @@ -25,7 +24,6 @@ use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionResetProcessor; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; @@ -66,17 +64,11 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $this->domainRepository, $this->persistenceManager ), - 'Prune content repository' => new ContentRepositoryPruningProcessor( - $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ContentStreamPrunerFactory() - ) - ), 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceMetadataAndRoleRepository), - 'Reset all projections' => new ProjectionResetProcessor( + 'Prune content repository' => new ContentRepositoryPruningProcessor( $this->contentRepositoryRegistry->buildService( $contentRepositoryId, - new SubscriptionServiceFactory() + new ContentRepositoryMaintainerFactory() ) ) ]); diff --git a/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature b/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature index 640ac7e62bf..c9fde1e6e6b 100644 --- a/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature +++ b/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature @@ -113,7 +113,7 @@ Feature: Basic routing functionality (match & resolve document nodes in one dime # !!! when caches were still enabled (without calling DocumentUriPathFinder->disableCache()), the replay below will # show really "interesting" (non-correct) results. This was bug #4253. - When I replay the "Neos\Neos\FrontendRouting\Projection\DocumentUriPathProjection" projection + When I replay the "Neos.Neos:DocumentUriPathProjection" projection Then the node "sir-david-nodenborough" in content stream "cs-identifier" and dimension "{}" should resolve to URL "/david-nodenborough-updated-b" And the node "earl-o-documentbourgh" in content stream "cs-identifier" and dimension "{}" should resolve to URL "/david-nodenborough-updated-b/earl-document" From bace8ffed072862b24b5763b51b38a6b66020fd5 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 19:32:05 +0100 Subject: [PATCH 076/142] TASK: Rename `ProjectionStatus` and introduce `ProjectionSubscriptionStatus` Like the `ProjectionSubscription` projections subscribers for projections will have an explicit state: `ProjectionSubscriptionStatus`. If its extended to allow other subscribers another status type should be introduced. replayRequired todo remove we cannot figurea that out in the status after all! --- .../DoctrineDbalContentGraphProjection.php | 12 ++--- .../Projection/HypergraphProjection.php | 12 ++--- .../TestSuite/DebugEventProjection.php | 8 +-- .../AbstractSubscriptionEngineTestCase.php | 10 ++-- .../Subscription/CatchUpHookErrorTest.php | 12 ++--- .../Subscription/ProjectionErrorTest.php | 26 ++++----- .../SubscriptionActiveStatusTest.php | 8 +-- .../SubscriptionBootingStatusTest.php | 6 +-- .../SubscriptionDetachedStatusTest.php | 20 +++---- .../SubscriptionGetStatusTest.php | 22 ++++---- .../SubscriptionNewStatusTest.php | 18 +++---- .../Subscription/SubscriptionResetTest.php | 4 +- .../Subscription/SubscriptionSetupTest.php | 50 ++++++++--------- .../Projection/ProjectionInterface.php | 4 +- .../Projection/ProjectionSetupStatus.php | 38 +++++++++++++ ...Type.php => ProjectionSetupStatusType.php} | 5 +- .../Classes/Projection/ProjectionStatus.php | 54 ------------------- .../Service/ContentRepositoryMaintainer.php | 4 +- .../Engine/SubscriptionEngine.php | 14 ++--- ...s.php => ProjectionSubscriptionStatus.php} | 13 +++-- .../Subscription/Subscriber/Subscribers.php | 6 +++ ...nStatuses.php => SubscriptionStatuses.php} | 22 +++++--- .../Classes/Command/CrCommandController.php | 17 +++--- .../Projection/DocumentUriPathProjection.php | 12 ++--- .../ChangeProjection.php | 12 ++--- 25 files changed, 204 insertions(+), 205 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatus.php rename Neos.ContentRepository.Core/Classes/Projection/{ProjectionStatusType.php => ProjectionSetupStatusType.php} (75%) delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php rename Neos.ContentRepository.Core/Classes/Subscription/{SubscriptionAndProjectionStatus.php => ProjectionSubscriptionStatus.php} (70%) rename Neos.ContentRepository.Core/Classes/Subscription/{SubscriptionAndProjectionStatuses.php => SubscriptionStatuses.php} (56%) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 4e998fde99a..a436c61d14f 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -67,7 +67,7 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeTags; use Neos\ContentRepository\Core\Projection\ContentGraph\Timestamps; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Node\NodeName; @@ -112,23 +112,23 @@ public function setUp(): void } } - public function status(): ProjectionStatus + public function setUpStatus(): ProjectionSetupStatus { try { $this->dbal->connect(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionStatus::ok(); + return ProjectionSetupStatus::ok(); } public function resetState(): void diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 5961442b9e6..12640cfdd72 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -42,7 +42,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -89,22 +89,22 @@ public function setUp(): void '); } - public function status(): ProjectionStatus + public function setUpStatus(): ProjectionSetupStatus { try { $this->getDatabaseConnection()->connect(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionStatus::ok(); + return ProjectionSetupStatus::ok(); } /** diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 7f71796e96d..4fcd47f66fd 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -14,7 +14,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Annotations as Flow; @@ -52,13 +52,13 @@ public function setUp(): void } } - public function status(): ProjectionStatus + public function setUpStatus(): ProjectionSetupStatus { $requiredSqlStatements = $this->determineRequiredSqlStatements(); if ($requiredSqlStatements !== []) { - return ProjectionStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); + return ProjectionSetupStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); } - return ProjectionStatus::ok(); + return ProjectionSetupStatus::ok(); } /** diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index c4e9506aee9..bc48af8b01c 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -14,12 +14,12 @@ use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory; @@ -134,7 +134,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); } - final protected function subscriptionStatus(string $subscriptionId): ?SubscriptionAndProjectionStatus + final protected function subscriptionStatus(string $subscriptionId): ?ProjectionSubscriptionStatus { return $this->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } @@ -156,12 +156,12 @@ final protected function expectOkayStatus($subscriptionId, SubscriptionStatus $s { $actual = $this->subscriptionStatus($subscriptionId); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString($subscriptionId), subscriptionStatus: $status, subscriptionPosition: $sequenceNumber, subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ), $actual ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 9948144bbad..eeb3300229e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -5,9 +5,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -37,12 +37,12 @@ public function error_onBeforeEvent_projectionIsNotRun() $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); - $expectedFailure = SubscriptionAndProjectionStatus::create( + $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ); self::assertEmpty( @@ -83,12 +83,12 @@ public function error_onAfterEvent_projectionIsRolledBack() // TODO pass the error subscription status to onAfterCatchUp, so that in case of an error it can be prevented that mails f.x. will be sent? $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); - $expectedFailure = SubscriptionAndProjectionStatus::create( + $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ); self::assertEmpty( diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 8f82f1ef9e9..03085882caf 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -6,13 +6,13 @@ use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Command\CreateRootWorkspace; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\Subscription\Engine\Error; use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -28,7 +28,7 @@ public function projectionWithError() $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -41,12 +41,12 @@ public function projectionWithError() $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $exception = new \RuntimeException('This projection is kaputt.') ); - $expectedStatusForFailedProjection = SubscriptionAndProjectionStatus::create( + $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ); $result = $this->subscriptionEngine->catchUpActive(); @@ -80,7 +80,7 @@ public function fixFailedProjection() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); @@ -93,12 +93,12 @@ public function fixFailedProjection() null // okay again ); - $expectedFailure = SubscriptionAndProjectionStatus::create( + $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ); $result = $this->subscriptionEngine->catchUpActive(); @@ -153,12 +153,12 @@ public function projectionIsRolledBackAfterError() $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); - $expectedFailure = SubscriptionAndProjectionStatus::create( + $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ); self::assertEmpty( @@ -230,12 +230,12 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() $result = $this->subscriptionEngine->catchUpActive(); self::assertTrue($result->hasFailed()); - $expectedFailure = SubscriptionAndProjectionStatus::create( + $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ); self::assertEquals( @@ -280,7 +280,7 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php index 421d9a4c212..a23b4e0080d 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -24,7 +24,7 @@ public function setupProjectionsAndCatchup() $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -53,7 +53,7 @@ public function setupProjectionsAndCatchup() public function filteringCatchUpActive() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); @@ -88,7 +88,7 @@ public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->subscriptionEngine->setup(); $result = $this->subscriptionEngine->boot(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php index a89370bd57e..7c8a005dd83 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -24,7 +24,7 @@ public function existingEventStoreEventsAreCaughtUpOnBoot() $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); @@ -44,7 +44,7 @@ public function existingEventStoreEventsAreCaughtUpOnBoot() public function filteringCatchUpBoot() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 855b68d473b..68a00a94969 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -4,9 +4,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; @@ -26,7 +26,7 @@ public function resetContentRepositoryRegistry(): void public function projectionIsDetachedOnCatchupActive() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); $this->subscriptionEngine->setup(); @@ -54,12 +54,12 @@ public function projectionIsDetachedOnCatchupActive() // todo result should reflect that there was an detachment? Throw error in CR? self::assertEquals(ProcessedResult::success(1), $result); - $expectedDetachedState = SubscriptionAndProjectionStatus::create( + $expectedDetachedState = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::DETACHED, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: null // not calculate-able at this point! + setupStatus: null // not calculate-able at this point! ); self::assertEquals( $expectedDetachedState, @@ -94,7 +94,7 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() { $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); $this->subscriptionEngine->setup(); @@ -120,12 +120,12 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() // todo result should reflect that there was an detachment? self::assertNull($result->errors); - $expectedDetachedState = SubscriptionAndProjectionStatus::create( + $expectedDetachedState = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::DETACHED, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: null, - projectionStatus: null // not calculate-able at this point! + setupStatus: null // not calculate-able at this point! ); self::assertEquals( $expectedDetachedState, @@ -147,12 +147,12 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() $this->setupContentRepositoryDependencies($this->contentRepository->id); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::DETACHED, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: null, - projectionStatus: ProjectionStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! + setupStatus: ProjectionSetupStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! ), $this->subscriptionStatus('Vendor.Package:FakeProjection') ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index 000944385f3..5090d3ee72f 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -5,10 +5,10 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; @@ -46,31 +46,31 @@ public function statusOnEmptyDatabase() $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + $this->fakeProjection->expects(self::once())->method('setUpStatus')->willReturn(ProjectionSetupStatus::setupRequired('fake needs setup.')); $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); - $expected = SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( + $expected = SubscriptionStatuses::fromArray([ + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('contentGraph'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ), - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('fake needs setup.'), + setupStatus: ProjectionSetupStatus::setupRequired('fake needs setup.'), ), - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), + setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements'), ), ]); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index 7cc7c2c1670..2d7db9869fa 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -6,10 +6,10 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; @@ -30,7 +30,7 @@ public function resetContentRepositoryRegistry(): void public function newProjectionIsFoundWhenConfigurationIsAdded() { $this->fakeProjection->expects(self::exactly(2))->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); $this->subscriptionEngine->setup(); @@ -43,10 +43,10 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); - $newFakeProjection->expects(self::exactly(3))->method('status')->willReturnOnConsecutiveCalls( - ProjectionStatus::setupRequired('Set me up'), - ProjectionStatus::ok(), - ProjectionStatus::ok(), + $newFakeProjection->expects(self::exactly(3))->method('setUpStatus')->willReturnOnConsecutiveCalls( + ProjectionSetupStatus::setupRequired('Set me up'), + ProjectionSetupStatus::ok(), + ProjectionSetupStatus::ok(), ); FakeProjectionFactory::setProjection( @@ -73,12 +73,12 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() self::assertNull($result->errors); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Set me up') + setupStatus: ProjectionSetupStatus::setupRequired('Set me up') ), $this->subscriptionStatus('Vendor.Package:NewFakeProjection') ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php index cbebb621ceb..b5eb56b12d9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -16,7 +16,7 @@ final class SubscriptionResetTest extends AbstractSubscriptionEngineTestCase public function filteringReset() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index d3b3e6d5ada..4bfaf10ad59 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -4,10 +4,10 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -23,30 +23,30 @@ public function setupOnEmptyDatabase() $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::exactly(2))->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); - $expected = SubscriptionAndProjectionStatuses::fromArray([ - SubscriptionAndProjectionStatus::create( + $expected = SubscriptionStatuses::fromArray([ + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('contentGraph'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ), - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ), - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::ok(), + setupStatus: ProjectionSetupStatus::ok(), ), ]); @@ -65,7 +65,7 @@ public function setupOnEmptyDatabase() public function filteringSetup() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); $this->eventStore->setup(); @@ -77,12 +77,12 @@ public function filteringSetup() $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::ok() + setupStatus: ProjectionSetupStatus::ok() ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -92,7 +92,7 @@ public function filteringSetup() public function setupIsInvokedForBootingSubscribers() { $this->fakeProjection->expects(self::exactly(2))->method('setUp'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); // hard reset, so that the tests actually need sql migrations $this->secondFakeProjection->dropTables(); @@ -110,12 +110,12 @@ public function setupIsInvokedForBootingSubscribers() $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements') ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -133,7 +133,7 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); // hard reset, so that the tests actually need sql migrations $this->secondFakeProjection->dropTables(); @@ -144,12 +144,12 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() self::assertNull($result->errors); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements') ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -176,12 +176,12 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); self::assertEquals( - SubscriptionAndProjectionStatus::create( + ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::ACTIVE, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: null, - projectionStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements') ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -198,19 +198,19 @@ public function failingSetupWillMarkProjectionAsErrored() $this->fakeProjection->expects(self::once())->method('setUp')->willThrowException( $exception = new \RuntimeException('Projection could not be setup') ); - $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Needs setup')); + $this->fakeProjection->expects(self::once())->method('setUpStatus')->willReturn(ProjectionSetupStatus::setupRequired('Needs setup')); $this->eventStore->setup(); $result = $this->subscriptionEngine->setup(); self::assertSame('Projection could not be setup', $result->errors?->first()->message); - $expectedFailure = SubscriptionAndProjectionStatus::create( + $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::NEW, $exception), - projectionStatus: ProjectionStatus::setupRequired('Needs setup'), + setupStatus: ProjectionSetupStatus::setupRequired('Needs setup'), ); self::assertEquals( diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 7e195ff4644..1b5a8519102 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -26,9 +26,9 @@ interface ProjectionInterface public function setUp(): void; /** - * Determines the status of the projection (not to confuse with {@see getState()}) + * Determines the setup status of the projection */ - public function status(): ProjectionStatus; + public function setUpStatus(): ProjectionSetupStatus; public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatus.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatus.php new file mode 100644 index 00000000000..fd9d4061713 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatus.php @@ -0,0 +1,38 @@ +type, $details); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 1db74979dae..8ea8f51c241 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -8,7 +8,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\Error\Messages\Error; use Neos\EventStore\EventStoreInterface; @@ -55,7 +55,7 @@ public function eventStoreStatus(): EventStoreStatus return $this->eventStore->status(); } - public function subscriptionStatuses(): SubscriptionAndProjectionStatuses + public function subscriptionStatuses(): SubscriptionStatuses { return $this->subscriptionEngine->subscriptionStatuses(); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 5d8bb9ac086..dadca3f87ad 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -11,8 +11,8 @@ use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionAndProjectionStatuses; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; @@ -105,26 +105,26 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } - public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null): SubscriptionAndProjectionStatuses + public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null): SubscriptionStatuses { $statuses = []; try { $subscriptions = $this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()); } catch (TableNotFoundException) { // the schema is not setup - thus there are no subscribers - return SubscriptionAndProjectionStatuses::createEmpty(); + return SubscriptionStatuses::createEmpty(); } foreach ($subscriptions as $subscription) { $subscriber = $this->subscribers->contain($subscription->id) ? $this->subscribers->get($subscription->id) : null; - $statuses[] = SubscriptionAndProjectionStatus::create( + $statuses[] = ProjectionSubscriptionStatus::create( subscriptionId: $subscription->id, subscriptionStatus: $subscription->status, subscriptionPosition: $subscription->position, subscriptionError: $subscription->error, - projectionStatus: $subscriber?->projection->status(), + setupStatus: $subscriber?->projection->setUpStatus(), ); } - return SubscriptionAndProjectionStatuses::fromArray($statuses); + return SubscriptionStatuses::fromArray($statuses); } private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php similarity index 70% rename from Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php rename to Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php index 56fca0ac88a..eb9a2218c1d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php @@ -4,30 +4,33 @@ namespace Neos\ContentRepository\Core\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\EventStore\Model\Event\SequenceNumber; /** * @api */ -final readonly class SubscriptionAndProjectionStatus +final readonly class ProjectionSubscriptionStatus { private function __construct( public SubscriptionId $subscriptionId, public SubscriptionStatus $subscriptionStatus, public SequenceNumber $subscriptionPosition, public SubscriptionError|null $subscriptionError, - public ProjectionStatus|null $projectionStatus, + public ProjectionSetupStatus|null $setupStatus, ) { } + /** + * @internal + */ public static function create( SubscriptionId $subscriptionId, SubscriptionStatus $subscriptionStatus, SequenceNumber $subscriptionPosition, SubscriptionError|null $subscriptionError, - ProjectionStatus|null $projectionStatus + ProjectionSetupStatus|null $setupStatus ): self { - return new self($subscriptionId, $subscriptionStatus, $subscriptionPosition, $subscriptionError, $projectionStatus); + return new self($subscriptionId, $subscriptionStatus, $subscriptionPosition, $subscriptionError, $setupStatus); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php index ba40fbddb1a..17288fc927a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php @@ -7,6 +7,12 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionId; /** + * A collection of the registered subscribers. + * + * Currently only projections are the available subscribers, but when the concept is extended, + * other *Subscriber value objects will also be hold in this set. + * Like a possible "ListeningSubscriber" to only listen to events without the capabilities of a full-blown projection. + * * @implements \IteratorAggregate * @internal */ diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php similarity index 56% rename from Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php rename to Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php index 1c3351ed666..b372bc4062b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionAndProjectionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php @@ -4,19 +4,27 @@ namespace Neos\ContentRepository\Core\Subscription; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; + /** + * A collection of the states of the subscribers. + * + * Currently only projections are the available subscribers, but when the concept is extended, + * other *SubscriptionStatus value objects will also be hold in this set. + * Like "ListeningSubscriptionStatus" if a "ListeningSubscriber" is introduced. + * * @api - * @implements \IteratorAggregate + * @implements \IteratorAggregate */ -final readonly class SubscriptionAndProjectionStatuses implements \IteratorAggregate +final readonly class SubscriptionStatuses implements \IteratorAggregate { /** - * @var array $statuses + * @var array $statuses */ private array $statuses; private function __construct( - SubscriptionAndProjectionStatus ...$statuses, + ProjectionSubscriptionStatus ...$statuses, ) { $this->statuses = $statuses; } @@ -27,14 +35,14 @@ public static function createEmpty(): self } /** - * @param array $statuses + * @param array $statuses */ public static function fromArray(array $statuses): self { return new self(...$statuses); } - public function first(): ?SubscriptionAndProjectionStatus + public function first(): ?ProjectionSubscriptionStatus { foreach ($this->statuses as $status) { return $status; @@ -58,7 +66,7 @@ public function isOk(): bool if ($status->subscriptionStatus === SubscriptionStatus::ERROR) { return false; } - if ($status->projectionStatus?->type !== ProjectionStatusType::OK) { + if ($status->setupStatus?->type !== ProjectionSetupStatusType::OK) { return false; } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index d7a03eed6bb..6f1d1b0f04c 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -3,7 +3,7 @@ namespace Neos\ContentRepositoryRegistry\Command; -use Neos\ContentRepository\Core\Projection\ProjectionStatusType; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -73,21 +73,20 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->outputLine(); $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); - foreach ($subscriptionStatuses as $subscriptionStatus) { + foreach ($subscriptionStatuses as $projectionSubscriptionStatus) { // todo reimplement 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 - $this->output('Projection "%s": ', [$subscriptionStatus->subscriptionId->value]); - $projectionStatus = $subscriptionStatus->projectionStatus; + $this->output('Projection "%s": ', [$projectionSubscriptionStatus->subscriptionId->value]); + $projectionStatus = $projectionSubscriptionStatus->setupStatus; if ($projectionStatus === null) { $this->outputLine('No status available.'); // todo this means detached? continue; } $this->outputLine(match ($projectionStatus->type) { - ProjectionStatusType::OK => 'OK', - ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', - ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', - ProjectionStatusType::ERROR => 'ERROR', + ProjectionSetupStatusType::OK => 'OK', + ProjectionSetupStatusType::SETUP_REQUIRED => 'Setup required!', + ProjectionSetupStatusType::ERROR => 'ERROR', }); - if ($verbose && ($projectionStatus->type !== ProjectionStatusType::OK || $projectionStatus->details)) { + if ($verbose && ($projectionStatus->type !== ProjectionSetupStatusType::OK || $projectionStatus->details)) { $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); foreach ($lines as $line) { $this->outputLine(' ' . $line); diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index 016814020bc..194dca2bab1 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -28,7 +28,7 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\EventStore\Model\EventEnvelope; @@ -65,22 +65,22 @@ public function setUp(): void } } - public function status(): ProjectionStatus + public function setUpStatus(): ProjectionSetupStatus { try { $this->dbal->connect(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionStatus::ok(); + return ProjectionSetupStatus::ok(); } /** diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index ead06f7b588..269ebf29598 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -40,7 +40,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\EventStore\Model\EventEnvelope; @@ -75,22 +75,22 @@ public function setUp(): void } } - public function status(): ProjectionStatus + public function setUpStatus(): ProjectionSetupStatus { try { $this->dbal->connect(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionStatus::ok(); + return ProjectionSetupStatus::ok(); } /** From 8ff0f61fae1f52cb34dd6fb49a407a3995dd62d4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 19:54:48 +0100 Subject: [PATCH 077/142] TASK: Introduce `DetachedSubscriptionStatus` as the projection setup status cannot be calculated ... and when extending the system to support multiple subscribers, we cannot know their original classification but have to use a special empty placeholder like: `DetachedSubscriptionStatus` This also makes the `$projectionStatus === null` detached case more explicit when using status. --- .../AbstractSubscriptionEngineTestCase.php | 3 +- .../SubscriptionDetachedStatusTest.php | 31 ++++++++------- .../DetachedSubscriptionStatus.php | 34 ++++++++++++++++ .../Engine/SubscriptionEngine.php | 13 ++++++- .../ProjectionSubscriptionStatus.php | 4 +- .../Subscription/SubscriptionStatuses.php | 26 ++++++++----- .../Classes/Command/CrCommandController.php | 39 +++++++++++-------- 7 files changed, 104 insertions(+), 46 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index bc48af8b01c..2ca859b0182 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -17,6 +17,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; @@ -134,7 +135,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); } - final protected function subscriptionStatus(string $subscriptionId): ?ProjectionSubscriptionStatus + final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null { return $this->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 68a00a94969..88dbd65e056 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -5,6 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -45,8 +46,15 @@ public function projectionIsDetachedOnCatchupActive() $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); $this->setupContentRepositoryDependencies($this->contentRepository->id); - // todo status is stale??, should be DETACHED, and also cr:setup should marke detached projections?!! - // $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + self::assertEquals( + DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + // the state is still active as we do not mutate it during the setup call! + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::none() + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); $this->fakeProjection->expects(self::never())->method('apply'); // catchup to mark detached subscribers @@ -54,15 +62,12 @@ public function projectionIsDetachedOnCatchupActive() // todo result should reflect that there was an detachment? Throw error in CR? self::assertEquals(ProcessedResult::success(1), $result); - $expectedDetachedState = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), - subscriptionStatus: SubscriptionStatus::DETACHED, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - setupStatus: null // not calculate-able at this point! - ); self::assertEquals( - $expectedDetachedState, + $expectedDetachedState = DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::none() + ), $this->subscriptionStatus('Vendor.Package:FakeProjection') ); @@ -120,12 +125,10 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() // todo result should reflect that there was an detachment? self::assertNull($result->errors); - $expectedDetachedState = ProjectionSubscriptionStatus::create( + $expectedDetachedState = DetachedSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::DETACHED, - subscriptionPosition: SequenceNumber::fromInteger(1), - subscriptionError: null, - setupStatus: null // not calculate-able at this point! + subscriptionPosition: SequenceNumber::fromInteger(1) ); self::assertEquals( $expectedDetachedState, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php new file mode 100644 index 00000000000..fd61ddf9f7a --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php @@ -0,0 +1,34 @@ +subscribers->contain($subscription->id) ? $this->subscribers->get($subscription->id) : null; + if (!$this->subscribers->contain($subscription->id)) { + $statuses[] = DetachedSubscriptionStatus::create( + $subscription->id, + $subscription->status, + $subscription->position + ); + continue; + } + $subscriber = $this->subscribers->get($subscription->id); $statuses[] = ProjectionSubscriptionStatus::create( subscriptionId: $subscription->id, subscriptionStatus: $subscription->status, subscriptionPosition: $subscription->position, subscriptionError: $subscription->error, - setupStatus: $subscriber?->projection->setUpStatus(), + setupStatus: $subscriber->projection->setUpStatus(), ); } return SubscriptionStatuses::fromArray($statuses); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php index eb9a2218c1d..13c366fe308 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php @@ -17,7 +17,7 @@ private function __construct( public SubscriptionStatus $subscriptionStatus, public SequenceNumber $subscriptionPosition, public SubscriptionError|null $subscriptionError, - public ProjectionSetupStatus|null $setupStatus, + public ProjectionSetupStatus $setupStatus, ) { } @@ -29,7 +29,7 @@ public static function create( SubscriptionStatus $subscriptionStatus, SequenceNumber $subscriptionPosition, SubscriptionError|null $subscriptionError, - ProjectionSetupStatus|null $setupStatus + ProjectionSetupStatus $setupStatus ): self { return new self($subscriptionId, $subscriptionStatus, $subscriptionPosition, $subscriptionError, $setupStatus); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php index b372bc4062b..d8dcddabc14 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php @@ -13,18 +13,21 @@ * other *SubscriptionStatus value objects will also be hold in this set. * Like "ListeningSubscriptionStatus" if a "ListeningSubscriber" is introduced. * + * In case the subscriber is not available currently - e.g. will be detached, a {@see DetachedSubscriptionStatus} will be returned. + * Note that ProjectionSubscriptionStatus with status == Detached can be returned, if the projection is installed again! + * * @api - * @implements \IteratorAggregate + * @implements \IteratorAggregate */ final readonly class SubscriptionStatuses implements \IteratorAggregate { /** - * @var array $statuses + * @var array $statuses */ private array $statuses; private function __construct( - ProjectionSubscriptionStatus ...$statuses, + ProjectionSubscriptionStatus|DetachedSubscriptionStatus ...$statuses, ) { $this->statuses = $statuses; } @@ -35,14 +38,14 @@ public static function createEmpty(): self } /** - * @param array $statuses + * @param array $statuses */ public static function fromArray(array $statuses): self { return new self(...$statuses); } - public function first(): ?ProjectionSubscriptionStatus + public function first(): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null { foreach ($this->statuses as $status) { return $status; @@ -63,11 +66,14 @@ public function isEmpty(): bool public function isOk(): bool { foreach ($this->statuses as $status) { - if ($status->subscriptionStatus === SubscriptionStatus::ERROR) { - return false; - } - if ($status->setupStatus?->type !== ProjectionSetupStatusType::OK) { - return false; + // ignores DetachedSubscriptionStatus + if ($status instanceof ProjectionSubscriptionStatus) { + if ($status->subscriptionStatus === SubscriptionStatus::ERROR) { + return false; + } + if ($status->setupStatus->type !== ProjectionSetupStatusType::OK) { + return false; + } } } return true; diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 6f1d1b0f04c..fdae52a57ad 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -6,7 +6,10 @@ use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Annotations as Flow; @@ -73,25 +76,27 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->outputLine(); $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); - foreach ($subscriptionStatuses as $projectionSubscriptionStatus) { - // todo reimplement 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 - $this->output('Projection "%s": ', [$projectionSubscriptionStatus->subscriptionId->value]); - $projectionStatus = $projectionSubscriptionStatus->setupStatus; - if ($projectionStatus === null) { - $this->outputLine('No status available.'); // todo this means detached? - continue; + foreach ($subscriptionStatuses as $subscriptionStatus) { + if ($subscriptionStatus instanceof DetachedSubscriptionStatus) { + $this->output('Subscriber "%s" %s detached.', [$subscriptionStatus->subscriptionId->value, $subscriptionStatus->subscriptionStatus === SubscriptionStatus::DETACHED ? 'is' : 'will be']); } - $this->outputLine(match ($projectionStatus->type) { - ProjectionSetupStatusType::OK => 'OK', - ProjectionSetupStatusType::SETUP_REQUIRED => 'Setup required!', - ProjectionSetupStatusType::ERROR => 'ERROR', - }); - if ($verbose && ($projectionStatus->type !== ProjectionSetupStatusType::OK || $projectionStatus->details)) { - $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); - foreach ($lines as $line) { - $this->outputLine(' ' . $line); + if ($subscriptionStatus instanceof ProjectionSubscriptionStatus) { + // todo reimplement 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 + $this->output('Projection "%s": ', [$subscriptionStatus->subscriptionId->value]); + $projectionStatus = $subscriptionStatus->setupStatus; + + $this->outputLine(match ($projectionStatus->type) { + ProjectionSetupStatusType::OK => 'OK', + ProjectionSetupStatusType::SETUP_REQUIRED => 'Setup required!', + ProjectionSetupStatusType::ERROR => 'ERROR', + }); + if ($verbose && ($projectionStatus->type !== ProjectionSetupStatusType::OK || $projectionStatus->details)) { + $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' ' . $line); + } + $this->outputLine(); } - $this->outputLine(); } } if ($eventStoreStatus->type !== StatusType::OK || !$subscriptionStatuses->isOk()) { From 9675572da094d5ff7e1fc5ba974fb8356991cfb6 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 19:58:06 +0100 Subject: [PATCH 078/142] TASK: Inline `pruneAllWorkspacesAndContentStreamsFromEventStream` into CR Maintainer --- .../Service/ContentRepositoryMaintainer.php | 62 +++++++++++++++++-- .../ContentRepositoryMaintainerFactory.php | 7 +-- .../Classes/Service/ContentStreamPruner.php | 55 ---------------- 3 files changed, 59 insertions(+), 65 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 8ea8f51c241..bf83d89ea80 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -5,6 +5,8 @@ namespace Neos\ContentRepository\Core\Service; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; +use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; @@ -12,7 +14,11 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\Error\Messages\Error; use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\Event\EventType; +use Neos\EventStore\Model\Event\EventTypes; +use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventStore\Status as EventStoreStatus; +use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\VirtualStreamName; /** @@ -27,8 +33,7 @@ */ public function __construct( private EventStoreInterface $eventStore, - private SubscriptionEngine $subscriptionEngine, - private ContentStreamPruner $contentStreamPruner + private SubscriptionEngine $subscriptionEngine ) { } @@ -119,8 +124,13 @@ public function catchupProjection(SubscriptionId $subscriptionId, \Closure|null */ public function prune(): Error|null { - // todo move pruneAllWorkspacesAndContentStreamsFromEventStream here. - $this->contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); + // prune all streams: + foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { + $this->eventStore->deleteStream($contentStreamStreamName); + } + foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { + $this->eventStore->deleteStream($workspaceStreamName); + } $resetResult = $this->subscriptionEngine->reset(); if ($resetResult->errors !== null) { return self::createErrorForReason('reset', $resetResult->errors); @@ -143,4 +153,48 @@ private static function createErrorForReason(string $method, Errors $errors): Er } return new Error(join("\n", $message)); } + + /** + * @return list + */ + private function findAllContentStreamStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('ContentStreamWasCreated'), + EventType::fromString('ContentStreamWasForked') + ) + ) + ); + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } + + /** + * @return list + */ + private function findAllWorkspaceStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('RootWorkspaceWasCreated'), + EventType::fromString('WorkspaceWasCreated') + ) + ) + ); + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php index e4ddcf4e76d..234b01fcb1d 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php @@ -18,12 +18,7 @@ public function build( ): ContentRepositoryMaintainer { return new ContentRepositoryMaintainer( $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->subscriptionEngine, - new ContentStreamPruner( - $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventNormalizer, - $serviceFactoryDependencies->subscriptionEngine, - ) + $serviceFactoryDependencies->subscriptionEngine ); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index a71f8a4d7d2..5f1991c0a77 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -4,7 +4,6 @@ namespace Neos\ContentRepository\Core\Service; -use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; @@ -27,7 +26,6 @@ use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventType; use Neos\EventStore\Model\Event\EventTypes; -use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -200,15 +198,6 @@ public function pruneRemovedFromEventStream(\Closure $outputFn): void } } - public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void - { - foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { - $this->eventStore->deleteStream($contentStreamStreamName); - } - foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { - $this->eventStore->deleteStream($workspaceStreamName); - } - } /** * Find all removed content streams that are unused in the event stream @@ -411,48 +400,4 @@ private function findAllContentStreams(): array } return $cs; } - - /** - * @return list - */ - private function findAllContentStreamStreamNames(): array - { - $events = $this->eventStore->load( - VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), - EventStreamFilter::create( - EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch - EventType::fromString('ContentStreamWasCreated'), - EventType::fromString('ContentStreamWasForked') - ) - ) - ); - $allStreamNames = []; - foreach ($events as $eventEnvelope) { - $allStreamNames[] = $eventEnvelope->streamName; - } - return array_unique($allStreamNames, SORT_REGULAR); - } - - /** - * @return list - */ - private function findAllWorkspaceStreamNames(): array - { - $events = $this->eventStore->load( - VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), - EventStreamFilter::create( - EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch - EventType::fromString('RootWorkspaceWasCreated'), - EventType::fromString('WorkspaceWasCreated') - ) - ) - ); - $allStreamNames = []; - foreach ($events as $eventEnvelope) { - $allStreamNames[] = $eventEnvelope->streamName; - } - return array_unique($allStreamNames, SORT_REGULAR); - } } From 655ac3c43f8d15f074c91e5bfbda6cabc77d06ca Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:26:57 +0100 Subject: [PATCH 079/142] TASK: Reimplement 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 Under consideration of the new `ProjectionSubscriptionStatus` --- .../Subscription/SubscriptionStatuses.php | 18 ----- .../Classes/Command/CrCommandController.php | 73 +++++++++++++++---- .../Domain/Service/SiteImportService.php | 13 +++- 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php index d8dcddabc14..e6860cdd045 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php @@ -4,8 +4,6 @@ namespace Neos\ContentRepository\Core\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; - /** * A collection of the states of the subscribers. * @@ -62,20 +60,4 @@ public function isEmpty(): bool { return $this->statuses === []; } - - public function isOk(): bool - { - foreach ($this->statuses as $status) { - // ignores DetachedSubscriptionStatus - if ($status instanceof ProjectionSubscriptionStatus) { - if ($status->subscriptionStatus === SubscriptionStatus::ERROR) { - return false; - } - if ($status->setupStatus->type !== ProjectionSetupStatusType::OK) { - return false; - } - } - } - return true; - } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index fdae52a57ad..ee609417152 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -62,44 +62,85 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - $eventStoreStatus = $contentRepositoryMaintainer->eventStoreStatus(); + $hasErrors = false; + $setupRequired = false; + $bootingRequired = false; $this->output('Event Store: '); $this->outputLine(match ($eventStoreStatus->type) { StatusType::OK => 'OK', StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); + $hasErrors |= $eventStoreStatus->type === StatusType::ERROR; if ($verbose && $eventStoreStatus->details !== '') { $this->outputFormatted($eventStoreStatus->details, [], 2); } $this->outputLine(); - + $this->outputLine('Subscriptions:'); $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); - foreach ($subscriptionStatuses as $subscriptionStatus) { - if ($subscriptionStatus instanceof DetachedSubscriptionStatus) { - $this->output('Subscriber "%s" %s detached.', [$subscriptionStatus->subscriptionId->value, $subscriptionStatus->subscriptionStatus === SubscriptionStatus::DETACHED ? 'is' : 'will be']); + if ($subscriptionStatuses->isEmpty()) { + $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); + $this->quit(1); + } + foreach ($subscriptionStatuses as $status) { + if ($status instanceof DetachedSubscriptionStatus) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Subscription: '); + $this->output('%s DETACHED', [$status->subscriptionId->value, $status->subscriptionStatus === SubscriptionStatus::DETACHED ? 'is' : 'will be']); + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); } - if ($subscriptionStatus instanceof ProjectionSubscriptionStatus) { - // todo reimplement 40e8d35e09ee690406c6a9cfc823c775d4ee3b51 - $this->output('Projection "%s": ', [$subscriptionStatus->subscriptionId->value]); - $projectionStatus = $subscriptionStatus->setupStatus; - - $this->outputLine(match ($projectionStatus->type) { + if ($status instanceof ProjectionSubscriptionStatus) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Projection: '); + $this->output(match ($status->subscriptionStatus) { + SubscriptionStatus::NEW => 'NEW', + SubscriptionStatus::BOOTING => 'BOOTING', + SubscriptionStatus::ACTIVE => 'ACTIVE', + SubscriptionStatus::DETACHED => 'DETACHED', + SubscriptionStatus::ERROR => 'ERROR', + }); + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $bootingRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; + // detached can be reattached via setup: + $setupRequired |= $status->subscriptionStatus === SubscriptionStatus::DETACHED; + if ($verbose && $status->subscriptionError !== null) { + $lines = explode(chr(10), $status->subscriptionError->errorMessage ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' %s', [$line]); + } + } + $this->output(' Setup: '); + $this->outputLine(match ($status->setupStatus->type) { ProjectionSetupStatusType::OK => 'OK', - ProjectionSetupStatusType::SETUP_REQUIRED => 'Setup required!', + ProjectionSetupStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', ProjectionSetupStatusType::ERROR => 'ERROR', }); - if ($verbose && ($projectionStatus->type !== ProjectionSetupStatusType::OK || $projectionStatus->details)) { - $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); + $hasErrors |= $status->setupStatus->type === ProjectionSetupStatusType::ERROR; + $setupRequired |= $status->setupStatus->type === ProjectionSetupStatusType::SETUP_REQUIRED; + if ($verbose && ($status->setupStatus->type !== ProjectionSetupStatusType::OK || $status->setupStatus->details)) { + $lines = explode(chr(10), $status->setupStatus->details ?: 'No details available.'); foreach ($lines as $line) { - $this->outputLine(' ' . $line); + $this->outputLine(' ' . $line); } $this->outputLine(); } } } - if ($eventStoreStatus->type !== StatusType::OK || !$subscriptionStatuses->isOk()) { + if ($verbose) { + $this->outputLine(); + if ($setupRequired) { + $this->outputLine('Setup required, please run ./flow cr:setup'); + } + if ($bootingRequired) { + $this->outputLine('Catchup needed for projections, please run ./flow cr:projectioncatchup [projection-name]'); + } + if ($hasErrors) { + $this->outputLine('Some projections are not okay'); + } + } + if ($hasErrors) { $this->quit(1); } } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index e0489cc6440..6d0bc8f7b44 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -17,10 +17,12 @@ use Doctrine\DBAL\Exception as DBALException; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; +use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -94,10 +96,17 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $contentRepositoryMaintainer, ContentRepositoryId $contentRepositoryId): void { $eventStoreStatus = $contentRepositoryMaintainer->eventStoreStatus(); - $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); - if ($eventStoreStatus->type !== StatusType::OK || !$subscriptionStatuses->isOk()) { + if ($eventStoreStatus->type !== StatusType::OK) { throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepositoryId->value)); } + $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); + foreach ($subscriptionStatuses as $status) { + if ($status instanceof ProjectionSubscriptionStatus) { + if ($status->setupStatus->type !== ProjectionSetupStatusType::OK) { + throw new \RuntimeException(sprintf('Projection %s in content repository %s is not setup correctly, please run `./flow cr:setup`', $status->subscriptionId->value, $contentRepositoryId->value)); + } + } + } } private function requireDataBaseSchemaToBeSetup(): void From e235e69652037ee2f8273795229bc94efc686df4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 24 Nov 2024 21:12:26 +0100 Subject: [PATCH 080/142] TASK: Document new `ContentRepositoryMaintainer` --- .../Service/ContentRepositoryMaintainer.php | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index bf83d89ea80..e1adc7dc2b3 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -4,6 +4,7 @@ namespace Neos\ContentRepository\Core\Service; +use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; @@ -22,7 +23,33 @@ use Neos\EventStore\Model\EventStream\VirtualStreamName; /** - * API to set up and manage a content repository + * Set up and manage a content repository + * + * Initialisation / Tear down + * -------------------------- + * The method {@see setUp} sets up the content repository like event store and projection database tables. + * It is non-destructive. + * + * Resetting a content repository with {@see prune} method will purge the event stream and reset all projection states. + * + * Staus information + * ----------------- + * The status of the content repository e.g. if a setup is required or if all subscriptions are active and their position + * can be examined with two methods: + * + * The event store status is available via {@see eventStoreStatus}, while the subscription status are returned + * via {@see subscriptionStatuses}. Further documentation in {@see SubscriptionStatuses}. + * + * Replay / Catchup of projections + * ------------------------------- + * The methods {@see replayProjection}, {@see replayAllProjections} and {@see catchupProjection} + * can be leveraged to interact with the projection catchup. In the happy path no interaction is necessary, + * as {@see ContentRepository::handle()} triggers the projections after applying the events. + * + * For initialising on a new database - which contains events already - a replay will make sure that the projections + * are emptied and reapply the events. + * + * The explicit catchup of a projection is only required when adding new projections after installation, of after fixing a projection error. * * @api */ @@ -102,6 +129,8 @@ public function replayAllProjections(\Closure|null $progressCallback = null): Er */ public function catchupProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { + // todo if a projection is in error state and will be explicit caught up here we might as well attempt that without saying the user should setup? + // also setup then can avoid doing the repairing! $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); if ($bootResult->errors !== null) { return self::createErrorForReason('catchup', $bootResult->errors); From 611ca3794fba12c40db4802291d2a14938d77204 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:13:50 +0100 Subject: [PATCH 081/142] TASK: Rename `ProjectionSetupStatus` back to `ProjectionStatus` reverts partially bace8ffed072862b24b5763b51b38a6b66020fd5 --- .../DoctrineDbalContentGraphProjection.php | 12 ++++---- .../Projection/HypergraphProjection.php | 12 ++++---- .../TestSuite/DebugEventProjection.php | 8 +++--- .../AbstractSubscriptionEngineTestCase.php | 4 +-- .../Subscription/CatchUpHookErrorTest.php | 6 ++-- .../Subscription/ProjectionErrorTest.php | 16 +++++------ .../SubscriptionActiveStatusTest.php | 8 +++--- .../SubscriptionBootingStatusTest.php | 6 ++-- .../SubscriptionDetachedStatusTest.php | 8 +++--- .../SubscriptionGetStatusTest.php | 10 +++---- .../SubscriptionNewStatusTest.php | 14 +++++----- .../Subscription/SubscriptionResetTest.php | 4 +-- .../Subscription/SubscriptionSetupTest.php | 28 +++++++++---------- .../Projection/ProjectionInterface.php | 4 +-- .../Projection/ProjectionSetupStatusType.php | 15 ---------- ...onSetupStatus.php => ProjectionStatus.php} | 14 ++++++---- .../Projection/ProjectionStatusType.php | 25 +++++++++++++++++ .../Engine/SubscriptionEngine.php | 2 +- .../ProjectionSubscriptionStatus.php | 6 ++-- .../Classes/Command/CrCommandController.php | 14 +++++----- .../Domain/Service/SiteImportService.php | 4 +-- .../Projection/DocumentUriPathProjection.php | 12 ++++---- .../ChangeProjection.php | 12 ++++---- 23 files changed, 129 insertions(+), 115 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatusType.php rename Neos.ContentRepository.Core/Classes/Projection/{ProjectionSetupStatus.php => ProjectionStatus.php} (56%) create mode 100644 Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index a436c61d14f..4e998fde99a 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -67,7 +67,7 @@ use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeTags; use Neos\ContentRepository\Core\Projection\ContentGraph\Timestamps; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Node\NodeName; @@ -112,23 +112,23 @@ public function setUp(): void } } - public function setUpStatus(): ProjectionSetupStatus + public function status(): ProjectionStatus { try { $this->dbal->connect(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionSetupStatus::ok(); + return ProjectionStatus::ok(); } public function resetState(): void diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 12640cfdd72..5961442b9e6 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -42,7 +42,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -89,22 +89,22 @@ public function setUp(): void '); } - public function setUpStatus(): ProjectionSetupStatus + public function status(): ProjectionStatus { try { $this->getDatabaseConnection()->connect(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionSetupStatus::ok(); + return ProjectionStatus::ok(); } /** diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 4fcd47f66fd..7f71796e96d 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -14,7 +14,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Annotations as Flow; @@ -52,13 +52,13 @@ public function setUp(): void } } - public function setUpStatus(): ProjectionSetupStatus + public function status(): ProjectionStatus { $requiredSqlStatements = $this->determineRequiredSqlStatements(); if ($requiredSqlStatements !== []) { - return ProjectionSetupStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); + return ProjectionStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); } - return ProjectionSetupStatus::ok(); + return ProjectionStatus::ok(); } /** diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 2ca859b0182..b80bf57dd81 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -14,7 +14,7 @@ use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; @@ -162,7 +162,7 @@ final protected function expectOkayStatus($subscriptionId, SubscriptionStatus $s subscriptionStatus: $status, subscriptionPosition: $sequenceNumber, subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ), $actual ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index eeb3300229e..3396ec98639 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; @@ -42,7 +42,7 @@ public function error_onBeforeEvent_projectionIsNotRun() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ); self::assertEmpty( @@ -88,7 +88,7 @@ public function error_onAfterEvent_projectionIsRolledBack() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ); self::assertEmpty( diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 03085882caf..53ded4f33c3 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -6,7 +6,7 @@ use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Command\CreateRootWorkspace; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\Subscription\Engine\Error; @@ -28,7 +28,7 @@ public function projectionWithError() $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -46,7 +46,7 @@ public function projectionWithError() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ); $result = $this->subscriptionEngine->catchUpActive(); @@ -80,7 +80,7 @@ public function fixFailedProjection() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); @@ -98,7 +98,7 @@ public function fixFailedProjection() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ); $result = $this->subscriptionEngine->catchUpActive(); @@ -158,7 +158,7 @@ public function projectionIsRolledBackAfterError() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ); self::assertEmpty( @@ -235,7 +235,7 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ); self::assertEquals( @@ -280,7 +280,7 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php index a23b4e0080d..421d9a4c212 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -24,7 +24,7 @@ public function setupProjectionsAndCatchup() $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); @@ -53,7 +53,7 @@ public function setupProjectionsAndCatchup() public function filteringCatchUpActive() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); @@ -88,7 +88,7 @@ public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->subscriptionEngine->setup(); $result = $this->subscriptionEngine->boot(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php index 7c8a005dd83..a89370bd57e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -24,7 +24,7 @@ public function existingEventStoreEventsAreCaughtUpOnBoot() $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); @@ -44,7 +44,7 @@ public function existingEventStoreEventsAreCaughtUpOnBoot() public function filteringCatchUpBoot() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 88dbd65e056..dabdcce9cac 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; @@ -27,7 +27,7 @@ public function resetContentRepositoryRegistry(): void public function projectionIsDetachedOnCatchupActive() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); $this->subscriptionEngine->setup(); @@ -99,7 +99,7 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() { $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); $this->subscriptionEngine->setup(); @@ -155,7 +155,7 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() subscriptionStatus: SubscriptionStatus::DETACHED, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! + setupStatus: ProjectionStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! ), $this->subscriptionStatus('Vendor.Package:FakeProjection') ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index 5090d3ee72f..4c6d7085568 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; @@ -46,7 +46,7 @@ public function statusOnEmptyDatabase() $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); - $this->fakeProjection->expects(self::once())->method('setUpStatus')->willReturn(ProjectionSetupStatus::setupRequired('fake needs setup.')); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); @@ -56,21 +56,21 @@ public function statusOnEmptyDatabase() subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ), ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::setupRequired('fake needs setup.'), + setupStatus: ProjectionStatus::setupRequired('fake needs setup.'), ), ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements'), + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), ), ]); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index 2d7db9869fa..ed2d57fe2b1 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -6,7 +6,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; @@ -30,7 +30,7 @@ public function resetContentRepositoryRegistry(): void public function newProjectionIsFoundWhenConfigurationIsAdded() { $this->fakeProjection->expects(self::exactly(2))->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); $this->subscriptionEngine->setup(); @@ -43,10 +43,10 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); - $newFakeProjection->expects(self::exactly(3))->method('setUpStatus')->willReturnOnConsecutiveCalls( - ProjectionSetupStatus::setupRequired('Set me up'), - ProjectionSetupStatus::ok(), - ProjectionSetupStatus::ok(), + $newFakeProjection->expects(self::exactly(3))->method('status')->willReturnOnConsecutiveCalls( + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::ok(), + ProjectionStatus::ok(), ); FakeProjectionFactory::setProjection( @@ -78,7 +78,7 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::setupRequired('Set me up') + setupStatus: ProjectionStatus::setupRequired('Set me up') ), $this->subscriptionStatus('Vendor.Package:NewFakeProjection') ); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php index b5eb56b12d9..cbebb621ceb 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -16,7 +16,7 @@ final class SubscriptionResetTest extends AbstractSubscriptionEngineTestCase public function filteringReset() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 4bfaf10ad59..5982db88625 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; @@ -23,7 +23,7 @@ public function setupOnEmptyDatabase() $this->fakeProjection->expects(self::once())->method('setUp'); $this->subscriptionEngine->setup(); - $this->fakeProjection->expects(self::exactly(2))->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); $expected = SubscriptionStatuses::fromArray([ @@ -32,21 +32,21 @@ public function setupOnEmptyDatabase() subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ), ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ), ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok(), + setupStatus: ProjectionStatus::ok(), ), ]); @@ -65,7 +65,7 @@ public function setupOnEmptyDatabase() public function filteringSetup() { $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); $this->eventStore->setup(); @@ -82,7 +82,7 @@ public function filteringSetup() subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::ok() + setupStatus: ProjectionStatus::ok() ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -92,7 +92,7 @@ public function filteringSetup() public function setupIsInvokedForBootingSubscribers() { $this->fakeProjection->expects(self::exactly(2))->method('setUp'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); // hard reset, so that the tests actually need sql migrations $this->secondFakeProjection->dropTables(); @@ -115,7 +115,7 @@ public function setupIsInvokedForBootingSubscribers() subscriptionStatus: SubscriptionStatus::BOOTING, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements') + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -133,7 +133,7 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() $this->fakeProjection->expects(self::exactly(2))->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); - $this->fakeProjection->expects(self::any())->method('setUpStatus')->willReturn(ProjectionSetupStatus::ok()); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); // hard reset, so that the tests actually need sql migrations $this->secondFakeProjection->dropTables(); @@ -149,7 +149,7 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() subscriptionStatus: SubscriptionStatus::NEW, subscriptionPosition: SequenceNumber::none(), subscriptionError: null, - setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements') + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -181,7 +181,7 @@ public function setupIsInvokedForPreviouslyActiveSubscribers() subscriptionStatus: SubscriptionStatus::ACTIVE, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: null, - setupStatus: ProjectionSetupStatus::setupRequired('Requires 1 SQL statements') + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); @@ -198,7 +198,7 @@ public function failingSetupWillMarkProjectionAsErrored() $this->fakeProjection->expects(self::once())->method('setUp')->willThrowException( $exception = new \RuntimeException('Projection could not be setup') ); - $this->fakeProjection->expects(self::once())->method('setUpStatus')->willReturn(ProjectionSetupStatus::setupRequired('Needs setup')); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Needs setup')); $this->eventStore->setup(); @@ -210,7 +210,7 @@ public function failingSetupWillMarkProjectionAsErrored() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::NEW, $exception), - setupStatus: ProjectionSetupStatus::setupRequired('Needs setup'), + setupStatus: ProjectionStatus::setupRequired('Needs setup'), ); self::assertEquals( diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 1b5a8519102..aff57389895 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -26,9 +26,9 @@ interface ProjectionInterface public function setUp(): void; /** - * Determines the setup status of the projection + * Determines the setup status of the projection. E.g. are the database tables created or any columns missing. */ - public function setUpStatus(): ProjectionSetupStatus; + public function status(): ProjectionStatus; public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatusType.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatusType.php deleted file mode 100644 index 3b6baf62696..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionSetupStatusType.php +++ /dev/null @@ -1,15 +0,0 @@ -status, subscriptionPosition: $subscription->position, subscriptionError: $subscription->error, - setupStatus: $subscriber->projection->setUpStatus(), + setupStatus: $subscriber->projection->status(), ); } return SubscriptionStatuses::fromArray($statuses); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php index 13c366fe308..0924c717b73 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\Core\Subscription; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\Event\SequenceNumber; /** @@ -17,7 +17,7 @@ private function __construct( public SubscriptionStatus $subscriptionStatus, public SequenceNumber $subscriptionPosition, public SubscriptionError|null $subscriptionError, - public ProjectionSetupStatus $setupStatus, + public ProjectionStatus $setupStatus, ) { } @@ -29,7 +29,7 @@ public static function create( SubscriptionStatus $subscriptionStatus, SequenceNumber $subscriptionPosition, SubscriptionError|null $subscriptionError, - ProjectionSetupStatus $setupStatus + ProjectionStatus $setupStatus ): self { return new self($subscriptionId, $subscriptionStatus, $subscriptionPosition, $subscriptionError, $setupStatus); } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index ee609417152..0b0c64e7c9b 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -3,7 +3,7 @@ namespace Neos\ContentRepositoryRegistry\Command; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; +use Neos\ContentRepository\Core\Projection\ProjectionStatusType; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; @@ -113,13 +113,13 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo } $this->output(' Setup: '); $this->outputLine(match ($status->setupStatus->type) { - ProjectionSetupStatusType::OK => 'OK', - ProjectionSetupStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', - ProjectionSetupStatusType::ERROR => 'ERROR', + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', + ProjectionStatusType::ERROR => 'ERROR', }); - $hasErrors |= $status->setupStatus->type === ProjectionSetupStatusType::ERROR; - $setupRequired |= $status->setupStatus->type === ProjectionSetupStatusType::SETUP_REQUIRED; - if ($verbose && ($status->setupStatus->type !== ProjectionSetupStatusType::OK || $status->setupStatus->details)) { + $hasErrors |= $status->setupStatus->type === ProjectionStatusType::ERROR; + $setupRequired |= $status->setupStatus->type === ProjectionStatusType::SETUP_REQUIRED; + if ($verbose && ($status->setupStatus->type !== ProjectionStatusType::OK || $status->setupStatus->details)) { $lines = explode(chr(10), $status->setupStatus->details ?: 'No details available.'); foreach ($lines as $line) { $this->outputLine(' ' . $line); diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 6d0bc8f7b44..6e7bc3bd57e 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -17,7 +17,7 @@ use Doctrine\DBAL\Exception as DBALException; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatusType; +use Neos\ContentRepository\Core\Projection\ProjectionStatusType; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -102,7 +102,7 @@ private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $ $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); foreach ($subscriptionStatuses as $status) { if ($status instanceof ProjectionSubscriptionStatus) { - if ($status->setupStatus->type !== ProjectionSetupStatusType::OK) { + if ($status->setupStatus->type !== ProjectionStatusType::OK) { throw new \RuntimeException(sprintf('Projection %s in content repository %s is not setup correctly, please run `./flow cr:setup`', $status->subscriptionId->value, $contentRepositoryId->value)); } } diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index 194dca2bab1..016814020bc 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -28,7 +28,7 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\EventStore\Model\EventEnvelope; @@ -65,22 +65,22 @@ public function setUp(): void } } - public function setUpStatus(): ProjectionSetupStatus + public function status(): ProjectionStatus { try { $this->dbal->connect(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionSetupStatus::ok(); + return ProjectionStatus::ok(); } /** diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index 269ebf29598..ead06f7b588 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -40,7 +40,7 @@ use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionSetupStatus; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\EventStore\Model\EventEnvelope; @@ -75,22 +75,22 @@ public function setUp(): void } } - public function setUpStatus(): ProjectionSetupStatus + public function status(): ProjectionStatus { try { $this->dbal->connect(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to connect to database: %s', $e->getMessage())); } try { $requiredSqlStatements = $this->determineRequiredSqlStatements(); } catch (\Throwable $e) { - return ProjectionSetupStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); + return ProjectionStatus::error(sprintf('Failed to determine required SQL statements: %s', $e->getMessage())); } if ($requiredSqlStatements !== []) { - return ProjectionSetupStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); + return ProjectionStatus::setupRequired(sprintf('The following SQL statement%s required: %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', implode(chr(10), $requiredSqlStatements))); } - return ProjectionSetupStatus::ok(); + return ProjectionStatus::ok(); } /** From 0b8a3b55755a19cc212a232d9491eda32e1251fd Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:17:20 +0100 Subject: [PATCH 082/142] TASK: Rename `SubscriptionStatuses` to `SubscriptionStatusCollection` --- .../AbstractSubscriptionEngineTestCase.php | 2 +- .../SubscriptionGetStatusTest.php | 8 +++---- .../Subscription/SubscriptionSetupTest.php | 6 ++--- .../Service/ContentRepositoryMaintainer.php | 8 +++---- .../Engine/SubscriptionEngine.php | 8 +++---- ...s.php => SubscriptionStatusCollection.php} | 24 +++++++++---------- .../Classes/Command/CrCommandController.php | 6 ++--- .../Domain/Service/SiteImportService.php | 4 ++-- 8 files changed, 33 insertions(+), 33 deletions(-) rename Neos.ContentRepository.Core/Classes/Subscription/{SubscriptionStatuses.php => SubscriptionStatusCollection.php} (74%) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index b80bf57dd81..bd9a8ce4de2 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -137,7 +137,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null { - return $this->subscriptionEngine->subscriptionStatuses(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + return $this->subscriptionEngine->subscriptionStatus(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } final protected function commitExampleContentStreamEvent(): void diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index 4c6d7085568..c8be53f9d98 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -8,7 +8,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; @@ -25,7 +25,7 @@ public function statusOnEmptyDatabase() keepSchema: false ); - $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); self::assertTrue($actualStatuses->isEmpty()); self::assertNull( @@ -48,9 +48,9 @@ public function statusOnEmptyDatabase() $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); - $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); - $expected = SubscriptionStatuses::fromArray([ + $expected = SubscriptionStatusCollection::fromArray([ ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('contentGraph'), subscriptionStatus: SubscriptionStatus::BOOTING, diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 5982db88625..dd021dc8183 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -7,7 +7,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -24,9 +24,9 @@ public function setupOnEmptyDatabase() $this->subscriptionEngine->setup(); $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); - $actualStatuses = $this->subscriptionEngine->subscriptionStatuses(); + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); - $expected = SubscriptionStatuses::fromArray([ + $expected = SubscriptionStatusCollection::fromArray([ ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('contentGraph'), subscriptionStatus: SubscriptionStatus::BOOTING, diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index e1adc7dc2b3..6929e706be7 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -11,7 +11,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\Error\Messages\Error; use Neos\EventStore\EventStoreInterface; @@ -38,7 +38,7 @@ * can be examined with two methods: * * The event store status is available via {@see eventStoreStatus}, while the subscription status are returned - * via {@see subscriptionStatuses}. Further documentation in {@see SubscriptionStatuses}. + * via {@see SubscriptionStatusCollection}. Further documentation in {@see SubscriptionStatusCollection}. * * Replay / Catchup of projections * ------------------------------- @@ -87,9 +87,9 @@ public function eventStoreStatus(): EventStoreStatus return $this->eventStore->status(); } - public function subscriptionStatuses(): SubscriptionStatuses + public function subscriptionStatus(): SubscriptionStatusCollection { - return $this->subscriptionEngine->subscriptionStatuses(); + return $this->subscriptionEngine->subscriptionStatus(); } public function replayProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 12a15e9d24c..e86537f47d8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -13,7 +13,7 @@ use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatuses; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; @@ -106,14 +106,14 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } - public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null): SubscriptionStatuses + public function subscriptionStatus(SubscriptionCriteria|null $criteria = null): SubscriptionStatusCollection { $statuses = []; try { $subscriptions = $this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()); } catch (TableNotFoundException) { // the schema is not setup - thus there are no subscribers - return SubscriptionStatuses::createEmpty(); + return SubscriptionStatusCollection::createEmpty(); } foreach ($subscriptions as $subscription) { if (!$this->subscribers->contain($subscription->id)) { @@ -133,7 +133,7 @@ public function subscriptionStatuses(SubscriptionCriteria|null $criteria = null) setupStatus: $subscriber->projection->status(), ); } - return SubscriptionStatuses::fromArray($statuses); + return SubscriptionStatusCollection::fromArray($statuses); } private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusCollection.php similarity index 74% rename from Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php rename to Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusCollection.php index e6860cdd045..4ee36d68ded 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatuses.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusCollection.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * A collection of the states of the subscribers. + * A collection of the status of the subscribers. * * Currently only projections are the available subscribers, but when the concept is extended, * other *SubscriptionStatus value objects will also be hold in this set. @@ -17,17 +17,17 @@ * @api * @implements \IteratorAggregate */ -final readonly class SubscriptionStatuses implements \IteratorAggregate +final readonly class SubscriptionStatusCollection implements \IteratorAggregate { /** - * @var array $statuses + * @var array $items */ - private array $statuses; + private array $items; private function __construct( - ProjectionSubscriptionStatus|DetachedSubscriptionStatus ...$statuses, + ProjectionSubscriptionStatus|DetachedSubscriptionStatus ...$items, ) { - $this->statuses = $statuses; + $this->items = $items; } public static function createEmpty(): self @@ -36,16 +36,16 @@ public static function createEmpty(): self } /** - * @param array $statuses + * @param array $items */ - public static function fromArray(array $statuses): self + public static function fromArray(array $items): self { - return new self(...$statuses); + return new self(...$items); } public function first(): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null { - foreach ($this->statuses as $status) { + foreach ($this->items as $status) { return $status; } return null; @@ -53,11 +53,11 @@ public function first(): ProjectionSubscriptionStatus|DetachedSubscriptionStatus public function getIterator(): \Traversable { - yield from $this->statuses; + yield from $this->items; } public function isEmpty(): bool { - return $this->statuses === []; + return $this->items === []; } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 0b0c64e7c9b..47e8a3198c2 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -78,12 +78,12 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo } $this->outputLine(); $this->outputLine('Subscriptions:'); - $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); - if ($subscriptionStatuses->isEmpty()) { + $subscriptionStatusCollection = $contentRepositoryMaintainer->subscriptionStatus(); + if ($subscriptionStatusCollection->isEmpty()) { $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); $this->quit(1); } - foreach ($subscriptionStatuses as $status) { + foreach ($subscriptionStatusCollection as $status) { if ($status instanceof DetachedSubscriptionStatus) { $this->outputLine(' %s:', [$status->subscriptionId->value]); $this->output(' Subscription: '); diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 6e7bc3bd57e..bd6a54d17d0 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -99,8 +99,8 @@ private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $ if ($eventStoreStatus->type !== StatusType::OK) { throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepositoryId->value)); } - $subscriptionStatuses = $contentRepositoryMaintainer->subscriptionStatuses(); - foreach ($subscriptionStatuses as $status) { + $subscriptionStatusCollection = $contentRepositoryMaintainer->subscriptionStatus(); + foreach ($subscriptionStatusCollection as $status) { if ($status instanceof ProjectionSubscriptionStatus) { if ($status->setupStatus->type !== ProjectionStatusType::OK) { throw new \RuntimeException(sprintf('Projection %s in content repository %s is not setup correctly, please run `./flow cr:setup`', $status->subscriptionId->value, $contentRepositoryId->value)); From 51f0cf6762ebe6aad99e3bbde3530a09e3e83ba3 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:21:28 +0100 Subject: [PATCH 083/142] TASK: Leave warning hint for why we do a replay see https://github.com/neos/neos-development-collection/pull/5378#discussion_r1855529168 --- Neos.Neos/Classes/Domain/Service/SiteImportService.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index bd6a54d17d0..7aa8c3afd2b 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -83,8 +83,9 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - // todo we do a replay here even though it will redo the live workspace creation. But otherwise the catchup hooks are not skipped because it seems like a regular catchup - 'Catchup all projections' => new ProjectionReplayProcessor($contentRepositoryMaintainer), + // WARNING! We do a replay here even though it will redo the live workspace creation. But otherwise the catchup hooks cannot determine that they need to be skipped as it seems like a regular catchup + // In case we allow to import events into other root workspaces, or don't expect live to be empty (see Import events), this would need to be adjusted, as otherwise existing data will be replayed + 'Replay all projections' => new ProjectionReplayProcessor($contentRepositoryMaintainer), ]); foreach ($processors as $processorLabel => $processor) { From a2a24111359df26dfd2239324a8b233d9bfb6804 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:33:53 +0100 Subject: [PATCH 084/142] TASK: Warn in `catchupProjection` if projection is not ready to be caught up --- .../Service/ContentRepositoryMaintainer.php | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 6929e706be7..64feda60dcf 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -11,6 +11,8 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\Error\Messages\Error; @@ -129,22 +131,29 @@ public function replayAllProjections(\Closure|null $progressCallback = null): Er */ public function catchupProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { - // todo if a projection is in error state and will be explicit caught up here we might as well attempt that without saying the user should setup? - // also setup then can avoid doing the repairing! - $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); - if ($bootResult->errors !== null) { - return self::createErrorForReason('catchup', $bootResult->errors); + $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionCriteria::create([$subscriptionId]))->first(); + + if ($subscriptionStatus === null) { + return new Error(sprintf('Projection "%s" is not registered.', $subscriptionId->value)); } - if ($bootResult->numberOfProcessedEvents > 0) { - // the projection was bootet + + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::BOOTING) { + $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); + if ($bootResult->errors !== null) { + return self::createErrorForReason('booting', $bootResult->errors); + } return null; } - // todo the projection was active, and we might still want to catch it up ... find reason for this? And combine boot and catchup? - $catchupResult = $this->subscriptionEngine->catchUpActive(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); - if ($catchupResult->errors !== null) { - return self::createErrorForReason('catchup', $catchupResult->errors); + + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::ACTIVE) { + $catchupResult = $this->subscriptionEngine->catchUpActive(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); + if ($catchupResult->errors !== null) { + return self::createErrorForReason('catchup', $catchupResult->errors); + } + return null; } - return null; + + return new Error(sprintf('Cannot catch-up projection "%s" with state %s. Please setup the content repository first.', $subscriptionId->value, $subscriptionStatus->subscriptionStatus->name)); } /** From 63d1589f1321ba26a1db718474d432a022c13270 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 19:06:06 +0100 Subject: [PATCH 085/142] TASK: Document `catchupProjection` correctly --- .../Service/ContentRepositoryMaintainer.php | 18 +++++++++--------- .../Classes/Command/CrCommandController.php | 6 ++++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 64feda60dcf..e7f5729fd12 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -49,9 +49,9 @@ * as {@see ContentRepository::handle()} triggers the projections after applying the events. * * For initialising on a new database - which contains events already - a replay will make sure that the projections - * are emptied and reapply the events. + * are emptied and reapply the events. After registering a new projection a setup is needed and a replay of this projection. * - * The explicit catchup of a projection is only required when adding new projections after installation, of after fixing a projection error. + * The explicit catchup of a projection is only required after fixing a projection error. * * @api */ @@ -121,13 +121,13 @@ public function replayAllProjections(\Closure|null $progressCallback = null): Er } /** - * Catchup one specific projection. + * Catchup one specific projection for debugging or fixing it. * - * The explicit catchup is required for new projections in the booting state. + * The explicit catchup is only needed for projections in the booting state with an advanced position. + * So in the case of an error or for a detached projection, the setup will move the projection back to booting keeping its current position. + * Running a full replay would work but might be overkill, instead this catchup will just attempt boot the projection back to active. * - * We don't offer an API to catch up all projections catchupAllProjection as we would have to distinct between booting or catchup if its active already. - * - * This method is only needed in rare cases for debugging or after installing a new projection or fixing its errors. + * We don't offer an API to catch up all projections at once (like catchupAllProjections). Instead, a replayAll can be used. */ public function catchupProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { @@ -201,7 +201,7 @@ private function findAllContentStreamStreamNames(): array VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), EventStreamFilter::create( EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch + // we are only interested in the creation events to limit the amount of events to fetch EventType::fromString('ContentStreamWasCreated'), EventType::fromString('ContentStreamWasForked') ) @@ -223,7 +223,7 @@ private function findAllWorkspaceStreamNames(): array VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), EventStreamFilter::create( EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch + // we are only interested in the creation events to limit the amount of events to fetch EventType::fromString('RootWorkspaceWasCreated'), EventType::fromString('WorkspaceWasCreated') ) diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 47e8a3198c2..6cc53d9d14d 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -247,9 +247,11 @@ public function projectionReplayAllCommand(string $contentRepository = 'default' } /** - * Catchup one specific projection. + * Catchup one specific projection for debugging or fixing it. * - * The explicit catchup is required for new projections in the booting state, after installing a new projection or fixing its errors. + * The explicit catchup is only needed for projections in the booting state with an advanced position. + * So in the case of an error or for a detached projection, the setup will move the projection back to booting keeping its current position. + * Running a full replay would work but might be overkill, instead this catchup will just attempt boot the projection back to active. * * @param string $projection Identifier of the projection to catchup like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") * @param string $contentRepository Identifier of the Content Repository instance to operate on From 2297c141b9dc4371f4118abef2d689346f5fbad6 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 19:33:01 +0100 Subject: [PATCH 086/142] TASK: Reintroduce `ContentRepositoryStatus` object and expose current event store position for debugging and status information --- .../Service/ContentRepositoryMaintainer.php | 24 +++++---- .../ContentRepositoryStatus.php | 50 +++++++++++++++++++ .../Classes/Command/CrCommandController.php | 21 ++++---- .../Domain/Service/SiteImportService.php | 7 ++- 4 files changed, 77 insertions(+), 25 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index e7f5729fd12..0f21315d736 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -8,19 +8,20 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamEventStreamName; use Neos\ContentRepository\Core\Feature\WorkspaceEventStreamName; +use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryStatus; use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\Error\Messages\Error; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventType; use Neos\EventStore\Model\Event\EventTypes; +use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\Event\StreamName; -use Neos\EventStore\Model\EventStore\Status as EventStoreStatus; use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -37,10 +38,10 @@ * Staus information * ----------------- * The status of the content repository e.g. if a setup is required or if all subscriptions are active and their position - * can be examined with two methods: + * can be examined with {@see status} * - * The event store status is available via {@see eventStoreStatus}, while the subscription status are returned - * via {@see SubscriptionStatusCollection}. Further documentation in {@see SubscriptionStatusCollection}. + * The event store status is available via {@see ContentRepositoryStatus::$eventStoreStatus}, and the subscription status + * via {@see ContentRepositoryStatus::$subscriptionStatus}. Further documentation in {@see SubscriptionStatusCollection}. * * Replay / Catchup of projections * ------------------------------- @@ -84,14 +85,15 @@ public function setUp(): Error|null return null; } - public function eventStoreStatus(): EventStoreStatus + public function status(): ContentRepositoryStatus { - return $this->eventStore->status(); - } + $lastEventEnvelope = current(iterator_to_array($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1))) ?: null; - public function subscriptionStatus(): SubscriptionStatusCollection - { - return $this->subscriptionEngine->subscriptionStatus(); + return ContentRepositoryStatus::create( + $this->eventStore->status(), + $lastEventEnvelope?->sequenceNumber ?? SequenceNumber::none(), + $this->subscriptionEngine->subscriptionStatus() + ); } public function replayProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php new file mode 100644 index 00000000000..d54faac3a52 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php @@ -0,0 +1,50 @@ +contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - $eventStoreStatus = $contentRepositoryMaintainer->eventStoreStatus(); + $crStatus = $contentRepositoryMaintainer->status(); $hasErrors = false; $setupRequired = false; $bootingRequired = false; - $this->output('Event Store: '); - $this->outputLine(match ($eventStoreStatus->type) { + $this->outputLine('Event Store:'); + $this->output(' Setup: '); + $this->outputLine(match ($crStatus->eventStoreStatus->type) { StatusType::OK => 'OK', StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - $hasErrors |= $eventStoreStatus->type === StatusType::ERROR; - if ($verbose && $eventStoreStatus->details !== '') { - $this->outputFormatted($eventStoreStatus->details, [], 2); + $this->output(' Position: %d', [$crStatus->eventStorePosition->value]); + $hasErrors |= $crStatus->eventStoreStatus->type === StatusType::ERROR; + if ($verbose && $crStatus->eventStoreStatus->details !== '') { + $this->outputFormatted($crStatus->eventStoreStatus->details, [], 2); } $this->outputLine(); $this->outputLine('Subscriptions:'); - $subscriptionStatusCollection = $contentRepositoryMaintainer->subscriptionStatus(); - if ($subscriptionStatusCollection->isEmpty()) { + if ($crStatus->subscriptionStatus->isEmpty()) { $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); $this->quit(1); } - foreach ($subscriptionStatusCollection as $status) { + foreach ($crStatus->subscriptionStatus as $status) { if ($status instanceof DetachedSubscriptionStatus) { $this->outputLine(' %s:', [$status->subscriptionId->value]); $this->output(' Subscription: '); @@ -134,7 +135,7 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->outputLine('Setup required, please run ./flow cr:setup'); } if ($bootingRequired) { - $this->outputLine('Catchup needed for projections, please run ./flow cr:projectioncatchup [projection-name]'); + $this->outputLine('Catchup or replay needed for BOOTING projections'); } if ($hasErrors) { $this->outputLine('Some projections are not okay'); diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 7aa8c3afd2b..2c8d5aca54a 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -96,12 +96,11 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $contentRepositoryMaintainer, ContentRepositoryId $contentRepositoryId): void { - $eventStoreStatus = $contentRepositoryMaintainer->eventStoreStatus(); - if ($eventStoreStatus->type !== StatusType::OK) { + $status = $contentRepositoryMaintainer->status(); + if ($status->eventStoreStatus->type !== StatusType::OK) { throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepositoryId->value)); } - $subscriptionStatusCollection = $contentRepositoryMaintainer->subscriptionStatus(); - foreach ($subscriptionStatusCollection as $status) { + foreach ($status->subscriptionStatus as $status) { if ($status instanceof ProjectionSubscriptionStatus) { if ($status->setupStatus->type !== ProjectionStatusType::OK) { throw new \RuntimeException(sprintf('Projection %s in content repository %s is not setup correctly, please run `./flow cr:setup`', $status->subscriptionId->value, $contentRepositoryId->value)); From 1220f822aac149a38337f1bf0aaf670d5136f18a Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 25 Nov 2024 19:36:18 +0100 Subject: [PATCH 087/142] TASK: Swap Projection and Setup in output so that Setup comes first --- .../Classes/Command/CrCommandController.php | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 37ec012b47a..6aa5af59a81 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -73,7 +73,7 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - $this->output(' Position: %d', [$crStatus->eventStorePosition->value]); + $this->outputLine(' Position: %d', [$crStatus->eventStorePosition->value]); $hasErrors |= $crStatus->eventStoreStatus->type === StatusType::ERROR; if ($verbose && $crStatus->eventStoreStatus->details !== '') { $this->outputFormatted($crStatus->eventStoreStatus->details, [], 2); @@ -93,6 +93,21 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo } if ($status instanceof ProjectionSubscriptionStatus) { $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Setup: '); + $this->outputLine(match ($status->setupStatus->type) { + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', + ProjectionStatusType::ERROR => 'ERROR', + }); + $hasErrors |= $status->setupStatus->type === ProjectionStatusType::ERROR; + $setupRequired |= $status->setupStatus->type === ProjectionStatusType::SETUP_REQUIRED; + if ($verbose && ($status->setupStatus->type !== ProjectionStatusType::OK || $status->setupStatus->details)) { + $lines = explode(chr(10), $status->setupStatus->details ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' ' . $line); + } + $this->outputLine(); + } $this->output(' Projection: '); $this->output(match ($status->subscriptionStatus) { SubscriptionStatus::NEW => 'NEW', @@ -112,21 +127,6 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->outputLine(' %s', [$line]); } } - $this->output(' Setup: '); - $this->outputLine(match ($status->setupStatus->type) { - ProjectionStatusType::OK => 'OK', - ProjectionStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', - ProjectionStatusType::ERROR => 'ERROR', - }); - $hasErrors |= $status->setupStatus->type === ProjectionStatusType::ERROR; - $setupRequired |= $status->setupStatus->type === ProjectionStatusType::SETUP_REQUIRED; - if ($verbose && ($status->setupStatus->type !== ProjectionStatusType::OK || $status->setupStatus->details)) { - $lines = explode(chr(10), $status->setupStatus->details ?: 'No details available.'); - foreach ($lines as $line) { - $this->outputLine(' ' . $line); - } - $this->outputLine(); - } } } if ($verbose) { From 800fd5317d2b0fdd28e008ff6c66eb2d13b97498 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:24:43 +0100 Subject: [PATCH 088/142] WIP: Introduce `cr:reactivateSubscription` in hindsight that well revert that setup updates the error state or detached state --- .../Service/ContentRepositoryMaintainer.php | 60 +++++++++---------- .../Engine/SubscriptionEngine.php | 7 ++- .../Classes/Command/CrCommandController.php | 27 +++++---- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 0f21315d736..4b79aa012e2 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -13,7 +13,6 @@ use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\Error\Messages\Error; @@ -43,16 +42,28 @@ * The event store status is available via {@see ContentRepositoryStatus::$eventStoreStatus}, and the subscription status * via {@see ContentRepositoryStatus::$subscriptionStatus}. Further documentation in {@see SubscriptionStatusCollection}. * - * Replay / Catchup of projections - * ------------------------------- - * The methods {@see replayProjection}, {@see replayAllProjections} and {@see catchupProjection} - * can be leveraged to interact with the projection catchup. In the happy path no interaction is necessary, - * as {@see ContentRepository::handle()} triggers the projections after applying the events. + * Projection subscriptions + * ------------------------ + * + * This maintainer offers also the public API to interact with the projection catchup. In the happy path, + * no interaction is necessary, as {@see ContentRepository::handle()} triggers the projections after applying the events. + * + * Special cases: + * + * *Replay* * * For initialising on a new database - which contains events already - a replay will make sure that the projections - * are emptied and reapply the events. After registering a new projection a setup is needed and a replay of this projection. + * are emptied and reapply the events. This can be triggered via {@see replayProjection} or {@see replayAllProjections} + * + * And after registering a new projection a setup as well as a replay of this projection is also required. * - * The explicit catchup of a projection is only required after fixing a projection error. + * *Reactivate* + * + * In case a projection is detached but is reinstalled a reactivation is needed via {@see reactivateSubscription} + * + * Also in case a projection runs into the error status, its code needs to be fixed, and it can also be attempted to be reactivated. + * + * Note that in both cases a projection replay would also work, but with the difference that the projection is reset as well. * * @api */ @@ -123,39 +134,22 @@ public function replayAllProjections(\Closure|null $progressCallback = null): Er } /** - * Catchup one specific projection for debugging or fixing it. - * - * The explicit catchup is only needed for projections in the booting state with an advanced position. - * So in the case of an error or for a detached projection, the setup will move the projection back to booting keeping its current position. - * Running a full replay would work but might be overkill, instead this catchup will just attempt boot the projection back to active. + * Reactivate a projection * - * We don't offer an API to catch up all projections at once (like catchupAllProjections). Instead, a replayAll can be used. + * The explicit catchup is only needed for projections in the error or detached status with an advanced position. + * Running a full replay would work but might be overkill, instead this reactivation will just attempt + * catchup the projection back to active from its current position. */ - public function catchupProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionCriteria::create([$subscriptionId]))->first(); if ($subscriptionStatus === null) { - return new Error(sprintf('Projection "%s" is not registered.', $subscriptionId->value)); + return new Error(sprintf('Subscription "%s" is not registered.', $subscriptionId->value)); } - if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::BOOTING) { - $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); - if ($bootResult->errors !== null) { - return self::createErrorForReason('booting', $bootResult->errors); - } - return null; - } - - if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::ACTIVE) { - $catchupResult = $this->subscriptionEngine->catchUpActive(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback); - if ($catchupResult->errors !== null) { - return self::createErrorForReason('catchup', $catchupResult->errors); - } - return null; - } - - return new Error(sprintf('Cannot catch-up projection "%s" with state %s. Please setup the content repository first.', $subscriptionId->value, $subscriptionStatus->subscriptionStatus->name)); + // todo implement https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L624 + return null; } /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index e86537f47d8..d6a0ce79379 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -28,7 +28,12 @@ use Neos\ContentRepository\Core\Subscription\Subscriptions; /** - * @internal public API is the {@see ContentRepository::handle()} and the {@see ContentRepositoryMaintainer} + * This is the internal core for the catchup + * + * All functionality is low level and well encapsulated and abstracted by the {@see ContentRepositoryMaintainer} + * It presents the only API way to interact with catchup and offers more maintenance tasks. + * + * @internal implementation detail of {@see ContentRepository::handle()} and the {@see ContentRepositoryMaintainer} */ final class SubscriptionEngine { diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 6aa5af59a81..e4eb21b1c75 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -64,6 +64,7 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); $crStatus = $contentRepositoryMaintainer->status(); $hasErrors = false; + $reactivationRequired = false; $setupRequired = false; $bootingRequired = false; $this->outputLine('Event Store:'); @@ -118,9 +119,9 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo }); $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $reactivationRequired |= $status->subscriptionStatus === SubscriptionStatus::ERROR; $bootingRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; - // detached can be reattached via setup: - $setupRequired |= $status->subscriptionStatus === SubscriptionStatus::DETACHED; + $reactivationRequired |= $status->subscriptionStatus === SubscriptionStatus::DETACHED; if ($verbose && $status->subscriptionError !== null) { $lines = explode(chr(10), $status->subscriptionError->errorMessage ?: 'No details available.'); foreach ($lines as $line) { @@ -135,10 +136,10 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->outputLine('Setup required, please run ./flow cr:setup'); } if ($bootingRequired) { - $this->outputLine('Catchup or replay needed for BOOTING projections'); + $this->outputLine('Replay needed for BOOTING projections, please run ./flow cr:projectionreplay [subscription-id]'); } - if ($hasErrors) { - $this->outputLine('Some projections are not okay'); + if ($reactivationRequired) { + $this->outputLine('Reactivation of ERROR or DETACHED projection required, please run ./flow cr:reactivatesubscription [subscription-id]'); } } if ($hasErrors) { @@ -248,31 +249,31 @@ public function projectionReplayAllCommand(string $contentRepository = 'default' } /** - * Catchup one specific projection for debugging or fixing it. + * Reactivate a projection * - * The explicit catchup is only needed for projections in the booting state with an advanced position. - * So in the case of an error or for a detached projection, the setup will move the projection back to booting keeping its current position. - * Running a full replay would work but might be overkill, instead this catchup will just attempt boot the projection back to active. + * The explicit catchup is only needed for projections in the error or detached status with an advanced position. + * Running a full replay would work but might be overkill, instead this reactivation will just attempt + * catchup the projection back to active from its current position. * - * @param string $projection Identifier of the projection to catchup like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") + * @param string $projection Identifier of the projection to reactivate like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) */ - public function projectionCatchupCommand(string $projection, string $contentRepository = 'default', bool $quiet = false): void + public function reactivateSubscriptionCommand(string $projection, string $contentRepository = 'default', bool $quiet = false): void { $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); $progressCallback = null; if (!$quiet) { - $this->outputLine('Catchup projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); + $this->outputLine('Reactivate projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); // render memory consumption and time remaining $this->output->getProgressBar()->setFormat('debug'); $this->output->progressStart(); $progressCallback = fn () => $this->output->progressAdvance(); } - $result = $contentRepositoryMaintainer->catchupProjection(SubscriptionId::fromString($projection), progressCallback: $progressCallback); + $result = $contentRepositoryMaintainer->reactivateSubscription(SubscriptionId::fromString($projection), progressCallback: $progressCallback); if (!$quiet) { $this->output->progressFinish(); From a0c9f904ac9deaf1ea803b07f3736ac238837a55 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:12:11 +0100 Subject: [PATCH 089/142] TASK: Dont crash on status when the event store is not setup --- .../Classes/Service/ContentRepositoryMaintainer.php | 10 ++++++++-- .../ContentRepository/ContentRepositoryStatus.php | 9 +++++++-- .../Classes/Command/CrCommandController.php | 13 +++++++++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 4b79aa012e2..261509a8576 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -23,6 +23,7 @@ use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\VirtualStreamName; +use Doctrine\DBAL\Exception as DBALException; /** * Set up and manage a content repository @@ -98,11 +99,16 @@ public function setUp(): Error|null public function status(): ContentRepositoryStatus { - $lastEventEnvelope = current(iterator_to_array($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1))) ?: null; + try { + $lastEventEnvelope = current(iterator_to_array($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1))) ?: null; + $sequenceNumber = $lastEventEnvelope?->sequenceNumber ?? SequenceNumber::none(); + } catch (DBALException) { + $sequenceNumber = null; + } return ContentRepositoryStatus::create( $this->eventStore->status(), - $lastEventEnvelope?->sequenceNumber ?? SequenceNumber::none(), + $sequenceNumber, $this->subscriptionEngine->subscriptionStatus() ); } diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php index d54faac3a52..3c2cd61e244 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php @@ -26,9 +26,14 @@ */ final readonly class ContentRepositoryStatus { + /** + * @param EventStoreStatus $eventStoreStatus + * @param SequenceNumber|null $eventStorePosition The position of the event store. NULL if an error occurred, see error state of $eventStoreStatus + * @param SubscriptionStatusCollection $subscriptionStatus + */ private function __construct( public EventStoreStatus $eventStoreStatus, - public SequenceNumber $eventStorePosition, + public SequenceNumber|null $eventStorePosition, public SubscriptionStatusCollection $subscriptionStatus, ) { } @@ -38,7 +43,7 @@ private function __construct( */ public static function create( EventStoreStatus $eventStoreStatus, - SequenceNumber $eventStorePosition, + SequenceNumber|null $eventStorePosition, SubscriptionStatusCollection $subscriptionStatus, ): self { return new self( diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index e4eb21b1c75..17589ac1c0a 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -74,7 +74,11 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - $this->outputLine(' Position: %d', [$crStatus->eventStorePosition->value]); + if ($crStatus->eventStorePosition) { + $this->outputLine(' Position: %d', [$crStatus->eventStorePosition->value]); + } else { + $this->outputLine(' Position: Loading failed!'); + } $hasErrors |= $crStatus->eventStoreStatus->type === StatusType::ERROR; if ($verbose && $crStatus->eventStoreStatus->details !== '') { $this->outputFormatted($crStatus->eventStoreStatus->details, [], 2); @@ -117,7 +121,12 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo SubscriptionStatus::DETACHED => 'DETACHED', SubscriptionStatus::ERROR => 'ERROR', }); - $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + if ($crStatus->eventStorePosition?->value > $status->subscriptionPosition->value) { + // projection is behind + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } else { + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; $reactivationRequired |= $status->subscriptionStatus === SubscriptionStatus::ERROR; $bootingRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; From 8c079d9657dbccf190585df81dbff005a1a1e2b6 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:48:09 +0100 Subject: [PATCH 090/142] TASK: Split projection replay into separate SubscriptionCommandController this allows us to keep the namings short and precise instead of introducing `cr:subscriptionreplayall` see also: https://neos-project.slack.com/archives/C04PYL8H3/p1732629147379509 --- .../Service/ContentRepositoryMaintainer.php | 36 ++-- .../Features/Bootstrap/CRTestSuiteTrait.php | 2 +- .../Classes/Command/CrCommandController.php | 168 +++++----------- .../Command/SubscriptionCommandController.php | 180 ++++++++++++++++++ .../Processors/ProjectionReplayProcessor.php | 2 +- 5 files changed, 248 insertions(+), 140 deletions(-) create mode 100644 Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 261509a8576..2bc514668a1 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -30,10 +30,10 @@ * * Initialisation / Tear down * -------------------------- - * The method {@see setUp} sets up the content repository like event store and projection database tables. + * The method {@see setUp} sets up the content repository like event store and subscription database tables. * It is non-destructive. * - * Resetting a content repository with {@see prune} method will purge the event stream and reset all projection states. + * Resetting a content repository with {@see prune} method will purge the event stream and reset all subscription states. * * Staus information * ----------------- @@ -43,28 +43,28 @@ * The event store status is available via {@see ContentRepositoryStatus::$eventStoreStatus}, and the subscription status * via {@see ContentRepositoryStatus::$subscriptionStatus}. Further documentation in {@see SubscriptionStatusCollection}. * - * Projection subscriptions - * ------------------------ + * Subscriptions (mainly projections) + * ---------------------------------- * - * This maintainer offers also the public API to interact with the projection catchup. In the happy path, - * no interaction is necessary, as {@see ContentRepository::handle()} triggers the projections after applying the events. + * This maintainer offers also the public API to interact with the subscription catchup. In the happy path, + * no interaction is necessary, as {@see ContentRepository::handle()} triggers the subscriptions after applying the events. * * Special cases: * * *Replay* * - * For initialising on a new database - which contains events already - a replay will make sure that the projections - * are emptied and reapply the events. This can be triggered via {@see replayProjection} or {@see replayAllProjections} + * For initialising on a new database - which contains events already - a replay will make sure that the subscriptions + * are emptied and reapply the events. This can be triggered via {@see replaySubscription} or {@see replayAllSubscriptions} * - * And after registering a new projection a setup as well as a replay of this projection is also required. + * And after registering a new subscription a setup as well as a replay of this subscription is also required. * * *Reactivate* * - * In case a projection is detached but is reinstalled a reactivation is needed via {@see reactivateSubscription} + * In case a subscription is detached but is reinstalled a reactivation is needed via {@see reactivateSubscription} * - * Also in case a projection runs into the error status, its code needs to be fixed, and it can also be attempted to be reactivated. + * Also in case a subscription runs into the error status, its code needs to be fixed, and it can also be attempted to be reactivated. * - * Note that in both cases a projection replay would also work, but with the difference that the projection is reset as well. + * Note that in both cases a subscription replay would also work, but with the difference that the subscription is reset as well. * * @api */ @@ -113,7 +113,7 @@ public function status(): ContentRepositoryStatus ); } - public function replayProjection(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + public function replaySubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { $resetResult = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([$subscriptionId])); if ($resetResult->errors !== null) { @@ -126,7 +126,7 @@ public function replayProjection(SubscriptionId $subscriptionId, \Closure|null $ return null; } - public function replayAllProjections(\Closure|null $progressCallback = null): Error|null + public function replayAllSubscriptions(\Closure|null $progressCallback = null): Error|null { $resetResult = $this->subscriptionEngine->reset(); if ($resetResult->errors !== null) { @@ -140,11 +140,11 @@ public function replayAllProjections(\Closure|null $progressCallback = null): Er } /** - * Reactivate a projection + * Reactivate a subscription * - * The explicit catchup is only needed for projections in the error or detached status with an advanced position. + * The explicit catchup is only needed for subscriptions in the error or detached status with an advanced position. * Running a full replay would work but might be overkill, instead this reactivation will just attempt - * catchup the projection back to active from its current position. + * catchup the subscription back to active from its current position. */ public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { @@ -159,7 +159,7 @@ public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure| } /** - * WARNING: Removes all events from the content repository and resets the projections + * WARNING: Removes all events from the content repository and resets the subscriptions * This operation cannot be undone. */ public function prune(): Error|null diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index 777d5a9c1f3..c06fa32af4c 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -257,7 +257,7 @@ abstract protected function getContentRepositoryService( public function iReplayTheProjection(string $projectionName): void { $contentRepositoryMaintainer = $this->getContentRepositoryService(new ContentRepositoryMaintainerFactory()); - $result = $contentRepositoryMaintainer->replayProjection(SubscriptionId::fromString($projectionName)); + $result = $contentRepositoryMaintainer->replaySubscription(SubscriptionId::fromString($projectionName)); Assert::assertNull($result); } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 17589ac1c0a..30af699ad72 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -8,7 +8,6 @@ use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\EventStore\Model\EventStore\StatusType; @@ -16,6 +15,23 @@ use Neos\Flow\Cli\CommandController; use Symfony\Component\Console\Output\Output; +/** + * Set up a content repository + * + * *Initialisation* + * + * The command "./flow cr:setup" sets up the content repository like event store and subscription database tables. + * It is non-destructive. + * + * Note that a reset is not implemented here but for the Neos CMS use-case provided via "./flow site:pruneAll" + * + * *Staus information* + * + * The status of the content repository e.g. if a setup is required or if all subscriptions are active and their position + * can be examined with "./flow cr:status" + * + * See also {@see ContentRepositoryMaintainer} for more information. + */ final class CrCommandController extends CommandController { #[Flow\Inject()] @@ -145,10 +161,10 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->outputLine('Setup required, please run ./flow cr:setup'); } if ($bootingRequired) { - $this->outputLine('Replay needed for BOOTING projections, please run ./flow cr:projectionreplay [subscription-id]'); + $this->outputLine('Replay needed for BOOTING projections, please run ./flow subscription:replay [subscription-id]'); } if ($reactivationRequired) { - $this->outputLine('Reactivation of ERROR or DETACHED projection required, please run ./flow cr:reactivatesubscription [subscription-id]'); + $this->outputLine('Reactivation of ERROR or DETACHED projection required, please run ./flow subscription:reactivate [subscription-id]'); } } if ($hasErrors) { @@ -159,51 +175,35 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo /** * Replays the specified projection of a Content Repository by resetting its state and performing a full catchup. * - * @param string $projection Identifier of the projection to replay like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") + * @param string $projection Identifier of the projection to replay * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $force Replay the projection without confirmation. This may take some time! * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + * @internal + * @deprecated with Neos 9 Beta 17, please use ./flow subscription:replay instead */ public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); - $this->quit(1); - } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the projection "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $projection, $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - - $progressCallback = null; - if (!$quiet) { - $this->outputLine('Replaying events for projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); - // render memory consumption and time remaining - $this->output->getProgressBar()->setFormat('debug'); - $this->output->progressStart(); - $progressCallback = fn () => $this->output->progressAdvance(); - } - - $result = $contentRepositoryMaintainer->replayProjection(SubscriptionId::fromString($projection), progressCallback: $progressCallback); - - if (!$quiet) { - $this->output->progressFinish(); - $this->outputLine(); - } - - if ($result !== null) { - $this->outputLine('%s', [$result->getMessage()]); + $this->outputLine('Please use ./flow subscription:replay instead!'); + $subscriptionId = match($projection) { + 'doctrineDbalContentGraph', + 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjection' => 'contentGraph', + 'documentUriPathProjection' => 'Neos.Neos:DocumentUriPathProjection', + 'change' => 'Neos.Neos:PendingChangesProjection', + default => null + }; + if ($subscriptionId === null) { + $this->outputLine('Invalid --projection specified. Could not map legacy argument.'); $this->quit(1); - } elseif (!$quiet) { - $this->outputLine('Done.'); } + $this->forward( + 'replay', + SubscriptionCommandController::class, + array_merge( + ['subscription' => $subscriptionId], + compact('contentRepository', 'force', 'quiet') + ) + ); } /** @@ -212,88 +212,16 @@ public function projectionReplayCommand(string $projection, string $contentRepos * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $force Replay the projection without confirmation. This may take some time! * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + * @internal + * @deprecated with Neos 9 Beta 17, please use ./flow subscription:replayall instead */ public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); - $this->quit(1); - } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - - $progressCallback = null; - if (!$quiet) { - $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); - // render memory consumption and time remaining - // todo maybe reintroduce pretty output: https://github.com/neos/neos-development-collection/pull/5010 but without using highestSequenceNumber - $this->output->getProgressBar()->setFormat('debug'); - $this->output->progressStart(); - $progressCallback = fn () => $this->output->progressAdvance(); - } - - $result = $contentRepositoryMaintainer->replayAllProjections(progressCallback: $progressCallback); - - if (!$quiet) { - $this->output->progressFinish(); - $this->outputLine(); - } - - if ($result !== null) { - $this->outputLine('%s', [$result->getMessage()]); - $this->quit(1); - } elseif (!$quiet) { - $this->outputLine('Done.'); - } - } - - /** - * Reactivate a projection - * - * The explicit catchup is only needed for projections in the error or detached status with an advanced position. - * Running a full replay would work but might be overkill, instead this reactivation will just attempt - * catchup the projection back to active from its current position. - * - * @param string $projection Identifier of the projection to reactivate like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") - * @param string $contentRepository Identifier of the Content Repository instance to operate on - * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - */ - public function reactivateSubscriptionCommand(string $projection, string $contentRepository = 'default', bool $quiet = false): void - { - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - - $progressCallback = null; - if (!$quiet) { - $this->outputLine('Reactivate projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); - // render memory consumption and time remaining - $this->output->getProgressBar()->setFormat('debug'); - $this->output->progressStart(); - $progressCallback = fn () => $this->output->progressAdvance(); - } - - $result = $contentRepositoryMaintainer->reactivateSubscription(SubscriptionId::fromString($projection), progressCallback: $progressCallback); - - if (!$quiet) { - $this->output->progressFinish(); - $this->outputLine(); - } - - if ($result !== null) { - $this->outputLine('%s', [$result->getMessage()]); - $this->quit(1); - } elseif (!$quiet) { - $this->outputLine('Done.'); - } + $this->outputLine('Please use ./flow subscription:replayall instead!'); + $this->forward( + 'replayall', + SubscriptionCommandController::class, + compact('contentRepository', 'force', 'quiet') + ); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php new file mode 100644 index 00000000000..4da391f52c7 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php @@ -0,0 +1,180 @@ +output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this subscription. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the subscription "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $subscription, $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for subscription "%s" of Content Repository "%s" ...', [$subscription, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replaySubscription(SubscriptionId::fromString($subscription), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Replays all projections of the specified Content Repository by resetting their states and performing a full catchup + * + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $force Replay all subscriptions without confirmation. This may take some time! + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function replayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void + { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay all subscriptions. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); + // render memory consumption and time remaining + // todo maybe reintroduce pretty output: https://github.com/neos/neos-development-collection/pull/5010 but without using highestSequenceNumber + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replayAllSubscriptions(progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Reactivate a subscription + * + * The explicit catchup is only needed for projections in the error or detached status with an advanced position. + * Running a full replay would work but might be overkill, instead this reactivation will just attempt + * catchup the subscription back to active from its current position. + * + * @param string $subscription Identifier of the subscription to reactivate like it was configured (e.g. "contentGraph", "Vendor.Package:YourProjection") + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function reactivateCommand(string $subscription, string $contentRepository = 'default', bool $quiet = false): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Reactivate subscription "%s" of Content Repository "%s" ...', [$subscription, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->reactivateSubscription(SubscriptionId::fromString($subscription), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php index b97a37eb27d..bb44d1f9643 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php @@ -19,6 +19,6 @@ public function __construct( public function run(ProcessingContext $context): void { - $this->contentRepositoryMaintainer->replayAllProjections(); + $this->contentRepositoryMaintainer->replayAllSubscriptions(); } } From 4c65d81917b706f3448bf8119eebb516000bc28b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:13:25 +0100 Subject: [PATCH 091/142] TASK: Status also shows new subscriptions even if they are not persistent yet --- .../AbstractSubscriptionEngineTestCase.php | 4 +-- .../SubscriptionNewStatusTest.php | 26 ++++++++++++------- .../Service/ContentRepositoryMaintainer.php | 15 ++++++++--- .../Engine/SubscriptionEngine.php | 21 +++++++++++++-- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index bd9a8ce4de2..9aeb1c43b60 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -19,7 +19,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -137,7 +137,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null { - return $this->subscriptionEngine->subscriptionStatus(SubscriptionCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + return $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); } final protected function commitExampleContentStreamEvent(): void diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php index ed2d57fe2b1..86c8ae2e0d5 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -43,7 +43,8 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); - $newFakeProjection->expects(self::exactly(3))->method('status')->willReturnOnConsecutiveCalls( + $newFakeProjection->expects(self::exactly(4))->method('status')->willReturnOnConsecutiveCalls( + ProjectionStatus::setupRequired('Set me up'), ProjectionStatus::setupRequired('Set me up'), ProjectionStatus::ok(), ProjectionStatus::ok(), @@ -65,21 +66,26 @@ public function newProjectionIsFoundWhenConfigurationIsAdded() $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); $this->setupContentRepositoryDependencies($this->contentRepository->id); - // todo status doesnt find this projection yet? - self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + $expectedNewState = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Set me up') + ); + + // status predicts the NEW state already (without creating it in the db) + self::assertEquals( + $expectedNewState, + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); // do something that finds new subscriptions, trigger a setup on a specific projection: $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); self::assertNull($result->errors); self::assertEquals( - ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), - subscriptionStatus: SubscriptionStatus::NEW, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: null, - setupStatus: ProjectionStatus::setupRequired('Set me up') - ), + $expectedNewState, $this->subscriptionStatus('Vendor.Package:NewFakeProjection') ); diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 2bc514668a1..5a36a287eac 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -12,8 +12,8 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\Error\Messages\Error; use Neos\EventStore\EventStoreInterface; @@ -115,6 +115,13 @@ public function status(): ContentRepositoryStatus public function replaySubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { + $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create([$subscriptionId]))->first(); + if ($subscriptionStatus === null) { + return new Error(sprintf('Subscription "%s" is not registered.', $subscriptionId->value)); + } + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { + return new Error(sprintf('Subscription "%s" is not setup and cannot be replayed.', $subscriptionId->value)); + } $resetResult = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([$subscriptionId])); if ($resetResult->errors !== null) { return self::createErrorForReason('reset', $resetResult->errors); @@ -148,11 +155,13 @@ public function replayAllSubscriptions(\Closure|null $progressCallback = null): */ public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null { - $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionCriteria::create([$subscriptionId]))->first(); - + $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create([$subscriptionId]))->first(); if ($subscriptionStatus === null) { return new Error(sprintf('Subscription "%s" is not registered.', $subscriptionId->value)); } + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { + return new Error(sprintf('Subscription "%s" is not setup and cannot be reactivated.', $subscriptionId->value)); + } // todo implement https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L624 return null; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index d6a0ce79379..e9616c204bc 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -111,11 +111,11 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } - public function subscriptionStatus(SubscriptionCriteria|null $criteria = null): SubscriptionStatusCollection + public function subscriptionStatus(SubscriptionEngineCriteria|null $criteria = null): SubscriptionStatusCollection { $statuses = []; try { - $subscriptions = $this->subscriptionStore->findByCriteria($criteria ?? SubscriptionCriteria::noConstraints()); + $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create(ids: $criteria?->ids)); } catch (TableNotFoundException) { // the schema is not setup - thus there are no subscribers return SubscriptionStatusCollection::createEmpty(); @@ -138,6 +138,23 @@ public function subscriptionStatus(SubscriptionCriteria|null $criteria = null): setupStatus: $subscriber->projection->status(), ); } + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; + } + if ($criteria?->ids?->contain($subscriber->id) === false) { + // this might be a NEW subscription but we dont return it as status is filtered. + continue; + } + // this NEW state is not persisted yet + $statuses[] = ProjectionSubscriptionStatus::create( + subscriptionId: $subscriber->id, + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: $subscriber->projection->status(), + ); + } return SubscriptionStatusCollection::fromArray($statuses); } From 8c9c0e8cfc241e91b84eafb4e54b3ed115d41923 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:17:28 +0100 Subject: [PATCH 092/142] TASK: Refine todos skipBooting was removed via 0ac8751e843a5a88103bbac5a5615238374a083d --- .../Classes/Service/ContentRepositoryMaintainer.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 5a36a287eac..4adb2301ca9 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -88,7 +88,8 @@ public function setUp(): Error|null return self::createErrorForReason('setup', $setupResult->errors); } if ($eventStoreIsEmpty) { - // todo reintroduce skipBooting flag, and also notify if the flag is not set, e.g. because there are events + // note: possibly introduce $skipBooting flag instead + // see https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L42 $bootResult = $this->subscriptionEngine->boot(); if ($bootResult->errors !== null) { return self::createErrorForReason('initial catchup', $bootResult->errors); @@ -184,7 +185,7 @@ public function prune(): Error|null if ($resetResult->errors !== null) { return self::createErrorForReason('reset', $resetResult->errors); } - // todo reintroduce skipBooting flag to reset + // note: possibly introduce $skipBooting flag like for setup $bootResult = $this->subscriptionEngine->boot(); if ($bootResult->errors !== null) { return self::createErrorForReason('booting', $bootResult->errors); @@ -194,7 +195,6 @@ public function prune(): Error|null private static function createErrorForReason(string $method, Errors $errors): Error { - // todo log throwable via flow???, but we are here in the CORE ... $message = []; $message[] = sprintf('%s produced the following error%s', $method, $errors->count() === 1 ? '' : 's'); foreach ($errors as $error) { From 4424483e3fe3e3b79a7cc7cb80d73a5445631a41 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:27:35 +0100 Subject: [PATCH 093/142] TASK: Declare SubscriptionEngine and friends as internal only things in `SubscriptionStatusCollection` are API --- .../Classes/Subscription/DetachedSubscriptionStatus.php | 4 ++-- .../Classes/Subscription/Engine/Error.php | 2 +- .../Classes/Subscription/Engine/Errors.php | 2 +- .../Classes/Subscription/Engine/ProcessedResult.php | 2 +- .../Classes/Subscription/Engine/Result.php | 2 +- .../Classes/Subscription/Engine/SubscriptionEngine.php | 2 +- .../Subscription/Engine/SubscriptionEngineCriteria.php | 2 +- .../Classes/Subscription/Engine/SubscriptionManager.php | 4 +++- .../Classes/Subscription/Exception/CatchUpFailed.php | 1 + .../SubscriptionEngineAlreadyProcessingException.php | 3 --- .../Classes/Subscription/ProjectionSubscriptionStatus.php | 4 ++-- .../Classes/Subscription/Store/SubscriptionCriteria.php | 2 +- .../Classes/Subscription/Store/SubscriptionStoreInterface.php | 2 +- .../Classes/Subscription/Subscriber/ProjectionSubscriber.php | 2 +- .../Classes/Subscription/Subscriber/Subscribers.php | 2 +- .../Classes/Subscription/Subscription.php | 2 +- .../Classes/Subscription/SubscriptionError.php | 2 +- .../Classes/Subscription/SubscriptionId.php | 2 +- .../Classes/Subscription/SubscriptionIds.php | 2 +- .../Classes/Subscription/SubscriptionStatus.php | 2 +- .../Classes/Subscription/SubscriptionStatusFilter.php | 2 +- .../Classes/Subscription/Subscriptions.php | 2 +- 22 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php index fd61ddf9f7a..7ab2ad36ac0 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php @@ -10,7 +10,7 @@ * Note that the SubscriptionStatus might not be actually Detached yet, as for the marking of detached, * setup or catchup has to be run. * - * @api + * @api part of the subscription status */ final readonly class DetachedSubscriptionStatus { @@ -22,7 +22,7 @@ private function __construct( } /** - * @internal + * @internal implementation detail of the catchup */ public static function create( SubscriptionId $subscriptionId, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php index 699dac846f3..546f82c5d3c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php @@ -7,7 +7,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionId; /** - * @internal + * @internal implementation detail of the catchup */ final class Error { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index bf7dd3668b9..11b9fffc3cf 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @api + * @internal implementation detail of the catchup */ final readonly class Errors implements \IteratorAggregate, \Countable { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php index 61f1a07e840..e90e6975fa2 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; /** - * @api + * @internal implementation detail of the catchup */ final readonly class ProcessedResult { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php index dc8da2ef08b..dab71033f97 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; /** - * @api + * @internal implementation detail of the catchup */ final readonly class Result { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index e9616c204bc..25cf464e0bc 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -33,7 +33,7 @@ * All functionality is low level and well encapsulated and abstracted by the {@see ContentRepositoryMaintainer} * It presents the only API way to interact with catchup and offers more maintenance tasks. * - * @internal implementation detail of {@see ContentRepository::handle()} and the {@see ContentRepositoryMaintainer} + * @internal implementation detail of the catchup. See {@see ContentRepository::handle()} and {@see ContentRepositoryMaintainer} */ final class SubscriptionEngine { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php index 3cb3946aa17..068f5a15a98 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php @@ -8,7 +8,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionIds; /** - * @internal + * @internal implementation detail of the catchup */ final class SubscriptionEngineCriteria { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php index 4fd33df2388..2daa7e9db3f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php @@ -9,7 +9,9 @@ use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\Subscriptions; -/** @internal */ +/** + * @internal implementation detail of the catchup + */ final class SubscriptionManager { /** @var \SplObjectStorage */ diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php index c18aaf9c3ba..2cee008bda8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php @@ -6,6 +6,7 @@ /** * Only thrown if there is no way to recover the started catchup. The transaction will be rolled back. + * * @api */ final class CatchUpFailed extends \RuntimeException diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php index 9631724adfb..51ab9bcbf2f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php @@ -9,9 +9,6 @@ */ final class SubscriptionEngineAlreadyProcessingException extends \RuntimeException { - /** - * @internal - */ public function __construct() { parent::__construct('Subscription engine is already processing'); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php index 0924c717b73..3e443da6171 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/ProjectionSubscriptionStatus.php @@ -8,7 +8,7 @@ use Neos\EventStore\Model\Event\SequenceNumber; /** - * @api + * @api part of the subscription status */ final readonly class ProjectionSubscriptionStatus { @@ -22,7 +22,7 @@ private function __construct( } /** - * @internal + * @internal implementation detail of the catchup */ public static function create( SubscriptionId $subscriptionId, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php index 8eca4a8ed79..5bb7c30b2ea 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionCriteria.php @@ -11,7 +11,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; /** - * @api + * @internal implementation detail of the catchup */ final readonly class SubscriptionCriteria { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 4b6a827f8e6..0127769fde8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -8,7 +8,7 @@ use Neos\ContentRepository\Core\Subscription\Subscriptions; /** - * @api + * @internal only API for custom content repository integrations */ interface SubscriptionStoreInterface { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php index 117b52265bc..7e15a8fdcba 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php @@ -13,7 +13,7 @@ use Neos\EventStore\Model\EventEnvelope; /** - * @internal + * @internal implementation detail of the catchup */ final class ProjectionSubscriber { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php index 17288fc927a..eba25e39a1a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php @@ -14,7 +14,7 @@ * Like a possible "ListeningSubscriber" to only listen to events without the capabilities of a full-blown projection. * * @implements \IteratorAggregate - * @internal + * @internal implementation detail of the catchup */ final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 7a6c22ab82f..902f7210db7 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -11,7 +11,7 @@ /** * Note: This class is mutable by design! * - * @internal + * @internal implementation detail of the catchup */ final class Subscription { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php index 5c53135dc66..1adc7e2cfe1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionError.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @api + * @api part of the subscription status */ final class SubscriptionError { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php index 777814c275e..668c85f0b78 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @api + * @api identifier for a registered subscription */ final class SubscriptionId { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php index 7b672d887ec..9e81ae56ef2 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @api + * @internal implementation detail of the catchup */ final class SubscriptionIds implements \IteratorAggregate, \Countable, \JsonSerializable { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php index f1b29c7e572..487f742fbab 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -5,7 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription; /** - * @api + * @api part of the subscription status */ enum SubscriptionStatus : string { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php index adc3fa4e51b..e531c180329 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php @@ -6,7 +6,7 @@ /** * @implements \IteratorAggregate - * @api + * @internal implementation detail of the catchup */ final class SubscriptionStatusFilter implements \IteratorAggregate { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php index 03a09abe0f1..3bba94d0018 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -8,7 +8,7 @@ /** * @implements \IteratorAggregate - * @api + * @internal implementation detail of the catchup */ final class Subscriptions implements \IteratorAggregate, \Countable, \JsonSerializable { From baa5e4aa280935c806d32bc02ebf3beb38544f29 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:28:24 +0100 Subject: [PATCH 094/142] TASK: Add error code to `SubscriptionEngineAlreadyProcessingException` --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 2 +- .../SubscriptionEngineAlreadyProcessingException.php | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 25cf464e0bc..7f8e1f43aa1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -378,7 +378,7 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu private function processExclusively(\Closure $closure): mixed { if ($this->processing) { - throw new SubscriptionEngineAlreadyProcessingException(); + throw new SubscriptionEngineAlreadyProcessingException('Subscription engine is already processing', 1732714075); } $this->processing = true; try { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php index 51ab9bcbf2f..4747a062f31 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php @@ -9,8 +9,4 @@ */ final class SubscriptionEngineAlreadyProcessingException extends \RuntimeException { - public function __construct() - { - parent::__construct('Subscription engine is already processing'); - } } From 66e54bceba557256880d7bfde1b6539ab32e1c37 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:37:41 +0100 Subject: [PATCH 095/142] TASK: Allow cr registry to implement internal subscription store because its framework --- phpstan-baseline.neon | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index aab9e3e26d4..2477370866c 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -1,5 +1,20 @@ parameters: ignoreErrors: + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionIds\\:\\:toStringArray\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionStatusFilter\\:\\:isEmpty\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionStatusFilter\\:\\:toStringArray\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + - message: "#^Method Neos\\\\Neos\\\\Controller\\\\Backend\\\\MenuHelper\\:\\:buildModuleList\\(\\) return type has no value type specified in iterable type array\\.$#" count: 1 From 51d39e59c5ebc6bbcd4604fad1b06590b698192b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:59:20 +0100 Subject: [PATCH 096/142] TASK: Rename to `SubscriptionReplayProcessor` --- ...ionReplayProcessor.php => SubscriptionReplayProcessor.php} | 2 +- Neos.Neos/Classes/Domain/Service/SiteImportService.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename Neos.ContentRepositoryRegistry/Classes/Processors/{ProjectionReplayProcessor.php => SubscriptionReplayProcessor.php} (87%) diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php similarity index 87% rename from Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php rename to Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php index bb44d1f9643..83ce99eaca0 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionReplayProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php @@ -10,7 +10,7 @@ /** * @internal */ -final readonly class ProjectionReplayProcessor implements ProcessorInterface +final readonly class SubscriptionReplayProcessor implements ProcessorInterface { public function __construct( private ContentRepositoryMaintainer $contentRepositoryMaintainer, diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 2c8d5aca54a..ebe76134d19 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -30,7 +30,7 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionReplayProcessor; +use Neos\ContentRepositoryRegistry\Processors\SubscriptionReplayProcessor; use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; @@ -85,7 +85,7 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), // WARNING! We do a replay here even though it will redo the live workspace creation. But otherwise the catchup hooks cannot determine that they need to be skipped as it seems like a regular catchup // In case we allow to import events into other root workspaces, or don't expect live to be empty (see Import events), this would need to be adjusted, as otherwise existing data will be replayed - 'Replay all projections' => new ProjectionReplayProcessor($contentRepositoryMaintainer), + 'Replay all subscriptions' => new SubscriptionReplayProcessor($contentRepositoryMaintainer), ]); foreach ($processors as $processorLabel => $processor) { From b2c1a29423004c6170e9aaf773c206d7341af618 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:04:56 +0100 Subject: [PATCH 097/142] TASK: Improve legacy projectionReplayCommand stub --- .../Classes/Command/CrCommandController.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 30af699ad72..c9ab42e077b 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -184,7 +184,6 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo */ public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - $this->outputLine('Please use ./flow subscription:replay instead!'); $subscriptionId = match($projection) { 'doctrineDbalContentGraph', 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjection' => 'contentGraph', @@ -193,9 +192,10 @@ public function projectionReplayCommand(string $projection, string $contentRepos default => null }; if ($subscriptionId === null) { - $this->outputLine('Invalid --projection specified. Could not map legacy argument.'); + $this->outputLine('Invalid --projection specified. Please use ./flow subscription:replay [contentGraph|Neos.Neos:DocumentUriPathProjection|...] directly.'); $this->quit(1); } + $this->outputLine('Please use ./flow subscription:replay %s instead!', [$subscriptionId]); $this->forward( 'replay', SubscriptionCommandController::class, From d84c2a4d4d2eaa01ccc15db06fcce7e3437217cf Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 19:13:25 +0100 Subject: [PATCH 098/142] TASK: Add test for Subscription & Cr Commands (and thus CRMaintainer) --- .../AbstractSubscriptionEngineTestCase.php | 3 + .../SubscriptionGetStatusTest.php | 12 +- .../ContentRepositoryStatus.php | 2 +- .../Classes/Command/CrCommandController.php | 6 +- ...sitoryMaintenanceCommandControllerTest.php | 142 ++++++++++++++++++ 5 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 9aeb1c43b60..ca9c4de33eb 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -36,6 +36,9 @@ use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; +/** + * @internal, only for tests of the Neos.* namespace + */ abstract class AbstractSubscriptionEngineTestCase extends TestCase // we don't use Flows functional test case as it would reset the database afterwards { protected ContentRepository $contentRepository; diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php index c8be53f9d98..0a624d4c08b 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -6,12 +6,15 @@ use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; +use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\EventStore\Model\EventStore\StatusType; final class SubscriptionGetStatusTest extends AbstractSubscriptionEngineTestCase { @@ -25,8 +28,13 @@ public function statusOnEmptyDatabase() keepSchema: false ); - $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); - self::assertTrue($actualStatuses->isEmpty()); + $crMaintainer = $this->getObject(ContentRepositoryRegistry::class)->buildService($this->contentRepository->id, new ContentRepositoryMaintainerFactory()); + + $status = $crMaintainer->status(); + + self::assertEquals(StatusType::SETUP_REQUIRED, $status->eventStoreStatus->type); + self::assertNull($status->eventStorePosition); + self::assertTrue($status->subscriptionStatus->isEmpty()); self::assertNull( $this->subscriptionStatus('contentGraph') diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php index 3c2cd61e244..db546bf9f9f 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php @@ -28,7 +28,7 @@ { /** * @param EventStoreStatus $eventStoreStatus - * @param SequenceNumber|null $eventStorePosition The position of the event store. NULL if an error occurred, see error state of $eventStoreStatus + * @param SequenceNumber|null $eventStorePosition The position of the event store. NULL if an error occurred in which case a setup must likely be done, see $eventStoreStatus * @param SubscriptionStatusCollection $subscriptionStatus */ private function __construct( diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index c9ab42e077b..c51957ed96c 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -47,10 +47,14 @@ final class CrCommandController extends CommandController * To check if the content repository needs to be setup look into cr:status. * That command will also display information what is about to be migrated. * + * @param bool $quiet If set, no output is generated. This is useful if only the exit code (0 = all OK, 1 = errors or warnings) is of interest * @param string $contentRepository Identifier of the Content Repository to set up */ - public function setupCommand(string $contentRepository = 'default'): void + public function setupCommand(string $contentRepository = 'default', bool $quiet = false): void { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); diff --git a/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php b/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php new file mode 100644 index 00000000000..cf8de0c0b88 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php @@ -0,0 +1,142 @@ +crController = $this->getObject(CrCommandController::class); + $this->subscriptionController = $this->getObject(SubscriptionCommandController::class); + + $this->response = new Response(); + $this->bufferedOutput = new BufferedOutput(); + + ObjectAccess::setProperty($this->crController, 'response', $this->response, true); + ObjectAccess::getProperty($this->crController, 'output', true)->setOutput($this->bufferedOutput); + + ObjectAccess::setProperty($this->subscriptionController, 'response', $this->response, true); + ObjectAccess::getProperty($this->subscriptionController, 'output', true)->setOutput($this->bufferedOutput); + } + + /** @test */ + public function setupOnEmptyEventStore(): void + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + // projections are marked active because the event store is empty + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + } + + /** @test */ + public function setupOnModifiedEventStore(): void + { + $this->eventStore->setup(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->subscriptionController->replayCommand(subscription: 'contentGraph', contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->subscriptionController->replayAllCommand(contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function projectionInError(): void + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::any())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->secondFakeProjection->injectSaboteur(fn () => throw new \RuntimeException('This projection is kaputt.')); + + try { + $this->contentRepository->handle(CreateRootWorkspace::create( + WorkspaceName::forLive(), + ContentStreamId::create() + )); + } catch (\RuntimeException) { + } + + self::assertEquals( + SubscriptionStatus::ERROR, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')?->subscriptionStatus + ); + + try { + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + } catch (StopCommandException) { + } + // exit error code because one projection has a failure + self::assertEquals(1, $this->response->getExitCode()); + self::assertEmpty($this->bufferedOutput->fetch()); + + // repair projection + $this->secondFakeProjection->killSaboteur(); + $this->subscriptionController->reactivateCommand(subscription: 'Vendor.Package:SecondFakeProjection', contentRepository: $this->contentRepository->id->value, quiet: true); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + } +} From ac425ff2a6d6d5133fea72bd637e488a8c5d207e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 20:30:38 +0100 Subject: [PATCH 099/142] TASK: Remove `SubscriptionManager` and make subscriptions immutable This removes any magic from the code flow and makes transactions, locking, and update more explicit and easier to follow. --- .../Service/ContentRepositoryMaintainer.php | 4 +- .../Classes/Subscription/Engine/Errors.php | 2 +- .../Engine/SubscriptionEngine.php | 233 ++++++++++-------- .../Engine/SubscriptionManager.php | 76 ------ .../Store/SubscriptionStoreInterface.php | 13 +- .../Classes/Subscription/Subscription.php | 49 +--- .../DoctrineSubscriptionStore.php | 19 +- 7 files changed, 156 insertions(+), 240 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 4adb2301ca9..a8e725dc3b5 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -35,8 +35,8 @@ * * Resetting a content repository with {@see prune} method will purge the event stream and reset all subscription states. * - * Staus information - * ----------------- + * Status information + * ------------------ * The status of the content repository e.g. if a setup is required or if all subscriptions are active and their position * can be examined with {@see status} * diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index 11b9fffc3cf..8e8d3b3853f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -21,7 +21,7 @@ private function __construct( if ($errors === []) { throw new \InvalidArgumentException('Errors must not be empty.', 1731612542); } - $this->errors = $errors; + $this->errors = array_values($errors); } /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 7f8e1f43aa1..0038935ec7d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -13,6 +13,13 @@ use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; +use Neos\ContentRepository\Core\Subscription\Subscription; +use Neos\ContentRepository\Core\Subscription\SubscriptionError; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; @@ -20,12 +27,6 @@ use Neos\EventStore\Model\EventEnvelope; use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Log\LoggerInterface; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; -use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; -use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; -use Neos\ContentRepository\Core\Subscription\Subscription; -use Neos\ContentRepository\Core\Subscription\Subscriptions; /** * This is the internal core for the catchup @@ -38,7 +39,6 @@ final class SubscriptionEngine { private bool $processing = false; - private readonly SubscriptionManager $subscriptionManager; public function __construct( private readonly EventStoreInterface $eventStore, @@ -47,7 +47,6 @@ public function __construct( private readonly EventNormalizer $eventNormalizer, private readonly LoggerInterface|null $logger = null, ) { - $this->subscriptionManager = new SubscriptionManager($this->subscriptionStore); } public function setup(SubscriptionEngineCriteria|null $criteria = null): Result @@ -58,7 +57,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result $this->subscriptionStore->setup(); $this->discoverNewSubscriptions(); - $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, SubscriptionStatus::ACTIVE, @@ -76,7 +75,6 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result $errors[] = $error; } } - $this->subscriptionManager->flush(); return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } @@ -95,7 +93,7 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result $criteria ??= SubscriptionEngineCriteria::noConstraints(); $this->logger?->info('Subscription Engine: Start to reset.'); - $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::any())); + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::any())); if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions to reset.'); return Result::success(); @@ -107,7 +105,6 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result $errors[] = $error; } } - $this->subscriptionManager->flush(); return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } @@ -115,7 +112,7 @@ public function subscriptionStatus(SubscriptionEngineCriteria|null $criteria = n { $statuses = []; try { - $subscriptions = $this->subscriptionStore->findByCriteria(SubscriptionCriteria::create(ids: $criteria?->ids)); + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::create(ids: $criteria?->ids)); } catch (TableNotFoundException) { // the schema is not setup - thus there are no subscribers return SubscriptionStatusCollection::createEmpty(); @@ -158,21 +155,16 @@ public function subscriptionStatus(SubscriptionEngineCriteria|null $criteria = n return SubscriptionStatusCollection::fromArray($statuses); } - private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, Subscription $subscription): Error|null + private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, SubscriptionId $subscriptionId): Error|null { - $subscriber = $this->subscribers->get($subscription->id); + $subscriber = $this->subscribers->get($subscriptionId); try { $subscriber->handle($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); - $subscription->fail($e); - $this->subscriptionManager->update($subscription); - return Error::fromSubscriptionIdAndException($subscription->id, $e); + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscriptionId->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + return Error::fromSubscriptionIdAndException($subscriptionId, $e); } - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); - $subscription->set( - position: $eventEnvelope->sequenceNumber - ); + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscriptionId->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); return null; } @@ -184,19 +176,21 @@ private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domai */ private function discoverNewSubscriptions(): void { - $this->subscriptionManager->findForAndUpdate( - SubscriptionCriteria::noConstraints(), - function (Subscriptions $subscriptions) { - foreach ($this->subscribers as $subscriber) { - if ($subscriptions->contain($subscriber->id)) { - continue; - } - $subscription = Subscription::createFromSubscriber($subscriber); - $this->subscriptionManager->add($subscription); - $this->logger?->info(sprintf('Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', $subscriber->id->value)); - } + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::noConstraints()); + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; } - ); + $subscription = new Subscription( + $subscriber->id, + SubscriptionStatus::NEW, + SequenceNumber::fromInteger(0), + null, + null + ); + $this->subscriptionStore->add($subscription); + $this->logger?->info(sprintf('Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', $subscriber->id->value)); + } } /** @@ -207,10 +201,12 @@ private function setupSubscription(Subscription $subscription): ?Error { if (!$this->subscribers->contain($subscription->id)) { // mark detached subscriptions as we cannot set up - $subscription->set( + $this->subscriptionStore->update( + $subscription->id, status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: $subscription->error ); - $this->subscriptionManager->update($subscription); $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); return null; } @@ -221,35 +217,30 @@ private function setupSubscription(Subscription $subscription): ?Error } catch (\Throwable $e) { // todo wrap in savepoint to ensure error do not mess up the projection? $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); - $subscription->fail($e); - $this->subscriptionManager->update($subscription); + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::ERROR, + $subscription->position, + SubscriptionError::fromPreviousStatusAndException($subscription->status, $e) + ); return Error::fromSubscriptionIdAndException($subscription->id, $e); } if ($subscription->status === SubscriptionStatus::ACTIVE) { $this->logger?->debug(sprintf('Subscription Engine: Active subscriber "%s" for "%s" has been re-setup.', $subscriber::class, $subscription->id->value)); return null; - } - if ($subscription->status === SubscriptionStatus::ERROR) { - $this->logger?->debug(sprintf('Subscription Engine: Failed subscriber "%s" for "%s" has been re-setup, set to %s. Previous error: %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->error?->errorMessage)); - $subscription->set( - status: SubscriptionStatus::BOOTING + } else { + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::BOOTING, + $subscription->position, + null ); - $subscription->unsetError(); - $this->subscriptionManager->update($subscription); - return null; } $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->status->name)); - $subscription->set( - status: SubscriptionStatus::BOOTING - ); - $this->subscriptionManager->update($subscription); return null; } - /** - * TODO - */ private function resetSubscription(Subscription $subscription): ?Error { $subscriber = $this->subscribers->get($subscription->id); @@ -259,12 +250,12 @@ private function resetSubscription(Subscription $subscription): ?Error $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); return Error::fromSubscriptionIdAndException($subscription->id, $e); } - $subscription->set( - status: SubscriptionStatus::BOOTING, - position: SequenceNumber::none() + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::BOOTING, + position: SequenceNumber::none(), + subscriptionError: null ); - $subscription->unsetError(); - $this->subscriptionManager->update($subscription); $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the resetState method has been executed.', $subscriber::class, $subscription->id->value)); return null; } @@ -273,26 +264,30 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs { $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); - return $this->subscriptionManager->findForAndUpdate( - SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus), - function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosure) { - foreach ($subscriptions as $subscription) { + $subscriptionEngineCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus); + return $this->subscriptionStore->transactional( + function () use ($subscriptionEngineCriteria, $subscriptionStatus, $progressClosure) { + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionEngineCriteria); + foreach ($subscriptionsToCatchup as $subscription) { if (!$this->subscribers->contain($subscription->id)) { // mark detached subscriptions as we cannot handle them and exclude them from catchup - $subscription->set( + $this->subscriptionStore->update( + $subscription->id, status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, ); - $this->subscriptionManager->update($subscription); $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); - $subscriptions = $subscriptions->without($subscription->id); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); } } - if ($subscriptions->isEmpty()) { - $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); return ProcessedResult::success(0); } - foreach ($subscriptions as $subscription) { + + foreach ($subscriptionsToCatchup as $subscription) { try { $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { @@ -302,48 +297,68 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu throw new CatchUpFailed($message, 1732374000, $e); } } - $startSequenceNumber = $subscriptions->lowestPosition()?->next() ?? SequenceNumber::none(); + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); - /** @var list $errors */ + /** @var array $errors */ $errors = []; $numberOfProcessedEvents = 0; - try { - $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); - foreach ($eventStream as $eventEnvelope) { - $sequenceNumber = $eventEnvelope->sequenceNumber; - if ($numberOfProcessedEvents > 0) { - $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); - } - if ($progressClosure !== null) { - $progressClosure($eventEnvelope); + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; + + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } + if ($progressClosure !== null) { + $progressClosure($eventEnvelope); + } + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; } - $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - foreach ($subscriptions as $subscription) { - if ($subscription->status !== $subscriptionStatus) { - continue; - } - if ($subscription->position->value >= $sequenceNumber->value) { - $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); - continue; - } - $this->subscriptionStore->createSavepoint(); - $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription); - if (!$error) { - $this->subscriptionStore->releaseSavepoint(); - continue; - } + $this->subscriptionStore->createSavepoint(); + $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); + if ($error !== null) { + // ERROR Case: + // 1.) roll back the partially applied event on the subscriber $this->subscriptionStore->rollbackSavepoint(); + // 2.) for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // 3.) update the subscription error state on either its unchanged or new position (if some events worked) + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $error->throwable + ), + ); + // 4.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed + // todo put the ERROR $subscriptionStatus into the after hook, so it can properly be reacted upon + try { + $this->subscribers->get($subscription->id)->onAfterCatchUp(); + } catch (\Throwable $e) { + // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" had an error and also failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732733740, $e); + } $errors[] = $error; + continue; } - $numberOfProcessedEvents++; - } - } finally { - foreach ($subscriptions as $subscription) { - $this->subscriptionManager->update($subscription); + // HAPPY Case: + $this->subscriptionStore->releaseSavepoint(); + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; } + $numberOfProcessedEvents++; } - foreach ($subscriptions as $subscription) { + foreach ($subscriptionsToCatchup as $subscription) { try { $this->subscribers->get($subscription->id)->onAfterCatchUp(); } catch (\Throwable $e) { @@ -352,15 +367,15 @@ function (Subscriptions $subscriptions) use ($subscriptionStatus, $progressClosu $this->logger?->critical($message); throw new CatchUpFailed($message, 1732374000, $e); } - if ($subscription->status !== $subscriptionStatus) { - continue; - } - + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ACTIVE, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); if ($subscription->status !== SubscriptionStatus::ACTIVE) { - $subscription->set( - status: SubscriptionStatus::ACTIVE, - ); - $this->subscriptionManager->update($subscription); $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php deleted file mode 100644 index 2daa7e9db3f..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionManager.php +++ /dev/null @@ -1,76 +0,0 @@ - */ - private \SplObjectStorage $forAdd; - - /** @var \SplObjectStorage */ - private \SplObjectStorage $forUpdate; - - public function __construct( - private readonly SubscriptionStoreInterface $subscriptionStore, - ) { - $this->forAdd = new \SplObjectStorage(); - $this->forUpdate = new \SplObjectStorage(); - } - - /** - * @template T - * @param \Closure(Subscriptions):T $closure - * @return T - */ - public function findForAndUpdate(SubscriptionCriteria $criteria, \Closure $closure): mixed - { - return $this->subscriptionStore->transactional( - /** @return T */ - function () use ($closure, $criteria): mixed { - try { - return $closure($this->subscriptionStore->findByCriteria($criteria)); - } finally { - $this->flush(); - } - }, - ); - } - - public function add(Subscription $subscription): void - { - $this->forAdd->attach($subscription); - } - - public function update(Subscription $subscription): void - { - $this->forUpdate->attach($subscription); - } - - public function flush(): void - { - foreach ($this->forAdd as $subscription) { - $this->subscriptionStore->add($subscription); - } - - foreach ($this->forUpdate as $subscription) { - if ($this->forAdd->contains($subscription)) { - continue; - } - - $this->subscriptionStore->update($subscription); - } - - $this->forAdd = new \SplObjectStorage(); - $this->forUpdate = new \SplObjectStorage(); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 0127769fde8..85f31072717 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -5,7 +5,11 @@ namespace Neos\ContentRepository\Core\Subscription\Store; use Neos\ContentRepository\Core\Subscription\Subscription; +use Neos\ContentRepository\Core\Subscription\SubscriptionError; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\Subscriptions; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; +use Neos\EventStore\Model\Event\SequenceNumber; /** * @internal only API for custom content repository integrations @@ -14,11 +18,16 @@ interface SubscriptionStoreInterface { public function setup(): void; - public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions; + public function findByCriteriaForUpdate(SubscriptionCriteria $criteria): Subscriptions; public function add(Subscription $subscription): void; - public function update(Subscription $subscription): void; + public function update( + SubscriptionId $subscriptionId, + SubscriptionStatus $status, + SequenceNumber $position, + SubscriptionError|null $subscriptionError, + ): void; /** * @template T diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php index 902f7210db7..ccfb08f07e5 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -4,60 +4,19 @@ namespace Neos\ContentRepository\Core\Subscription; -use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; -use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionSubscriber; use Neos\EventStore\Model\Event\SequenceNumber; /** - * Note: This class is mutable by design! - * * @internal implementation detail of the catchup */ -final class Subscription +final readonly class Subscription { public function __construct( - public readonly SubscriptionId $id, + public SubscriptionId $id, public SubscriptionStatus $status, public SequenceNumber $position, - public SubscriptionError|null $error = null, - public readonly \DateTimeImmutable|null $lastSavedAt = null, + public SubscriptionError|null $error, + public \DateTimeImmutable|null $lastSavedAt, ) { } - - /** - * @internal Only the {@see SubscriptionEngine} is supposed to instantiate subscriptions - */ - public static function createFromSubscriber(ProjectionSubscriber $subscriber): self - { - return new self( - $subscriber->id, - SubscriptionStatus::NEW, - SequenceNumber::fromInteger(0), - ); - } - - /** - * @internal Only the {@see SubscriptionEngine} is supposed to mutate subscriptions - */ - public function set( - SubscriptionStatus $status = null, - SequenceNumber $position = null - ): void { - $this->status = $status ?? $this->status; - $this->position = $position ?? $this->position; - } - - public function unsetError(): void - { - $this->error = null; - } - - /** - * @internal Only the {@see SubscriptionEngine} is supposed to mutate subscriptions - */ - public function fail(\Throwable $exception): void - { - $this->error = SubscriptionError::fromPreviousStatusAndException($this->status, $exception); - $this->status = SubscriptionStatus::ERROR; - } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 234f1b44838..9c7c9e92005 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -64,7 +64,7 @@ public function setup(): void } } - public function findByCriteria(SubscriptionCriteria $criteria): Subscriptions + public function findByCriteriaForUpdate(SubscriptionCriteria $criteria): Subscriptions { $queryBuilder = $this->dbal->createQueryBuilder() ->select('*') @@ -107,15 +107,24 @@ public function add(Subscription $subscription): void ); } - public function update(Subscription $subscription): void - { - $row = self::toDatabase($subscription); + public function update( + SubscriptionId $subscriptionId, + SubscriptionStatus $status, + SequenceNumber $position, + SubscriptionError|null $subscriptionError, + ): void { + $row = []; $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $row['status'] = $status->name; + $row['position'] = $position->value; + $row['error_message'] = $subscriptionError?->errorMessage; + $row['error_previous_status'] = $subscriptionError?->previousStatus?->name; + $row['error_trace'] = $subscriptionError?->errorTrace; $this->dbal->update( $this->tableName, $row, [ - 'id' => $subscription->id->value, + 'id' => $subscriptionId->value, ] ); } From eb0d792d38a3b9fec8628258f19bf8a3a42baed6 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 21:21:24 +0100 Subject: [PATCH 100/142] FEATURE: Implement `reactivateSubscription` Reverts a8f246be1bbffa5259cd135b27bdfa4cb9446ba9 and 73e10975cb872a2013a7acf43edd8f498bee8e0e as setup doesnt do that task anymore. Similar to: https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L624 --- .../Subscription/ProjectionErrorTest.php | 153 ++++++++--- .../SubscriptionDetachedStatusTest.php | 10 +- .../Service/ContentRepositoryMaintainer.php | 24 +- .../Engine/SubscriptionEngine.php | 254 ++++++++++-------- 4 files changed, 272 insertions(+), 169 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 53ded4f33c3..78d3054c6b1 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -12,13 +12,14 @@ use Neos\ContentRepository\Core\Subscription\Engine\Error; use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; +use Neos\ContentRepository\Core\Subscription\Engine\Result; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; -use PHPUnit\Framework\MockObject\Stub\Exception as WillThrowException; final class ProjectionErrorTest extends AbstractSubscriptionEngineTestCase { @@ -76,11 +77,13 @@ public function projectionWithError() } /** @test */ - public function fixFailedProjection() + public function fixFailedProjectionViaReset() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); @@ -88,13 +91,15 @@ public function fixFailedProjection() $this->commitExampleContentStreamEvent(); // catchup active tries to apply the commited event - $this->fakeProjection->expects(self::exactly(2))->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnOnConsecutiveCalls( - new WillThrowException($exception = new \RuntimeException('This projection is kaputt.')), - null // okay again - ); + $exception = new \RuntimeException('This projection is kaputt.'); + $this->secondFakeProjection->injectSaboteur(function (EventEnvelope $eventEnvelope) use ($exception) { + self::assertEquals(SequenceNumber::fromInteger(1), $eventEnvelope->sequenceNumber); + self::assertEquals('ContentStreamWasCreated', $eventEnvelope->event->type->value); + throw $exception; + }); $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), @@ -106,40 +111,121 @@ public function fixFailedProjection() self::assertEquals( $expectedFailure, - $this->subscriptionStatus('Vendor.Package:FakeProjection') + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); - // catchup active does not change anything + $this->secondFakeProjection->killSaboteur(); + + $result = $this->subscriptionEngine->reset(); + self::assertNull($result->errors); + + // expect the subscriptionError to be reset to null + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function irreparableProjection() + { + // test ways NOT to fix a projection :) + $this->eventStore->setup(); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + $this->secondFakeProjection->injectSaboteur(function (EventEnvelope $eventEnvelope) use ($exception) { + self::assertEquals(SequenceNumber::fromInteger(1), $eventEnvelope->sequenceNumber); + self::assertEquals('ContentStreamWasCreated', $eventEnvelope->event->type->value); + throw $exception; + }); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + // catchup active tries to apply the commited event + $result = $this->subscriptionEngine->catchUpActive(); + // but fails + self::assertTrue($result->hasFailed()); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // a second catchup active does not change anything $result = $this->subscriptionEngine->catchUpActive(); self::assertEquals(ProcessedResult::success(0), $result); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + // boot neither $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); - // still the same state + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // setup neither + $result = $this->subscriptionEngine->setup(); + self::assertEquals(Result::success(), $result); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // reactivation will attempt to retry fix this, but can only work if the projection is repaired and will lead to an error otherwise: + $result = $this->subscriptionEngine->reactivate(); self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:FakeProjection') + ProcessedResult::failed(1, Errors::fromArray([ + Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception) + ])), + $result ); - $this->fakeProjection->expects(self::once())->method('resetState'); + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + // previous state is now an error too also error: + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ERROR, $exception), + setupStatus: ProjectionStatus::ok(), + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + // expect the subscriptionError to be reset to null $result = $this->subscriptionEngine->reset(); self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - // expect the subscriptionError to be reset to null - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - + // but booting will rethrow that error :D $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertTrue($result->hasFailed()); + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + // previous state is now booting + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::BOOTING, $exception), + setupStatus: ProjectionStatus::ok(), + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); } /** @test */ public function projectionIsRolledBackAfterError() { $this->eventStore->setup(); - $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); $result = $this->subscriptionEngine->setup(); self::assertNull($result->errors); @@ -184,16 +270,11 @@ public function projectionIsRolledBackAfterError() $this->secondFakeProjection->killSaboteur(); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); - - // subscriptionError is reset - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); - - // catchup after fix - $result = $this->subscriptionEngine->boot(); + // reactivate and catchup + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); self::assertEquals( [SequenceNumber::fromInteger(1)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() @@ -204,7 +285,7 @@ public function projectionIsRolledBackAfterError() public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() { $this->eventStore->setup(); - $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); @@ -255,20 +336,12 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() $this->secondFakeProjection->killSaboteur(); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); - - // subscriptionError is reset, but the position is preserved - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - // catchup after fix - $result = $this->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); self::assertNull($result->errors); + // subscriptionError is reset, and the position is advanced if there were events + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); self::assertEquals( [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index dabdcce9cac..5b073a4db92 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -7,6 +7,7 @@ use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; @@ -97,7 +98,7 @@ public function projectionIsDetachedOnCatchupActive() /** @test */ public function projectionIsDetachedOnSetupAndReattachedIfPossible() { - $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('setUp'); $this->fakeProjection->expects(self::once())->method('apply'); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); @@ -160,9 +161,10 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() $this->subscriptionStatus('Vendor.Package:FakeProjection') ); - // setup does re-attach as the projection is found again - $this->subscriptionEngine->setup(); + // reactivate does re-attach as the projection if its found again + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')])); + self::assertNull($result->errors); - $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index a8e725dc3b5..9b879a814cb 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -85,14 +85,14 @@ public function setUp(): Error|null $eventStoreIsEmpty = iterator_count($this->eventStore->load(VirtualStreamName::all())->limit(1)) === 0; $setupResult = $this->subscriptionEngine->setup(); if ($setupResult->errors !== null) { - return self::createErrorForReason('setup', $setupResult->errors); + return self::createErrorForReason('Setup failed:', $setupResult->errors); } if ($eventStoreIsEmpty) { // note: possibly introduce $skipBooting flag instead // see https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L42 $bootResult = $this->subscriptionEngine->boot(); if ($bootResult->errors !== null) { - return self::createErrorForReason('initial catchup', $bootResult->errors); + return self::createErrorForReason('Initial catchup failed:', $bootResult->errors); } } return null; @@ -125,11 +125,11 @@ public function replaySubscription(SubscriptionId $subscriptionId, \Closure|null } $resetResult = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([$subscriptionId])); if ($resetResult->errors !== null) { - return self::createErrorForReason('reset', $resetResult->errors); + return self::createErrorForReason('Reset failed:', $resetResult->errors); } $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), $progressCallback); if ($bootResult->errors !== null) { - return self::createErrorForReason('catchup', $bootResult->errors); + return self::createErrorForReason('Catchup failed:', $bootResult->errors); } return null; } @@ -138,11 +138,11 @@ public function replayAllSubscriptions(\Closure|null $progressCallback = null): { $resetResult = $this->subscriptionEngine->reset(); if ($resetResult->errors !== null) { - return self::createErrorForReason('reset', $resetResult->errors); + return self::createErrorForReason('Reset failed:', $resetResult->errors); } $bootResult = $this->subscriptionEngine->boot(progressCallback: $progressCallback); if ($bootResult->errors !== null) { - return self::createErrorForReason('catchup', $bootResult->errors); + return self::createErrorForReason('Catchup failed:', $bootResult->errors); } return null; } @@ -163,8 +163,10 @@ public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure| if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { return new Error(sprintf('Subscription "%s" is not setup and cannot be reactivated.', $subscriptionId->value)); } - - // todo implement https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L624 + $reactivateResult = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([$subscriptionId]), $progressCallback); + if ($reactivateResult->errors !== null) { + return self::createErrorForReason('Could not reactivate subscriber:', $reactivateResult->errors); + } return null; } @@ -183,12 +185,12 @@ public function prune(): Error|null } $resetResult = $this->subscriptionEngine->reset(); if ($resetResult->errors !== null) { - return self::createErrorForReason('reset', $resetResult->errors); + return self::createErrorForReason('Reset failed:', $resetResult->errors); } // note: possibly introduce $skipBooting flag like for setup $bootResult = $this->subscriptionEngine->boot(); if ($bootResult->errors !== null) { - return self::createErrorForReason('booting', $bootResult->errors); + return self::createErrorForReason('Catchup failed:', $bootResult->errors); } return null; } @@ -196,7 +198,7 @@ public function prune(): Error|null private static function createErrorForReason(string $method, Errors $errors): Error { $message = []; - $message[] = sprintf('%s produced the following error%s', $method, $errors->count() === 1 ? '' : 's'); + $message[] = sprintf('%s: Following error%s', $method, $errors->count() === 1 ? '' : 's'); foreach ($errors as $error) { $message[] = sprintf(' Subscription "%s": %s', $error->subscriptionId->value, $error->message); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 0038935ec7d..8d556ec8017 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -19,6 +19,7 @@ use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\Subscriptions; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; @@ -60,9 +61,7 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ SubscriptionStatus::NEW, SubscriptionStatus::BOOTING, - SubscriptionStatus::ACTIVE, - SubscriptionStatus::DETACHED, - SubscriptionStatus::ERROR, + SubscriptionStatus::ACTIVE ]))); if ($subscriptions->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions found.'); // todo not happy? Because there must be at least the content graph?!! @@ -80,12 +79,47 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { - return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::BOOTING, $progressCallback)); + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + return $this->processExclusively(fn () => $this->subscriptionStore->transactional( + function () use ($criteria, $progressCallback) { + $this->logger?->info('Subscription Engine: Start catching up subscriptions in state "BOOTING".'); + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate( + SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::BOOTING) + ); + return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); + }) + ); } public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { - return $this->processExclusively(fn () => $this->catchUpSubscriptions($criteria ?? SubscriptionEngineCriteria::noConstraints(), SubscriptionStatus::ACTIVE, $progressCallback)); + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + return $this->processExclusively(fn () => $this->subscriptionStore->transactional( + function () use ($criteria, $progressCallback) { + $this->logger?->info('Subscription Engine: Start catching up subscriptions in state "ACTIVE".'); + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate( + SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::ACTIVE) + ); + return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); + }) + ); + } + + public function reactivate(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult + { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + return $this->processExclusively(fn () => $this->subscriptionStore->transactional( + function () use ($criteria, $progressCallback) { + $this->logger?->info('Subscription Engine: Start catching up subscriptions in state "ACTIVE".'); + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate( + SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ + SubscriptionStatus::ERROR, + SubscriptionStatus::DETACHED, + ])) + ); + return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); + }) + ); } public function reset(SubscriptionEngineCriteria|null $criteria = null): Result @@ -260,129 +294,121 @@ private function resetSubscription(Subscription $subscription): ?Error return null; } - private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatus $subscriptionStatus, \Closure $progressClosure = null): ProcessedResult + private function catchUpSubscriptions(Subscriptions $subscriptionsToCatchup, \Closure $progressClosure = null): ProcessedResult { - $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in state "%s".', $subscriptionStatus->value)); - - $subscriptionEngineCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $subscriptionStatus); - return $this->subscriptionStore->transactional( - function () use ($subscriptionEngineCriteria, $subscriptionStatus, $progressClosure) { - $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionEngineCriteria); - foreach ($subscriptionsToCatchup as $subscription) { - if (!$this->subscribers->contain($subscription->id)) { - // mark detached subscriptions as we cannot handle them and exclude them from catchup - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::DETACHED, - position: $subscription->position, - subscriptionError: null, - ); - $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - } - } + foreach ($subscriptionsToCatchup as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + } + } - if ($subscriptionsToCatchup->isEmpty()) { - $this->logger?->info(sprintf('Subscription Engine: No subscriptions in state "%s". Finishing catch up', $subscriptionStatus->value)); - return ProcessedResult::success(0); - } + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); + return ProcessedResult::success(0); + } - foreach ($subscriptionsToCatchup as $subscription) { - try { - $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); - } catch (\Throwable $e) { - // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); - $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732374000, $e); - } - } - $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); - $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + foreach ($subscriptionsToCatchup as $subscription) { + try { + $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); + } + } + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); - /** @var array $errors */ - $errors = []; - $numberOfProcessedEvents = 0; - /** @var array $highestSequenceNumberForSubscriber */ - $highestSequenceNumberForSubscriber = []; + /** @var array $errors */ + $errors = []; + $numberOfProcessedEvents = 0; + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; - $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); - foreach ($eventStream as $eventEnvelope) { - $sequenceNumber = $eventEnvelope->sequenceNumber; - if ($numberOfProcessedEvents > 0) { - $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); - } - if ($progressClosure !== null) { - $progressClosure($eventEnvelope); - } - $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - foreach ($subscriptionsToCatchup as $subscription) { - if ($subscription->position->value >= $sequenceNumber->value) { - $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); - continue; - } - $this->subscriptionStore->createSavepoint(); - $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); - if ($error !== null) { - // ERROR Case: - // 1.) roll back the partially applied event on the subscriber - $this->subscriptionStore->rollbackSavepoint(); - // 2.) for the leftover events we are not including this failed subscription for catchup - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // 3.) update the subscription error state on either its unchanged or new position (if some events worked) - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ERROR, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: SubscriptionError::fromPreviousStatusAndException( - $subscription->status, - $error->throwable - ), - ); - // 4.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed - // todo put the ERROR $subscriptionStatus into the after hook, so it can properly be reacted upon - try { - $this->subscribers->get($subscription->id)->onAfterCatchUp(); - } catch (\Throwable $e) { - // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" had an error and also failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); - $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732733740, $e); - } - $errors[] = $error; - continue; - } - // HAPPY Case: - $this->subscriptionStore->releaseSavepoint(); - $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; - } - $numberOfProcessedEvents++; + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } + if ($progressClosure !== null) { + $progressClosure($eventEnvelope); + } + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; } - foreach ($subscriptionsToCatchup as $subscription) { + $this->subscriptionStore->createSavepoint(); + $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); + if ($error !== null) { + // ERROR Case: + // 1.) roll back the partially applied event on the subscriber + $this->subscriptionStore->rollbackSavepoint(); + // 2.) for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // 3.) update the subscription error state on either its unchanged or new position (if some events worked) + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $error->throwable + ), + ); + // 4.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed + // todo put the ERROR $subscriptionStatus into the after hook, so it can properly be reacted upon try { $this->subscribers->get($subscription->id)->onAfterCatchUp(); } catch (\Throwable $e) { // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); + $message = sprintf('Subscriber "%s" had an error and also failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732374000, $e); - } - // after catchup mark all subscriptions as active, so they are triggered automatically now. - // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ACTIVE, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: null, - ); - if ($subscription->status !== SubscriptionStatus::ACTIVE) { - $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); + throw new CatchUpFailed($message, 1732733740, $e); } + $errors[] = $error; + continue; } - $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); - return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); + // HAPPY Case: + $this->subscriptionStore->releaseSavepoint(); + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; } - ); + $numberOfProcessedEvents++; + } + foreach ($subscriptionsToCatchup as $subscription) { + try { + $this->subscribers->get($subscription->id)->onAfterCatchUp(); + } catch (\Throwable $e) { + // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); + } + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ACTIVE, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); + if ($subscription->status !== SubscriptionStatus::ACTIVE) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); + } + } + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); + return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); } /** From dc5ff1038b89ca656eb01a420e4763059b455f70 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 27 Nov 2024 21:56:34 +0100 Subject: [PATCH 101/142] TASK: Move transactional logic _on_ projection as it does not belong to the subscription store ... which technically only coincidentally uses the same connection and dbal instance see https://github.com/neos/neos-development-collection/pull/5321#issuecomment-2495670705 --- .../DoctrineDbalContentGraphProjection.php | 3 ++ .../Projection/HypergraphProjection.php | 3 ++ .../TestSuite/DebugEventProjection.php | 3 ++ .../AbstractSubscriptionEngineTestCase.php | 1 + .../ProjectionTransactionTrait.php | 37 +++++++++++++++++++ .../Projection/ProjectionInterface.php | 10 +++++ .../Engine/SubscriptionEngine.php | 10 ++--- .../Store/SubscriptionStoreInterface.php | 6 --- .../Subscriber/ProjectionSubscriber.php | 8 ++-- .../DoctrineSubscriptionStore.php | 15 -------- .../Projection/DocumentUriPathProjection.php | 3 ++ .../ChangeProjection.php | 3 ++ 12 files changed, 71 insertions(+), 31 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 4e998fde99a..025f596f5cb 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -62,6 +62,7 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; +use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; @@ -80,6 +81,8 @@ */ final class DoctrineDbalContentGraphProjection implements ContentGraphProjectionInterface { + use ProjectionTransactionTrait; + use ContentStream; use NodeMove; use NodeRemoval; diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 5961442b9e6..69ab40e5ce3 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -40,6 +40,7 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; +use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; @@ -52,6 +53,8 @@ */ final class HypergraphProjection implements ContentGraphProjectionInterface { + use ProjectionTransactionTrait; + use ContentStreamForking; use NodeCreation; use SubtreeTagging; diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 7f71796e96d..959d00db7aa 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -12,6 +12,7 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; +use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; @@ -29,6 +30,8 @@ */ final class DebugEventProjection implements ProjectionInterface { + use ProjectionTransactionTrait; + private DebugEventProjectionState $state; private \Closure|null $saboteur = null; diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index ca9c4de33eb..60d06f40064 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -65,6 +65,7 @@ public function setUp(): void $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + $this->fakeProjection->expects(self::any())->method('transactional')->willReturnCallback(fn ($fn) => $fn())->willReturnCallback(fn ($fn) => $fn()); FakeProjectionFactory::setProjection( 'default', diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php b/Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php new file mode 100644 index 00000000000..2a6b9d6ebc2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php @@ -0,0 +1,37 @@ +dbal->isTransactionActive() === false) { + /** @phpstan-ignore argument.templateType */ + $this->dbal->transactional($closure); + return; + } + // technically we could leverage nested transactions from dbal, which effectively does the same. + // but that requires us to enable this globally first via setNestTransactionsWithSavepoints also making this explicit is more transparent: + $this->dbal->createSavepoint('PROJECTION'); + try { + $closure(); + } catch (\Throwable $e) { + // roll back the partially applied event on the projection + $this->dbal->rollbackSavepoint('PROJECTION'); + throw $e; + } + $this->dbal->releaseSavepoint('PROJECTION'); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index aff57389895..19090c17cc1 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -30,6 +30,16 @@ public function setUp(): void; */ public function status(): ProjectionStatus; + /** + * Must invoke the closure which will update the catchup hooks and {@see apply}. + * Additionally, to guarantee exactly once delivery and also to behave correct during exceptions (even fatal ones), + * a database transaction should be started, or if a transaction is already active on the same connection save points + * must be used and rolled back on error. + * + * @param-immediately-invoked-callable $closure + */ + public function transactional(\Closure $closure): void; + public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 8d556ec8017..93dd501e908 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -349,15 +349,12 @@ private function catchUpSubscriptions(Subscriptions $subscriptionsToCatchup, \Cl $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); continue; } - $this->subscriptionStore->createSavepoint(); $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); if ($error !== null) { // ERROR Case: - // 1.) roll back the partially applied event on the subscriber - $this->subscriptionStore->rollbackSavepoint(); - // 2.) for the leftover events we are not including this failed subscription for catchup + // 1.) for the leftover events we are not including this failed subscription for catchup $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // 3.) update the subscription error state on either its unchanged or new position (if some events worked) + // 2.) update the subscription error state on either its unchanged or new position (if some events worked) $this->subscriptionStore->update( $subscription->id, status: SubscriptionStatus::ERROR, @@ -367,7 +364,7 @@ private function catchUpSubscriptions(Subscriptions $subscriptionsToCatchup, \Cl $error->throwable ), ); - // 4.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed + // 3.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed // todo put the ERROR $subscriptionStatus into the after hook, so it can properly be reacted upon try { $this->subscribers->get($subscription->id)->onAfterCatchUp(); @@ -381,7 +378,6 @@ private function catchUpSubscriptions(Subscriptions $subscriptionsToCatchup, \Cl continue; } // HAPPY Case: - $this->subscriptionStore->releaseSavepoint(); $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; } $numberOfProcessedEvents++; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 85f31072717..b7b0540415b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -35,10 +35,4 @@ public function update( * @return T */ public function transactional(\Closure $closure): mixed; - - public function createSavepoint(): void; - - public function releaseSavepoint(): void; - - public function rollbackSavepoint(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php index 7e15a8fdcba..ee19dbc548c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php @@ -34,9 +34,11 @@ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void public function handle(EventInterface $event, EventEnvelope $eventEnvelope): void { - $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); - $this->projection->apply($event, $eventEnvelope); - $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); + $this->projection->transactional(function () use ($event, $eventEnvelope) { + $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); + $this->projection->apply($event, $eventEnvelope); + $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); + }); } public function onAfterCatchUp(): void diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 9c7c9e92005..fe5f038fda4 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -169,19 +169,4 @@ public function transactional(\Closure $closure): mixed { return $this->dbal->transactional($closure); } - - public function createSavepoint(): void - { - $this->dbal->createSavepoint('SUBSCRIBER'); - } - - public function releaseSavepoint(): void - { - $this->dbal->releaseSavepoint('SUBSCRIBER'); - } - - public function rollbackSavepoint(): void - { - $this->dbal->rollbackSavepoint('SUBSCRIBER'); - } } diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index 016814020bc..2c2dbe5101a 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -25,6 +25,7 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; +use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ProjectionInterface; @@ -40,6 +41,8 @@ */ final class DocumentUriPathProjection implements ProjectionInterface, WithMarkStaleInterface { + use ProjectionTransactionTrait; + public const COLUMN_TYPES_DOCUMENT_URIS = [ 'shortcutTarget' => Types::JSON, ]; diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index ead06f7b588..af0f3e894f7 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -39,6 +39,7 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; +use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; @@ -52,6 +53,8 @@ */ class ChangeProjection implements ProjectionInterface { + use ProjectionTransactionTrait; + /** * @var ChangeFinder|null Cache for the ChangeFinder returned by {@see getState()}, * so that always the same instance is returned From 3448e21d57dbf2530d0a9d9d9e6b87e13da11fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Mu=CC=88ller?= Date: Sun, 1 Dec 2024 13:37:06 +0100 Subject: [PATCH 102/142] SubscriptionEngineTest postgresql compatible --- .../AbstractSubscriptionEngineTestCase.php | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 60d06f40064..5b0d03f8c07 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -5,6 +5,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Platforms\SqlitePlatform; use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; @@ -122,21 +125,36 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor final protected function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void { - $connection->prepare('SET FOREIGN_KEY_CHECKS = 0;')->executeStatement(); - foreach ($connection->createSchemaManager()->listTableNames() as $tableNames) { - if (!str_starts_with($tableNames, sprintf('cr_%s_', $contentRepositoryId->value))) { + $preDeleteStatement = match (true) { + $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform => 'SET FOREIGN_KEY_CHECKS = 0;', + default => '', + }; + + if ($preDeleteStatement !== '') { + $connection->prepare($preDeleteStatement)->executeStatement(); + } + + $truncateDropStatement = match (true) { + $connection->getDatabasePlatform() instanceof PostgreSQLPlatform => '%s TABLE `%s` CASCADE', + default => '%s TABLE `%s`', + }; + + foreach ($connection->createSchemaManager()->listTableNames() as $tableName) { + if (!str_starts_with($tableName, sprintf('cr_%s_', $contentRepositoryId->value))) { // speedup deletion, only delete current cr continue; } - if ($keepSchema) { - // truncate is faster - $sql = 'TRUNCATE TABLE ' . $tableNames; - } else { - $sql = 'DROP TABLE ' . $tableNames; - } + // truncate is faster + $sql = sprintf($truncateDropStatement, $keepSchema ? 'TRUNCATE' : 'DROP', $tableName); $connection->prepare($sql)->executeStatement(); } - $connection->prepare('SET FOREIGN_KEY_CHECKS = 1;')->executeStatement(); + + $postDeleteStatement = match (true) { + $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform => 'SET FOREIGN_KEY_CHECKS = 1;', + default => '', + }; + + $connection->prepare($postDeleteStatement)->executeStatement(); } final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null From 5abdb0acbe8b9bf0f4102afc47105bd903136ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Mu=CC=88ller?= Date: Sun, 1 Dec 2024 13:46:17 +0100 Subject: [PATCH 103/142] Add missing empty string check --- .../Subscription/AbstractSubscriptionEngineTestCase.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 5b0d03f8c07..b5ffa5c89a3 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -134,7 +134,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository $connection->prepare($preDeleteStatement)->executeStatement(); } - $truncateDropStatement = match (true) { + $truncateOrDropStatement = match (true) { $connection->getDatabasePlatform() instanceof PostgreSQLPlatform => '%s TABLE `%s` CASCADE', default => '%s TABLE `%s`', }; @@ -145,7 +145,7 @@ final protected function resetDatabase(Connection $connection, ContentRepository continue; } // truncate is faster - $sql = sprintf($truncateDropStatement, $keepSchema ? 'TRUNCATE' : 'DROP', $tableName); + $sql = sprintf($truncateOrDropStatement, $keepSchema ? 'TRUNCATE' : 'DROP', $tableName); $connection->prepare($sql)->executeStatement(); } @@ -154,7 +154,9 @@ final protected function resetDatabase(Connection $connection, ContentRepository default => '', }; - $connection->prepare($postDeleteStatement)->executeStatement(); + if ($postDeleteStatement !== '') { + $connection->prepare($postDeleteStatement)->executeStatement(); + } } final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null From 2b82f62c120e42315143556444e0a725264c37f3 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:21:44 +0100 Subject: [PATCH 104/142] TASK: Declare `EventNormalizer` as internal --- .../Classes/EventStore/EventNormalizer.php | 47 +++++++++++-------- .../Factory/ContentRepositoryFactory.php | 12 +++-- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php index 14943d68c97..2648f055470 100644 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php +++ b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php @@ -4,7 +4,6 @@ namespace Neos\ContentRepository\Core\EventStore; -use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\Events as DomainEvents; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasClosed; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasReopened; @@ -42,7 +41,6 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\EventData; use Neos\EventStore\Model\Event\EventId; @@ -52,28 +50,29 @@ /** * Central authority to convert Content Repository domain events to Event Store EventData and EventType, vice versa. * - * For normalizing (from classes to event store), this is called from {@see ContentRepository::normalizeEvent()}. + * - For normalizing (from classes to event store) + * - For denormalizing (from event store to classes) * - * For denormalizing (from event store to classes), this is called in the individual projections; f.e. - * {@see ProjectionInterface::apply()}. - * - * @api because inside projections, you get an instance of EventNormalizer to handle events. + * @internal inside projections the event will already be denormalized */ -final class EventNormalizer +final readonly class EventNormalizer { - /** - * @var array,EventType> - */ - private array $fullClassNameToShortEventType = []; - /** - * @var array> - */ - private array $shortEventTypeToFullClassName = []; + private function __construct( + /** + * @var array,EventType> + */ + private array $fullClassNameToShortEventType, + /** + * @var array> + */ + private array $shortEventTypeToFullClassName, + ) { + } /** - * @internal never instanciate this object yourself + * @internal never instantiate this object yourself */ - public function __construct() + public static function create(): self { $supportedEventClassNames = [ ContentStreamWasClosed::class, @@ -113,12 +112,20 @@ public function __construct() WorkspaceBaseWorkspaceWasChanged::class, ]; + $fullClassNameToShortEventType = []; + $shortEventTypeToFullClassName = []; + foreach ($supportedEventClassNames as $fullEventClassName) { $shortEventClassName = substr($fullEventClassName, strrpos($fullEventClassName, '\\') + 1); - $this->fullClassNameToShortEventType[$fullEventClassName] = EventType::fromString($shortEventClassName); - $this->shortEventTypeToFullClassName[$shortEventClassName] = $fullEventClassName; + $fullClassNameToShortEventType[$fullEventClassName] = EventType::fromString($shortEventClassName); + $shortEventTypeToFullClassName[$shortEventClassName] = $fullEventClassName; } + + return new self( + fullClassNameToShortEventType: $fullClassNameToShortEventType, + shortEventTypeToFullClassName: $shortEventTypeToFullClassName + ); } /** diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index ea4593a6a70..ae9d7ef4fad 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -57,11 +57,12 @@ final class ContentRepositoryFactory private SubscriptionEngine $subscriptionEngine; private ContentGraphProjectionInterface $contentGraphProjection; private ProjectionStates $additionalProjectionStates; + private EventNormalizer $eventNormalizer; // guards against recursion and memory overflow private bool $isBuilding = false; - // The following properties store "singleton" references of objects for this content repository + // The "singleton" reference for this content repository private ?ContentRepository $contentRepositoryRuntimeCache = null; /** @@ -89,6 +90,7 @@ public function __construct( ); $eventNormalizer = new EventNormalizer(); $this->subscriberFactoryDependencies = new SubscriberFactoryDependencies( + $this->eventNormalizer = EventNormalizer::create(); $contentRepositoryId, $eventNormalizer, $nodeTypeManager, @@ -107,7 +109,7 @@ public function __construct( $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); $subscribers[] = $this->buildContentGraphSubscriber(); - $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $eventNormalizer, $logger); + $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $this->eventNormalizer, $logger); } private function buildContentGraphSubscriber(): ProjectionSubscriber @@ -166,7 +168,7 @@ public function getOrBuild(): ContentRepository $commandSimulatorFactory = new CommandSimulatorFactory( $this->contentGraphProjection, - $this->subscriberFactoryDependencies->eventNormalizer, + $this->eventNormalizer, $commandBusForRebaseableCommands ); @@ -174,7 +176,7 @@ public function getOrBuild(): ContentRepository new WorkspaceCommandHandler( $commandSimulatorFactory, $this->eventStore, - $this->subscriberFactoryDependencies->eventNormalizer, + $this->eventNormalizer, ) ); $authProvider = $this->authProviderFactory->build($this->contentRepositoryId, $contentGraphReadModel); @@ -189,7 +191,7 @@ public function getOrBuild(): ContentRepository $this->contentRepositoryId, $publicCommandBus, $this->eventStore, - $this->subscriberFactoryDependencies->eventNormalizer, + $this->eventNormalizer, $this->subscriptionEngine, $this->subscriberFactoryDependencies->nodeTypeManager, $this->subscriberFactoryDependencies->interDimensionalVariationGraph, From 4353f0fca4cc5412062e0422c9099c5a7fb4d832 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:31:53 +0100 Subject: [PATCH 105/142] TASK: Simplify `SubscriberFactoryDependencies` to only contain api things As we reduced the dependencies there, we now have to save their instances in the `ContentRepositoryFactory` and can ignore the `SubscriberFactoryDependencies` then. --- .../Factory/ContentRepositoryFactory.php | 73 ++++++++++--------- ...ntRepositoryServiceFactoryDependencies.php | 24 +++--- .../Factory/SubscriberFactoryDependencies.php | 26 +++++-- 3 files changed, 73 insertions(+), 50 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index ae9d7ef4fad..fde8bdd29a7 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -53,11 +53,13 @@ */ final class ContentRepositoryFactory { - private SubscriberFactoryDependencies $subscriberFactoryDependencies; private SubscriptionEngine $subscriptionEngine; private ContentGraphProjectionInterface $contentGraphProjection; private ProjectionStates $additionalProjectionStates; private EventNormalizer $eventNormalizer; + private ContentDimensionZookeeper $contentDimensionZookeeper; + private InterDimensionalVariationGraph $interDimensionalVariationGraph; + private PropertyConverter $propertyConverter; // guards against recursion and memory overflow private bool $isBuilding = false; @@ -71,8 +73,8 @@ final class ContentRepositoryFactory public function __construct( private readonly ContentRepositoryId $contentRepositoryId, private readonly EventStoreInterface $eventStore, - NodeTypeManager $nodeTypeManager, - ContentDimensionSourceInterface $contentDimensionSource, + private readonly NodeTypeManager $nodeTypeManager, + private readonly ContentDimensionSourceInterface $contentDimensionSource, Serializer $propertySerializer, private readonly AuthProviderFactoryInterface $authProviderFactory, private readonly ClockInterface $clock, @@ -83,31 +85,29 @@ public function __construct( private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, LoggerInterface|null $logger = null, ) { - $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); - $interDimensionalVariationGraph = new InterDimensionalVariationGraph( + $this->contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); + $this->interDimensionalVariationGraph = new InterDimensionalVariationGraph( $contentDimensionSource, - $contentDimensionZookeeper + $this->contentDimensionZookeeper ); - $eventNormalizer = new EventNormalizer(); - $this->subscriberFactoryDependencies = new SubscriberFactoryDependencies( $this->eventNormalizer = EventNormalizer::create(); + $this->propertyConverter = new PropertyConverter($propertySerializer); + $subscriberFactoryDependencies = SubscriberFactoryDependencies::create( $contentRepositoryId, - $eventNormalizer, $nodeTypeManager, $contentDimensionSource, - $contentDimensionZookeeper, - $interDimensionalVariationGraph, - new PropertyConverter($propertySerializer), + $this->interDimensionalVariationGraph, + $this->propertyConverter, ); $subscribers = []; $additionalProjectionStates = []; foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { - $subscriber = $additionalSubscriberFactory->build($this->subscriberFactoryDependencies); + $subscriber = $additionalSubscriberFactory->build($subscriberFactoryDependencies); $additionalProjectionStates[] = $subscriber->projection->getState(); $subscribers[] = $subscriber; } $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); - $this->contentGraphProjection = $contentGraphProjectionFactory->build($this->subscriberFactoryDependencies); + $this->contentGraphProjection = $contentGraphProjectionFactory->build($subscriberFactoryDependencies); $subscribers[] = $this->buildContentGraphSubscriber(); $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $this->eventNormalizer, $logger); } @@ -120,9 +120,9 @@ private function buildContentGraphSubscriber(): ProjectionSubscriber $this->contentGraphCatchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( $this->contentRepositoryId, $this->contentGraphProjection->getState(), - $this->subscriberFactoryDependencies->nodeTypeManager, - $this->subscriberFactoryDependencies->contentDimensionSource, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->nodeTypeManager, + $this->contentDimensionSource, + $this->interDimensionalVariationGraph, )), ); } @@ -150,19 +150,19 @@ public function getOrBuild(): ContentRepository $commandBusForRebaseableCommands = new CommandBus( $commandHandlingDependencies, new NodeAggregateCommandHandler( - $this->subscriberFactoryDependencies->nodeTypeManager, - $this->subscriberFactoryDependencies->contentDimensionZookeeper, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, - $this->subscriberFactoryDependencies->propertyConverter, + $this->nodeTypeManager, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, + $this->propertyConverter, ), new DimensionSpaceCommandHandler( - $this->subscriberFactoryDependencies->contentDimensionZookeeper, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, ), new NodeDuplicationCommandHandler( - $this->subscriberFactoryDependencies->nodeTypeManager, - $this->subscriberFactoryDependencies->contentDimensionZookeeper, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->nodeTypeManager, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, ) ); @@ -183,9 +183,9 @@ public function getOrBuild(): ContentRepository $commandHooks = $this->commandHooksFactory->build(CommandHooksFactoryDependencies::create( $this->contentRepositoryId, $this->contentGraphProjection->getState(), - $this->subscriberFactoryDependencies->nodeTypeManager, - $this->subscriberFactoryDependencies->contentDimensionSource, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, + $this->nodeTypeManager, + $this->contentDimensionSource, + $this->interDimensionalVariationGraph, )); $this->contentRepositoryRuntimeCache = new ContentRepository( $this->contentRepositoryId, @@ -193,9 +193,9 @@ public function getOrBuild(): ContentRepository $this->eventStore, $this->eventNormalizer, $this->subscriptionEngine, - $this->subscriberFactoryDependencies->nodeTypeManager, - $this->subscriberFactoryDependencies->interDimensionalVariationGraph, - $this->subscriberFactoryDependencies->contentDimensionSource, + $this->nodeTypeManager, + $this->interDimensionalVariationGraph, + $this->contentDimensionSource, $authProvider, $this->clock, $contentGraphReadModel, @@ -220,10 +220,15 @@ public function getOrBuild(): ContentRepository public function buildService( ContentRepositoryServiceFactoryInterface $serviceFactory ): ContentRepositoryServiceInterface { - $serviceFactoryDependencies = ContentRepositoryServiceFactoryDependencies::create( - $this->subscriberFactoryDependencies, + $this->contentRepositoryId, $this->eventStore, + $this->eventNormalizer, + $this->nodeTypeManager, + $this->contentDimensionSource, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, + $this->propertyConverter, $this->getOrBuild(), $this->contentGraphProjection->getState(), $this->subscriptionEngine, diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index 756440449f3..4bc420a7986 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -34,7 +34,6 @@ final readonly class ContentRepositoryServiceFactoryDependencies { private function __construct( - // These properties are from ProjectionFactoryDependencies public ContentRepositoryId $contentRepositoryId, public EventStoreInterface $eventStore, public EventNormalizer $eventNormalizer, @@ -45,7 +44,6 @@ private function __construct( public PropertyConverter $propertyConverter, public ContentRepository $contentRepository, public ContentGraphReadModelInterface $contentGraphReadModel, - // we don't need CommandBus, because this is included in ContentRepository->handle() public SubscriptionEngine $subscriptionEngine, ) { } @@ -54,21 +52,27 @@ private function __construct( * @internal */ public static function create( - SubscriberFactoryDependencies $projectionFactoryDependencies, + ContentRepositoryId $contentRepositoryId, EventStoreInterface $eventStore, + EventNormalizer $eventNormalizer, + NodeTypeManager $nodeTypeManager, + ContentDimensionSourceInterface $contentDimensionSource, + ContentDimensionZookeeper $contentDimensionZookeeper, + InterDimensionalVariationGraph $interDimensionalVariationGraph, + PropertyConverter $propertyConverter, ContentRepository $contentRepository, ContentGraphReadModelInterface $contentGraphReadModel, SubscriptionEngine $subscriptionEngine, ): self { return new self( - $projectionFactoryDependencies->contentRepositoryId, + $contentRepositoryId, $eventStore, - $projectionFactoryDependencies->eventNormalizer, - $projectionFactoryDependencies->nodeTypeManager, - $projectionFactoryDependencies->contentDimensionSource, - $projectionFactoryDependencies->contentDimensionZookeeper, - $projectionFactoryDependencies->interDimensionalVariationGraph, - $projectionFactoryDependencies->propertyConverter, + $eventNormalizer, + $nodeTypeManager, + $contentDimensionSource, + $contentDimensionZookeeper, + $interDimensionalVariationGraph, + $propertyConverter, $contentRepository, $contentGraphReadModel, $subscriptionEngine, diff --git a/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php index 8a6af1f94a4..da09885a2fa 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php @@ -15,27 +15,41 @@ namespace Neos\ContentRepository\Core\Factory; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; -use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; -use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\EventStore\EventStoreInterface; /** * @api because it is used inside the ProjectionsFactory */ final readonly class SubscriberFactoryDependencies { - public function __construct( + private function __construct( public ContentRepositoryId $contentRepositoryId, - public EventNormalizer $eventNormalizer, public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, - public ContentDimensionZookeeper $contentDimensionZookeeper, public InterDimensionalVariationGraph $interDimensionalVariationGraph, public PropertyConverter $propertyConverter, ) { } + + /** + * @internal + */ + public static function create( + ContentRepositoryId $contentRepositoryId, + NodeTypeManager $nodeTypeManager, + ContentDimensionSourceInterface $contentDimensionSource, + InterDimensionalVariationGraph $interDimensionalVariationGraph, + PropertyConverter $propertyConverter + ): self { + return new self( + $contentRepositoryId, + $nodeTypeManager, + $contentDimensionSource, + $interDimensionalVariationGraph, + $propertyConverter + ); + } } From ea3eaa6a97316fb9bcfd2538301cabdec10e15d2 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:32:39 +0100 Subject: [PATCH 106/142] TASK: Introduce `getPropertyConverter` to denote that this is really internal --- .../src/DoctrineDbalContentGraphProjectionFactory.php | 2 +- .../src/HypergraphProjectionFactory.php | 2 +- .../Classes/Factory/SubscriberFactoryDependencies.php | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php index 0d69c3c6a22..f63308efc43 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php @@ -34,7 +34,7 @@ public function build( $nodeFactory = new NodeFactory( $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->propertyConverter, + $projectionFactoryDependencies->getPropertyConverter(), $dimensionSpacePointsRepository ); diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php index a70b0d5d148..b2775d8b69c 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php @@ -36,7 +36,7 @@ public function build( $nodeFactory = new NodeFactory( $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->propertyConverter + $projectionFactoryDependencies->getPropertyConverter() ); return new HypergraphProjection( diff --git a/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php index da09885a2fa..1bd43162bc4 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php @@ -30,7 +30,7 @@ private function __construct( public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, public InterDimensionalVariationGraph $interDimensionalVariationGraph, - public PropertyConverter $propertyConverter, + private PropertyConverter $propertyConverter, ) { } @@ -52,4 +52,12 @@ public static function create( $propertyConverter ); } + + /** + * @internal only to be used for custom content graph integrations to build a node property collection + */ + public function getPropertyConverter(): PropertyConverter + { + return $this->propertyConverter; + } } From e750d93b04ef7e3d98d0c87d7e86e0acc66acc0c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:09:21 +0100 Subject: [PATCH 107/142] TASK: Migrate checkpoints to subscriptions via `migrateevents:migrateCheckpointsToSubscriptions` --- .../Classes/Subscription/Subscriptions.php | 8 +++ .../MigrateEventsCommandController.php | 16 +++++ .../Classes/Service/EventMigrationService.php | 61 +++++++++++++++++++ .../Service/EventMigrationServiceFactory.php | 1 + 4 files changed, 86 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php index 3bba94d0018..73760148746 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -53,6 +53,14 @@ public function isEmpty(): bool return $this->subscriptionsById === []; } + public function first(): ?Subscription + { + foreach ($this->subscriptionsById as $subscription) { + return $subscription; + } + return null; + } + public function count(): int { return count($this->subscriptionsById); diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php index 01cbc8ccf87..821da7d8dc3 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php @@ -168,4 +168,20 @@ public function copyNodesStatusCommand(string $contentRepository = 'default'): v $eventMigrationService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->eventMigrationServiceFactory); $eventMigrationService->copyNodesStatus($this->outputLine(...)); } + + /** + * Migrates the checkpoint tables to ACTIVE subscriptions + * + * Needed for #5321: https://github.com/neos/neos-development-collection/pull/5321 + * + * Included in November 2024 - before final Neos 9.0 release + * + * @param string $contentRepository Identifier of the Content Repository to migrate + */ + public function migrateCheckpointsToSubscriptionsCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $eventMigrationService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->eventMigrationServiceFactory); + $eventMigrationService->migrateCheckpointsToSubscriptions($this->outputLine(...)); + } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php index 34a23dc5fc1..22c2283435f 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php @@ -28,6 +28,12 @@ use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscription; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\Command\MigrateEventsCommandController; use Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory; use Neos\EventStore\EventStoreInterface; @@ -44,6 +50,7 @@ use Neos\Neos\Domain\Model\WorkspaceClassification; use Neos\Neos\Domain\Model\WorkspaceRole; use Neos\Neos\Domain\Model\WorkspaceRoleSubjectType; +use Doctrine\DBAL\Exception as DBALException; /** * Content Repository service to perform migrations of events. @@ -59,6 +66,7 @@ final class EventMigrationService implements ContentRepositoryServiceInterface private array $eventsModified = []; public function __construct( + private readonly SubscriptionEngine $subscriptionEngine, private readonly ContentRepositoryId $contentRepositoryId, private readonly EventStoreInterface $eventStore, private readonly Connection $connection @@ -903,6 +911,59 @@ public function copyNodesStatus(\Closure $outputFn): void $outputFn('NOTE: To reduce the number of matched content streams and to cleanup the event store run `./flow contentStream:removeDangling` and `./flow contentStream:pruneRemovedFromEventStream`'); } + public function migrateCheckpointsToSubscriptions(\Closure $outputFn): void + { + /** @var SubscriptionStoreInterface $subscriptionStore */ + $subscriptionStore = (new \ReflectionClass($this->subscriptionEngine))->getProperty('subscriptionStore')->getValue($this->subscriptionEngine); + $subscriptionStore->setup(); + + foreach ([ + 'contentGraph' => 'cr_%s_p_graph_checkpoint', + 'Neos.Neos:DocumentUriPathProjection' => 'cr_%s_p_neos_documenturipath_checkpoint', + 'Neos.Neos:PendingChangesProjection' => 'cr_%s_p_neos_change_checkpoint', + ] as $subscriberId => $projectionCheckpointTablePattern) { + $projectionCheckpointTable = sprintf($projectionCheckpointTablePattern, $this->contentRepositoryId->value); + try { + $rows = $this->connection->fetchAllAssociative("SELECT appliedsequencenumber from {$projectionCheckpointTable}"); + } catch (DBALException $e) { + $outputFn(sprintf('Could not migrate subscriber %s, please replay: %s', $subscriberId, $e->getMessage())); + continue; + } + $first = reset($rows); + if (count($rows) !== 1 || !isset($first['appliedsequencenumber'])) { + $outputFn(sprintf('Could not migrate subscriber %s, please replay. Expected exactly one appliedsequencenumber', $subscriberId)); + continue; + } + + $subscription = $subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::create([SubscriptionId::fromString($subscriberId)]))->first(); + + if ($subscription === null) { + $subscriptionStore->add( + new Subscription( + SubscriptionId::fromString($subscriberId), + SubscriptionStatus::ACTIVE, + SequenceNumber::fromInteger((int)$first['appliedsequencenumber']), + null, + null + ) + ); + $outputFn(sprintf('Added subscriber %s with active and position %s', $subscriberId, $first['appliedsequencenumber'])); + } else { + if ($subscription->status === SubscriptionStatus::ACTIVE) { + $outputFn(sprintf('Subscriber %s is already active', $subscriberId)); + continue; + } + $subscriptionStore->update( + SubscriptionId::fromString($subscriberId), + SubscriptionStatus::ACTIVE, + SequenceNumber::fromInteger((int)$first['appliedsequencenumber']), + null + ); + $outputFn(sprintf('Updated subscriber %s to active and position %s', $subscriberId, $first['appliedsequencenumber'])); + } + } + } + /** ------------------------ */ /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php index f61935dccaa..2e2dcea0f3f 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php @@ -32,6 +32,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor } return new EventMigrationService( + $serviceFactoryDependencies->subscriptionEngine, $serviceFactoryDependencies->contentRepositoryId, $serviceFactoryDependencies->eventStore, $this->connection From 33b717aefee91b4f845b492d743c632cafeb7c6d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:39:33 +0100 Subject: [PATCH 108/142] TASK: Disable running subscription test on PostgreSQL ... because the content graph is not finished yet and the dbal version is not really dbal and postgres compatible: https://github.com/neos/neos-development-collection/issues/3855 --- .../Subscription/AbstractSubscriptionEngineTestCase.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index b5ffa5c89a3..73b11b5ef81 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -58,6 +58,9 @@ abstract class AbstractSubscriptionEngineTestCase extends TestCase // we don't u public function setUp(): void { + if ($this->getObject(Connection::class)->getDatabasePlatform() instanceof PostgreSQLPlatform) { + $this->markTestSkipped('TODO: The content graph is not available in postgres currently: https://github.com/neos/neos-development-collection/issues/3855'); + } $contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); $this->resetDatabase( From 9286afce210564dbe2e46cc3bae586d79ef7a9d0 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:40:08 +0100 Subject: [PATCH 109/142] TASK: Remove obsolete todo --- .../Functional/Subscription/SubscriptionDetachedStatusTest.php | 2 +- .../Classes/Subscription/Exception/CatchUpFailed.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php index 5b073a4db92..1adfb18e041 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -156,7 +156,7 @@ public function projectionIsDetachedOnSetupAndReattachedIfPossible() subscriptionStatus: SubscriptionStatus::DETACHED, subscriptionPosition: SequenceNumber::fromInteger(1), subscriptionError: null, - setupStatus: ProjectionStatus::ok() // state _IS_ calculate-able at this point, todo better reflect meaning: is detached, but re-attachable! + setupStatus: ProjectionStatus::ok() ), $this->subscriptionStatus('Vendor.Package:FakeProjection') ); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php index 2cee008bda8..9e76c3db051 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php @@ -6,7 +6,7 @@ /** * Only thrown if there is no way to recover the started catchup. The transaction will be rolled back. - * + * * @api */ final class CatchUpFailed extends \RuntimeException From 34fd834a2853d5973e0f9960425df3c4bd8105fe Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 21:02:49 +0100 Subject: [PATCH 110/142] TASK: Dont use `new EventNormalizer()` in tests --- .../Tests/Behavior/Bootstrap/FeatureContext.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 7b8ef635cf2..fa8c3b68f4c 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -103,24 +103,26 @@ public function iRunTheEventMigration(RootNodeTypeMapping $rootNodeTypeMapping = $propertyMapper = $this->getObject(PropertyMapper::class); // HACK to access the property converter - $propertyConverterAccess = new class implements ContentRepositoryServiceFactoryInterface { + $crInternalsAccess = new class implements ContentRepositoryServiceFactoryInterface { public PropertyConverter|null $propertyConverter; + public EventNormalizer|null $eventNormalizer; public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { $this->propertyConverter = $serviceFactoryDependencies->propertyConverter; + $this->eventNormalizer = $serviceFactoryDependencies->eventNormalizer; return new class implements ContentRepositoryServiceInterface { }; } }; - $this->getContentRepositoryService($propertyConverterAccess); + $this->getContentRepositoryService($crInternalsAccess); $eventExportProcessor = new EventExportProcessor( $nodeTypeManager, $propertyMapper, - $propertyConverterAccess->propertyConverter, + $crInternalsAccess->propertyConverter, $this->currentContentRepository->getVariationGraph(), - $this->getObject(EventNormalizer::class), + $crInternalsAccess->eventNormalizer, $rootNodeTypeMapping ?? RootNodeTypeMapping::fromArray(['/sites' => NodeTypeNameFactory::NAME_SITES]), $this->nodeDataRows ); From caa70bf4a33486e28aa7e2e302a8799ce263f53c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:28:08 +0100 Subject: [PATCH 111/142] TASK: Fix php cs --- .../Subscription/Engine/SubscriptionEngine.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 93dd501e908..e54a6dbc112 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -87,8 +87,8 @@ function () use ($criteria, $progressCallback) { SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::BOOTING) ); return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); - }) - ); + } + )); } public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult @@ -101,8 +101,8 @@ function () use ($criteria, $progressCallback) { SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::ACTIVE) ); return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); - }) - ); + } + )); } public function reactivate(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult @@ -118,8 +118,8 @@ function () use ($criteria, $progressCallback) { ])) ); return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); - }) - ); + } + )); } public function reset(SubscriptionEngineCriteria|null $criteria = null): Result From 54b24b8027f2588b575aceb462d40fce53bf8de4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:29:15 +0100 Subject: [PATCH 112/142] BUGFIX: Ensure `onAfterCatchUp` is always executed _after_ the projection is persisted. Even in error case. --- .../Subscription/CatchUpHookErrorTest.php | 60 ++++++++++++++++++- .../CatchUpHook/CatchUpHookInterface.php | 20 +++++-- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 3396ec98639..84e051864be 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -156,7 +156,7 @@ public function error_onBeforeCatchUp_abortsCatchup() } /** @test */ - public function error_onAfterCatchUp_abortsCatchupAndRollBack() + public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -190,10 +190,64 @@ public function error_onAfterCatchUp_abortsCatchupAndRollBack() self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onAfterCatchUp: This catchup hook is kaputt.'); - // still the initial status + // one event is applied! + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted_withProjectionError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - // must be empty because full rollback + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $innerException = new \RuntimeException('Inner event handling is kaputt.') + );; + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + new \RuntimeException('This catchup hook is kaputt.') + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedFailure = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $expectedFailure = $e; + } + self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" had an error and also failed onAfterCatchUp: This catchup hook is kaputt.'); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $innerException), + setupStatus: ProjectionStatus::ok(), + ); + + // projection is still marked as error + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index 6dd4301f2a7..cedadbc79ea 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -5,7 +5,8 @@ namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Projection\ProjectionInterface; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; @@ -21,25 +22,34 @@ interface CatchUpHookInterface { /** * This hook is called at the beginning of a catch-up run; - * AFTER the Database Lock is acquired ({@see SubscriptionEngine::catchUpActive()}). + * AFTER the Database Lock is acquired, BEFORE any projection was opened. + * + * Its important that no errors are thrown, as they will cause the catchup to directly halt with a {@see CatchUpFailed} exception. */ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; /** * This hook is called for every event during the catchup process, **before** the projection - * is updated. Thus, this hook runs AFTER the database lock is acquired. + * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. + * + * Any errors will cause the transaction being rolled back, and the projection goes into {@see SubscriptionStatus::ERROR} state. */ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; /** * This hook is called for every event during the catchup process, **after** the projection - * is updated. Thus, this hook runs AFTER the database lock is acquired. + * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. + * + * Any errors will cause the transaction being rolled back, and the projection goes into {@see SubscriptionStatus::ERROR} state. */ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; /** * This hook is called at the END of a catch-up run - * BEFORE the Database Lock is released ({@see SubscriptionEngine::catchUpActive()}). + * BEFORE the Database Lock is released, but AFTER the transaction is commited. + * + * Its important that no errors are thrown, as they will cause the catchup to directly halt with a {@see CatchUpFailed} exception. + * The projections and their new position will already be persisted and there is no rollback. */ public function onAfterCatchUp(): void; } From 4a7b0585d35ccfcab32ea6f586039d6abfe5cca0 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:45:53 +0100 Subject: [PATCH 113/142] TASK: Reactivate `ParallelWritingInWorkspacesTest` Replaces: https://github.com/neos/neos-development-collection/pull/5376 We agreed on using the default timeout for `FOR UPDATE` locks in the database. This test only failed previously because of the `NOWAIT`. The `NOWAIT` was only needed when we had a lot of catchup processes which always did a for update for EACH event which was not a good combination. Now when getting the lock once the default wait of the database is more sane and easier to implement than fetching the lowest position and always waiting. --- .../ParallelWritingInWorkspacesTest.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php index 58d0ce6acc9..06b508c094e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php @@ -38,10 +38,7 @@ use PHPUnit\Framework\Assert; /** - * This tests ensures that the subscribers are updated without any locking problems: - * - * Exception in subscriber "contentGraph" while catching up: - * An exception occurred while executing a query: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction. + * This tests ensures that the subscribers are updated without any locking problems (and to test via {@see DebugEventProjection} that locking is used at all!) * * To test that we utilise two processes committing and catching up to a lot of events. * The is archived by creating nodes in a loop which have tethered nodes as this will lead to a lot of events being emitted in a fast way. From c47d1813500fa4befd165ac819ed45206c1af38b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:29:03 +0100 Subject: [PATCH 114/142] TASK: Test `ProjectionTransactionTrait` when using external projections CASE: $this->dbal->isTransactionActive() === false --- .../AbstractSubscriptionEngineTestCase.php | 24 ++- .../ExternalProjectionErrorTest.php | 36 +++++ .../Subscription/ProjectionErrorTest.php | 139 +---------------- .../ProjectionRollbackTestTrait.php | 144 ++++++++++++++++++ 4 files changed, 204 insertions(+), 139 deletions(-) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 73b11b5ef81..2a3476bc987 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -44,6 +44,8 @@ */ abstract class AbstractSubscriptionEngineTestCase extends TestCase // we don't use Flows functional test case as it would reset the database afterwards { + protected static ContentRepositoryId $contentRepositoryId; + protected ContentRepository $contentRepository; protected SubscriptionEngine $subscriptionEngine; @@ -56,16 +58,20 @@ abstract class AbstractSubscriptionEngineTestCase extends TestCase // we don't u protected CatchUpHookInterface&MockObject $catchupHookForFakeProjection; + public static function setUpBeforeClass(): void + { + static::$contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); + } + public function setUp(): void { if ($this->getObject(Connection::class)->getDatabasePlatform() instanceof PostgreSQLPlatform) { $this->markTestSkipped('TODO: The content graph is not available in postgres currently: https://github.com/neos/neos-development-collection/issues/3855'); } - $contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); $this->resetDatabase( $this->getObject(Connection::class), - $contentRepositoryId, + self::$contentRepositoryId, keepSchema: true ); @@ -78,10 +84,12 @@ public function setUp(): void $this->fakeProjection ); - $this->secondFakeProjection = new DebugEventProjection( - sprintf('cr_%s_debug_projection', $contentRepositoryId->value), - $this->getObject(Connection::class) - ); + if (!isset($this->secondFakeProjection)) { + $this->secondFakeProjection = new DebugEventProjection( + sprintf('cr_%s_debug_projection', self::$contentRepositoryId->value), + $this->getObject(Connection::class) + ); + } FakeProjectionFactory::setProjection( 'second', @@ -98,9 +106,9 @@ public function setUp(): void FakeNodeTypeManagerFactory::setConfiguration([]); FakeContentDimensionSourceFactory::setWithoutDimensions(); - $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($contentRepositoryId); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance(self::$contentRepositoryId); - $this->setupContentRepositoryDependencies($contentRepositoryId); + $this->setupContentRepositoryDependencies(self::$contentRepositoryId); } final protected function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php new file mode 100644 index 00000000000..de79f4f4db1 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php @@ -0,0 +1,36 @@ +getObject(EntityManagerInterface::class); + + if (!isset(self::$secondConnection)) { + self::$secondConnection = DriverManager::getConnection( + $entityManager->getConnection()->getParams(), + $entityManager->getConfiguration(), + $entityManager->getEventManager() + ); + } + + $this->secondFakeProjection = new DebugEventProjection( + sprintf('cr_%s_debug_projection', self::$contentRepositoryId->value), + self::$secondConnection + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 78d3054c6b1..909131e7898 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -13,7 +13,6 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\Result; -use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -23,12 +22,15 @@ final class ProjectionErrorTest extends AbstractSubscriptionEngineTestCase { + use ProjectionRollbackTestTrait; + /** @test */ public function projectionWithError() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->subscriptionEngine->setup(); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); @@ -84,8 +86,10 @@ public function fixFailedProjectionViaReset() $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->fakeProjection->expects(self::once())->method('resetState'); $this->fakeProjection->expects(self::exactly(2))->method('apply'); - $this->subscriptionEngine->setup(); - $this->subscriptionEngine->boot(); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); // commit an event $this->commitExampleContentStreamEvent(); @@ -221,133 +225,6 @@ public function irreparableProjection() ); } - /** @test */ - public function projectionIsRolledBackAfterError() - { - $this->eventStore->setup(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); - $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); - - // commit an event - $this->commitExampleContentStreamEvent(); - - $exception = new \RuntimeException('This projection is kaputt.'); - - $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); - - $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionStatus::ok(), - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionEngine->catchUpActive(); - self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // should be empty as we need an exact once delivery - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // - // fix projection and catchup - // - - $this->secondFakeProjection->killSaboteur(); - - // reactivate and catchup - $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - - /** @test */ - public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() - { - $this->eventStore->setup(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::exactly(2))->method('apply'); - $this->subscriptionEngine->setup(); - $this->subscriptionEngine->boot(); - - // commit two events - $this->commitExampleContentStreamEvent(); - $this->commitExampleContentStreamEvent(); - - $exception = new \RuntimeException('Event 2 is kaputt.'); - - // fail at the second event - $this->secondFakeProjection->injectSaboteur( - fn (EventEnvelope $eventEnvelope) => - $eventEnvelope->sequenceNumber->value === 2 - ? throw $exception - : null - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - - $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::fromInteger(1), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionStatus::ok(), - ); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // the first successful event is applied and committet: - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // - // fix projection and catchup - // - - $this->secondFakeProjection->killSaboteur(); - - // catchup after fix - $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); - self::assertNull($result->errors); - - // subscriptionError is reset, and the position is advanced if there were events - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); - self::assertEquals( - [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - /** @test */ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() { diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php new file mode 100644 index 00000000000..835173717f0 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php @@ -0,0 +1,144 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + // reactivate and catchup + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('Event 2 is kaputt.'); + + // fail at the second event + $this->secondFakeProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw $exception + : null + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // the first successful event is applied and committet: + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + // catchup after fix + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); + self::assertNull($result->errors); + + // subscriptionError is reset, and the position is advanced if there were events + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} From fd9faa45c0f12fe0d1d3057ff30584c4474502fe Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:14:42 +0100 Subject: [PATCH 115/142] TASK: Implement that `onAfterCatchUp` called _after_ everything is persisted Test provided by 54b24b8027f2588b575aceb462d40fce53bf8de4 This partially reverts moving `$this->subscriptionStore->transactional` outside again from eb0d792d38a3b9fec8628258f19bf8a3a42baed6, as we have to do a task afterwards still --- .../Subscription/CatchUpHookErrorTest.php | 2 +- .../Engine/SubscriptionEngine.php | 227 ++++++++---------- 2 files changed, 107 insertions(+), 122 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 84e051864be..92842259256 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -233,7 +233,7 @@ public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted_withPro } self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); - self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" had an error and also failed onAfterCatchUp: This catchup hook is kaputt.'); + self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onAfterCatchUp: This catchup hook is kaputt.'); $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index e54a6dbc112..902b4b223c5 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -80,46 +80,25 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { $criteria ??= SubscriptionEngineCriteria::noConstraints(); - return $this->processExclusively(fn () => $this->subscriptionStore->transactional( - function () use ($criteria, $progressCallback) { - $this->logger?->info('Subscription Engine: Start catching up subscriptions in state "BOOTING".'); - $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate( - SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::BOOTING) - ); - return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); - } - )); + return $this->processExclusively( + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::BOOTING]), $progressCallback) + ); } public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { $criteria ??= SubscriptionEngineCriteria::noConstraints(); - return $this->processExclusively(fn () => $this->subscriptionStore->transactional( - function () use ($criteria, $progressCallback) { - $this->logger?->info('Subscription Engine: Start catching up subscriptions in state "ACTIVE".'); - $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate( - SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatus::ACTIVE) - ); - return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); - } - )); + return $this->processExclusively( + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), $progressCallback) + ); } public function reactivate(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult { $criteria ??= SubscriptionEngineCriteria::noConstraints(); - return $this->processExclusively(fn () => $this->subscriptionStore->transactional( - function () use ($criteria, $progressCallback) { - $this->logger?->info('Subscription Engine: Start catching up subscriptions in state "ACTIVE".'); - $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate( - SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ - SubscriptionStatus::ERROR, - SubscriptionStatus::DETACHED, - ])) - ); - return $this->catchUpSubscriptions($subscriptionsToCatchup, $progressCallback); - } - )); + return $this->processExclusively( + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR, SubscriptionStatus::DETACHED]), $progressCallback) + ); } public function reset(SubscriptionEngineCriteria|null $criteria = null): Result @@ -294,95 +273,112 @@ private function resetSubscription(Subscription $subscription): ?Error return null; } - private function catchUpSubscriptions(Subscriptions $subscriptionsToCatchup, \Closure $progressClosure = null): ProcessedResult + private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatusFilter $status, \Closure $progressClosure = null): ProcessedResult { - foreach ($subscriptionsToCatchup as $subscription) { - if (!$this->subscribers->contain($subscription->id)) { - // mark detached subscriptions as we cannot handle them and exclude them from catchup - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::DETACHED, - position: $subscription->position, - subscriptionError: null, - ); - $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - } - } + $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in states %s.', join(',', $status->toStringArray()))); + $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = Subscriptions::none(); - if ($subscriptionsToCatchup->isEmpty()) { - $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); - return ProcessedResult::success(0); - } + $subscriptionCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $status); - foreach ($subscriptionsToCatchup as $subscription) { - try { - $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); - } catch (\Throwable $e) { - // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); - $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732374000, $e); + $result = $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressClosure, &$subscriptionsToInvokeBeforeAndAfterCatchUpHooks) { + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); + foreach ($subscriptionsToCatchup as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + } } - } - $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); - $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); - /** @var array $errors */ - $errors = []; - $numberOfProcessedEvents = 0; - /** @var array $highestSequenceNumberForSubscriber */ - $highestSequenceNumberForSubscriber = []; - - $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); - foreach ($eventStream as $eventEnvelope) { - $sequenceNumber = $eventEnvelope->sequenceNumber; - if ($numberOfProcessedEvents > 0) { - $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); + return ProcessedResult::success(0); } - if ($progressClosure !== null) { - $progressClosure($eventEnvelope); + + $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = $subscriptionsToCatchup; + foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { + try { + $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. + $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); + $this->logger?->critical($message); + throw new CatchUpFailed($message, 1732374000, $e); // todo throw here or in delegating hook? + } } - $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - foreach ($subscriptionsToCatchup as $subscription) { - if ($subscription->position->value >= $sequenceNumber->value) { - $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); - continue; + + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + + /** @var array $errors */ + $errors = []; + $numberOfProcessedEvents = 0; + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; + + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); } - $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); - if ($error !== null) { - // ERROR Case: - // 1.) for the leftover events we are not including this failed subscription for catchup - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // 2.) update the subscription error state on either its unchanged or new position (if some events worked) - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ERROR, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: SubscriptionError::fromPreviousStatusAndException( - $subscription->status, - $error->throwable - ), - ); - // 3.) invoke onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent we want to "shutdown" this catchup iteration event though we know it failed - // todo put the ERROR $subscriptionStatus into the after hook, so it can properly be reacted upon - try { - $this->subscribers->get($subscription->id)->onAfterCatchUp(); - } catch (\Throwable $e) { - // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" had an error and also failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); - $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732733740, $e); + if ($progressClosure !== null) { + $progressClosure($eventEnvelope); + } + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; + } + $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); + if ($error !== null) { + // ERROR Case: + // 1.) for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // 2.) update the subscription error state on either its unchanged or new position (if some events worked) + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $error->throwable + ), + ); + $errors[] = $error; + continue; } - $errors[] = $error; - continue; + // HAPPY Case: + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; } - // HAPPY Case: - $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; + $numberOfProcessedEvents++; } - $numberOfProcessedEvents++; - } - foreach ($subscriptionsToCatchup as $subscription) { + foreach ($subscriptionsToCatchup as $subscription) { + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ACTIVE, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); + if ($subscription->status !== SubscriptionStatus::ACTIVE) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); + } + } + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); + return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); + }); + + // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? + foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { try { $this->subscribers->get($subscription->id)->onAfterCatchUp(); } catch (\Throwable $e) { @@ -391,20 +387,9 @@ private function catchUpSubscriptions(Subscriptions $subscriptionsToCatchup, \Cl $this->logger?->critical($message); throw new CatchUpFailed($message, 1732374000, $e); } - // after catchup mark all subscriptions as active, so they are triggered automatically now. - // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ACTIVE, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: null, - ); - if ($subscription->status !== SubscriptionStatus::ACTIVE) { - $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); - } } - $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); - return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); + + return $result; } /** From 175ab4cefe37a6d6565801fe38856f8f6ce4fce8 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:18:03 +0100 Subject: [PATCH 116/142] Move savepoint creation back on the subscription store This reverts commit dc5ff1038b89ca656eb01a420e4763059b455f70. We discussed that we dont want to ensure exactly once delivery for external projections. Adding transaction logic via traits for the projections increases logic there. And while with alot of effort we could bring exactly once delivery to work for most cases php could still decide to die in the small timeframe where we commit the one transaction and then the second. So there is no gurantee except when using a dedicated store: https://github.com/neos/neos-development-collection/pull/5377 --- .../DoctrineDbalContentGraphProjection.php | 3 -- .../Projection/HypergraphProjection.php | 3 -- .../TestSuite/DebugEventProjection.php | 3 -- .../AbstractSubscriptionEngineTestCase.php | 1 - .../ProjectionTransactionTrait.php | 37 ------------------- .../Projection/ProjectionInterface.php | 10 ----- .../Engine/SubscriptionEngine.php | 9 ++++- .../Store/SubscriptionStoreInterface.php | 6 +++ .../Subscriber/ProjectionSubscriber.php | 8 ++-- .../DoctrineSubscriptionStore.php | 15 ++++++++ .../Projection/DocumentUriPathProjection.php | 3 -- .../ChangeProjection.php | 3 -- 12 files changed, 31 insertions(+), 70 deletions(-) delete mode 100644 Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 025f596f5cb..4e998fde99a 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -62,7 +62,6 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; @@ -81,8 +80,6 @@ */ final class DoctrineDbalContentGraphProjection implements ContentGraphProjectionInterface { - use ProjectionTransactionTrait; - use ContentStream; use NodeMove; use NodeRemoval; diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 69ab40e5ce3..5961442b9e6 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -40,7 +40,6 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; @@ -53,8 +52,6 @@ */ final class HypergraphProjection implements ContentGraphProjectionInterface { - use ProjectionTransactionTrait; - use ContentStreamForking; use NodeCreation; use SubtreeTagging; diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 959d00db7aa..7f71796e96d 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -12,7 +12,6 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; -use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; @@ -30,8 +29,6 @@ */ final class DebugEventProjection implements ProjectionInterface { - use ProjectionTransactionTrait; - private DebugEventProjectionState $state; private \Closure|null $saboteur = null; diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 2a3476bc987..13364c83e69 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -77,7 +77,6 @@ public function setUp(): void $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); - $this->fakeProjection->expects(self::any())->method('transactional')->willReturnCallback(fn ($fn) => $fn())->willReturnCallback(fn ($fn) => $fn()); FakeProjectionFactory::setProjection( 'default', diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php b/Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php deleted file mode 100644 index 2a6b9d6ebc2..00000000000 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/ProjectionTransactionTrait.php +++ /dev/null @@ -1,37 +0,0 @@ -dbal->isTransactionActive() === false) { - /** @phpstan-ignore argument.templateType */ - $this->dbal->transactional($closure); - return; - } - // technically we could leverage nested transactions from dbal, which effectively does the same. - // but that requires us to enable this globally first via setNestTransactionsWithSavepoints also making this explicit is more transparent: - $this->dbal->createSavepoint('PROJECTION'); - try { - $closure(); - } catch (\Throwable $e) { - // roll back the partially applied event on the projection - $this->dbal->rollbackSavepoint('PROJECTION'); - throw $e; - } - $this->dbal->releaseSavepoint('PROJECTION'); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 19090c17cc1..aff57389895 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -30,16 +30,6 @@ public function setUp(): void; */ public function status(): ProjectionStatus; - /** - * Must invoke the closure which will update the catchup hooks and {@see apply}. - * Additionally, to guarantee exactly once delivery and also to behave correct during exceptions (even fatal ones), - * a database transaction should be started, or if a transaction is already active on the same connection save points - * must be used and rolled back on error. - * - * @param-immediately-invoked-callable $closure - */ - public function transactional(\Closure $closure): void; - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 902b4b223c5..eb28d4970d1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -337,12 +337,16 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); continue; } + + $this->subscriptionStore->createSavepoint(); $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); if ($error !== null) { // ERROR Case: - // 1.) for the leftover events we are not including this failed subscription for catchup + // 1.) roll back the partially applied event on the subscriber + $this->subscriptionStore->rollbackSavepoint(); + // 2.) for the leftover events we are not including this failed subscription for catchup $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // 2.) update the subscription error state on either its unchanged or new position (if some events worked) + // 3.) update the subscription error state on either its unchanged or new position (if some events worked) $this->subscriptionStore->update( $subscription->id, status: SubscriptionStatus::ERROR, @@ -356,6 +360,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs continue; } // HAPPY Case: + $this->subscriptionStore->releaseSavepoint(); $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; } $numberOfProcessedEvents++; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index b7b0540415b..85f31072717 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -35,4 +35,10 @@ public function update( * @return T */ public function transactional(\Closure $closure): mixed; + + public function createSavepoint(): void; + + public function releaseSavepoint(): void; + + public function rollbackSavepoint(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php index ee19dbc548c..7e15a8fdcba 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php @@ -34,11 +34,9 @@ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void public function handle(EventInterface $event, EventEnvelope $eventEnvelope): void { - $this->projection->transactional(function () use ($event, $eventEnvelope) { - $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); - $this->projection->apply($event, $eventEnvelope); - $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); - }); + $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); + $this->projection->apply($event, $eventEnvelope); + $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); } public function onAfterCatchUp(): void diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index fe5f038fda4..9c7c9e92005 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -169,4 +169,19 @@ public function transactional(\Closure $closure): mixed { return $this->dbal->transactional($closure); } + + public function createSavepoint(): void + { + $this->dbal->createSavepoint('SUBSCRIBER'); + } + + public function releaseSavepoint(): void + { + $this->dbal->releaseSavepoint('SUBSCRIBER'); + } + + public function rollbackSavepoint(): void + { + $this->dbal->rollbackSavepoint('SUBSCRIBER'); + } } diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index 2c2dbe5101a..016814020bc 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -25,7 +25,6 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ProjectionInterface; @@ -41,8 +40,6 @@ */ final class DocumentUriPathProjection implements ProjectionInterface, WithMarkStaleInterface { - use ProjectionTransactionTrait; - public const COLUMN_TYPES_DOCUMENT_URIS = [ 'shortcutTarget' => Types::JSON, ]; diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index af0f3e894f7..ead06f7b588 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -39,7 +39,6 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; -use Neos\ContentRepository\Core\Infrastructure\ProjectionTransactionTrait; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; @@ -53,8 +52,6 @@ */ class ChangeProjection implements ProjectionInterface { - use ProjectionTransactionTrait; - /** * @var ChangeFinder|null Cache for the ChangeFinder returned by {@see getState()}, * so that always the same instance is returned From 582c5e6617ca63d7cea93bf43bce431b16c85744 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:35:44 +0100 Subject: [PATCH 117/142] TASK: Adjust tests that exactly once delivery is not possible for external projections see 175ab4cefe37a6d6565801fe38856f8f6ce4fce8 --- .../ExternalProjectionErrorTest.php | 64 +++++++- .../Subscription/ProjectionErrorTest.php | 139 ++++++++++++++++- .../ProjectionRollbackTestTrait.php | 144 ------------------ 3 files changed, 193 insertions(+), 154 deletions(-) delete mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php index de79f4f4db1..6de1555d27e 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php @@ -8,11 +8,21 @@ use Doctrine\DBAL\DriverManager; use Doctrine\ORM\EntityManagerInterface; use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionError; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; +use Neos\EventStore\Model\Event\SequenceNumber; +/** + * Note that this test only documents the current state, there were ideas to guarantee exactly once delivery for external projections + * but this would mean additional complexity and is technically never truly possible with two connections. + * The most likely path this might work fully is by the introduction of a decentralized subscription store concept, + * see https://github.com/neos/neos-development-collection/pull/5377, but this is out of scope for now until we find good reasons. + */ final class ExternalProjectionErrorTest extends AbstractSubscriptionEngineTestCase { - use ProjectionRollbackTestTrait; - static Connection $secondConnection; /** @before */ @@ -33,4 +43,54 @@ public function injectExternalFakeProjection(): void self::$secondConnection ); } + + public static function tearDownAfterClass(): void + { + self::$secondConnection->close(); + } + + /** @test */ + public function externalProjectionIsNotRolledBackAfterError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // not empty as the projection is commited directly + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 909131e7898..78d3054c6b1 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -13,6 +13,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\Result; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -22,15 +23,12 @@ final class ProjectionErrorTest extends AbstractSubscriptionEngineTestCase { - use ProjectionRollbackTestTrait; - /** @test */ public function projectionWithError() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); + $this->subscriptionEngine->setup(); $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $result = $this->subscriptionEngine->boot(); self::assertEquals(ProcessedResult::success(0), $result); @@ -86,10 +84,8 @@ public function fixFailedProjectionViaReset() $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); $this->fakeProjection->expects(self::once())->method('resetState'); $this->fakeProjection->expects(self::exactly(2))->method('apply'); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); - $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); // commit an event $this->commitExampleContentStreamEvent(); @@ -225,6 +221,133 @@ public function irreparableProjection() ); } + /** @test */ + public function projectionIsRolledBackAfterError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // should be empty as we need an exact once delivery + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + // reactivate and catchup + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('Event 2 is kaputt.'); + + // fail at the second event + $this->secondFakeProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw $exception + : null + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hasFailed()); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // the first successful event is applied and committet: + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // + // fix projection and catchup + // + + $this->secondFakeProjection->killSaboteur(); + + // catchup after fix + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); + self::assertNull($result->errors); + + // subscriptionError is reset, and the position is advanced if there were events + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + /** @test */ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() { diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php deleted file mode 100644 index 835173717f0..00000000000 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionRollbackTestTrait.php +++ /dev/null @@ -1,144 +0,0 @@ -eventStore->setup(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); - $result = $this->subscriptionEngine->setup(); - self::assertNull($result->errors); - $result = $this->subscriptionEngine->boot(); - self::assertNull($result->errors); - - // commit an event - $this->commitExampleContentStreamEvent(); - - $exception = new \RuntimeException('This projection is kaputt.'); - - $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); - - $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionStatus::ok(), - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionEngine->catchUpActive(); - self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // should be empty as we need an exact once delivery - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // - // fix projection and catchup - // - - $this->secondFakeProjection->killSaboteur(); - - // reactivate and catchup - $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } - - /** @test */ - public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() - { - $this->eventStore->setup(); - $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::exactly(2))->method('apply'); - $this->subscriptionEngine->setup(); - $this->subscriptionEngine->boot(); - - // commit two events - $this->commitExampleContentStreamEvent(); - $this->commitExampleContentStreamEvent(); - - $exception = new \RuntimeException('Event 2 is kaputt.'); - - // fail at the second event - $this->secondFakeProjection->injectSaboteur( - fn (EventEnvelope $eventEnvelope) => - $eventEnvelope->sequenceNumber->value === 2 - ? throw $exception - : null - ); - - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - $result = $this->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); - - $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::fromInteger(1), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionStatus::ok(), - ); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); - - // the first successful event is applied and committet: - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // - // fix projection and catchup - // - - $this->secondFakeProjection->killSaboteur(); - - // catchup after fix - $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); - self::assertNull($result->errors); - - // subscriptionError is reset, and the position is advanced if there were events - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); - self::assertEquals( - [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - } -} From bfb4655173ef7c437aa33f007a4ffd0160705e9b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:29:51 +0100 Subject: [PATCH 118/142] TASK: Change that hooks are not executed in the same savepoint and do not put the projection in ERROR state ... instead a full rollback is done (which might be still changed) - calls "internals" of `ProjectionSubscriber` directly from the subscription engine - inline `handleEvent` method --- .../AbstractSubscriptionEngineTestCase.php | 2 +- .../Subscription/CatchUpHookErrorTest.php | 71 ++++++++----------- .../CatchUpHook/CatchUpHookInterface.php | 16 +++-- .../Engine/SubscriptionEngine.php | 31 ++++---- .../Subscription/Exception/CatchUpFailed.php | 4 +- .../Subscriber/ProjectionSubscriber.php | 28 ++------ 6 files changed, 61 insertions(+), 91 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index 13364c83e69..cd3b5c8bb92 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -187,7 +187,7 @@ final protected function commitExampleContentStreamEvent(): void ); } - final protected function expectOkayStatus($subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void + final protected function expectOkayStatus(string $subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void { $actual = $this->subscriptionStatus($subscriptionId); self::assertEquals( diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 92842259256..0ad8b30ac33 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -24,38 +24,31 @@ public function error_onBeforeEvent_projectionIsNotRun() $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); - // commit an event + // commit two events, we expect neither to be catchupd correctly because handing on the first fails + $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - // Todo test that onBeforeEvent|onAfterEvent are in the same transaction and that a rollback will also revert their state $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); - - $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); - - $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionStatus::ok(), - ); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $result = $this->subscriptionEngine->catchUpActive(); - self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); + $actualException = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $actualException = $e; + } + self::assertSame($exception, $actualException); + self::assertSame('This catchup hook is kaputt.', $actualException->getMessage()); - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); // must be still empty because apply was never called self::assertEmpty( @@ -72,7 +65,8 @@ public function error_onAfterEvent_projectionIsRolledBack() $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); - // commit an event + // commit an events, we expect neither to be catchupd correctly because handing on the first fails + $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); @@ -80,30 +74,23 @@ public function error_onAfterEvent_projectionIsRolledBack() $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); - // TODO pass the error subscription status to onAfterCatchUp, so that in case of an error it can be prevented that mails f.x. will be sent? - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); - - $expectedFailure = ProjectionSubscriptionStatus::create( - subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), - subscriptionStatus: SubscriptionStatus::ERROR, - subscriptionPosition: SequenceNumber::none(), - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), - setupStatus: ProjectionStatus::ok(), - ); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $result = $this->subscriptionEngine->catchUpActive(); - self::assertSame($result->errors?->first()->message, 'This catchup hook is kaputt.'); - - self::assertEquals( - $expectedFailure, - $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') - ); + $actualException = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $actualException = $e; + } + self::assertSame($exception, $actualException); + self::assertSame('This catchup hook is kaputt.', $actualException->getMessage()); - // should be empty as we need an exact once delivery + // will be empty again because the full transaction was rolled back + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); @@ -172,6 +159,7 @@ public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted() $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent'); + // todo test that other catchup hooks are still run and all errors are collected! $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( new \RuntimeException('This catchup hook is kaputt.') ); @@ -214,13 +202,14 @@ public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted_withPro $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( - $innerException = new \RuntimeException('Inner event handling is kaputt.') - );; + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( new \RuntimeException('This catchup hook is kaputt.') ); + $innerException = new \RuntimeException('Projection is kaputt.'); + $this->secondFakeProjection->injectSaboteur(fn () => throw $innerException); + self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index cedadbc79ea..83e9e392c41 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -6,7 +6,6 @@ use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; @@ -22,9 +21,10 @@ interface CatchUpHookInterface { /** * This hook is called at the beginning of a catch-up run; - * AFTER the Database Lock is acquired, BEFORE any projection was opened. + * AFTER the Database Lock is acquired, BEFORE any projection was called. * - * Its important that no errors are thrown, as they will cause the catchup to directly halt with a {@see CatchUpFailed} exception. + * Note that any errors thrown will cause the catchup to directly halt, + * and no projections or their subscriber state are updated. */ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; @@ -32,7 +32,8 @@ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; * This hook is called for every event during the catchup process, **before** the projection * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. * - * Any errors will cause the transaction being rolled back, and the projection goes into {@see SubscriptionStatus::ERROR} state. + * Note that any errors thrown will cause the catchup to directly halt, + * and no projections or their subscriber state are updated, as the transaction is rolled back. */ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; @@ -40,7 +41,8 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even * This hook is called for every event during the catchup process, **after** the projection * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. * - * Any errors will cause the transaction being rolled back, and the projection goes into {@see SubscriptionStatus::ERROR} state. + * Note that any errors thrown will cause the catchup to directly halt, + * and no projections or their subscriber state are updated, as the transaction is rolled back. */ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; @@ -48,8 +50,8 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event * This hook is called at the END of a catch-up run * BEFORE the Database Lock is released, but AFTER the transaction is commited. * - * Its important that no errors are thrown, as they will cause the catchup to directly halt with a {@see CatchUpFailed} exception. - * The projections and their new position will already be persisted and there is no rollback. + * Note that any errors thrown will bubble up and do not implicate the projection. + * The projection and their new status and position will already be persisted without rollback. */ public function onAfterCatchUp(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index eb28d4970d1..cab0ec97b4b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -168,19 +168,6 @@ public function subscriptionStatus(SubscriptionEngineCriteria|null $criteria = n return SubscriptionStatusCollection::fromArray($statuses); } - private function handleEvent(EventEnvelope $eventEnvelope, EventInterface $domainEvent, SubscriptionId $subscriptionId): Error|null - { - $subscriber = $this->subscribers->get($subscriptionId); - try { - $subscriber->handle($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscriptionId->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); - return Error::fromSubscriptionIdAndException($subscriptionId, $e); - } - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscriptionId->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); - return null; - } - /** * Find all subscribers that don't have a corresponding subscription. * For each match a subscription is added @@ -304,7 +291,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = $subscriptionsToCatchup; foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { try { - $this->subscribers->get($subscription->id)->onBeforeCatchUp($subscription->status); + $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); @@ -337,11 +324,17 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); continue; } + $subscriber = $this->subscribers->get($subscription->id); + $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); $this->subscriptionStore->createSavepoint(); - $error = $this->handleEvent($eventEnvelope, $domainEvent, $subscription->id); - if ($error !== null) { + try { + $subscriber->projection->apply($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { // ERROR Case: + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + $error = Error::fromSubscriptionIdAndException($subscription->id, $e); + // 1.) roll back the partially applied event on the subscriber $this->subscriptionStore->rollbackSavepoint(); // 2.) for the leftover events we are not including this failed subscription for catchup @@ -360,8 +353,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs continue; } // HAPPY Case: + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); $this->subscriptionStore->releaseSavepoint(); $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; + + $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); } $numberOfProcessedEvents++; } @@ -383,9 +379,10 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs }); // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? + // note that a catchup error in onAfterEvent would bubble up directly and never invoke onAfterCatchUp foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { try { - $this->subscribers->get($subscription->id)->onAfterCatchUp(); + $this->subscribers->get($subscription->id)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. $message = sprintf('Subscriber "%s" failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php index 9e76c3db051..68cce1e2ed1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php @@ -5,7 +5,9 @@ namespace Neos\ContentRepository\Core\Subscription\Exception; /** - * Only thrown if there is no way to recover the started catchup. The transaction will be rolled back. + * Only thrown if there is no way to recover the started catchup. + * + * Todo move to delegating hook! * * @api */ diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php index 7e15a8fdcba..09cbd20a7a1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/ProjectionSubscriber.php @@ -4,43 +4,23 @@ namespace Neos\ContentRepository\Core\Subscription\Subscriber; -use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\Subscription\SubscriptionId; -use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; -use Neos\EventStore\Model\EventEnvelope; /** * @internal implementation detail of the catchup */ -final class ProjectionSubscriber +final readonly class ProjectionSubscriber { /** * @param ProjectionInterface $projection */ public function __construct( - public readonly SubscriptionId $id, - public readonly ProjectionInterface $projection, - private readonly ?CatchUpHookInterface $catchUpHook + public SubscriptionId $id, + public ProjectionInterface $projection, + public ?CatchUpHookInterface $catchUpHook ) { } - - public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void - { - $this->catchUpHook?->onBeforeCatchUp($subscriptionStatus); - } - - public function handle(EventInterface $event, EventEnvelope $eventEnvelope): void - { - $this->catchUpHook?->onBeforeEvent($event, $eventEnvelope); - $this->projection->apply($event, $eventEnvelope); - $this->catchUpHook?->onAfterEvent($event, $eventEnvelope); - } - - public function onAfterCatchUp(): void - { - $this->catchUpHook?->onAfterCatchUp(); - } } From aa2e7b13632fbe4f50cd75ce70a85990213eda45 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 3 Dec 2024 20:08:29 +0100 Subject: [PATCH 119/142] TASK: Prevent catchup hooks from halting the projections updates bfb4655173ef7c437aa33f007a4ffd0160705e9b to not do a full rollback if catchup hooks fail but continue and collect their errors, which will be rethrown later with the advantage that the catchup worked! --- .../Subscription/CatchUpHookErrorTest.php | 250 +++++++++++------- .../Subscription/ProjectionErrorTest.php | 4 +- .../CatchUpHook/CatchUpHookFailed.php | 36 +++ .../CatchUpHook/CatchUpHookInterface.php | 28 +- .../CatchUpHook/DelegatingCatchUpHook.php | 55 +++- .../Classes/Subscription/Engine/Error.php | 10 +- .../Engine/SubscriptionEngine.php | 47 ++-- .../Subscription/Exception/CatchUpFailed.php | 16 -- 8 files changed, 287 insertions(+), 159 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php delete mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 0ad8b30ac33..f0f481002c3 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -5,206 +5,261 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFailed; use Neos\ContentRepository\Core\Projection\ProjectionStatus; -use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; +use Neos\ContentRepository\Core\Subscription\Engine\Error; +use Neos\ContentRepository\Core\Subscription\Engine\Errors; +use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\EventStore\Model\EventEnvelope; final class CatchUpHookErrorTest extends AbstractSubscriptionEngineTestCase { /** @test */ - public function error_onBeforeEvent_projectionIsNotRun() + public function error_onBeforeEvent_isIgnoredAndCollected() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); - // commit two events, we expect neither to be catchupd correctly because handing on the first fails + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( - $exception = new \RuntimeException('This catchup hook is kaputt.') - ); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + $exception = new \RuntimeException('This catchup hook is kaputt.'); + $this->catchupHookForFakeProjection->expects($invokedCount = self::exactly(2))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { + match ($invokedCount->getInvocationCount()) { + 1 => self::assertSame(1, $eventEnvelope->sequenceNumber->value), + 2 => self::assertSame(2, $eventEnvelope->sequenceNumber->value), + }; + throw $exception; + }); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $actualException = null; - try { - $this->subscriptionEngine->catchUpActive(); - } catch (\Throwable $e) { - $actualException = $e; - } - self::assertSame($exception, $actualException); - self::assertSame('This catchup hook is kaputt.', $actualException->getMessage()); + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onBeforeEvent": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + // two errors for both of the events + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); - // must be still empty because apply was never called - self::assertEmpty( + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } /** @test */ - public function error_onAfterEvent_projectionIsRolledBack() + public function error_onAfterEvent_isIgnoredAndCollected() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); - // commit an events, we expect neither to be catchupd correctly because handing on the first fails + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( - $exception = new \RuntimeException('This catchup hook is kaputt.') - ); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $exception = new \RuntimeException('This catchup hook is kaputt.'); + $this->catchupHookForFakeProjection->expects($invokedCount = self::exactly(2))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { + match ($invokedCount->getInvocationCount()) { + 1 => self::assertSame(1, $eventEnvelope->sequenceNumber->value), + 2 => self::assertSame(2, $eventEnvelope->sequenceNumber->value), + }; + throw $exception; + }); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); // todo asset no parameters! self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $actualException = null; - try { - $this->subscriptionEngine->catchUpActive(); - } catch (\Throwable $e) { - $actualException = $e; - } - self::assertSame($exception, $actualException); - self::assertSame('This catchup hook is kaputt.', $actualException->getMessage()); + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onAfterEvent": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); - // will be empty again because the full transaction was rolled back - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - self::assertEmpty( + // two errors for both of the events + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } /** @test */ - public function error_onBeforeCatchUp_abortsCatchup() + public function error_onBeforeCatchUp_isIgnoredAndCollected() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::never())->method('apply'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - // commit an event + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willThrowException( - new \RuntimeException('This catchup hook is kaputt.') + $exception = new \RuntimeException('This catchup hook is kaputt.') ); - $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); - - $this->secondFakeProjection->injectSaboteur(fn () => self::fail('Projection apply is not expected to be called!')); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $expectedFailure = null; - try { - $this->subscriptionEngine->catchUpActive(); - } catch (\Throwable $e) { - $expectedFailure = $e; - } - self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); - - self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onBeforeCatchUp: This catchup hook is kaputt.'); + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onBeforeCatchUp": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); - // still the initial status - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); - // must be still empty because apply was never called - self::assertEmpty( + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } /** @test */ - public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted() + public function error_onAfterCatchUp_isIgnoredAndCollected() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - // commit an event + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); // todo test that other catchup hooks are still run and all errors are collected! $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( - new \RuntimeException('This catchup hook is kaputt.') + $exception = new \RuntimeException('This catchup hook is kaputt.') ); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $expectedFailure = null; - try { - $this->subscriptionEngine->catchUpActive(); - } catch (\Throwable $e) { - $expectedFailure = $e; - } - self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onAfterCatchUp": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); - self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onAfterCatchUp: This catchup hook is kaputt.'); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); - // one event is applied! - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); self::assertEquals( - [SequenceNumber::fromInteger(1)], + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } /** @test */ - public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted_withProjectionError() + public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); - $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); $this->subscriptionEngine->setup(); $this->subscriptionEngine->boot(); $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); - // commit an event + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent'); + // only the onBeforeEvent hook will be invoked as afterward the projection errored + $this->catchupHookForFakeProjection->expects(self::exactly(1))->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( - new \RuntimeException('This catchup hook is kaputt.') + $exception = new \RuntimeException('This catchup hook is kaputt.') ); $innerException = new \RuntimeException('Projection is kaputt.'); @@ -214,15 +269,26 @@ public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted_withPro $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $expectedFailure = null; - try { - $this->subscriptionEngine->catchUpActive(); - } catch (\Throwable $e) { - $expectedFailure = $e; - } - self::assertInstanceOf(CatchUpFailed::class, $expectedFailure); + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onAfterCatchUp": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); + + // two errors for both of the events + $result = $this->subscriptionEngine->catchUpActive(); - self::assertSame($expectedFailure->getMessage(), 'Subscriber "Vendor.Package:SecondFakeProjection" failed onAfterCatchUp: This catchup hook is kaputt.'); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $innerException), + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), @@ -232,7 +298,7 @@ public function error_onAfterCatchUp_crashesAfterProjectionsArePersisted_withPro setupStatus: ProjectionStatus::ok(), ); - // projection is still marked as error + // projection is marked as error self::assertEquals( $expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 78d3054c6b1..2b98d6289f2 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -51,7 +51,7 @@ public function projectionWithError() ); $result = $this->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::forSubscription(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); self::assertEquals( $expectedStatusForFailedProjection, @@ -183,7 +183,7 @@ public function irreparableProjection() $result = $this->subscriptionEngine->reactivate(); self::assertEquals( ProcessedResult::failed(1, Errors::fromArray([ - Error::fromSubscriptionIdAndException(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception) + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception) ])), $result ); diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php new file mode 100644 index 00000000000..ccdaa36a79d --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php @@ -0,0 +1,36 @@ + + * @api + */ +final class CatchUpHookFailed extends \RuntimeException implements \IteratorAggregate +{ + /** + * @internal + * @param array<\Throwable> $additionalExceptions + */ + public function __construct( + string $message, + int $code, + \Throwable $exception, + private readonly array $additionalExceptions + ) { + parent::__construct($message, $code, $exception); + } + + public function getIterator(): \Traversable + { + $previous = $this->getPrevious(); + if ($previous !== null) { + yield $previous; + } + yield from $this->additionalExceptions; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index 83e9e392c41..b95c9e26d7e 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -10,7 +10,7 @@ use Neos\EventStore\Model\EventEnvelope; /** - * This is an internal API with which you can hook into the catch-up process of a Projection. + * This is an api with which you can hook into the catch-up process of a projection. * * To register such a CatchUpHook, create a corresponding {@see CatchUpHookFactoryInterface} * and pass it to {@see ProjectionFactoryInterface::build()}. @@ -23,8 +23,10 @@ interface CatchUpHookInterface * This hook is called at the beginning of a catch-up run; * AFTER the Database Lock is acquired, BEFORE any projection was called. * - * Note that any errors thrown will cause the catchup to directly halt, - * and no projections or their subscriber state are updated. + * Note that any errors thrown will be ignored and the catchup will start as normal. + * The collect errors will be returned and rethrown by the content repository. + * + * @throws CatchUpHookFailed */ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; @@ -32,8 +34,10 @@ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; * This hook is called for every event during the catchup process, **before** the projection * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. * - * Note that any errors thrown will cause the catchup to directly halt, - * and no projections or their subscriber state are updated, as the transaction is rolled back. + * Note that any errors thrown will be ignored and the catchup will continue as normal. + * The collect errors will be returned and rethrown by the content repository. + * + * @throws CatchUpHookFailed */ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; @@ -41,8 +45,10 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even * This hook is called for every event during the catchup process, **after** the projection * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. * - * Note that any errors thrown will cause the catchup to directly halt, - * and no projections or their subscriber state are updated, as the transaction is rolled back. + * Note that any errors thrown will be ignored and the catchup will continue as normal. + * The collect errors will be returned and rethrown by the content repository. + * + * @throws CatchUpHookFailed */ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; @@ -50,8 +56,12 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event * This hook is called at the END of a catch-up run * BEFORE the Database Lock is released, but AFTER the transaction is commited. * - * Note that any errors thrown will bubble up and do not implicate the projection. - * The projection and their new status and position will already be persisted without rollback. + * The projection and their new status and position are already persisted. + * + * Note that any errors thrown will be ignored and the catchup will finish as normal. + * The collect errors will be returned and rethrown by the content repository. + * + * @throws CatchUpHookFailed */ public function onAfterCatchUp(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php index 12a2e73c9b8..a7e0d109e22 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php @@ -14,7 +14,7 @@ * * @internal */ -final class DelegatingCatchUpHook implements CatchUpHookInterface +final readonly class DelegatingCatchUpHook implements CatchUpHookInterface { /** * @var CatchUpHookInterface[] @@ -29,29 +29,62 @@ public function __construct( public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeCatchUp($subscriptionStatus); - } + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onBeforeCatchUp($subscriptionStatus), + 'onBeforeCatchUp' + ); } public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeEvent($eventInstance, $eventEnvelope); - } + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onBeforeEvent($eventInstance, $eventEnvelope), + 'onBeforeEvent' + ); } public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onAfterEvent($eventInstance, $eventEnvelope); - } + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onAfterEvent($eventInstance, $eventEnvelope), + 'onAfterEvent' + ); } public function onAfterCatchUp(): void { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onAfterCatchUp(), + 'onAfterCatchUp' + ); + } + + /** + * @param \Closure(CatchUpHookInterface): void $closure + * @return void + */ + private function delegateHooks(\Closure $closure, string $hookName): void + { + /** @var array<\Throwable> $errors */ + $errors = []; + $firstFailedCatchupHook = null; foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onAfterCatchUp(); + try { + $closure($catchUpHook); + } catch (\Throwable $e) { + $firstFailedCatchupHook ??= substr(strrchr($catchUpHook::class, '\\') ?: '', 1); + $errors[] = $e; + } + } + if ($errors !== []) { + $firstError = array_shift($errors); + $additionalMessage = $errors !== [] ? sprintf(' (and %d other)', count($errors)) : ''; + throw new CatchUpHookFailed( + sprintf('Hook "%s"%s failed "%s": %s', $firstFailedCatchupHook, $additionalMessage, $hookName, $firstError->getMessage()), + 1733243960, + $firstError, + $errors + ); } } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php index 546f82c5d3c..98ae41bd1d1 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php @@ -9,16 +9,16 @@ /** * @internal implementation detail of the catchup */ -final class Error +final readonly class Error { private function __construct( - public readonly SubscriptionId $subscriptionId, - public readonly string $message, - public readonly \Throwable $throwable, + public SubscriptionId $subscriptionId, + public string $message, + public \Throwable $throwable, ) { } - public static function fromSubscriptionIdAndException(SubscriptionId $subscriptionId, \Throwable $exception): self + public static function forSubscription(SubscriptionId $subscriptionId, \Throwable $exception): self { return new self( $subscriptionId, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index cab0ec97b4b..7cbddd43cbb 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -6,11 +6,9 @@ use Doctrine\DBAL\Exception\TableNotFoundException; use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; -use Neos\ContentRepository\Core\Subscription\Exception\CatchUpFailed; use Neos\ContentRepository\Core\Subscription\Exception\SubscriptionEngineAlreadyProcessingException; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; @@ -18,14 +16,12 @@ use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\SubscriptionError; -use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\Subscriptions; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\SequenceNumber; -use Neos\EventStore\Model\EventEnvelope; use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Log\LoggerInterface; @@ -223,7 +219,7 @@ private function setupSubscription(Subscription $subscription): ?Error $subscription->position, SubscriptionError::fromPreviousStatusAndException($subscription->status, $e) ); - return Error::fromSubscriptionIdAndException($subscription->id, $e); + return Error::forSubscription($subscription->id, $e); } if ($subscription->status === SubscriptionStatus::ACTIVE) { @@ -248,7 +244,7 @@ private function resetSubscription(Subscription $subscription): ?Error $subscriber->projection->resetState(); } catch (\Throwable $e) { $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); - return Error::fromSubscriptionIdAndException($subscription->id, $e); + return Error::forSubscription($subscription->id, $e); } $this->subscriptionStore->update( $subscription->id, @@ -267,7 +263,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $subscriptionCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $status); - $result = $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressClosure, &$subscriptionsToInvokeBeforeAndAfterCatchUpHooks) { + $numberOfProcessedEvents = 0; + /** @var array $errors */ + $errors = []; + + $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressClosure, &$subscriptionsToInvokeBeforeAndAfterCatchUpHooks, &$numberOfProcessedEvents, &$errors) { $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); foreach ($subscriptionsToCatchup as $subscription) { if (!$this->subscribers->contain($subscription->id)) { @@ -285,7 +285,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs if ($subscriptionsToCatchup->isEmpty()) { $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); - return ProcessedResult::success(0); + return; } $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = $subscriptionsToCatchup; @@ -293,19 +293,13 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { - // analog to onAfterCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" failed onBeforeCatchUp: %s', $subscription->id->value, $e->getMessage()); - $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732374000, $e); // todo throw here or in delegating hook? + $errors[] = Error::forSubscription($subscription->id, $e); } } $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); - /** @var array $errors */ - $errors = []; - $numberOfProcessedEvents = 0; /** @var array $highestSequenceNumberForSubscriber */ $highestSequenceNumberForSubscriber = []; @@ -326,14 +320,19 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } $subscriber = $this->subscribers->get($subscription->id); - $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); + try { + $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } + $this->subscriptionStore->createSavepoint(); try { $subscriber->projection->apply($domainEvent, $eventEnvelope); } catch (\Throwable $e) { // ERROR Case: $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); - $error = Error::fromSubscriptionIdAndException($subscription->id, $e); + $error = Error::forSubscription($subscription->id, $e); // 1.) roll back the partially applied event on the subscriber $this->subscriptionStore->rollbackSavepoint(); @@ -357,7 +356,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->subscriptionStore->releaseSavepoint(); $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; - $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); + try { + $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } } $numberOfProcessedEvents++; } @@ -375,7 +378,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } } $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); - return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); }); // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? @@ -384,14 +386,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscription->id)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { - // analog to onBeforeCatchUp, we tolerate no exceptions here and consider it a critical developer error. - $message = sprintf('Subscriber "%s" failed onAfterCatchUp: %s', $subscription->id->value, $e->getMessage()); - $this->logger?->critical($message); - throw new CatchUpFailed($message, 1732374000, $e); + $errors[] = Error::forSubscription($subscription->id, $e); } } - return $result; + return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); } /** diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php deleted file mode 100644 index 68cce1e2ed1..00000000000 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpFailed.php +++ /dev/null @@ -1,16 +0,0 @@ - Date: Tue, 3 Dec 2024 21:39:53 +0100 Subject: [PATCH 120/142] TASK: Introduce dedicated `CatchUpHadErrors` exception --- .../Subscription/ProjectionErrorTest.php | 11 +-- .../Classes/ContentRepository.php | 9 ++- .../Classes/Service/ContentStreamPruner.php | 2 +- .../Subscription/Engine/ProcessedResult.php | 20 +----- .../Exception/CatchUpHadErrors.php | 68 +++++++++++++++++++ 5 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 2b98d6289f2..bcf4be11ab6 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -14,6 +14,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\ProcessedResult; use Neos\ContentRepository\Core\Subscription\Engine\Result; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpHadErrors; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionError; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -107,7 +108,7 @@ public function fixFailedProjectionViaReset() ); $result = $this->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); + self::assertTrue($result->hadErrors()); self::assertEquals( $expectedFailure, @@ -161,7 +162,7 @@ public function irreparableProjection() // catchup active tries to apply the commited event $result = $this->subscriptionEngine->catchUpActive(); // but fails - self::assertTrue($result->hasFailed()); + self::assertTrue($result->hadErrors()); self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); // a second catchup active does not change anything @@ -207,7 +208,7 @@ public function irreparableProjection() // but booting will rethrow that error :D $result = $this->subscriptionEngine->boot(); - self::assertTrue($result->hasFailed()); + self::assertTrue($result->hadErrors()); self::assertEquals( ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), @@ -309,7 +310,7 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() ); $result = $this->subscriptionEngine->catchUpActive(); - self::assertTrue($result->hasFailed()); + self::assertTrue($result->hadErrors()); $expectedFailure = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), @@ -367,7 +368,7 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( } catch (\RuntimeException $exception) { $handleException = $exception; } - self::assertNotNull($handleException); + self::assertInstanceOf(CatchUpHadErrors::class, $exception); self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); self::assertSame($originalException, $handleException->getPrevious()); diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 460cc969898..6c5b4f239d4 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -41,6 +41,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpHadErrors; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Exception\ConcurrencyException; use Psr\Clock\ClockInterface; @@ -98,7 +99,9 @@ public function handle(CommandInterface $command): void $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); $catchUpResult = $this->subscriptionEngine->catchUpActive(); - $catchUpResult->throwOnFailure(); + if ($catchUpResult->hadErrors()) { + throw CatchUpHadErrors::createFromErrors($catchUpResult->errors); + } return; } @@ -128,7 +131,9 @@ public function handle(CommandInterface $command): void // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. $catchUpResult = $this->subscriptionEngine->catchUpActive(); - $catchUpResult->throwOnFailure(); + if ($catchUpResult->hadErrors()) { + throw CatchUpHadErrors::createFromErrors($catchUpResult->errors); + } } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 5f1991c0a77..50ca89b5a86 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -161,7 +161,7 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta if ($danglingContentStreamsPresent) { $result = $this->subscriptionEngine->catchUpActive(); - if ($result->hasFailed()) { + if ($result->hadErrors()) { $outputFn('Catchup after removing unused content streams led to errors. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.'); } } else { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php index e90e6975fa2..eabe5c5270a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -26,26 +26,8 @@ public static function failed(int $numberOfProcessedEvents, Errors $errors): sel } /** @phpstan-assert-if-true !null $this->errors */ - public function hasFailed(): bool + public function hadErrors(): bool { return $this->errors !== null; } - - public function throwOnFailure(): void - { - /** @var Error[] $errors */ - $errors = iterator_to_array($this->errors ?? []); - if ($errors === []) { - return; - } - $firstError = array_shift($errors); - - $additionalFailedSubscribers = array_map(fn (Error $error) => $error->subscriptionId->value, $errors); - - $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' | And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); - $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); - - // todo custom exception! - throw new \RuntimeException($exceptionMessage, 1732132930, $firstError->throwable); - } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php new file mode 100644 index 00000000000..7545c74706c --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php @@ -0,0 +1,68 @@ + + * @api + */ +final class CatchUpHadErrors extends \RuntimeException implements \IteratorAggregate +{ + /** + * @internal + * @param array<\Throwable> $additionalExceptions + */ + private function __construct( + string $message, + int $code, + \Throwable $exception, + private readonly array $additionalExceptions + ) { + parent::__construct($message, $code, $exception); + } + + /** + * @internal + */ + public static function createFromErrors(Errors $errors): self + { + /** @var non-empty-array $errors */ + $errors = iterator_to_array($errors); + $firstError = array_shift($errors); + + $additionalFailedSubscribers = array_map(fn (Error $error) => $error->subscriptionId->value, $errors); + + $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' | And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); + $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); + + throw new self($exceptionMessage, 1732132930, $firstError->throwable, array_map(fn (Error $error) => $error->throwable, $errors)); + } + + public function getIterator(): \Traversable + { + $previous = $this->getPrevious(); + if ($previous !== null) { + yield $previous; + } + yield from $this->additionalExceptions; + } +} From 4c41482f8e07206d86d917b2a472f27b0449da4c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:08:23 +0100 Subject: [PATCH 121/142] TASK: Introduce test to assert behaviour when catchup hooks use the persistence manager --- .../CatchUpHookWithPersistenceTest.php | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php new file mode 100644 index 00000000000..22019865219 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php @@ -0,0 +1,120 @@ +truncateAndSetupFlowEntities(); + } + + /** @test */ + public function commitOnConnection_onAfterEvent() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events. but only the first will never be seen + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () { + $this->getObject(Connection::class)->commit(); + }); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $actualException = null; + try { + $this->subscriptionEngine->catchUpActive(); + } catch (\Throwable $e) { + $actualException = $e; + } + self::assertInstanceOf(\Doctrine\DBAL\ConnectionException::class, $actualException); + + // todo invalid state, use own connection for cr?! + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function persistAll_onAfterEvent_willUseTheTransaction() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit one event + $this->commitExampleContentStreamEvent(); + + $persistentResource = new PersistentResource(); + $persistentResource->disableLifecycleEvents(); + $persistentResource->setFilename($expectedName = 'test_cr_catchup.empty'); + $persistentResource->setFileSize(0); + $persistentResource->setCollectionName('default'); + $persistentResource->setMediaType('text/plain'); + $persistentResource->setSha1($sha1 = '67f22467d829a254d53fa5cf019787c23c57bbef'); + + self::assertTrue($this->getObject(PersistenceManagerInterface::class)->isNewObject($persistentResource)); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () use ($persistentResource) { + $this->getObject(ResourceRepository::class)->add($persistentResource); + $this->getObject(PersistenceManagerInterface::class)->persistAll(); + }); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // check that the object was persisted and re-fetch it from the database + self::assertFalse($this->getObject(PersistenceManagerInterface::class)->isNewObject($persistentResource)); + $this->getObject(PersistenceManagerInterface::class)->clearState(); + + $actuallyPersisted = $this->getObject(ResourceRepository::class)->findOneBySha1($sha1); + + self::assertEquals($sha1, $actuallyPersisted->getSha1()); + self::assertEquals($expectedName, $actuallyPersisted->getFilename()); + } +} From 21318b28d4cd2ba930b903b0f7c6576849bf7bd4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:10:39 +0100 Subject: [PATCH 122/142] TASK: Explain behaviour when handling multiple commands --- .../Classes/CommandHandler/Commands.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php b/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php index c76eb1955e7..85cee65f834 100644 --- a/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php +++ b/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php @@ -4,6 +4,8 @@ namespace Neos\ContentRepository\Core\CommandHandler; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpHadErrors; + /** * @api can be used as collection of commands to be individually handled: * @@ -11,6 +13,9 @@ * $contentRepository->handle($command); * } * + * Note that as they are separate commands, they might individually fail due to constraints + * or a projection or catchup failing during the first catchup with {@see CatchUpHadErrors} + * * @implements \IteratorAggregate */ final readonly class Commands implements \IteratorAggregate, \Countable From 47fc20b6487c8038f41f66d28f1955b98e1b5e72 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 7 Dec 2024 20:23:13 +0100 Subject: [PATCH 123/142] FEATURE: Introduce batching in subscription engine --- .../TestSuite/DebugEventProjectionState.php | 13 +- .../Subscription/CatchUpHookErrorTest.php | 53 ++++ .../Subscription/CatchUpHookTest.php | 120 +++++++++ .../Subscription/SubscriptionBatchingTest.php | 64 +++++ .../Classes/ContentRepository.php | 14 +- .../Service/ContentRepositoryMaintainer.php | 8 +- .../Engine/SubscriptionEngine.php | 239 ++++++++++-------- 7 files changed, 394 insertions(+), 117 deletions(-) create mode 100644 Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php index 02dde323ad5..da23036635c 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php @@ -26,7 +26,18 @@ public function __construct( public function findAppliedSequenceNumbers(): iterable { return array_map( - fn ($value) => SequenceNumber::fromInteger((int)$value['sequenceNumber']), + fn (int $value) => SequenceNumber::fromInteger($value), + $this->findAppliedSequenceNumberValues() + ); + } + + /** + * @return iterable + */ + public function findAppliedSequenceNumberValues(): iterable + { + return array_map( + fn ($value) => (int)$value['sequenceNumber'], $this->dbal->fetchAllAssociative("SELECT sequenceNumber from {$this->tableNamePrefix}") ); } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index f0f481002c3..851428d14ea 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -307,4 +307,57 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } + + /** @test */ + public function error_onAfterEvent_stopsEngineAfterFirstBatch() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + + // commit two events. we expect that the hook will throw the first event and due to the batching its halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $exception = new \RuntimeException('This catchup hook is kaputt.'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $exception + ); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onAfterEvent": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); + + // one error + $result = $this->subscriptionEngine->boot(batchSize: 1); + self::assertEquals( + ProcessedResult::failed( + 1, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); + + // only one event is applied + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php index 56b9d632597..34508353906 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -7,6 +7,7 @@ use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\EventStore\Model\EventEnvelope; final class CatchUpHookTest extends AbstractSubscriptionEngineTestCase { @@ -47,4 +48,123 @@ public function catchUpHooksAreExecutedAndCanAccessTheCorrectProjectionsState() $expectOneHandledEvent(); } + + /** @test */ + public function catchUpBeforeAndAfterCatchupAreRunForZeroEvents() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(0)); + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + } + + /** @test */ + public function catchUpBeforeAndAfterCatchupAreNotRunIfNoSubscriberMatches() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + self::assertEquals(0, $result->numberOfProcessedEvents); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + } + + public function provideValidBatchSizes(): iterable + { + yield 'none' => [null]; + yield 'one' => [1]; + yield 'two' => [2]; + yield 'four' => [4]; + yield 'ten' => [10]; + } + + /** + * @dataProvider provideValidBatchSizes + * @test + */ + public function catchUpHooksWithBatching(int|null $batchSize) + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(4))->method('apply'); + $this->subscriptionEngine->setup(); + + // commit events (will be batched in chunks of two) + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects($i = self::exactly(4))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + self::assertEquals([], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 2 => [ + self::assertEquals(2, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 3 => [ + self::assertEquals(3, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 4 => [ + self::assertEquals(4, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + }; + }); + $this->catchupHookForFakeProjection->expects($i = self::exactly(4))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 2 => [ + self::assertEquals(2, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 3 => [ + self::assertEquals(3, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 4 => [ + self::assertEquals(4, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3,4], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + }; + }); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + + $result = $this->subscriptionEngine->boot(batchSize: $batchSize); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + self::assertEquals([1,2,3,4], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php new file mode 100644 index 00000000000..960a755cc6d --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php @@ -0,0 +1,64 @@ +eventStore->setup(); + // commit three events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(batchSize: 1); + self::assertEquals(ProcessedResult::success(2), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function invalidBatchSizes() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + + $e = null; + try { + $this->subscriptionEngine->boot(batchSize: 0); + } catch (\Throwable $e) { + } + self::assertInstanceOf(\InvalidArgumentException::class, $e); + self::assertEquals(1733597950, $e->getCode()); + + try { + $this->subscriptionEngine->catchUpActive(batchSize: -1); + } catch (\Throwable $e) { + } + + self::assertInstanceOf(\InvalidArgumentException::class, $e); + self::assertEquals(1733597950, $e->getCode()); + } +} diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 6c5b4f239d4..f308279ac42 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -98,9 +98,9 @@ public function handle(CommandInterface $command): void if ($toPublish instanceof EventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); - $catchUpResult = $this->subscriptionEngine->catchUpActive(); - if ($catchUpResult->hadErrors()) { - throw CatchUpHadErrors::createFromErrors($catchUpResult->errors); + $fullCatchUpResult = $this->subscriptionEngine->catchUpActive(); // NOTE: we don't batch here, to ensure the catchup is run completely and any errors don't stop it. + if ($fullCatchUpResult->hadErrors()) { + throw CatchUpHadErrors::createFromErrors($fullCatchUpResult->errors); } return; } @@ -115,7 +115,7 @@ public function handle(CommandInterface $command): void // we pass the exception into the generator (->throw), so it could be try-caught and reacted upon: // // try { - // yield EventsToPublish(...); + // yield new EventsToPublish(...); // } catch (ConcurrencyException $e) { // yield $this->reopenContentStream(); // throw $e; @@ -130,9 +130,9 @@ public function handle(CommandInterface $command): void } finally { // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. - $catchUpResult = $this->subscriptionEngine->catchUpActive(); - if ($catchUpResult->hadErrors()) { - throw CatchUpHadErrors::createFromErrors($catchUpResult->errors); + $fullCatchUpResult = $this->subscriptionEngine->catchUpActive(); // NOTE: we don't batch here, to ensure the catchup is run completely and any errors don't stop it. + if ($fullCatchUpResult->hadErrors()) { + throw CatchUpHadErrors::createFromErrors($fullCatchUpResult->errors); } } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 9b879a814cb..8337dae565b 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -70,6 +70,8 @@ */ final readonly class ContentRepositoryMaintainer implements ContentRepositoryServiceInterface { + private const REPLAY_BATCH_SIZE = 500; + /** * @internal please use the {@see ContentRepositoryMaintainerFactory} instead! */ @@ -127,7 +129,7 @@ public function replaySubscription(SubscriptionId $subscriptionId, \Closure|null if ($resetResult->errors !== null) { return self::createErrorForReason('Reset failed:', $resetResult->errors); } - $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), $progressCallback); + $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback, batchSize: self::REPLAY_BATCH_SIZE); if ($bootResult->errors !== null) { return self::createErrorForReason('Catchup failed:', $bootResult->errors); } @@ -140,7 +142,7 @@ public function replayAllSubscriptions(\Closure|null $progressCallback = null): if ($resetResult->errors !== null) { return self::createErrorForReason('Reset failed:', $resetResult->errors); } - $bootResult = $this->subscriptionEngine->boot(progressCallback: $progressCallback); + $bootResult = $this->subscriptionEngine->boot(progressCallback: $progressCallback, batchSize: self::REPLAY_BATCH_SIZE); if ($bootResult->errors !== null) { return self::createErrorForReason('Catchup failed:', $bootResult->errors); } @@ -163,7 +165,7 @@ public function reactivateSubscription(SubscriptionId $subscriptionId, \Closure| if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { return new Error(sprintf('Subscription "%s" is not setup and cannot be reactivated.', $subscriptionId->value)); } - $reactivateResult = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([$subscriptionId]), $progressCallback); + $reactivateResult = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback, batchSize: self::REPLAY_BATCH_SIZE); if ($reactivateResult->errors !== null) { return self::createErrorForReason('Could not reactivate subscriber:', $reactivateResult->errors); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 7cbddd43cbb..c146f8ecd33 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -73,27 +73,27 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); } - public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult + public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null, int $batchSize = null): ProcessedResult { $criteria ??= SubscriptionEngineCriteria::noConstraints(); return $this->processExclusively( - fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::BOOTING]), $progressCallback) + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::BOOTING]), $progressCallback, $batchSize) ); } - public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult + public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null, int $batchSize = null): ProcessedResult { $criteria ??= SubscriptionEngineCriteria::noConstraints(); return $this->processExclusively( - fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), $progressCallback) + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), $progressCallback, $batchSize) ); } - public function reactivate(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null): ProcessedResult + public function reactivate(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null, int $batchSize = null): ProcessedResult { $criteria ??= SubscriptionEngineCriteria::noConstraints(); return $this->processExclusively( - fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR, SubscriptionStatus::DETACHED]), $progressCallback) + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ERROR, SubscriptionStatus::DETACHED]), $progressCallback, $batchSize) ); } @@ -256,8 +256,16 @@ private function resetSubscription(Subscription $subscription): ?Error return null; } - private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatusFilter $status, \Closure $progressClosure = null): ProcessedResult + /** + * @param \Closure|null $progressCallback The callback that is invoked for every {@see EventEnvelope} that is processed per subscriber + * @param int|null $batchSize Number of events to process before the transaction is commited and reopened. (defaults to all events). + */ + private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatusFilter $status, \Closure|null $progressCallback, int|null $batchSize): ProcessedResult { + if ($batchSize !== null && $batchSize <= 0) { + throw new \InvalidArgumentException(sprintf('Invalid batchSize %d specified, must be either NULL or a positive integer.', $batchSize), 1733597950); + } + $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in states %s.', join(',', $status->toStringArray()))); $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = Subscriptions::none(); @@ -267,118 +275,137 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs /** @var array $errors */ $errors = []; - $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressClosure, &$subscriptionsToInvokeBeforeAndAfterCatchUpHooks, &$numberOfProcessedEvents, &$errors) { - $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); - foreach ($subscriptionsToCatchup as $subscription) { - if (!$this->subscribers->contain($subscription->id)) { - // mark detached subscriptions as we cannot handle them and exclude them from catchup - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::DETACHED, - position: $subscription->position, - subscriptionError: null, - ); - $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - } - } + do { + /** + * If batching is enabled, the {@see $continueBatching} flag will indicate that the last run was stopped and continuation is necessary to handle the rest of the events. + * It's possible that batching stops at the last event, in that case the transaction is still reopened to set the active state correctly. + */ + $continueBatching = $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressCallback, $batchSize, &$subscriptionsToInvokeBeforeAndAfterCatchUpHooks, &$numberOfProcessedEvents, &$errors) { + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); + if ($numberOfProcessedEvents === 0) { + // first batch + foreach ($subscriptionsToCatchup as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + } + } - if ($subscriptionsToCatchup->isEmpty()) { - $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); - return; - } + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); + return false; + } - $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = $subscriptionsToCatchup; - foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { - try { - $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); - } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = $subscriptionsToCatchup; + foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { + try { + $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } + } } - } - $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); - $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); - /** @var array $highestSequenceNumberForSubscriber */ - $highestSequenceNumberForSubscriber = []; + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; - $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); - foreach ($eventStream as $eventEnvelope) { - $sequenceNumber = $eventEnvelope->sequenceNumber; - if ($numberOfProcessedEvents > 0) { - $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); - } - if ($progressClosure !== null) { - $progressClosure($eventEnvelope); - } - $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - foreach ($subscriptionsToCatchup as $subscription) { - if ($subscription->position->value >= $sequenceNumber->value) { - $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); - continue; + $continueBatching = false; + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); } - $subscriber = $this->subscribers->get($subscription->id); - - try { - $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + if ($progressCallback !== null) { + $progressCallback($eventEnvelope); } - - $this->subscriptionStore->createSavepoint(); - try { - $subscriber->projection->apply($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - // ERROR Case: - $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); - $error = Error::forSubscription($subscription->id, $e); - - // 1.) roll back the partially applied event on the subscriber - $this->subscriptionStore->rollbackSavepoint(); - // 2.) for the leftover events we are not including this failed subscription for catchup - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // 3.) update the subscription error state on either its unchanged or new position (if some events worked) - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ERROR, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: SubscriptionError::fromPreviousStatusAndException( - $subscription->status, - $error->throwable - ), - ); - $errors[] = $error; - continue; + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; + } + $subscriber = $this->subscribers->get($subscription->id); + + try { + $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } + + $this->subscriptionStore->createSavepoint(); + try { + $subscriber->projection->apply($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + // ERROR Case: + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + $error = Error::forSubscription($subscription->id, $e); + + // 1.) roll back the partially applied event on the subscriber + $this->subscriptionStore->rollbackSavepoint(); + // 2.) for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // 3.) update the subscription error state on either its unchanged or new position (if some events worked) + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $error->throwable + ), + ); + $errors[] = $error; + continue; + } + // HAPPY Case: + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + $this->subscriptionStore->releaseSavepoint(); + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; + + try { + $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } } - // HAPPY Case: - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); - $this->subscriptionStore->releaseSavepoint(); - $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; - - try { - $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $numberOfProcessedEvents++; + if ($batchSize !== null && $numberOfProcessedEvents % $batchSize === 0) { + $continueBatching = true; + $this->logger?->info(sprintf('Subscription Engine: Batch completed with %d events', $numberOfProcessedEvents)); + break; } } - $numberOfProcessedEvents++; - } - foreach ($subscriptionsToCatchup as $subscription) { - // after catchup mark all subscriptions as active, so they are triggered automatically now. - // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ACTIVE, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: null, - ); - if ($subscription->status !== SubscriptionStatus::ACTIVE) { - $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting.', $subscription->id->value)); + foreach ($subscriptionsToCatchup as $subscription) { + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: $continueBatching === false ? SubscriptionStatus::ACTIVE : $subscription->status, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); + if ($continueBatching === false && $subscription->status !== SubscriptionStatus::ACTIVE) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting', $subscription->id->value)); + } } + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); + return $continueBatching; + }); + if ($errors !== []) { + break; } - $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); - }); + } while ($continueBatching === true); // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? // note that a catchup error in onAfterEvent would bubble up directly and never invoke onAfterCatchUp From 6b59fdbfe0d324b035935c1d2d051e223cd95ae3 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sat, 7 Dec 2024 21:04:13 +0100 Subject: [PATCH 124/142] FEATURE: Introduce `onAfterBatchCompleted` hook Previously the `CatchUpHookInterface` contained a `onBeforeBatchCompleted` which was explained as follows: This hook is called directly before the database lock is RELEASED It can happen that this method is called multiple times, even without having seen Events in the meantime. If there exist more events which need to be processed, the database lock is directly acquired again after it is released. ---------- `onAfterBatchCompleted` behaves similar just that it runs when the lock was released. This allows us to run all the tasks that `onAfterCatchUp` could also do (using possibly transactions and commiting work) We dont ensure that `onAfterBatchCompleted` is only called if there are more events to handle, as this would complicate the code and technically without acquiring a lock there is never a guarantee for that, so hooks might as well encounter the case more often and have to deal with that. --- .../RaceTrackerCatchUpHook.php | 4 ++ .../Subscription/CatchUpHookErrorTest.php | 61 ++++++++++++++++++- .../Subscription/CatchUpHookTest.php | 51 ++++++++++++++-- .../Subscription/ProjectionErrorTest.php | 35 +++++++++++ .../CatchUpHook/CatchUpHookInterface.php | 11 ++++ .../CatchUpHook/DelegatingCatchUpHook.php | 8 +++ .../Engine/SubscriptionEngine.php | 17 ++++-- .../FlushSubgraphCachePoolCatchUpHook.php | 4 ++ .../CatchUpHook/AssetUsageCatchUpHook.php | 4 ++ .../CatchUpHook/RouterCacheHook.php | 5 ++ ...phProjectorCatchUpHookForCacheFlushing.php | 4 ++ 11 files changed, 192 insertions(+), 12 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php index 9f809c0b9ac..d491a718fff 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php @@ -127,6 +127,10 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event { } + public function onAfterBatchCompleted(): void + { + } + public function onAfterCatchUp(): void { // we only want to track relevant lock release calls (i.e. if we were in the event processing loop before) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 851428d14ea..755ea0b05c5 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -43,6 +43,7 @@ public function error_onBeforeEvent_isIgnoredAndCollected() throw $exception; }); $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( @@ -100,7 +101,8 @@ public function error_onAfterEvent_isIgnoredAndCollected() }; throw $exception; }); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); // todo asset no parameters! + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); // todo assert no parameters?! self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() @@ -154,6 +156,7 @@ public function error_onBeforeCatchUp_isIgnoredAndCollected() ); $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( @@ -186,6 +189,59 @@ public function error_onBeforeCatchUp_isIgnoredAndCollected() ); } + /** @test */ + public function error_onAfterBatchCompleted_isIgnoredAndCollected() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "" failed "onAfterBatchCompleted": This catchup hook is kaputt.', + 1733243960, + $exception, + [] + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + /** @test */ public function error_onAfterCatchUp_isIgnoredAndCollected() { @@ -204,6 +260,7 @@ public function error_onAfterCatchUp_isIgnoredAndCollected() $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); // todo test that other catchup hooks are still run and all errors are collected! $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') @@ -258,6 +315,7 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() // only the onBeforeEvent hook will be invoked as afterward the projection errored $this->catchupHookForFakeProjection->expects(self::exactly(1))->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); @@ -326,6 +384,7 @@ public function error_onAfterEvent_stopsEngineAfterFirstBatch() $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( $exception ); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php index 34508353906..0129ca9cba2 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -37,6 +37,7 @@ public function catchUpHooksAreExecutedAndCanAccessTheCorrectProjectionsState() $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectOneHandledEvent); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); $expectNoHandledEvents(); @@ -60,6 +61,7 @@ public function catchUpBeforeAndAfterCatchupAreRunForZeroEvents() $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); $result = $this->subscriptionEngine->boot(); @@ -80,6 +82,7 @@ public function catchUpBeforeAndAfterCatchupAreNotRunIfNoSubscriberMatches() $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeCatchUp'); $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); $result = $this->subscriptionEngine->catchUpActive(); @@ -92,18 +95,51 @@ public function catchUpBeforeAndAfterCatchupAreNotRunIfNoSubscriberMatches() public function provideValidBatchSizes(): iterable { - yield 'none' => [null]; - yield 'one' => [1]; - yield 'two' => [2]; - yield 'four' => [4]; - yield 'ten' => [10]; + yield 'none' => [ + 'batchSize' => null, + 'onAfterBatchCompletedInvocations' => [ + [1,2,3,4] + ], + ]; + yield 'one' => [ + 'batchSize' => 1, + 'onAfterBatchCompletedInvocations' => [ + [1], + [1,2], + [1,2,3], + [1,2,3,4], + [1,2,3,4], + ], + ]; + yield 'two' => [ + 'batchSize' => 2, + 'onAfterBatchCompletedInvocations' => [ + [1,2], + [1,2,3,4], + [1,2,3,4], + ], + ]; + yield 'four' => [ + 'batchSize' => 4, + // we have two calls as the batch size exactly matches the events and we are running again to see if we handled everything. + 'onAfterBatchCompletedInvocations' => [ + [1,2,3,4], + [1,2,3,4], + ], + ]; + yield 'ten' => [ + 'batchSize' => 10, + 'onAfterBatchCompletedInvocations' => [ + [1,2,3,4], + ], + ]; } /** * @dataProvider provideValidBatchSizes * @test */ - public function catchUpHooksWithBatching(int|null $batchSize) + public function catchUpHooksWithBatching(int|null $batchSize, array $onAfterBatchCompletedInvocations) { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -157,6 +193,9 @@ public function catchUpHooksWithBatching(int|null $batchSize) ], }; }); + $this->catchupHookForFakeProjection->expects($i = self::exactly(\count($onAfterBatchCompletedInvocations)))->method('onAfterBatchCompleted')->willReturnCallback(function () use ($i, $onAfterBatchCompletedInvocations) { + self::assertEquals($onAfterBatchCompletedInvocations[$i->getInvocationCount() - 1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + }); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index bcf4be11ab6..1b1dd3bdfe4 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -393,4 +393,39 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } + + /** @test */ + public function projectionError_stopsEngineAfterFirstBatch() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This projection is kaputt.') + ); + $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::BOOTING, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + $result = $this->subscriptionEngine->boot(batchSize: 1); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::forSubscription(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + } } diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index b95c9e26d7e..6d71b122ade 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -52,6 +52,17 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even */ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void; + /** + * This hook is called for each batch of processed events during the catchup process, **after** the projection + * is updated and the transaction is commited. + * + * It can happen that this method is called even without having seen Events in the meantime. + * + * If there exist more events which need to be processed, the database lock + * is directly acquired again after it is released. + */ + public function onAfterBatchCompleted(): void; + /** * This hook is called at the END of a catch-up run * BEFORE the Database Lock is released, but AFTER the transaction is commited. diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php index a7e0d109e22..af24811e9b4 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php @@ -51,6 +51,14 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event ); } + public function onAfterBatchCompleted(): void + { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onAfterBatchCompleted(), + 'onAfterBatchCompleted' + ); + } + public function onAfterCatchUp(): void { $this->delegateHooks( diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index c146f8ecd33..32dfed4199a 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -267,7 +267,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in states %s.', join(',', $status->toStringArray()))); - $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = Subscriptions::none(); + $subscriptionsToInvokeAroundCatchUpHooks = Subscriptions::none(); $subscriptionCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $status); @@ -280,7 +280,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs * If batching is enabled, the {@see $continueBatching} flag will indicate that the last run was stopped and continuation is necessary to handle the rest of the events. * It's possible that batching stops at the last event, in that case the transaction is still reopened to set the active state correctly. */ - $continueBatching = $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressCallback, $batchSize, &$subscriptionsToInvokeBeforeAndAfterCatchUpHooks, &$numberOfProcessedEvents, &$errors) { + $continueBatching = $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressCallback, $batchSize, &$subscriptionsToInvokeAroundCatchUpHooks, &$numberOfProcessedEvents, &$errors) { $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); if ($numberOfProcessedEvents === 0) { // first batch @@ -303,8 +303,8 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs return false; } - $subscriptionsToInvokeBeforeAndAfterCatchUpHooks = $subscriptionsToCatchup; - foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { + $subscriptionsToInvokeAroundCatchUpHooks = $subscriptionsToCatchup; + foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { try { $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { @@ -402,6 +402,13 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); return $continueBatching; }); + foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { + try { + $this->subscribers->get($subscription->id)->catchUpHook?->onAfterBatchCompleted(); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } + } if ($errors !== []) { break; } @@ -409,7 +416,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? // note that a catchup error in onAfterEvent would bubble up directly and never invoke onAfterCatchUp - foreach ($subscriptionsToInvokeBeforeAndAfterCatchUpHooks as $subscription) { + foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { try { $this->subscribers->get($subscription->id)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { diff --git a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php index 3ff7ca35222..3644ecfa414 100644 --- a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php +++ b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php @@ -37,6 +37,10 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event $this->subgraphCachePool->reset(); } + public function onAfterBatchCompleted(): void + { + } + public function onAfterCatchUp(): void { $this->subgraphCachePool->reset(); diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php index 8c2ee16b5d2..657ad850095 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php @@ -88,6 +88,10 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } + public function onAfterBatchCompleted(): void + { + } + public function onAfterCatchUp(): void { } diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php index b6bd864021b..b26f90ebef2 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php @@ -60,6 +60,11 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } + public function onAfterBatchCompleted(): void + { + // Nothing to do here + } + public function onAfterCatchUp(): void { // Nothing to do here diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php index 6f60eb626d2..937ff4dc56c 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php @@ -245,6 +245,10 @@ private function scheduleCacheFlushJobForWorkspaceName( ); } + public function onAfterBatchCompleted(): void + { + } + public function onAfterCatchUp(): void { foreach ($this->flushNodeAggregateRequestsOnAfterCatchUp as $request) { From d04b8f3ba4f2b29bba54c8a913c059a2a5ec3f3d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:01:40 +0100 Subject: [PATCH 125/142] TASK: Trivial cosmetic changes --- .../Classes/Feature/ContentStreamHandling.php | 3 +++ .../Factory/SubscriptionStore/DoctrineSubscriptionStore.php | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php index a228ca7c864..a16aff28aa0 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php @@ -18,6 +18,9 @@ use Neos\EventStore\Model\Event\Version; use Neos\EventStore\Model\EventStream\ExpectedVersion; +/** + * @internal + */ trait ContentStreamHandling { /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 9c7c9e92005..06ed1292fe7 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -154,7 +154,9 @@ private static function fromDatabase(array $row): Subscription $subscriptionError = null; } $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); - assert($lastSavedAt instanceof DateTimeImmutable); + if ($lastSavedAt === false) { + throw new \RuntimeException(sprintf('last_saved_at %s is not a valid date', $row['last_saved_at']), 1733602968); + } return new Subscription( SubscriptionId::fromString($row['id']), From b852617832cc8ced22d25f5d1e8c38f53c7c28c9 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:48:29 +0100 Subject: [PATCH 126/142] TASK: Update documentation of `CatchUpHookInterface` --- .../CatchUpHook/CatchUpHookInterface.php | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php index 6d71b122ade..aac6b2461be 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookInterface.php @@ -20,10 +20,10 @@ interface CatchUpHookInterface { /** - * This hook is called at the beginning of a catch-up run; - * AFTER the Database Lock is acquired, BEFORE any projection was called. + * This hook is called at the beginning of a catch-up run, **after** the database lock is acquired, + * but **before** any projection was called. * - * Note that any errors thrown will be ignored and the catchup will start as normal. + * Note that any errors thrown will be collected and the current catchup batch will be finished as normal. * The collect errors will be returned and rethrown by the content repository. * * @throws CatchUpHookFailed @@ -32,9 +32,9 @@ public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void; /** * This hook is called for every event during the catchup process, **before** the projection - * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. + * is updated but in the same transaction. * - * Note that any errors thrown will be ignored and the catchup will continue as normal. + * Note that any errors thrown will be collected and the current catchup batch will be finished as normal. * The collect errors will be returned and rethrown by the content repository. * * @throws CatchUpHookFailed @@ -43,9 +43,9 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even /** * This hook is called for every event during the catchup process, **after** the projection - * is updated but in the same transaction: {@see ProjectionInterface::transactional()}. + * is updated but in the same transaction, * - * Note that any errors thrown will be ignored and the catchup will continue as normal. + * Note that any errors thrown will be collected and the current catchup batch will be finished as normal. * The collect errors will be returned and rethrown by the content repository. * * @throws CatchUpHookFailed @@ -54,22 +54,23 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event /** * This hook is called for each batch of processed events during the catchup process, **after** the projection - * is updated and the transaction is commited. + * and their new position is updated and the transaction is commited. * - * It can happen that this method is called even without having seen Events in the meantime. + * The database lock is directly acquired again after it is released if the batching needs to continue. + * It can happen that this method is called even without having seen events in the meantime. * - * If there exist more events which need to be processed, the database lock - * is directly acquired again after it is released. + * Note that any errors thrown will be collected but no further batch is started. + * The collect errors will be returned and rethrown by the content repository. + * + * @throws CatchUpHookFailed */ public function onAfterBatchCompleted(): void; /** - * This hook is called at the END of a catch-up run - * BEFORE the Database Lock is released, but AFTER the transaction is commited. - * - * The projection and their new status and position are already persisted. + * This hook is called at the END of a catch-up run, **after** the projection + * and their new position is updated and the transaction is commited. * - * Note that any errors thrown will be ignored and the catchup will finish as normal. + * Note that any errors thrown will be collected and the catchup will finish as normal. * The collect errors will be returned and rethrown by the content repository. * * @throws CatchUpHookFailed From 8daa8364505e2da6aa2a4b43cbf7ee4cdcda2994 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:08:08 +0100 Subject: [PATCH 127/142] TASK: Introduce test that we cannot roll-back setup Similarly to that doctrine migrations (`flow doctrine:migrate`) are also cannot be rollback in case of an error So we cannot wrap projection->setUp in transaction and errors (like in a migration) will be directly commited. Thus, it must be acted with care. --- .../TestSuite/DebugEventProjection.php | 3 + .../Subscription/SubscriptionSetupTest.php | 67 +++++++++++++++++++ .../Engine/SubscriptionEngine.php | 4 +- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php index 7f71796e96d..556e2d34d41 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -50,6 +50,9 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } + if ($this->saboteur) { + ($this->saboteur)(); + } } public function status(): ProjectionStatus diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index dd021dc8183..4db05520350 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -5,6 +5,8 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Projection\ProjectionStatus; +use Neos\ContentRepository\Core\Subscription\Engine\Error; +use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; @@ -218,4 +220,69 @@ public function failingSetupWillMarkProjectionAsErrored() $this->subscriptionStatus('Vendor.Package:FakeProjection') ); } + + /** @test */ + public function failingSetupWillNotRollbackProjection() + { + // we cannot wrap the schema creation in transactions as CREATE TABLE would for example lead to an implicit commit + // and cannot be rolled back: https://dev.mysql.com/doc/refman/8.0/en/implicit-commit.html + + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + $this->eventStore->setup(); + + // initial setup for FakeProjection + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // regular work + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // then an update is fetched - but the migration contains an error: + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + $exception = new \RuntimeException('Setup failed after it did some sql queries!'); + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + self::assertNull($result->errors); + + $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + // as we cant roll back, the migration was (possibly partially) made: + setupStatus: ProjectionStatus::ok() + ); + + $result = $this->subscriptionEngine->setup(); + self::assertEquals(Errors::fromArray([Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception)]), $result->errors); + + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 32dfed4199a..3b3c259f6e0 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -60,7 +60,8 @@ public function setup(SubscriptionEngineCriteria|null $criteria = null): Result SubscriptionStatus::ACTIVE ]))); if ($subscriptions->isEmpty()) { - $this->logger?->info('Subscription Engine: No subscriptions found.'); // todo not happy? Because there must be at least the content graph?!! + // should not happen as this means the contentGraph is unavailable, see status information. + $this->logger?->info('Subscription Engine: No subscriptions found.'); return Result::success(); } $errors = []; @@ -211,7 +212,6 @@ private function setupSubscription(Subscription $subscription): ?Error try { $subscriber->projection->setUp(); } catch (\Throwable $e) { - // todo wrap in savepoint to ensure error do not mess up the projection? $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); $this->subscriptionStore->update( $subscription->id, From 8f55975ed2b4ce2a39059c81f198d4ff9327a37c Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:47:37 +0100 Subject: [PATCH 128/142] TASK: Remove use of save-points for projections see https://github.com/neos/neos-development-collection/pull/5321#issuecomment-2531212129 the save-point will only be used for REAL projection errors now and never rolled back if catchup errors occur. With that change in code the save-points are less important because a real projection error should better be thrown at the start before any statements, and even if some statements were issued and a full rollback is done its unlikely that a reactivateSubscription helps that case. Instead, to repair projections you should replay --- .../TestSuite/DebugEventProjectionState.php | 8 +- .../Subscription/CatchUpHookErrorTest.php | 6 +- .../Subscription/ProjectionErrorTest.php | 94 ++++++------------- .../Engine/SubscriptionEngine.php | 9 +- .../Store/SubscriptionStoreInterface.php | 6 -- .../DoctrineSubscriptionStore.php | 15 --- 6 files changed, 42 insertions(+), 96 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php index da23036635c..2c739009f9e 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php @@ -21,9 +21,9 @@ public function __construct( } /** - * @return iterable + * @return array */ - public function findAppliedSequenceNumbers(): iterable + public function findAppliedSequenceNumbers(): array { return array_map( fn (int $value) => SequenceNumber::fromInteger($value), @@ -32,9 +32,9 @@ public function findAppliedSequenceNumbers(): iterable } /** - * @return iterable + * @return array */ - public function findAppliedSequenceNumberValues(): iterable + public function findAppliedSequenceNumberValues(): array { return array_map( fn ($value) => (int)$value['sequenceNumber'], diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 755ea0b05c5..c829c856606 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -361,8 +361,10 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() $expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + // partially applied event because the error is thrown at the end and the projection is not rolled back + self::assertEquals( + [1], + $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues() ); } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 1b1dd3bdfe4..da46bb9eaee 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -25,7 +25,7 @@ final class ProjectionErrorTest extends AbstractSubscriptionEngineTestCase { /** @test */ - public function projectionWithError() + public function projectionWithErrorCanBeReactivated() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -40,9 +40,19 @@ public function projectionWithError() $this->commitExampleContentStreamEvent(); // catchup active tries to apply the commited event - $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( - $exception = new \RuntimeException('This projection is kaputt.') - ); + $exception = new \RuntimeException('This projection is kaputt.'); + $this->fakeProjection->expects($i = self::exactly(2))->method('apply')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i, $exception) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + throw $exception + ], + 2 => [ + // on second call is repaired: + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + ] + }; + }); $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), subscriptionStatus: SubscriptionStatus::ERROR, @@ -60,21 +70,15 @@ public function projectionWithError() ); $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); - // todo test retry if reimplemented: https://github.com/patchlevel/event-sourcing/blob/6826d533fd4762220f0397bc7afc589abb8c901b/src/Subscription/RetryStrategy/RetryStrategy.php - // // CatchUp 2 with retry - // $result = $this->subscriptionEngine->catchUpActive(); - // self::assertTrue($result->hasFailed()); - // self::assertEquals($result->errors->first()->message, 'Something really wrong.'); - // self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'Something really wrong.'); + // + // fix projection and catchup + // - // no retry, nothing to do. - $result = $this->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::success(0), $result); - self::assertEquals($this->subscriptionStatus('Vendor.Package:FakeProjection')->subscriptionError->errorMessage, 'This projection is kaputt.'); - self::assertEquals( - $expectedStatusForFailedProjection, - $this->subscriptionStatus('Vendor.Package:FakeProjection') - ); + // reactivate and catchup + $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')])); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); } /** @test */ @@ -182,12 +186,8 @@ public function irreparableProjection() // reactivation will attempt to retry fix this, but can only work if the projection is repaired and will lead to an error otherwise: $result = $this->subscriptionEngine->reactivate(); - self::assertEquals( - ProcessedResult::failed(1, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception) - ])), - $result - ); + self::assertEquals(1, $result->numberOfProcessedEvents); + self::assertEquals('Must not happen! Debug projection detected duplicate event 1 of type ContentStreamWasCreated', $result->errors->first()?->message); self::assertEquals( ProjectionSubscriptionStatus::create( @@ -195,7 +195,7 @@ public function irreparableProjection() subscriptionStatus: SubscriptionStatus::ERROR, subscriptionPosition: SequenceNumber::none(), // previous state is now an error too also error: - subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ERROR, $exception), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ERROR, $result->errors->first()->throwable), setupStatus: ProjectionStatus::ok(), ), $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') @@ -223,7 +223,7 @@ public function irreparableProjection() } /** @test */ - public function projectionIsRolledBackAfterError() + public function projectionWithError() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -260,30 +260,15 @@ public function projectionIsRolledBackAfterError() $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); - // should be empty as we need an exact once delivery - self::assertEmpty( - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // - // fix projection and catchup - // - - $this->secondFakeProjection->killSaboteur(); - - // reactivate and catchup - $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); - self::assertNull($result->errors); - - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + // because the error is thrown after the even the state is commited self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + [1], + $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues() ); } /** @test */ - public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() + public function projectionWithErrorAfterSecondEvent() { $this->eventStore->setup(); $this->fakeProjection->expects(self::once())->method('setUp'); @@ -325,24 +310,7 @@ public function projectionIsRolledBackAfterErrorButKeepsSuccessFullEvents() $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') ); - // the first successful event is applied and committet: - self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() - ); - - // - // fix projection and catchup - // - - $this->secondFakeProjection->killSaboteur(); - - // catchup after fix - $result = $this->subscriptionEngine->reactivate(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')])); - self::assertNull($result->errors); - - // subscriptionError is reset, and the position is advanced if there were events - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + // the first successful event is applied and committet, but the second partially applied event is also applied: self::assertEquals( [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 3b3c259f6e0..e50e7004e05 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -343,7 +343,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $errors[] = Error::forSubscription($subscription->id, $e); } - $this->subscriptionStore->createSavepoint(); try { $subscriber->projection->apply($domainEvent, $eventEnvelope); } catch (\Throwable $e) { @@ -351,11 +350,10 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); $error = Error::forSubscription($subscription->id, $e); - // 1.) roll back the partially applied event on the subscriber - $this->subscriptionStore->rollbackSavepoint(); - // 2.) for the leftover events we are not including this failed subscription for catchup + // for the leftover events we are not including this failed subscription for catchup $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // 3.) update the subscription error state on either its unchanged or new position (if some events worked) + // update the subscription error state on either its unchanged or new position (if some events worked) + // note that the possibly partially applied event will not be rolled back. $this->subscriptionStore->update( $subscription->id, status: SubscriptionStatus::ERROR, @@ -370,7 +368,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } // HAPPY Case: $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); - $this->subscriptionStore->releaseSavepoint(); $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; try { diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index 85f31072717..b7b0540415b 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -35,10 +35,4 @@ public function update( * @return T */ public function transactional(\Closure $closure): mixed; - - public function createSavepoint(): void; - - public function releaseSavepoint(): void; - - public function rollbackSavepoint(): void; } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 06ed1292fe7..252feebc4c1 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -171,19 +171,4 @@ public function transactional(\Closure $closure): mixed { return $this->dbal->transactional($closure); } - - public function createSavepoint(): void - { - $this->dbal->createSavepoint('SUBSCRIBER'); - } - - public function releaseSavepoint(): void - { - $this->dbal->releaseSavepoint('SUBSCRIBER'); - } - - public function rollbackSavepoint(): void - { - $this->dbal->rollbackSavepoint('SUBSCRIBER'); - } } From 79d4ec7c50fca5a05b4ffc6dfb7c9843f935197f Mon Sep 17 00:00:00 2001 From: Bastian Waidelich Date: Tue, 10 Dec 2024 20:12:04 +0100 Subject: [PATCH 129/142] Fix `DoctrineSubscriptionStore` compatibility with SQLite and PostgreSQL --- .../SubscriptionStore/DoctrineSubscriptionStore.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index fe5f038fda4..628ace96f21 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -7,6 +7,7 @@ use DateTimeImmutable; use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\Column; use Doctrine\DBAL\Schema\Schema; @@ -44,11 +45,11 @@ public function setup(): void 'charset' => 'utf8mb4' ]); $tableSchema = new Table($this->tableName, [ - (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), + (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH), (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), - (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), + (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32), (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), - (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32)->setPlatformOption('charset', 'ascii')->setPlatformOption('collation', 'ascii_general_ci'), + (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32), (new Column('error_trace', Type::getType(Types::TEXT)))->setNotnull(false), (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), ]); @@ -70,7 +71,9 @@ public function findByCriteriaForUpdate(SubscriptionCriteria $criteria): Subscri ->select('*') ->from($this->tableName) ->orderBy('id'); - $queryBuilder->forUpdate(); + if (!$this->dbal->getDatabasePlatform() instanceof SqlitePlatform) { + $queryBuilder->forUpdate(); + } if ($criteria->ids !== null) { $queryBuilder->andWhere('id IN (:ids)') ->setParameter( From c3e329128c3c93366afd55d918e51cd48bb6a979 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 11:34:57 +0100 Subject: [PATCH 130/142] TASK: Improve errors for catch hooks to only the first error the exception Adds test for these cases Additionally, we have to introduce a `FakeCatchUpHookFactory2` to have two hooks on one projection and prevent: > a CatchUpHookFactory of type "Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory" already exists in this set --- .../Configuration/Testing/Settings.yaml | 8 +- .../AbstractSubscriptionEngineTestCase.php | 18 +- .../Subscription/CatchUpHookErrorTest.php | 230 +++++++++++++----- .../Subscription/CatchUpHookTest.php | 182 ++++++++++---- .../CatchUpHookWithPersistenceTest.php | 16 +- .../CatchUpHook/CatchUpHookFailed.php | 26 +- .../CatchUpHook/DelegatingCatchUpHook.php | 18 +- .../Classes/Fakes/FakeCatchUpHookFactory2.php | 32 +++ 8 files changed, 380 insertions(+), 150 deletions(-) create mode 100644 Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index 7f2fdbc0cc4..24d25b27065 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -65,14 +65,18 @@ Neos: factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory options: instanceId: default + catchUpHooks: + 'Vendor.Package:FakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory 'Vendor.Package:SecondFakeProjection': factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory options: instanceId: second catchUpHooks: - 'Vendor.Package:FakeCatchupHook': + 'Vendor.Package:SecondFakeCatchupHook': factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory - + 'Vendor.Package:AdditionalSecondFakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory2 Flow: object: includeClasses: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php index cd3b5c8bb92..c630985ba7c 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -27,6 +27,7 @@ use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory2; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; @@ -58,6 +59,10 @@ abstract class AbstractSubscriptionEngineTestCase extends TestCase // we don't u protected CatchUpHookInterface&MockObject $catchupHookForFakeProjection; + protected CatchUpHookInterface&MockObject $catchupHookForSecondFakeProjection; + + protected CatchUpHookInterface&MockObject $additionalCatchupHookForSecondFakeProjection; + public static function setUpBeforeClass(): void { static::$contentRepositoryId = ContentRepositoryId::fromString('t_subscription'); @@ -96,10 +101,21 @@ public function setUp(): void ); $this->catchupHookForFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + FakeCatchUpHookFactory::setCatchupHook( + $this->fakeProjection->getState(), + $this->catchupHookForFakeProjection + ); + $this->catchupHookForSecondFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); FakeCatchUpHookFactory::setCatchupHook( $this->secondFakeProjection->getState(), - $this->catchupHookForFakeProjection + $this->catchupHookForSecondFakeProjection + ); + + $this->additionalCatchupHookForSecondFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + FakeCatchUpHookFactory2::setCatchupHook( + $this->secondFakeProjection->getState(), + $this->additionalCatchupHookForSecondFakeProjection ); FakeNodeTypeManagerFactory::setConfiguration([]); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index c829c856606..24f0f960e0a 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -32,29 +32,28 @@ public function error_onBeforeEvent_isIgnoredAndCollected() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); $exception = new \RuntimeException('This catchup hook is kaputt.'); - $this->catchupHookForFakeProjection->expects($invokedCount = self::exactly(2))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { + $this->catchupHookForSecondFakeProjection->expects($invokedCount = self::exactly(2))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { match ($invokedCount->getInvocationCount()) { 1 => self::assertSame(1, $eventEnvelope->sequenceNumber->value), 2 => self::assertSame(2, $eventEnvelope->sequenceNumber->value), }; throw $exception; }); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onBeforeEvent": This catchup hook is kaputt.', + 'Hook "onBeforeEvent" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); // two errors for both of the events @@ -91,28 +90,27 @@ public function error_onAfterEvent_isIgnoredAndCollected() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); $exception = new \RuntimeException('This catchup hook is kaputt.'); - $this->catchupHookForFakeProjection->expects($invokedCount = self::exactly(2))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { + $this->catchupHookForSecondFakeProjection->expects($invokedCount = self::exactly(2))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { match ($invokedCount->getInvocationCount()) { 1 => self::assertSame(1, $eventEnvelope->sequenceNumber->value), 2 => self::assertSame(2, $eventEnvelope->sequenceNumber->value), }; throw $exception; }); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); // todo assert no parameters?! + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); // todo assert no parameters?! self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onAfterEvent": This catchup hook is kaputt.', + 'Hook "onAfterEvent" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); // two errors for both of the events @@ -151,23 +149,22 @@ public function error_onBeforeCatchUp_isIgnoredAndCollected() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willThrowException( + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onBeforeCatchUp": This catchup hook is kaputt.', + 'Hook "onBeforeCatchUp" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); $result = $this->subscriptionEngine->catchUpActive(); @@ -204,23 +201,22 @@ public function error_onAfterBatchCompleted_isIgnoredAndCollected() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willThrowException( + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onAfterBatchCompleted": This catchup hook is kaputt.', + 'Hook "onAfterBatchCompleted" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); $result = $this->subscriptionEngine->catchUpActive(); @@ -257,12 +253,11 @@ public function error_onAfterCatchUp_isIgnoredAndCollected() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - // todo test that other catchup hooks are still run and all errors are collected! - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); @@ -271,10 +266,9 @@ public function error_onAfterCatchUp_isIgnoredAndCollected() ); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onAfterCatchUp": This catchup hook is kaputt.', + 'Hook "onAfterCatchUp" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); $result = $this->subscriptionEngine->catchUpActive(); @@ -311,12 +305,12 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp'); // only the onBeforeEvent hook will be invoked as afterward the projection errored - $this->catchupHookForFakeProjection->expects(self::exactly(1))->method('onBeforeEvent'); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + $this->catchupHookForSecondFakeProjection->expects(self::exactly(1))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( $exception = new \RuntimeException('This catchup hook is kaputt.') ); @@ -328,10 +322,9 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() ); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onAfterCatchUp": This catchup hook is kaputt.', + 'Hook "onAfterCatchUp" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); // two errors for both of the events @@ -380,14 +373,14 @@ public function error_onAfterEvent_stopsEngineAfterFirstBatch() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); $exception = new \RuntimeException('This catchup hook is kaputt.'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( $exception ); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() @@ -396,10 +389,9 @@ public function error_onAfterEvent_stopsEngineAfterFirstBatch() $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); $expectedWrappedException = new CatchUpHookFailed( - 'Hook "" failed "onAfterEvent": This catchup hook is kaputt.', + 'Hook "onAfterEvent" failed: "": This catchup hook is kaputt.', 1733243960, - $exception, - [] + $exception ); // one error @@ -421,4 +413,128 @@ public function error_onAfterEvent_stopsEngineAfterFirstBatch() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); } + + /** @test */ + public function error_onAfterEvent_withMultipleFailingHooks() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $firstException = new \RuntimeException('First catchup hook is kaputt.'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $firstException + ); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $secondException = new \RuntimeException('Second catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $secondException + ); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals( + ProcessedResult::failed( + 1, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:FakeProjection'), new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.', + 1733243960, + $firstException + )), + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": Second catchup hook is kaputt.', + 1733243960, + $secondException + )), + ]) + ), + $result + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterEvent_withMultipleFailingHooksOnOneProjection() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $firstException = new \RuntimeException('First catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $firstException + ); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $secondException = new \RuntimeException('Second catchup hook is kaputt.'); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $secondException + ); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals( + ProcessedResult::failed( + 1, + Errors::fromArray([ + Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.;' . PHP_EOL . + '"": Second catchup hook is kaputt.', + 1733243960, + $firstException + )), + ]) + ), + $result + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php index 0129ca9cba2..2da14da2606 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -5,6 +5,8 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; @@ -34,11 +36,25 @@ public function catchUpHooksAreExecutedAndCanAccessTheCorrectProjectionsState() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectOneHandledEvent); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); + + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectOneHandledEvent); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); $expectNoHandledEvents(); @@ -58,12 +74,20 @@ public function catchUpBeforeAndAfterCatchupAreRunForZeroEvents() $this->fakeProjection->expects(self::never())->method('apply'); $this->subscriptionEngine->setup(); + // first projection hooks $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $result = $this->subscriptionEngine->boot(); self::assertNull($result->errors); @@ -79,12 +103,20 @@ public function catchUpBeforeAndAfterCatchupAreNotRunIfNoSubscriberMatches() $this->fakeProjection->expects(self::never())->method('apply'); $this->subscriptionEngine->setup(); + // first projection hooks $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeCatchUp'); $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterCatchUp'); + $result = $this->subscriptionEngine->catchUpActive(); self::assertNull($result->errors); self::assertEquals(0, $result->numberOfProcessedEvents); @@ -93,6 +125,47 @@ public function catchUpBeforeAndAfterCatchupAreNotRunIfNoSubscriberMatches() self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); } + /** @test */ + public function catchHooksAreOnlyRunForMatchingSubscriber() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $result = $this->subscriptionEngine->catchUpActive(SubscriptionEngineCriteria::create( + [SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')] + )); + self::assertNull($result->errors); + self::assertEquals(1, $result->numberOfProcessedEvents); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + public function provideValidBatchSizes(): iterable { yield 'none' => [ @@ -152,52 +225,65 @@ public function catchUpHooksWithBatching(int|null $batchSize, array $onAfterBatc $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); + // first projection hooks $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); - $this->catchupHookForFakeProjection->expects($i = self::exactly(4))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { - match($i->getInvocationCount()) { - 1 => [ - self::assertEquals(1, $eventEnvelope->sequenceNumber->value), - self::assertEquals([], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - 2 => [ - self::assertEquals(2, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - 3 => [ - self::assertEquals(3, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - 4 => [ - self::assertEquals(4, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - }; - }); - $this->catchupHookForFakeProjection->expects($i = self::exactly(4))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { - match($i->getInvocationCount()) { - 1 => [ - self::assertEquals(1, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - 2 => [ - self::assertEquals(2, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - 3 => [ - self::assertEquals(3, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - 4 => [ - self::assertEquals(4, $eventEnvelope->sequenceNumber->value), - self::assertEquals([1,2,3,4], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) - ], - }; - }); - $this->catchupHookForFakeProjection->expects($i = self::exactly(\count($onAfterBatchCompletedInvocations)))->method('onAfterBatchCompleted')->willReturnCallback(function () use ($i, $onAfterBatchCompletedInvocations) { - self::assertEquals($onAfterBatchCompletedInvocations[$i->getInvocationCount() - 1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); - }); + $this->catchupHookForFakeProjection->expects(self::exactly(4))->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(4))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(\count($onAfterBatchCompletedInvocations)))->method('onAfterBatchCompleted'); $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + // second projection hooks + foreach ([ + $this->catchupHookForSecondFakeProjection, + $this->additionalCatchupHookForSecondFakeProjection, + ] as $catchUpHookMock) { + $catchUpHookMock->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $catchUpHookMock->expects($i = self::exactly(4))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + self::assertEquals([], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 2 => [ + self::assertEquals(2, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 3 => [ + self::assertEquals(3, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 4 => [ + self::assertEquals(4, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + }; + }); + $catchUpHookMock->expects($i = self::exactly(4))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 2 => [ + self::assertEquals(2, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 3 => [ + self::assertEquals(3, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 4 => [ + self::assertEquals(4, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3,4], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + }; + }); + $catchUpHookMock->expects($i = self::exactly(\count($onAfterBatchCompletedInvocations)))->method('onAfterBatchCompleted')->willReturnCallback(function () use ($i, $onAfterBatchCompletedInvocations) { + self::assertEquals($onAfterBatchCompletedInvocations[$i->getInvocationCount() - 1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + }); + $catchUpHookMock->expects(self::once())->method('onAfterCatchUp'); + } + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); $result = $this->subscriptionEngine->boot(batchSize: $batchSize); diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php index 22019865219..56e0906bb44 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php @@ -38,12 +38,12 @@ public function commitOnConnection_onAfterEvent() $this->commitExampleContentStreamEvent(); $this->commitExampleContentStreamEvent(); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () { + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () { $this->getObject(Connection::class)->commit(); }); - $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() @@ -87,13 +87,13 @@ public function persistAll_onAfterEvent_willUseTheTransaction() self::assertTrue($this->getObject(PersistenceManagerInterface::class)->isNewObject($persistentResource)); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () use ($persistentResource) { + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () use ($persistentResource) { $this->getObject(ResourceRepository::class)->add($persistentResource); $this->getObject(PersistenceManagerInterface::class)->persistAll(); }); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); self::assertEmpty( $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php index ccdaa36a79d..487bbd9e40e 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php @@ -5,32 +5,10 @@ namespace Neos\ContentRepository\Core\Projection\CatchUpHook; /** - * Thrown if a delegated catchup hook fails + * Thrown if a catchup hook fails * - * @implements \IteratorAggregate<\Throwable> * @api */ -final class CatchUpHookFailed extends \RuntimeException implements \IteratorAggregate +final class CatchUpHookFailed extends \RuntimeException { - /** - * @internal - * @param array<\Throwable> $additionalExceptions - */ - public function __construct( - string $message, - int $code, - \Throwable $exception, - private readonly array $additionalExceptions - ) { - parent::__construct($message, $code, $exception); - } - - public function getIterator(): \Traversable - { - $previous = $this->getPrevious(); - if ($previous !== null) { - yield $previous; - } - yield from $this->additionalExceptions; - } } diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php index af24811e9b4..0d66ec34601 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/DelegatingCatchUpHook.php @@ -73,25 +73,23 @@ public function onAfterCatchUp(): void */ private function delegateHooks(\Closure $closure, string $hookName): void { - /** @var array<\Throwable> $errors */ + $firstError = null; + /** @var list $errors */ $errors = []; - $firstFailedCatchupHook = null; foreach ($this->catchUpHooks as $catchUpHook) { try { $closure($catchUpHook); } catch (\Throwable $e) { - $firstFailedCatchupHook ??= substr(strrchr($catchUpHook::class, '\\') ?: '', 1); - $errors[] = $e; + $firstError ??= $e; + $failedCatchupHookName = substr(strrchr($catchUpHook::class, '\\') ?: '', 1); + $errors[] = sprintf('"%s": %s', $failedCatchupHookName, $e->getMessage()); } } - if ($errors !== []) { - $firstError = array_shift($errors); - $additionalMessage = $errors !== [] ? sprintf(' (and %d other)', count($errors)) : ''; + if ($firstError !== null) { throw new CatchUpHookFailed( - sprintf('Hook "%s"%s failed "%s": %s', $firstFailedCatchupHook, $additionalMessage, $hookName, $firstError->getMessage()), + sprintf('Hook "%s" failed: %s', $hookName, join(";\n", $errors)), 1733243960, - $firstError, - $errors + $firstError ); } } diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php new file mode 100644 index 00000000000..cc5610ed1d9 --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php @@ -0,0 +1,32 @@ + + * @internal helper to configure custom catchup hook mocks for testing + */ +final class FakeCatchUpHookFactory2 implements CatchUpHookFactoryInterface +{ + /** + * @var array + */ + private static array $catchupHooks; + + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface + { + return static::$catchupHooks[spl_object_hash($dependencies->projectionState)] ?? throw new \RuntimeException('No catchup hook defined for Fake.'); + } + + public static function setCatchupHook(ProjectionStateInterface $projectionState, CatchUpHookInterface $catchUpHook): void + { + self::$catchupHooks[spl_object_hash($projectionState)] = $catchUpHook; + } +} From 542bd4e1d7fad2225a9febf85af8909f81539ad7 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:10:01 +0100 Subject: [PATCH 131/142] TASK: Replace `transactional` function use with explicit start and stop This way we reduce complexity as we dont have to pass too many parameters by reference. Also it makes the case for `onBeforeCatchUp` (the first iteration) more explicit We dont need to worry about rollbacks as no errors should ever be thrown uncatched during catchup. --- .../Engine/SubscriptionEngine.php | 234 +++++++++--------- .../Store/SubscriptionStoreInterface.php | 9 +- .../DoctrineSubscriptionStore.php | 9 +- 3 files changed, 130 insertions(+), 122 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index e50e7004e05..cfcd5317846 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -267,7 +267,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in states %s.', join(',', $status->toStringArray()))); - $subscriptionsToInvokeAroundCatchUpHooks = Subscriptions::none(); $subscriptionCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $status); @@ -275,130 +274,132 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs /** @var array $errors */ $errors = []; - do { + $this->subscriptionStore->beginTransaction(); + + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); + foreach ($subscriptionsToCatchup as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + } + } + + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); + $this->subscriptionStore->commit(); + return ProcessedResult::success(0); + } + + $subscriptionsToInvokeAroundCatchUpHooks = $subscriptionsToCatchup; + foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { + try { + $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); + } + } + + while (true) { /** * If batching is enabled, the {@see $continueBatching} flag will indicate that the last run was stopped and continuation is necessary to handle the rest of the events. * It's possible that batching stops at the last event, in that case the transaction is still reopened to set the active state correctly. */ - $continueBatching = $this->subscriptionStore->transactional(function () use ($subscriptionCriteria, $progressCallback, $batchSize, &$subscriptionsToInvokeAroundCatchUpHooks, &$numberOfProcessedEvents, &$errors) { - $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); - if ($numberOfProcessedEvents === 0) { - // first batch - foreach ($subscriptionsToCatchup as $subscription) { - if (!$this->subscribers->contain($subscription->id)) { - // mark detached subscriptions as we cannot handle them and exclude them from catchup - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::DETACHED, - position: $subscription->position, - subscriptionError: null, - ); - $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - } - } - - if ($subscriptionsToCatchup->isEmpty()) { - $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); - return false; - } + $continueBatching = false; - $subscriptionsToInvokeAroundCatchUpHooks = $subscriptionsToCatchup; - foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { - try { - $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); - } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); - } - } - } - - $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); - $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); - /** @var array $highestSequenceNumberForSubscriber */ - $highestSequenceNumberForSubscriber = []; + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; - $continueBatching = false; - $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); - foreach ($eventStream as $eventEnvelope) { - $sequenceNumber = $eventEnvelope->sequenceNumber; - if ($numberOfProcessedEvents > 0) { - $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } + if ($progressCallback !== null) { + $progressCallback($eventEnvelope); + } + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; } - if ($progressCallback !== null) { - $progressCallback($eventEnvelope); + $subscriber = $this->subscribers->get($subscription->id); + + try { + $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); } - $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); - foreach ($subscriptionsToCatchup as $subscription) { - if ($subscription->position->value >= $sequenceNumber->value) { - $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); - continue; - } - $subscriber = $this->subscribers->get($subscription->id); - - try { - $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); - } - - try { - $subscriber->projection->apply($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - // ERROR Case: - $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); - $error = Error::forSubscription($subscription->id, $e); - - // for the leftover events we are not including this failed subscription for catchup - $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); - // update the subscription error state on either its unchanged or new position (if some events worked) - // note that the possibly partially applied event will not be rolled back. - $this->subscriptionStore->update( - $subscription->id, - status: SubscriptionStatus::ERROR, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: SubscriptionError::fromPreviousStatusAndException( - $subscription->status, - $error->throwable - ), - ); - $errors[] = $error; - continue; - } - // HAPPY Case: - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); - $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; - - try { - $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); - } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); - } + + try { + $subscriber->projection->apply($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + // ERROR Case: + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + $error = Error::forSubscription($subscription->id, $e); + + // for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // update the subscription error state on either its unchanged or new position (if some events worked) + // note that the possibly partially applied event will not be rolled back. + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $error->throwable + ), + ); + $errors[] = $error; + continue; } - $numberOfProcessedEvents++; - if ($batchSize !== null && $numberOfProcessedEvents % $batchSize === 0) { - $continueBatching = true; - $this->logger?->info(sprintf('Subscription Engine: Batch completed with %d events', $numberOfProcessedEvents)); - break; + // HAPPY Case: + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; + + try { + $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = Error::forSubscription($subscription->id, $e); } } - foreach ($subscriptionsToCatchup as $subscription) { - // after catchup mark all subscriptions as active, so they are triggered automatically now. - // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position - $this->subscriptionStore->update( - $subscription->id, - status: $continueBatching === false ? SubscriptionStatus::ACTIVE : $subscription->status, - position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, - subscriptionError: null, - ); - if ($continueBatching === false && $subscription->status !== SubscriptionStatus::ACTIVE) { - $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting', $subscription->id->value)); - } + $numberOfProcessedEvents++; + if ($batchSize !== null && $numberOfProcessedEvents % $batchSize === 0) { + $continueBatching = true; + $this->logger?->info(sprintf('Subscription Engine: Batch completed with %d events', $numberOfProcessedEvents)); + break; } - $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); - return $continueBatching; - }); + } + foreach ($subscriptionsToCatchup as $subscription) { + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: $continueBatching === false ? SubscriptionStatus::ACTIVE : $subscription->status, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); + if ($continueBatching === false && $subscription->status !== SubscriptionStatus::ACTIVE) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting', $subscription->id->value)); + } + } + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); + + // todo test that this is done before onAfterBatchCompleted + $this->subscriptionStore->commit(); + foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { try { $this->subscribers->get($subscription->id)->catchUpHook?->onAfterBatchCompleted(); @@ -406,10 +407,15 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $errors[] = Error::forSubscription($subscription->id, $e); } } - if ($errors !== []) { + + if ($continueBatching === true && $errors === []) { + // start new batch + $this->subscriptionStore->beginTransaction(); + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); + } else { break; } - } while ($continueBatching === true); + } // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? // note that a catchup error in onAfterEvent would bubble up directly and never invoke onAfterCatchUp diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php index b7b0540415b..b47c385a7c2 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -29,10 +29,7 @@ public function update( SubscriptionError|null $subscriptionError, ): void; - /** - * @template T - * @param \Closure():T $closure - * @return T - */ - public function transactional(\Closure $closure): mixed; + public function beginTransaction(): void; + + public function commit(): void; } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index ab043b527b6..3d578afed52 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -170,8 +170,13 @@ private static function fromDatabase(array $row): Subscription ); } - public function transactional(\Closure $closure): mixed + public function beginTransaction(): void { - return $this->dbal->transactional($closure); + $this->dbal->beginTransaction(); + } + + public function commit(): void + { + $this->dbal->commit(); } } From efa4165e52088fca11ef631d6445ecd337f4232f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:33:30 +0100 Subject: [PATCH 132/142] TASK: Ensure that subscriptions are not catchup'd if "onBeforeCatchUp" was not invoked this does not happen under normal circumstances but is technically possible as we release and reaquire a lock during batching. --- .../Engine/SubscriptionEngine.php | 26 ++++++++++--------- .../Classes/Subscription/Subscriptions.php | 8 ++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index cfcd5317846..fdfd948982c 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -16,7 +16,6 @@ use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; use Neos\ContentRepository\Core\Subscription\Subscription; use Neos\ContentRepository\Core\Subscription\SubscriptionError; -use Neos\ContentRepository\Core\Subscription\Subscriptions; use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; use Neos\ContentRepository\Core\Subscription\SubscriptionStatusFilter; @@ -297,10 +296,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs return ProcessedResult::success(0); } - $subscriptionsToInvokeAroundCatchUpHooks = $subscriptionsToCatchup; - foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { + $subscriptionIdsToInvokeAroundCatchUpHooks = $subscriptionsToCatchup->getIds(); + foreach ($subscriptionsToCatchup as $subscription) { + $subscriber = $this->subscribers->get($subscription->id); try { - $this->subscribers->get($subscription->id)->catchUpHook?->onBeforeCatchUp($subscription->status); + $subscriber->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { $errors[] = Error::forSubscription($subscription->id, $e); } @@ -334,6 +334,10 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); continue; } + if (!$subscriptionIdsToInvokeAroundCatchUpHooks->contain($subscription->id)) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" with status "%s" was not part of the first batch, continue catch up.', $subscription->id->value, $subscription->status->value)); + continue; + } $subscriber = $this->subscribers->get($subscription->id); try { @@ -400,11 +404,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs // todo test that this is done before onAfterBatchCompleted $this->subscriptionStore->commit(); - foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { + foreach ($subscriptionIdsToInvokeAroundCatchUpHooks as $subscriptionId) { try { - $this->subscribers->get($subscription->id)->catchUpHook?->onAfterBatchCompleted(); + $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterBatchCompleted(); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $errors[] = Error::forSubscription($subscriptionId, $e); } } @@ -417,13 +421,11 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } } - // todo do we need want to invoke for failed projections onAfterCatchUp, as onBeforeCatchUp was invoked already and to be consistent to "shutdown" this catchup iteration? - // note that a catchup error in onAfterEvent would bubble up directly and never invoke onAfterCatchUp - foreach ($subscriptionsToInvokeAroundCatchUpHooks as $subscription) { + foreach ($subscriptionIdsToInvokeAroundCatchUpHooks as $subscriptionId) { try { - $this->subscribers->get($subscription->id)->catchUpHook?->onAfterCatchUp(); + $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $errors[] = Error::forSubscription($subscriptionId, $e); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php index 73760148746..f3316a486d8 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -109,6 +109,14 @@ public function with(Subscription $subscription): self return new self([...$this->subscriptionsById, $subscription->id->value => $subscription]); } + public function getIds(): SubscriptionIds + { + return SubscriptionIds::fromArray(array_map( + fn (Subscription $subscription) => $subscription->id, + iterator_to_array($this->subscriptionsById) + )); + } + /** * @return iterable */ From 202519c606c63d60b8d67fd787cd9bb68e8357c7 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:39:58 +0100 Subject: [PATCH 133/142] TASK: Add test to ensure transaction is not active during onAfterBatchCompleted and onAfterCatchUp --- .../Subscription/CatchUpHookTest.php | 19 ++++++++++++++----- .../Engine/SubscriptionEngine.php | 1 - 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php index 2da14da2606..6693f1c2563 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -4,6 +4,7 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Functional\Subscription; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngineCriteria; use Neos\ContentRepository\Core\Subscription\SubscriptionId; @@ -36,12 +37,20 @@ public function catchUpHooksAreExecutedAndCanAccessTheCorrectProjectionsState() $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() ); + $expectNoTransactionActive = fn () => self::assertFalse( + $this->getObject(Connection::class)->isTransactionActive(), 'Expected no transaction to be active' + ); + + $expectTransactionActive = fn () => self::assertTrue( + $this->getObject(Connection::class)->isTransactionActive(), 'Expected transaction to be active' + ); + // first projection hooks - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); - $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); - $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectNoTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectNoTransactionActive); // second projection hooks $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index fdfd948982c..8715b4585e4 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -401,7 +401,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); - // todo test that this is done before onAfterBatchCompleted $this->subscriptionStore->commit(); foreach ($subscriptionIdsToInvokeAroundCatchUpHooks as $subscriptionId) { From cc299c3811aad13693e71d829bbec6efc598512b Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:05:46 +0100 Subject: [PATCH 134/142] TASK: Only collect first Throwable object instance during catchup As the exception log and `CatchUpHadErrors` only accept one previous exception and to reduce possible bloat if to many exceptions are thrown. Instead, the exceptions should additionally be logged (next commit). --- .../Subscription/CatchUpHookErrorTest.php | 53 +++++++++++-------- .../Subscription/ProjectionErrorTest.php | 4 +- .../Subscription/SubscriptionSetupTest.php | 2 +- .../Classes/Subscription/Engine/Error.php | 13 +++-- .../Engine/SubscriptionEngine.php | 19 ++++--- .../Exception/CatchUpHadErrors.php | 29 ++-------- 6 files changed, 53 insertions(+), 67 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 24f0f960e0a..99229f8fd86 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -62,8 +62,8 @@ public function error_onBeforeEvent_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), ]) ), $result @@ -119,8 +119,8 @@ public function error_onAfterEvent_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), ]) ), $result @@ -172,7 +172,7 @@ public function error_onBeforeCatchUp_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), ]) ), $result @@ -224,7 +224,7 @@ public function error_onAfterBatchCompleted_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), ]) ), $result @@ -276,7 +276,7 @@ public function error_onAfterCatchUp_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), ]) ), $result @@ -334,8 +334,8 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() ProcessedResult::failed( 2, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $innerException), - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $innerException->getMessage(), $innerException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), ]) ), $result @@ -400,7 +400,7 @@ public function error_onAfterEvent_stopsEngineAfterFirstBatch() ProcessedResult::failed( 1, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), ]) ), $result @@ -455,16 +455,20 @@ public function error_onAfterEvent_withMultipleFailingHooks() ProcessedResult::failed( 1, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:FakeProjection'), new CatchUpHookFailed( + Error::create( + SubscriptionId::fromString('Vendor.Package:FakeProjection'), 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.', - 1733243960, - $firstException - )), - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), new CatchUpHookFailed( + new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.', + 1733243960, + $firstException + ) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), 'Hook "onAfterEvent" failed: "": Second catchup hook is kaputt.', - 1733243960, - $secondException - )), + null + ), ]) ), $result @@ -519,12 +523,15 @@ public function error_onAfterEvent_withMultipleFailingHooksOnOneProjection() ProcessedResult::failed( 1, Errors::fromArray([ - Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), new CatchUpHookFailed( - 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.;' . PHP_EOL . + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $message = 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.;' . PHP_EOL . '"": Second catchup hook is kaputt.', - 1733243960, - $firstException - )), + new CatchUpHookFailed( + $message, + 1733243960, + $firstException + )), ]) ), $result diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index da46bb9eaee..72db4f9c3c1 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -62,7 +62,7 @@ public function projectionWithErrorCanBeReactivated() ); $result = $this->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::forSubscription(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception->getMessage(), $exception)])), $result); self::assertEquals( $expectedStatusForFailedProjection, @@ -388,7 +388,7 @@ public function projectionError_stopsEngineAfterFirstBatch() ); $result = $this->subscriptionEngine->boot(batchSize: 1); - self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::forSubscription(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception)])), $result); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception->getMessage(), $exception)])), $result); self::assertEquals( $expectedStatusForFailedProjection, diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 4db05520350..614a5ccbd52 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -278,7 +278,7 @@ public function failingSetupWillNotRollbackProjection() ); $result = $this->subscriptionEngine->setup(); - self::assertEquals(Errors::fromArray([Error::forSubscription(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception)]), $result->errors); + self::assertEquals(Errors::fromArray([Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception->getMessage(), $exception)]), $result->errors); self::assertEquals( $expectedStatusForFailedProjection, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php index 98ae41bd1d1..07cedb6e8f4 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php @@ -14,16 +14,19 @@ private function __construct( public SubscriptionId $subscriptionId, public string $message, - public \Throwable $throwable, + public \Throwable|null $throwable, ) { } - public static function forSubscription(SubscriptionId $subscriptionId, \Throwable $exception): self - { + public static function create( + SubscriptionId $subscriptionId, + string $message, + \Throwable|null $throwable, + ): self { return new self( $subscriptionId, - $exception->getMessage(), - $exception, + $message, + $throwable ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 8715b4585e4..43577301268 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -218,7 +218,7 @@ private function setupSubscription(Subscription $subscription): ?Error $subscription->position, SubscriptionError::fromPreviousStatusAndException($subscription->status, $e) ); - return Error::forSubscription($subscription->id, $e); + return Error::create($subscription->id, $e->getMessage(), $e); } if ($subscription->status === SubscriptionStatus::ACTIVE) { @@ -243,7 +243,7 @@ private function resetSubscription(Subscription $subscription): ?Error $subscriber->projection->resetState(); } catch (\Throwable $e) { $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); - return Error::forSubscription($subscription->id, $e); + return Error::create($subscription->id, $e->getMessage(), $e); } $this->subscriptionStore->update( $subscription->id, @@ -302,7 +302,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); } } @@ -343,7 +343,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); } try { @@ -351,7 +351,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs } catch (\Throwable $e) { // ERROR Case: $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); - $error = Error::forSubscription($subscription->id, $e); // for the leftover events we are not including this failed subscription for catchup $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); @@ -363,10 +362,10 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, subscriptionError: SubscriptionError::fromPreviousStatusAndException( $subscription->status, - $error->throwable + $e ), ); - $errors[] = $error; + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); continue; } // HAPPY Case: @@ -376,7 +375,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscription->id, $e); + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); } } $numberOfProcessedEvents++; @@ -407,7 +406,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterBatchCompleted(); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscriptionId, $e); + $errors[] = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); } } @@ -424,7 +423,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { - $errors[] = Error::forSubscription($subscriptionId, $e); + $errors[] = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php index 7545c74706c..87a7314f6e4 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php @@ -20,26 +20,12 @@ * projection again, as it has to be fixed and reactivated first. * * - A catchup hook contains an error. In this case the projections is further updated and also all further catchup errors - * collected. This results in a {@see CatchUpHookFailed} exception in this exception collection. + * collected. This results in a {@see CatchUpHookFailed} exception. * - * @implements \IteratorAggregate<\Throwable> * @api */ -final class CatchUpHadErrors extends \RuntimeException implements \IteratorAggregate +final class CatchUpHadErrors extends \RuntimeException { - /** - * @internal - * @param array<\Throwable> $additionalExceptions - */ - private function __construct( - string $message, - int $code, - \Throwable $exception, - private readonly array $additionalExceptions - ) { - parent::__construct($message, $code, $exception); - } - /** * @internal */ @@ -54,15 +40,6 @@ public static function createFromErrors(Errors $errors): self $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' | And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); - throw new self($exceptionMessage, 1732132930, $firstError->throwable, array_map(fn (Error $error) => $error->throwable, $errors)); - } - - public function getIterator(): \Traversable - { - $previous = $this->getPrevious(); - if ($previous !== null) { - yield $previous; - } - yield from $this->additionalExceptions; + throw new self($exceptionMessage, 1732132930, $firstError->throwable); } } From 431732e20ec445d774f266a19ba66b0d4517616f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:38:06 +0100 Subject: [PATCH 135/142] TASK: Improve message of `CatchUpHadErrors` by concatenating all errors (and clamping). --- .../Subscription/ProjectionErrorTest.php | 2 +- .../Exception/CatchUpHadErrors.php | 22 ++--- .../Subscription/CatchUpHadErrorsTest.php | 86 +++++++++++++++++++ 3 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index 72db4f9c3c1..b0864427f88 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -337,7 +337,7 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( $handleException = $exception; } self::assertInstanceOf(CatchUpHadErrors::class, $exception); - self::assertEquals('Exception in subscriber "Vendor.Package:FakeProjection" while catching up: This projection is kaputt.', $handleException->getMessage()); + self::assertEquals('Exception while catching up: "Vendor.Package:FakeProjection": This projection is kaputt.', $handleException->getMessage()); self::assertSame($originalException, $handleException->getPrevious()); // workspace is created. The fake projection failed on the first event, but other projections succeed: diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php index 87a7314f6e4..ab50707f43f 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php @@ -26,20 +26,22 @@ */ final class CatchUpHadErrors extends \RuntimeException { + private const CLAMP_ERRORS = 5; + /** * @internal */ public static function createFromErrors(Errors $errors): self { - /** @var non-empty-array $errors */ - $errors = iterator_to_array($errors); - $firstError = array_shift($errors); - - $additionalFailedSubscribers = array_map(fn (Error $error) => $error->subscriptionId->value, $errors); - - $additionalErrors = $additionalFailedSubscribers === [] ? '' : sprintf(' | And subscribers %s with additional errors.', join(', ', $additionalFailedSubscribers)); - $exceptionMessage = sprintf('Exception in subscriber "%s" while catching up: %s%s', $firstError->subscriptionId->value, $firstError->message, $additionalErrors); - - throw new self($exceptionMessage, 1732132930, $firstError->throwable); + $additionalMessage = ''; + $lines = []; + foreach ($errors as $error) { + $lines[] = sprintf('"%s": %s', $error->subscriptionId->value, $error->message); + if (count($lines) >= self::CLAMP_ERRORS) { + $additionalMessage = sprintf('%sAnd %d other exceptions, see log.', ";\n", count($errors) - self::CLAMP_ERRORS); + break; + } + } + return new self(sprintf('Exception while catching up: %s%s', join(";\n", $lines), $additionalMessage), 1732132930, $errors->first()->throwable); } } diff --git a/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php b/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php new file mode 100644 index 00000000000..0e9aed206cb --- /dev/null +++ b/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php @@ -0,0 +1,86 @@ +getMessage(), $expectedWrappedException), + ]); + + // assert shape if the error had been thrown by the cr: + $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); + self::assertEquals($expectedWrappedException, $catchupHadErrors->getPrevious()); + self::assertEquals('Exception while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeCatchup" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); + } + + public function testWithTwoHookErrors() + { + $exception = new \RuntimeException('This catchup hook is kaputt.'); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + $errors = Errors::fromArray([ + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), + ]); + + // assert shape if the error had been thrown by the cr: + $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); + self::assertEquals($expectedWrappedException, $catchupHadErrors->getPrevious()); + self::assertEquals('Exception while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.; +"Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); + } + + public function testWith10Errors() + { + $exception = new \RuntimeException('Message why A failed'); + $errors = Errors::fromArray([ + Error::create(SubscriptionId::fromString('Vendor.Package:A'), $exception->getMessage(), $exception), + Error::create(SubscriptionId::fromString('Vendor.Package:B'), 'Message why B failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:C'), 'Message why C failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:D'), 'Message why D failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:E'), 'Message why E failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:F'), 'Message why F failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:G'), 'Message why G failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:H'), 'Message why H failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:I'), 'Message why I failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:J'), 'Message why J failed', null), + ]); + + // assert shape if the error had been thrown by the cr: + $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); + self::assertEquals($exception, $catchupHadErrors->getPrevious()); + self::assertEquals(<<getMessage()); + } +} From 3fa41229e1935df1eb4959a7d287e9103a1bcfa8 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:58:53 +0100 Subject: [PATCH 136/142] TASK: Log catchup hook errors directly and improve error output during replay --- .../Service/ContentRepositoryMaintainer.php | 9 ++++--- .../Classes/Subscription/Engine/Errors.php | 16 +++++++++++++ .../Engine/SubscriptionEngine.php | 24 ++++++++++++++----- .../Exception/CatchUpHadErrors.php | 14 +---------- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php index 8337dae565b..3c54d5ce7f9 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -199,11 +199,10 @@ public function prune(): Error|null private static function createErrorForReason(string $method, Errors $errors): Error { - $message = []; - $message[] = sprintf('%s: Following error%s', $method, $errors->count() === 1 ? '' : 's'); - foreach ($errors as $error) { - $message[] = sprintf(' Subscription "%s": %s', $error->subscriptionId->value, $error->message); - } + $message = [ + sprintf('%s Following error%s', $method, $errors->count() === 1 ? '' : 's'), + ...array_map(fn (string $line) => ' ' . $line, explode("\n", $errors->getClampedMessage())) + ]; return new Error(join("\n", $message)); } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index 8e8d3b3853f..b9b576ef150 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -10,6 +10,8 @@ */ final readonly class Errors implements \IteratorAggregate, \Countable { + private const CLAMP_ERRORS = 5; + /** * @var non-empty-array */ @@ -48,4 +50,18 @@ public function first(): Error return $error; } } + + public function getClampedMessage(): string + { + $additionalMessage = ''; + $lines = []; + foreach ($this->errors as $error) { + $lines[] = sprintf('"%s": %s', $error->subscriptionId->value, $error->message); + if (count($lines) >= self::CLAMP_ERRORS) { + $additionalMessage = sprintf('%sAnd %d other exceptions, see log.', ";\n", count($this->errors) - self::CLAMP_ERRORS); + break; + } + } + return join(";\n", $lines) . $additionalMessage; + } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 43577301268..83ea4553cff 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -302,7 +302,8 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { - $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $this->logCatchupHookError($error); } } @@ -343,13 +344,15 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $this->logCatchupHookError($error); } try { $subscriber->projection->apply($domainEvent, $eventEnvelope); } catch (\Throwable $e) { // ERROR Case: + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); // for the leftover events we are not including this failed subscription for catchup @@ -365,7 +368,6 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $e ), ); - $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); continue; } // HAPPY Case: @@ -375,7 +377,8 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $this->logCatchupHookError($error); } } $numberOfProcessedEvents++; @@ -406,7 +409,8 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterBatchCompleted(); } catch (\Throwable $e) { - $errors[] = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); + $this->logCatchupHookError($error); } } @@ -423,13 +427,21 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { - $errors[] = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); + $this->logCatchupHookError($error); } } return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); } + private function logCatchupHookError(Error $error): void + { + $this->logger?->error( + sprintf('Subscription Engine: Subscription %s has error in catchup hook: %s', $error->subscriptionId->value, $error->message) + ); + } + /** * @template T * @param \Closure(): T $closure diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php index ab50707f43f..cc84bf8276d 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php @@ -5,7 +5,6 @@ namespace Neos\ContentRepository\Core\Subscription\Exception; use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFailed; -use Neos\ContentRepository\Core\Subscription\Engine\Error; use Neos\ContentRepository\Core\Subscription\Engine\Errors; /** @@ -26,22 +25,11 @@ */ final class CatchUpHadErrors extends \RuntimeException { - private const CLAMP_ERRORS = 5; - /** * @internal */ public static function createFromErrors(Errors $errors): self { - $additionalMessage = ''; - $lines = []; - foreach ($errors as $error) { - $lines[] = sprintf('"%s": %s', $error->subscriptionId->value, $error->message); - if (count($lines) >= self::CLAMP_ERRORS) { - $additionalMessage = sprintf('%sAnd %d other exceptions, see log.', ";\n", count($errors) - self::CLAMP_ERRORS); - break; - } - } - return new self(sprintf('Exception while catching up: %s%s', join(";\n", $lines), $additionalMessage), 1732132930, $errors->first()->throwable); + return new self(sprintf('Exception while catching up: %s', $errors->getClampedMessage()), 1732132930, $errors->first()->throwable); } } From 2e76381d1e4e53d3b2338b4c60150935e38c8611 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:15:29 +0100 Subject: [PATCH 137/142] TASK: Add position of error to catchup error for better debug information --- .../Subscription/CatchUpHookErrorTest.php | 82 +++++++++++++++---- .../Subscription/ProjectionErrorTest.php | 16 +++- .../Subscription/SubscriptionSetupTest.php | 7 +- .../Classes/Subscription/Engine/Error.php | 6 +- .../Classes/Subscription/Engine/Errors.php | 2 +- .../Engine/SubscriptionEngine.php | 16 ++-- .../Exception/CatchUpHadErrors.php | 2 +- .../Subscription/CatchUpHadErrorsTest.php | 54 +++++++----- 8 files changed, 137 insertions(+), 48 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php index 99229f8fd86..8dabe601728 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -62,8 +62,18 @@ public function error_onBeforeEvent_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + SequenceNumber::fromInteger(2) + ), ]) ), $result @@ -119,8 +129,18 @@ public function error_onAfterEvent_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + SequenceNumber::fromInteger(2) + ), ]) ), $result @@ -172,7 +192,12 @@ public function error_onBeforeCatchUp_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), ]) ), $result @@ -224,7 +249,12 @@ public function error_onAfterBatchCompleted_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), ]) ), $result @@ -276,7 +306,12 @@ public function error_onAfterCatchUp_isIgnoredAndCollected() ProcessedResult::failed( 2, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), ]) ), $result @@ -334,8 +369,18 @@ public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() ProcessedResult::failed( 2, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $innerException->getMessage(), $innerException), - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $innerException->getMessage(), + $innerException, + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + null + ), ]) ), $result @@ -400,7 +445,12 @@ public function error_onAfterEvent_stopsEngineAfterFirstBatch() ProcessedResult::failed( 1, Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + SequenceNumber::fromInteger(1) + ), ]) ), $result @@ -462,12 +512,14 @@ public function error_onAfterEvent_withMultipleFailingHooks() 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.', 1733243960, $firstException - ) + ), + SequenceNumber::fromInteger(1) ), Error::create( SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), 'Hook "onAfterEvent" failed: "": Second catchup hook is kaputt.', - null + null, + SequenceNumber::fromInteger(1) ), ]) ), @@ -526,12 +578,14 @@ public function error_onAfterEvent_withMultipleFailingHooksOnOneProjection() Error::create( SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $message = 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.;' . PHP_EOL . - '"": Second catchup hook is kaputt.', + '"": Second catchup hook is kaputt.', new CatchUpHookFailed( $message, 1733243960, $firstException - )), + ), + SequenceNumber::fromInteger(1) + ), ]) ), $result diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php index b0864427f88..2f2551dcb53 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -62,7 +62,12 @@ public function projectionWithErrorCanBeReactivated() ); $result = $this->subscriptionEngine->catchUpActive(); - self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception->getMessage(), $exception)])), $result); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create( + SubscriptionId::fromString('Vendor.Package:FakeProjection'), + $exception->getMessage(), + $exception, + SequenceNumber::fromInteger(1) + )])), $result); self::assertEquals( $expectedStatusForFailedProjection, @@ -337,7 +342,7 @@ public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle( $handleException = $exception; } self::assertInstanceOf(CatchUpHadErrors::class, $exception); - self::assertEquals('Exception while catching up: "Vendor.Package:FakeProjection": This projection is kaputt.', $handleException->getMessage()); + self::assertEquals('Error while catching up: Event 1 in "Vendor.Package:FakeProjection": This projection is kaputt.', $handleException->getMessage()); self::assertSame($originalException, $handleException->getPrevious()); // workspace is created. The fake projection failed on the first event, but other projections succeed: @@ -388,7 +393,12 @@ public function projectionError_stopsEngineAfterFirstBatch() ); $result = $this->subscriptionEngine->boot(batchSize: 1); - self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create(SubscriptionId::fromString('Vendor.Package:FakeProjection'), $exception->getMessage(), $exception)])), $result); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create( + SubscriptionId::fromString('Vendor.Package:FakeProjection'), + $exception->getMessage(), + $exception, + SequenceNumber::fromInteger(1) + )])), $result); self::assertEquals( $expectedStatusForFailedProjection, diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php index 614a5ccbd52..20269e80b15 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -278,7 +278,12 @@ public function failingSetupWillNotRollbackProjection() ); $result = $this->subscriptionEngine->setup(); - self::assertEquals(Errors::fromArray([Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $exception->getMessage(), $exception)]), $result->errors); + self::assertEquals(Errors::fromArray([Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $exception->getMessage(), + $exception, + null + )]), $result->errors); self::assertEquals( $expectedStatusForFailedProjection, diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php index 07cedb6e8f4..a2d025df795 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Error.php @@ -5,6 +5,7 @@ namespace Neos\ContentRepository\Core\Subscription\Engine; use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\EventStore\Model\Event\SequenceNumber; /** * @internal implementation detail of the catchup @@ -15,6 +16,7 @@ private function __construct( public SubscriptionId $subscriptionId, public string $message, public \Throwable|null $throwable, + public SequenceNumber|null $position, ) { } @@ -22,11 +24,13 @@ public static function create( SubscriptionId $subscriptionId, string $message, \Throwable|null $throwable, + SequenceNumber|null $position, ): self { return new self( $subscriptionId, $message, - $throwable + $throwable, + $position ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php index b9b576ef150..d651f85c725 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Errors.php @@ -56,7 +56,7 @@ public function getClampedMessage(): string $additionalMessage = ''; $lines = []; foreach ($this->errors as $error) { - $lines[] = sprintf('"%s": %s', $error->subscriptionId->value, $error->message); + $lines[] = sprintf('%s"%s": %s', $error->position ? 'Event ' . $error->position->value . ' in ' : '', $error->subscriptionId->value, $error->message); if (count($lines) >= self::CLAMP_ERRORS) { $additionalMessage = sprintf('%sAnd %d other exceptions, see log.', ";\n", count($this->errors) - self::CLAMP_ERRORS); break; diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 83ea4553cff..400947409d3 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -218,7 +218,7 @@ private function setupSubscription(Subscription $subscription): ?Error $subscription->position, SubscriptionError::fromPreviousStatusAndException($subscription->status, $e) ); - return Error::create($subscription->id, $e->getMessage(), $e); + return Error::create($subscription->id, $e->getMessage(), $e, null); } if ($subscription->status === SubscriptionStatus::ACTIVE) { @@ -243,7 +243,7 @@ private function resetSubscription(Subscription $subscription): ?Error $subscriber->projection->resetState(); } catch (\Throwable $e) { $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); - return Error::create($subscription->id, $e->getMessage(), $e); + return Error::create($subscription->id, $e->getMessage(), $e, null); } $this->subscriptionStore->update( $subscription->id, @@ -302,7 +302,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onBeforeCatchUp($subscription->status); } catch (\Throwable $e) { - $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, position: null); $this->logCatchupHookError($error); } } @@ -344,7 +344,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, $eventEnvelope->sequenceNumber); $this->logCatchupHookError($error); } @@ -352,7 +352,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs $subscriber->projection->apply($domainEvent, $eventEnvelope); } catch (\Throwable $e) { // ERROR Case: - $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, $eventEnvelope->sequenceNumber); $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); // for the leftover events we are not including this failed subscription for catchup @@ -377,7 +377,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); } catch (\Throwable $e) { - $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, $eventEnvelope->sequenceNumber); $this->logCatchupHookError($error); } } @@ -409,7 +409,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterBatchCompleted(); } catch (\Throwable $e) { - $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null, position: null); $this->logCatchupHookError($error); } } @@ -427,7 +427,7 @@ private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, Subs try { $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterCatchUp(); } catch (\Throwable $e) { - $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null); + $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null, position: null); $this->logCatchupHookError($error); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php index cc84bf8276d..68fe0eea6bf 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php @@ -30,6 +30,6 @@ final class CatchUpHadErrors extends \RuntimeException */ public static function createFromErrors(Errors $errors): self { - return new self(sprintf('Exception while catching up: %s', $errors->getClampedMessage()), 1732132930, $errors->first()->throwable); + return new self(sprintf('Error%s while catching up: %s', $errors->count() > 1 ? 's' : '', $errors->getClampedMessage()), 1732132930, $errors->first()->throwable); } } diff --git a/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php b/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php index 0e9aed206cb..36f2e968e8f 100644 --- a/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php +++ b/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php @@ -9,6 +9,7 @@ use Neos\ContentRepository\Core\Subscription\Engine\Errors; use Neos\ContentRepository\Core\Subscription\Exception\CatchUpHadErrors; use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\EventStore\Model\Event\SequenceNumber; use PHPUnit\Framework\TestCase; class CatchUpHadErrorsTest extends TestCase @@ -24,13 +25,18 @@ public function testSimple() ); $errors = Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), ]); // assert shape if the error had been thrown by the cr: $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); self::assertEquals($expectedWrappedException, $catchupHadErrors->getPrevious()); - self::assertEquals('Exception while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeCatchup" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); + self::assertEquals('Error while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeCatchup" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); } public function testWithTwoHookErrors() @@ -44,14 +50,24 @@ public function testWithTwoHookErrors() ); $errors = Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), $expectedWrappedException), - Error::create(SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), $expectedWrappedException->getMessage(), null), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + null + ), ]); // assert shape if the error had been thrown by the cr: $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); self::assertEquals($expectedWrappedException, $catchupHadErrors->getPrevious()); - self::assertEquals('Exception while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.; + self::assertEquals('Errors while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.; "Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); } @@ -59,26 +75,26 @@ public function testWith10Errors() { $exception = new \RuntimeException('Message why A failed'); $errors = Errors::fromArray([ - Error::create(SubscriptionId::fromString('Vendor.Package:A'), $exception->getMessage(), $exception), - Error::create(SubscriptionId::fromString('Vendor.Package:B'), 'Message why B failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:C'), 'Message why C failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:D'), 'Message why D failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:E'), 'Message why E failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:F'), 'Message why F failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:G'), 'Message why G failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:H'), 'Message why H failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:I'), 'Message why I failed', null), - Error::create(SubscriptionId::fromString('Vendor.Package:J'), 'Message why J failed', null), + Error::create(SubscriptionId::fromString('Vendor.Package:A'), $exception->getMessage(), $exception, null), + Error::create(SubscriptionId::fromString('Vendor.Package:B'), 'Message why B failed', null, SequenceNumber::fromInteger(1)), + Error::create(SubscriptionId::fromString('Vendor.Package:C'), 'Message why C failed', null, SequenceNumber::fromInteger(1)), + Error::create(SubscriptionId::fromString('Vendor.Package:D'), 'Message why D failed', null, SequenceNumber::fromInteger(3)), + Error::create(SubscriptionId::fromString('Vendor.Package:E'), 'Message why E failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:F'), 'Message why F failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:G'), 'Message why G failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:H'), 'Message why H failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:I'), 'Message why I failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:J'), 'Message why J failed', null, null), ]); // assert shape if the error had been thrown by the cr: $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); self::assertEquals($exception, $catchupHadErrors->getPrevious()); self::assertEquals(<<getMessage()); From cd384e517a4b17e11dd9367d889318662d18fc26 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:19:31 +0100 Subject: [PATCH 138/142] TASK: Credit patchlevel in SubscriptionEngine --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 400947409d3..0d4e22a24b2 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -30,6 +30,9 @@ * All functionality is low level and well encapsulated and abstracted by the {@see ContentRepositoryMaintainer} * It presents the only API way to interact with catchup and offers more maintenance tasks. * + * This implementation is heavily inspired and adjusted from the event-sourcing package of "patchlevel": + * {@link https://github.com/patchlevel/event-sourcing/} + * * @internal implementation detail of the catchup. See {@see ContentRepository::handle()} and {@see ContentRepositoryMaintainer} */ final class SubscriptionEngine From d53c361071bb3c0d4087aec308e52a89e1873749 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:31:45 +0100 Subject: [PATCH 139/142] TASK: Adjust commitOnConnection_onAfterEvent test to batching without batching we had invalid state because the position is none (0) while the event is applied and persisted an the projection. --- .../CatchUpHookWithPersistenceTest.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php index 56e0906bb44..5acdb583039 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php @@ -51,17 +51,21 @@ public function commitOnConnection_onAfterEvent() $actualException = null; try { - $this->subscriptionEngine->catchUpActive(); + $this->subscriptionEngine->catchUpActive(batchSize: 1); } catch (\Throwable $e) { $actualException = $e; } + // To solve this we would need to use an own connection for all CORE cr parts. self::assertInstanceOf(\Doctrine\DBAL\ConnectionException::class, $actualException); + self::assertEquals('There is no active transaction.', $actualException->getMessage()); - // todo invalid state, use own connection for cr?! - $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + self::assertFalse($this->getObject(Connection::class)->isTransactionActive()); + + // partially applied event because the error is thrown at the end and the projection is not rolled back + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); self::assertEquals( - [SequenceNumber::fromInteger(1)], - $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + [1], + $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues() ); } From 24890b9d95731094ae806b0a27a7615b67baa2c4 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:37:54 +0100 Subject: [PATCH 140/142] TASK: Ensure status is serialized by value as we use SubscriptionStatus::from --- .../Subscription/Engine/SubscriptionEngine.php | 2 +- .../Classes/Subscription/SubscriptionStatus.php | 15 +++++++++++++++ .../DoctrineSubscriptionStore.php | 8 ++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index 0d4e22a24b2..d78df391057 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -235,7 +235,7 @@ private function setupSubscription(Subscription $subscription): ?Error null ); } - $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->name, $subscription->status->name)); + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->value, $subscription->status->name)); return null; } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php index 487f742fbab..1af1aa4f1a9 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -9,9 +9,24 @@ */ enum SubscriptionStatus : string { + /** + * New subscribers e.g a newly installed package: will not be run on catchup active as it doesn't have its schema setup + */ case NEW = 'NEW'; + /** + * Subscriber was set up and can be catch-up via boot, but will not run on active + */ case BOOTING = 'BOOTING'; + /** + * Active subscribers will always be run if new events are commited + */ case ACTIVE = 'ACTIVE'; + /** + * Subscribers that are uninstalled will be detached and have to be reactivated + */ case DETACHED = 'DETACHED'; + /** + * Subscribers that are run into an error during catchup or setup + */ case ERROR = 'ERROR'; } diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php index 3d578afed52..525f9bd2639 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -118,10 +118,10 @@ public function update( ): void { $row = []; $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); - $row['status'] = $status->name; + $row['status'] = $status->value; $row['position'] = $position->value; $row['error_message'] = $subscriptionError?->errorMessage; - $row['error_previous_status'] = $subscriptionError?->previousStatus?->name; + $row['error_previous_status'] = $subscriptionError?->previousStatus?->value; $row['error_trace'] = $subscriptionError?->errorTrace; $this->dbal->update( $this->tableName, @@ -138,10 +138,10 @@ public function update( private static function toDatabase(Subscription $subscription): array { return [ - 'status' => $subscription->status->name, + 'status' => $subscription->status->value, 'position' => $subscription->position->value, 'error_message' => $subscription->error?->errorMessage, - 'error_previous_status' => $subscription->error?->previousStatus?->name, + 'error_previous_status' => $subscription->error?->previousStatus?->value, 'error_trace' => $subscription->error?->errorTrace, ]; } From ca7990730af37ba105d97527305f0c776bc89b86 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:41:29 +0100 Subject: [PATCH 141/142] BUGFIX: Ensure that replay does not reset new or detached projections --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index d78df391057..bca537c3de3 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -112,6 +112,11 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result } $errors = []; foreach ($subscriptions as $subscription) { + if ($subscription->status === SubscriptionStatus::NEW + || !$this->subscribers->contain($subscription->id)) { + // todo test this case! And mark projections as detached? + continue; + } $error = $this->resetSubscription($subscription); if ($error !== null) { $errors[] = $error; From dc972320bf3bf0b0bd5321d3b3be0f0639397d8a Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:08:52 +0100 Subject: [PATCH 142/142] TASK: Make phpcs happy --- .../Classes/Subscription/Engine/SubscriptionEngine.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php index bca537c3de3..14656eff243 100644 --- a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngine.php @@ -112,8 +112,10 @@ public function reset(SubscriptionEngineCriteria|null $criteria = null): Result } $errors = []; foreach ($subscriptions as $subscription) { - if ($subscription->status === SubscriptionStatus::NEW - || !$this->subscribers->contain($subscription->id)) { + if ( + $subscription->status === SubscriptionStatus::NEW + || !$this->subscribers->contain($subscription->id) + ) { // todo test this case! And mark projections as detached? continue; }