From 36c73b5336c49ac7ee1a042aa468757ffb3b2a52 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sat, 5 May 2018 20:48:37 -0400 Subject: [PATCH 1/8] Adding improved synchronization process --- Annotations/SyncParent.php | 17 +++++ EventListener/DoctrineEventSubscriber.php | 93 ++++++++++------------- Interfaces/SyncEntityInterface.php | 6 ++ Service/SyncService.php | 2 +- 4 files changed, 63 insertions(+), 55 deletions(-) create mode 100755 Annotations/SyncParent.php create mode 100755 Interfaces/SyncEntityInterface.php diff --git a/Annotations/SyncParent.php b/Annotations/SyncParent.php new file mode 100755 index 0000000..1886d34 --- /dev/null +++ b/Annotations/SyncParent.php @@ -0,0 +1,17 @@ +container = $container; $this->syncService = $this->container->get('nti.sync'); } @@ -28,72 +34,51 @@ public function getSubscribedEvents() public function onFlush(OnFlushEventArgs $args) { + $em = $args->getEntityManager(); $uow = $em->getUnitOfWork(); - $somethingChanged = false; - - $identityMap = $uow->getIdentityMap(); - - // ehhehehehee LOL - $deletedEntities = count($uow->getScheduledEntityDeletions()) > 0; - $insertedEntities = count($uow->getScheduledEntityInsertions()) > 0; - - foreach($identityMap as $map) { - foreach($map as $object) { - $changes = $uow->getEntityChangeSet($object); - if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]))) { - $somethingChanged = true; - break; - } - } - - if($somethingChanged) { - break; + foreach ($uow->getScheduledEntityUpdates() as $entity) { + // Check if the entity should be synchronized + if (!($entity instanceof SyncEntityInterface)) { + continue; } + $this->processEntity($em, $entity); } - if($deletedEntities || $insertedEntities) - $somethingChanged = true; - - foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { - $changes = $uow->getEntityChangeSet($entity); - - if(count($changes) == 1 && isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]) && !$somethingChanged) { - $oid = spl_object_hash($entity); - $uow->clearEntityChangeSet($oid); - } else { - $this->handleEntityChange($em, $entity); - } - } + } - foreach ($uow->getScheduledEntityInsertions() as $keyEntity => $entity) { - $this->handleEntityChange($em, $entity); - } + public function preRemove(LifecycleEventArgs $args) + { } - public function preRemove(LifecycleEventArgs $args) { - $entity = $args->getEntity(); - $class = get_class($entity); - $id = null; + private function processEntity(EntityManagerInterface $em, $entity) + { + $uow = $em->getUnitOfWork(); - if(method_exists($entity, 'getId')) { - $id = $entity->getId(); - } + $timestamp = time(); - $this->syncService->addToDeleteSyncState($class, $id); - } + // Check if this class itself has a lastTimestamp + if(method_exists($entity, 'setLastTimestamp')) { + $entity->setLastTimestamp($timestamp); + $uow->computeChangeSet($em->getClassMetadata(get_class($entity)), $entity); + } - private function handleEntityChange(EntityManagerInterface $em, $entity) { - if(method_exists($entity, 'getLastTimestamp')) { - $timestamp = $entity->getLastTimestamp() ?? time(); - } else { - $timestamp = time(); + // Check if there are any relationships that should notified + $annotationReader = new AnnotationReader(); + $reflection = new \ReflectionClass(get_class($entity)); + + /** @var \ReflectionProperty $property */ + foreach ($reflection->getProperties() as $property) { + /** @var SyncParent $annotation */ + if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { + $getter = $annotation->getter; + $parent = $entity->$getter(); + if($parent instanceof SyncEntityInterface) { + $this->processEntity($em, $parent); + } + } } - $class = get_class($entity); - $this->syncService->updateSyncState($em, $class, $timestamp); } - - } diff --git a/Interfaces/SyncEntityInterface.php b/Interfaces/SyncEntityInterface.php new file mode 100755 index 0000000..3b54861 --- /dev/null +++ b/Interfaces/SyncEntityInterface.php @@ -0,0 +1,6 @@ + $result["data"], 'deletes' => json_decode($this->container->get('jms_serializer')->serialize($deletes, 'json'), true), 'newItems' => json_decode($this->container->get('jms_serializer')->serialize($newItems, 'json'), true), - 'failedItems' => json_decode($this->container->get('jms_serializer')->serialize($failedItems, 'json'), true), + 'failedItems' => json_decode($this->container->get('jms_serializer')->serialize($failedItems, 'json'), true), SyncState::REAL_LAST_TIMESTAMP => $result[SyncState::REAL_LAST_TIMESTAMP], ); } From cf38cabba40aae7ff7469221009ff3f1f0c5f822 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Tue, 8 May 2018 06:20:24 -0400 Subject: [PATCH 2/8] Working with deleted entities --- Annotations/SyncEntity.php | 13 ++ Entity/SyncFailedItemState.php | 2 +- Entity/SyncMapping.php | 2 +- EventListener/DoctrineEventSubscriber.php | 84 ------------- EventSubscriber/DoctrineEventSubscriber.php | 125 ++++++++++++++++++++ Interfaces/SyncEntityInterface.php | 6 - Resources/config/services.yml | 2 +- Service/SyncService.php | 38 +----- 8 files changed, 144 insertions(+), 128 deletions(-) create mode 100755 Annotations/SyncEntity.php mode change 100644 => 100755 Entity/SyncFailedItemState.php mode change 100644 => 100755 Entity/SyncMapping.php delete mode 100755 EventListener/DoctrineEventSubscriber.php create mode 100755 EventSubscriber/DoctrineEventSubscriber.php delete mode 100755 Interfaces/SyncEntityInterface.php diff --git a/Annotations/SyncEntity.php b/Annotations/SyncEntity.php new file mode 100755 index 0000000..1786a19 --- /dev/null +++ b/Annotations/SyncEntity.php @@ -0,0 +1,13 @@ +container = $container; - $this->syncService = $this->container->get('nti.sync'); - } - - public function getSubscribedEvents() - { - return array( - 'onFlush', - 'preRemove', - ); - } - - public function onFlush(OnFlushEventArgs $args) - { - - $em = $args->getEntityManager(); - $uow = $em->getUnitOfWork(); - - foreach ($uow->getScheduledEntityUpdates() as $entity) { - // Check if the entity should be synchronized - if (!($entity instanceof SyncEntityInterface)) { - continue; - } - $this->processEntity($em, $entity); - } - - } - - public function preRemove(LifecycleEventArgs $args) - { - - } - - private function processEntity(EntityManagerInterface $em, $entity) - { - $uow = $em->getUnitOfWork(); - - $timestamp = time(); - - // Check if this class itself has a lastTimestamp - if(method_exists($entity, 'setLastTimestamp')) { - $entity->setLastTimestamp($timestamp); - $uow->computeChangeSet($em->getClassMetadata(get_class($entity)), $entity); - } - - // Check if there are any relationships that should notified - $annotationReader = new AnnotationReader(); - $reflection = new \ReflectionClass(get_class($entity)); - - /** @var \ReflectionProperty $property */ - foreach ($reflection->getProperties() as $property) { - /** @var SyncParent $annotation */ - if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { - $getter = $annotation->getter; - $parent = $entity->$getter(); - if($parent instanceof SyncEntityInterface) { - $this->processEntity($em, $parent); - } - } - } - } -} diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php new file mode 100755 index 0000000..0f58359 --- /dev/null +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -0,0 +1,125 @@ +container = $container; + $this->syncService = $this->container->get('nti.sync'); + } + + public function getSubscribedEvents() + { + return array( + 'onFlush', + ); + } + + public function onFlush(OnFlushEventArgs $args) + { + + $em = $args->getEntityManager(); + $uow = $em->getUnitOfWork(); + + foreach ($uow->getScheduledEntityUpdates() as $entity) { + $this->processEntity($em, $entity); + } + + foreach ($uow->getScheduledEntityInsertions() as $entity) { + $this->processEntity($em, $entity); + } + + foreach ($uow->getScheduledEntityDeletions() as $entity) { + $this->processEntity($em, $entity); + $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); + } + + /** @var PersistentCollection $collectionUpdate */ + foreach ($uow->getScheduledCollectionUpdates() as $collectionUpdate) { + foreach($collectionUpdate as $entity) { + $this->processEntity($em, $entity); + } + } + + /** @var PersistentCollection $collectionDeletion */ + foreach($uow->getScheduledCollectionDeletions() as $collectionDeletion) { + foreach($collectionDeletion as $entity) { + $this->processEntity($em, $entity); + $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); + } + } + + } + + private function processEntity(EntityManagerInterface $em, $entity) + { + + $reflection = new \ReflectionClass(ClassUtils::getClass($entity)); + $annotationReader = new AnnotationReader(); + $syncEntityAnnotation = $annotationReader->getClassAnnotation($reflection, SyncEntity::class); + // Check if the entity should be synchronized + if (!$syncEntityAnnotation) { + return; + } + + $uow = $em->getUnitOfWork(); + $timestamp = time(); + + // Update the mapping's sync state if exists + $mapping = $em->getRepository(SyncMapping::class)->findOneBy(array("class" => get_class($entity))); + if($mapping) { + $syncState = $em->getRepository(SyncState::class)->findOneBy(array("mapping" => $mapping)); + if(!$syncState) { + $syncState = new SyncState(); + $syncState->setMapping($mapping); + $em->persist($syncState); + } + $syncState->setTimestamp($timestamp); + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + } + + // Check if this class itself has a lastTimestamp + if(method_exists($entity, 'setLastTimestamp')) { + $entity->setLastTimestamp($timestamp); + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(get_class($entity)), $entity); + } + + // Notify relationships + /** @var \ReflectionProperty $property */ + foreach ($reflection->getProperties() as $property) { + + /** @var SyncParent $annotation */ + if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { + $getter = $annotation->getter; + $parent = $entity->$getter(); + dump($parent); + die; + // Using ClassUtils as $parent is actually a Proxy of the class + $reflrectionParent = new \ReflectionClass(ClassUtils::getClass($parent)); + $syncParentAnnotation = $annotationReader->getClassAnnotation($reflrectionParent, SyncEntity::class); + if(!$syncParentAnnotation) { + continue; + } + $this->processEntity($em, $parent); + } + } + } +} diff --git a/Interfaces/SyncEntityInterface.php b/Interfaces/SyncEntityInterface.php deleted file mode 100755 index 3b54861..0000000 --- a/Interfaces/SyncEntityInterface.php +++ /dev/null @@ -1,6 +0,0 @@ -em->getRepository(SyncDeleteState::class)->findFromTimestamp($mappingName, $timestamp); $newItems = $this->em->getRepository(SyncNewItemState::class)->findFromTimestampAndMapping($mappingName, $timestamp); - /** - * Failed Items Synchronization - */ - $failedItems = $this->em->getRepository('NTISyncBundle:SyncFailedItemState')->findFromTimestampAndMapping($mappingName, $timestamp); + $failedItems = $this->em->getRepository(SyncFailedItemState::class)->findFromTimestampAndMapping($mappingName, $timestamp); /** @var SyncRepositoryInterface $repository */ $repository = $this->em->getRepository($syncMapping->getClass()); @@ -88,30 +86,6 @@ public function getFromMappings($mappings) { return $changes; } - public function updateSyncState(EntityManagerInterface $em, $class, $timestamp) { - - $mapping = $em->getRepository(SyncMapping::class)->findOneBy(array("class" => $class)); - if(!$mapping) { - return; - } - - $syncState = $em->getRepository(SyncState::class)->findOneBy(array("mapping" => $mapping)); - - $uow = $em->getUnitOfWork(); - - if(!$syncState) { - $syncState = new SyncState(); - $syncState->setMapping($mapping); - $syncState->setTimestamp($timestamp); - $em->persist($syncState); - $uow->computeChangeSet($em->getClassMetadata(SyncState::class), $syncState); - } else { - $syncState->setTimestamp($timestamp); - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); - } - - } - /** * Create a new SyncDeleteState for the given class/id * @@ -122,6 +96,7 @@ public function addToDeleteSyncState($class, $id) { $this->em = $this->container->get('doctrine')->getManager(); + /** @var SyncMapping $mapping */ $mapping = $this->em->getRepository(SyncMapping::class)->findOneBy(array("class" => $class)); if(!$mapping) { return; @@ -133,12 +108,5 @@ public function addToDeleteSyncState($class, $id) { $deleteEntry->setTimestamp(time()); $this->em->persist($deleteEntry); - - try { - $this->em->flush(); - } catch (\Exception $ex) { - error_log("Unable to register deletion of object: " . $class . " with ID " . $id); - error_log($ex->getMessage()); - } } } From 112de243327288761ba53c4ec9fbf6b306ec3c65 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Thu, 10 May 2018 10:07:27 -0400 Subject: [PATCH 3/8] Removing dump --- EventSubscriber/DoctrineEventSubscriber.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index 0f58359..b8f2865 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -110,8 +110,6 @@ private function processEntity(EntityManagerInterface $em, $entity) if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { $getter = $annotation->getter; $parent = $entity->$getter(); - dump($parent); - die; // Using ClassUtils as $parent is actually a Proxy of the class $reflrectionParent = new \ReflectionClass(ClassUtils::getClass($parent)); $syncParentAnnotation = $annotationReader->getClassAnnotation($reflrectionParent, SyncEntity::class); From 3694219e7209a32e7737777374101bfcf86bc053 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Mon, 21 May 2018 09:11:19 -0400 Subject: [PATCH 4/8] Changing get_class to avoid proxies --- .gitignore | 1 + .idea/SyncBundle.iml | 10 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 196 ++++++++++++++++++++ EventSubscriber/DoctrineEventSubscriber.php | 4 +- 7 files changed, 229 insertions(+), 2 deletions(-) create mode 100755 .idea/SyncBundle.iml create mode 100755 .idea/misc.xml create mode 100755 .idea/modules.xml create mode 100755 .idea/vcs.xml create mode 100755 .idea/workspace.xml diff --git a/.gitignore b/.gitignore index 3a2a5ba..a6a6a28 100755 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ # Backup entities generated with doctrine:generate:entities command **/Entity/*~ +./.idea diff --git a/.idea/SyncBundle.iml b/.idea/SyncBundle.iml new file mode 100755 index 0000000..940f6f9 --- /dev/null +++ b/.idea/SyncBundle.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100755 index 0000000..28a804d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100755 index 0000000..549aacb --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100755 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100755 index 0000000..5661775 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + $PROJECT_DIR$/composer.json + + + + + + + + + + + + + + + + + + + + + + + + true + DEFINITION_ORDER + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - 1523390869087 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 0548505809513d18d13829c919fe6048980b63cf Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sun, 27 May 2018 19:29:14 -0400 Subject: [PATCH 6/8] Added computeChangeSet to deleted items --- Service/SyncService.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Service/SyncService.php b/Service/SyncService.php index 7a7129e..6d55d43 100755 --- a/Service/SyncService.php +++ b/Service/SyncService.php @@ -108,5 +108,7 @@ public function addToDeleteSyncState($class, $id) { $deleteEntry->setTimestamp(time()); $this->em->persist($deleteEntry); + $uow = $this->em->getUnitOfWork(); + $uow->computeChangeSet($this->em->getClassMetadata(SyncDeleteState::class), $deleteEntry); } } From fdd26decb6fd7be0c62bbd190e620378a1021567 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sun, 27 May 2018 19:58:55 -0400 Subject: [PATCH 7/8] Added entity state verification --- EventSubscriber/DoctrineEventSubscriber.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index 8e2dac6..ae38a46 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -9,6 +9,7 @@ use Doctrine\ORM\Event\LifecycleEventArgs; use Doctrine\ORM\Event\OnFlushEventArgs; use Doctrine\ORM\PersistentCollection; +use Doctrine\ORM\UnitOfWork; use NTI\SyncBundle\Annotations\SyncEntity; use NTI\SyncBundle\Annotations\SyncParent; use NTI\SyncBundle\Entity\SyncMapping; @@ -93,13 +94,17 @@ private function processEntity(EntityManagerInterface $em, $entity) $em->persist($syncState); } $syncState->setTimestamp($timestamp); - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + if($uow->getEntityState($syncState) == UnitOfWork::STATE_MANAGED) { + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + } } // Check if this class itself has a lastTimestamp if(method_exists($entity, 'setLastTimestamp')) { $entity->setLastTimestamp($timestamp); - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); + if($uow->getEntityState($entity) == UnitOfWork::STATE_MANAGED) { + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); + } } // Notify relationships From a992791c02eb273d08c2c7da1b05d86030f0eab9 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Wed, 27 Jun 2018 16:41:32 +0000 Subject: [PATCH 8/8] Added fix for bug when deleting entities --- EventSubscriber/DoctrineEventSubscriber.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index ae38a46..c06d84d 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -49,8 +49,9 @@ public function onFlush(OnFlushEventArgs $args) } foreach ($uow->getScheduledEntityDeletions() as $entity) { - $this->processEntity($em, $entity); + $this->processEntity($em, $entity, true); $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); + } /** @var PersistentCollection $collectionUpdate */ @@ -63,14 +64,14 @@ public function onFlush(OnFlushEventArgs $args) /** @var PersistentCollection $collectionDeletion */ foreach($uow->getScheduledCollectionDeletions() as $collectionDeletion) { foreach($collectionDeletion as $entity) { - $this->processEntity($em, $entity); + $this->processEntity($em, $entity, true); $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); } } } - private function processEntity(EntityManagerInterface $em, $entity) + private function processEntity(EntityManagerInterface $em, $entity, $deleting = false) { $reflection = new \ReflectionClass(ClassUtils::getClass($entity)); @@ -100,11 +101,9 @@ private function processEntity(EntityManagerInterface $em, $entity) } // Check if this class itself has a lastTimestamp - if(method_exists($entity, 'setLastTimestamp')) { + if(!$deleting && method_exists($entity, 'setLastTimestamp')) { $entity->setLastTimestamp($timestamp); - if($uow->getEntityState($entity) == UnitOfWork::STATE_MANAGED) { - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); - } + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); } // Notify relationships