From 781eb49db908fd6d51cb6fc0ff1fad1623b69dfe Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Fri, 16 Mar 2018 10:04:32 -0400 Subject: [PATCH 01/23] Added fix for entities that just change their last timestamp --- EventListener/DoctrineEventSubscriber.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index 8700e95..9aadd06 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -32,6 +32,13 @@ public function onFlush(OnFlushEventArgs $args) $uow = $em->getUnitOfWork(); foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { + + $changes = $uow->getEntityChangeSet($entity); + + if(count($changes) == 1 && isset($changes["lastTimestamp"])) { + $uow->detach($entity); + continue; + } $this->handleEntityChange($em, $entity); } @@ -64,4 +71,4 @@ private function handleEntityChange(EntityManagerInterface $em, $entity) { $this->syncService->updateSyncState($em, $class, $timestamp); } -} \ No newline at end of file +} From ae8d4f82f5e3417e7638f324dd3613cb3e7db4b2 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sun, 18 Mar 2018 21:17:17 -0400 Subject: [PATCH 02/23] Fixed entity manager persisting invoice after detaching --- EventListener/DoctrineEventSubscriber.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index 9aadd06..a2acc12 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -36,7 +36,8 @@ public function onFlush(OnFlushEventArgs $args) $changes = $uow->getEntityChangeSet($entity); if(count($changes) == 1 && isset($changes["lastTimestamp"])) { - $uow->detach($entity); + $oid = spl_object_hash($entity); + $uow->clearEntityChangeSet($oid); continue; } $this->handleEntityChange($em, $entity); From a73c8f7fb27481f5d260a30bdb9e570b560533cc Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Thu, 29 Mar 2018 14:10:09 -0400 Subject: [PATCH 03/23] Adding temporary fix to handle child changes --- EventListener/DoctrineEventSubscriber.php | 50 ++++++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index a2acc12..29ec183 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -26,19 +26,54 @@ public function getSubscribedEvents() ); } + public function getChangedOids($entities, &$oids) { + + foreach($entities as $entity) { + + $reflection = new \ReflectionObject($entity); + $methods = $reflection->getMethods(); + + foreach($methods as $method) { + + // Getter must not have a parameter!! + if(strpos($method->getName(), "get") !== false && count($method->getParameters()) <= 0) { + $result = $method->invoke($entity); + if(is_object($result)) { + $reflection2 = new \ReflectionClass(get_class($result)); + $annotations = $reflection2->getDocComment(); + if(strpos($annotations, '@ORM\Entity') !== false) { + $oid = spl_object_hash($result); + if(method_exists($result, 'setLastTimestamp')) { + $oids[] = $oid; + } + $this->getChangedOids(array($result), $oids); + } + } + } + } + } + } + public function onFlush(OnFlushEventArgs $args) { $em = $args->getEntityManager(); $uow = $em->getUnitOfWork(); + + $oids = array(); + $this->getChangedOids($uow->getScheduledEntityUpdates(), $oids); + foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { $changes = $uow->getEntityChangeSet($entity); - if(count($changes) == 1 && isset($changes["lastTimestamp"])) { + if (count($changes) == 1 && isset($changes["lastTimestamp"])) { $oid = spl_object_hash($entity); - $uow->clearEntityChangeSet($oid); - continue; + if(!in_array($oid, $oids)) { +// dump(get_class($entity)." is not really chaning anything..."); + $uow->clearEntityChangeSet($oid); + continue; + } } $this->handleEntityChange($em, $entity); } @@ -46,11 +81,13 @@ public function onFlush(OnFlushEventArgs $args) foreach ($uow->getScheduledEntityInsertions() as $keyEntity => $entity) { $this->handleEntityChange($em, $entity); } +// +// dump($oids); +// dump("Finished"); +// die; } - public function preRemove(LifecycleEventArgs $args) - { - + public function preRemove(LifecycleEventArgs $args) { $entity = $args->getEntity(); $class = get_class($entity); $id = null; @@ -72,4 +109,5 @@ private function handleEntityChange(EntityManagerInterface $em, $entity) { $this->syncService->updateSyncState($em, $class, $timestamp); } + } From a9c4d511cff162d82a30934a7bea35ef588f4a45 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Mon, 2 Apr 2018 11:40:45 -0400 Subject: [PATCH 04/23] Fixing issue that doesn't track relationship changes --- EventListener/DoctrineEventSubscriber.php | 65 ++++++++--------------- 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index 29ec183..e1b8577 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -25,66 +25,45 @@ public function getSubscribedEvents() 'preRemove', ); } - - public function getChangedOids($entities, &$oids) { - - foreach($entities as $entity) { - - $reflection = new \ReflectionObject($entity); - $methods = $reflection->getMethods(); - - foreach($methods as $method) { - - // Getter must not have a parameter!! - if(strpos($method->getName(), "get") !== false && count($method->getParameters()) <= 0) { - $result = $method->invoke($entity); - if(is_object($result)) { - $reflection2 = new \ReflectionClass(get_class($result)); - $annotations = $reflection2->getDocComment(); - if(strpos($annotations, '@ORM\Entity') !== false) { - $oid = spl_object_hash($result); - if(method_exists($result, 'setLastTimestamp')) { - $oids[] = $oid; - } - $this->getChangedOids(array($result), $oids); - } - } - } - } - } - } - + public function onFlush(OnFlushEventArgs $args) { $em = $args->getEntityManager(); $uow = $em->getUnitOfWork(); + $somethingChanged = false; - $oids = array(); - $this->getChangedOids($uow->getScheduledEntityUpdates(), $oids); + $identityMap = $uow->getIdentityMap(); - foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { + foreach($identityMap as $map) { + foreach($map as $object) { + $changes = $uow->getEntityChangeSet($object); + if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]))) { + $somethingChanged = true; + break; + } + } + + if($somethingChanged) { + break; + } + } + foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { $changes = $uow->getEntityChangeSet($entity); - if (count($changes) == 1 && isset($changes["lastTimestamp"])) { + if(count($changes) == 1 && isset($changes["lastTimestamp"]) && !$somethingChanged) { $oid = spl_object_hash($entity); - if(!in_array($oid, $oids)) { -// dump(get_class($entity)." is not really chaning anything..."); - $uow->clearEntityChangeSet($oid); - continue; - } + $uow->clearEntityChangeSet($oid); + } else { + $this->handleEntityChange($em, $entity); } - $this->handleEntityChange($em, $entity); } foreach ($uow->getScheduledEntityInsertions() as $keyEntity => $entity) { $this->handleEntityChange($em, $entity); } -// -// dump($oids); -// dump("Finished"); -// die; + } public function preRemove(LifecycleEventArgs $args) { From d14f79b97921d3f7fe18d6e3775bb52dd4b801df Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Tue, 10 Apr 2018 09:10:28 -0400 Subject: [PATCH 05/23] Fixing temporarly collection changes sync state not updating --- EventListener/DoctrineEventSubscriber.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index e1b8577..3a44566 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -25,7 +25,7 @@ public function getSubscribedEvents() 'preRemove', ); } - + public function onFlush(OnFlushEventArgs $args) { $em = $args->getEntityManager(); @@ -35,6 +35,10 @@ public function onFlush(OnFlushEventArgs $args) $identityMap = $uow->getIdentityMap(); + // ehhehehehee LOL + $deletedEntities = count($uow->getScheduledEntityDeletions()) > 0; + $insertedEntities = count($uow->getScheduledEntityInsertions()) > 0; + foreach($identityMap as $map) { foreach($map as $object) { $changes = $uow->getEntityChangeSet($object); @@ -49,6 +53,9 @@ public function onFlush(OnFlushEventArgs $args) } } + if($deletedEntities || $insertedEntities) + $somethingChanged = true; + foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { $changes = $uow->getEntityChangeSet($entity); @@ -89,4 +96,4 @@ private function handleEntityChange(EntityManagerInterface $em, $entity) { } -} +} \ No newline at end of file From c2e44045f429f53f871a7717fbeeb456232f4367 Mon Sep 17 00:00:00 2001 From: ntidev Date: Fri, 4 May 2018 12:08:44 -0400 Subject: [PATCH 06/23] Update DoctrineEventSubscriber.php Temporarily adding the "lastLogin", this suscriber will have to be redone with serialization comparison instead of entity changesets as Doctrine is not very reliable detecting changes... --- EventListener/DoctrineEventSubscriber.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index 3a44566..2b274bb 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -42,7 +42,7 @@ public function onFlush(OnFlushEventArgs $args) foreach($identityMap as $map) { foreach($map as $object) { $changes = $uow->getEntityChangeSet($object); - if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]))) { + if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]))) { $somethingChanged = true; break; } @@ -59,7 +59,7 @@ public function onFlush(OnFlushEventArgs $args) foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { $changes = $uow->getEntityChangeSet($entity); - if(count($changes) == 1 && isset($changes["lastTimestamp"]) && !$somethingChanged) { + if(count($changes) == 1 && isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]) && !$somethingChanged) { $oid = spl_object_hash($entity); $uow->clearEntityChangeSet($oid); } else { @@ -96,4 +96,4 @@ private function handleEntityChange(EntityManagerInterface $em, $entity) { } -} \ No newline at end of file +} From 834ee643a000df574270713a28fd7d8486148cb6 Mon Sep 17 00:00:00 2001 From: ntidev Date: Fri, 4 May 2018 12:16:38 -0400 Subject: [PATCH 07/23] Update DoctrineEventSubscriber.php Typo fix --- EventListener/DoctrineEventSubscriber.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index 2b274bb..d5f126c 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -42,7 +42,7 @@ public function onFlush(OnFlushEventArgs $args) foreach($identityMap as $map) { foreach($map as $object) { $changes = $uow->getEntityChangeSet($object); - if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]))) { + if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"])) { $somethingChanged = true; break; } From d5407158576d701c63dfa654d471fbc1138d66e6 Mon Sep 17 00:00:00 2001 From: ntidev Date: Fri, 4 May 2018 12:58:45 -0400 Subject: [PATCH 08/23] Update DoctrineEventSubscriber.php --- EventListener/DoctrineEventSubscriber.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EventListener/DoctrineEventSubscriber.php b/EventListener/DoctrineEventSubscriber.php index d5f126c..2b274bb 100755 --- a/EventListener/DoctrineEventSubscriber.php +++ b/EventListener/DoctrineEventSubscriber.php @@ -42,7 +42,7 @@ public function onFlush(OnFlushEventArgs $args) foreach($identityMap as $map) { foreach($map as $object) { $changes = $uow->getEntityChangeSet($object); - if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"])) { + if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]))) { $somethingChanged = true; break; } From 36c73b5336c49ac7ee1a042aa468757ffb3b2a52 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sat, 5 May 2018 20:48:37 -0400 Subject: [PATCH 09/23] Adding improved synchronization process --- Annotations/SyncParent.php | 17 +++++ EventListener/DoctrineEventSubscriber.php | 93 ++++++++++------------- Interfaces/SyncEntityInterface.php | 6 ++ Service/SyncService.php | 2 +- 4 files changed, 63 insertions(+), 55 deletions(-) create mode 100755 Annotations/SyncParent.php create mode 100755 Interfaces/SyncEntityInterface.php diff --git a/Annotations/SyncParent.php b/Annotations/SyncParent.php new file mode 100755 index 0000000..1886d34 --- /dev/null +++ b/Annotations/SyncParent.php @@ -0,0 +1,17 @@ +container = $container; $this->syncService = $this->container->get('nti.sync'); } @@ -28,72 +34,51 @@ public function getSubscribedEvents() public function onFlush(OnFlushEventArgs $args) { + $em = $args->getEntityManager(); $uow = $em->getUnitOfWork(); - $somethingChanged = false; - - $identityMap = $uow->getIdentityMap(); - - // ehhehehehee LOL - $deletedEntities = count($uow->getScheduledEntityDeletions()) > 0; - $insertedEntities = count($uow->getScheduledEntityInsertions()) > 0; - - foreach($identityMap as $map) { - foreach($map as $object) { - $changes = $uow->getEntityChangeSet($object); - if(count($changes) > 1 || (count($changes) > 0 && !isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]))) { - $somethingChanged = true; - break; - } - } - - if($somethingChanged) { - break; + foreach ($uow->getScheduledEntityUpdates() as $entity) { + // Check if the entity should be synchronized + if (!($entity instanceof SyncEntityInterface)) { + continue; } + $this->processEntity($em, $entity); } - if($deletedEntities || $insertedEntities) - $somethingChanged = true; - - foreach ($uow->getScheduledEntityUpdates() as $keyEntity => $entity) { - $changes = $uow->getEntityChangeSet($entity); - - if(count($changes) == 1 && isset($changes["lastTimestamp"]) && !isset($changes["lastLogin"]) && !$somethingChanged) { - $oid = spl_object_hash($entity); - $uow->clearEntityChangeSet($oid); - } else { - $this->handleEntityChange($em, $entity); - } - } + } - foreach ($uow->getScheduledEntityInsertions() as $keyEntity => $entity) { - $this->handleEntityChange($em, $entity); - } + public function preRemove(LifecycleEventArgs $args) + { } - public function preRemove(LifecycleEventArgs $args) { - $entity = $args->getEntity(); - $class = get_class($entity); - $id = null; + private function processEntity(EntityManagerInterface $em, $entity) + { + $uow = $em->getUnitOfWork(); - if(method_exists($entity, 'getId')) { - $id = $entity->getId(); - } + $timestamp = time(); - $this->syncService->addToDeleteSyncState($class, $id); - } + // Check if this class itself has a lastTimestamp + if(method_exists($entity, 'setLastTimestamp')) { + $entity->setLastTimestamp($timestamp); + $uow->computeChangeSet($em->getClassMetadata(get_class($entity)), $entity); + } - private function handleEntityChange(EntityManagerInterface $em, $entity) { - if(method_exists($entity, 'getLastTimestamp')) { - $timestamp = $entity->getLastTimestamp() ?? time(); - } else { - $timestamp = time(); + // Check if there are any relationships that should notified + $annotationReader = new AnnotationReader(); + $reflection = new \ReflectionClass(get_class($entity)); + + /** @var \ReflectionProperty $property */ + foreach ($reflection->getProperties() as $property) { + /** @var SyncParent $annotation */ + if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { + $getter = $annotation->getter; + $parent = $entity->$getter(); + if($parent instanceof SyncEntityInterface) { + $this->processEntity($em, $parent); + } + } } - $class = get_class($entity); - $this->syncService->updateSyncState($em, $class, $timestamp); } - - } diff --git a/Interfaces/SyncEntityInterface.php b/Interfaces/SyncEntityInterface.php new file mode 100755 index 0000000..3b54861 --- /dev/null +++ b/Interfaces/SyncEntityInterface.php @@ -0,0 +1,6 @@ + $result["data"], 'deletes' => json_decode($this->container->get('jms_serializer')->serialize($deletes, 'json'), true), 'newItems' => json_decode($this->container->get('jms_serializer')->serialize($newItems, 'json'), true), - 'failedItems' => json_decode($this->container->get('jms_serializer')->serialize($failedItems, 'json'), true), + 'failedItems' => json_decode($this->container->get('jms_serializer')->serialize($failedItems, 'json'), true), SyncState::REAL_LAST_TIMESTAMP => $result[SyncState::REAL_LAST_TIMESTAMP], ); } From cf38cabba40aae7ff7469221009ff3f1f0c5f822 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Tue, 8 May 2018 06:20:24 -0400 Subject: [PATCH 10/23] Working with deleted entities --- Annotations/SyncEntity.php | 13 ++ Entity/SyncFailedItemState.php | 2 +- Entity/SyncMapping.php | 2 +- EventListener/DoctrineEventSubscriber.php | 84 ------------- EventSubscriber/DoctrineEventSubscriber.php | 125 ++++++++++++++++++++ Interfaces/SyncEntityInterface.php | 6 - Resources/config/services.yml | 2 +- Service/SyncService.php | 38 +----- 8 files changed, 144 insertions(+), 128 deletions(-) create mode 100755 Annotations/SyncEntity.php mode change 100644 => 100755 Entity/SyncFailedItemState.php mode change 100644 => 100755 Entity/SyncMapping.php delete mode 100755 EventListener/DoctrineEventSubscriber.php create mode 100755 EventSubscriber/DoctrineEventSubscriber.php delete mode 100755 Interfaces/SyncEntityInterface.php diff --git a/Annotations/SyncEntity.php b/Annotations/SyncEntity.php new file mode 100755 index 0000000..1786a19 --- /dev/null +++ b/Annotations/SyncEntity.php @@ -0,0 +1,13 @@ +container = $container; - $this->syncService = $this->container->get('nti.sync'); - } - - public function getSubscribedEvents() - { - return array( - 'onFlush', - 'preRemove', - ); - } - - public function onFlush(OnFlushEventArgs $args) - { - - $em = $args->getEntityManager(); - $uow = $em->getUnitOfWork(); - - foreach ($uow->getScheduledEntityUpdates() as $entity) { - // Check if the entity should be synchronized - if (!($entity instanceof SyncEntityInterface)) { - continue; - } - $this->processEntity($em, $entity); - } - - } - - public function preRemove(LifecycleEventArgs $args) - { - - } - - private function processEntity(EntityManagerInterface $em, $entity) - { - $uow = $em->getUnitOfWork(); - - $timestamp = time(); - - // Check if this class itself has a lastTimestamp - if(method_exists($entity, 'setLastTimestamp')) { - $entity->setLastTimestamp($timestamp); - $uow->computeChangeSet($em->getClassMetadata(get_class($entity)), $entity); - } - - // Check if there are any relationships that should notified - $annotationReader = new AnnotationReader(); - $reflection = new \ReflectionClass(get_class($entity)); - - /** @var \ReflectionProperty $property */ - foreach ($reflection->getProperties() as $property) { - /** @var SyncParent $annotation */ - if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { - $getter = $annotation->getter; - $parent = $entity->$getter(); - if($parent instanceof SyncEntityInterface) { - $this->processEntity($em, $parent); - } - } - } - } -} diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php new file mode 100755 index 0000000..0f58359 --- /dev/null +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -0,0 +1,125 @@ +container = $container; + $this->syncService = $this->container->get('nti.sync'); + } + + public function getSubscribedEvents() + { + return array( + 'onFlush', + ); + } + + public function onFlush(OnFlushEventArgs $args) + { + + $em = $args->getEntityManager(); + $uow = $em->getUnitOfWork(); + + foreach ($uow->getScheduledEntityUpdates() as $entity) { + $this->processEntity($em, $entity); + } + + foreach ($uow->getScheduledEntityInsertions() as $entity) { + $this->processEntity($em, $entity); + } + + foreach ($uow->getScheduledEntityDeletions() as $entity) { + $this->processEntity($em, $entity); + $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); + } + + /** @var PersistentCollection $collectionUpdate */ + foreach ($uow->getScheduledCollectionUpdates() as $collectionUpdate) { + foreach($collectionUpdate as $entity) { + $this->processEntity($em, $entity); + } + } + + /** @var PersistentCollection $collectionDeletion */ + foreach($uow->getScheduledCollectionDeletions() as $collectionDeletion) { + foreach($collectionDeletion as $entity) { + $this->processEntity($em, $entity); + $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); + } + } + + } + + private function processEntity(EntityManagerInterface $em, $entity) + { + + $reflection = new \ReflectionClass(ClassUtils::getClass($entity)); + $annotationReader = new AnnotationReader(); + $syncEntityAnnotation = $annotationReader->getClassAnnotation($reflection, SyncEntity::class); + // Check if the entity should be synchronized + if (!$syncEntityAnnotation) { + return; + } + + $uow = $em->getUnitOfWork(); + $timestamp = time(); + + // Update the mapping's sync state if exists + $mapping = $em->getRepository(SyncMapping::class)->findOneBy(array("class" => get_class($entity))); + if($mapping) { + $syncState = $em->getRepository(SyncState::class)->findOneBy(array("mapping" => $mapping)); + if(!$syncState) { + $syncState = new SyncState(); + $syncState->setMapping($mapping); + $em->persist($syncState); + } + $syncState->setTimestamp($timestamp); + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + } + + // Check if this class itself has a lastTimestamp + if(method_exists($entity, 'setLastTimestamp')) { + $entity->setLastTimestamp($timestamp); + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(get_class($entity)), $entity); + } + + // Notify relationships + /** @var \ReflectionProperty $property */ + foreach ($reflection->getProperties() as $property) { + + /** @var SyncParent $annotation */ + if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { + $getter = $annotation->getter; + $parent = $entity->$getter(); + dump($parent); + die; + // Using ClassUtils as $parent is actually a Proxy of the class + $reflrectionParent = new \ReflectionClass(ClassUtils::getClass($parent)); + $syncParentAnnotation = $annotationReader->getClassAnnotation($reflrectionParent, SyncEntity::class); + if(!$syncParentAnnotation) { + continue; + } + $this->processEntity($em, $parent); + } + } + } +} diff --git a/Interfaces/SyncEntityInterface.php b/Interfaces/SyncEntityInterface.php deleted file mode 100755 index 3b54861..0000000 --- a/Interfaces/SyncEntityInterface.php +++ /dev/null @@ -1,6 +0,0 @@ -em->getRepository(SyncDeleteState::class)->findFromTimestamp($mappingName, $timestamp); $newItems = $this->em->getRepository(SyncNewItemState::class)->findFromTimestampAndMapping($mappingName, $timestamp); - /** - * Failed Items Synchronization - */ - $failedItems = $this->em->getRepository('NTISyncBundle:SyncFailedItemState')->findFromTimestampAndMapping($mappingName, $timestamp); + $failedItems = $this->em->getRepository(SyncFailedItemState::class)->findFromTimestampAndMapping($mappingName, $timestamp); /** @var SyncRepositoryInterface $repository */ $repository = $this->em->getRepository($syncMapping->getClass()); @@ -88,30 +86,6 @@ public function getFromMappings($mappings) { return $changes; } - public function updateSyncState(EntityManagerInterface $em, $class, $timestamp) { - - $mapping = $em->getRepository(SyncMapping::class)->findOneBy(array("class" => $class)); - if(!$mapping) { - return; - } - - $syncState = $em->getRepository(SyncState::class)->findOneBy(array("mapping" => $mapping)); - - $uow = $em->getUnitOfWork(); - - if(!$syncState) { - $syncState = new SyncState(); - $syncState->setMapping($mapping); - $syncState->setTimestamp($timestamp); - $em->persist($syncState); - $uow->computeChangeSet($em->getClassMetadata(SyncState::class), $syncState); - } else { - $syncState->setTimestamp($timestamp); - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); - } - - } - /** * Create a new SyncDeleteState for the given class/id * @@ -122,6 +96,7 @@ public function addToDeleteSyncState($class, $id) { $this->em = $this->container->get('doctrine')->getManager(); + /** @var SyncMapping $mapping */ $mapping = $this->em->getRepository(SyncMapping::class)->findOneBy(array("class" => $class)); if(!$mapping) { return; @@ -133,12 +108,5 @@ public function addToDeleteSyncState($class, $id) { $deleteEntry->setTimestamp(time()); $this->em->persist($deleteEntry); - - try { - $this->em->flush(); - } catch (\Exception $ex) { - error_log("Unable to register deletion of object: " . $class . " with ID " . $id); - error_log($ex->getMessage()); - } } } From 112de243327288761ba53c4ec9fbf6b306ec3c65 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Thu, 10 May 2018 10:07:27 -0400 Subject: [PATCH 11/23] Removing dump --- EventSubscriber/DoctrineEventSubscriber.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index 0f58359..b8f2865 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -110,8 +110,6 @@ private function processEntity(EntityManagerInterface $em, $entity) if (null !== ($annotation = $annotationReader->getPropertyAnnotation($property, SyncParent::class))) { $getter = $annotation->getter; $parent = $entity->$getter(); - dump($parent); - die; // Using ClassUtils as $parent is actually a Proxy of the class $reflrectionParent = new \ReflectionClass(ClassUtils::getClass($parent)); $syncParentAnnotation = $annotationReader->getClassAnnotation($reflrectionParent, SyncEntity::class); From 3694219e7209a32e7737777374101bfcf86bc053 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Mon, 21 May 2018 09:11:19 -0400 Subject: [PATCH 12/23] Changing get_class to avoid proxies --- .gitignore | 1 + .idea/SyncBundle.iml | 10 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 196 ++++++++++++++++++++ EventSubscriber/DoctrineEventSubscriber.php | 4 +- 7 files changed, 229 insertions(+), 2 deletions(-) create mode 100755 .idea/SyncBundle.iml create mode 100755 .idea/misc.xml create mode 100755 .idea/modules.xml create mode 100755 .idea/vcs.xml create mode 100755 .idea/workspace.xml diff --git a/.gitignore b/.gitignore index 3a2a5ba..a6a6a28 100755 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ # Backup entities generated with doctrine:generate:entities command **/Entity/*~ +./.idea diff --git a/.idea/SyncBundle.iml b/.idea/SyncBundle.iml new file mode 100755 index 0000000..940f6f9 --- /dev/null +++ b/.idea/SyncBundle.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100755 index 0000000..28a804d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100755 index 0000000..549aacb --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100755 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100755 index 0000000..5661775 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + $PROJECT_DIR$/composer.json + + + + + + + + + + + + + + + + + + + + + + + + true + DEFINITION_ORDER + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - 1523390869087 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 0548505809513d18d13829c919fe6048980b63cf Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sun, 27 May 2018 19:29:14 -0400 Subject: [PATCH 14/23] Added computeChangeSet to deleted items --- Service/SyncService.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Service/SyncService.php b/Service/SyncService.php index 7a7129e..6d55d43 100755 --- a/Service/SyncService.php +++ b/Service/SyncService.php @@ -108,5 +108,7 @@ public function addToDeleteSyncState($class, $id) { $deleteEntry->setTimestamp(time()); $this->em->persist($deleteEntry); + $uow = $this->em->getUnitOfWork(); + $uow->computeChangeSet($this->em->getClassMetadata(SyncDeleteState::class), $deleteEntry); } } From fdd26decb6fd7be0c62bbd190e620378a1021567 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Sun, 27 May 2018 19:58:55 -0400 Subject: [PATCH 15/23] Added entity state verification --- EventSubscriber/DoctrineEventSubscriber.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index 8e2dac6..ae38a46 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -9,6 +9,7 @@ use Doctrine\ORM\Event\LifecycleEventArgs; use Doctrine\ORM\Event\OnFlushEventArgs; use Doctrine\ORM\PersistentCollection; +use Doctrine\ORM\UnitOfWork; use NTI\SyncBundle\Annotations\SyncEntity; use NTI\SyncBundle\Annotations\SyncParent; use NTI\SyncBundle\Entity\SyncMapping; @@ -93,13 +94,17 @@ private function processEntity(EntityManagerInterface $em, $entity) $em->persist($syncState); } $syncState->setTimestamp($timestamp); - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + if($uow->getEntityState($syncState) == UnitOfWork::STATE_MANAGED) { + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + } } // Check if this class itself has a lastTimestamp if(method_exists($entity, 'setLastTimestamp')) { $entity->setLastTimestamp($timestamp); - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); + if($uow->getEntityState($entity) == UnitOfWork::STATE_MANAGED) { + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); + } } // Notify relationships From a992791c02eb273d08c2c7da1b05d86030f0eab9 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Wed, 27 Jun 2018 16:41:32 +0000 Subject: [PATCH 16/23] Added fix for bug when deleting entities --- EventSubscriber/DoctrineEventSubscriber.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index ae38a46..c06d84d 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -49,8 +49,9 @@ public function onFlush(OnFlushEventArgs $args) } foreach ($uow->getScheduledEntityDeletions() as $entity) { - $this->processEntity($em, $entity); + $this->processEntity($em, $entity, true); $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); + } /** @var PersistentCollection $collectionUpdate */ @@ -63,14 +64,14 @@ public function onFlush(OnFlushEventArgs $args) /** @var PersistentCollection $collectionDeletion */ foreach($uow->getScheduledCollectionDeletions() as $collectionDeletion) { foreach($collectionDeletion as $entity) { - $this->processEntity($em, $entity); + $this->processEntity($em, $entity, true); $this->container->get('nti.sync')->addToDeleteSyncState(ClassUtils::getClass($entity), $entity->getId()); } } } - private function processEntity(EntityManagerInterface $em, $entity) + private function processEntity(EntityManagerInterface $em, $entity, $deleting = false) { $reflection = new \ReflectionClass(ClassUtils::getClass($entity)); @@ -100,11 +101,9 @@ private function processEntity(EntityManagerInterface $em, $entity) } // Check if this class itself has a lastTimestamp - if(method_exists($entity, 'setLastTimestamp')) { + if(!$deleting && method_exists($entity, 'setLastTimestamp')) { $entity->setLastTimestamp($timestamp); - if($uow->getEntityState($entity) == UnitOfWork::STATE_MANAGED) { - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); - } + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(ClassUtils::getClass($entity)), $entity); } // Notify relationships From e84f8a90b32ec63bfc4bf471c70a463a852a5544 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Wed, 18 Jul 2018 00:36:33 -0400 Subject: [PATCH 17/23] Update DoctrineEventSubscriber.php Fixing bug when the SyncState is created from the code it tried to recompute the change set on the new entity but it should be just computing the changeset --- EventSubscriber/DoctrineEventSubscriber.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EventSubscriber/DoctrineEventSubscriber.php b/EventSubscriber/DoctrineEventSubscriber.php index c06d84d..acd0f97 100755 --- a/EventSubscriber/DoctrineEventSubscriber.php +++ b/EventSubscriber/DoctrineEventSubscriber.php @@ -96,7 +96,11 @@ private function processEntity(EntityManagerInterface $em, $entity, $deleting = } $syncState->setTimestamp($timestamp); if($uow->getEntityState($syncState) == UnitOfWork::STATE_MANAGED) { - $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + if($syncState->getId()) { + $uow->recomputeSingleEntityChangeSet($em->getClassMetadata(SyncState::class), $syncState); + } else { + $uow->computeChangeSet($em->getClassMetadata(SyncState::class), $syncState); + } } } From 2a90752e0ef588f05c32a54bc2fb205e0383200d Mon Sep 17 00:00:00 2001 From: ntidev Date: Wed, 18 Jul 2018 15:37:24 -0400 Subject: [PATCH 18/23] Update README.md Updating documentation --- README.md | 115 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b95289d..128b294 100755 --- a/README.md +++ b/README.md @@ -40,14 +40,100 @@ Below are a list of things that need to be considered in order to implement this bundle: -1. Entities need to have a method called `getlastTimestamp()` in order for this bundle to work properly. As the name implies, it should return the `lastTimestamp` of when the object was last updated and cannot be null. (You need to handle this lastTimestamp property, for example, by using LifecyclecCallbacks) -2. Entities to be synced must have a repository implementing the `SyncRepositoryInterface`. (see below for more information) -3. The mapping (`SyncMapping`) needs to be configured foreach entity as it is the list used as reference for the lookup +1. Any Entity that needs to be taken into account during the synchronization process must have the `@NTI\SyncEntity` annotation at the class level. +2. `ManyToOne` relationships that should alter the last synchronization timestamp of their parents should use the annotation `@NTI\SyncParent(getter="[Getter Name]")` (see example below for more information). +3. Entities to be synced must have a repository implementing the `SyncRepositoryInterface` (see below for more information). +4. The mapping `SyncMapping` needs to be configured foreach entity as it is the list used as reference for the lookup. +5. If the entity is going to be synched FROM the client, then a service must be defined in the `SyncMapping` database entry. Also, this method needs to implement the interface `SyncServiceInterface`. -## Background process +## Tracking Changes -The bundle takes care of tracking the changes made to the entities by using a `DoctrineEventListener` which listenes to the `PreUpdate`, `PrePersist`, and `PreRemove` events. When any of these events is fired on an Entity that contains a `SyncMapping` defined, the bundle will call the `getlastTimestamp()` on this entity and use this value as the last `timtestamp` that the entity in general was updated. +The way that the bundle tracks changes in the synchronization is as follows: +1. The bundle has a `DoctrineEventListener` listening to the `onFlush` event. +2. Once the event is fired, the bundle will grab every entitty that has the `@NTI\SyncEntity` annotation. +3. If the entity has a `SyncMapping` defined, the system will update the `last_timestamp` field of this mapping to the current `time()`. +4. If the entity has a method called `setLastTimestamp()` it will be called with the `time()` as a parameter and the changes will be recomputed or computed. +5. All the properties of the entity will be examined in search for a property that contains the annotation `@NTI\SyncParent(getter="[Getter Name]")`. + If found, the getter will be called, if the result is an object that also has the `@NTI\SyncEntity`, it will be processed again starting from point #3. This process occurrs recursively. + +## Class Examples + + ``` + lastTimestamp = $lastTimestamp; + return $this; + } + + /** + * Get lastTimestamp + * @return integer + */ + public function getLastTimestamp() + { + return $this->lastTimestamp; + } + + } + +An example of a class using a `ManyToOne` where the child also needs the parent's `last_timestamp` to be updated can be defined as: + + ``` + product; + } + } + Below is the general process that the bundles goes through to keep track of the synchronization state: ![Synchronization Process - Server](/Images/SynchronizationProcess-Server.PNG?raw=true "Synchronization State Process on the Server") @@ -111,7 +197,7 @@ Besides implementing the interface, in the database `nti_sync_mapping` the mappi First, the idea is to get a summary of the changes and mappings from the server: ``` -GET /nti/sync/getSummary +GET /nti/sync/summary ``` To which the server will respond with the following structure: @@ -146,7 +232,6 @@ Content-Type: application/json ] } ``` -Note: This request can also be done using a query and GET instead. After receiving the request, if a mapping with the specified name exists, the system will call the repository's findFromTimestamp implementation and return the following result (Using a Product entity as an example): @@ -207,7 +292,7 @@ After receiving the request, if a mapping with the specified name exists, the sy }, "classId": 137, "timestamp": 1512080747, - "errors": "{"has_error":true,"additional_errors":null,"code":403,"message":"This inventory report is already closed. Further operations are not allowed.","data":null,"redirect":null}" + "errors": [...errors provided...] }, ... ], @@ -217,16 +302,16 @@ After receiving the request, if a mapping with the specified name exists, the sy ``` -The server will return the both the `changes` , `newItems`, `failedItems` ,and the `deletes`. The `changes` will contain the `data` portion of the array returned by +The server will return the both the `changes` , `newItems`, `failedItems` , and the `deletes`. The `changes` will contain the `data` portion of the array returned by the repository's implementation of `SyncRepositoryInterface`. The `deletes` will contain the list of `SyncDeleteState` that were recorded since the specified timestamp. The `newItems` will contain the list of `SyncNewItemState` which means the new items that were created since the provided timestamp including the UUID that was given at the time (This is helpful to third party devices when first pulling the information they can verify if an item was already created but they don't have the ID of that item in their local storage and avoid creating duplicates in the server). The `failedItems` will contain the list of `SyncFailedItemState`, each item in this list -contains an `errors` JSON property with the list of errors founds processing the creation or update of the entity. +contains an `errors` property with the errors founds processing the creation or update of the entity. The `_real_last_timestamp` should be used as it can help with paginating the results for a full-sync and help the client get the real last timestamp of the last object in the response. This has to be obtained in the repository and can be done -by simply looping through the array of objects and getting the latest updatedOn. +by simply getting the last item from the repository's result and calling the `getLastTimestamp()`. From this point on, the client must keep a track of the `_real_last_timestamp` in order to perform a sync in the future. @@ -236,10 +321,10 @@ Below is the general idea over the push/pull process: ![Synchronization Process - Push/Pull](/Images/SynchronizationProcess-PushPull.PNG?raw=true "Synchronization Push Pull Process") -###Server Side +### Server Side In the `SyncMapping` for each mapped entity a service should be specified. This service must implement the `SyncServiceInterface`. -###Client Side +### Client Side In order to handle a push from a third party device it must provide the following structure in its request: ``` @@ -277,9 +362,7 @@ The server then returns the following structure: } ``` - - - ## Todo * Handle deletes from third parties +* `ManyToMany` relationships are tricky and can lead to performance issues From f639ce15650413239f1ab079611d6e63e9757002 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Wed, 18 Jul 2018 20:00:15 -0400 Subject: [PATCH 19/23] Update SyncService.php Incorrect variable causing exception --- Service/SyncService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Service/SyncService.php b/Service/SyncService.php index 6d55d43..5326367 100755 --- a/Service/SyncService.php +++ b/Service/SyncService.php @@ -68,7 +68,7 @@ public function getFromMappings($mappings) { /** @var SyncRepositoryInterface $repository */ $repository = $this->em->getRepository($syncMapping->getClass()); if(!($repository instanceof SyncRepositoryInterface)) { - error_log("The repository for the class {$mapping->getClass()} does not implement the SyncRepositoryInterface."); + error_log("The repository for the class {$syncMapping->getClass()} does not implement the SyncRepositoryInterface."); continue; } From f879325264c3b8d11a6c9df1196de3b5a3ce8689 Mon Sep 17 00:00:00 2001 From: ntidev Date: Thu, 19 Jul 2018 08:20:05 -0400 Subject: [PATCH 20/23] Update README.md Adding required step for setup --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 128b294..749978c 100755 --- a/README.md +++ b/README.md @@ -44,7 +44,10 @@ Below are a list of things that need to be considered in order to implement this 2. `ManyToOne` relationships that should alter the last synchronization timestamp of their parents should use the annotation `@NTI\SyncParent(getter="[Getter Name]")` (see example below for more information). 3. Entities to be synced must have a repository implementing the `SyncRepositoryInterface` (see below for more information). 4. The mapping `SyncMapping` needs to be configured foreach entity as it is the list used as reference for the lookup. -5. If the entity is going to be synched FROM the client, then a service must be defined in the `SyncMapping` database entry. Also, this method needs to implement the interface `SyncServiceInterface`. +5. The `SyncState` should be created for each mapping. This can be done with this query after creating all the `SyncMapping`: + ``` + `INSERT INTO nti_sync_mapping(mapping_id, timestamp) SELECT id, 0 FROM sync_nti_mapping;` +6. If the entity is going to be synched FROM the client, then a service must be defined in the `SyncMapping` database entry. Also, this method needs to implement the interface `SyncServiceInterface`. ## Tracking Changes From e53b3776b4cab19a86bd411a5a5ef29527bf619a Mon Sep 17 00:00:00 2001 From: ntidev Date: Thu, 19 Jul 2018 08:20:59 -0400 Subject: [PATCH 21/23] Update README.md Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 749978c..a726b09 100755 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Below are a list of things that need to be considered in order to implement this 4. The mapping `SyncMapping` needs to be configured foreach entity as it is the list used as reference for the lookup. 5. The `SyncState` should be created for each mapping. This can be done with this query after creating all the `SyncMapping`: ``` - `INSERT INTO nti_sync_mapping(mapping_id, timestamp) SELECT id, 0 FROM sync_nti_mapping;` + `INSERT INTO nti_sync_state(mapping_id, timestamp) SELECT id, 0 FROM sync_nti_mapping;` 6. If the entity is going to be synched FROM the client, then a service must be defined in the `SyncMapping` database entry. Also, this method needs to implement the interface `SyncServiceInterface`. ## Tracking Changes From b6d53befa4463474a4df28d758cbfe7a2e9b68f2 Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Mon, 23 Jul 2018 23:11:42 -0400 Subject: [PATCH 22/23] Added the _total_count field so the client knows if there are more items to sync in the section --- Entity/SyncState.php | 1 + Service/SyncService.php | 1 + 2 files changed, 2 insertions(+) diff --git a/Entity/SyncState.php b/Entity/SyncState.php index 1e47788..e7999ec 100755 --- a/Entity/SyncState.php +++ b/Entity/SyncState.php @@ -13,6 +13,7 @@ class SyncState { const REAL_LAST_TIMESTAMP = "_real_last_timestamp"; + const TOTAL_COUNT = "_total_count"; /** * @var int * diff --git a/Service/SyncService.php b/Service/SyncService.php index 5326367..bf3542f 100755 --- a/Service/SyncService.php +++ b/Service/SyncService.php @@ -80,6 +80,7 @@ public function getFromMappings($mappings) { 'newItems' => json_decode($this->container->get('jms_serializer')->serialize($newItems, 'json'), true), 'failedItems' => json_decode($this->container->get('jms_serializer')->serialize($failedItems, 'json'), true), SyncState::REAL_LAST_TIMESTAMP => $result[SyncState::REAL_LAST_TIMESTAMP], + SyncState::TOTAL_COUNT => $result[SyncState::TOTAL_COUNT], ); } From ed995131ef9a78b3a71421e3636163ab55c6ca6b Mon Sep 17 00:00:00 2001 From: Benjamin Vison Date: Tue, 24 Jul 2018 22:39:17 -0400 Subject: [PATCH 23/23] Added support for limit, pagination, and changed the request and response structure to a more standarized way. Also. created a base sync repository that can be extended --- Controller/SyncController.php | 6 +- Interfaces/SyncRepositoryInterface.php | 19 ++- Models/SyncPullRequestData.php | 153 +++++++++++++++++++++++++ Models/SyncPullResponseData.php | 94 +++++++++++++++ Repository/SyncRepository.php | 64 +++++++++++ Service/SyncService.php | 25 ++-- 6 files changed, 332 insertions(+), 29 deletions(-) create mode 100755 Models/SyncPullRequestData.php create mode 100755 Models/SyncPullResponseData.php create mode 100755 Repository/SyncRepository.php diff --git a/Controller/SyncController.php b/Controller/SyncController.php index 2989afd..3a3c4de 100755 --- a/Controller/SyncController.php +++ b/Controller/SyncController.php @@ -52,9 +52,11 @@ public function pullAction(Request $request) { $mappings = (isset($data["mappings"])) ? $data["mappings"] : array(); } - $changes = $this->get('nti.sync')->getFromMappings($mappings); + $resultData = $this->get('nti.sync')->getFromMappings($mappings); - return new JsonResponse($changes, 200); + $resultData = json_decode($this->container->get('jms_serializer')->serialize($resultData, 'json'), true); + + return new JsonResponse($resultData, 200); } /** diff --git a/Interfaces/SyncRepositoryInterface.php b/Interfaces/SyncRepositoryInterface.php index 6053bdb..375ced0 100755 --- a/Interfaces/SyncRepositoryInterface.php +++ b/Interfaces/SyncRepositoryInterface.php @@ -2,6 +2,8 @@ namespace NTI\SyncBundle\Interfaces; +use NTI\SyncBundle\Models\SyncPullRequestData; +use NTI\SyncBundle\Models\SyncPullResponseData; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -9,25 +11,20 @@ * @package NTI\SyncBundle\Interfaces */ interface SyncRepositoryInterface { + /** - * This function should return a plain array containing the results to be sent to the client + * This function should return an instance of SyncPullResponseData containing the results to be sent to the client * when a sync is requested. The container is also passed as a parameter in order to give additional * flexibility to the repository when making decision on what to show to the client. For example, if the user * making the request only has access to a portion of the data, this can be handled via the container in this method * of the repository. * - * The resulting structure should be the following: - * - * array( - * "data" => (array of objects), - * SyncState::REAL_LAST_TIMESTAMP => (last updated_on date from the array of objects), - * ) + * The resulting structure should be an instance of SyncPullRequestData * * - * @param $timestamp * @param ContainerInterface $container - * @param array $serializationGroups - * @return mixed + * @param SyncPullRequestData $requestData + * @return SyncPullResponseData */ - public function findFromTimestamp($timestamp, ContainerInterface $container, $serializationGroups = array()); + public function findFromTimestamp(ContainerInterface $container, SyncPullRequestData $requestData); } diff --git a/Models/SyncPullRequestData.php b/Models/SyncPullRequestData.php new file mode 100755 index 0000000..afd34e5 --- /dev/null +++ b/Models/SyncPullRequestData.php @@ -0,0 +1,153 @@ + than the timestamp provided + * and all of those 300 results have the same timestamp (for example, it is normal to + * set the inital timestamp to 0 when first installing this bundle) this would cause + * a loop and the client would always sync the same 50 results over and over again. + * + * For this, the client can send the `page` parameter, which then can be used in the repository to offset the results. + * + */ + private $page = 1; + + /** + * @var array|string + * @JMS\Type("array") + * + * The serialization groups that the process should use when returning the results + */ + private $serializationGroups = array("sync_basic"); + + /** + * @return string + */ + public function getMapping() + { + return $this->mapping; + } + + /** + * @param string $mapping + * @return SyncPullRequestData + */ + public function setMapping($mapping) + { + $this->mapping = $mapping; + return $this; + } + + /** + * @return int + */ + public function getTimestamp() + { + return $this->timestamp; + } + + /** + * @param int $timestamp + * @return SyncPullRequestData + */ + public function setTimestamp($timestamp) + { + $this->timestamp = $timestamp; + return $this; + } + + /** + * @return int + */ + public function getLimit() + { + return $this->limit; + } + + /** + * @param int $limit + * @return SyncPullRequestData + */ + public function setLimit($limit) + { + $this->limit = $limit; + return $this; + } + + /** + * @return int + */ + public function getPage() + { + return $this->page; + } + + /** + * @param int $page + * @return SyncPullRequestData + */ + public function setPage($page) + { + $this->page = $page; + return $this; + } + + /** + * @return array|string + */ + public function getSerializationGroups() + { + return $this->serializationGroups; + } + + /** + * @param array|string $serializationGroups + * @return SyncPullRequestData + */ + public function setSerializationGroups($serializationGroups) + { + $this->serializationGroups = $serializationGroups; + return $this; + } + + +} \ No newline at end of file diff --git a/Models/SyncPullResponseData.php b/Models/SyncPullResponseData.php new file mode 100755 index 0000000..6fb45eb --- /dev/null +++ b/Models/SyncPullResponseData.php @@ -0,0 +1,94 @@ +realLastTimestamp; + } + + /** + * @param int $realLastTimestamp + * @return SyncPullResponseData + */ + public function setRealLastTimestamp(int $realLastTimestamp): SyncPullResponseData + { + $this->realLastTimestamp = $realLastTimestamp; + return $this; + } + + /** + * @return array + */ + public function getData(): array + { + return $this->data; + } + + /** + * @param array $data + * @return SyncPullResponseData + */ + public function setData(array $data): SyncPullResponseData + { + $this->data = $data; + return $this; + } + + /** + * @return int + */ + public function getTotalCount(): int + { + return $this->totalCount; + } + + /** + * @param int $totalCount + * @return SyncPullResponseData + */ + public function setTotalCount(int $totalCount): SyncPullResponseData + { + $this->totalCount = $totalCount; + return $this; + } + + +} \ No newline at end of file diff --git a/Repository/SyncRepository.php b/Repository/SyncRepository.php new file mode 100755 index 0000000..675c648 --- /dev/null +++ b/Repository/SyncRepository.php @@ -0,0 +1,64 @@ +getTimestamp(); + $serializationGroups = $requestData->getSerializationGroups(); + $page = $requestData->getPage() > 0 ? $requestData->getPage() - 1 : 0; + $limit = $requestData->getLimit(); + + // Joins + $qb = $this->createQueryBuilder('i'); + $qb->andWhere($qb->expr()->gte('i.lastTimestamp', $timestamp)); + $qb->orderBy('i.lastTimestamp', 'asc'); + + /** + * This should be set BEFORE getting the total count, that way the client will receive + * the actual items that are left for it to sync, not the total amount from a timestamp. + * @Ref the "page"parameter in SyncPulLRequestData + */ + $qb->setFirstResult($page * $limit); + + // Total records + $totalCountQb = clone $qb; + $totalCountQb->select('COUNT(i.id)'); + $totalCountQuery = $totalCountQb->getQuery(); + + try { + $totalCount = intval($totalCountQuery->getSingleScalarResult()); + } catch (\Exception $e) { + $totalCount = 0; + } + + $qb->setMaxResults($limit); + + $items = $qb->getQuery()->getResult(); + + $realLastTimestamp = count($items) <= 0 ? $timestamp : $items[count($items) - 1]->getLastTimestamp(); + + $itemsArray = json_decode($container->get('jms_serializer')->serialize($items, 'json', SerializationContext::create()->setGroups($serializationGroups)), true); + + $result = new SyncPullResponseData(); + $result->setData($itemsArray); + $result->setRealLastTimestamp($realLastTimestamp); + $result->setTotalCount($totalCount); + + return $result; + } + +} \ No newline at end of file diff --git a/Service/SyncService.php b/Service/SyncService.php index bf3542f..d8f8496 100755 --- a/Service/SyncService.php +++ b/Service/SyncService.php @@ -10,6 +10,7 @@ use NTI\SyncBundle\Entity\SyncNewItemState; use NTI\SyncBundle\Entity\SyncState; use NTI\SyncBundle\Interfaces\SyncRepositoryInterface; +use NTI\SyncBundle\Models\SyncPullRequestData; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -47,23 +48,17 @@ public function getFromMappings($mappings) { foreach($mappings as $mapping) { - if(!isset($mapping["timestamp"]) || !isset($mapping["mapping"])) { - continue; - } - - $timestamp = $mapping["timestamp"]; - $mappingName = $mapping["mapping"]; - $serializationGroup = (isset($mapping["serializer"])) ? $mapping["serializer"] : "sync_basic"; + $requestData = $this->container->get('jms_serializer')->deserialize(json_encode($mapping), SyncPullRequestData::class, 'json'); - $syncMapping = $this->em->getRepository(SyncMapping::class)->findOneBy(array("name" => $mappingName)); + $syncMapping = $this->em->getRepository(SyncMapping::class)->findOneBy(array("name" => $requestData->getMapping())); if(!$syncMapping) { continue; } - $deletes = $this->em->getRepository(SyncDeleteState::class)->findFromTimestamp($mappingName, $timestamp); - $newItems = $this->em->getRepository(SyncNewItemState::class)->findFromTimestampAndMapping($mappingName, $timestamp); - $failedItems = $this->em->getRepository(SyncFailedItemState::class)->findFromTimestampAndMapping($mappingName, $timestamp); + $deletes = $this->em->getRepository(SyncDeleteState::class)->findFromTimestamp($requestData->getMapping(), $requestData->getTimestamp()); + $newItems = $this->em->getRepository(SyncNewItemState::class)->findFromTimestampAndMapping($requestData->getMapping(), $requestData->getTimestamp()); + $failedItems = $this->em->getRepository(SyncFailedItemState::class)->findFromTimestampAndMapping($requestData->getMapping(), $requestData->getTimestamp()); /** @var SyncRepositoryInterface $repository */ $repository = $this->em->getRepository($syncMapping->getClass()); @@ -72,15 +67,13 @@ public function getFromMappings($mappings) { continue; } - $result = $repository->findFromTimestamp($timestamp, $this->container, $serializationGroup); + $result = $repository->findFromTimestamp($this->container, $requestData); - $changes[$mappingName] = array( - 'changes' => $result["data"], + $changes[$requestData->getMapping()] = array( + 'changes' => $result, 'deletes' => json_decode($this->container->get('jms_serializer')->serialize($deletes, 'json'), true), 'newItems' => json_decode($this->container->get('jms_serializer')->serialize($newItems, 'json'), true), 'failedItems' => json_decode($this->container->get('jms_serializer')->serialize($failedItems, 'json'), true), - SyncState::REAL_LAST_TIMESTAMP => $result[SyncState::REAL_LAST_TIMESTAMP], - SyncState::TOTAL_COUNT => $result[SyncState::TOTAL_COUNT], ); }