Ignore state machine not found errors when syncing the HSM tree (#7341)

## Why?

The node may not be found if the state machine was deleted in terminal a
state and this cluster is syncing from an older cluster that doesn't
delete the state machine on completion and we need to be backwards
compatible with the previous release.

service/history/ndc/hsm_state_replicator.go

@@ -187,40 +187,12 @@ func (r *HSMStateReplicatorImpl) syncHSMNode(
 		incomingNodePath := incomingNode.Path()
 		currentNode, err := currentHSM.Child(incomingNodePath)
 		if err != nil {
-			// The node may not be found if:
-			// State machine was deleted in terminal state.
+			// The node may not be found if the state machine was deleted in terminal a state and this cluster is
+			// syncing from an older cluster that doesn't delete the state machine on completion.
 			// Both state machine creation and deletion are always associated with an event, so any missing state
 			// machine must have a corresponding event in history.
 			if errors.Is(err, hsm.ErrStateMachineNotFound) {
-				notFoundErr := err
-				// Get the last items from both version histories
-				currentVersionHistory, err := versionhistory.GetCurrentVersionHistory(
-					mutableState.GetExecutionInfo().GetVersionHistories(),
-				)
-				if err != nil {
-					return err
-				}
-				lastLocalItem, err := versionhistory.GetLastVersionHistoryItem(currentVersionHistory)
-				if err != nil {
-					return err
-				}
-				lastIncomingItem, err := versionhistory.GetLastVersionHistoryItem(request.EventVersionHistory)
-				if err != nil {
-					return err
-				}
-
-				// Only accept "not found" if our version history is ahead
-				if versionhistory.CompareVersionHistoryItem(lastLocalItem, lastIncomingItem) > 0 {
-					r.logger.Debug("State machine not found - likely deleted in terminal state",
-						tag.WorkflowNamespaceID(mutableState.GetExecutionInfo().NamespaceId),
-						tag.WorkflowID(mutableState.GetExecutionInfo().WorkflowId),
-						tag.WorkflowRunID(mutableState.GetExecutionInfo().OriginalExecutionRunId),
-					)
-					return nil
-				}
-
-				// Otherwise, we might be missing events
-				return notFoundErr
+				return nil
 			}
 			return err
 		}

service/history/ndc/hsm_state_replicator_test.go

@@ -741,20 +741,20 @@ func (s *hsmStateReplicatorSuite) TestSyncHSM_StateMachineNotFound() {
 			expectedError: consts.ErrDuplicate,
 		},
 		{
-			name: "incoming version higher - return notFoundErr",
+			name: "incoming version higher - ignored",
 			versionHistory: &historyspb.VersionHistory{
 				Items: []*historyspb.VersionHistoryItem{
 					{EventId: 50, Version: baseVersion - 100},
 					{EventId: 102, Version: baseVersion},
 				},
 			},
-			expectedError: hsm.ErrStateMachineNotFound,
+			expectedError: consts.ErrDuplicate,
 		},
 	}
 
 	for _, tc := range testCases {
 		tc := tc
-		s.Run(tc.name, func() {
+		s.T().Run(tc.name, func(t *testing.T) {
 			lastVersion := tc.versionHistory.Items[len(tc.versionHistory.Items)-1].Version
 
 			err := s.nDCHSMStateReplicator.SyncHSMState(context.Background(), &shard.SyncHSMRequest{
@@ -781,9 +781,9 @@ func (s *hsmStateReplicatorSuite) TestSyncHSM_StateMachineNotFound() {
 			})
 
 			if tc.expectedError != nil {
-				s.ErrorIs(err, tc.expectedError)
+				require.ErrorIs(t, err, tc.expectedError)
 			} else {
-				s.NoError(err)
+				require.NoError(t, err)
 			}
 		})
 	}