fixed various edgecases and update submodule NOREC4DNA to support imp…

…licit type-casting for xor_numpy

app.py

@@ -799,8 +799,8 @@ def callback_handler(packet_tag_chunk_input, canvas_json_state, canvas_image_con
         except ValueError as ve:
             filename = ve.args[1]
         return (filename, dash.no_update,
-            dash.no_update, dash.no_update, dash.no_update, dash.no_update, dash.no_update, dash.no_update,
-            dash.no_update, dash.no_update, dash.no_update, dash.no_update, canvas_image_content, kaitai_view)
+                dash.no_update, dash.no_update, dash.no_update, dash.no_update, dash.no_update, dash.no_update,
+                dash.no_update, dash.no_update, dash.no_update, dash.no_update, canvas_image_content, kaitai_view)
     elif trigger_id == "packet-tag-chunk-invalid-button" or trigger_id == "packet-tag-chunk-valid-button":
         try:
             packet_tag_chunk_input = int(packet_tag_chunk_input)
@@ -871,14 +871,12 @@ def callback_handler(packet_tag_chunk_input, canvas_json_state, canvas_image_con
             _file = _file.rename(Path(working_dir + "/" + stem + _file.suffix))
             res += f"{_file.name}, "
         res += "]"
-        matrix_3d = np.dstack([mapping[x].b for x in differing_gepp_ids])
+        tmp = [mapping[x].b for x in differing_gepp_ids]
+        if len(tmp) == 0:
+            return "No differing solutions found!", dash.no_update
+        matrix_3d = np.dstack(tmp)
         most_common_vals, has_single_val = fast_most_common_matrix(matrix_3d)
-        # TODO: find the most common (!) value of each byte to predict the correct value
-        # TODO: detect changes between each differing solution to find which packet was invalid
-        #  (and which were the same for all solutinos)
         # with most_common_vals - gepp_backup.b we can calculate the rows AND columns that differ form the average
-        # TODO: ideally we would want to have an additional boolean matrix containing information about if all
-        #  matrices had the same value at that position (using this, we could detect if a row as correct)
         comp_mat = gepp_backup.b - most_common_vals
         semi_automatic_solver.decoder.GEPP = gepp_backup
         # calculate the invalid packet using by treating all rows from comp_mat with a sum() != 0 as invalid:

repair_algorithms/RandomShuffleRepair.py

@@ -106,6 +106,10 @@ def find_packet_shuffle(self, *args, **kwargs):
                                 self.semi_automatic_solver.initial_b.copy()[permutation])
                 tmp_gepp.solve()
 
+                # ensure that the order is correct / comparable
+                tmp_gepp.A = np.squeeze(tmp_gepp.A[tmp_gepp.result_mapping])
+                tmp_gepp.b = np.squeeze(tmp_gepp.b[tmp_gepp.result_mapping])
+                tmp_gepp.result_mapping = np.arange(tmp_gepp.b.shape[0])
                 # store the solution
                 self.solutions.append(tmp_gepp)
 
@@ -131,6 +135,9 @@ def find_packet_shuffle(self, *args, **kwargs):
         for sol_i, solution in enumerate(self.solutions[1:]):
             diff: numpy.array = np.array([helper.xor_numpy(self.solutions[0].b[i], solution.b[i]) for i in
                                           range(0, self.semi_automatic_solver.decoder.number_of_chunks)], dtype="uint8")
+            if not np.any(diff):
+                # the solutions are identical
+                continue
             unique_diffs = np.unique(np.vstack((unique_diffs, np.unique(diff, axis=0))), axis=0)
         # remove all zero rows:
         unique_diffs = unique_diffs[~np.all(unique_diffs == 0, axis=1)]
@@ -157,7 +164,10 @@ def find_packet_shuffle(self, *args, **kwargs):
 
         for sol_i, solution in enumerate(self.solutions):
             for cmp_sol_i, cmp_solution in enumerate(self.solutions):  # self.solution[sol_i:] instead of all?
-                if cmp_sol_i <= sol_i or np.array_equal(solution.b, cmp_solution.b):
+                if cmp_sol_i <= sol_i or (
+                        np.array_equal(solution.b, cmp_solution.b) and
+                        np.array_equal(solution.chunk_to_used_packets[self.perms[sol_i]],
+                                       cmp_solution.chunk_to_used_packets[self.perms[cmp_sol_i]])):
                     continue
                 for row_i, row in enumerate(solution.b[:self.semi_automatic_solver.decoder.number_of_chunks]):
                     possible_packets_sol = [self.perms[sol_i][i] for i, x in
@@ -205,7 +215,7 @@ def find_packet_shuffle(self, *args, **kwargs):
                                 print("WARNING: diff_bytes not in self.intersects.keys()!")
                                 self.intersects[diff_bytes] = np.array(possible_packets, dtype=np.uint64)
                             self.intersects[diff_bytes] = np.intersect1d(self.intersects[diff_bytes],
-                                                                         possible_packets, assume_unique=True)
+                                                                         possible_packets)
                             self.intersects[diff_bytes] = np.setdiff1d(self.intersects[diff_bytes],
                                                                        np.array(packet_intersect_for_row,
                                                                                 dtype=np.uint64),