From cec2b8c6a003da19113ffeca16619459f6871cb0 Mon Sep 17 00:00:00 2001 From: jasongellis Date: Thu, 23 Sep 2021 12:45:58 +0100 Subject: [PATCH 01/12] Add complexity measurement function --- pylithics/src/utils.py | 52 ++++++++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 22 +++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pylithics/src/utils.py b/pylithics/src/utils.py index c8e1145..7ebc3f8 100644 --- a/pylithics/src/utils.py +++ b/pylithics/src/utils.py @@ -4,6 +4,7 @@ import scipy.ndimage as ndi import pylithics.src.plotting as plot import math +import itertools def mask_image(binary_array, contour, innermask=False): @@ -658,3 +659,54 @@ def shape_detection(contour): shape = "arrow" # otherwise, we assume the shape is an arrow return shape, vertices + +def complexity_estimator(contour_df): + + adjacency_list = [] + for i in range(0, contour_df.shape[0]): + if contour_df.iloc[i]["parent_index"] == -1: + adjacency_list.append(0) + else: + # list coordinates for the contour we are interested on + contour_coordinate = contour_df.iloc[i]["contour"] + + # list of coordinates of each of the siblings that we are interested on (list of list) + contour_coordinate_siblings = contour_df[contour_df["parent_index"] == contour_df.iloc[i]["parent_index"]]['contour'].values + + count = 0 + for sibling_contour in contour_coordinate_siblings: + + # compare contour_coordinate with sibling_contour + adjacent = complexity_measure(contour_coordinate,sibling_contour) + + if adjacent == True: + count = count + 1 + + adjacency_list.append(count) + + contour_df['complexity'] = adjacency_list + + return contour_df + + +def complexity_measure(contour_coordinates1, contour_coordinates2): + + if np.array_equal(contour_coordinates1, contour_coordinates2): + return False + else: + new_pairs = [list(zip(x,contour_coordinates2)) for x in itertools.permutations(contour_coordinates2,len(contour_coordinates2))] + + print (new_pairs) + #dist = math.hypot(x2 - x1, y2 - y1) + return True + + + + + + + + + + + diff --git a/tests/test_utils.py b/tests/test_utils.py index b9a6655..07fdfe3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -12,7 +12,7 @@ from pylithics.src.utils import mask_image, contour_characterisation, classify_distributions, shape_detection, \ get_high_level_parent_and_hierarchy, pixulator, classify_surfaces, subtract_masked_image, measure_vertices, \ get_angles, \ - measure_arrow_angle, contour_selection + measure_arrow_angle, contour_selection, complexity_estimator # Global loads for all tests image_array = read_image(os.path.join('tests', 'test_images'), 'test') @@ -196,3 +196,23 @@ def test_shape_detection(): shape = shape_detection(cont) assert shape == ('arrow', 4) + +def test_complexity_estimator(): + image_array = read_image(os.path.join('tests', 'test_images'), 'test') + + filename_config = os.path.join('tests', 'test_config.yml') + + # Read YAML file + with open(filename_config, 'r') as config_file: + config_file = yaml.load(config_file) + + image_processed = process_image(image_array, config_file) + + config_file['conversion_px'] = 0.1 # hardcoded for now + binary_edge_sobel, _ = detect_lithic(image_processed, config_file) + + contours = find_lithic_contours(binary_edge_sobel, config_file) + + contour_complexity = complexity_estimator(contours) + return True + From 1dc6ad9b64f872db130005ee38f3f8cde6f4f3b4 Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Wed, 29 Sep 2021 12:28:07 +0100 Subject: [PATCH 02/12] adding complexity measure --- pylithics/src/utils.py | 47 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/pylithics/src/utils.py b/pylithics/src/utils.py index 7ebc3f8..dfa3fab 100644 --- a/pylithics/src/utils.py +++ b/pylithics/src/utils.py @@ -4,7 +4,7 @@ import scipy.ndimage as ndi import pylithics.src.plotting as plot import math -import itertools +from scipy.spatial.distance import cdist def mask_image(binary_array, contour, innermask=False): @@ -661,6 +661,21 @@ def shape_detection(contour): return shape, vertices def complexity_estimator(contour_df): + """ + + Function that estimate a complexity measure. Complexity is measured as the number of adjacent contours + for each contour. + + Parameters + ---------- + contour_df: dataframe + Dataframe with all contour information for an image. + Returns + ------- + + A copy of the contour_df dataframe with a new measure of complexity + + """ adjacency_list = [] for i in range(0, contour_df.shape[0]): @@ -690,15 +705,37 @@ def complexity_estimator(contour_df): def complexity_measure(contour_coordinates1, contour_coordinates2): + """ + Decide if two contours are adjacent based on distance between its coordinates. + Parameters + ---------- + contour_coordinates1: list of lists + Pixel coordinates for a contour of a single flake scar + or outline of a lithic object detected by contour finding + contour_coordinates2: list of lists + Pixel coordinates for a contour of a single flake scar + or outline of a lithic object detected by contour finding + + Returns + ------- + + A boolean + + """ + + # if they are the same contour they are not adjacent if np.array_equal(contour_coordinates1, contour_coordinates2): return False else: - new_pairs = [list(zip(x,contour_coordinates2)) for x in itertools.permutations(contour_coordinates2,len(contour_coordinates2))] + # get minimum distance between contours + min_dist = np.min(cdist(contour_coordinates1, contour_coordinates2)) - print (new_pairs) - #dist = math.hypot(x2 - x1, y2 - y1) - return True + # if the minimum distance found is less than a threshold then they are adjacent + if min_dist < 20: + return True + else: + return False From d9090cbe593a7c3cb6a9d3b3a499a67f68d5712f Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Wed, 29 Sep 2021 12:28:18 +0100 Subject: [PATCH 03/12] adding test for complexity measure --- tests/test_utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 07fdfe3..06df2ed 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -198,13 +198,6 @@ def test_shape_detection(): assert shape == ('arrow', 4) def test_complexity_estimator(): - image_array = read_image(os.path.join('tests', 'test_images'), 'test') - - filename_config = os.path.join('tests', 'test_config.yml') - - # Read YAML file - with open(filename_config, 'r') as config_file: - config_file = yaml.load(config_file) image_processed = process_image(image_array, config_file) @@ -214,5 +207,8 @@ def test_complexity_estimator(): contours = find_lithic_contours(binary_edge_sobel, config_file) contour_complexity = complexity_estimator(contours) - return True + + assert contour_complexity['complexity'].iloc[7] == 4 + + From 7b8d12da858b229a379e75df2e5fd597b1179dd6 Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:42:44 +0100 Subject: [PATCH 04/12] adding function that plots complexity --- pylithics/src/plotting.py | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pylithics/src/plotting.py b/pylithics/src/plotting.py index 99e61ee..2b13192 100644 --- a/pylithics/src/plotting.py +++ b/pylithics/src/plotting.py @@ -191,6 +191,10 @@ def plot_results(id, image_array, contours_df, output_dir): output_lithic = os.path.join(output_dir, id + "_lithium_angles.png") plot_angles(image_array, contours_df, output_lithic) + # plot scar strike angle + output_lithic = os.path.join(output_dir, id + "_complexity_polygon_count.png") + plot_complexity(image_array, contours_df, output_lithic) + def plot_thresholding(image_array, threshold, binary_array, output_file=''): @@ -279,3 +283,43 @@ def plot_template_arrow(image_array, template_array, value): ax[1].set_yticks([]) plt.figtext(0.4, 0.9, str(value)) plt.show() + +def plot_complexity(image_array, contours_df, output_path): + """ + Plot the contours from the lithic surfaces and display complexity and polygon count measurements. + + Parameters + ---------- + image_array: array + Original image array (0 to 255) + contours_df: dataframe + Dataframe with detected contours and extra information about them. + output_path: str + Path to output directory to save processed images + + """ + fig_x_size = fig_size(image_array) + fig, ax = plt.subplots(figsize=(fig_x_size, 20)) + ax.imshow(image_array, cmap=plt.cm.gray) + + # selecting only scars with a complexity measure > 0 + contours_angles_df = contours_df[(contours_df['parent_index'] != -1) & (contours_df['complexity']>0)] + cmap_list = plt.cm.get_cmap('tab20', contours_angles_df.shape[0]) + + if contours_angles_df.shape[0] == 0: + warnings.warn("Warning: No scars with complexity measure, no complexity output figure will be saved.'") + return None + + i = 0 + for contour, complexity, polygon_count in \ + contours_angles_df[['contour', 'complexity','polygon_count']].itertuples(index=False): + text = "Complexity: " + str(complexity)+", Polygon Count: "+str(polygon_count) + ax.plot(contour[:, 0], contour[:, 1], label=text, linewidth=5, color=cmap_list(i)) + i = i + 1 + + ax.set_xticks([]) + ax.set_yticks([]) + plt.legend(bbox_to_anchor=(1.02, 0), loc="lower left", borderaxespad=0, fontsize=11) + plt.title("Scar complexity and polygon count measurements", fontsize=30) + plt.savefig(output_path) + plt.close(fig) From d88dc1a9bdd343c8954bbba09ff3e5daa3bf49f2 Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:43:05 +0100 Subject: [PATCH 05/12] changing minimum complexity threshold --- pylithics/src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylithics/src/utils.py b/pylithics/src/utils.py index dfa3fab..4057996 100644 --- a/pylithics/src/utils.py +++ b/pylithics/src/utils.py @@ -732,7 +732,7 @@ def complexity_measure(contour_coordinates1, contour_coordinates2): min_dist = np.min(cdist(contour_coordinates1, contour_coordinates2)) # if the minimum distance found is less than a threshold then they are adjacent - if min_dist < 20: + if min_dist < 50: return True else: return False From a7554797a2dede3d7fde49f8411d9947c63939b2 Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:43:18 +0100 Subject: [PATCH 06/12] adding complexity to data output --- pylithics/src/read_and_process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pylithics/src/read_and_process.py b/pylithics/src/read_and_process.py index 30c1c4c..626c579 100644 --- a/pylithics/src/read_and_process.py +++ b/pylithics/src/read_and_process.py @@ -210,9 +210,9 @@ def data_output(contour_df, config_file): scars_objects_list = [] scar_id = 0 - for index, area_px, area_mm, width_mm, height_mm, angle, polygon_count in scars_df[ + for index, area_px, area_mm, width_mm, height_mm, angle, polygon_count, complexity in scars_df[ ['index', 'area_px', 'area_mm', - 'width_mm', 'height_mm', 'angle', 'polygon_count']].itertuples(index=False): + 'width_mm', 'height_mm', 'angle', 'polygon_count','complexity']].itertuples(index=False): scars_objects = {} scars_objects['scar_id'] = scar_id @@ -224,6 +224,7 @@ def data_output(contour_df, config_file): scars_objects['total_area_px'] / outer_objects['total_area_px'], 2) scars_objects['scar_angle'] = angle scars_objects["polygon_count"] = polygon_count + scars_objects["complexity"] = complexity scars_objects_list.append(scars_objects) scar_id = scar_id + 1 From 67d7cad88b7f11bde6e1a64682adf6e7a619ba54 Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:43:26 +0100 Subject: [PATCH 07/12] adding complexity to pipeline --- pylithics/scripts/run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pylithics/scripts/run.py b/pylithics/scripts/run.py index 3ecac6a..851274c 100644 --- a/pylithics/scripts/run.py +++ b/pylithics/scripts/run.py @@ -9,7 +9,7 @@ find_lithic_contours, detect_lithic, process_image, data_output, \ get_scars_angles, find_arrows from pylithics.src.plotting import plot_results, plot_thresholding -from pylithics.src.utils import pixulator, get_angles +from pylithics.src.utils import pixulator, get_angles, complexity_estimator def run_pipeline(id_list, metadata_df, input_dir, output_dir, config_file, get_arrows): @@ -115,6 +115,10 @@ def run_characterisation(input_dir, output_dir, config_file, arrows, debug=False # find contours contours = find_lithic_contours(binary_array, config_file) + # measure complexity on scars + contours = complexity_estimator(contours) + + # if this lithic has arrows do processing to detect and measure arrow angle if arrows: From d2395f48af129be0c53617957ed7e37a0618eb7a Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:46:52 +0100 Subject: [PATCH 08/12] refactoring complexity plot --- pylithics/src/plotting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pylithics/src/plotting.py b/pylithics/src/plotting.py index 2b13192..f348e7e 100644 --- a/pylithics/src/plotting.py +++ b/pylithics/src/plotting.py @@ -303,16 +303,16 @@ def plot_complexity(image_array, contours_df, output_path): ax.imshow(image_array, cmap=plt.cm.gray) # selecting only scars with a complexity measure > 0 - contours_angles_df = contours_df[(contours_df['parent_index'] != -1) & (contours_df['complexity']>0)] - cmap_list = plt.cm.get_cmap('tab20', contours_angles_df.shape[0]) + contours_complexity_df = contours_df[(contours_df['parent_index'] != -1) & (contours_df['complexity']>0)] + cmap_list = plt.cm.get_cmap('tab20', contours_complexity_df.shape[0]) - if contours_angles_df.shape[0] == 0: + if contours_complexity_df.shape[0] == 0: warnings.warn("Warning: No scars with complexity measure, no complexity output figure will be saved.'") return None i = 0 for contour, complexity, polygon_count in \ - contours_angles_df[['contour', 'complexity','polygon_count']].itertuples(index=False): + contours_complexity_df[['contour', 'complexity','polygon_count']].itertuples(index=False): text = "Complexity: " + str(complexity)+", Polygon Count: "+str(polygon_count) ax.plot(contour[:, 0], contour[:, 1], label=text, linewidth=5, color=cmap_list(i)) i = i + 1 From 31c42677cf516e02a2f7a07a8e929f777f8e63ad Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:53:54 +0100 Subject: [PATCH 09/12] fixing test --- tests/test_pipeline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 82e33eb..68598d2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,7 +5,7 @@ import yaml from pylithics.src.read_and_process import read_image, detect_lithic, \ find_lithic_contours, process_image, get_scars_angles, data_output, find_arrows -from pylithics.src.utils import get_angles +from pylithics.src.utils import get_angles, complexity_estimator def test_pipeline(): @@ -28,6 +28,9 @@ def test_pipeline(): # find contours contours = find_lithic_contours(binary_array, config_file) + # add complexity measure + contours = complexity_estimator(contours) + # in case we dont have arrows contours = get_scars_angles(image_processed, contours) @@ -60,6 +63,9 @@ def test_arrow_pipeline(): # find contours contours = find_lithic_contours(binary_array, config_file) + # add complexity measure + contours = complexity_estimator(contours) + # get the templates for the arrows templates = find_arrows(image_array, image_processed, False) From 0039e34029b1b4f53b511ab55e52f9a65ac0db4f Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 10:55:41 +0100 Subject: [PATCH 10/12] increasing minimum trheshold --- pylithics/src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylithics/src/utils.py b/pylithics/src/utils.py index 4057996..9df05af 100644 --- a/pylithics/src/utils.py +++ b/pylithics/src/utils.py @@ -732,7 +732,7 @@ def complexity_measure(contour_coordinates1, contour_coordinates2): min_dist = np.min(cdist(contour_coordinates1, contour_coordinates2)) # if the minimum distance found is less than a threshold then they are adjacent - if min_dist < 50: + if min_dist < 70: return True else: return False From ea282e685cbad4a1f0d54eaf588a3b223c7ec5db Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 11:35:22 +0100 Subject: [PATCH 11/12] fixing broken test --- tests/test_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 68598d2..7706c0d 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -38,7 +38,7 @@ def test_pipeline(): json_output = data_output(contours, config_file) assert len(json_output) == 4 - assert contours.shape == (11, 16) + assert contours.shape == (11, 17) assert binary_array.shape == (1841, 1665) assert len(json_output['lithic_contours']) == 4 From 3770295e3a99bd7ed7c86c786ac99ca0991e283b Mon Sep 17 00:00:00 2001 From: crangelsmith Date: Thu, 30 Sep 2021 11:47:54 +0100 Subject: [PATCH 12/12] fixing broken test and removing unsued column in contour dataframe --- pylithics/src/read_and_process.py | 1 - tests/test_pipeline.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pylithics/src/read_and_process.py b/pylithics/src/read_and_process.py index 626c579..16b3b84 100644 --- a/pylithics/src/read_and_process.py +++ b/pylithics/src/read_and_process.py @@ -313,7 +313,6 @@ def get_scars_angles(image_array, contour_df, templates = pd.DataFrame()): if templates.shape[0] == 0: # if there is no templates in the dataframe assing nan to angles. - contour_df['arrow_index'] = -1 contour_df['angle'] = np.nan # TODO: DO SOMETHING WITH RIPPLES diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 7706c0d..bd9a5ba 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -38,7 +38,7 @@ def test_pipeline(): json_output = data_output(contours, config_file) assert len(json_output) == 4 - assert contours.shape == (11, 17) + assert contours.shape == (11, 16) assert binary_array.shape == (1841, 1665) assert len(json_output['lithic_contours']) == 4 @@ -76,5 +76,5 @@ def test_arrow_pipeline(): contours_final = get_scars_angles(image_processed, contours, arrow_df) assert len(templates) == 4 - assert contours_final.shape == (11, 15) + assert contours_final.shape == (11, 16) assert arrow_df.shape == (4, 2)