From 3a1008f9efb442b342473d5ac9faa7801429cafb Mon Sep 17 00:00:00 2001 From: Max Dorzweiler Date: Wed, 22 May 2024 02:21:19 +0200 Subject: [PATCH] This is the improved version of image-tiling. Signed-off-by: Max Dorzweiler --- component-library/input/image-tiling.cwl | 57 ++++++ component-library/input/image-tiling.ipynb | 182 ++++++++++++++++++ component-library/input/image-tiling.job.yaml | 30 +++ component-library/input/image-tiling.yaml | 31 +++ 4 files changed, 300 insertions(+) create mode 100644 component-library/input/image-tiling.cwl create mode 100644 component-library/input/image-tiling.ipynb create mode 100644 component-library/input/image-tiling.job.yaml create mode 100644 component-library/input/image-tiling.yaml diff --git a/component-library/input/image-tiling.cwl b/component-library/input/image-tiling.cwl new file mode 100644 index 00000000..2943f14b --- /dev/null +++ b/component-library/input/image-tiling.cwl @@ -0,0 +1,57 @@ +cwlVersion: v1.2 +class: CommandLineTool + +baseCommand: "claimed" + +inputs: + component: + type: string + default: docker.io/mdorzweiler/claimed-image-tiling:0.1 + inputBinding: + position: 1 + prefix: --component + log_level: + type: string + default: "INFO" + inputBinding: + position: 2 + prefix: --log_level + directory_path: + type: string + default: None + inputBinding: + position: 3 + prefix: --directory_path + destination: + type: string + default: None + inputBinding: + position: 4 + prefix: --destination + tile_size_x: + type: string + default: None + inputBinding: + position: 5 + prefix: --tile_size_x + tile_size_y: + type: string + default: None + inputBinding: + position: 6 + prefix: --tile_size_y + stride_x: + type: string + default: None + inputBinding: + position: 7 + prefix: --stride_x + stride_y: + type: string + default: None + inputBinding: + position: 8 + prefix: --stride_y + + +outputs: [] diff --git a/component-library/input/image-tiling.ipynb b/component-library/input/image-tiling.ipynb new file mode 100644 index 00000000..04886717 --- /dev/null +++ b/component-library/input/image-tiling.ipynb @@ -0,0 +1,182 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2e8a0fac-adbd-4428-90ea-869aee2b95bf", + "metadata": {}, + "source": [ + "## Xview Dataset clipping\n", + "This is a component designed to clip the dataset provided by xview. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c0421aaf-4da4-4a66-b626-e8962315afe8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: Pillow in /Users/maxdorzweiler/anaconda3/lib/python3.11/site-packages (9.4.0)\n" + ] + } + ], + "source": [ + "!pip install Pillow" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0b7ad533-dabc-488e-bf65-59a54a85f3f9", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import os\n", + "from PIL import Image\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c8df047b-6167-43f1-9a31-d836580180ac", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#directory_path is the path to the folder with the unzipped .tif images from xview dataset \n", + "directory_path = os.environ.get(\"directory_path\")\n", + "\n", + "#destination is the path to the folder which saves all the extracted tiles. \n", + "destination = os.environ.get(\"destination\")\n", + "\n", + "#Each image is cropped using a rectangular window with edge_length tile_size_x and tile_size_y which has to be given in number of pixels \n", + "tile_size_x = os.environ.get(\"tile_size_x\")\n", + "tile_size_y = os.environ.get(\"tile_size_y\")\n", + "\n", + "#stride_x is the length in pixels the sliding window is moved to the right after each step\n", + "#stride_y is the length in pixels the sliding window is moved down after completing a row\n", + "#For tumbling window stride_x must equal tile_size_x and stride_y must equal tile_size_y\n", + "stride_x = os.environ.get(\"stride_x\")\n", + "stride_y = os.environ.get(\"stride_y\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8ed74228-937d-4d5d-b6d9-97026af3eb50", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def clear_directory(directory):\n", + " for item in os.listdir(directory):\n", + " item_path = os.path.join(directory, item)\n", + " # Check if the item is a file\n", + " if os.path.isfile(item_path):\n", + " # If it's a file, remove it\n", + " os.remove(item_path)\n", + " # If it's a directory, recursively clear it\n", + " elif os.path.isdir(item_path):\n", + " clear_directory(item_path)\n", + " os.rmdir(item_path)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "d6e89f94-490a-4c1a-80cd-f2ab30adb374", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def sliding_window(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y):\n", + " \n", + " if os.path.isdir(destination):\n", + " clear_directory(destination)\n", + " else: \n", + " os.makedirs(destination)\n", + " \n", + " \n", + " for item in os.listdir(directory_path):\n", + " if item == \".DS_Store\":\n", + " continue\n", + " item_path = os.path.join(directory_path, item)\n", + " clipped_item = os.path.join(destination, item)\n", + " os.makedirs(clipped_item)\n", + " \n", + " image = Image.open(item_path)\n", + " width, height = image.size\n", + " \n", + " x_range = [0]\n", + " while(x_range[-1] + stride_x + tile_size_x < width):\n", + " x_range += [x_range[-1] + stride_x]\n", + " \n", + " y_range = [0]\n", + " while(y_range[-1] + stride_y + tile_size_y < height):\n", + " y_range += [y_range[-1] + stride_y]\n", + " \n", + " counter = 0\n", + " for x in x_range:\n", + " for y in y_range:\n", + " cropped = image.crop((x,y, x+tile_size_x, y+tile_size_y))\n", + "\n", + " file_name = \"{}\".format(y)\n", + " file_name += \"_{}\".format(x) + \".tif\"\n", + " path = os.path.join(clipped_item, file_name)\n", + " \n", + " cropped.save(path)\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ce60bf47-7235-43a1-a4f7-5b81080d617d", + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "stat: path should be string, bytes, os.PathLike or integer, not NoneType", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m sliding_window(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y)\n", + "Cell \u001b[0;32mIn[9], line 3\u001b[0m, in \u001b[0;36msliding_window\u001b[0;34m(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msliding_window\u001b[39m(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y):\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39misdir(destination):\n\u001b[1;32m 4\u001b[0m clear_directory(destination)\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m: \n", + "File \u001b[0;32m:42\u001b[0m, in \u001b[0;36misdir\u001b[0;34m(s)\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: stat: path should be string, bytes, os.PathLike or integer, not NoneType" + ] + } + ], + "source": [ + "sliding_window(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/component-library/input/image-tiling.job.yaml b/component-library/input/image-tiling.job.yaml new file mode 100644 index 00000000..6e6b85b3 --- /dev/null +++ b/component-library/input/image-tiling.job.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: image-tiling +spec: + template: + spec: + containers: + - name: image-tiling + image: docker.io/mdorzweiler/claimed-image-tiling:0.1 + workingDir: /opt/app-root/src/ + command: ["/opt/app-root/bin/ipython","claimed_image-tiling.ipynb"] + env: + - name: log_level + value: value_of_log_level + - name: directory_path + value: value_of_directory_path + - name: destination + value: value_of_destination + - name: tile_size_x + value: value_of_tile_size_x + - name: tile_size_y + value: value_of_tile_size_y + - name: stride_x + value: value_of_stride_x + - name: stride_y + value: value_of_stride_y + restartPolicy: OnFailure + imagePullSecrets: + - name: image_pull_secret \ No newline at end of file diff --git a/component-library/input/image-tiling.yaml b/component-library/input/image-tiling.yaml new file mode 100644 index 00000000..c65e7dea --- /dev/null +++ b/component-library/input/image-tiling.yaml @@ -0,0 +1,31 @@ +name: image-tiling +description: "## Xview Dataset clipping – CLAIMED V0.1" + +inputs: +- {name: log_level, type: String, description: "update log level", default: "INFO"} +- {name: directory_path, type: String, description: "directory_path is the path to the folder with the unzipped .tif images from xview dataset"} +- {name: destination, type: String, description: "destination is the path to the folder which saves all the extracted tiles."} +- {name: tile_size_x, type: String, description: "Each image is cropped using a rectangular window with edge_length tile_size_x and tile_size_y which has to be given in number of pixels"} +- {name: tile_size_y, type: String, description: ""} +- {name: stride_x, type: String, description: "For tumbling window stride_x must equal tile_size_x and stride_y must equal tile_size_y"} +- {name: stride_y, type: String, description: ""} + + +outputs: + + +implementation: + container: + image: docker.io/mdorzweiler/claimed-image-tiling:0.1 + command: + - sh + - -ec + - | + ipython ./claimed_image-tiling.ipynb log_level="${0}" directory_path="${1}" destination="${2}" tile_size_x="${3}" tile_size_y="${4}" stride_x="${5}" stride_y="${6}" + - {inputValue: log_level} + - {inputValue: directory_path} + - {inputValue: destination} + - {inputValue: tile_size_x} + - {inputValue: tile_size_y} + - {inputValue: stride_x} + - {inputValue: stride_y}