Partial pipeline example

.gitignore

@@ -17,3 +17,6 @@ dist/
 
 # Documentation
 site/
+
+# Jupyter
+.ipynb_checkpoints

examples/partial-pipeline/DEMO-exceldata.ipynb

@@ -0,0 +1,236 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "11544030-eb82-4bfa-8b6e-c6833b66e750",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from oteutils.kb import *\n",
+    "from oteutils.utils import *\n",
+    "# Connect to the OTEAPI services\n",
+    "client = OTEPipelines(url='http://localhost:8080')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "0a5d8ac6-37a9-4347-9ce0-88cf46f91471",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create configurations for filters\n",
+    "\n",
+    "# Fetch mappings as triples from .ttl file\n",
+    "tsp=Triplestore(backend=\"rdflib\")\n",
+    "tsp.parse('./mappings/Vipcoat_fetch.ttl')\n",
+    "dataMappings=[]\n",
+    "for (s, p, o) in tsp.triples(None,MAP.mapsTo,None): \n",
+    "    dataMappings.append((s, p, o))\n",
+    "# print(dataMappings)\n",
+    "\n",
+    "# Mapping configuration for the mapping filter\n",
+    "generatorMappingConfig = MappingConfig(\n",
+    "    mappingType=\"mappings\",\n",
+    "    triples = dataMappings,\n",
+    "    configuration={\n",
+    "    }     \n",
+    ")\n",
+    "\n",
+    "# Configuration that fetches data and stores to a json file\n",
+    "generatorFilterConfig = FunctionConfig(\n",
+    "    functionType=\"dlite/translate\",\n",
+    "    configuration={'metadata':'http://onto-ns.com/meta/vipcoat/demo/0.2/data',\n",
+    "                    'storage_path':\"/dome_demo/models/vipcoat_fetch_properties.json\",\n",
+    "                   \"fileDestination\": \"/oteapi-dlite/output_dlite.json\"} \n",
+    ")\n",
+    "\n",
+    "\n",
+    "createcollectionfilter = FilterConfig(\n",
+    "      filterType=\"dlite/create-collection\",\n",
+    "    description=\"\",\n",
+    "    configuration={},\n",
+    ")\n",
+    "\n",
+    "\n",
+    "# Create Filters\n",
+    "generatorMappings= client.create_mapping(generatorMappingConfig)\n",
+    "generatorFilter=client.create_function(generatorFilterConfig)\n",
+    "createCollection=client.create_filter(createcollectionfilter)\n",
+    "\n",
+    "# Build Pipeline\n",
+    "generatorPipeline =generatorMappings >> generatorFilter >> createCollection \n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "14c74d9c-1e14-4dd2-8f6b-e3d924918aad",
+   "metadata": {},
+   "source": [
+    "## Fetch Data Pipeline w.r.t resource URL "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1bb836c2-dbe5-4817-89b2-18849c215c21",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Please connect to: client = OTEPipelines(url='http://localhost:8080') use otelib pypi package\n",
+      "\n",
+      "    \n",
+      "        f_1 = client.create_dataresource(ResourceConfig(configuration={'excel_config': {'worksheet': 'EIS_VALUES', 'header_row': '1', 'row_from': '2', 'row_to': '10'}, 'metadata': 'http://onto-ns.com/meta/vipcoat/demo/0.2/EIS', 'storage_path': '/dome_demo/models/vipcoat_EIS_properties.json'},downloadUrl='file:///dome_demo/data/vipcoatEIS.xlsx' ,mediaType='application/vnd.dlite-xlsx'))\n",
+      "    \n",
+      "    \n",
+      "    \n",
+      "    \n",
+      "\n",
+      "    \n",
+      "    \n",
+      "         f_2 = client.create_mapping(MappingConfig(configuration={},mappingType='mappings' ,triples=[('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Impedance_img', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#imginaryImpedanceValue'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Impedance_Phase', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#phaseOfImpedance'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Frequency', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#frequency'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Impedance_real', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#realImpedanceValue'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Electrode_potential', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#electrodePotential'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Time', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#time'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Absolute_Impedance', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#absoluteImpedanceValue')]))\n",
+      "    \n",
+      "    \n",
+      "    \n",
+      "\n",
+      "storedPipeline = f_1  >>   f_2   \n"
+     ]
+    }
+   ],
+   "source": [
+    "# File links given as download urls in the configuration of the data resources\n",
+    "\n",
+    "mpr_file=\"https://vipcoat-bucket.s3.eu-west-2.amazonaws.com/experiments/raw/2-amino-4-methylthiazole|CC1=CSC(=N1)N/101022_1mM_2amino4methylthiazole_run1/101022_1mM_2amino4methylthiazole_run1_11_PEIS_C01.mpr\"\n",
+    "excel_file=\"file:///dome_demo/data/vipcoatEIS.xlsx\"\n",
+    "\n",
+    "pipe_iri = find_pipeline_by_config_iri(excel_file)\n",
+    "print(fetch_pipeline(pipe_iri))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b62a9a12-cfb4-4651-9675-81b88f51f4e5",
+   "metadata": {},
+   "source": [
+    "## Copy the above filter configurations and Pipeline and run with the generator pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f79a8991-1e16-4bb4-98bf-85849834791d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "# Please connect to: client = OTEPipelines(url='http://localhost:8080') use otelib pypi package\n",
+    "\n",
+    "    \n",
+    "f_1 = client.create_dataresource(ResourceConfig(configuration={'storage_path': '/dome_demo/models/vipcoat_EIS_properties.json', 'metadata': 'http://onto-ns.com/meta/vipcoat/demo/0.2/EIS', 'excel_config': {'worksheet': 'EIS_VALUES', 'header_row': '1', 'row_from': '2', 'row_to': '10'}},downloadUrl='file:///dome_demo/data/vipcoatEIS.xlsx' ,mediaType='application/vnd.dlite-xlsx'))\n",
+    "    \n",
+    "    \n",
+    "    \n",
+    "f_2 = client.create_mapping(MappingConfig(configuration={},mappingType='mappings' ,triples=[('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Impedance_Phase', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#phaseOfImpedance'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Absolute_Impedance', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#absoluteImpedanceValue'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Impedance_real', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#realImpedanceValue'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Impedance_img', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#imginaryImpedanceValue'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Time', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#time'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Electrode_potential', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#electrodePotential'), ('http://onto-ns.com/meta/vipcoat/demo/0.2/EIS#Frequency', 'http://emmo.info/domain-mappings#mapsTo', 'http://onto-ns.com/meta/vipcoat/demo/0.2/properties#frequency')]))\n",
+    "    \n",
+    "    \n",
+    "    \n",
+    "\n",
+    "storedPipeline = f_1  >>   f_2    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "fd4d1aaf-70d1-4628-be35-4ac6714ede48",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "b'{\"collection_id\":\"66e6707a-fa9a-48a7-8bd1-3f7419fdb04a\"}'"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Build a new pipeline by merging the fetched one with the generator Pipeline\n",
+    "pipeline = storedPipeline >> generatorPipeline\n",
+    "\n",
+    "# Execute the pipeline\n",
+    "pipeline.get()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f0281799-4470-4a89-b68b-b6db53eae0fd",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: './oteapi-dlite/output_dlite.json'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[6], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m## Display the data stored in the generated json file\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mjson\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m f \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m./oteapi-dlite/output_dlite.json\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m data \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mload(f)\n\u001b[1;32m      6\u001b[0m f\u001b[38;5;241m.\u001b[39mclose()\n",
+      "File \u001b[0;32m~/.venv/ote/lib/python3.10/site-packages/IPython/core/interactiveshell.py:284\u001b[0m, in \u001b[0;36m_modified_open\u001b[0;34m(file, *args, **kwargs)\u001b[0m\n\u001b[1;32m    277\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[1;32m    278\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    279\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    280\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    281\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    282\u001b[0m     )\n\u001b[0;32m--> 284\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mio_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: './oteapi-dlite/output_dlite.json'"
+     ]
+    }
+   ],
+   "source": [
+    "## Display the data stored in the generated json file\n",
+    "\n",
+    "import json\n",
+    "f = open('./oteapi-dlite/output_dlite.json')\n",
+    "data = json.load(f)\n",
+    "f.close()\n",
+    "# print(data)\n",
+    "import pandas as pd\n",
+    "df = pd.DataFrame(data=data)\n",
+    "print(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f8e58dfa-da62-437c-8d0a-39dd7f3887ba",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/partial-pipeline/README.md

@@ -0,0 +1,37 @@
+# Proof of Concept Demonstrator for Partial pipelines 
+
+A demonstrator for showing data interoperability between data from a provider and data required by a data consumer (who might require it in another form), by making use of OTEAPI (OntoTrans) and  DLite.
+
+## User Stories
+
+User stories are explained in DEMO.ipynb
+
+
+
+### How to execute?
+
+docker-compose up --build -d
+
+Note: on Ubuntu, it has to be run as super user:
+sudo docker-compose up --build -d
+
+Go to http://localhost:3030/#/manage
+login: 
+user:admin
+pwd:secret
+
+![Alt text](image.png)
+create OTE2 database in fuseki:
+![Alt text](image-1.png)
+
+and run Jupyter lab and use DEMO.ipynb
+
+Jupyter lab can be installed by running (in Ubuntu):
+sudo apt install jupyter
+pip install jupyterlab
+
+### some packages to be installed :
+Go to oteutils folder:
+
+- pip install -r requirements.txt
+

examples/partial-pipeline/createpipeline.py

@@ -0,0 +1,18 @@
+
+
+# Please connect to: client = OTEPipelines(url='http://localhost:8080') use otelib pypi package
+{% for h in hh %}
+    {% if h.ft == 'ResourceConfig' %}
+        f_{{ loop.index }} = client.create_dataresource(ResourceConfig(configuration={{h.config.configuration}},downloadUrl={{"'" + h.config.downloadUrl + "'"}} ,mediaType={{"'" +h.config.mediaType + "'"}}))
+    {% endif %}
+    {% if h.ft == 'MappingConfig' %}
+         f_{{ loop.index }} = client.create_mapping(MappingConfig(configuration={{h.config.configuration}},mappingType={{"'" + h.config.mappingType + "'"}} ,triples={{h.config.triples }}))
+    {% endif %}
+    {% if h.ft == 'FunctionConfig' %}
+         f_{{ loop.index }} = client.create_function(FunctionConfig(configuration={{h.config.configuration}},functionType={{"'" + h.config.functionType + "'"}}))
+    {% endif %}
+    {% if h.ft == 'FilterConfig' %}
+         f_{{ loop.index }} = client.create_filter(FilterConfig(configuration={{h.config.configuration}},filterType={{"'" + h.config.filterType + "'"}}))
+    {% endif %}
+{% endfor %}
+storedPipeline = {% for i in range(hh|length) %}f_{{ i + 1 }} {% if hh|length > i+1 %} >> {% endif %}  {% endfor %}

examples/partial-pipeline/data/Hitachi_imageName1.txt

@@ -0,0 +1,43 @@
+[SemImageFile]
+InstructName=S-3400
+SerialNumber=340503-01
+DataNumber=S3400
+SampleName=
+Format=tif
+ImageName=Hitachi_imageName1.tif
+Directory=C:\data\experiment1
+Date=07/12/2023
+Time=12:05:16
+Media=NE[user_data ]
+DataSize=2560x1920
+PixelSize=248.0469
+SignalName=SE
+AcceleratingVoltage=10000 Volt
+DecelerationVoltage=0 Volt
+Magnification=200
+WorkingDistance=6100 um
+EmissionCurrent=75000 nA
+LensMode=
+PhotoSize=1000
+Vacuum=
+MicronMarker=200000
+SubMagnification=0
+SubSignalName=
+SpecimenBias=0 V
+Condencer1=20000
+ScanSpeed=Slow1
+CalibrationScanSpeed=25
+ColorMode=Grayscale
+ColorPalette=
+ScreenMode=Small Screen
+Comment=
+KeyWord1= 
+KeyWord2= 
+Condition=Vacc=10kV   Mag=x200   WD=6.1mm
+DataDisplayCombine=1
+StageType=5
+StagePositionX=76097000
+StagePositionY=27701000
+StagePositionR=0
+StagePositionZ=31000000
+StagePositionT=0

examples/partial-pipeline/data/JEOL_imageName2.txt

@@ -0,0 +1,24 @@
+$CM_FORMAT Bitmap
+$CM_VERSION 1.4
+$CM_COMMENT 
+$CM_DATE 12/4/2015
+$CM_TIME 14:31:24 PM
+$CM_OPERATOR GENERAL
+$CM_COMPANY JEOL
+$CM_IMAGEID : 00847A084726
+$CM_INSTRUMENT JAMP 9500F -- 9500
+$CM_ACCEL_VOLT 10.0
+$CM_MAG 500
+$CM_SIGNAL SEI
+$$SM_FILM_NUMBER 0
+$$SM_WD 23.5
+$$SM_MICRON_BAR 53
+$$SM_MICRON_MARKER 10µm
+$$SM_FONT_SIZE 24 48
+$$SM_DISPLAY_MODE STANDARD
+$$SM_COLUMN_MODE SEM
+$$SM_SEI_DETECTOR_MODE 0
+$$SM_SEI_DETECTOR_LEVEL 0
+$SM_GB_GUN_VOLT 0.0
+$SM_GB_BIAS_VOLT 0.0
+$SM_COLUM_ECP_ANGLE 0.0