Merge pull request #568 from EOxServer/configurable-scl-values

Allow cloud mask as parameter to cloud coverage process

autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json

@@ -0,0 +1 @@
+{"result": {"2020-02-15T07:59:36.409000+00:00": 0.33}}

autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json

@@ -0,0 +1 @@
+{"result": {"2020-02-15T07:59:36.409000+00:00": 0.2569736339319832}}

autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py

@@ -120,6 +120,42 @@ def getRequest(self):
         return (params, "xml")
 
 
+class WPS20ExecuteCloudCoverageCustomMask(
+    ContentTypeCheckMixIn, testbase.JSONTestCase
+):
+    fixtures = testbase.JSONTestCase.fixtures + ["scl_cloud_coverages.json"]
+
+    expectedContentType = "application/json; charset=utf-8"
+
+    def getRequest(self):
+        params = """<wps:Execute
+        version="2.0.0"
+        service="WPS"
+        response="raw"
+        mode="sync"
+        xmlns:wps="http://www.opengis.net/wps/2.0"
+        xmlns:ows="http://www.opengis.net/ows/2.0" >
+          <ows:Identifier>CloudCoverage</ows:Identifier>
+          <wps:Input id="begin_time"><wps:Data>2020-01-01</wps:Data></wps:Input>
+          <wps:Input id="end_time"><wps:Data>2020-05-31</wps:Data></wps:Input>
+          <wps:Input id="geometry">
+            <wps:Data>
+              <wps:ComplexData mimeType="text/plain">POLYGON ((69.19913354922439908 80.1406125504016984, 69.19921132386413376 80.13719046625288911, 69.20360559100976161 80.13719046625288911, 69.20364447832963606 80.14065143772157285, 69.20364447832963606 80.14065143772157285, 69.19913354922439908 80.1406125504016984))</wps:ComplexData>
+            </wps:Data>
+          </wps:Input>
+          <wps:Input id="cloud_mask">
+            <wps:Data>
+              <wps:ComplexData mimeType="text/plain">[1, 2, 3, 8]</wps:ComplexData>
+            </wps:Data>
+          </wps:Input>
+          <wps:Output id="result" >
+          </wps:Output>
+        </wps:Execute>
+        """
+        return (params, "xml")
+
+
+
 class WPS20ExecuteCloudCoverageEmptyResponse(
     ContentTypeCheckMixIn, testbase.JSONTestCase
 ):
@@ -237,3 +273,37 @@ def getRequest(self):
         </wps:Execute>
         """
         return (params, "xml")
+
+class WPS20ExecuteCloudCoverageOnCLMCustomMask(
+    ContentTypeCheckMixIn, testbase.JSONTestCase
+):
+    fixtures = testbase.JSONTestCase.fixtures + ["clm_cloud_coverages.json"]
+
+    expectedContentType = "application/json; charset=utf-8"
+
+    def getRequest(self):
+        params = """<wps:Execute
+        version="2.0.0"
+        service="WPS"
+        response="raw"
+        mode="sync"
+        xmlns:wps="http://www.opengis.net/wps/2.0"
+        xmlns:ows="http://www.opengis.net/ows/2.0" >
+          <ows:Identifier>CloudCoverage</ows:Identifier>
+          <wps:Input id="begin_time"><wps:Data>2020-01-01</wps:Data></wps:Input>
+          <wps:Input id="end_time"><wps:Data>2020-05-31</wps:Data></wps:Input>
+          <wps:Input id="geometry">
+            <wps:Data>
+              <wps:ComplexData mimeType="text/plain">POLYGON((16.689481892710717 47.49088479184637,16.685862090779757 47.49375416408526,16.717934765940697 47.50045333252222,16.689481892710717 47.49088479184637))</wps:ComplexData>
+            </wps:Data>
+          </wps:Input>
+          <wps:Input id="cloud_mask">
+            <wps:Data>
+              <wps:ComplexData mimeType="text/plain">7</wps:ComplexData>
+            </wps:Data>
+          </wps:Input>
+          <wps:Output id="result" >
+          </wps:Output>
+        </wps:Execute>
+        """
+        return (params, "xml")

eoxserver/services/ows/wps/processes/get_cloud_coverage.py

@@ -29,15 +29,17 @@
 import concurrent
 import functools
 from datetime import datetime
+import json
 from uuid import uuid4
-from typing import List, Callable, Optional
+from typing import List, Callable, Optional, Any
 
 from osgeo import ogr, osr
 
 from eoxserver.core import Component
 from eoxserver.contrib import gdal
 from eoxserver.resources.coverages import models
 from eoxserver.backends.access import gdal_open
+from eoxserver.services.ows.wps.exceptions import InvalidInputValueError
 from eoxserver.services.ows.wps.parameters import (
     LiteralData,
     ComplexData,
@@ -74,6 +76,12 @@ class CloudCoverageProcess(Component):
             title="Geometry",
             formats=[FormatText()],
         ),
+        "cloud_mask": ComplexData(
+            "cloud_mask",
+            optional=True,
+            title="Values of data which are interpreted as cloud",
+            formats=[FormatJSON()],
+        ),
     }
 
     outputs = {
@@ -91,6 +99,13 @@ class CloudCoverageProcess(Component):
     SCL_LAYER_THIN_CIRRUS = 10
     SCL_LAYER_SATURATED_OR_DEFECTIVE = 1
 
+    DEFAULT_SCL_CLOUD_MASK = [
+        SCL_LAYER_CLOUD_MEDIUM_PROBABILITY,
+        SCL_LAYER_CLOUD_HIGH_PROBABILITY,
+        SCL_LAYER_THIN_CIRRUS,
+        SCL_LAYER_SATURATED_OR_DEFECTIVE,
+    ]
+
     # https://labo.obs-mip.fr/multitemp/sentinel-2/majas-native-sentinel-2-format/#English
     # anything nonzero should be cloud, however that includes also cloud shadows which
     # have a lot of false positives (or shadows that are not visible to the naked eye)
@@ -108,10 +123,19 @@ def execute(
         begin_time,
         end_time,
         geometry,
+        cloud_mask,
         result,
     ):
         wkt_geometry = geometry[0].text
 
+        if cloud_mask:
+            # NOTE: cloud mask could be list or integer bitmask based on type,
+            #       so just accept json
+            try:
+                cloud_mask = json.loads(cloud_mask[0].text)
+            except ValueError:
+                raise InvalidInputValueError("cloud_mask", "Invalid cloud mask value")
+
         # TODO Use queue object for more complex query if parent_product__footprint is not enough
         relevant_coverages = models.Coverage.objects.filter(
             parent_product__begin_time__lte=end_time,
@@ -145,6 +169,7 @@ def execute(
                     calculation_fun=calculation_fun,
                     wkt_geometry=wkt_geometry,
                     no_data_value=no_data_value,
+                    cloud_mask=cloud_mask,
                 ),
                 [coverage.arraydata_items.get() for coverage in coverages],
             )
@@ -165,38 +190,39 @@ def execute(
 def cloud_coverage_ratio_in_geometry(
     data_item: models.ArrayDataItem,
     wkt_geometry: str,
-    calculation_fun: Callable[[List[int]], float],
+    calculation_fun: Callable[[List[int], Any], float],
     no_data_value: Optional[int],
+    cloud_mask: Any,
 ) -> float:
     histogram = _histogram_in_geometry(
         data_item=data_item,
         wkt_geometry=wkt_geometry,
         no_data_value=no_data_value,
     )
-    return calculation_fun(histogram)
+    return calculation_fun(histogram, cloud_mask)
 
 
-def cloud_coverage_ratio_for_CLM(histogram: List[int]) -> float:
+def cloud_coverage_ratio_for_CLM(histogram: List[int], cloud_mask: Any) -> float:
+    cloud_mask = (
+        cloud_mask
+        if cloud_mask is not None
+        else CloudCoverageProcess.CLM_MASK_ONLY_CLOUD
+    )
     num_is_cloud = sum(
-        value
-        for index, value in enumerate(histogram)
-        if index & CloudCoverageProcess.CLM_MASK_ONLY_CLOUD > 0
+        value for index, value in enumerate(histogram) if index & cloud_mask > 0
     )
 
     num_pixels = sum(histogram)
     return ((num_is_cloud / num_pixels)) if num_pixels != 0 else 0.0
 
 
-def cloud_coverage_ratio_for_SCL(histogram: List[int]) -> float:
-    num_cloud = sum(
-        histogram[scl_value]
-        for scl_value in [
-            CloudCoverageProcess.SCL_LAYER_CLOUD_MEDIUM_PROBABILITY,
-            CloudCoverageProcess.SCL_LAYER_CLOUD_HIGH_PROBABILITY,
-            CloudCoverageProcess.SCL_LAYER_THIN_CIRRUS,
-            CloudCoverageProcess.SCL_LAYER_SATURATED_OR_DEFECTIVE,
-        ]
+def cloud_coverage_ratio_for_SCL(histogram: List[int], cloud_mask: Any) -> float:
+    cloud_mask = (
+        cloud_mask
+        if cloud_mask is not None
+        else CloudCoverageProcess.DEFAULT_SCL_CLOUD_MASK
     )
+    num_cloud = sum(histogram[scl_value] for scl_value in cloud_mask)
 
     num_no_data = histogram[CloudCoverageProcess.SCL_LAYER_NO_DATA]