From e1fa3e84fe0e89c670e37db7f5f315598032d24c Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 4 Oct 2023 12:40:42 +0200 Subject: [PATCH] Allow cloud mask as parameter to cloud coverage process --- .../WPS20ExecuteCloudCoverageCustomMask.json | 1 + ...20ExecuteCloudCoverageOnCLMCustomMask.json | 1 + .../tests/wps/test_v20_cloud_coverage.py | 70 +++++++++++++++++++ .../ows/wps/processes/get_cloud_coverage.py | 58 ++++++++++----- 4 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json create mode 100644 autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json new file mode 100644 index 000000000..09cd12ef1 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.33}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json new file mode 100644 index 000000000..b4da905b2 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.2569736339319832}} \ No newline at end of file diff --git a/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py b/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py index 118f717d7..d40c35738 100644 --- a/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py +++ b/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py @@ -120,6 +120,42 @@ def getRequest(self): return (params, "xml") +class WPS20ExecuteCloudCoverageCustomMask( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["scl_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON ((69.19913354922439908 80.1406125504016984, 69.19921132386413376 80.13719046625288911, 69.20360559100976161 80.13719046625288911, 69.20364447832963606 80.14065143772157285, 69.20364447832963606 80.14065143772157285, 69.19913354922439908 80.1406125504016984)) + + + + + [1, 2, 3, 8] + + + + + + """ + return (params, "xml") + + + class WPS20ExecuteCloudCoverageEmptyResponse( ContentTypeCheckMixIn, testbase.JSONTestCase ): @@ -237,3 +273,37 @@ def getRequest(self): """ return (params, "xml") + +class WPS20ExecuteCloudCoverageOnCLMCustomMask( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["clm_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((16.689481892710717 47.49088479184637,16.685862090779757 47.49375416408526,16.717934765940697 47.50045333252222,16.689481892710717 47.49088479184637)) + + + + + 7 + + + + + + """ + return (params, "xml") diff --git a/eoxserver/services/ows/wps/processes/get_cloud_coverage.py b/eoxserver/services/ows/wps/processes/get_cloud_coverage.py index b94f3ebfe..f332711e7 100644 --- a/eoxserver/services/ows/wps/processes/get_cloud_coverage.py +++ b/eoxserver/services/ows/wps/processes/get_cloud_coverage.py @@ -29,8 +29,9 @@ import concurrent import functools from datetime import datetime +import json from uuid import uuid4 -from typing import List, Callable, Optional +from typing import List, Callable, Optional, Any from osgeo import ogr, osr @@ -38,6 +39,7 @@ from eoxserver.contrib import gdal from eoxserver.resources.coverages import models from eoxserver.backends.access import gdal_open +from eoxserver.services.ows.wps.exceptions import InvalidInputValueError from eoxserver.services.ows.wps.parameters import ( LiteralData, ComplexData, @@ -74,6 +76,12 @@ class CloudCoverageProcess(Component): title="Geometry", formats=[FormatText()], ), + "cloud_mask": ComplexData( + "cloud_mask", + optional=True, + title="Values of data which are interpreted as cloud", + formats=[FormatJSON()], + ), } outputs = { @@ -91,6 +99,13 @@ class CloudCoverageProcess(Component): SCL_LAYER_THIN_CIRRUS = 10 SCL_LAYER_SATURATED_OR_DEFECTIVE = 1 + DEFAULT_SCL_CLOUD_MASK = [ + SCL_LAYER_CLOUD_MEDIUM_PROBABILITY, + SCL_LAYER_CLOUD_HIGH_PROBABILITY, + SCL_LAYER_THIN_CIRRUS, + SCL_LAYER_SATURATED_OR_DEFECTIVE, + ] + # https://labo.obs-mip.fr/multitemp/sentinel-2/majas-native-sentinel-2-format/#English # anything nonzero should be cloud, however that includes also cloud shadows which # have a lot of false positives (or shadows that are not visible to the naked eye) @@ -108,10 +123,19 @@ def execute( begin_time, end_time, geometry, + cloud_mask, result, ): wkt_geometry = geometry[0].text + if cloud_mask: + # NOTE: cloud mask could be list or integer bitmask based on type, + # so just accept json + try: + cloud_mask = json.loads(cloud_mask[0].text) + except ValueError: + raise InvalidInputValueError("cloud_mask", "Invalid cloud mask value") + # TODO Use queue object for more complex query if parent_product__footprint is not enough relevant_coverages = models.Coverage.objects.filter( parent_product__begin_time__lte=end_time, @@ -145,6 +169,7 @@ def execute( calculation_fun=calculation_fun, wkt_geometry=wkt_geometry, no_data_value=no_data_value, + cloud_mask=cloud_mask, ), [coverage.arraydata_items.get() for coverage in coverages], ) @@ -165,38 +190,39 @@ def execute( def cloud_coverage_ratio_in_geometry( data_item: models.ArrayDataItem, wkt_geometry: str, - calculation_fun: Callable[[List[int]], float], + calculation_fun: Callable[[List[int], Any], float], no_data_value: Optional[int], + cloud_mask: Any, ) -> float: histogram = _histogram_in_geometry( data_item=data_item, wkt_geometry=wkt_geometry, no_data_value=no_data_value, ) - return calculation_fun(histogram) + return calculation_fun(histogram, cloud_mask) -def cloud_coverage_ratio_for_CLM(histogram: List[int]) -> float: +def cloud_coverage_ratio_for_CLM(histogram: List[int], cloud_mask: Any) -> float: + cloud_mask = ( + cloud_mask + if cloud_mask is not None + else CloudCoverageProcess.CLM_MASK_ONLY_CLOUD + ) num_is_cloud = sum( - value - for index, value in enumerate(histogram) - if index & CloudCoverageProcess.CLM_MASK_ONLY_CLOUD > 0 + value for index, value in enumerate(histogram) if index & cloud_mask > 0 ) num_pixels = sum(histogram) return ((num_is_cloud / num_pixels)) if num_pixels != 0 else 0.0 -def cloud_coverage_ratio_for_SCL(histogram: List[int]) -> float: - num_cloud = sum( - histogram[scl_value] - for scl_value in [ - CloudCoverageProcess.SCL_LAYER_CLOUD_MEDIUM_PROBABILITY, - CloudCoverageProcess.SCL_LAYER_CLOUD_HIGH_PROBABILITY, - CloudCoverageProcess.SCL_LAYER_THIN_CIRRUS, - CloudCoverageProcess.SCL_LAYER_SATURATED_OR_DEFECTIVE, - ] +def cloud_coverage_ratio_for_SCL(histogram: List[int], cloud_mask: Any) -> float: + cloud_mask = ( + cloud_mask + if cloud_mask is not None + else CloudCoverageProcess.DEFAULT_SCL_CLOUD_MASK ) + num_cloud = sum(histogram[scl_value] for scl_value in cloud_mask) num_no_data = histogram[CloudCoverageProcess.SCL_LAYER_NO_DATA]