Merge pull request #803 from no23reason/dho-cq-754-flex-fun-emancipat…

…ion-redux

feat: split the gooddata-flight-server into two pacakges

.envrc

@@ -9,6 +9,7 @@ export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-api-client/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-sdk/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-pandas/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flight-server/"
+export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flexfun/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-dbt/"
 
 export PATH="${PATH}:${SCRIPT_DIR}/gooddata-sdk/bin"

.github/workflows/rw-collect-changes.yaml

@@ -47,3 +47,4 @@ jobs:
               - 'gooddata-api-client/**'
               - 'gooddata-dbt/**'
               - 'gooddata-flight-server/**'
+              - 'gooddata-flexfun/**'

.github/workflows/rw-python-tests.yaml

@@ -24,7 +24,7 @@ jobs:
         if: ${{ matrix.python_version == 'py312' }}
         uses: codecov/codecov-action@v3
         with:
-          files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml
+          files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml,./gooddata-flexfun/coverage.xml
   lint-and-format-check:
     runs-on: ubuntu-latest
     if: ${{inputs.changed-python-modules == 'true'}}

.sonar.settings

@@ -1,4 +1,4 @@
 # (C) 2021 GoodData Corporation
-sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas
+sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexfun
 sonar.exclusions=gooddata-api-client/**/*
 sonar.python.version=3.9, 3.10, 3.11, 3.12

.sonarcloud.properties

@@ -1,4 +1,4 @@
 # (C) 2022 GoodData Corporation
-sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas
+sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexfun
 sonar.exclusions=gooddata-api-client/**/*
 sonar.python.version=3.9, 3.10, 3.11, 3.12

gooddata-flexfun/.gitignore

@@ -0,0 +1,69 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.venv/
+.python-version
+.pytest_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+docs/_autosummary/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints
+
+test_data

gooddata-flexfun/.readthedocs.yaml

@@ -0,0 +1,19 @@
+# (C) 2024 GoodData Corporation
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: gooddata-flexfun/docs/conf.py
+
+python:
+  install:
+    - requirements: gooddata-flexfun/docs/requirements.txt

gooddata-flexfun/Makefile

@@ -0,0 +1,19 @@
+# (C) 2024 GoodData Corporation
+include ../project_common.mk
+
+.PHONY: docs
+docs:
+	tox $(TOX_FLAGS) -e docs
+
+
+.PHONY: dev-certs
+dev-certs:
+	mkdir -p test_data
+	# create CA with self-signed certificate > generate keys + cert requests for server and client > sign
+	cd test_data && \
+		rm -f *.pem && \
+		openssl req -x509 -newkey rsa:4096 -days 3650 -nodes -keyout ca-key.pem -out ca-cert.pem -subj "/C=NA/ST=Unknown/L=Unknown/O=Unknown/OU=Unknown/CN=localhost/[email protected]" && \
+		openssl req -newkey rsa:4096 -nodes -keyout server-key.pem -out server-req.pem -subj "/C=NA/ST=Unknown/L=Unknown/O=Unknown/OU=Computer/CN=localhost/[email protected]" && \
+		openssl req -newkey rsa:4096 -nodes -keyout client-key.pem -out client-req.pem -subj "/C=NA/ST=Unknown/L=Unknown/O=Unknown/OU=Computer/CN=testClient1/[email protected]" && \
+		openssl x509 -req -in server-req.pem -days 3650 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem && \
+		openssl x509 -req -in client-req.pem -days 3650 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem

gooddata-flexfun/README.md

@@ -0,0 +1,63 @@
+# GoodData FlexFun
+
+The GoodData FlexFun package is a GoodData Flight Server-compatible extension
+that provides a framework for hosting custom pluggable functions called FlexFuns.
+These can be used to act as a dataset for GoodData FlightRPC data sources.
+
+## What is a FlexFun?
+
+In essence, FlexFun is a class that provides a set of methods that can be called by the GoodData Cloud when data is
+requested from the corresponding FlightRPC data source dataset.
+
+Each FlexFun provides a `name` used to identify the FlexFun in the GoodData Cloud.
+FlexFuns can provide a set of `metadata` that can further influence when and how they are called by GoodData Cloud.
+They also provide a `schema` (defined in terms
+of [pyarrow.Schema](https://arrow.apache.org/docs/python/generated/pyarrow.Schema.html))
+that describes the shape of data that the FlexFun can provide.
+Finally, they provide a set of methods that can be called to provide data in response to queries:
+* `call` - called to provide data in response to a query
+* `cancel` - called to cancel a query if GoodData Cloud decides to stop requesting data (e.g. if there is a timeout)
+* `on_load` - called when the FlexFun is created before any `call` or `cancel` methods are called
+
+## Usage
+
+Install the package alongside the gooddata-flight-server using pip:
+
+```bash
+pip install gooddata-flight-server gooddata-flexfun
+```
+
+Next, update the GoodData Flight Server configuration to load the FlexFun methods.
+
+```toml
+[flexfun]
+
+# specify one or more modules that contain your FlexFun implementations
+#
+functions = [
+    "flexfun.your_function"
+]
+```
+
+Then when running the GoodData Flight Server, use the `--methods-provider` option to load the FlexFun methods.
+For example:
+
+```bash
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+SERVER_CMD="${SCRIPT_DIR}/.venv/bin/gooddata-flight-server"
+
+export PYTHONPATH="${SCRIPT_DIR}/src"
+export CONFIG_ENV="${1:-dev}"
+
+$SERVER_CMD start \
+              --methods-provider gooddata_flexfun \
+              --config \
+                config/${CONFIG_ENV}.server.toml \
+                config/flexfun.config.toml \
+              --logging-config config/default.logging.ini \
+              --dev-log
+```
+
+This will start the GoodData Flight Server with the FlexFun methods loaded.

gooddata-flexfun/gooddata_flexfun/__init__.py

@@ -0,0 +1,17 @@
+# (C) 2024 GoodData Corporation
+
+from gooddata_flexfun.flexfun.flex_fun import FlexFun
+from gooddata_flexfun.flexfun.flex_fun_execution_context import (
+    ExecutionContext,
+    ExecutionContextAbsoluteDateFilter,
+    ExecutionContextAttribute,
+    ExecutionContextAttributeSorting,
+    ExecutionContextNegativeAttributeFilter,
+    ExecutionContextPositiveAttributeFilter,
+    ExecutionContextRelativeDateFilter,
+    ExecutionRequest,
+    ExecutionType,
+    LabelElementsExecutionRequest,
+    ReportExecutionRequest,
+)
+from gooddata_flexfun.flexfun.flight_methods import create_flexfun_flight_methods

gooddata-flight-server/gooddata_flight_server/flexfun/flex_fun.py → gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun.py

@@ -3,9 +3,7 @@
 from typing import Optional
 
 import pyarrow
-
-from gooddata_flight_server.server.base import ServerContext
-from gooddata_flight_server.tasks.base import ArrowData
+from gooddata_flight_server import ArrowData, ServerContext
 
 
 class FlexFun(abc.ABC):

gooddata-flight-server/gooddata_flight_server/flexfun/flex_fun_registry.py → gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_registry.py

@@ -3,10 +3,9 @@
 from collections.abc import Iterable
 
 import structlog
+from gooddata_flight_server import ErrorInfo, ServerContext
 
-from gooddata_flight_server.errors.error_info import ErrorInfo
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
-from gooddata_flight_server.server.base import ServerContext
+from gooddata_flexfun.flexfun.flex_fun import FlexFun
 
 
 class FlexFunRegistry:

gooddata-flight-server/gooddata_flight_server/flexfun/flex_fun_task.py → gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_task.py

@@ -4,15 +4,9 @@
 
 import pyarrow
 import structlog
+from gooddata_flight_server import ArrowData, FlightDataTaskResult, Task, TaskError, TaskResult
 
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
-from gooddata_flight_server.tasks.base import ArrowData
-from gooddata_flight_server.tasks.task import Task
-from gooddata_flight_server.tasks.task_error import TaskError
-from gooddata_flight_server.tasks.task_result import (
-    FlightDataTaskResult,
-    TaskResult,
-)
+from gooddata_flexfun.flexfun.flex_fun import FlexFun
 
 _LOGGER = structlog.get_logger("gooddata_flexfun.task")
 

gooddata-flight-server/gooddata_flight_server/flexfun/flight_methods.py → gooddata-flexfun/gooddata_flexfun/flexfun/flight_methods.py

@@ -5,22 +5,21 @@
 import orjson
 import pyarrow.flight
 import structlog
-
-from gooddata_flight_server.errors.error_code import ErrorCode
-from gooddata_flight_server.errors.error_info import ErrorInfo
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
-from gooddata_flight_server.flexfun.flex_fun_registry import FlexFunRegistry
-from gooddata_flight_server.flexfun.flex_fun_task import FlexFunTask
-from gooddata_flight_server.server.base import ServerContext
-from gooddata_flight_server.server.flight_rpc.server_methods import (
-    FlightServerMethods,
-)
-from gooddata_flight_server.tasks.base import TaskWaitTimeoutError
-from gooddata_flight_server.tasks.task_result import (
+from gooddata_flight_server import (
+    ErrorCode,
+    ErrorInfo,
     FlightDataTaskResult,
+    FlightServerMethods,
+    ServerContext,
     TaskExecutionResult,
+    TaskWaitTimeoutError,
+    flight_server_methods,
 )
 
+from gooddata_flexfun.flexfun.flex_fun import FlexFun
+from gooddata_flexfun.flexfun.flex_fun_registry import FlexFunRegistry
+from gooddata_flexfun.flexfun.flex_fun_task import FlexFunTask
+
 _LOGGER = structlog.get_logger("gooddata_flexfun.rpc")
 _DEFAULT_TASK_WAIT = 60.0
 
@@ -244,6 +243,7 @@ def _on_end(_: Optional[pyarrow.ArrowException]) -> None:
 _FLEXFUN_FUNCTION_LIST = "functions"
 
 
+@flight_server_methods
 def create_flexfun_flight_methods(ctx: ServerContext) -> FlightServerMethods:
     """
     This factory creates implementation of Flight RPC methods that realize the FlexFun server.

gooddata-flexfun/mypy.ini

@@ -0,0 +1,26 @@
+# (C) 2024 GoodData Corporation
+ [mypy]
+ plugins = pydantic.mypy
+ disallow_untyped_defs = True
+ warn_redundant_casts = True
+ strict_equality = True
+ no_implicit_optional = True
+ python_version = 3.9
+
+ [mypy-gooddata_api_client.*]
+ ignore_missing_imports = True
+
+ [mypy-gooddata_sdk.*]
+ ignore_missing_imports = True
+
+ [mypy-gooddata_flight_server.*]
+ ignore_missing_imports = True
+
+ [mypy-pyarrow.*]
+ ignore_missing_imports = True
+
+ [mypy-orjson.*]
+ ignore_missing_imports = True
+
+ [mypy-structlog.*]
+ ignore_missing_imports = True

gooddata-flexfun/requirements.txt

@@ -0,0 +1,6 @@
+ orjson>=3.9.15,<4.0.0
+ pyarrow>=16.1.0
+ structlog>=24.0.0,<25.0.0
+
+ setuptools~=74.1.2
+ typing_extensions~=4.12.2