refactor: split gooddata-flight-server

Split GoodData FlexFun-specific stuff into a separate package. This leaves gooddata-flight-server to be a GoodData-agnostic server implementation better encapsulating the responsibilities.

JIRA: CQ-754
risk: low

.envrc

@@ -9,6 +9,7 @@ export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-api-client/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-sdk/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-pandas/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flight-server/"
+export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flexfun/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-dbt/"
 
 export PATH="${PATH}:${SCRIPT_DIR}/gooddata-sdk/bin"

.github/workflows/rw-collect-changes.yaml

@@ -47,3 +47,4 @@ jobs:
               - 'gooddata-api-client/**'
               - 'gooddata-dbt/**'
               - 'gooddata-flight-server/**'
+              - 'gooddata-flexfun/**'

.github/workflows/rw-python-tests.yaml

@@ -24,7 +24,7 @@ jobs:
         if: ${{ matrix.python_version == 'py312' }}
         uses: codecov/codecov-action@v3
         with:
-          files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml
+          files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml,./gooddata-flexfun/coverage.xml
   lint-and-format-check:
     runs-on: ubuntu-latest
     if: ${{inputs.changed-python-modules == 'true'}}

gooddata-flexfun/.gitignore

@@ -0,0 +1,69 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.venv/
+.python-version
+.pytest_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+docs/_autosummary/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints
+
+test_data

gooddata-flexfun/.readthedocs.yaml

@@ -0,0 +1,19 @@
+# (C) 2024 GoodData Corporation
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: gooddata-flexfun/docs/conf.py
+
+python:
+  install:
+    - requirements: gooddata-flexfun/docs/requirements.txt

gooddata-flexfun/Makefile

@@ -0,0 +1,19 @@
+# (C) 2021 GoodData Corporation
+include ../project_common.mk
+
+.PHONY: docs
+docs:
+	tox $(TOX_FLAGS) -e docs
+
+
+.PHONY: dev-certs
+dev-certs:
+	mkdir -p test_data
+	# create CA with self-signed certificate > generate keys + cert requests for server and client > sign
+	cd test_data && \
+		rm -f *.pem && \
+		openssl req -x509 -newkey rsa:4096 -days 3650 -nodes -keyout ca-key.pem -out ca-cert.pem -subj "/C=NA/ST=Unknown/L=Unknown/O=Unknown/OU=Unknown/CN=localhost/[email protected]" && \
+		openssl req -newkey rsa:4096 -nodes -keyout server-key.pem -out server-req.pem -subj "/C=NA/ST=Unknown/L=Unknown/O=Unknown/OU=Computer/CN=localhost/[email protected]" && \
+		openssl req -newkey rsa:4096 -nodes -keyout client-key.pem -out client-req.pem -subj "/C=NA/ST=Unknown/L=Unknown/O=Unknown/OU=Computer/CN=testClient1/[email protected]" && \
+		openssl x509 -req -in server-req.pem -days 3650 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem && \
+		openssl x509 -req -in client-req.pem -days 3650 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem

gooddata-flexfun/README.md

@@ -0,0 +1,3 @@
+# GoodData Flex function
+
+...

gooddata-flexfun/gooddata_flexfun/__init__.py

@@ -0,0 +1,16 @@
+# (C) 2024 GoodData Corporation
+from gooddata_flexfun.flex_fun import FlexFun
+from gooddata_flexfun.flex_fun_execution_context import (
+    ExecutionContext,
+    ExecutionContextAbsoluteDateFilter,
+    ExecutionContextAttribute,
+    ExecutionContextAttributeSorting,
+    ExecutionContextFilter,
+    ExecutionContextNegativeAttributeFilter,
+    ExecutionContextPositiveAttributeFilter,
+    ExecutionContextRelativeDateFilter,
+    ExecutionRequest,
+)
+from gooddata_flexfun.flex_fun_registry import FlexFunRegistry
+from gooddata_flexfun.flex_fun_task import FlexFunTask
+from gooddata_flexfun.flight_methods import create_flexfun_flight_methods

gooddata-flight-server/gooddata_flight_server/flexfun/flex_fun.py → gooddata-flexfun/gooddata_flexfun/flex_fun.py

@@ -3,9 +3,7 @@
 from typing import Optional
 
 import pyarrow
-
-from gooddata_flight_server.server.base import ServerContext
-from gooddata_flight_server.tasks.base import ArrowData
+from gooddata_flight_server import ArrowData, ServerContext
 
 
 class FlexFun(abc.ABC):
@@ -26,8 +24,6 @@ class FlexFun(abc.ABC):
 
     Programming detail: a new instance of the FlexFun will be created
     for every call using the `create` method.
-
-    TODO: rename the class... i'm not very creative right now
     """
 
     Name: Optional[str] = None

gooddata-flight-server/gooddata_flight_server/flexfun/flex_fun_registry.py → gooddata-flexfun/gooddata_flexfun/flex_fun_registry.py

@@ -3,10 +3,9 @@
 from collections.abc import Iterable
 
 import structlog
+from gooddata_flight_server import ErrorInfo, ServerContext
 
-from gooddata_flight_server.errors.error_info import ErrorInfo
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
-from gooddata_flight_server.server.base import ServerContext
+from gooddata_flexfun.flex_fun import FlexFun
 
 
 class FlexFunRegistry:

gooddata-flight-server/gooddata_flight_server/flexfun/flex_fun_task.py → gooddata-flexfun/gooddata_flexfun/flex_fun_task.py

@@ -4,15 +4,9 @@
 
 import pyarrow
 import structlog
+from gooddata_flight_server import ArrowData, FlightDataTaskResult, Task, TaskError, TaskResult
 
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
-from gooddata_flight_server.tasks.base import ArrowData
-from gooddata_flight_server.tasks.task import Task
-from gooddata_flight_server.tasks.task_error import TaskError
-from gooddata_flight_server.tasks.task_result import (
-    FlightDataTaskResult,
-    TaskResult,
-)
+from gooddata_flexfun.flex_fun import FlexFun
 
 _LOGGER = structlog.get_logger("gooddata_flexfun.task")
 

gooddata-flight-server/gooddata_flight_server/flexfun/flight_methods.py → gooddata-flexfun/gooddata_flexfun/flight_methods.py

@@ -5,22 +5,20 @@
 import orjson
 import pyarrow.flight
 import structlog
-
-from gooddata_flight_server.errors.error_code import ErrorCode
-from gooddata_flight_server.errors.error_info import ErrorInfo
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
-from gooddata_flight_server.flexfun.flex_fun_registry import FlexFunRegistry
-from gooddata_flight_server.flexfun.flex_fun_task import FlexFunTask
-from gooddata_flight_server.server.base import ServerContext
-from gooddata_flight_server.server.flight_rpc.server_methods import (
-    FlightServerMethods,
-)
-from gooddata_flight_server.tasks.base import TaskWaitTimeoutError
-from gooddata_flight_server.tasks.task_result import (
+from gooddata_flight_server import (
+    ErrorCode,
+    ErrorInfo,
     FlightDataTaskResult,
+    FlightServerMethods,
+    ServerContext,
     TaskExecutionResult,
+    TaskWaitTimeoutError,
 )
 
+from gooddata_flexfun.flex_fun import FlexFun
+from gooddata_flexfun.flex_fun_registry import FlexFunRegistry
+from gooddata_flexfun.flex_fun_task import FlexFunTask
+
 _LOGGER = structlog.get_logger("gooddata_flexfun.rpc")
 _DEFAULT_TASK_WAIT = 60.0
 

gooddata-flexfun/gooddata_flexfun/py.typed

@@ -0,0 +1 @@
+Mark package as supporting typing. See https://www.python.org/dev/peps/pep-0561/ for details.

gooddata-flexfun/mypy.ini

@@ -0,0 +1,35 @@
+# (C) 2024 GoodData Corporation
+[mypy]
+plugins = pydantic.mypy
+disallow_untyped_defs = True
+warn_redundant_casts = True
+strict_equality = True
+no_implicit_optional = True
+python_version = 3.9
+
+[mypy-gooddata_api_client.*]
+ignore_missing_imports = True
+
+[mypy-gooddata_sdk.*]
+ignore_missing_imports = True
+
+[mypy-pyarrow.*]
+ignore_missing_imports = True
+
+[mypy-orjson.*]
+ignore_missing_imports = True
+
+[mypy-dynaconf.*]
+ignore_missing_imports = True
+
+[mypy-structlog.*]
+ignore_missing_imports = True
+
+[mypy-opentelemetry.*]
+ignore_missing_imports = True
+
+[mypy-prometheus_client.*]
+ignore_missing_imports = True
+
+[mypy-readerwriterlock.*]
+ignore_missing_imports = True

gooddata-flexfun/requirements.txt

@@ -0,0 +1,7 @@
+attrs~=23.2.0
+orjson>=3.9.15,<4.0.0
+pyarrow>=16.1.0
+structlog>=24.0.0,<25.0.0
+
+setuptools~=74.1.2
+typing_extensions~=4.12.2

gooddata-flexfun/setup.py

@@ -0,0 +1,69 @@
+# (C) 2024 GoodData Corporation
+from pathlib import Path
+
+from setuptools import find_packages, setup
+
+this_directory = Path(__file__).parent
+long_description = (this_directory / "README.md").read_text(encoding="utf-8")
+
+REQUIRES = [
+    "attrs>=21.4.0,<=23.2.0",
+    "dynaconf>=3.1.11,<4.0.0",
+    "gooddata-flight-server~=1.26.0",
+    "gooddata-sdk~=1.26.0",
+    "orjson>=3.9.15,<4.0.0",
+    "pyarrow>=16.1.0",
+    "structlog>=24.0.0,<25.0.0",
+]
+
+setup(
+    name="gooddata-flexfun",
+    description="Flight RPC server to host custom functions for GoodData Cloud",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    version="1.26.0",
+    author="GoodData",
+    author_email="[email protected]",
+    license="MIT",
+    license_file="LICENSE.txt",
+    license_files=("LICENSE.txt",),
+    install_requires=REQUIRES,
+    packages=find_packages(exclude=["tests*"]),
+    include_package_data=True,
+    python_requires=">=3.9.0",
+    project_urls={
+        "Documentation": "https://gooddata-flexfun.readthedocs.io/en/v1.26.0",
+        "Source": "https://github.com/gooddata/gooddata-python-sdk",
+    },
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Environment :: Console",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Topic :: Database",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Software Development",
+        "Typing :: Typed",
+    ],
+    keywords=[
+        "gooddata",
+        "flight",
+        "rpc",
+        "flight rpc",
+        "custom functions",
+        "analytics",
+        "headless",
+        "business",
+        "intelligence",
+        "headless-bi",
+        "cloud",
+        "native",
+        "semantic",
+        "layer",
+        "sql",
+        "metrics",
+    ],
+)

gooddata-flexfun/test-requirements.txt

@@ -0,0 +1,2 @@
+pytest~=8.2.2
+pytest-cov~=5.0.0

gooddata-flexfun/tests/assert_error_info.py

@@ -0,0 +1,12 @@
+#  (C) 2024 GoodData Corporation
+import pyarrow.flight
+from gooddata_flight_server.errors.error_code import ErrorCode
+from gooddata_flight_server.errors.error_info import ErrorInfo
+
+
+def assert_error_code(code: int, err: pyarrow.flight.FlightError):
+    info = ErrorInfo.maybe_from_pyarrow_error(err)
+    assert info is not None, "The error does not contain the ErrorInfo."
+
+    if code != info.code:
+        raise AssertionError(f"Expected error code '{ErrorCode.name(code)}'. Got: '{ErrorCode.name(info.code)}'")

gooddata-flight-server/tests/flexfun/test_registry.py → gooddata-flexfun/tests/flexfun/test_registry.py

@@ -1,9 +1,8 @@
 #  (C) 2024 GoodData Corporation
 import pyarrow.flight
 import pytest
-from gooddata_flight_server.errors.error_code import ErrorCode
-from gooddata_flight_server.flexfun.flex_fun_registry import FlexFunRegistry
-from gooddata_flight_server.server.base import ServerContext
+from gooddata_flexfun.flex_fun_registry import FlexFunRegistry
+from gooddata_flight_server import ErrorCode, ServerContext
 
 from tests.assert_error_info import assert_error_code
 from tests.flexfun.testing_funs import Fun1, Fun2

gooddata-flight-server/tests/flexfun/testing_funs.py → gooddata-flexfun/tests/flexfun/testing_funs.py

@@ -1,7 +1,7 @@
 #  (C) 2024 GoodData Corporation
 
 import pyarrow
-from gooddata_flight_server.flexfun.flex_fun import FlexFun
+from gooddata_flexfun.flex_fun import FlexFun
 from gooddata_flight_server.server.base import ServerContext
 from gooddata_flight_server.tasks.base import ArrowData