✨(backends) import backends dynamically

We want to automatically discover backends in the data/backends
sub-directories for CLI and LRS usage.
We also now handle import failures gracefully, thus backends with
unmet dependencies are excluded.

.env.dist

@@ -62,6 +62,12 @@ RALPH_BACKENDS__DATA__ES__TEST_HOSTS=http://elasticsearch:9200
 RALPH_BACKENDS__DATA__ES__TEST_INDEX=test-index-foo
 RALPH_BACKENDS__DATA__ES__TEST_FORWARDING_INDEX=test-index-foo-2
 
+# ES lrs backend
+
+# Same options as for the ES data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__ES__ instead. Example:
+# RALPH_BACKENDS__LRS__ES__HOSTS=http://elasticsearch:9200
+
 # MONGO data backend
 
 RALPH_BACKENDS__DATA__MONGO__CONNECTION_URI=mongodb://mongo:27017/
@@ -76,7 +82,13 @@ RALPH_BACKENDS__DATA__MONGO__TEST_FORWARDING_COLLECTION=foo-2
 RALPH_BACKENDS__DATA__MONGO__TEST_DATABASE=statements
 RALPH_BACKENDS__DATA__MONGO__TEST_CONNECTION_URI=mongodb://mongo:27017/
 
-# ClickHouse data backend
+# MONGO lrs backend
+
+# Same options as for the MONGO data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__MONGO__ instead. Example:
+# RALPH_BACKENDS__LRS__MONGO__CONNECTION_URI=mongodb://mongo:27017/
+
+# CLICKHOUSE data backend
 
 RALPH_BACKENDS__DATA__CLICKHOUSE__HOST=clickhouse
 RALPH_BACKENDS__DATA__CLICKHOUSE__PORT=8123
@@ -92,6 +104,30 @@ RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_HOST=clickhouse
 RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_PORT=8123
 RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
 
+# CLICKHOUSE lrs backend
+
+# Same options as for the CLICKHOUSE data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__CLICKHOUSE__ instead. Example:
+# RALPH_BACKENDS__LRS__CLICKHOUSE__HOST=clickhouse
+
+# Additional options specific to the CLICKHOUSE lrs backend:
+# RALPH_BACKENDS__LRS__CLICKHOUSE__IDS_CHUNK_SIZE=8123
+
+# FS data backend
+
+# RALPH_BACKENDS__DATA__FS__DEFAULT_CHUNK_SIZE=4096
+# RALPH_BACKENDS__DATA__FS__DEFAULT_DIRECTORY_PATH=.
+# RALPH_BACKENDS__DATA__FS__DEFAULT_QUERY_STRING=*
+# RALPH_BACKENDS__DATA__FS__LOCALE_ENCODING=utf8
+
+# FS lrs backend
+
+# Same options as for the FS data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__FS__ instead. Example:
+# RALPH_BACKENDS__LRS__FS__DEFAULT_DIRECTORY_PATH=.
+
+# Additional options specific to the FS lrs backend:
+# RALPH_BACKENDS__LRS__FS__DEFAULT_LRS_FILE=fs_lrs.jsonl
 
 # LRS HTTP backend
 

CHANGELOG.md

@@ -14,6 +14,10 @@ and this project adheres to
 - Implement xAPI LMS Profile statements validation
 - `EdX` to `xAPI` converters for enrollment events
 - Helm: Add variable ``ingress.hosts``
+- Backends: Add `Writable` and `Listable` interfaces to distinguish supported
+  functionalities among `data` backends
+- Backends: Add `get_backends` function to automatically discover backends
+  for CLI and LRS usage
 
 ### Changed
 

Makefile

@@ -85,7 +85,7 @@ arnold-bootstrap: \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a $(ARNOLD_APP) create_app_vaults && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a elasticsearch create_app_vaults && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) decrypt
-	sed -i 's/^# RALPH_BACKENDS__DATA__ES/RALPH_BACKENDS__DATA__ES/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
+	sed -i 's/^# RALPH_BACKENDS__/RALPH_BACKENDS__/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
 	source .k3d-cluster.env.sh && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) encrypt
 	echo "skip_verification: True" > $(ARNOLD_APP_VARS)

src/helm/ralph/vault.yaml

@@ -1,3 +1,5 @@
+RALPH_BACKENDS__LRS__ES__HOSTS: http://elasticsearch:9200
+RALPH_BACKENDS__LRS__ES__INDEX: statements
 RALPH_BACKENDS__DATA__ES__HOSTS: http://elasticsearch:9200
 RALPH_BACKENDS__DATA__ES__INDEX: statements
 RALPH_SENTRY_DSN: https://[email protected]/1234567

src/ralph/api/routers/health.py

@@ -6,19 +6,18 @@
 from fastapi import APIRouter, status
 from fastapi.responses import JSONResponse
 
-from ralph.backends.conf import backends_settings
+from ralph.backends.loader import get_lrs_backends
 from ralph.backends.lrs.base import BaseAsyncLRSBackend, BaseLRSBackend
 from ralph.conf import settings
-from ralph.utils import await_if_coroutine, get_backend_instance
+from ralph.utils import await_if_coroutine, get_backend_class
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
-BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_instance(
-    backend_type=backends_settings.BACKENDS.LRS,
-    backend_name=settings.RUNSERVER_BACKEND,
-)
+BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class(
+    backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND
+)()
 
 
 @router.get("/__lbheartbeat__")

src/ralph/api/routers/statements.py

@@ -27,9 +27,10 @@
 from ralph.api.auth.user import AuthenticatedUser
 from ralph.api.forwarding import forward_xapi_statements, get_active_xapi_forwardings
 from ralph.api.models import ErrorDetail, LaxStatement
-from ralph.backends.conf import backends_settings
+from ralph.backends.loader import get_lrs_backends
 from ralph.backends.lrs.base import (
     AgentParameters,
+    BaseAsyncLRSBackend,
     BaseLRSBackend,
     RalphStatementsQuery,
 )
@@ -45,7 +46,7 @@
 from ralph.models.xapi.base.common import IRI
 from ralph.utils import (
     await_if_coroutine,
-    get_backend_instance,
+    get_backend_class,
     now,
     statements_are_equivalent,
 )
@@ -58,10 +59,9 @@
 )
 
 
-BACKEND_CLIENT: BaseLRSBackend = get_backend_instance(
-    backend_type=backends_settings.BACKENDS.LRS,
-    backend_name=settings.RUNSERVER_BACKEND,
-)
+BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class(
+    backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND
+)()
 
 POST_PUT_RESPONSES = {
     400: {

src/ralph/backends/data/async_es.py

@@ -31,7 +31,6 @@ class AsyncESDataBackend(BaseAsyncDataBackend, AsyncWritable, AsyncListable):
     name = "async_es"
     query_model = ESQuery
     settings_class = ESDataBackendSettings
-    default_operation_type = BaseOperationType.INDEX
 
     def __init__(self, settings: Optional[ESDataBackendSettings] = None):
         """Instantiate the asynchronous Elasticsearch client.

src/ralph/backends/data/async_mongo.py

@@ -21,6 +21,8 @@
 from ralph.utils import parse_bytes_to_dict
 
 from ..data.base import (
+    AsyncListable,
+    AsyncWritable,
     BaseAsyncDataBackend,
     DataBackendStatus,
     async_enforce_query_checks,
@@ -29,13 +31,12 @@
 logger = logging.getLogger(__name__)
 
 
-class AsyncMongoDataBackend(BaseAsyncDataBackend):
+class AsyncMongoDataBackend(BaseAsyncDataBackend, AsyncWritable, AsyncListable):
     """Async MongoDB data backend."""
 
     name = "async_mongo"
     query_model = MongoQuery
     settings_class = MongoDataBackendSettings
-    default_operation_type = BaseOperationType.INDEX
 
     def __init__(self, settings: Optional[MongoDataBackendSettings] = None):
         """Instantiate the asynchronous MongoDB client.

src/ralph/backends/data/base.py

@@ -147,7 +147,6 @@ def list(
 class BaseDataBackend(ABC):
     """Base data backend interface."""
 
-    type = "data"
     name = "base"
     query_model = BaseQuery
     settings_class = BaseDataBackendSettings
@@ -329,7 +328,6 @@ async def list(
 class BaseAsyncDataBackend(ABC):
     """Base async data backend interface."""
 
-    type = "data"
     name = "base"
     query_model = BaseQuery
     settings_class = BaseDataBackendSettings

src/ralph/backends/http/async_lrs.py

@@ -3,28 +3,23 @@
 import asyncio
 import json
 import logging
-from datetime import datetime
 from itertools import chain
-from typing import Iterable, Iterator, List, Literal, Optional, Union
+from typing import Iterable, Iterator, List, Optional, Union
 from urllib.parse import ParseResult, parse_qs, urljoin, urlparse
-from uuid import UUID
 
 from httpx import AsyncClient, HTTPError, HTTPStatusError, RequestError
 from more_itertools import chunked
-from pydantic import AnyHttpUrl, BaseModel, Field, NonNegativeInt, parse_obj_as
+from pydantic import AnyHttpUrl, BaseModel, Field, parse_obj_as
 from pydantic.types import PositiveInt
 
+from ralph.backends.lrs.base import LRSStatementsQuery
 from ralph.conf import BaseSettingsConfig, HeadersParameters
 from ralph.exceptions import BackendException, BackendParameterException
-from ralph.models.xapi.base.agents import BaseXapiAgent
-from ralph.models.xapi.base.common import IRI
-from ralph.models.xapi.base.groups import BaseXapiGroup
 from ralph.utils import gather_with_limited_concurrency
 
 from .base import (
     BaseHTTPBackend,
     BaseHTTPBackendSettings,
-    BaseQuery,
     HTTPBackendStatus,
     OperationType,
     enforce_query_checks,
@@ -72,31 +67,6 @@ class StatementResponse(BaseModel):
     more: Optional[str]
 
 
-class LRSStatementsQuery(BaseQuery):
-    """Pydantic model for LRS query on Statements resource query parameters.
-
-    LRS Specification:
-    https://github.com/adlnet/xAPI-Spec/blob/1.0.3/xAPI-Communication.md#213-get-statements
-    """
-
-    # pylint: disable=too-many-instance-attributes
-
-    statement_id: Optional[str] = Field(None, alias="statementId")
-    voided_statement_id: Optional[str] = Field(None, alias="voidedStatementId")
-    agent: Optional[Union[BaseXapiAgent, BaseXapiGroup]]
-    verb: Optional[IRI]
-    activity: Optional[IRI]
-    registration: Optional[UUID]
-    related_activities: Optional[bool] = False
-    related_agents: Optional[bool] = False
-    since: Optional[datetime]
-    until: Optional[datetime]
-    limit: Optional[NonNegativeInt] = 0
-    format: Optional[Literal["ids", "exact", "canonical"]] = "exact"
-    attachments: Optional[bool] = False
-    ascending: Optional[bool] = False
-
-
 class AsyncLRSHTTPBackend(BaseHTTPBackend):
     """Asynchronous LRS HTTP backend."""
 

src/ralph/backends/http/base.py

@@ -81,7 +81,6 @@ class Config:
 class BaseHTTPBackend(ABC):
     """Base HTTP backend interface."""
 
-    type = "http"
     name = "base"
     query = BaseQuery