Migrate to pydantic v2

.circleci/config.yml

@@ -158,7 +158,7 @@ jobs:
             - v1-dependencies-<< parameters.python-image >>-{{ .Revision }}
       - run:
           name: Install development dependencies
-          command: pip install --user .[backend-clickhouse,backend-es,backend-ldp,backend-lrs,backend-mongo,backend-swift,backend-ws,cli,dev,lrs]
+          command: pip install --user .[backend-clickhouse,backend-es,backend-ldp,backend-lrs,backend-mongo,backend-s3,backend-swift,backend-ws,cli,dev,lrs]
       - save_cache:
           paths:
             - ~/.local
@@ -495,8 +495,17 @@ jobs:
           command: |
             git config --global user.email "[email protected]"
             git config --global user.name "FUN MOOC Bot"
-            ~/.local/bin/mkdocs gh-deploy
-
+            # Deploy docs with either:
+            # - DOCS_VERSION: 1.1 (for git tag v1.1.2)
+            # - DOCS_ALIAS: latest
+            # or
+            # - DOCS_VERSION: dev (for master branch)
+            # - No DOCS_ALIAS
+            DOCS_VERSION=$([[ -z "$CIRCLE_TAG" ]] && echo $CIRCLE_BRANCH || echo ${CIRCLE_TAG} | sed 's/^v\([0-9]\.[0-9]*\)\..*/\1/')
+            DOCS_ALIAS=$([[ -z "$CIRCLE_TAG" ]] && echo "" || echo "latest")
+            echo "DOCS_VERSION: ${DOCS_VERSION}"
+            echo "DOCS_ALIAS: ${DOCS_ALIAS}"
+            ~/.local/bin/mike deploy --push --update-aliases ${DOCS_VERSION} ${DOCS_ALIAS}
   # Make a new github release
   release:
     docker:
@@ -670,8 +679,8 @@ workflows:
               only: /^v.*/
 
       # Publish the documentation website to GitHub Pages.
-      # Only do it for master as tagged releases are supposed to tag their own version of the
-      # documentation in the release commit on master before they go out.
+      # Only do it for master and for tagged releases with a tag starting with
+      # the letter v.
       - deploy-docs:
           requires:
             - tray
@@ -680,7 +689,7 @@ workflows:
             branches:
               only: master
             tags:
-              only: /.*/
+              only: /^v.*/
 
       # Release
       - release:

.env.dist

@@ -4,7 +4,7 @@ RALPH_APP_DIR=/app/.ralph
 # Uncomment lines (by removing # characters at the beginning of target lines)
 # to define environment variables associated to the backend(s) you need.
 
-# LDP storage backend
+# LDP data backend
 #
 # You need to generate an API token for your OVH's account and fill the service
 # name and stream id you are targeting.
@@ -13,63 +13,96 @@ RALPH_APP_DIR=/app/.ralph
 # define them for convenience purpose during development, but they can be
 # passed as CLI options.
 
-# RALPH_BACKENDS__STORAGE__LDP__ENDPOINT=
-# RALPH_BACKENDS__STORAGE__LDP__APPLICATION_KEY=
-# RALPH_BACKENDS__STORAGE__LDP__APPLICATION_SECRET=
-# RALPH_BACKENDS__STORAGE__LDP__CONSUMER_KEY=
-# RALPH_BACKENDS__STORAGE__LDP__SERVICE_NAME=
-# RALPH_BACKENDS__STORAGE__LDP__STREAM_ID=
-
-# Swift storage backend
-
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_AUTH_URL=http://swift:35357/v3/
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_IDENTITY_API_VERSION=3
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_USER_DOMAIN_NAME=Default
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_PROJECT_DOMAIN_NAME=Default
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_TENANT_ID=cd238e84310a46e58af7f1d515887d88
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_TENANT_NAME=RegionOne
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_USERNAME=demo
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_PASSWORD=demo
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_REGION_NAME=RegionOne
-# RALPH_BACKENDS__STORAGE__SWIFT__OS_STORAGE_URL=http://swift:8080/v1/KEY_cd238e84310a46e58af7f1d515887d88/test_container
-
-# S3 storage backend
-
-# RALPH_BACKENDS__STORAGE__S3__ACCESS_KEY_ID=
-# RALPH_BACKENDS__STORAGE__S3__SECRET_ACCESS_KEY=
-# RALPH_BACKENDS__STORAGE__S3__SESSION_TOKEN=
-# RALPH_BACKENDS__STORAGE__S3__DEFAULT_REGION=
-# RALPH_BACKENDS__STORAGE__S3__BUCKET_NAME=
-# RALPH_BACKENDS__STORAGE__S3__ENDPOINT_URL=
-
-# ES database backend
-
-RALPH_BACKENDS__DATABASE__ES__HOSTS=http://elasticsearch:9200
-RALPH_BACKENDS__DATABASE__ES__INDEX=statements
-RALPH_BACKENDS__DATABASE__ES__TEST_HOSTS=http://elasticsearch:9200
-RALPH_BACKENDS__DATABASE__ES__TEST_INDEX=test-index-foo
-RALPH_BACKENDS__DATABASE__ES__TEST_FORWARDING_INDEX=test-index-foo-2
-
-# MONGO database backend
-
-RALPH_BACKENDS__DATABASE__MONGO__COLLECTION=foo
-RALPH_BACKENDS__DATABASE__MONGO__DATABASE=statements
-RALPH_BACKENDS__DATABASE__MONGO__CONNECTION_URI=mongodb://mongo:27017/
-RALPH_BACKENDS__DATABASE__MONGO__TEST_COLLECTION=foo
-RALPH_BACKENDS__DATABASE__MONGO__TEST_FORWARDING_COLLECTION=foo-2
-RALPH_BACKENDS__DATABASE__MONGO__TEST_DATABASE=statements
-RALPH_BACKENDS__DATABASE__MONGO__TEST_CONNECTION_URI=mongodb://mongo:27017/
-
-# ClickHouse database backend
-
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__HOST=clickhouse
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__PORT=8123
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__XAPI_DATABASE=xapi
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__EVENT_TABLE_NAME=xapi_events_all
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_DATABASE=test_statements
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_HOST=clickhouse
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_PORT=8123
-RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
+# RALPH_BACKENDS__DATA__LDP__APPLICATION_KEY=
+# RALPH_BACKENDS__DATA__LDP__APPLICATION_SECRET=
+# RALPH_BACKENDS__DATA__LDP__CONSUMER_KEY=
+# RALPH_BACKENDS__DATA__LDP__DEFAULT_STREAM_ID=
+# RALPH_BACKENDS__DATA__LDP__ENDPOINT=
+# RALPH_BACKENDS__DATA__LDP__REQUEST_TIMEOUT=
+# RALPH_BACKENDS__DATA__LDP__SERVICE_NAME=
+
+# Swift data backend
+
+# RALPH_BACKENDS__DATA__SWIFT__AUTH_URL=http://swift:35357/v3/
+# RALPH_BACKENDS__DATA__SWIFT__USERNAME=demo
+# RALPH_BACKENDS__DATA__SWIFT__PASSWORD=demo
+# RALPH_BACKENDS__DATA__SWIFT__IDENTITY_API_VERSION=3
+# RALPH_BACKENDS__DATA__SWIFT__TENANT_ID=cd238e84310a46e58af7f1d515887d88
+# RALPH_BACKENDS__DATA__SWIFT__TENANT_NAME=RegionOne
+# RALPH_BACKENDS__DATA__SWIFT__PROJECT_DOMAIN_NAME=Default
+# RALPH_BACKENDS__DATA__SWIFT__REGION_NAME=RegionOne
+# RALPH_BACKENDS__DATA__SWIFT__OBJECT_STORAGE_URL=http://swift:8080/v1/KEY_cd238e84310a46e58af7f1d515887d88/test_container
+# RALPH_BACKENDS__DATA__SWIFT__USER_DOMAIN_NAME=Default
+# RALPH_BACKENDS__DATA__SWIFT__DEFAULT_CONTAINER=
+# RALPH_BACKENDS__DATA__SWIFT__LOCALE_ENCODING=Default
+
+# S3 data backend
+
+# RALPH_BACKENDS__DATA__S3__ACCESS_KEY_ID=
+# RALPH_BACKENDS__DATA__S3__SECRET_ACCESS_KEY=
+# RALPH_BACKENDS__DATA__S3__SESSION_TOKEN=
+# RALPH_BACKENDS__DATA__S3__ENDPOINT_URL=
+# RALPH_BACKENDS__DATA__S3__DEFAULT_REGION=
+# RALPH_BACKENDS__DATA__S3__DEFAULT_BUCKET_NAME=
+# RALPH_BACKENDS__DATA__S3__DEFAULT_CHUNK_SIZE=
+# RALPH_BACKENDS__DATA__S3__LOCALE_ENCODING=
+
+# ES data backend
+
+RALPH_BACKENDS__DATA__ES__HOSTS=http://elasticsearch:9200
+RALPH_BACKENDS__DATA__ES__DEFAULT_INDEX=statements
+# RALPH_BACKENDS__DATA__ES__ALLOW_YELLOW_STATUS=False
+# RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__ca_certs=False
+# RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs=False
+# RALPH_BACKENDS__DATA__ES__DEFAULT_CHUNK_SIZE=500
+# RALPH_BACKENDS__DATA__ES__LOCALE_ENCODING=utf8
+# RALPH_BACKENDS__DATA__ES__POINT_IN_TIME_KEEP_ALIVE=1m
+# RALPH_BACKENDS__DATA__ES__REFRESH_AFTER_WRITE=False
+RALPH_BACKENDS__DATA__ES__TEST_HOSTS=http://elasticsearch:9200
+RALPH_BACKENDS__DATA__ES__TEST_INDEX=test-index-foo
+RALPH_BACKENDS__DATA__ES__TEST_FORWARDING_INDEX=test-index-foo-2
+
+# MONGO data backend
+
+RALPH_BACKENDS__DATA__MONGO__CONNECTION_URI=mongodb://mongo:27017/
+RALPH_BACKENDS__DATA__MONGO__DEFAULT_COLLECTION=foo
+RALPH_BACKENDS__DATA__MONGO__DEFAULT_DATABASE=statements
+# RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__document_class=
+# RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__tz_aware=False
+# RALPH_BACKENDS__DATA__MONGO__DEFAULT_CHUNK_SIZE=500
+# RALPH_BACKENDS__DATA__MONGO__LOCALE_ENCODING=utf8
+RALPH_BACKENDS__DATA__MONGO__TEST_COLLECTION=foo
+RALPH_BACKENDS__DATA__MONGO__TEST_FORWARDING_COLLECTION=foo-2
+RALPH_BACKENDS__DATA__MONGO__TEST_DATABASE=statements
+RALPH_BACKENDS__DATA__MONGO__TEST_CONNECTION_URI=mongodb://mongo:27017/
+
+# ClickHouse data backend
+
+RALPH_BACKENDS__DATA__CLICKHOUSE__HOST=clickhouse
+RALPH_BACKENDS__DATA__CLICKHOUSE__PORT=8123
+RALPH_BACKENDS__DATA__CLICKHOUSE__DATABASE=xapi
+RALPH_BACKENDS__DATA__CLICKHOUSE__EVENT_TABLE_NAME=xapi_events_all
+# RALPH_BACKENDS__DATA__CLICKHOUSE__USERNAME=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__PASSWORD=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__CLIENT_OPTIONS__date_time_input_format=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__CLIENT_OPTIONS__allow_experimental_object_type=
+# RALPH_BACKENDS__DATA__CLICKHOUSE__DEFAULT_CHUNK_SIZE=500
+# RALPH_BACKENDS__DATA__CLICKHOUSE__LOCALE_ENCODING=utf8
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_DATABASE=test_statements
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_HOST=clickhouse
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_PORT=8123
+RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
+
+
+# LRS HTTP backend
+
+RALPH_BACKENDS__HTTP__LRS__BASE_URL=http://ralph:[email protected]:8100/
+RALPH_BACKENDS__HTTP__LRS__USERNAME=ralph
+RALPH_BACKENDS__HTTP__LRS__PASSWORD=secret
+RALPH_BACKENDS__HTTP__LRS__HEADERS__X_EXPERIENCE_API_VERSION=1.0.3
+RALPH_BACKENDS__HTTP__LRS__HEADERS__CONTENT_TYPE=application/json
+RALPH_BACKENDS__HTTP__LRS__STATUS_ENDPOINT=/__heartbeat__
+RALPH_BACKENDS__HTTP__LRS__STATEMENTS_ENDPOINT=/xAPI/statements
 
 # Sentry
 
@@ -83,9 +116,9 @@ RALPH_BACKENDS__DATABASE__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
 
 # RALPH_CONVERTER_EDX_XAPI_UUID_NAMESPACE=
 
-# LRS API
+# LRS API 
 
-RALPH_RUNSERVER_AUTH_BACKEND=basic
+RALPH_RUNSERVER_AUTH_BACKENDS=Basic
 RALPH_RUNSERVER_AUTH_OIDC_AUDIENCE=http://localhost:8100
 RALPH_RUNSERVER_AUTH_OIDC_ISSUER_URI=http://learning-analytics-playground_keycloak_1:8080/auth/realms/fun-mooc
 RALPH_RUNSERVER_BACKEND=es

.gitignore

@@ -51,9 +51,6 @@ venv.bak/
 .pylint.d
 .pytest_cache
 
-# Test fixtures
-data/
-
 # Documentation site
 site/
 

CHANGELOG.md

@@ -13,19 +13,25 @@ and this project adheres to
 - Implement Pydantic model for LRS Statements resource query parameters
 - Implement xAPI LMS Profile statements validation
 - `EdX` to `xAPI` converters for enrollment events
+- Helm: Add variable ``ingress.hosts``
 
 ### Changed
 
+- Upgrade `cachetools` to `5.3.2`
+- Refactor `database` and `storage` backends under the unified `data` backend
+interface [BC]
+- Refactor LRS `query_statements` and `query_statements_by_ids` backends
+methods under the unified `lrs` backend interface [BC]
 - Refactor LRS Statements resource query parameters defined for `ralph` API
 - Helm chart: improve chart modularity
 - User credentials must now include an "agent" field which can be created
   using the cli
 - `GET /statements` now has "mine" option which matches statements that
 have an authority field matching that of the user
 - CLI: change `push` to `write` and `fetch` to `read` [BC]
-- Upgrade `fastapi` to `0.103.2`
+- Upgrade `fastapi` to `0.104.0`
 - Upgrade `more-itertools` to `10.1.0`
-- Upgrade `sentry_sdk` to `1.31.0`
+- Upgrade `sentry_sdk` to `1.32.0`
 - Upgrade `uvicorn` to `0.23.2`
 - API: Invalid parameters now return 400 status code
 - API: Forwarding PUT now uses PUT (instead of POST)
@@ -40,6 +46,13 @@ have an authority field matching that of the user
   with camelCase alias, in `LRSStatementsQuery`
 - API: Add `RALPH_LRS_RESTRICT_BY_AUTHORITY` option making `?mine=True`
   implicit
+- CLI: list cli usage strings in alphabetical order
+- Helm: Fix clickhouse version
+- Helm: improve volumes and ingress configurations
+- API: Add `RALPH_LRS_RESTRICT_BY_SCOPE` option enabling endpoint access
+  control by user scopes
+- API: Variable `RUNSERVER_AUTH_BACKEND` becomes `RUNSERVER_AUTH_BACKENDS`, and
+  multiple authentication methods are supported simultaneously
 
 ### Fixed
 
@@ -48,7 +61,9 @@ have an authority field matching that of the user
 
 ### Removed
 
+- `school`, `course`, `module` context extensions in Edx to xAPI base converter
 - `name` field in `VideoActivity` xAPI model mistakenly used in `video` profile
+- Helm: remove variable ``ingress.hostname`` and ``ingress.tls``
 
 ## [3.9.0] - 2023-07-21
 

Dockerfile

@@ -25,7 +25,7 @@ RUN apt-get update && \
         libffi-dev && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip install .[backend-clickhouse,backend-es,backend-ldp,backend-lrs,backend-mongo,backend-swift,backend-ws,cli,lrs]
+RUN pip install .[backend-clickhouse,backend-es,backend-ldp,backend-lrs,backend-mongo,backend-s3,backend-swift,backend-ws,cli,lrs]
 
 
 # -- Core --
@@ -59,6 +59,12 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; \
         rm -rf /var/lib/apt/lists/*; \
     fi;
 
+# Install git for documentation deployment
+RUN apt-get update && \
+    apt-get install -y \
+        git && \
+    rm -rf /var/lib/apt/lists/*;
+
 # Uninstall ralph and re-install it in editable mode along with development
 # dependencies
 RUN pip uninstall -y ralph-malph

Makefile

@@ -10,7 +10,14 @@ COMPOSE              = DOCKER_USER=$(DOCKER_USER) docker compose
 COMPOSE_RUN          = $(COMPOSE) run --rm
 COMPOSE_TEST_RUN     = $(COMPOSE_RUN)
 COMPOSE_TEST_RUN_APP = $(COMPOSE_TEST_RUN) app
-MKDOCS               = $(COMPOSE_RUN) --no-deps --publish "8000:8000" app mkdocs
+COMPOSE_RUN_DOCS     = $(COMPOSE_RUN) --no-deps --publish "8000:8000" app
+
+
+# -- Documentation
+DOCS_COMMITTER_NAME  = "FUN MOOC Bot"
+DOCS_COMMITTER_EMAIL = [email protected]
+MKDOCS               = $(COMPOSE_RUN_DOCS) mkdocs
+MIKE                 = GIT_COMMITTER_NAME=$(DOCS_COMMITTER_NAME) GIT_COMMITTER_EMAIL=$(DOCS_COMMITTER_EMAIL) $(COMPOSE_RUN_DOCS) mike
 
 # -- Elasticsearch
 ES_PROTOCOL = http
@@ -66,7 +73,7 @@ bin/init-cluster:
 		-u $(RALPH_LRS_AUTH_USER_NAME) \
 		-p $(RALPH_LRS_AUTH_USER_PWD) \
 		-s $(RALPH_LRS_AUTH_USER_SCOPE) \
-		-M $(RALPH_LRS_AUTH_USER_AGENT_MBOX)
+		-M $(RALPH_LRS_AUTH_USER_AGENT_MBOX) 
 		-w
 
 
@@ -79,7 +86,7 @@ arnold-bootstrap: \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a $(ARNOLD_APP) create_app_vaults && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a elasticsearch create_app_vaults && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) decrypt
-	sed -i 's/^# RALPH_BACKENDS__DATABASE__ES/RALPH_BACKENDS__DATABASE__ES/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
+	sed -i 's/^# RALPH_BACKENDS__DATA__ES/RALPH_BACKENDS__DATA__ES/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
 	source .k3d-cluster.env.sh && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) encrypt
 	echo "skip_verification: True" > $(ARNOLD_APP_VARS)
@@ -140,11 +147,14 @@ docs-build: ## build documentation site
 .PHONY: docs-build
 
 docs-deploy: ## deploy documentation site
-	@$(MKDOCS) gh-deploy
+# Using env variables GIT_COMMITTER_NAME and GIT_COMMITTER_EMAIL will work with mike 2.0
+# Until that you need to set local git config user.name and user.email manually
+	@echo "Deploying docs with version dev"
+	@${MIKE} deploy dev
 .PHONY: docs-deploy
 
-docs-serve: ## run mkdocs live server
-	@$(MKDOCS) serve --dev-addr 0.0.0.0:8000
+docs-serve: ## run mike live server
+	@$(MIKE) serve --dev-addr 0.0.0.0:8000
 .PHONY: docs-serve
 
 down: ## stop and remove backend containers
@@ -222,6 +232,11 @@ lint-pydocstyle: ## lint Python docstrings with pydocstyle
 	@$(COMPOSE_TEST_RUN_APP) pydocstyle
 .PHONY: lint-pydocstyle
 
+lint-mypy: ## lint back-end python sources with mypy
+	@echo 'lint:mypy started…'
+	@$(COMPOSE_TEST_RUN_APP) mypy
+.PHONY: lint-mypy
+
 logs: ## display app logs (follow mode)
 	@$(COMPOSE) logs -f app
 .PHONY: logs