Configured DevContainer for Python Client; Fixed HF Unittest; Integrated Unittests into VSCode; Quality of Life (#660)

* Configured backend and python client tests for vscode and moved all devcontainer files to .devcontainer

* removed old python-client/Dockerfile.dev and integrated into dev container

* added launch target for vuetify tests

* consolidated all tests into tests.yml

* readded LICENSE file to python client

* fixed minore issues in the application and added further documentation on frontend development

* made linter happy and fixed backend workflow

* fixed communicating HF Cache folder to hf

* removed default_hf_home_in_tira_host

---------

Co-authored-by: Tim Hagen <[email protected]>

Author: TheMrSheldon

Dockerfile.dev .devcontainer/Dockerfile.dev

@@ -7,28 +7,33 @@ RUN apt-get update && apt-get install -y sudo git locales
 RUN echo "en_US.UTF-8 UTF-8" | tee -a /etc/locale.gen && locale-gen
 
 ########################################################################################################################
-# Create User                                                                                                          #
+# Create Users                                                                                                         #
 ########################################################################################################################
-# Change root Password to 1234
-RUN echo 'root:1234' | chpasswd
-# Create new user: "dev" also with password 1234
-RUN useradd -ms /bin/bash dev && \
-    echo 'dev:1234' | chpasswd && \
+RUN <<EOF
+	# Create docker group for later
+	addgroup docker
+
+	# Change root Password to 1234
+	echo 'root:1234' | chpasswd
+	
+	# Create new user: "dev" also with password 1234
+	useradd -ms /bin/bash dev
+    echo 'dev:1234' | chpasswd
     usermod -aG sudo dev
+	usermod -aG docker dev
+EOF
 
 
 ########################################################################################################################
 # Frontend                                                                                                             #
 ########################################################################################################################
 USER root
 
-# https://stackoverflow.com/a/47680012
-RUN apt-get update && apt-get install -y curl gnupg2 && \
-    curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
-    echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
-
-RUN apt-get update && apt-get install -y npm yarn
-
+RUN <<EOF
+    apt-get update
+    apt-get install -y npm
+    npm install --global yarn
+EOF
 
 
 
@@ -38,20 +43,34 @@ RUN apt-get update && apt-get install -y npm yarn
 ########################################################################################################################
 ENV PIP_BREAK_SYSTEM_PACKAGES 1
 USER root
-RUN <<EOF
-apt-get update && apt-get install -y python3 python3-pip python3-dev pkg-config default-libmysqlclient-dev \
+RUN apt-get update && apt-get install -y python3 python3-pip python3-dev pkg-config default-libmysqlclient-dev \
     libpcre3-dev
-pip3 install black flake8 isort mypy
-EOF
-RUN <<EOF
 # Create a dummy secret
-mkdir -p "/etc/discourse/" && echo "I am so secret" > "/etc/discourse/client-api-key"
+COPY <<EOF /etc/discourse/client-api-key
+I am so secret
 EOF
 
 ########################################################################################################################
 # Client                                                                                                               #
 ########################################################################################################################
-# NOT YET
+USER root
+RUN <<EOF
+	# Install Java (some dependencies like python are already installed by the application module)
+    apt-get update
+    apt-get install -y openjdk-11-jdk
+	# Install Docker and set correct permissions to the docker.sock
+	apt-get install -y curl
+	curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+	chmod a+r /etc/apt/keyrings/docker.gpg
+	echo \
+		"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+		"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+		tee /etc/apt/sources.list.d/docker.list > /dev/null
+	apt-get update -y
+	apt-get install -y docker-ce-cli
+	touch /var/run/docker.sock
+	chown root:docker /var/run/docker.sock
+EOF
 
 
 

.devcontainer/devcontainer.json

@@ -6,7 +6,7 @@
 	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
 	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
 	"dockerComposeFile": [
-		"../docker-compose.dev.yml",
+		"./docker-compose.dev.yml",
 		"docker-compose.yml"
 	],
 
@@ -50,7 +50,7 @@
 	// "shutdownAction": "none",
 
 	// Uncomment the next line to run commands after the container is created.
-	"postCreateCommand": "cd frontend; yarn; cd ../application/; pip install -e .[test,dev]; make setup; make import-mock-data",
+	"postCreateCommand": "cd frontend; yarn; cd ../python-client; pip install -e .[test,dev]; cd ../application/; pip install -e .[test,dev]; make setup; make import-mock-data",
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -60,6 +60,14 @@
 				"json.format.keepLines": true,
 				"livePreview.portNumber": 3080,
 				"remote.autoForwardPorts": false,
+				"files.exclude": {
+					"**/__pycache__": true,
+					"**/.mypy_cache": true,
+					"**/.pytest_cache": true,
+					"**/*.egg-info": true,
+					"**/node_modules": true,
+					"application/.data-dumps": true
+				},
 				"launch": {
 					"version": "0.2.0",
 					"configurations": [
@@ -86,14 +94,22 @@
 						},
 						{
 							"name": "TIRA Backend Tests",
-							"type": "python",
+							"type": "debugpy",
 							"request": "launch",
 							"program": "${workspaceFolder:Backend}/src/manage.py",
 							"cwd": "${workspaceFolder:Backend}/test",
 							"args": [ "test", "--failfast", "--settings=settings_test" ],
 							"django": true,
 							"env": { "PYTHONPATH": ":../src:.", "DJANGO_SETTINGS_MODULE": "settings_test" },
 							"justMyCode": false
+						},
+						{
+							"name": "Frontend Tests",
+							"type": "node",
+							"request": "launch",
+							"runtimeExecutable": "yarn",
+							"args": [ "test" ],
+							"cwd": "${workspaceFolder:Frontend}"
 						}
 					],
 					"compounds": [
@@ -122,6 +138,7 @@
 				"ms-python.flake8",
 				"ms-python.mypy-type-checker",
 				"42Crunch.vscode-openapi",
+				"vuetifyjs.vuetify-vscode"
 			]
 		}
 	},

.devfiles/README.md .devcontainer/devfiles/README.md

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   devenv:
     build:
@@ -16,21 +15,21 @@ services:
     image: ghcr.io/authelia/authelia
     restart: unless-stopped
     volumes:
-      - ./.devfiles/authelia/configuration.dev.yml:/config/configuration.yml
-      - ./.devfiles/authelia/users-database.yml:/config/users_database.yml
+      - ./devfiles/authelia/configuration.dev.yml:/config/configuration.yml
+      - ./devfiles/authelia/users-database.yml:/config/users_database.yml
   nginx:
     image: lscr.io/linuxserver/nginx
     restart: unless-stopped
-    ports:
-      - "8080:8080"
-      - "8081:8081"
-      - "8082:8082"
+    #ports:
+    #  - "8080:8080"
+    #  - "8081:8081"
+    #  - "8082:8082"
     external_links:
       - "auth:auth.tira.local"
       - "devenv:www.tira.local"
     volumes:
-      - ./.devfiles/nginx/tira.conf:/config/nginx/site-confs/tira.conf
-      - ./.devfiles/nginx/tira-backend.conf:/config/nginx/site-confs/tira-backend.conf
-      - ./.devfiles/nginx/auth.conf:/config/nginx/site-confs/auth.conf
-      - ./.devfiles/nginx/snippets/:/config/nginx/snippets/
-      - ./.devfiles/nginx/certs/:/etc/nginx/certs/
+      - ./devfiles/nginx/tira.conf:/config/nginx/site-confs/tira.conf
+      - ./devfiles/nginx/tira-backend.conf:/config/nginx/site-confs/tira-backend.conf
+      - ./devfiles/nginx/auth.conf:/config/nginx/site-confs/auth.conf
+      - ./devfiles/nginx/snippets/:/config/nginx/snippets/
+      - ./devfiles/nginx/certs/:/etc/nginx/certs/

.devfiles/authelia/configuration.dev.yml .devcontainer/devfiles/authelia/configuration.dev.yml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   # Update this to the name of the service you want to work with in your docker-compose.yml file
   devenv:
@@ -13,7 +12,7 @@ services:
 
     volumes:
       # Update this to wherever you want VS Code to mount the folder of your project
-      - ..:/workspaces:cached
+      - ..:/workspaces/tira:cached
 
     # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
     # cap_add:

.devfiles/authelia/users-database.yml .devcontainer/devfiles/authelia/users-database.yml

@@ -0,0 +1,75 @@
+name: Unit Tests
+
+on: [push]
+
+jobs:
+  backend-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install TIRA Python Client from Repo
+        working-directory: ${{github.workspace}}/python-client
+        run: |
+          # Install tira from the repository since the pip version may not be up-to-date enough.
+          # The install musst be editable (-e) since importing from tira fails otherwise
+          pip3 install -e .[dev,test]
+      - name: Install dependencies
+        working-directory: ${{github.workspace}}/application
+        run: |
+          # Create a dummy DISRAPTOR_API_KEY
+          sudo bash -c 'mkdir -p "/etc/discourse/" && echo "I am so secret" > "/etc/discourse/client-api-key"'
+          pip3 install -e .[dev,test]
+          make setup
+      - name: Run backend tests
+        working-directory: ${{github.workspace}}/application/test
+        run: pytest
+
+  frontend-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+      - name: Install dependencies
+        working-directory: ${{github.workspace}}/frontend
+        run: yarn --frozen-lockfile
+      - name: Run frontend tests
+        working-directory: ${{github.workspace}}/frontend
+        run: yarn test
+
+  python-client-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Install Dependencies
+        working-directory: ${{github.workspace}}/python-client
+        run: |
+          sudo apt-get install -y openjdk-11-jdk
+          pip3 install .[test,dev]
+      - name: Running Tests
+        working-directory: ${{github.workspace}}/python-client
+        run: |
+          echo running on branch ${GITHUB_REF##*/}
+          pytest