diff --git a/config/app-config.json b/config/app-config.json index 31a546e8..55b89267 100644 --- a/config/app-config.json +++ b/config/app-config.json @@ -1,32 +1,61 @@ { "project": { - "id": "x0-system", + "id": "x0", "name": "x0-base-system", "git-repo": "https://github.com/WEBcodeX1/x0/" }, "installer": { - "type": "x0" + "type": "x0", + "subtype": "minikube", + "os:": "linux", + "minikube_driver": "docker" }, "database": { "name": "x0", "su_password": "changeme", - "x0_password": "changeme" + "repl_password": "changeme", + "x0_password": "changeme", + "size": "50Mi", + "replicas": 1 + }, + "test": { + "selenium": { + "run_servers": 1, + "run_images": [ + "chrome:latest" + ] + }, + "testenvs": [ "test" ] + }, + "env_list": [ "test" ], + "environments": { + "test": { + "kubernetes": { + "deployment": { + "image": "x0-app", + "replicas": 1, + "cpu": "200m", + "memory": "512Mi", + "autoscale": false + }, + "namespace": "x0-app", + "app_templates": [ "kubegres" ] + } + } }, - "env_list": [ "default" ], "vhosts": { "x0-app": { "apps": [ "x0" ], "env": { - "default": { - "dns": { - "hostname": "x0-app", - "domain": "x0.localnet" - } - }, "test": { "dns": { "hostname": "x0-app", "domain": "x0.localnet" + }, + "ip": { + "v4": { + "dns_register": false + } } } } diff --git a/doc/dev-tests.rst b/doc/dev-tests.rst index b37b6df0..0cdf477a 100644 --- a/doc/dev-tests.rst +++ b/doc/dev-tests.rst @@ -20,11 +20,17 @@ As a developer, you are strongly encouraged to write sufficient tests and add th 24.1. Test CI ------------- -Automated CI tests are triggered on: +On pushing to the ``current-release`` branch, current-release`s test-containers will +be run and tests executed within. -``git push --set-upstream origin current-release`` +An authenticated maintainer must ensure to push the relevant images to ``ghcr.io/webcodex1/`` +before pushing to the ``current-release`` branch by: -This happens when pushing to the **current-release** branch. +.. code-block:: bash + + git branch current-release + git checkout current-release + git push --set-upstream origin current-release Once the tests pass, a snapshot of the branch will be created and named: diff --git a/kubernetes/README.md b/kubernetes/README.md index c7a35a1d..d578dd6e 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -1,50 +1,25 @@ -# Kubernetes Minikube Installation Instructions +# Kubernetes Installation Instructions -The following installation instructions cover the installation of the -base *x0 system* on a kubernetes minikube cluster including CI / tests. +The *x0-system* ships **kubernetes-ready**. -In short, run the following inside Windows 11 (Professional) Powershell -(with Administrator privileges): system up and running. +## 1. Quick Start / Minikube -```powershell -# enable windows hyper-v feature -Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -``` - -Answer "Y"(es) to reboot. Afterwards Hyper-V is operational. +The kubernetes installer can be started with the following command +`cd ./setup/ && python3 Setup.py`. The configuration is taken from `/config/app-config.json`. ->[!NOTE] -> Install Git for Windows 64bit before you continue, it contains `curl` binary -> which is used / referenced by Setup.py +Per default the installer will set up a *Minikube* system on *Linux* using the *docker* +driver. ->[!NOTE] -> Also install the remaining dependencies (see 3. Dependencies). +Prerequisites: -```powershell -# clone repository and start setup -git clone https://github.com/WEBcodeX1/x0.git -cd .\x0\kubernetes\setup\ -python Setup.py -``` +- Non root user inside sudo group +- A running Docker setup +- Kubectl binary `apt-get install kubernetes-client` >[!NOTE] -> Add `x0-app.x0.localnet` to host DNS, access `http://x0-app.x0.localnet/python/Index.py&appid=test_base` - -The next chapters describe the installation process (also non Hyper-V setup) -in detail. +> Some installation steps require root (sudo) access (prompted). -## 1. Preface - -The minikube installation (version v1.34.0) currently is tested on Microsoft -Windows 11 23H2 (x64) using the *x0 kube installer*
(`./setup/Setup.py`). - -*x0 minikube* runs Kubernetes version: *v1.31.0* and Docker version: *27.2.0*. - -Minikubes primary purpose is to test an aplication / kubernetes metadata before -deploying it to a production cluster. The following section describes the -differences / limited *x0 kube installer* features. - -## 2. Production Cluster Difference +## 2. Production Cluster Differences 1. *x0 minikube* setup lacks enhanced Role Base Access Control functionality / enhanced private *Docker Registry* integration (e.g. GitLab). @@ -117,220 +92,17 @@ standalone-chrome container(s) in combination with Remote WebDriver are used. http://www.selenium.dev -## 6. Installation Preparation - -The following sections cover the installation process in detail. - -### 6.1. Backends - -The minikube system can be run on multiple backends / virtualization products, -called **driver**. - -Tested installations / drivers (MS Windows). - -- VMWare -- VirtualBox -- Microsoft Hyper-V - ->[!WARNING] -> Dispite all recommendations using docker as preferred driver, **we** do not do -> this for MS Windows (tm) systems. Hyper-V is your friend here. - -### 6.2. Internet-less Setup - -In some (iso-certified) environments internet-access is limited, even turned off -completely. We recommend using the VMWare driver (--driver=vmware) in this case. - ->[!WARNING] -> The addons installation is very picky with the image verification. -> Be sure that base iso and correct docker images are locally cached correctly. -> It is possible to install minikube on an internet connected client and copy the -> cached data afterwards to the internet-less system. - ->[!NOTE] -> In some environments / more complex network setups (multiple NIC / routed) -> VMWare NAT NIC settings have to be adapted manually (DNS, Port Forwarding ...). - -### 6.3. Global Recommendations - -Be sure to run the minikube setup / installation as Administrator User. - ->[!WARNING] -> Always manage the cluster from an unprivileged account (Powershell) using -> Administrator rights. - -### 6.4. Hyper-V - -However, the Virtualbox and VMWare drivers did misbehave on multiple machines -directly connected to the internet. Only MS Hyper-V did work out of the box here. - -The following message indicates no external network connectivity. This should not -be the case with `--driver=hyperv`. - -> ! Failing to connect to https://registry.k8s.io/ from both inside the minikube -> VM and host machine - ->[!NOTE] -> Also modern Windows *Virtual Based Security* works out of the box with enabled -> hardware acceleration. With VMWare and VirtualBox driver the Windows feature -> *Virtual Based Security* should be turned off completely. - -The following powershell command will install and enable Windows Hyper-V. - -```powershell -# enable windows hyper-v feature -Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -``` - -### 6.5. VMWare - -First, we recommend to disable Windows feature *Virtual Based Security*. -Without VMs do not use VT-x / VT-d CPU hardware capabilities. - -Also add the following VMWare path to the systems environment variable `Path`. - -```powershell -C:\Program Files (x86)\VMware\VMware Workstation -``` - -### 6.6. Kubectl - -Also `kubectl` cluster administration utility is required by *x0 kube installer*. - -https://kubernetes.io/docs/tasks/tools/install-kubectl-windows/ - -### 6.7. Python - -Install Python for Windows 3.13 globally (all users). - -On the first installer page. - -- Set checkbox "Use admin privileges when installing py.exe" -- Set checkbox "Add python.exe to PATH" - -Click "Customize installation". +## 6. Post Installation -Important checkboxes **must** be set. +Check the following. -- "py launcher" -- "for all users (requires admin privileges)" - -## 7. Installation - -Generate the cluster now and install *x0-system* inside. - -### Download Docker Images - -Only important for a non-internet environment. - -Minikube connected to the internet will download all Docker images -automatically. - -Download required mages from https://docker.webcodex.de/x0/ manually (an own -Docker sub-registry is in preparation). - -### 7.1. Import Docker Base Images - -```powershell -# import docker images (system / addons) -minikube image load docker.registry.2.8.3.tar -minikube image load docker.kube-registry-proxy.v0.0.6.tar -minikube image load docker.minikube-ingress-dns.v0.0.3.tar -minikube image load docker.ingress-nginx.controller.v1.11.2.tar -``` - -### 7.2. Generate minikube Cluster - -1. Option 1 (MS Hyper-V) - -```powershell -# generate minikube cluster (ms hyper-v) -minikube.exe start --driver=hyperv -``` - -2. Option 2 (VMWare) - -```powershell -# generate minikube cluster (vmware driver) -minikube.exe start --driver=vmware -``` - -After successful cluster creation the following lines should appear. - -> * minikube v1.34.0 on Microsoft Windows 11 Pro ... - -> * Done! kubectl is now configured to use "minikube" cluster and "default" -> namespace by default - -Perfect, cluster is up and running. Now, check which external IPv4 address the -cluster is using. - -```powershell -# check minikube cluster ip -minikube.exe ip -``` - -This should output the current configured external IP address. -Remember the address for later DNS usage. - -### 7.3. Install Addons - -```powershell -# enable addons "registry", "ingress" and "ingress-dns" -minikube addons enable registry -minikube addons enable ingress -minikube addons enable ingress-dns - -# check addons enabled -minikube addons list -``` - -### 7.4. Import 3d Party Docker Images - -```powershell -# import docker images (selenium test framework) -minikube image load docker.selenium-standalone-chrome.tar - -# import docker images (kubegres and postgres) -minikube image load docker.reactivetechio.kubegres.1.19.tar -minikube image load docker.postgres.14.tar -``` - -### 7.5. Import x0 Docker Images - ->[!WARNING] -> This step is **mandatory** / images **must** be downloaded and installed manually. - -```powershell -# import docker images (x0 system) -minikube image load docker.x0-app.tar -minikube image load docker.x0-db-install.tar -minikube image load docker.x0-test.tar -``` - -### 7.6. Run Setup / x0-Installer - -Now its time to run the installer. - -The installer uses `../../config/app-config.json` as configuration source. - -```powershell -# start setup / installation -cd .\setup\ -python.exe Setup.py -``` - -## 8. Post Installation - -After installation, the following kubernetes objects should have been generated. - -### 8.1. Namespaces +### 6.1. Namespaces Get all existing namespaces. -```powershell +```bash # get namespaces -kubectl.exe get namespace -o wide +kubectl get namespace -o wide ``` Check, if the following namespaces, especially "x0-app" exists. @@ -345,13 +117,13 @@ Check, if the following namespaces, especially "x0-app" exists. | kubegres-system | Active | | x0-app | Active | -### 8.2. Pods +### 6.2. Pods Get all existing pods from namespace "x0-app". -```powershell +```bash # get pods -kubectl.exe get pods -n x0-app +kubectl get pods -n x0-app ``` Check if the following pods exist, especially "READY" and "STATUS". @@ -366,13 +138,13 @@ Also an internal IPv4 address must be assigned. | x0-test-deployment-... | 1/1 | Running | 10.244.x.x | | x0-test-test-run | 0/1 | Completed | 10.244.x.x | -### 8.3. Service +### 6.3. Service Get all services from namespace "x0-app". -```powershell +```bash # get service -kubectl.exe get service -o wide -n x0-app +kubectl get service -o wide -n x0-app ``` Check if the following services exist. @@ -384,13 +156,13 @@ Check if the following services exist. | selenium-server-0 | ClusterIP | 10.103.x.x | 4444/TCP | | x0-x0-app-test-svc | ClusterIP | 10.104.x.x | 80/TCP | -### 8.4. Endpoints +### 6.4. Endpoints Get all endpoints from namespace "x0-app". -```powershell +```bash # get endpoints -kubectl.exe get endpoints -o wide -n x0-app +kubectl get endpoints -o wide -n x0-app ``` Check if the following endpoints exist and the IP-mapping to pods is correct. @@ -402,35 +174,27 @@ Check if the following endpoints exist and the IP-mapping to pods is correct. | selenium-server-0 | 10.244.x.x:4444 | | x0-x0-app-test-svc | 10.244.x.x:80 | -### 8.5. Ingress +### 6.5. Ingress Get ingress from namespace "x0-app". -```powershell +```bash # get ingress -kubectl.exe get ingress -o wide -n x0-app +kubectl get ingress -o wide -n x0-app ``` -Check if the following ingress exists. +Check if the following ingress exists (IP addresses / subnets may differ). | NAME | CLASS | HOSTS | ADDRESS | | ------------------------------- | ------------------ | ------------------ | ------------------ | | | | | | | x0-x0-app-test-minikube-ingress | nginx | x0-app.x0.localnet | 172.17.84.42 | -### 8.6. DNS - ->[!WARNING] -> Note kubernetes is using **4 IP scopes** - -1. Hypervisor Host IP (NAT) -2. Ingress: Docker Network (172.17.x.x) -3. Kube Proxy Subnets (10.103.x.x, 10.104.x.x) -4. Internal PODs (10.244.x.x) +### 6.6. DNS Get the Hypervisor Host IP. -```powershell +```bash # get external ip address minikube ip ``` @@ -453,14 +217,14 @@ Also check the following test(s) and example(s). To be 100% sure everything is working correctly, check the test-pods output. -```powershell +```bash # get test pods logs -kubectl.exe logs x0-test-test-run -o wide -n x0-app +kubectl logs x0-test-test-run -o wide -n x0-app ``` The last lines should look like this. -```powershell +```bash ------------------ generated xml file: /tmp/pytest-junit.xml ------------------- -======================== 40 passed in 81.53s (0:01:21) ========================= +======================== 51 passed in 81.53s (0:01:21) ========================= ``` diff --git a/kubernetes/setup/Setup.py b/kubernetes/setup/Setup.py index 33a7560f..950e4271 100755 --- a/kubernetes/setup/Setup.py +++ b/kubernetes/setup/Setup.py @@ -24,7 +24,7 @@ def log_message(log_prefix, msg): logging.info('{}:{}'.format(log_prefix, msg)) -def prepare_minikube_hyperv(os_type, subtype, driver, offline_install): +def prepare_minikube(os_type, subtype, driver, offline_install): if os_type == 'mswindows' and subtype == 'minikube' and driver == 'hyperv' and offline_install is False: @@ -68,6 +68,25 @@ def prepare_minikube_hyperv(os_type, subtype, driver, offline_install): ) res = subprocess.run(cmd_image_load, shell=True) + if os_type == 'linux' and subtype == 'minikube' and driver == 'docker' and offline_install is False: + + get_installer_cmd = 'curl -LO https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-amd64' + res = subprocess.run(get_installer_cmd, shell=True) + + install_cmd = 'sudo install minikube-linux-amd64 /usr/local/bin/minikube' + res = subprocess.run(install_cmd, shell=True) + + cmd_create_cluster = 'minikube start --driver=docker' + res = subprocess.run(cmd_create_cluster, shell=True) + + res = subprocess.run('minikube addons enable registry', shell=True) + res = subprocess.run('minikube addons enable ingress', shell=True) + res = subprocess.run('minikube addons enable ingress-dns', shell=True) + + res = subprocess.run('minikube image pull reactivetechio/kubegres:1.19', shell=True) + res = subprocess.run('minikube image pull postgres:14', shell=True) + res = subprocess.run('minikube image pull selenium/standalone-chrome:latest', shell=True) + def gen_kubernetes_templates(ConfRef, environment, tpl_group='app'): config = ConfRef.getConfig() @@ -439,7 +458,7 @@ def getRuntimeData(self): print("Install sub type:{}".format(install_subtype)) # prepare win - prepare_minikube_hyperv(os_type, install_subtype, minikube_driver, install_offline) + prepare_minikube(os_type, install_subtype, minikube_driver, install_offline) load_balancers = get_loadbalancers(CH) log_message(log_prefix, 'LoadBalancers:{}'.format(load_balancers)) diff --git a/kubernetes/template/01-deployment.yaml b/kubernetes/template/01-deployment.yaml index 2cee65a7..670a5232 100644 --- a/kubernetes/template/01-deployment.yaml +++ b/kubernetes/template/01-deployment.yaml @@ -19,7 +19,7 @@ spec: automountServiceAccountToken: false containers: - name: ${x0_APP_ID}-${x0_APP_ENV}-app - image: docker.io/library/${x0_KUBERNETES_IMAGE} + image: ghcr.io/webcodex1/${x0_KUBERNETES_IMAGE} imagePullPolicy: IfNotPresent resources: requests: diff --git a/kubernetes/template/11-db-install.yaml b/kubernetes/template/11-db-install.yaml index 072c372d..e376c856 100644 --- a/kubernetes/template/11-db-install.yaml +++ b/kubernetes/template/11-db-install.yaml @@ -8,6 +8,6 @@ spec: automountServiceAccountToken: false containers: - name: db-install - image: docker.io/library/${x0_DB_NAME}-db-install:latest + image: ghcr.io/webcodex1/${x0_DB_NAME}-db-install:latest imagePullPolicy: IfNotPresent restartPolicy: Never diff --git a/kubernetes/template/14-test-run.yaml b/kubernetes/template/14-test-run.yaml index 4cc63ef1..d809e4a2 100644 --- a/kubernetes/template/14-test-run.yaml +++ b/kubernetes/template/14-test-run.yaml @@ -8,7 +8,7 @@ spec: automountServiceAccountToken: false containers: - name: test-run - image: docker.io/library/${x0_APP_ID}-test:latest + image: ghcr.io/webcodex1/${x0_APP_ID}-test:latest imagePullPolicy: IfNotPresent env: - name: RUN_NAMESPACE