diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..4ba66bf1eb --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,49 @@ +version: 2.1 + +jobs: + build: + docker: + - image: circleci/python:3.7.3-stretch + + working_directory: ~/repo + + steps: + - checkout + + # Set up Docker environment + - setup_remote_docker + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + ls + cd $(eval echo "$CIRCLE_WORKING_DIRECTORY/project-ml-microservice-kubernetes") + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + # run tests! + - run: + name: run tests + command: | + cd $(eval echo "$CIRCLE_WORKING_DIRECTORY/project-ml-microservice-kubernetes") + . venv/bin/activate + make test + # run lints! + - run: + name: run lint + command: | + cd $(eval echo "$CIRCLE_WORKING_DIRECTORY/project-ml-microservice-kubernetes") + . venv/bin/activate + make lint diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..112032326c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.devops/ +*.env \ No newline at end of file diff --git a/project-ml-microservice-kubernetes/.circleci/config.yaml b/project-ml-microservice-kubernetes/.circleci/config.yaml new file mode 100644 index 0000000000..4d9f9448c8 --- /dev/null +++ b/project-ml-microservice-kubernetes/.circleci/config.yaml @@ -0,0 +1,48 @@ +version: 2.1 + +jobs: + build: + docker: + - image: circleci/python:3.7.3-stretch + + working_directory: ~/repo + + steps: + - checkout + + # Set up Docker environment + - setup_remote_docker + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + cd $(eval echo "$CIRCLE_WORKING_DIRECTORY/DevOps_Microservices/project-ml-microservice-kubernetes") + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + # run tests! + - run: + name: run tests + command: | + cd $(eval echo "$CIRCLE_WORKING_DIRECTORY/DevOps_Microservices/project-ml-microservice-kubernetes") + . venv/bin/activate + make test + # run lints! + - run: + name: run lint + command: | + cd $(eval echo "$CIRCLE_WORKING_DIRECTORY/DevOps_Microservices/project-ml-microservice-kubernetes") + . venv/bin/activate + make lint diff --git a/project-ml-microservice-kubernetes/Dockerfile b/project-ml-microservice-kubernetes/Dockerfile index 8f70001b57..4a1935be99 100644 --- a/project-ml-microservice-kubernetes/Dockerfile +++ b/project-ml-microservice-kubernetes/Dockerfile @@ -2,17 +2,22 @@ FROM python:3.7.3-stretch ## Step 1: # Create a working directory +WORKDIR /app ## Step 2: # Copy source code to working directory +COPY requirements.txt ./ ## Step 3: # Install packages from requirements.txt # hadolint ignore=DL3013 +RUN pip install --no-cache-dir -r requirements.txt +COPY . . ## Step 4: # Expose port 80 +EXPOSE 80 ## Step 5: # Run app.py at container launch - +CMD [ "python", "app.py" ] diff --git a/project-ml-microservice-kubernetes/Makefile b/project-ml-microservice-kubernetes/Makefile index 759f77a425..ad653bbcc0 100644 --- a/project-ml-microservice-kubernetes/Makefile +++ b/project-ml-microservice-kubernetes/Makefile @@ -9,6 +9,7 @@ setup: # Create python virtualenv & source it # source ~/.devops/bin/activate python3 -m venv ~/.devops + source ~/.devops/bin/activate install: # This should be run from inside a virtualenv @@ -22,8 +23,10 @@ test: lint: # See local hadolint install instructions: https://github.com/hadolint/hadolint + wget -O ./hadolint https://github.com/hadolint/hadolint/releases/download/v1.16.3/hadolint-Linux-x86_64 &&\ + chmod +x ./hadolint # This is linter for Dockerfiles - hadolint Dockerfile + ./hadolint Dockerfile # This is a linter for Python source code linter: https://www.pylint.org/ # This should be run from inside a virtualenv pylint --disable=R,C,W1203,W1202 app.py diff --git a/project-ml-microservice-kubernetes/README.md b/project-ml-microservice-kubernetes/README.md index a5bd8fee6e..5ba3e285ab 100644 --- a/project-ml-microservice-kubernetes/README.md +++ b/project-ml-microservice-kubernetes/README.md @@ -1,4 +1,5 @@ - + +[![lx0612](https://circleci.com/gh/lx0612/DevOps_Microservices.svg?style=svg)](https://app.circleci.com/pipelines/github/lx0612/DevOps_Microservices) ## Project Overview @@ -44,7 +45,24 @@ source .devops/bin/activate ### Kubernetes Steps -* Setup and Configure Docker locally -* Setup and Configure Kubernetes locally -* Create Flask app in Container -* Run via kubectl +1. **Setup and Configure Docker** + - Go to the Docker Desktop website at https://www.docker.com/products/docker-desktop/ and follow the instructions to install Docker Desktop. + - Verify : `docker --version`. + +2. **Setup and Configure Kubernetes** + - For Windows users, the recommended way is to use Docker Desktop. Open Docker Desktop, go to Settings, navigate to Kubernetes, and check "Enable Kubernetes." + - Verify the Kubernetes configuration by running: `kubectl version --output json`. + +3. **Create Flask App in a Container** + - Build the Docker image for the Flask app using the following command: `docker build --tag udacity-pj4:v1.0.0 .` + - Run the container with the created image: `docker run -d --rm -p 8000:80 udacity-pj4:v1.0.0` + +4. **Deploy Flask App via Kubernetes** + - Create an environment file `.env` and set variable `DOCKER_PASSWORD=`. + - run: `source .env`. + - Export your Docker Hub ID using: `export docker_path=`. + - Log in to Docker Hub to push the image: `echo "$DOCKER_PASSWORD" | docker login --username $docker_path --password-stdin`. + - Tag and push the Docker image to Docker Hub: `docker image tag udacity-pj4:v1.0.0 $docker_path/udacity-pj4:v1.0.0 && docker image push $docker_path/udacity-pj4:v1.0.0`. + - Create a Kubernetes deployment: `kubectl create deploy udacity-pj4 --image="$docker_path/udacity-pj4:v1.0.0"`. + - Check whether the pod is in the READY state: `kubectl get pods`. + - Wait pods ready, forward the port to access the Flask app locally: `kubectl port-forward deployment.apps/udacity-pj4 8000:80`. diff --git a/project-ml-microservice-kubernetes/app.py b/project-ml-microservice-kubernetes/app.py index 7d583b72e5..730ef514c5 100644 --- a/project-ml-microservice-kubernetes/app.py +++ b/project-ml-microservice-kubernetes/app.py @@ -1,4 +1,8 @@ -from flask import Flask, request, jsonify +from flask import ( + Flask, + request, + jsonify + ) from flask.logging import create_logger import logging @@ -63,6 +67,7 @@ def predict(): # get an output prediction from the pretrained model, clf prediction = list(clf.predict(scaled_payload)) # TO DO: Log the output prediction value + LOG.info(f'prediction value: {prediction}') return jsonify({'prediction': prediction}) if __name__ == "__main__": diff --git a/project-ml-microservice-kubernetes/output_txt_files/docker_out.txt b/project-ml-microservice-kubernetes/output_txt_files/docker_out.txt index 79b164d89c..4e12d1e8f6 100644 --- a/project-ml-microservice-kubernetes/output_txt_files/docker_out.txt +++ b/project-ml-microservice-kubernetes/output_txt_files/docker_out.txt @@ -1 +1,10 @@ - \ No newline at end of file +[2023-11-18 17:43:35,145] INFO in app: JSON payload: +{'CHAS': {'0': 0}, 'RM': {'0': 6.575}, 'TAX': {'0': 296.0}, 'PTRATIO': {'0': 15.3}, 'B': {'0': 396.9}, 'LSTAT': {'0': 4.98}} +[2023-11-18 17:43:35,157] INFO in app: Inference payload DataFrame: + CHAS RM TAX PTRATIO B LSTAT +0 0 6.575 296.0 15.3 396.9 4.98 +[2023-11-18 17:43:35,165] INFO in app: Scaling Payload: + CHAS RM TAX PTRATIO B LSTAT +0 0 6.575 296.0 15.3 396.9 4.98 +[2023-11-18 17:43:35,168] INFO in app: prediction value: [20.35373177134412] +172.17.0.1 - - [18/Nov/2023 17:43:35] "POST /predict HTTP/1.1" 200 - \ No newline at end of file diff --git a/project-ml-microservice-kubernetes/output_txt_files/kubernetes_out.txt b/project-ml-microservice-kubernetes/output_txt_files/kubernetes_out.txt index a79241a757..335d7d6d62 100644 --- a/project-ml-microservice-kubernetes/output_txt_files/kubernetes_out.txt +++ b/project-ml-microservice-kubernetes/output_txt_files/kubernetes_out.txt @@ -1 +1,29 @@ - \ No newline at end of file + +### K8s: +voclabs:~/environment/DevOps_Microservices/project-ml-microservice-kubernetes (master) $ kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system coredns-5dd5756b68-rt42b 1/1 Running 0 21m +kube-system etcd-minikube 1/1 Running 0 21m +kube-system kube-apiserver-minikube 1/1 Running 0 21m +kube-system kube-controller-manager-minikube 1/1 Running 0 21m +kube-system kube-proxy-qgfbj 1/1 Running 0 21m +kube-system kube-scheduler-minikube 1/1 Running 0 21m +kube-system storage-provisioner 1/1 Running 1 (20m ago) 21m +voclabs:~/environment/DevOps_Microservices/project-ml-microservice-kubernetes (master) $ ./run_kubernetes.sh +lx96/udacity-pj4:v1.0.0 +deployment.apps/udacity-pj4 created +NAME READY STATUS RESTARTS AGE +udacity-pj4-7759b46ccd-hv79b 0/1 ContainerCreating 0 0s +error: unable to forward port because pod is not running. Current status=Pending +voclabs:~/environment/DevOps_Microservices/project-ml-microservice-kubernetes (master) $ kubectl port-forward deployment.apps/udacity-pj4 --address 0.0.0.0 8000:80 +Forwarding from 0.0.0.0:8000 -> 80 +Handling connection for 8000 + +### Call API +voclabs:~/environment/DevOps_Microservices/project-ml-microservice-kubernetes (master) $ ./make_prediction.sh +Port: 8000 +{ + "prediction": [ + 20.35373177134412 + ] +} \ No newline at end of file diff --git a/project-ml-microservice-kubernetes/run_docker.sh b/project-ml-microservice-kubernetes/run_docker.sh index 65c3f832e4..55f697569e 100755 --- a/project-ml-microservice-kubernetes/run_docker.sh +++ b/project-ml-microservice-kubernetes/run_docker.sh @@ -3,10 +3,16 @@ ## Complete the following steps to get Docker running locally # Step 1: +image_name=udacity-pj4 +image_tag=v1.0.0 +container_name=udacity-pj4 + # Build image and add a descriptive tag +docker build -t $image_name:$image_tag . -# Step 2: # List docker images +docker image list -# Step 3: # Run flask app +docker run --name $container_name -p 8000:80 --rm $image_name:$image_tag + diff --git a/project-ml-microservice-kubernetes/run_kubernetes.sh b/project-ml-microservice-kubernetes/run_kubernetes.sh index b041b10827..5134d94600 100755 --- a/project-ml-microservice-kubernetes/run_kubernetes.sh +++ b/project-ml-microservice-kubernetes/run_kubernetes.sh @@ -1,18 +1,26 @@ #!/usr/bin/env bash # This tags and uploads an image to Docker Hub +image_name=udacity-pj4 +image_tag=v1.0.0 +deployment_name=udacity-pj4 # Step 1: # This is your Docker ID/path -# dockerpath=<> +docker_path=lx96 +echo $docker_path/$image_name:$image_tag # Step 2 # Run the Docker Hub container with kubernetes +kubectl create deploy $deployment_name --image=$docker_path/$image_name:$image_tag # Step 3: # List kubernetes pods +kubectl get pods # Step 4: # Forward the container port to a host +kubectl port-forward deployment.apps/$deployment_name --address 0.0.0.0 8000:80 + diff --git a/project-ml-microservice-kubernetes/upload_docker.sh b/project-ml-microservice-kubernetes/upload_docker.sh index 19baeafe4b..8fbf7608b1 100755 --- a/project-ml-microservice-kubernetes/upload_docker.sh +++ b/project-ml-microservice-kubernetes/upload_docker.sh @@ -1,15 +1,27 @@ -#!/usr/bin/env bash -# This file tags and uploads an image to Docker Hub +#!/bin/bash -# Assumes that an image is built via `run_docker.sh` +image_name=udacity-pj4 +image_tag=v1.0.0 -# Step 1: -# Create dockerpath -# dockerpath= +# Create docker_path +docker_path=lx96 -# Step 2: -# Authenticate & tag -echo "Docker ID and Image: $dockerpath" +# Add variable DOCKER_PASSWORD +source .env + +# Log in to Docker +echo docker login --username "$docker_path" --password-stdin $DOCKER_PASSWORD + +# Check if login was successful +if [ $? -eq 0 ]; then + echo "Docker authentication successful!" +else + echo "Docker authentication failed!" +fi + +echo "Docker ID and Image: $docker_path" + +docker image tag "$image_name:$image_tag" "$docker_path/$image_name:$image_tag" -# Step 3: # Push image to a docker repository +docker push "$docker_path/$image_name:$image_tag" \ No newline at end of file