Merge branch 'develop' to master for 0.1.0 release

splunk · Mar 20, 2020 · cb761d6 · cb761d6
2 parents 1be429a + 78b6874
commit cb761d6
Show file tree

Hide file tree

Showing 109 changed files with 50,288 additions and 2,526 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,16 +1,181 @@
-version: 2
-jobs:
-  compile-binary:
+# See https://circleci.com/blog/using-circleci-workflows-to-replicate-docker-hub-automated-builds/
+version: 2.1
+workflows:
+  version: 2
+  build-and-push:
+    jobs:
+      - build-image
+      - unit-tests
+      - vulnerability-scan:
+          requires:
+            - build-image
+      - push-latest:
+          requires:
+            - build-image
+            - unit-tests
+            - vulnerability-scan
+          filters:
+            branches:
+              only:
+                - master
+      - push-edge:
+          requires:
+            - build-image
+            - unit-tests
+            - vulnerability-scan
+          filters:
+            branches:
+              only:
+                - develop
+      - push-release:
+          requires:
+            - build-image
+            - unit-tests
+            - vulnerability-scan
+          filters:
+            tags:
+              only: /^\d+\.\d+\.\d+/
+            branches:
+              ignore: /.*/
+executors:
+  golang-builder:
+    environment:
+      IMAGE_NAME: splunk/splunk-operator
+      IMAGE_FILENAME: splunk-operator
+    working_directory: /opt/app-root/src/splunk-operator
     docker:
-      - image: splunk/splunk-operator-builder:master
+      - image: splunk/splunk-operator-builder:develop
+  docker-publisher:
+    environment:
+      IMAGE_NAME: splunk/splunk-operator
+      IMAGE_FILENAME: splunk-operator
+    docker:
+      - image: circleci/buildpack-deps:buster
+  classic-machine:
+    environment:
+      IMAGE_NAME: splunk/splunk-operator
+      IMAGE_FILENAME: splunk-operator
+    machine:
+      image: circleci/classic:latest
+commands:
+  save_image:
+    description: "Save image"
+    steps:
+      - run:
+          name: "Save image"
+          command: |
+            mkdir -p /tmp/images
+            docker image save -o /tmp/images/${IMAGE_FILENAME}-${CIRCLE_SHA1}.tar ${IMAGE_NAME}:${CIRCLE_SHA1}
+  load_image:
+    description: "Load image"
+    steps:
+      - run:
+          name: "Load image"
+          command: docker load -i /tmp/images/${IMAGE_FILENAME}-${CIRCLE_SHA1}.tar
+  push_image:
+    description: "Load, tag and push an image"
+    parameters:
+      tag:
+        type: string
+    steps:
+      - load_image
+      - run: 
+          name: Tag image
+          command: |
+            docker tag ${IMAGE_NAME}:${CIRCLE_SHA1} ${IMAGE_NAME}:<< parameters.tag >>
+      - run:
+          name: Push latest image to DockerHub
+          command: |
+            echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+            docker push ${IMAGE_NAME}:<< parameters.tag >>
+jobs:
+  build-image:
+    executor: golang-builder
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run: 
+          name: Pull base image updates
+          command: docker pull registry.access.redhat.com/ubi8/ubi-minimal:latest
+      - run: 
+          name: Build splunk/splunk-operator image
+          command: operator-sdk build --verbose ${IMAGE_NAME}:${CIRCLE_SHA1}
+      - save_image
+      - persist_to_workspace:
+          name: Persist images to workspace
+          root: /tmp
+          paths:
+            - images
+      - store_artifacts:
+          name: Save images as artifacts
+          path: /tmp/images
+  unit-tests:
+    executor: golang-builder
     steps:
       - checkout
+      - run: 
+          name: Check source formatting
+          command: X=`make fmt` && if [[ "x$X" != "x" ]]; then echo $X && false; fi
+      - run: 
+          name: Lint source code
+          command: X=`make lint` && if [[ "x$X" != "x" ]]; then echo $X && false; fi
       - run:
-          name: Build splunk-operator binary
-          command: go build -v -o ./build/_output/bin/splunk-operator ./cmd/manager
-workflows:
-  version: 2
-  build:
-    jobs:
-      - compile-binary
-
+          name: Run package tests
+          command: make test
+      - run:
+          name: Upload coverage.out
+          command: goveralls -coverprofile=coverage.out -service=circle-ci -repotoken $COVERALLS_TOKEN
+      - store_artifacts:
+          name: Save coverage.out as artifact
+          path: coverage.out
+  vulnerability-scan:
+    executor: classic-machine
+    steps:
+      - checkout
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - load_image
+      - run:
+          name: Setup clair scanner
+          command: make setup_clair_scanner
+      - run:
+          name: Scan container image
+          command: make run_clair_scan
+      - run:
+          name: Stop clair scanner
+          command: make stop_clair_scanner
+      - store_artifacts:
+          name: Save scan results as artifacts
+          path: clair-scanner-logs
+  push-latest:
+    executor: docker-publisher
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: false
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - push_image:
+          tag: "latest"
+  push-edge:
+    executor: docker-publisher
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: false
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - push_image:
+          tag: "edge"
+  push-release:
+    executor: docker-publisher
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: false
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - push_image:
+          tag: "${CIRCLE_TAG}"
diff --git a/.gitignore b/.gitignore
@@ -84,3 +84,5 @@ push_targets
 splunk-enterprise.lic
 splunk-operator-*.yaml
 splunk-operator-*.tar.gz
+clair-scanner
+clair-scanner-logs
diff --git a/Makefile b/Makefile
@@ -1,46 +1,106 @@
 # Makefile for Splunk Operator
 
-.PHONY: all image package local clean run fmt lint
+.PHONY: all builder builder-image image package local clean run fmt lint
+
+# Security Scanner Variables
+SCANNER_DATE := `date +%Y-%m-%d`
+SCANNER_DATE_YEST := `TZ=GMT+24 +%Y:%m:%d`
+SCANNER_VERSION := v8
+SCANNER_LOCALIP := $(shell ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | awk '{print $1}' | head -n 1)
+ifeq ($(shell uname), Linux)
+	SCANNER_FILE = clair-scanner_linux_amd64
+else ifeq ($(shell uname), Darwin)
+	SCANNER_FILE = clair-scanner_darwin_amd64
+else
+	SCANNER_FILE = clair-scanner_windows_amd64.exe
+endif
 
 all: image
 
-builder-image:
+builder:
 	@echo Creating container image to build splunk-operator
 	@docker build -f ./build/Dockerfile.builder -t splunk/splunk-operator-builder .
 
-builder: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
-	@echo Using container to build splunk-operator
+builder-image:
+	@echo Using builder container to build splunk-operator
 	@mkdir -p ./build/_output/bin
-	@docker run -v ${PWD}:/opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "cd /opt/app-root/src/splunk-operator && go build -v -o ./build/_output/bin/splunk-operator ./cmd/manager"
+	@docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/opt/app-root/src/splunk-operator -w /opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "operator-sdk build --verbose splunk/splunk-operator"
 
-image: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
-	@echo Building splunk-operator image
-	@operator-sdk build splunk/splunk-operator
+builder-test:
+	@echo Running unit tests for splunk-operator inside of builder container
+	@docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/opt/app-root/src/splunk-operator -w /opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "go test -v -covermode=count -coverprofile=coverage.out --timeout=300s github.com/splunk/splunk-operator/pkg/splunk/resources github.com/splunk/splunk-operator/pkg/splunk/spark github.com/splunk/splunk-operator/pkg/splunk/enterprise github.com/splunk/splunk-operator/pkg/splunk/reconcile github.com/splunk/splunk-operator/pkg/splunk/client"
 
-package: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
-	@build/package.sh
+image:
+	@echo Building splunk-operator image
+	@operator-sdk build --verbose splunk/splunk-operator
 
-local: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
+local:
+	@echo Building splunk-operator-local binary only
 	@mkdir -p ./build/_output/bin
 	@go build -v -o ./build/_output/bin/splunk-operator-local ./cmd/manager
 
-clean:
+test:
+	@echo Running unit tests for splunk-operator
+	@go test -v -covermode=count -coverprofile=coverage.out --timeout=300s github.com/splunk/splunk-operator/pkg/splunk/resources github.com/splunk/splunk-operator/pkg/splunk/spark github.com/splunk/splunk-operator/pkg/splunk/enterprise github.com/splunk/splunk-operator/pkg/splunk/reconcile github.com/splunk/splunk-operator/pkg/splunk/client
+
+stop_clair_scanner:
+	@docker stop clair_db || true
+	@docker rm clair_db || true
+	@docker stop clair || true
+	@docker rm clair || true
+
+setup_clair_scanner: stop_clair_scanner
+	@mkdir -p clair-scanner-logs
+	@docker pull arminc/clair-db:${SCANNER_DATE} || docker pull arminc/clair-db:${SCANNER_DATE_YEST} || echo "WARNING: Failed to pull daily image, defaulting to latest" >> clair-scanner-logs/clair_setup_errors.log ; docker pull arminc/clair-db:latest
+	@docker run -d --name clair_db arminc/clair-db:${SCANNER_DATE} || docker run -d --name clair_db arminc/clair-db:${SCANNER_DATE_YEST} || docker run -d --name clair_db arminc/clair-db:latest
+	@docker run -p 6060:6060 --link clair_db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6
+	@wget https://github.com/arminc/clair-scanner/releases/download/${SCANNER_VERSION}/${SCANNER_FILE}
+	@mv ${SCANNER_FILE} clair-scanner
+	@chmod +x clair-scanner
+	@echo "Waiting for clair daemon to start"
+	@retries=0 ; while( ! wget -T 10 -q -O /dev/null http://0.0.0.0:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $$retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$$(($$retries+1)) ; done
+	@echo "Clair daemon started."
+
+run_clair_scan:
+	@./clair-scanner -c http://0.0.0.0:6060 --ip ${SCANNER_LOCALIP} -r clair-scanner-logs/results.json -l clair-scanner-logs/results.log splunk/splunk-operator
+
+generate:
+	@echo Running operator-sdk generate k8s
+	@operator-sdk generate k8s
+	@echo Running operator-sdk generate crds
+	@cp deploy/rbac.yaml deploy/role.yaml
+	@operator-sdk generate crds
+	@rm -f deploy/role.yaml deploy/crds/*_cr.yaml
+	@echo Rebuilding deploy/crds/combined.yaml
+	@echo "---" > deploy/crds/combined.yaml
+	@cat deploy/crds/enterprise.splunk.com_standalones_crd.yaml >> deploy/crds/combined.yaml
+	@echo "---" >> deploy/crds/combined.yaml
+	@cat deploy/crds/enterprise.splunk.com_licensemasters_crd.yaml >> deploy/crds/combined.yaml
+	@echo "---" >> deploy/crds/combined.yaml
+	@cat deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml >> deploy/crds/combined.yaml
+	@echo "---" >> deploy/crds/combined.yaml
+	@cat deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml >> deploy/crds/combined.yaml
+	@echo "---" >> deploy/crds/combined.yaml
+	@cat deploy/crds/enterprise.splunk.com_sparks_crd.yaml >> deploy/crds/combined.yaml
+	@echo Rebuilding deploy/all-in-one-scoped.yaml
+	@cat deploy/crds/combined.yaml deploy/rbac.yaml deploy/operator.yaml > deploy/all-in-one-scoped.yaml
+	@echo Rebuilding deploy/all-in-one-cluster.yaml
+	@cat deploy/crds/combined.yaml deploy/rbac.yaml deploy/cluster_operator.yaml > deploy/all-in-one-cluster.yaml
+
+package: lint fmt generate
+	@build/package.sh
+
+clean: stop_clair_scanner
 	@rm -rf ./build/_output
 	@docker rmi splunk/splunk-operator || true
+	@rm -f clair-scanner
+	@rm -rf clair-scanner-logs
 
 run:
-	@OPERATOR_NAME=splunk-operator operator-sdk up local
+	@OPERATOR_NAME=splunk-operator operator-sdk run --local
 
 fmt:
 	@gofmt -l -w `find ./ -name "*.go"`
 
 lint:
 	@golint ./...
-
-deploy/all-in-one-scoped.yaml: deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/operator.yaml
-	@echo Rebuilding deploy/all-in-one-scoped.yaml
-	@cat deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/operator.yaml > deploy/all-in-one-scoped.yaml
-
-deploy/all-in-one-cluster.yaml: deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/cluster_operator.yaml
-	@echo Rebuilding deploy/all-in-one-cluster.yaml
-	@cat deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/cluster_operator.yaml > deploy/all-in-one-cluster.yaml