refactor: use internal queue scheduler

.dockerignore

@@ -2,6 +2,8 @@ build/
 pb/api.pb.go
 server/ui
 worker/data
+server/cmd/wire_gen.go
+worker/cmd/wire_gen.go
 .DS_Store
 tmp/
 configs/

.gitignore

@@ -1,7 +1,9 @@
-/build
-pb/api.pb.go
-server/ui
-worker/data
+/build/
+/server/ui/
+/worker/data
+/server/cmd/wire_gen.go
+/worker/cmd/wire_gen.go
+/pb/api.pb.go
 .DS_Store
 tmp/
 configs/testing/**/*.pem

Dockerfile

@@ -8,7 +8,7 @@ WORKDIR /app/ui
 RUN yarn install && yarn build
 
 # stage 2 build
-FROM golang:1.14-alpine as build
+FROM golang:1.15-alpine as build
 
 ARG GIT_COMMIT=""
 ENV GIT_COMMIT=$GIT_COMMIT
@@ -21,9 +21,9 @@ COPY --from=ui /app/ui/dist /app/web/abstruse/dist
 
 COPY . /app/
 
-RUN go get github.com/jkuri/statik github.com/golang/protobuf/protoc-gen-go@v1.3
+RUN go get github.com/jkuri/statik github.com/golang/protobuf/protoc-gen-go github.com/google/wire/...
 
-RUN make protoc && make statik && make server
+RUN make protoc && make statik && make wire && make server
 
 # stage 3 image
 FROM scratch
@@ -33,12 +33,12 @@ LABEL maintainer="Jan Kuri <[email protected]>" \
   org.label-schema.name="abstruse-server" \
   org.label-schema.description="Distributed Continuous Intergration Platform" \
   org.label-schema.url="https://ci.abstruse.cc/" \
-  org.label-schema.vcs-url="https://github.com/jkuri/abstruse" \
+  org.label-schema.vcs-url="https://github.com/bleenco/abstruse" \
   org.label-schema.vendor="abstruse"
 
 COPY --from=build /etc/ssl/certs /etc/ssl/certs
 COPY --from=build /app/build/abstruse-server /usr/bin/abstruse-server
 
 ENTRYPOINT [ "/usr/bin/abstruse-server" ]
 
-EXPOSE 80 2379 2380
+EXPOSE 80

Dockerfile.worker

@@ -1,5 +1,5 @@
 # stage 1 build
-FROM golang:1.14-alpine as build
+FROM golang:1.15-alpine as build
 
 ARG GIT_COMMIT=""
 ENV GIT_COMMIT=$GIT_COMMIT
@@ -10,9 +10,9 @@ RUN apk --no-cache add git make protobuf protobuf-dev ca-certificates alpine-sdk
 
 COPY . /app/
 
-RUN go get github.com/jkuri/statik github.com/golang/protobuf/protoc-gen-go@v1.3
+RUN go get github.com/jkuri/statik github.com/golang/protobuf/protoc-gen-go github.com/google/wire/...
 
-RUN make protoc && make statik_worker && make worker
+RUN make protoc && make wire && make worker
 
 # stage 2 image
 FROM scratch
@@ -22,7 +22,7 @@ LABEL maintainer="Jan Kuri <[email protected]>" \
   org.label-schema.name="abstruse-worker" \
   org.label-schema.description="Distributed Continuous Intergration Platform" \
   org.label-schema.url="https://ci.abstruse.cc/" \
-  org.label-schema.vcs-url="https://github.com/jkuri/abstruse" \
+  org.label-schema.vcs-url="https://github.com/bleenco/abstruse" \
   org.label-schema.vendor="abstruse"
 
 COPY --from=build /etc/ssl/certs /etc/ssl/certs

Makefile

@@ -13,38 +13,35 @@ endif
 
 all: build
 
-build: build_ui statik statik_worker protoc server worker
+build: build_ui statik wire protoc server worker
 
 server:
 	@CGO_ENABLED=${CGO_ENABLED} go build -i -ldflags "-X ${ABSTRUSE_VERSION_PATH}.GitCommit=${GIT_COMMIT} -X ${ABSTRUSE_VERSION_PATH}.UIVersion=${ABSTRUSE_UI_VERSION} -X ${ABSTRUSE_VERSION_PATH}.BuildDate=${BUILD_DATE}" -o build/abstruse-server ./cmd/abstruse-server
 
 worker:
-	@CGO_ENABLED=${CGO_ENABLED} go build -ldflags "-X ${ABSTRUSE_VERSION_PATH}.GitCommit=${GIT_COMMIT} -X ${ABSTRUSE_VERSION_PATH}.UIVersion=${ABSTRUSE_UI_VERSION} -X ${ABSTRUSE_VERSION_PATH}.BuildDate=${BUILD_DATE}" -o build/abstruse-worker ./cmd/abstruse-worker
+	@CGO_ENABLED=${CGO_ENABLED} go build -i -ldflags "-X ${ABSTRUSE_VERSION_PATH}.GitCommit=${GIT_COMMIT} -X ${ABSTRUSE_VERSION_PATH}.UIVersion=${ABSTRUSE_UI_VERSION} -X ${ABSTRUSE_VERSION_PATH}.BuildDate=${BUILD_DATE}" -o build/abstruse-worker ./cmd/abstruse-worker
 
 build_ui:
 	@if [ ! -d "web/abstruse/dist" ]; then cd web/abstruse && yarn build; fi
 
-build_pty:
-	@if [ ! -r "build/worker-data/abstruse-pty" ]; then GOOS=linux GOARCH=amd64 go build -o build/worker-data/abstruse-pty ./cmd/abstruse-pty; fi
-
 statik:
 	@if [ ! -r "server/ui/statik.go" ]; then statik -dest ./server -p ui -src ./web/abstruse/dist; fi
 
-statik_worker: build_pty
-	@if [ ! -r "worker/data/statik.go" ]; then statik -dest ./worker -p data -src ./build/worker-data; fi
+wire:
+	@wire ./server/cmd/... ./worker/cmd/...
 
 install_dependencies:
-	@go get github.com/jkuri/statik github.com/golang/protobuf/protoc-gen-go@v1.3 github.com/cespare/reflex
+	@cd /tmp && go get github.com/golang/protobuf/protoc-gen-go github.com/jkuri/statik github.com/cespare/reflex github.com/google/wire/... && cd -
 	@cd web/abstruse && yarn install
 
 clean:
-	@rm -rf build/ web/abstruse/dist server/ui/ worker/data pb/api.pb.go
+	@rm -rf build/ web/abstruse/dist server/ui/ server/cmd/wire_gen.go worker/cmd/wire_gen.go
 
 dev:
-	@reflex -sr '\.go$$' -R '^web/' -R '^server/ui' -R '^worker/' -R '^configs/' -R '^tests/' -- sh -c 'make server && ./build/abstruse-server --log-level debug'
+	@reflex -sr '\.go$$' -R '^web/' -R '^server/ui' -R '^worker/' -R '^configs/' -R '^tests/' -- sh -c 'make server && ./build/abstruse-server --logger-level debug'
 
 dev_worker:
-	@reflex -sr '\.go$$' -R '^web/' -R '^server/' -R '^configs/' -R '^tests/' -- sh -c 'make worker && ./build/abstruse-worker --log-level debug'
+	@reflex -sr '\.go$$' -R '^web/' -R '^server/' -R '^configs/' -R '^tests/' -- sh -c 'make worker && ./build/abstruse-worker --logger-level debug'
 
 protoc:
 	@protoc ./pb/api.proto --go_out=plugins=grpc:./pb/
@@ -70,4 +67,4 @@ test-unit:
 test-e2e:
 	go run ./tests/e2e
 
-.PHONY: build server build_ui statik install_dependencies clean protoc dev test test-unit test-e2e statik_worker build_pty worker dev_worker docker docker_server docker_worker docker_push
+.PHONY: build server worker build_ui statik wire install_dependencies clean dev dev_worker protoc docker docker_server docker_worker docker_push test test-unit test-e2e

README.md

@@ -6,7 +6,7 @@
 
 [![Slack](https://slackin-xxrlxvwnmd.now.sh/badge.svg)](https://join.slack.com/t/abstruse/shared_invite/enQtNDI0MzUxMTQ1OTExLTMyNGRiYjllMTQ4NjkzZDkwNDM4NGIwMGM2YjA5NjFmNzI3MzdkMWExYWRlNWQ1N2NjNzI4NDlhOTFmNGM5ZTM)
 
-**Abstruse 2.x.x** is a lightweight, yet powerful distributed CI/CD written in Golang. It features embedded `etcd` server for scheduling and service discovery, integrated private `Docker Registry` for synchronization of nodes, and a new frontend design with useful features for better user experience. Its default configuration uses single node cluster with n workers, however, this cluster can be easily extended with more nodes if necessary. We used `gorm` package for implementing the storage, users can choose between `mysql`, `postgresql` and `mssql`.
+**Abstruse 2.x.x** is a lightweight, yet powerful distributed CI/CD written in Golang. Its default configuration uses single node cluster with n workers, however, this cluster can be easily extended with more nodes if necessary.
 
 This is a work in progress for v2.0.0 relase, if you are looking for Node.JS based v1.x.x version please check [here](https://github.com/bleenco/abstruse/tree/v1).
 

configs/demo/default/docker-compose.yml

@@ -22,7 +22,7 @@ services:
     depends_on:
       - mysql
     ports:
-      - "4400:80"
+      - "80:80"
     environment:
       - ABSTRUSE_HTTP_COMPRESS=true
       - ABSTRUSE_DB_HOST=10.20.20.10
@@ -31,14 +31,6 @@ services:
       - ABSTRUSE_DB_PASSWORD=demotest
       - ABSTRUSE_DB_NAME=abstruse_demo
       - ABSTRUSE_AUTH_JWTSECRET=fa27ef3ef6570e32a79e74deca7c1bc3
-      - ABSTRUSE_ETCD_HOST=0.0.0.0
-      - ABSTRUSE_ETCD_DATADIR=etcd/
-      - ABSTRUSE_ETCD_USERNAME=abstruse
-      - ABSTRUSE_ETCD_PASSWORD=235c36984614ec1cbc9f6172a7ba8c9b
-      - ABSTRUSE_ETCD_ROOTPASSWORD=1c45154abbf67419b5a20c9258c71775
-      - ABSTRUSE_REGISTRY_DATADIR=registry/
-      - ABSTRUSE_REGISTRY_USERNAME=abstruse
-      - ABSTRUSE_REGISTRY_PASSWORD=fc066d0290bd670fa4f2b38609ebce3d
     volumes:
       - "./testdata/abstruse-server:/abstruse"
     command:
@@ -55,13 +47,8 @@ services:
     depends_on:
       - abstruse-server
     environment:
-      - ABSTRUSE_GRPC_LISTENADDR=10.20.20.30:3330
-      - ABSTRUSE_ETCD_ADDR=10.20.20.20:2379
-      - ABSTRUSE_ETCD_USERNAME=abstruse
-      - ABSTRUSE_ETCD_PASSWORD=235c36984614ec1cbc9f6172a7ba8c9b
-      - ABSTRUSE_REGISTRY_ADDR=10.20.20.10
-      - ABSTRUSE_REGISTRY_USERNAME=abstruse
-      - ABSTRUSE_REGISTRY_PASSWORD=fc066d0290bd670fa4f2b38609ebce3d
+      - ABSTRUSE_GRPC_ADDR=0.0.0.0:3330
+      - ABSTRUSE_SERVER_ADDR=http://10.20.20.20:80
       - ABSTRUSE_SCHEDULER_MAXPARALLEL=2
       - ABSTRUSE_AUTH_JWTSECRET=fa27ef3ef6570e32a79e74deca7c1bc3
     volumes:

go.mod

@@ -1,57 +1,46 @@
 module github.com/bleenco/abstruse
 
-go 1.14
+go 1.15
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
-	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/Sirupsen/logrus v0.0.0-00010101000000-000000000000 // indirect
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
-	github.com/drone/go-scm v1.7.0
+	github.com/drone/go-scm v1.7.1
 	github.com/dustin/go-humanize v1.0.0
 	github.com/felixge/httpsnoop v1.0.1
-	github.com/garyburd/redigo v1.6.0 // indirect
-	github.com/go-chi/chi v4.1.2+incompatible
-	github.com/go-git/go-git/v5 v5.1.0
+	github.com/go-chi/chi v1.5.0
+	github.com/go-chi/cors v1.1.1
+	github.com/go-git/go-git/v5 v5.2.0
 	github.com/go-ole/go-ole v1.2.4 // indirect
-	github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee
-	github.com/gobwas/pool v0.2.0 // indirect
-	github.com/gobwas/ws v1.0.3
-	github.com/golang/protobuf v1.3.5
-	github.com/google/go-cmp v0.5.0 // indirect
-	github.com/google/uuid v1.1.1
-	github.com/gorilla/handlers v1.4.2 // indirect
-	github.com/gorilla/mux v1.7.4 // indirect
-	github.com/jinzhu/gorm v1.9.12
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.0.4
+	github.com/golang/protobuf v1.4.3
+	github.com/google/uuid v1.1.2
+	github.com/google/wire v0.4.0
+	github.com/jinzhu/gorm v1.9.16
 	github.com/jkuri/statik v0.3.0
 	github.com/jpillora/backoff v1.0.0
-	github.com/json-iterator/go v1.1.9 // indirect
-	github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381
+	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/mssola/user_agent v0.5.2
 	github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/shirou/gopsutil v2.20.5+incompatible
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.3
+	github.com/shirou/gopsutil v3.20.10+incompatible
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.0
-	go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
-	go.uber.org/zap v1.15.0
-	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
-	google.golang.org/grpc v1.29.1
+	go.uber.org/zap v1.10.0
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
+	golang.org/x/sys v0.0.0-20201126233918-771906719818 // indirect
+	golang.org/x/text v0.3.4 // indirect
+	google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 // indirect
+	google.golang.org/grpc v1.33.2
+	google.golang.org/protobuf v1.25.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.2.8
-	sigs.k8s.io/yaml v1.2.0 // indirect
 )
-
-replace github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.6.0