Chore: speedy docker (#191)

* Fix docker build and update stretto * Update lock file * chore: speedy docker * fix log name --------- Co-authored-by: Michael Danenberg <[email protected]>
rpcpool · May 2, 2024 · bd0f4c7 · bd0f4c7
1 parent b9cf47f
commit bd0f4c7
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 180 deletions.
diff --git a/Api.Dockerfile b/Api.Dockerfile
@@ -1,54 +1,4 @@
-FROM rust:1.75-bullseye AS chef
-RUN cargo install cargo-chef
-FROM chef AS planner
-
-RUN mkdir /rust
-COPY Cargo.toml /rust
-COPY core /rust/core
-COPY das_api /rust/das_api
-COPY digital_asset_types /rust/digital_asset_types
-COPY integration_tests /rust/integration_tests
-COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
-COPY migration /rust/migration
-COPY nft_ingester /rust/nft_ingester
-COPY ops /rust/ops
-COPY program_transformers /rust/program_transformers
-COPY tools /rust/tools
-COPY blockbuster rust/blockbuster
-
-WORKDIR /rust/das_api
-RUN cargo chef prepare --recipe-path /rust/das_api/recipe.json
-
-FROM chef AS builder
-RUN apt-get update -y && \
-    apt-get install -y build-essential make git
-
-RUN mkdir /rust
-COPY Cargo.toml /rust
-COPY core /rust/core
-COPY das_api /rust/das_api
-COPY digital_asset_types /rust/digital_asset_types
-COPY integration_tests /rust/integration_tests
-COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
-COPY migration /rust/migration
-COPY nft_ingester /rust/nft_ingester
-COPY ops /rust/ops
-COPY program_transformers /rust/program_transformers
-COPY tools /rust/tools
-COPY blockbuster rust/blockbuster
-
-WORKDIR /rust/das_api
-COPY --from=planner /rust/das_api/recipe.json recipe.json
-
-# Build dependencies - this is the caching Docker layer!
-# TODO: Fix this.  For now we are building without the cached dependencies as there's apparently
-# some problem with digital-asset-types feature activation.
-
-# RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target --all-features
-
-# Build application
-RUN cargo build --release
-
+FROM das-api/builder AS files
 FROM rust:1.75-slim-bullseye
 ARG APP=/usr/src/app
 RUN apt update \
@@ -59,7 +9,7 @@ ENV TZ=Etc/UTC \
 RUN groupadd $APP_USER \
     && useradd -g $APP_USER $APP_USER \
     && mkdir -p ${APP}
-COPY --from=builder /rust/target/release/das_api ${APP}
+COPY --from=files /das/das_api ${APP}
 RUN chown -R $APP_USER:$APP_USER ${APP}
 USER $APP_USER
 WORKDIR ${APP}

diff --git a/Builder.Dockerfile b/Builder.Dockerfile
@@ -0,0 +1,25 @@
+FROM rust:1.75-bullseye AS builder
+RUN apt-get update -y && \
+    apt-get install -y build-essential make git
+
+RUN mkdir /rust
+RUN mkdir /rust/bins
+COPY Cargo.toml /rust
+COPY core /rust/core
+COPY das_api /rust/das_api
+COPY digital_asset_types /rust/digital_asset_types
+COPY integration_tests /rust/integration_tests
+COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
+COPY migration /rust/migration
+COPY nft_ingester /rust/nft_ingester
+COPY ops /rust/ops
+COPY program_transformers /rust/program_transformers
+COPY tools /rust/tools
+COPY blockbuster rust/blockbuster
+WORKDIR /rust
+RUN --mount=type=cache,target=/rust/target,id=das-rust \
+  cargo build --release --bins && cp `find /rust/target/release -maxdepth 1 -type f | sed 's/^\.\///' | grep -v "\." ` /rust/bins
+
+FROM rust:1.75-slim-bullseye as final
+COPY --from=builder /rust/bins /das/
+CMD echo "Built the DAS API bins!"
diff --git a/Ingest.Dockerfile b/Ingest.Dockerfile
@@ -1,54 +1,4 @@
-FROM rust:1.75-bullseye AS chef
-RUN cargo install cargo-chef
-FROM chef AS planner
-
-RUN mkdir /rust
-COPY Cargo.toml /rust
-COPY core /rust/core
-COPY das_api /rust/das_api
-COPY digital_asset_types /rust/digital_asset_types
-COPY integration_tests /rust/integration_tests
-COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
-COPY migration /rust/migration
-COPY nft_ingester /rust/nft_ingester
-COPY ops /rust/ops
-COPY program_transformers /rust/program_transformers
-COPY tools /rust/tools
-COPY blockbuster /rust/blockbuster
-
-WORKDIR /rust/nft_ingester
-RUN cargo chef prepare --recipe-path /rust/nft_ingester/recipe.json
-
-FROM chef AS builder
-RUN apt-get update -y && \
-    apt-get install -y build-essential make git
-
-RUN mkdir /rust
-COPY Cargo.toml /rust
-COPY core /rust/core
-COPY das_api /rust/das_api
-COPY digital_asset_types /rust/digital_asset_types
-COPY integration_tests /rust/integration_tests
-COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
-COPY migration /rust/migration
-COPY nft_ingester /rust/nft_ingester
-COPY ops /rust/ops
-COPY program_transformers /rust/program_transformers
-COPY tools /rust/tools
-COPY blockbuster /rust/blockbuster
-
-WORKDIR /rust/nft_ingester
-COPY --from=planner /rust/nft_ingester/recipe.json recipe.json
-
-# Build dependencies - this is the caching Docker layer!
-# TODO: Fix this.  For now we are building without the cached dependencies as there's apparently
-# some problem with digital-asset-types feature activation.
-
-# RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target --all-features
-
-# Build application
-RUN cargo build --release
-
+FROM das-api/builder AS files
 FROM rust:1.75-slim-bullseye
 ARG APP=/usr/src/app
 RUN apt update \
@@ -59,7 +9,7 @@ ENV TZ=Etc/UTC \
 RUN groupadd $APP_USER \
     && useradd -g $APP_USER $APP_USER \
     && mkdir -p ${APP}
-COPY --from=builder /rust/target/release/nft_ingester ${APP}
+COPY --from=files /das/nft_ingester ${APP}
 RUN chown -R $APP_USER:$APP_USER ${APP}
 USER $APP_USER
 WORKDIR ${APP}

diff --git a/Load.Dockerfile b/Load.Dockerfile
@@ -1,51 +1,4 @@
-FROM rust:1.75-bullseye AS chef
-RUN cargo install cargo-chef
-FROM chef AS planner
-
-RUN mkdir /rust
-COPY Cargo.toml /rust
-COPY core /rust/core
-COPY das_api /rust/das_api
-COPY digital_asset_types /rust/digital_asset_types
-COPY integration_tests /rust/integration_tests
-COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
-COPY migration /rust/migration
-COPY nft_ingester /rust/nft_ingester
-COPY ops /rust/ops
-COPY program_transformers /rust/program_transformers
-COPY tools /rust/tools
-COPY blockbuster /rust/blockbuster
-
-WORKDIR /rust/tools/load_generation
-RUN cargo chef prepare --recipe-path /rust/tools/load_generation/recipe.json
-
-FROM chef AS builder
-RUN apt-get update -y && \
-    apt-get install -y build-essential make git
-
-RUN mkdir /rust
-COPY Cargo.toml /rust
-COPY core /rust/core
-COPY das_api /rust/das_api
-COPY digital_asset_types /rust/digital_asset_types
-COPY integration_tests /rust/integration_tests
-COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
-COPY migration /rust/migration
-COPY nft_ingester /rust/nft_ingester
-COPY ops /rust/ops
-COPY program_transformers /rust/program_transformers
-COPY tools /rust/tools
-COPY blockbuster /rust/blockbuster
-
-WORKDIR /rust/tools/load_generation
-COPY --from=planner /rust/tools/load_generation/recipe.json recipe.json
-
-# Build dependencies - this is the caching Docker layer!
-RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target
-
-# Build application
-RUN cargo build --release
-
+FROM das-api/builder AS files
 FROM rust:1.75-slim-bullseye
 ARG APP=/usr/src/app
 RUN apt update \
@@ -56,7 +9,7 @@ ENV TZ=Etc/UTC \
 RUN groupadd $APP_USER \
     && useradd -g $APP_USER $APP_USER \
     && mkdir -p ${APP}
-COPY --from=builder /rust/target/release/load_generation ${APP}
+COPY --from=files /das/load_generation ${APP}
 RUN chown -R $APP_USER:$APP_USER ${APP}
 USER $APP_USER
 WORKDIR ${APP}

diff --git a/Migrator.Dockerfile b/Migrator.Dockerfile
@@ -1,21 +1,7 @@
+FROM das-api/builder AS files
+
 FROM rust:1.75-bullseye
 COPY init.sql /init.sql
 ENV INIT_FILE_PATH=/init.sql
-
-COPY Cargo.toml /
-COPY ./core /core
-COPY ./das_api /das_api
-COPY ./digital_asset_types /digital_asset_types
-COPY ./integration_tests /integration_tests
-COPY ./metaplex-rpc-proxy /metaplex-rpc-proxy
-COPY ./migration /migration
-COPY ./nft_ingester /nft_ingester
-COPY ./ops /ops
-COPY ./program_transformers /program_transformers
-COPY ./tools /tools
-COPY ./blockbuster /blockbuster
-
-WORKDIR /migration
-RUN cargo build --release
-WORKDIR /target/release
-CMD /target/release/migration up -n 100
+COPY --from=files /das/migration /bins/migration
+CMD /bins/migration up -n 100
diff --git a/Proxy.Dockerfile b/Proxy.Dockerfile
@@ -1,4 +1,5 @@
 FROM rust:1.75-bullseye AS builder
+RUN cargo install wasm-pack
 
 RUN mkdir /rust
 COPY ./Cargo.toml /rust
@@ -15,10 +16,11 @@ COPY ./tools /rust/tools
 COPY ./blockbuster /rust/blockbuster
 
 WORKDIR /rust/metaplex-rpc-proxy
-RUN cargo install wasm-pack
-RUN wasm-pack build --release
+RUN mkdir /rust/wasm-out/
+RUN --mount=type=cache,target=/rust/target,id=das-wasm \
+    wasm-pack build --release && cp /rust/target/wasm32-unknown-unknown/release/metaplex_rpc_proxy.wasm /rust/wasm-out/
 
 FROM envoyproxy/envoy:v1.24.0
-COPY --from=builder /rust/target/wasm32-unknown-unknown/release/metaplex_rpc_proxy.wasm /etc/rpc_proxy.wasm
+COPY --from=builder /rust/wasm-out/metaplex_rpc_proxy.wasm /etc/rpc_proxy.wasm
 RUN apt-get update && apt-get install -y ca-certificates
 ENTRYPOINT /usr/local/bin/envoy -c /etc/envoy.yaml -l trace --service-cluster proxy
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,5 +1,10 @@
 version: "3.9"
 services:
+  builder:
+    image: das-api/builder
+    build:
+      context: .
+      dockerfile: Builder.Dockerfile
   graphite:
     image: graphiteapp/graphite-statsd
     container_name: graphite
@@ -13,6 +18,7 @@ services:
   migrator:
     depends_on:
       - db
+      - builder
     environment:
       DATABASE_URL: postgres://solana:solana@db/solana
     build:
@@ -21,6 +27,7 @@ services:
   ingester:
     depends_on:
       - migrator
+      - builder
       - redis
       - db
       - graphite
@@ -41,6 +48,7 @@ services:
     depends_on:
       - migrator
       - ingester
+      - builder
       - solana
     restart: always
     environment:
@@ -62,6 +70,7 @@ services:
   api:
     depends_on:
       - migrator
+      - builder
     restart: always
     environment:
       APP_DATABASE_URL: postgres://solana:solana@db/solana
@@ -79,7 +88,7 @@ services:
       - "6379:6379"
   db:
     image: 'postgres:14'
-    command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ,"-c","max_connections=200"]
+    command: [ "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ,"-c","max_connections=200" ]
     ports:
       - 5432:5432
     environment:
@@ -91,9 +100,9 @@ services:
   solana:
     image: ghcr.io/metaplex-foundation/plerkle-test-validator:v1.6.0-1.69.0-v1.16.6
     volumes:
-    - ./programs:/so/:ro
-    - ./ledger:/config:rw
-    - ./solana-test-validator-geyser-config:/plugin-config:rw
+      - ./programs:/so/:ro
+      - ./ledger:/config:rw
+      - ./solana-test-validator-geyser-config:/plugin-config:rw
     environment:
       RUST_LOG: error
       PLUGIN_MESSENGER_CONFIG.messenger_type: "Redis"
@@ -104,5 +113,5 @@ services:
       - "8899:8899"
       - "9900:9900"
 volumes:
-  grafana_data: {}
-  graphite_data: {}
+  grafana_data: { }
+  graphite_data: { }