diff --git a/Api.Dockerfile b/Api.Dockerfile index 162ceb19f..cbc4ee6ee 100644 --- a/Api.Dockerfile +++ b/Api.Dockerfile @@ -1,54 +1,4 @@ -FROM rust:1.75-bullseye AS chef -RUN cargo install cargo-chef -FROM chef AS planner - -RUN mkdir /rust -COPY Cargo.toml /rust -COPY core /rust/core -COPY das_api /rust/das_api -COPY digital_asset_types /rust/digital_asset_types -COPY integration_tests /rust/integration_tests -COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy -COPY migration /rust/migration -COPY nft_ingester /rust/nft_ingester -COPY ops /rust/ops -COPY program_transformers /rust/program_transformers -COPY tools /rust/tools -COPY blockbuster rust/blockbuster - -WORKDIR /rust/das_api -RUN cargo chef prepare --recipe-path /rust/das_api/recipe.json - -FROM chef AS builder -RUN apt-get update -y && \ - apt-get install -y build-essential make git - -RUN mkdir /rust -COPY Cargo.toml /rust -COPY core /rust/core -COPY das_api /rust/das_api -COPY digital_asset_types /rust/digital_asset_types -COPY integration_tests /rust/integration_tests -COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy -COPY migration /rust/migration -COPY nft_ingester /rust/nft_ingester -COPY ops /rust/ops -COPY program_transformers /rust/program_transformers -COPY tools /rust/tools -COPY blockbuster rust/blockbuster - -WORKDIR /rust/das_api -COPY --from=planner /rust/das_api/recipe.json recipe.json - -# Build dependencies - this is the caching Docker layer! -# TODO: Fix this. For now we are building without the cached dependencies as there's apparently -# some problem with digital-asset-types feature activation. - -# RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target --all-features - -# Build application -RUN cargo build --release - +FROM das-api/builder AS files FROM rust:1.75-slim-bullseye ARG APP=/usr/src/app RUN apt update \ @@ -59,7 +9,7 @@ ENV TZ=Etc/UTC \ RUN groupadd $APP_USER \ && useradd -g $APP_USER $APP_USER \ && mkdir -p ${APP} -COPY --from=builder /rust/target/release/das_api ${APP} +COPY --from=files /das/das_api ${APP} RUN chown -R $APP_USER:$APP_USER ${APP} USER $APP_USER WORKDIR ${APP} diff --git a/Builder.Dockerfile b/Builder.Dockerfile new file mode 100644 index 000000000..4ea85b7c9 --- /dev/null +++ b/Builder.Dockerfile @@ -0,0 +1,25 @@ +FROM rust:1.75-bullseye AS builder +RUN apt-get update -y && \ + apt-get install -y build-essential make git + +RUN mkdir /rust +RUN mkdir /rust/bins +COPY Cargo.toml /rust +COPY core /rust/core +COPY das_api /rust/das_api +COPY digital_asset_types /rust/digital_asset_types +COPY integration_tests /rust/integration_tests +COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy +COPY migration /rust/migration +COPY nft_ingester /rust/nft_ingester +COPY ops /rust/ops +COPY program_transformers /rust/program_transformers +COPY tools /rust/tools +COPY blockbuster rust/blockbuster +WORKDIR /rust +RUN --mount=type=cache,target=/rust/target,id=das-rust \ + cargo build --release --bins && cp `find /rust/target/release -maxdepth 1 -type f | sed 's/^\.\///' | grep -v "\." ` /rust/bins + +FROM rust:1.75-slim-bullseye as final +COPY --from=builder /rust/bins /das/ +CMD echo "Built the DAS API bins!" diff --git a/Ingest.Dockerfile b/Ingest.Dockerfile index 4a195cf76..a0a6023d6 100644 --- a/Ingest.Dockerfile +++ b/Ingest.Dockerfile @@ -1,54 +1,4 @@ -FROM rust:1.75-bullseye AS chef -RUN cargo install cargo-chef -FROM chef AS planner - -RUN mkdir /rust -COPY Cargo.toml /rust -COPY core /rust/core -COPY das_api /rust/das_api -COPY digital_asset_types /rust/digital_asset_types -COPY integration_tests /rust/integration_tests -COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy -COPY migration /rust/migration -COPY nft_ingester /rust/nft_ingester -COPY ops /rust/ops -COPY program_transformers /rust/program_transformers -COPY tools /rust/tools -COPY blockbuster /rust/blockbuster - -WORKDIR /rust/nft_ingester -RUN cargo chef prepare --recipe-path /rust/nft_ingester/recipe.json - -FROM chef AS builder -RUN apt-get update -y && \ - apt-get install -y build-essential make git - -RUN mkdir /rust -COPY Cargo.toml /rust -COPY core /rust/core -COPY das_api /rust/das_api -COPY digital_asset_types /rust/digital_asset_types -COPY integration_tests /rust/integration_tests -COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy -COPY migration /rust/migration -COPY nft_ingester /rust/nft_ingester -COPY ops /rust/ops -COPY program_transformers /rust/program_transformers -COPY tools /rust/tools -COPY blockbuster /rust/blockbuster - -WORKDIR /rust/nft_ingester -COPY --from=planner /rust/nft_ingester/recipe.json recipe.json - -# Build dependencies - this is the caching Docker layer! -# TODO: Fix this. For now we are building without the cached dependencies as there's apparently -# some problem with digital-asset-types feature activation. - -# RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target --all-features - -# Build application -RUN cargo build --release - +FROM das-api/builder AS files FROM rust:1.75-slim-bullseye ARG APP=/usr/src/app RUN apt update \ @@ -59,7 +9,7 @@ ENV TZ=Etc/UTC \ RUN groupadd $APP_USER \ && useradd -g $APP_USER $APP_USER \ && mkdir -p ${APP} -COPY --from=builder /rust/target/release/nft_ingester ${APP} +COPY --from=files /das/nft_ingester ${APP} RUN chown -R $APP_USER:$APP_USER ${APP} USER $APP_USER WORKDIR ${APP} diff --git a/Load.Dockerfile b/Load.Dockerfile index e4936cdb1..ae4ee3ec1 100644 --- a/Load.Dockerfile +++ b/Load.Dockerfile @@ -1,51 +1,4 @@ -FROM rust:1.75-bullseye AS chef -RUN cargo install cargo-chef -FROM chef AS planner - -RUN mkdir /rust -COPY Cargo.toml /rust -COPY core /rust/core -COPY das_api /rust/das_api -COPY digital_asset_types /rust/digital_asset_types -COPY integration_tests /rust/integration_tests -COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy -COPY migration /rust/migration -COPY nft_ingester /rust/nft_ingester -COPY ops /rust/ops -COPY program_transformers /rust/program_transformers -COPY tools /rust/tools -COPY blockbuster /rust/blockbuster - -WORKDIR /rust/tools/load_generation -RUN cargo chef prepare --recipe-path /rust/tools/load_generation/recipe.json - -FROM chef AS builder -RUN apt-get update -y && \ - apt-get install -y build-essential make git - -RUN mkdir /rust -COPY Cargo.toml /rust -COPY core /rust/core -COPY das_api /rust/das_api -COPY digital_asset_types /rust/digital_asset_types -COPY integration_tests /rust/integration_tests -COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy -COPY migration /rust/migration -COPY nft_ingester /rust/nft_ingester -COPY ops /rust/ops -COPY program_transformers /rust/program_transformers -COPY tools /rust/tools -COPY blockbuster /rust/blockbuster - -WORKDIR /rust/tools/load_generation -COPY --from=planner /rust/tools/load_generation/recipe.json recipe.json - -# Build dependencies - this is the caching Docker layer! -RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target - -# Build application -RUN cargo build --release - +FROM das-api/builder AS files FROM rust:1.75-slim-bullseye ARG APP=/usr/src/app RUN apt update \ @@ -56,7 +9,7 @@ ENV TZ=Etc/UTC \ RUN groupadd $APP_USER \ && useradd -g $APP_USER $APP_USER \ && mkdir -p ${APP} -COPY --from=builder /rust/target/release/load_generation ${APP} +COPY --from=files /das/load_generation ${APP} RUN chown -R $APP_USER:$APP_USER ${APP} USER $APP_USER WORKDIR ${APP} diff --git a/Migrator.Dockerfile b/Migrator.Dockerfile index a834c2e52..d275aa7d6 100644 --- a/Migrator.Dockerfile +++ b/Migrator.Dockerfile @@ -1,21 +1,7 @@ +FROM das-api/builder AS files + FROM rust:1.75-bullseye COPY init.sql /init.sql ENV INIT_FILE_PATH=/init.sql - -COPY Cargo.toml / -COPY ./core /core -COPY ./das_api /das_api -COPY ./digital_asset_types /digital_asset_types -COPY ./integration_tests /integration_tests -COPY ./metaplex-rpc-proxy /metaplex-rpc-proxy -COPY ./migration /migration -COPY ./nft_ingester /nft_ingester -COPY ./ops /ops -COPY ./program_transformers /program_transformers -COPY ./tools /tools -COPY ./blockbuster /blockbuster - -WORKDIR /migration -RUN cargo build --release -WORKDIR /target/release -CMD /target/release/migration up -n 100 \ No newline at end of file +COPY --from=files /das/migration /bins/migration +CMD /bins/migration up -n 100 \ No newline at end of file diff --git a/Proxy.Dockerfile b/Proxy.Dockerfile index 694349d9e..9679a632e 100644 --- a/Proxy.Dockerfile +++ b/Proxy.Dockerfile @@ -1,4 +1,5 @@ FROM rust:1.75-bullseye AS builder +RUN cargo install wasm-pack RUN mkdir /rust COPY ./Cargo.toml /rust @@ -15,10 +16,11 @@ COPY ./tools /rust/tools COPY ./blockbuster /rust/blockbuster WORKDIR /rust/metaplex-rpc-proxy -RUN cargo install wasm-pack -RUN wasm-pack build --release +RUN mkdir /rust/wasm-out/ +RUN --mount=type=cache,target=/rust/target,id=das-wasm \ + wasm-pack build --release && cp /rust/target/wasm32-unknown-unknown/release/metaplex_rpc_proxy.wasm /rust/wasm-out/ FROM envoyproxy/envoy:v1.24.0 -COPY --from=builder /rust/target/wasm32-unknown-unknown/release/metaplex_rpc_proxy.wasm /etc/rpc_proxy.wasm +COPY --from=builder /rust/wasm-out/metaplex_rpc_proxy.wasm /etc/rpc_proxy.wasm RUN apt-get update && apt-get install -y ca-certificates ENTRYPOINT /usr/local/bin/envoy -c /etc/envoy.yaml -l trace --service-cluster proxy diff --git a/docker-compose.yaml b/docker-compose.yaml index be60e2e56..364a00855 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,10 @@ version: "3.9" services: + builder: + image: das-api/builder + build: + context: . + dockerfile: Builder.Dockerfile graphite: image: graphiteapp/graphite-statsd container_name: graphite @@ -13,6 +18,7 @@ services: migrator: depends_on: - db + - builder environment: DATABASE_URL: postgres://solana:solana@db/solana build: @@ -21,6 +27,7 @@ services: ingester: depends_on: - migrator + - builder - redis - db - graphite @@ -41,6 +48,7 @@ services: depends_on: - migrator - ingester + - builder - solana restart: always environment: @@ -62,6 +70,7 @@ services: api: depends_on: - migrator + - builder restart: always environment: APP_DATABASE_URL: postgres://solana:solana@db/solana @@ -79,7 +88,7 @@ services: - "6379:6379" db: image: 'postgres:14' - command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ,"-c","max_connections=200"] + command: [ "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr" ,"-c","max_connections=200" ] ports: - 5432:5432 environment: @@ -91,9 +100,9 @@ services: solana: image: ghcr.io/metaplex-foundation/plerkle-test-validator:v1.6.0-1.69.0-v1.16.6 volumes: - - ./programs:/so/:ro - - ./ledger:/config:rw - - ./solana-test-validator-geyser-config:/plugin-config:rw + - ./programs:/so/:ro + - ./ledger:/config:rw + - ./solana-test-validator-geyser-config:/plugin-config:rw environment: RUST_LOG: error PLUGIN_MESSENGER_CONFIG.messenger_type: "Redis" @@ -104,5 +113,5 @@ services: - "8899:8899" - "9900:9900" volumes: - grafana_data: {} - graphite_data: {} + grafana_data: { } + graphite_data: { }