chore: document starting from a docker tarfile base image (#345)

Co-authored-by: thesayyn <[email protected]>

e2e/convert_docker_tarball/.gitignore

@@ -0,0 +1 @@
+image.tar

e2e/convert_docker_tarball/BUILD.bazel

@@ -0,0 +1,48 @@
+load("@aspect_bazel_lib//lib:run_binary.bzl", "run_binary")
+load("@container_structure_test//:defs.bzl", "container_structure_test")
+load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball")
+
+sh_binary(
+    name = "convert",
+    srcs = ["convert.bash"],
+    data = [
+        "@oci_crane_registry_toolchains//:current_toolchain",
+        "@oci_crane_toolchains//:current_toolchain",
+    ],
+)
+
+# Before building this example, you'll need to run ./create_base_image.bash to produce an
+# image.tar file. It's large so we .gitignore it.
+run_binary(
+    name = "base",
+    srcs = ["image.tar"],
+    args = [
+        "$@",
+        "$(location :image.tar)",
+        "$(CRANE_BIN)",
+        "$(LAUNCHER_WRAPPER)",
+    ],
+    out_dirs = ["oci"],
+    tool = ":convert",
+    toolchains = [
+        "@oci_crane_toolchains//:current_toolchain",
+        "@oci_crane_registry_toolchains//:current_toolchain",
+    ],
+)
+
+oci_image(
+    name = "image",
+    base = ":base",
+)
+
+oci_tarball(
+    name = "tar",
+    image = ":image",
+    repo_tags = [],
+)
+
+container_structure_test(
+    name = "test",
+    configs = ["test.yaml"],
+    image = ":image",
+)

e2e/convert_docker_tarball/Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    LANGUAGE=C.UTF-8
+
+RUN apt-get -y update \
+    && apt-get -y install jq \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

e2e/convert_docker_tarball/README.md

@@ -0,0 +1,5 @@
+# Convert a docker tarball as a base image
+
+In some cases, your legacy setup doesn't fetch a base image from a remote registry, instead you've produced your base image in a script and check or fetch the tarball.
+
+To generate the `image.tar` file, first run `create_base_image.bash`. Then build the example normally.

e2e/convert_docker_tarball/WORKSPACE

@@ -0,0 +1,30 @@
+workspace(name = "custom_registry_example")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+local_repository(
+    name = "rules_oci",
+    path = "../../",
+)
+
+load("@rules_oci//oci:dependencies.bzl", "rules_oci_dependencies")
+
+rules_oci_dependencies()
+
+load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "oci_register_toolchains")
+
+oci_register_toolchains(
+    name = "oci",
+    crane_version = LATEST_CRANE_VERSION,
+)
+
+http_archive(
+    name = "container_structure_test",
+    sha256 = "2da13da4c4fec9d4627d4084b122be0f4d118bd02dfa52857ff118fde88e4faa",
+    strip_prefix = "container-structure-test-1.16.0",
+    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/v1.16.0.zip"],
+)
+
+load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain")
+
+container_structure_test_register_toolchain(name = "st")

e2e/convert_docker_tarball/convert.bash

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Most rules_oci users will use oci_pull to fetch base layers from a remote registry.
+# However, you might build docker-format tarballs and uploaded them to Artifactory for example.
+# rules_oci expects an OCI format base image, so these need to be converted.
+#
+# This just requires push'ing the tarball into a locally-running registry which
+# understands both formats, then pull'ing back out in the oci format.
+#
+# Note, this is suboptimal because Bazel will still have to execute an action that has the entire
+# base image as an input. Large inputs cause network delays with remote execution.
+
+set -o pipefail -o errexit -o nounset
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+RUNFILES="$SCRIPT_DIR/$(basename $0).runfiles"
+
+TMP=$(mktemp -d)
+export HOME="$TMP"
+
+readonly OUTPUT="${1}"
+readonly TARBALL="${2}"
+readonly CRANE="${RUNFILES}/${3#"external/"}"
+readonly REGISTRY_LAUNCHER=${RUNFILES}/${4#"external/"}
+
+
+# Launch a registry instance at a random port
+source "${REGISTRY_LAUNCHER}"
+REGISTRY=$(start_registry $TMP $TMP/output.log)
+
+readonly REPOSITORY="${REGISTRY}/local" 
+
+
+REF=$(mktemp)
+"${CRANE}" push "${TARBALL}" "${REPOSITORY}" --image-refs="${REF}"
+
+"${CRANE}" pull "$(cat $REF)" "${OUTPUT}" --format=oci

e2e/convert_docker_tarball/create_base_image.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -o errexit -o nounset
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+# Export image
+docker build . -t temp --no-cache
+docker save temp -o image.tar
+

e2e/convert_docker_tarball/test.yaml

@@ -0,0 +1,23 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+  envVars:
+    - key: "LC_ALL"
+      value: "C.UTF-8"
+    - key: "LANG"
+      value: "C.UTF-8"
+    - key: "LANGUAGE"
+      value: "C.UTF-8"
+  entrypoint: []
+  cmd: ["/bin/bash"]
+
+commandTests:
+  - name: "jq should be installed"
+    command: "jq"
+    expectedError: ["jq - commandline JSON processor"]
+    exitCode: 2
+
+fileExistenceTests:
+  - name: "should not remove /var/lib/apt/lists"
+    path: /var/lib/apt/lists
+    shouldExist: true

e2e/crane_as_registry/WORKSPACE.bazel

@@ -32,11 +32,9 @@ rules_pkg_dependencies()
 
 http_archive(
     name = "container_structure_test",
-    sha256 = "6913d8b5fa701b07d30ca236d2ab9e456ea0da415266cd8910a4537fb9222445",
-    strip_prefix = "container-structure-test-e5848342114f9a435e9a54f45bef99a7703c9173",
-    # Note, this commit not on main, it comes from
-    # https://github.com/GoogleContainerTools/container-structure-test/pull/347
-    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/e5848342114f9a435e9a54f45bef99a7703c9173.zip"],
+    sha256 = "2da13da4c4fec9d4627d4084b122be0f4d118bd02dfa52857ff118fde88e4faa",
+    strip_prefix = "container-structure-test-1.16.0",
+    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/v1.16.0.zip"],
 )
 
 load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain")

e2e/custom_registry/WORKSPACE

@@ -39,11 +39,9 @@ oci_pull(
 
 http_archive(
     name = "container_structure_test",
-    sha256 = "6913d8b5fa701b07d30ca236d2ab9e456ea0da415266cd8910a4537fb9222445",
-    strip_prefix = "container-structure-test-e5848342114f9a435e9a54f45bef99a7703c9173",
-    # Note, this commit not on main, it comes from
-    # https://github.com/GoogleContainerTools/container-structure-test/pull/347
-    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/e5848342114f9a435e9a54f45bef99a7703c9173.zip"],
+    sha256 = "2da13da4c4fec9d4627d4084b122be0f4d118bd02dfa52857ff118fde88e4faa",
+    strip_prefix = "container-structure-test-1.16.0",
+    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/v1.16.0.zip"],
 )
 
 load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain")

e2e/smoke/WORKSPACE.bazel

@@ -9,11 +9,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "container_structure_test",
-    sha256 = "6913d8b5fa701b07d30ca236d2ab9e456ea0da415266cd8910a4537fb9222445",
-    strip_prefix = "container-structure-test-e5848342114f9a435e9a54f45bef99a7703c9173",
-    # Note, this commit not on main, it comes from
-    # https://github.com/GoogleContainerTools/container-structure-test/pull/347
-    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/e5848342114f9a435e9a54f45bef99a7703c9173.zip"],
+    sha256 = "2da13da4c4fec9d4627d4084b122be0f4d118bd02dfa52857ff118fde88e4faa",
+    strip_prefix = "container-structure-test-1.16.0",
+    urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/v1.16.0.zip"],
 )
 
 load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain")

internal_deps.bzl

@@ -61,11 +61,9 @@ def rules_oci_internal_deps():
 
     http_archive(
         name = "container_structure_test",
-        sha256 = "6913d8b5fa701b07d30ca236d2ab9e456ea0da415266cd8910a4537fb9222445",
-        strip_prefix = "container-structure-test-e5848342114f9a435e9a54f45bef99a7703c9173",
-        # Note, this commit not on main, it comes from
-        # https://github.com/GoogleContainerTools/container-structure-test/pull/347
-        urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/e5848342114f9a435e9a54f45bef99a7703c9173.zip"],
+        sha256 = "2da13da4c4fec9d4627d4084b122be0f4d118bd02dfa52857ff118fde88e4faa",
+        strip_prefix = "container-structure-test-1.16.0",
+        urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/v1.16.0.zip"],
     )
 
     http_archive(