fix(ci): Re-add custom builds workflow. Fixes integration test build …

…error. (vectordotdev#17024)

.github/actions/spelling/patterns.txt

@@ -215,3 +215,6 @@ k8s
 
 # ignore long runs of a single character:
 \b([A-Za-z])\g{-1}{3,}\b
+
+# ignore comment in package-msi.sh
+[#].*custom\.a28ecdc.*

.github/workflows/custom_builds.yml

@@ -0,0 +1,12 @@
+name: Custom Builds
+
+on:
+  workflow_dispatch: {}
+
+jobs:
+  Custom:
+    uses: ./.github/workflows/publish.yml
+    with:
+      git_ref: ${{ github.ref }}
+      channel: custom
+    secrets: inherit

.github/workflows/k8s_e2e.yml

@@ -67,7 +67,7 @@ jobs:
       - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
       - run: bash scripts/environment/prepare.sh
       - run: echo "::add-matcher::.github/matchers/rust.json"
-      - run: make package-deb-x86_64-unknown-linux-gnu
+      - run: VECTOR_VERSION="$(cargo vdev version)" make package-deb-x86_64-unknown-linux-gnu
       - uses: actions/upload-artifact@v3
         with:
           name: e2e-test-deb-package

.github/workflows/nightly.yml

@@ -10,5 +10,5 @@ jobs:
     uses: ./.github/workflows/publish.yml
     with:
       git_ref: ${{ github.ref }}
-      mode: Nightly
+      channel: nightly
     secrets: inherit

.github/workflows/publish.yml

@@ -6,7 +6,9 @@ on:
       git_ref:
         type: string
         required: true
-      mode:
+        # channel is the dir/namespace packages are organized into.
+        # Options are release/nightly/custom.
+      channel:
         type: string
         required: true
 
@@ -21,6 +23,7 @@ env:
   # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps
   # can be removed when we switch back to the upstream openssl-sys crate
   CARGO_NET_GIT_FETCH_WITH_CLI: true
+  CHANNEL: ${{ inputs.channel }}
 
 jobs:
   generate-publish-metadata:
@@ -545,7 +548,7 @@ jobs:
   publish-github:
     name: Publish to GitHub
     # We only publish to GitHub for versioned releases, not nightlies.
-    if: ${{ inputs.mode == 'Release' }}
+    if: ${{ inputs.channel == 'release' }}
     runs-on: ubuntu-20.04
     needs:
       - generate-publish-metadata
@@ -615,7 +618,7 @@ jobs:
   publish-homebrew:
     name: Publish to Homebrew
     # We only publish to Homebrew for versioned releases, not nightlies.
-    if: ${{ inputs.mode == 'Release' }}
+    if: ${{ inputs.channel == 'release' }}
     runs-on: ubuntu-20.04
     needs:
       - generate-publish-metadata
@@ -635,7 +638,7 @@ jobs:
   publish-cloudsmith:
     name: Publish to Cloudsmith
     # We only publish to CloudSmith for versioned releases, not nightlies.
-    if: ${{ inputs.mode == 'Release' }}
+    if: ${{ inputs.channel == 'release' }}
     runs-on: ubuntu-20.04
     needs:
       - generate-publish-metadata
@@ -748,7 +751,7 @@ jobs:
 
   publish-failure:
     name: Send Publish Failure Notification
-    if: failure()
+    if: ${{ inputs.channel != 'custom' }} && failure()
     runs-on: ubuntu-20.04
     needs:
       - generate-publish-metadata
@@ -773,4 +776,4 @@ jobs:
         DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
       uses: Ilshidur/[email protected]
       with:
-        args: "${{ inputs.mode }} failed: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}>"
+        args: "${{ inputs.channel }} failed: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}>"

.github/workflows/release.yml

@@ -11,5 +11,5 @@ jobs:
     uses: ./.github/workflows/publish.yml
     with:
       git_ref: ${{ github.ref }}
-      mode: Release
+      channel: release
     secrets: inherit

Makefile

@@ -527,37 +527,37 @@ package-armv7-unknown-linux-musleabihf: target/artifacts/vector-${VERSION}-armv7
 
 .PHONY: package-deb-x86_64-unknown-linux-gnu
 package-deb-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 GNU deb package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-gnu $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-gnu -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
 
 .PHONY: package-deb-x86_64-unknown-linux-musl
 package-deb-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 GNU deb package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-musl $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-musl -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
 
 .PHONY: package-deb-aarch64
 package-deb-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 deb package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=aarch64-unknown-linux-gnu $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=aarch64-unknown-linux-gnu -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
 
 .PHONY: package-deb-armv7-gnu
 package-deb-armv7-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7-unknown-linux-gnueabihf deb package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=armv7-unknown-linux-gnueabihf $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=armv7-unknown-linux-gnueabihf -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package deb
 
 # rpms
 
 .PHONY: package-rpm-x86_64-unknown-linux-gnu
 package-rpm-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 rpm package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-gnu $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-gnu -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
 
 .PHONY: package-rpm-x86_64-unknown-linux-musl
 package-rpm-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 musl rpm package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-musl $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=x86_64-unknown-linux-musl -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
 
 .PHONY: package-rpm-aarch64
 package-rpm-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 rpm package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=aarch64-unknown-linux-gnu $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=aarch64-unknown-linux-gnu -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
 
 .PHONY: package-rpm-armv7-gnu
 package-rpm-armv7-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7-unknown-linux-gnueabihf rpm package
-	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=armv7-unknown-linux-gnueabihf $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
+	$(CONTAINER_TOOL) run -v  $(PWD):/git/vectordotdev/vector/ -e TARGET=armv7-unknown-linux-gnueabihf -e VECTOR_VERSION $(ENVIRONMENT_UPSTREAM) cargo vdev package rpm
 
 ##@ Releasing
 

build.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, env, fs::File, io::Write, path::Path};
+use std::{collections::HashSet, env, fs::File, io::Write, path::Path, process::Command};
 
 struct TrackedEnv {
     tracked: HashSet<String>,
@@ -93,10 +93,27 @@ impl BuildConstants {
     }
 }
 
+fn git_short_hash() -> std::io::Result<String> {
+    let output_result = Command::new("git")
+        .args(["rev-parse", "--short", "HEAD"])
+        .output();
+
+    output_result.map(|output| {
+        let mut hash = String::from_utf8(output.stdout).expect("valid UTF-8");
+        hash.retain(|c| !c.is_ascii_whitespace());
+
+        hash
+    })
+}
+
 fn main() {
     // Always rerun if the build script itself changes.
     println!("cargo:rerun-if-changed=build.rs");
 
+    // re-run if the HEAD has changed. This is only necessary for non-release and nightly builds.
+    #[cfg(not(feature = "nightly"))]
+    println!("cargo:rerun-if-changed=.git/HEAD");
+
     #[cfg(feature = "protobuf-build")]
     {
         println!("cargo:rerun-if-changed=proto/dd_trace.proto");
@@ -162,6 +179,24 @@ fn main() {
         .expect("Cargo-provided environment variables should always exist!");
     let build_desc = tracker.get_env_var("VECTOR_BUILD_DESC");
 
+    // Get the git short hash of the HEAD.
+    // Note that if Vector is compiled within a container, proper git permissions must be set for
+    // the repo directory.
+    // In CI build workflows this will have been pre-configured by running the command
+    // "git config --global --add safe.directory /git/vectordotdev/vector", from the vdev package
+    // subcommands.
+    let git_short_hash = git_short_hash()
+        .map_err(|e| {
+            #[allow(clippy::print_stderr)]
+            {
+                eprintln!(
+                    "Unable to determine git short hash from rev-parse command: {}",
+                    e
+                );
+            }
+        })
+        .expect("git hash detection failed");
+
     // Gather up the constants and write them out to our build constants file.
     let mut constants = BuildConstants::new();
     constants.add_required_constant(
@@ -206,6 +241,11 @@ fn main() {
         "Special build description, related to versioned releases.",
         build_desc,
     );
+    constants.add_required_constant(
+        "GIT_SHORT_HASH",
+        "The short hash of the Git HEAD",
+        git_short_hash,
+    );
     constants
         .write_to_file("built.rs")
         .expect("Failed to write build-time constants file!");

distribution/msi/build.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 ARCHIVE_VERSION=$1
+PACKAGE_VERSION=$2
 
 echo "Copying ZIP archive..."
 
@@ -14,7 +15,7 @@ sed 's/$/\\/' < vector-"${ARCHIVE_VERSION}"-x86_64-pc-windows-msvc/LICENSE.txt >
 echo -e '\n}' >> LICENSE.rtf
 
 echo "Substituting version..."
-VERSION="${ARCHIVE_VERSION}" envsubst < vector.wxs.tmpl > vector.wxs
+VERSION="${PACKAGE_VERSION}" envsubst < vector.wxs.tmpl > vector.wxs
 
 echo "Building the MSI package..."
 heat dir vector-"${ARCHIVE_VERSION}"-x86_64-pc-windows-msvc \

scripts/build-docker.sh

@@ -78,6 +78,11 @@ elif [[ "$CHANNEL" == "nightly" ]]; then
     build distroless-static "$VERSION_TAG"
     build distroless-libc "$VERSION_TAG"
   done
+elif [[ "$CHANNEL" == "custom" ]]; then
+  build alpine "$VERSION"
+  build debian "$VERSION"
+  build distroless-static "$VERSION"
+  build distroless-libc "$VERSION"
 elif [[ "$CHANNEL" == "test" ]]; then
   build "${BASE:-"alpine"}" "${TAG:-"test"}"
   build "${BASE:-"distroless-libc"}" "${TAG:-"test"}"

scripts/integration/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     pkg-config \
     zlib1g-dev \
     unzip \
+    git \
   && rm -rf /var/lib/apt/lists/*
 
 RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.25 --locked

scripts/package-msi.sh

@@ -17,6 +17,21 @@ cp target/artifacts/vector-"${ARCHIVE_VERSION}"-x86_64-pc-windows-msvc.zip targe
 pushd target/msi-x64
 # shellcheck disable=SC2016
 powershell '$progressPreference = "silentlyContinue"; Expand-Archive vector-'"$ARCHIVE_VERSION"'-x86_64-pc-windows-msvc.zip'
-./build.sh "${ARCHIVE_VERSION}"
+
+# Building the MSI package requires the version to be purely numerical (eg 0.0.0),
+# which is not the case if with custom build workflow.
+# This specifically works around the following issue:
+#     C:\a\vector\vector\target\msi-x64\vector.wxs(6) : error CNDL0108 : The Product/@Version attribute's value, '0.29.0.custom.a28ecdc', is not a valid version.
+#     Legal version values should look like 'x.x.x.x' where x is an integer from 0 to 65534.
+# , by  changing "0.29.0.custom.a28ecdc" -> "0.29.0".
+CHANNEL="${CHANNEL:-"$(cargo vdev release channel)"}"
+
+if [[ "$CHANNEL" == "custom" ]]; then
+    PACKAGE_VERSION="${ARCHIVE_VERSION%.custom*}"
+else
+    PACKAGE_VERSION="${ARCHIVE_VERSION}"
+fi
+
+./build.sh "${ARCHIVE_VERSION}" "${PACKAGE_VERSION}"
 popd
 cp target/msi-x64/vector.msi target/artifacts/vector-"${ARCHIVE_VERSION}"-x64.msi

scripts/package-rpm.sh

@@ -17,7 +17,7 @@ TARGET="${TARGET:?"You must specify a target triple, ex: x86_64-apple-darwin"}"
 # Local vars
 #
 
-PACKAGE_VERSION="$(cargo vdev version)"
+PACKAGE_VERSION="${VECTOR_VERSION:-"$(cargo vdev version)"}"
 ARCHIVE_NAME="vector-$PACKAGE_VERSION-$TARGET.tar.gz"
 ARCHIVE_PATH="target/artifacts/$ARCHIVE_NAME"
 

scripts/release-s3.sh

@@ -21,7 +21,6 @@ export AWS_REGION=us-east-1
 
 td="$(mktemp -d)"
 cp -av "target/artifacts/." "$td"
-ls "$td"
 
 td_nightly="$(mktemp -d)"
 cp -av "target/artifacts/." "$td_nightly"
@@ -79,6 +78,7 @@ if [[ "$CHANNEL" == "nightly" ]]; then
   verify_artifact \
     "https://packages.timber.io/vector/nightly/latest/vector-nightly-x86_64-unknown-linux-gnu.tar.gz" \
     "$td_nightly/vector-nightly-x86_64-unknown-linux-gnu.tar.gz"
+
 elif [[ "$CHANNEL" == "latest" ]]; then
   VERSION_EXACT="$VERSION"
   # shellcheck disable=SC2001
@@ -128,6 +128,21 @@ elif [[ "$CHANNEL" == "latest" ]]; then
   verify_artifact \
     "https://packages.timber.io/vector/latest/vector-latest-x86_64-unknown-linux-gnu.tar.gz" \
     "$td/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz"
+
+elif [[ "$CHANNEL" == "custom" ]]; then
+
+  # Add custom files
+  echo "Uploading all artifacts to s3://packages.timber.io/vector/custom"
+  aws s3 cp "$td" "s3://packages.timber.io/vector/custom/$VERSION" --recursive --sse --acl public-read
+  echo "Uploaded archives"
+
+  # Verify that the files exist and can be downloaded
+  echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications"
+  sleep "$VERIFY_TIMEOUT"
+  verify_artifact \
+    "https://packages.timber.io/vector/custom/$VERSION/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" \
+    "$td/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz"
+
 fi
 
 #

src/lib.rs

@@ -130,7 +130,17 @@ pub fn vector_version() -> impl std::fmt::Display {
     let pkg_version = format!("{}-nightly", built_info::PKG_VERSION);
 
     #[cfg(not(feature = "nightly"))]
-    let pkg_version = built_info::PKG_VERSION;
+    let pkg_version = match built_info::DEBUG {
+        // If any debug info is included, consider it a non-release build.
+        "1" | "2" | "true" => {
+            format!(
+                "{}-custom-{}",
+                built_info::PKG_VERSION,
+                built_info::GIT_SHORT_HASH
+            )
+        }
+        _ => built_info::PKG_VERSION.to_string(),
+    };
 
     pkg_version
 }

vdev/src/app.rs

@@ -9,7 +9,7 @@ use indicatif::{ProgressBar, ProgressStyle};
 use log::LevelFilter;
 use once_cell::sync::{Lazy, OnceCell};
 
-use crate::{config::Config, platform, util};
+use crate::{config::Config, git, platform, util};
 
 // Use the `bash` interpreter included as part of the standard `git` install for our default shell
 // if nothing is specified in the environment.
@@ -46,8 +46,9 @@ pub fn set_repo_dir() -> Result<()> {
 }
 
 pub fn version() -> Result<String> {
-    let version = env::var("VERSION").or_else(|_| util::read_version())?;
-    let channel = env::var("CHANNEL").or_else(|_| util::release_channel().map(Into::into))?;
+    let mut version = util::get_version()?;
+
+    let channel = util::get_channel();
 
     if channel == "latest" {
         let head = util::git_head()?;
@@ -59,6 +60,14 @@ pub fn version() -> Result<String> {
         if tag != format!("v{version}") {
             bail!("On latest release channel and tag {tag:?} is different from Cargo.toml {version:?}. Aborting");
         }
+
+    // extend version for custom builds if not already
+    } else if channel == "custom" && !version.contains("custom") {
+        let sha = git::get_git_sha()?;
+
+        // use '.' instead of '-' or '_' to avoid issues with rpm and deb package naming
+        // format requirements.
+        version = format!("{version}.custom.{sha}");
     }
 
     Ok(version)