skip stdenv rebuilds

ofborg/src/nix.rs

@@ -4,12 +4,15 @@ use crate::ofborg::partition_result;
 
 use std::collections::HashMap;
 use std::env;
+use std::error::Error;
 use std::ffi::OsStr;
 use std::fmt;
 use std::fs;
+use std::io;
 use std::io::{BufRead, BufReader, Seek, SeekFrom};
 use std::path::Path;
 use std::process::{Command, Stdio};
+use tracing::{debug, info};
 
 use tempfile::tempfile;
 
@@ -137,6 +140,47 @@ impl Nix {
         n
     }
 
+    pub fn safely_query_cache_for_attr(
+        &self,
+        nixpkgs: &Path,
+        file: File,
+        attr: String,
+    ) -> Result<bool, Box<dyn Error>> {
+        let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]);
+        self.set_attrs_command(&mut command, file, vec![attr]);
+        let output = command
+            .stderr(Stdio::piped())
+            .stdout(Stdio::piped())
+            .output()?;
+        debug!("{}", String::from_utf8(output.stderr)?.trim());
+
+        let drv = String::from_utf8(output.stdout)?;
+        let output = Command::new("nix-store")
+            .args(&["-q", "--binding", "out"])
+            .arg(drv.trim())
+            .stderr(Stdio::piped())
+            .stdout(Stdio::piped())
+            .output()?;
+        debug!("{}", String::from_utf8(output.stderr)?.trim());
+        if !output.status.success() {
+            let err = io::Error::new(io::ErrorKind::Other, "Could not evaluate stdenv");
+            return Err(Box::new(err));
+        }
+
+        let out = String::from_utf8(output.stdout)?;
+        info!("stdenv {}", out);
+        let output = Command::new("nix-store")
+            .args(&["--option", "store", "https://cache.nixos.org"])
+            .args(&["-q", "--size"])
+            .arg(out.trim())
+            .stderr(Stdio::piped())
+            .stdout(Stdio::null())
+            .output()?;
+        debug!("{}", String::from_utf8(output.stderr)?.trim());
+
+        Ok(output.status.success())
+    }
+
     pub fn safely_partition_instantiable_attrs(
         &self,
         nixpkgs: &Path,

ofborg/src/tasks/build.rs

@@ -325,10 +325,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
             return;
         }
 
-        info!(
-            "Got path: {:?}, determining which ones we can build ",
-            refpath
-        );
+        info!("Determining which attributes we can build");
         let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs(
             refpath.as_ref(),
             buildfile,
@@ -341,6 +338,25 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
             .map(|(attr, _)| attr)
             .collect();
 
+        info!("Checking for stdenv rebuild");
+        match self.nix.safely_query_cache_for_attr(
+            refpath.as_ref(),
+            buildfile,
+            String::from("stdenv"),
+        ) {
+            Ok(false) => {
+                info!(
+                    "Skip build: '{}', Cannot build: '{}'",
+                    can_build.join(", "),
+                    cannot_build_attrs.join(", ")
+                );
+                actions.build_not_attempted(job.attrs.clone());
+                return;
+            }
+            Ok(true) => (),
+            Err(err) => error!("Failed to detect stdenv rebuild: {:?}", err),
+        }
+
         info!(
             "Can build: '{}', Cannot build: '{}'",
             can_build.join(", "),
@@ -397,7 +413,10 @@ mod tests {
     const SYSTEM: &str = "x86_64-darwin";
 
     fn nix() -> nix::Nix {
+        let path = env::var("PATH").unwrap();
+        let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path);
         let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned());
+        env::set_var("PATH", test_path);
         nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None)
     }
 
@@ -433,6 +452,21 @@ mod tests {
         hash.trim().to_owned()
     }
 
+    fn make_stdenv_pr_repo(bare: &Path, co: &Path) -> String {
+        let output = Command::new("bash")
+            .current_dir(tpath("./test-srcs"))
+            .arg("make-stdenv-pr.sh")
+            .arg(bare)
+            .arg(co)
+            .stderr(Stdio::null())
+            .stdout(Stdio::piped())
+            .output()
+            .expect("building the test PR failed");
+        let hash = String::from_utf8(output.stdout).expect("Should just be a hash");
+
+        hash.trim().to_owned()
+    }
+
     fn strip_escaped_ansi(string: &str) -> String {
         string
             .replace("‘", "'")
@@ -513,6 +547,45 @@ mod tests {
         assert_eq!(actions.next(), Some(worker::Action::Ack));
     }
 
+    #[test]
+    pub fn test_stdenv_rebuild() {
+        let p = TestScratch::new_dir("build-stdenv-build-working");
+        let bare_repo = TestScratch::new_dir("build-stdenv-build-bare");
+        let co_repo = TestScratch::new_dir("build-stdenv-build-co");
+
+        let head_sha = make_stdenv_pr_repo(&bare_repo.path(), &co_repo.path());
+        let worker = make_worker(&p.path());
+
+        let job = buildjob::BuildJob {
+            attrs: vec!["success".to_owned()],
+            pr: Pr {
+                head_sha,
+                number: 1,
+                target_branch: Some("staging".to_owned()),
+            },
+            repo: Repo {
+                clone_url: bare_repo.path().to_str().unwrap().to_owned(),
+                full_name: "test-git".to_owned(),
+                name: "nixos".to_owned(),
+                owner: "ofborg-test".to_owned(),
+            },
+            subset: None,
+            logs: Some((Some(String::from("logs")), Some(String::from("build.log")))),
+            statusreport: Some((Some(String::from("build-results")), None)),
+            request_id: "bogus-request-id".to_owned(),
+        };
+
+        let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new();
+
+        worker.consumer(&job, &mut dummyreceiver);
+
+        println!("Total actions: {:?}", dummyreceiver.actions.len());
+        let mut actions = dummyreceiver.actions.into_iter();
+        assert_contains_job(&mut actions, "skipped_attrs\":[\"success"); // First one to the github poster
+        assert_contains_job(&mut actions, "skipped_attrs\":[\"success"); // This one to the logs
+        assert_eq!(actions.next(), Some(worker::Action::Ack));
+    }
+
     #[test]
     pub fn test_all_jobs_skipped() {
         let p = TestScratch::new_dir("no-attempt");

ofborg/test-srcs/build-pr/default.nix

@@ -2,6 +2,11 @@ let
   builder = builtins.storePath <ofborg-test-bash>;
 in
 {
+  stdenv = import <nix/fetchurl.nix> {
+    url = "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz";
+    sha256 = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39";
+  };
+
   success = derivation {
     name = "success";
     system = builtins.currentSystem;

ofborg/test-srcs/build/default.nix

@@ -2,6 +2,11 @@ let
   builder = builtins.storePath <ofborg-test-bash>;
 in
 {
+  stdenv = import <nix/fetchurl.nix> {
+    url = "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz";
+    sha256 = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39";
+  };
+
   success = derivation {
     name = "success";
     system = builtins.currentSystem;

ofborg/test-srcs/make-stdenv-pr.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eu
+
+bare=$1
+co=$2
+
+makepr() {
+    git init --bare "$bare"
+    git clone "$bare" "$co"
+    git -C "$co" commit --no-gpg-sign --author "GrahamCOfBorg <[email protected]>" --allow-empty -m "empty master commit"
+    git -C "$co" push origin master
+
+    cp build/* "$co/"
+    git -C "$co" add .
+    git -C "$co" checkout -B staging
+    git -C "$co" commit --no-gpg-sign --author "GrahamCOfBorg <[email protected]>" -m "initial repo commit"
+    git -C "$co" push origin staging
+
+    cp stdenv-pr/*  "$co/"
+    git -C "$co" checkout -b my-cool-pr
+    git -C "$co" add .
+    git -C "$co" commit --no-gpg-sign --author "GrahamCOfBorg <[email protected]>" -m "check out this cool PR"
+    git -C "$co" push origin my-cool-pr:refs/pull/1/head
+}
+
+makepr >&2
+git -C "$co" rev-parse HEAD

ofborg/test-srcs/stdenv-pr/default.nix

@@ -0,0 +1,16 @@
+let
+  builder = builtins.storePath <ofborg-test-bash>;
+in
+{
+  stdenv = import <nix/fetchurl.nix> {
+    url = "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz";
+    sha256 = "0000000000000000000000000000000000000000000000000000000000000000";
+  };
+
+  success = derivation {
+    name = "success";
+    system = builtins.currentSystem;
+    inherit builder;
+    args = [ "-c" "echo hi; printf '1\n2\n3\n4\n'; echo ${toString builtins.currentTime} > $out" ];
+  };
+}