diff --git a/ofborg/src/nix.rs b/ofborg/src/nix.rs index 1180e37f..ed7800b2 100644 --- a/ofborg/src/nix.rs +++ b/ofborg/src/nix.rs @@ -4,12 +4,15 @@ use crate::ofborg::partition_result; use std::collections::HashMap; use std::env; +use std::error::Error; use std::ffi::OsStr; use std::fmt; use std::fs; +use std::io; use std::io::{BufRead, BufReader, Seek, SeekFrom}; use std::path::Path; use std::process::{Command, Stdio}; +use tracing::{debug, info}; use tempfile::tempfile; @@ -137,6 +140,47 @@ impl Nix { n } + pub fn safely_query_cache_for_attr( + &self, + nixpkgs: &Path, + file: File, + attr: String, + ) -> Result> { + let mut command = self.safe_command::<&OsStr>(&Operation::Instantiate, nixpkgs, &[], &[]); + self.set_attrs_command(&mut command, file, vec![attr]); + let output = command + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .output()?; + debug!("{}", String::from_utf8(output.stderr)?.trim()); + + let drv = String::from_utf8(output.stdout)?; + let output = Command::new("nix-store") + .args(&["-q", "--binding", "out"]) + .arg(drv.trim()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .output()?; + debug!("{}", String::from_utf8(output.stderr)?.trim()); + if !output.status.success() { + let err = io::Error::new(io::ErrorKind::Other, "Could not evaluate stdenv"); + return Err(Box::new(err)); + } + + let out = String::from_utf8(output.stdout)?; + info!("stdenv {}", out); + let output = Command::new("nix-store") + .args(&["--option", "store", "https://cache.nixos.org"]) + .args(&["-q", "--size"]) + .arg(out.trim()) + .stderr(Stdio::piped()) + .stdout(Stdio::null()) + .output()?; + debug!("{}", String::from_utf8(output.stderr)?.trim()); + + Ok(output.status.success()) + } + pub fn safely_partition_instantiable_attrs( &self, nixpkgs: &Path, diff --git a/ofborg/src/tasks/build.rs b/ofborg/src/tasks/build.rs index 597d9cc5..4225d37d 100644 --- a/ofborg/src/tasks/build.rs +++ b/ofborg/src/tasks/build.rs @@ -325,10 +325,7 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { return; } - info!( - "Got path: {:?}, determining which ones we can build ", - refpath - ); + info!("Determining which attributes we can build"); let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs( refpath.as_ref(), buildfile, @@ -341,6 +338,25 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { .map(|(attr, _)| attr) .collect(); + info!("Checking for stdenv rebuild"); + match self.nix.safely_query_cache_for_attr( + refpath.as_ref(), + buildfile, + String::from("stdenv"), + ) { + Ok(false) => { + info!( + "Skip build: '{}', Cannot build: '{}'", + can_build.join(", "), + cannot_build_attrs.join(", ") + ); + actions.build_not_attempted(job.attrs.clone()); + return; + } + Ok(true) => (), + Err(err) => error!("Failed to detect stdenv rebuild: {:?}", err), + } + info!( "Can build: '{}', Cannot build: '{}'", can_build.join(", "), @@ -397,7 +413,10 @@ mod tests { const SYSTEM: &str = "x86_64-darwin"; fn nix() -> nix::Nix { + let path = env::var("PATH").unwrap(); + let test_path = format!("{}/test-nix/bin:{}", env!("CARGO_MANIFEST_DIR"), path); let remote = env::var("NIX_REMOTE").unwrap_or("".to_owned()); + env::set_var("PATH", test_path); nix::Nix::new("x86_64-linux".to_owned(), remote, 1800, None) } @@ -433,6 +452,21 @@ mod tests { hash.trim().to_owned() } + fn make_stdenv_pr_repo(bare: &Path, co: &Path) -> String { + let output = Command::new("bash") + .current_dir(tpath("./test-srcs")) + .arg("make-stdenv-pr.sh") + .arg(bare) + .arg(co) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .output() + .expect("building the test PR failed"); + let hash = String::from_utf8(output.stdout).expect("Should just be a hash"); + + hash.trim().to_owned() + } + fn strip_escaped_ansi(string: &str) -> String { string .replace("‘", "'") @@ -513,6 +547,45 @@ mod tests { assert_eq!(actions.next(), Some(worker::Action::Ack)); } + #[test] + pub fn test_stdenv_rebuild() { + let p = TestScratch::new_dir("build-stdenv-build-working"); + let bare_repo = TestScratch::new_dir("build-stdenv-build-bare"); + let co_repo = TestScratch::new_dir("build-stdenv-build-co"); + + let head_sha = make_stdenv_pr_repo(&bare_repo.path(), &co_repo.path()); + let worker = make_worker(&p.path()); + + let job = buildjob::BuildJob { + attrs: vec!["success".to_owned()], + pr: Pr { + head_sha, + number: 1, + target_branch: Some("staging".to_owned()), + }, + repo: Repo { + clone_url: bare_repo.path().to_str().unwrap().to_owned(), + full_name: "test-git".to_owned(), + name: "nixos".to_owned(), + owner: "ofborg-test".to_owned(), + }, + subset: None, + logs: Some((Some(String::from("logs")), Some(String::from("build.log")))), + statusreport: Some((Some(String::from("build-results")), None)), + request_id: "bogus-request-id".to_owned(), + }; + + let mut dummyreceiver = notifyworker::DummyNotificationReceiver::new(); + + worker.consumer(&job, &mut dummyreceiver); + + println!("Total actions: {:?}", dummyreceiver.actions.len()); + let mut actions = dummyreceiver.actions.into_iter(); + assert_contains_job(&mut actions, "skipped_attrs\":[\"success"); // First one to the github poster + assert_contains_job(&mut actions, "skipped_attrs\":[\"success"); // This one to the logs + assert_eq!(actions.next(), Some(worker::Action::Ack)); + } + #[test] pub fn test_all_jobs_skipped() { let p = TestScratch::new_dir("no-attempt"); diff --git a/ofborg/test-srcs/build-pr/default.nix b/ofborg/test-srcs/build-pr/default.nix index 231e5840..260867e5 100644 --- a/ofborg/test-srcs/build-pr/default.nix +++ b/ofborg/test-srcs/build-pr/default.nix @@ -2,6 +2,11 @@ let builder = builtins.storePath ; in { + stdenv = import { + url = "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz"; + sha256 = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39"; + }; + success = derivation { name = "success"; system = builtins.currentSystem; diff --git a/ofborg/test-srcs/build/default.nix b/ofborg/test-srcs/build/default.nix index 6c57488f..033fbdd0 100644 --- a/ofborg/test-srcs/build/default.nix +++ b/ofborg/test-srcs/build/default.nix @@ -2,6 +2,11 @@ let builder = builtins.storePath ; in { + stdenv = import { + url = "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz"; + sha256 = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39"; + }; + success = derivation { name = "success"; system = builtins.currentSystem; diff --git a/ofborg/test-srcs/make-stdenv-pr.sh b/ofborg/test-srcs/make-stdenv-pr.sh new file mode 100755 index 00000000..d6c6149b --- /dev/null +++ b/ofborg/test-srcs/make-stdenv-pr.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -eu + +bare=$1 +co=$2 + +makepr() { + git init --bare "$bare" + git clone "$bare" "$co" + git -C "$co" commit --no-gpg-sign --author "GrahamCOfBorg " --allow-empty -m "empty master commit" + git -C "$co" push origin master + + cp build/* "$co/" + git -C "$co" add . + git -C "$co" checkout -B staging + git -C "$co" commit --no-gpg-sign --author "GrahamCOfBorg " -m "initial repo commit" + git -C "$co" push origin staging + + cp stdenv-pr/* "$co/" + git -C "$co" checkout -b my-cool-pr + git -C "$co" add . + git -C "$co" commit --no-gpg-sign --author "GrahamCOfBorg " -m "check out this cool PR" + git -C "$co" push origin my-cool-pr:refs/pull/1/head +} + +makepr >&2 +git -C "$co" rev-parse HEAD diff --git a/ofborg/test-srcs/stdenv-pr/default.nix b/ofborg/test-srcs/stdenv-pr/default.nix new file mode 100644 index 00000000..6192e19f --- /dev/null +++ b/ofborg/test-srcs/stdenv-pr/default.nix @@ -0,0 +1,16 @@ +let + builder = builtins.storePath ; +in +{ + stdenv = import { + url = "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz"; + sha256 = "0000000000000000000000000000000000000000000000000000000000000000"; + }; + + success = derivation { + name = "success"; + system = builtins.currentSystem; + inherit builder; + args = [ "-c" "echo hi; printf '1\n2\n3\n4\n'; echo ${toString builtins.currentTime} > $out" ]; + }; +}