Improve e2e testing with snapshots (#103)

* Improve e2e macro

* Fix tests, add helper macro

* Restore unfixed error

.gitignore

@@ -1,3 +1,4 @@
 **/target
 
-.vscode
+.vscode
+*.snap.new

Cargo.toml

@@ -25,6 +25,7 @@ unescaper = "0.1.4"
 tempdir = "0.3.7"
 indexmap = "2.2.5"
 gimli = { version = "0.28.1", default-features = false }
+insta = "1.36.1"
 
 [build-dependencies]
 clap = { version = "4.5.2", features = ["derive"] }

src/compilation/compiler.rs

@@ -164,7 +164,7 @@ impl Compiler {
     ///
     /// let mut compiler = Compiler::new().at("src");
     /// let m1 = compiler.compile("main").unwrap();
-    /// let m2 = compiler.compule("main").unwrap();
+    /// let m2 = compiler.compile("main").unwrap();
     /// assert_eq!(m1, m2);
     /// ```
     pub fn compile(&mut self, name: &str) -> miette::Result<Module> {

src/driver/tests/snapshots/ppl__driver__tests__multifile.run.snap

@@ -0,0 +1,5 @@
+---
+source: src/driver/tests/mod.rs
+expression: run_log
+---
+Hello World

src/e2e.rs

@@ -4,45 +4,71 @@ macro_rules! e2e {
     ($name: ident) => {
         #[test]
         fn $name() {
-            $crate::e2e::internal::e2e(file!(), stringify!($name))
+            use std::path::Path;
+
+            use insta::assert_snapshot;
+            use tempdir::TempDir;
+
+            // Compile-time check that file exists
+            include_bytes!(concat!(stringify!($name), "/", stringify!($name), ".ppl"));
+
+            let temp_dir = TempDir::new("ppl").unwrap();
+            let file = file!();
+            let name = stringify!($name);
+            let dir = Path::new(file).parent().unwrap().join(name);
+
+            let res = $crate::e2e::internal::compile(&temp_dir, name, &dir);
+            if let Err(err) = res {
+                assert_snapshot!(concat!(stringify!($name), ".error"), err);
+                return;
+            }
+
+            let run_log = $crate::e2e::internal::run(&temp_dir, name, &dir);
+            assert_snapshot!(concat!(stringify!($name), ".run"), run_log);
         }
     };
 }
 
+#[macro_export]
+macro_rules! e2es {
+    ($($name: ident),+) => {
+        $(
+            $crate::e2e!($name);
+        )+
+    };
+}
+
 #[cfg(test)]
 pub mod internal {
-    use std::fs;
+    use std::path::Path;
 
-    use crate::driver::commands::compile::OutputType;
+    use tempdir::TempDir;
 
-    pub fn e2e(file: &str, name: &str) {
-        use pretty_assertions::assert_str_eq;
-        use std::path::Path;
-        use tempdir::TempDir;
+    use crate::driver::commands::compile::OutputType;
 
-        let temp_dir = TempDir::new("ppl").unwrap();
+    pub fn compile(temp_dir: &TempDir, name: &str, dir: &Path) -> Result<(), String> {
         let temp_dir_path = temp_dir.path().to_str().unwrap();
 
         let ppl = concat!(env!("CARGO_MANIFEST_DIR"), "/target/debug/ppl");
-        let dir = Path::new(file).parent().unwrap().join(name);
         let file = format!("{name}.ppl");
         let output = std::process::Command::new(ppl)
             .args(&["compile", &file])
             .args(&["--output-dir", temp_dir_path])
-            .current_dir(&dir)
+            .current_dir(dir)
             .output()
             .expect("failed to run command");
 
         let stderr = String::from_utf8(output.stderr).expect("stderr is not utf8");
-        let expected_stderr = fs::read_to_string(format!("{dir}/stderr.log", dir = dir.display()))
-            .unwrap_or_default();
-        assert_str_eq!(stderr, expected_stderr, "compiler output should match");
-
-        if !expected_stderr.is_empty() {
-            return;
+        if !stderr.is_empty() {
+            return Err(stderr);
         }
 
+        Ok(())
+    }
+
+    pub fn run(temp_dir: &TempDir, name: &str, dir: &Path) -> String {
         let exe = temp_dir.path().join(OutputType::Executable.named(name));
+
         let output = std::process::Command::new(exe)
             .current_dir(&dir)
             .output()
@@ -53,10 +79,8 @@ pub mod internal {
 
         let run_log = format!("{stdout}{stderr}");
 
-        let expected_run_log =
-            fs::read_to_string(format!("{dir}/run.log", dir = dir.display())).unwrap_or_default();
-        assert_str_eq!(run_log, expected_run_log, "executable output should match");
-
         output.status.exit_ok().unwrap();
+
+        run_log
     }
 }

src/ir/tests/mod.rs

@@ -1,5 +1,7 @@
-use crate::e2e;
+use crate::e2es;
 
-e2e!(clone);
-e2e!(empty_constructor);
-e2e!(type_of);
+e2es! {
+    clone,
+    empty_constructor,
+    type_of
+}

src/ir/tests/snapshots/ppl__ir__tests__clone.run.snap

@@ -0,0 +1,8 @@
+---
+source: src/ir/tests/mod.rs
+expression: run_log
+---
+1
+1
+1
+2

src/ir/tests/snapshots/ppl__ir__tests__empty_constructor.run.snap

@@ -0,0 +1,5 @@
+---
+source: src/ir/tests/mod.rs
+expression: run_log
+---
+A

src/ir/tests/snapshots/ppl__ir__tests__type_of.run.snap

@@ -0,0 +1,5 @@
+---
+source: src/ir/tests/mod.rs
+expression: run_log
+---
+Integer

src/semantics/tests/mod.rs

@@ -1,30 +1,32 @@
-use crate::e2e;
+use crate::e2es;
 
-e2e!(array);
-e2e!(candidate_not_viable);
-e2e!(constraints);
-e2e!(constraints_in_constructor);
-e2e!(deref_member_ref);
-e2e!(destructor);
-e2e!(generics);
-e2e!(import_all);
-e2e!(integer);
-e2e!(integer_not_eq_rational);
-e2e!(memory);
-e2e!(missing_fields);
-e2e!(monomorphize);
-e2e!(monomorphize_predeclared);
-e2e!(multiple_errors);
-e2e!(multiple_initialization);
-e2e!(non_class_constructor);
-e2e!(plus_assign);
-e2e!(predeclare_function);
-e2e!(predeclare_vars);
-e2e!(rational);
-e2e!(reference_mut);
-e2e!(references);
-e2e!(specify_variable_ty);
-e2e!(string);
-e2e!(traits);
-e2e!(type_as_value);
-e2e!(wrong_initializer_type);
+e2es! {
+    array,
+    candidate_not_viable,
+    constraints,
+    constraints_in_constructor,
+    deref_member_ref,
+    destructor,
+    generics,
+    import_all,
+    integer,
+    integer_not_eq_rational,
+    memory,
+    missing_fields,
+    monomorphize,
+    monomorphize_predeclared,
+    multiple_errors,
+    multiple_initialization,
+    non_class_constructor,
+    plus_assign,
+    predeclare_function,
+    predeclare_vars,
+    rational,
+    reference_mut,
+    references,
+    specify_variable_ty,
+    string,
+    traits,
+    type_as_value,
+    wrong_initializer_type
+}

src/semantics/tests/snapshots/ppl__semantics__tests__array.run.snap

@@ -0,0 +1,5 @@
+---
+source: src/semantics/tests/mod.rs
+expression: run_log
+---
+1

src/semantics/tests/candidate_not_viable/stderr.log → src/semantics/tests/snapshots/ppl__semantics__tests__candidate_not_viable.error.snap

@@ -1,3 +1,7 @@
+---
+source: src/semantics/tests/mod.rs
+expression: err
+---
 Error: semantics::no_function
 
   × no operator `<:Rational> + <:Integer>`