Correctly handle should_panic doctest attribute

library/std/src/error.rs

@@ -123,7 +123,7 @@ use crate::fmt::{self, Write};
 /// the `Debug` output means `Report` is an ideal starting place for formatting errors returned
 /// from `main`.
 ///
-/// ```should_panic
+/// ```
 /// #![feature(error_reporter)]
 /// use std::error::Report;
 /// # use std::error::Error;
@@ -154,10 +154,14 @@ use crate::fmt::{self, Write};
 /// #     Err(SuperError { source: SuperErrorSideKick })
 /// # }
 ///
-/// fn main() -> Result<(), Report<SuperError>> {
+/// fn run() -> Result<(), Report<SuperError>> {
 ///     get_super_error()?;
 ///     Ok(())
 /// }
+///
+/// fn main() {
+///     assert!(run().is_err());
+/// }
 /// ```
 ///
 /// This example produces the following output:
@@ -170,7 +174,7 @@ use crate::fmt::{self, Write};
 /// output format. If you want to make sure your `Report`s are pretty printed and include backtrace
 /// you will need to manually convert and enable those flags.
 ///
-/// ```should_panic
+/// ```
 /// #![feature(error_reporter)]
 /// use std::error::Report;
 /// # use std::error::Error;
@@ -201,12 +205,16 @@ use crate::fmt::{self, Write};
 /// #     Err(SuperError { source: SuperErrorSideKick })
 /// # }
 ///
-/// fn main() -> Result<(), Report<SuperError>> {
+/// fn run() -> Result<(), Report<SuperError>> {
 ///     get_super_error()
 ///         .map_err(Report::from)
 ///         .map_err(|r| r.pretty(true).show_backtrace(true))?;
 ///     Ok(())
 /// }
+///
+/// fn main() {
+///     assert!(run().is_err());
+/// }
 /// ```
 ///
 /// This example produces the following output:

src/librustdoc/doctest.rs

@@ -773,7 +773,7 @@ fn run_test(
     match result {
         Err(e) => return Err(TestFailure::ExecutionError(e)),
         Ok(out) => {
-            if langstr.should_panic && out.status.success() {
+            if langstr.should_panic && out.status.code() != Some(101) {
                 return Err(TestFailure::UnexpectedRunPass);
             } else if !langstr.should_panic && !out.status.success() {
                 return Err(TestFailure::ExecutionFailure(out));
@@ -1083,7 +1083,7 @@ fn doctest_run_fn(
                 eprint!("Test compiled successfully, but it's marked `compile_fail`.");
             }
             TestFailure::UnexpectedRunPass => {
-                eprint!("Test executable succeeded, but it's marked `should_panic`.");
+                eprint!("Test didn't panic, but it's marked `should_panic`.");
             }
             TestFailure::MissingErrorCodes(codes) => {
                 eprint!("Some expected error codes were not found: {codes:?}");

src/librustdoc/doctest/runner.rs

@@ -129,13 +129,20 @@ mod __doctest_mod {{
     }}
 
     #[allow(unused)]
-    pub fn doctest_runner(bin: &std::path::Path, test_nb: usize) -> ExitCode {{
+    pub fn doctest_runner(bin: &std::path::Path, test_nb: usize, should_panic: bool) -> ExitCode {{
         let out = std::process::Command::new(bin)
             .env(self::RUN_OPTION, test_nb.to_string())
             .args(std::env::args().skip(1).collect::<Vec<_>>())
             .output()
             .expect(\"failed to run command\");
-        if !out.status.success() {{
+        if should_panic {{
+            if out.status.code() != Some(101) {{
+                eprintln!(\"Test didn't panic, but it's marked `should_panic`.\");
+                ExitCode::FAILURE
+            }} else {{
+                ExitCode::SUCCESS
+            }}
+        }} else if !out.status.success() {{
             if let Some(code) = out.status.code() {{
                 eprintln!(\"Test executable failed (exit status: {{code}}).\");
             }} else {{
@@ -257,7 +264,7 @@ fn main() {returns_result} {{
         "
 mod {test_id} {{
 pub const TEST: test::TestDescAndFn = test::TestDescAndFn::new_doctest(
-{test_name:?}, {ignore}, {file:?}, {line}, {no_run}, {should_panic},
+{test_name:?}, {ignore}, {file:?}, {line}, {no_run}, false,
 test::StaticTestFn(
     || {{{runner}}},
 ));
@@ -266,7 +273,6 @@ test::StaticTestFn(
         file = scraped_test.path(),
         line = scraped_test.line,
         no_run = scraped_test.langstr.no_run,
-        should_panic = !scraped_test.langstr.no_run && scraped_test.langstr.should_panic,
         // Setting `no_run` to `true` in `TestDesc` still makes the test run, so we simply
         // don't give it the function to run.
         runner = if not_running {
@@ -275,11 +281,12 @@ test::StaticTestFn(
             format!(
                 "
 if let Some(bin_path) = crate::__doctest_mod::doctest_path() {{
-    test::assert_test_result(crate::__doctest_mod::doctest_runner(bin_path, {id}))
+    test::assert_test_result(crate::__doctest_mod::doctest_runner(bin_path, {id}, {should_panic}))
 }} else {{
     test::assert_test_result(doctest_bundle::{test_id}::__main_fn())
 }}
 ",
+                should_panic = scraped_test.langstr.should_panic,
             )
         },
     )

tests/run-make/rustdoc-should-panic/rmake.rs

@@ -0,0 +1,36 @@
+// Ensure that `should_panic` doctests only succeed if the test actually panicked.
+// Regression test for <https://github.com/rust-lang/rust/issues/143009>.
+
+//@ needs-target-std
+
+use run_make_support::rustdoc;
+
+fn check_output(output: String, edition: &str) {
+    let should_contain = &[
+        "test test.rs - bad_exit_code (line 1) ... FAILED",
+        "test test.rs - did_not_panic (line 6) ... FAILED",
+        "test test.rs - did_panic (line 11) ... ok",
+        "---- test.rs - bad_exit_code (line 1) stdout ----
+Test executable failed (exit status: 1).",
+        "---- test.rs - did_not_panic (line 6) stdout ----
+Test didn't panic, but it's marked `should_panic`.",
+        "test result: FAILED. 1 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out;",
+    ];
+    for text in should_contain {
+        assert!(
+            output.contains(text),
+            "output doesn't contains (edition: {edition}) {:?}\nfull output: {output}",
+            text
+        );
+    }
+}
+
+fn main() {
+    check_output(rustdoc().input("test.rs").arg("--test").run_fail().stdout_utf8(), "2015");
+
+    // Same check with the merged doctest feature (enabled with the 2024 edition).
+    check_output(
+        rustdoc().input("test.rs").arg("--test").edition("2024").run_fail().stdout_utf8(),
+        "2024",
+    );
+}

tests/run-make/rustdoc-should-panic/test.rs

@@ -0,0 +1,14 @@
+/// ```
+/// std::process::exit(1);
+/// ```
+fn bad_exit_code() {}
+
+/// ```should_panic
+/// std::process::exit(1);
+/// ```
+fn did_not_panic() {}
+
+/// ```should_panic
+/// panic!("yeay");
+/// ```
+fn did_panic() {}

tests/rustdoc-ui/doctest/failed-doctest-should-panic-2021.stdout

@@ -5,7 +5,7 @@ test $DIR/failed-doctest-should-panic-2021.rs - Foo (line 10) ... FAILED
 failures:
 
 ---- $DIR/failed-doctest-should-panic-2021.rs - Foo (line 10) stdout ----
-Test executable succeeded, but it's marked `should_panic`.
+Test didn't panic, but it's marked `should_panic`.
 
 failures:
     $DIR/failed-doctest-should-panic-2021.rs - Foo (line 10)

tests/rustdoc-ui/doctest/failed-doctest-should-panic.stdout

@@ -1,11 +1,12 @@
 
 running 1 test
-test $DIR/failed-doctest-should-panic.rs - Foo (line 10) - should panic ... FAILED
+test $DIR/failed-doctest-should-panic.rs - Foo (line 10) ... FAILED
 
 failures:
 
 ---- $DIR/failed-doctest-should-panic.rs - Foo (line 10) stdout ----
-note: test did not panic as expected at $DIR/failed-doctest-should-panic.rs:10:0
+Test didn't panic, but it's marked `should_panic`.
+
 
 failures:
     $DIR/failed-doctest-should-panic.rs - Foo (line 10)

tests/rustdoc-ui/doctest/wrong-ast-2024.stdout

@@ -1,6 +1,6 @@
 
 running 1 test
-test $DIR/wrong-ast-2024.rs - three (line 18) - should panic ... ok
+test $DIR/wrong-ast-2024.rs - three (line 18) ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME