diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index c1fe9e8e20c63..ddd17f25fa0f1 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -157,10 +157,10 @@ This can be used to allow for factorization of test sets, making it easier to ru test sets by running the associated functions instead. Note that in the case of functions, the test set will be given the name of the called function. In the event that a nested test set has no failures, as happened here, it will be hidden in the -summary, unless the `verbose=true` option is passed: +summary: ```jldoctest testfoo; filter = r"[0-9\.]+s" -julia> @testset verbose = true "Foo Tests" begin +julia> @testset "Foo Tests" begin @testset "Animals" begin @test foo("cat") == 9 @test foo("dog") == foo("cat") @@ -172,10 +172,6 @@ julia> @testset verbose = true "Foo Tests" begin end; Test Summary: | Pass Total Time Foo Tests | 8 8 0.0s - Animals | 2 2 0.0s - Arrays 1 | 2 2 0.0s - Arrays 2 | 2 2 0.0s - Arrays 3 | 2 2 0.0s ``` If we do have a test failure, only the details for the failed test sets will be shown: @@ -207,6 +203,66 @@ Foo Tests | 3 1 4 0.0s ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored, 0 broken. ``` +You can use the `verbose` option to show results even for test sets that pass. +`verbose=true` (or equivalently `verbose=1`) displays results only for the top level of +nested sets: + +```julia-repl; filter = r"[0-9\.]+s" +julia> @testset "Foo Tests" verbose = true begin + @testset "Animals" begin + @testset "Felines" begin + @test foo("cat") == 9 + end + @testset "Canines" begin + @test foo("dog") == 9 + end + end + @testset "Arrays" begin + @testset "zeros" begin + @test foo(zeros(2)) == 4 + end + @testset "fill" begin + @test foo(fill(1.0, 4)) == 16 + end + end + end; +Test Summary: | Pass Total Time +Foo Tests | 4 4 0.0s + Animals | 2 2 0.0s + Arrays | 2 2 0.0s +``` + +whereas `verbose=2` will apply to all nested test sets: + +```julia-repl; filter = r"[0-9\.]+s" +julia> @testset "Foo Tests" verbose = 2 begin + @testset "Animals" begin + @testset "Felines" begin + @test foo("cat") == 9 + end + @testset "Canines" begin + @test foo("dog") == 9 + end + end + @testset "Arrays" begin + @testset "zeros" begin + @test foo(zeros(2)) == 4 + end + @testset "fill" begin + @test foo(fill(1.0, 4)) == 16 + end + end + end; +Test Summary: | Pass Total Time +Foo Tests | 4 4 0.0s + Animals | 2 2 0.0s + Felines | 1 1 0.0s + Canines | 1 1 0.0s + Arrays | 2 2 0.0s + zeros | 1 1 0.0s + fill | 1 1 0.0s +``` + ## Testing Log Statements One can use the [`@test_logs`](@ref) macro to test log statements, or use a [`TestLogger`](@ref). diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 79fdb89399d42..c17095e0f806c 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1053,6 +1053,21 @@ end #----------------------------------------------------------------------- +""" + VerbosityLevel + +Different possible levels of verbosity for test output as reported by DefaultTestSet. + +Options are `only_failures`, `this_testset`, and `this_testset_and_children`. +""" +@enum VerbosityLevel only_failures=0 this_testset=1 this_testset_and_children=2 + +# The established user interface for setting verbosity level is `verbose = true/false`, +# hence the below conversion. Note that by virtue of being an Enum, we can also use the Ints +# 0, 1, 2 to construct a `VerbosityLevel`. +VerbosityLevel(x::Bool) = x ? this_testset : only_failures + + """ DefaultTestSet @@ -1065,7 +1080,7 @@ mutable struct DefaultTestSet <: AbstractTestSet results::Vector{Any} n_passed::Int anynonpass::Bool - verbose::Bool + verbose::VerbosityLevel showtiming::Bool time_start::Float64 time_end::Union{Float64,Nothing} @@ -1073,7 +1088,18 @@ mutable struct DefaultTestSet <: AbstractTestSet file::Union{String,Nothing} rng::Union{Nothing,AbstractRNG} end -function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing, rng = nothing) +function DefaultTestSet(desc::AbstractString; verbose = nothing, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing, rng = nothing) + if isnothing(verbose) + parent_ts = get_testset() + if parent_ts isa DefaultTestSet && (parent_ts.verbose == this_testset_and_children) + verbose = this_testset_and_children + else + verbose = only_failures + end + elseif !(verbose isa VerbosityLevel) + verbose = VerbosityLevel(verbose) + end + if isnothing(failfast) # pass failfast state into child testsets parent_ts = get_testset() @@ -1133,7 +1159,7 @@ Defaults to the empty tuple. """ function results end -print_verbose(ts::DefaultTestSet) = ts.verbose +print_verbose(ts::DefaultTestSet) = ts.verbose > only_failures results(ts::DefaultTestSet) = ts.results # When a DefaultTestSet finishes, it records itself to its parent @@ -1275,7 +1301,7 @@ function get_alignment(ts::DefaultTestSet, depth::Int) # The minimum width at this depth is ts_width = 2*depth + length(ts.description) # If not verbose and all passing, no need to look at children - !ts.verbose && !ts.anynonpass && return ts_width + (ts.verbose == only_failures) && !ts.anynonpass && return ts_width # Return the maximum of this width and the minimum width # for all children (if they exist) isempty(ts.results) && return ts_width diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 995d2c983437c..b8b2ef4daf960 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1252,72 +1252,149 @@ let ex = :(something_complex + [1, 2, 3]) end @testset "verbose option" begin - expected = r""" - Test Summary: | Pass Total Time - Parent | 9 9 \s*\d*.\ds - Child 1 | 3 3 \s*\d*.\ds - Child 1.1 (long name) | 1 1 \s*\d*.\ds - Child 1.2 | 1 1 \s*\d*.\ds - Child 1.3 | 1 1 \s*\d*.\ds - Child 2 | 3 3 \s*\d*.\ds - Child 3 | 3 3 \s*\d*.\ds - Child 3.1 | 1 1 \s*\d*.\ds - Child 3.2 | 1 1 \s*\d*.\ds - Child 3.3 | 1 1 \s*\d*.\ds - """ - - mktemp() do f, _ - write(f, + @testset "verbose = true" begin + expected = r""" + Test Summary: | Pass Total Time + Parent | 9 9 \s*\d*.\ds + Child 1 | 3 3 \s*\d*.\ds + Child 1.1 (long name) | 1 1 \s*\d*.\ds + Child 1.2 | 1 1 \s*\d*.\ds + Child 1.3 | 1 1 \s*\d*.\ds + Child 2 | 3 3 \s*\d*.\ds + Child 3 | 3 3 \s*\d*.\ds + Child 3.1 | 1 1 \s*\d*.\ds + Child 3.2 | 1 1 \s*\d*.\ds + Child 3.3 | 1 1 \s*\d*.\ds """ - using Test - @testset "Parent" verbose = true begin - @testset "Child 1" verbose = true begin - @testset "Child 1.1 (long name)" begin - @test 1 == 1 - end + mktemp() do f, _ + write(f, + """ + using Test - @testset "Child 1.2" begin - @test 1 == 1 - end + @testset "Parent" verbose = true begin + @testset "Child 1" verbose = true begin + @testset "Child 1.1 (long name)" begin + @test 1 == 1 + end - @testset "Child 1.3" begin - @test 1 == 1 - end - end + @testset "Child 1.2" begin + @test 1 == 1 + end - @testset "Child 2" begin - @testset "Child 2.1" begin - @test 1 == 1 + @testset "Child 1.3" begin + @test 1 == 1 + end end - @testset "Child 2.2" begin - @test 1 == 1 - end + @testset "Child 2" begin + @testset "Child 2.1" begin + @test 1 == 1 + end - @testset "Child 2.3" begin - @test 1 == 1 - end - end + @testset "Child 2.2" begin + @test 1 == 1 + end - @testset "Child 3" verbose = true begin - @testset "Child 3.1" begin - @test 1 == 1 + @testset "Child 2.3" begin + @test 1 == 1 + end end - @testset "Child 3.2" begin - @test 1 == 1 - end + @testset "Child 3" verbose = true begin + @testset "Child 3.1" begin + @test 1 == 1 + end + + @testset "Child 3.2" begin + @test 1 == 1 + end - @testset "Child 3.3" begin - @test 1 == 1 + @testset "Child 3.3" begin + @test 1 == 1 + end end end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end + + @testset "verbose = 2" begin + expected = r""" + Test Summary: \| Pass Total Time + Parent \| 9 9 \s*\d*\.\ds + Child 1 \| 3 3 \s*\d*\.\ds + Child 1\.1 \(long name\) \| 2 2 \s*\d*\.\ds + Child 1\.2 \| 1 1 \s*\d*\.\ds + Child 2 \| 3 3 \s*\d*\.\ds + Child 2\.1 \| 1 1 \s*\d*\.\ds + Child 2\.1\.1 \| 1 1 \s*\d*\.\ds + Child 2\.2 \| 1 1 \s*\d*\.\ds + Child 2\.3 \| 1 1 \s*\d*\.\ds + Child 3 \| 3 3 \s*\d*\.\ds + """ + + mktemp() do f, _ + write(f, + """ + using Test + + @testset "Parent" verbose = 2 begin + # Setting the verbosity to just 1 overrides the inherited, higher level of 2. + @testset "Child 1" verbose = 1 begin + @testset "Child 1.1 (long name)" begin + @testset "Child 1.1.1" begin + @test 1 == 1 + end + + @testset "Child 1.1.2" begin + @test 1 == 1 + end + end + + @testset "Child 1.2" begin + @test 1 == 1 + end + end + + @testset "Child 2" begin + @testset "Child 2.1" begin + @testset "Child 2.1.1" begin + @test 1 == 1 + end + end + + @testset "Child 2.2" begin + @test 1 == 1 + end + + @testset "Child 2.3" begin + @test 1 == 1 + end + end + + @testset "Child 3" verbose = false begin + @testset "Child 3.1" begin + @test 1 == 1 + end + + @testset "Child 3.2" begin + @test 1 == 1 + end + + @testset "Child 3.3" begin + @test 1 == 1 + end + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) end - """) - cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` - result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) - @test occursin(expected, result) end end