diff --git a/docs/src/index.md b/docs/src/index.md index 3d13341..7b474a4 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -151,10 +151,10 @@ This can be used to allow for factorization of test sets, making it easier to ru test sets by running the associated functions instead. Note that in the case of functions, the test set will be given the name of the called function. In the event that a nested test set has no failures, as happened here, it will be hidden in the -summary, unless the `verbose=true` option is passed: +summary: ```jldoctest testfoo; filter = r"[0-9\.]+s" -julia> @testset verbose = true "Foo Tests" begin +julia> @testset "Foo Tests" begin @testset "Animals" begin @test foo("cat") == 9 @test foo("dog") == foo("cat") @@ -166,10 +166,6 @@ julia> @testset verbose = true "Foo Tests" begin end; Test Summary: | Pass Total Time Foo Tests | 8 8 0.0s - Animals | 2 2 0.0s - Arrays 1 | 2 2 0.0s - Arrays 2 | 2 2 0.0s - Arrays 3 | 2 2 0.0s ``` If we do have a test failure, only the details for the failed test sets will be shown: @@ -201,6 +197,66 @@ Foo Tests | 3 1 4 0.0s ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored, 0 broken. ``` +You can use the `verbose` option to show results even for test sets that pass. +`verbose=true` (or equivalently `verbose=1`) displays results only for the top level of +nested sets: + +```julia-repl; filter = r"[0-9\.]+s" +julia> @testset "Foo Tests" verbose = true begin + @testset "Animals" begin + @testset "Felines" begin + @test foo("cat") == 9 + end + @testset "Canines" begin + @test foo("dog") == 9 + end + end + @testset "Arrays" begin + @testset "zeros" begin + @test foo(zeros(2)) == 4 + end + @testset "fill" begin + @test foo(fill(1.0, 4)) == 16 + end + end + end; +Test Summary: | Pass Total Time +Foo Tests | 4 4 0.0s + Animals | 2 2 0.0s + Arrays | 2 2 0.0s +``` + +whereas `verbose=2` will apply to all nested test sets: + +```julia-repl; filter = r"[0-9\.]+s" +julia> @testset "Foo Tests" verbose = 2 begin + @testset "Animals" begin + @testset "Felines" begin + @test foo("cat") == 9 + end + @testset "Canines" begin + @test foo("dog") == 9 + end + end + @testset "Arrays" begin + @testset "zeros" begin + @test foo(zeros(2)) == 4 + end + @testset "fill" begin + @test foo(fill(1.0, 4)) == 16 + end + end + end; +Test Summary: | Pass Total Time +Foo Tests | 4 4 0.0s + Animals | 2 2 0.0s + Felines | 1 1 0.0s + Canines | 1 1 0.0s + Arrays | 2 2 0.0s + zeros | 1 1 0.0s + fill | 1 1 0.0s +``` + ## Testing Log Statements One can use the [`@test_logs`](@ref) macro to test log statements, or use a [`TestLogger`](@ref). diff --git a/src/Test.jl b/src/Test.jl index 2628382..062d847 100644 --- a/src/Test.jl +++ b/src/Test.jl @@ -1053,6 +1053,20 @@ end #----------------------------------------------------------------------- +""" + VerbosityLevel + +Different possible levels of verbosity for test output as reported by DefaultTestSet. + +Options are `only_failures`, `this_testset`, and `this_testset_and_children`. +""" +@enum VerbosityLevel only_failures=0 this_testset=1 this_testset_and_children=2 + +# The established user interface for setting verbosity level is `verbose = true/false`, +# hence the below conversion. Note that by virtue of being an Enum, we can also use the Ints +# 0, 1, 2 to construct a `VerbosityLevel`. +VerbosityLevel(x::Bool) = x ? this_testset : only_failures + """ DefaultTestSet @@ -1065,14 +1079,25 @@ mutable struct DefaultTestSet <: AbstractTestSet results::Vector{Any} n_passed::Int anynonpass::Bool - verbose::Bool + verbose::VerbosityLevel showtiming::Bool time_start::Float64 time_end::Union{Float64,Nothing} failfast::Bool file::Union{String,Nothing} end -function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing) +function DefaultTestSet(desc::AbstractString; verbose = nothing, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing) + if isnothing(verbose) + parent_ts = get_testset() + if parent_ts isa DefaultTestSet && (parent_ts.verbose == this_testset_and_children) + verbose = this_testset_and_children + else + verbose = only_failures + end + elseif !(verbose isa VerbosityLevel) + verbose = VerbosityLevel(verbose) + end + if isnothing(failfast) # pass failfast state into child testsets parent_ts = get_testset() @@ -1230,7 +1255,7 @@ function get_alignment(ts::DefaultTestSet, depth::Int) # The minimum width at this depth is ts_width = 2*depth + length(ts.description) # If not verbose and all passing, no need to look at children - !ts.verbose && !ts.anynonpass && return ts_width + (ts.verbose == only_failures) && !ts.anynonpass && return ts_width # Return the maximum of this width and the minimum width # for all children (if they exist) isempty(ts.results) && return ts_width @@ -1344,7 +1369,7 @@ function print_counts(ts::DefaultTestSet, depth, align, # Only print results at lower levels if we had failures or if the user # wants. - if (np + nb != subtotal) || (ts.verbose) + if (np + nb != subtotal) || (ts.verbose >= this_testset) for t in ts.results if isa(t, DefaultTestSet) print_counts(t, depth + 1, align, diff --git a/test/runtests.jl b/test/runtests.jl index 0388e21..facc406 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1200,72 +1200,149 @@ let ex = :(something_complex + [1, 2, 3]) end @testset "verbose option" begin - expected = r""" - Test Summary: | Pass Total Time - Parent | 9 9 \s*\d*.\ds - Child 1 | 3 3 \s*\d*.\ds - Child 1.1 (long name) | 1 1 \s*\d*.\ds - Child 1.2 | 1 1 \s*\d*.\ds - Child 1.3 | 1 1 \s*\d*.\ds - Child 2 | 3 3 \s*\d*.\ds - Child 3 | 3 3 \s*\d*.\ds - Child 3.1 | 1 1 \s*\d*.\ds - Child 3.2 | 1 1 \s*\d*.\ds - Child 3.3 | 1 1 \s*\d*.\ds - """ - - mktemp() do f, _ - write(f, + @testset "verbose = true" begin + expected = r""" + Test Summary: | Pass Total Time + Parent | 9 9 \s*\d*.\ds + Child 1 | 3 3 \s*\d*.\ds + Child 1.1 (long name) | 1 1 \s*\d*.\ds + Child 1.2 | 1 1 \s*\d*.\ds + Child 1.3 | 1 1 \s*\d*.\ds + Child 2 | 3 3 \s*\d*.\ds + Child 3 | 3 3 \s*\d*.\ds + Child 3.1 | 1 1 \s*\d*.\ds + Child 3.2 | 1 1 \s*\d*.\ds + Child 3.3 | 1 1 \s*\d*.\ds """ - using Test - @testset "Parent" verbose = true begin - @testset "Child 1" verbose = true begin - @testset "Child 1.1 (long name)" begin - @test 1 == 1 - end + mktemp() do f, _ + write(f, + """ + using Test - @testset "Child 1.2" begin - @test 1 == 1 - end + @testset "Parent" verbose = true begin + @testset "Child 1" verbose = 1 begin + @testset "Child 1.1 (long name)" begin + @test 1 == 1 + end - @testset "Child 1.3" begin - @test 1 == 1 - end - end + @testset "Child 1.2" begin + @test 1 == 1 + end - @testset "Child 2" begin - @testset "Child 2.1" begin - @test 1 == 1 + @testset "Child 1.3" begin + @test 1 == 1 + end end - @testset "Child 2.2" begin - @test 1 == 1 + @testset "Child 2" begin + @testset "Child 2.1" begin + @test 1 == 1 + end + + @testset "Child 2.2" begin + @test 1 == 1 + end + + @testset "Child 2.3" begin + @test 1 == 1 + end end - @testset "Child 2.3" begin - @test 1 == 1 + @testset "Child 3" verbose = true begin + @testset "Child 3.1" begin + @test 1 == 1 + end + + @testset "Child 3.2" begin + @test 1 == 1 + end + + @testset "Child 3.3" begin + @test 1 == 1 + end end end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end + + @testset "verbose = 2" begin + expected = r""" + Test Summary: \| Pass Total Time + Parent \| 9 9 \s*\d*\.\ds + Child 1 \| 3 3 \s*\d*\.\ds + Child 1\.1 \(long name\) \| 2 2 \s*\d*\.\ds + Child 1\.2 \| 1 1 \s*\d*\.\ds + Child 2 \| 3 3 \s*\d*\.\ds + Child 2\.1 \| 1 1 \s*\d*\.\ds + Child 2\.1\.1 \| 1 1 \s*\d*\.\ds + Child 2\.2 \| 1 1 \s*\d*\.\ds + Child 2\.3 \| 1 1 \s*\d*\.\ds + Child 3 \| 3 3 \s*\d*\.\ds + """ + + mktemp() do f, _ + write(f, + """ + using Test - @testset "Child 3" verbose = true begin - @testset "Child 3.1" begin - @test 1 == 1 + @testset "Parent" verbose = 2 begin + # Setting the verbosity to just 1 overrides the inherited, higher level of 2. + @testset "Child 1" verbose = 1 begin + @testset "Child 1.1 (long name)" begin + @testset "Child 1.1.1" begin + @test 1 == 1 + end + + @testset "Child 1.1.2" begin + @test 1 == 1 + end + end + + @testset "Child 1.2" begin + @test 1 == 1 + end end - @testset "Child 3.2" begin - @test 1 == 1 + @testset "Child 2" begin + @testset "Child 2.1" begin + @testset "Child 2.1.1" begin + @test 1 == 1 + end + end + + @testset "Child 2.2" begin + @test 1 == 1 + end + + @testset "Child 2.3" begin + @test 1 == 1 + end end - @testset "Child 3.3" begin - @test 1 == 1 + @testset "Child 3" verbose = false begin + @testset "Child 3.1" begin + @test 1 == 1 + end + + @testset "Child 3.2" begin + @test 1 == 1 + end + + @testset "Child 3.3" begin + @test 1 == 1 + end end end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) end - """) - cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` - result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) - @test occursin(expected, result) end end