Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option to make verbose = true apply recursively to nested test sets #57084

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Add option to make verbose = true apply recursively to nested test sets #57084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 62 additions & 6 deletions stdlib/Test/docs/src/index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,10 @@ This can be used to allow for factorization of test sets, making it easier to ru
test sets by running the associated functions instead.
Note that in the case of functions, the test set will be given the name of the called function.
In the event that a nested test set has no failures, as happened here, it will be hidden in the
summary, unless the `verbose=true` option is passed:
summary:

```jldoctest testfoo; filter = r"[0-9\.]+s"
julia> @testset verbose = true "Foo Tests" begin
julia> @testset "Foo Tests" begin
@testset "Animals" begin
@test foo("cat") == 9
@test foo("dog") == foo("cat")
Expand All @@ -172,10 +172,6 @@ julia> @testset verbose = true "Foo Tests" begin
end;
Test Summary: | Pass Total Time
Foo Tests | 8 8 0.0s
Animals | 2 2 0.0s
Arrays 1 | 2 2 0.0s
Arrays 2 | 2 2 0.0s
Arrays 3 | 2 2 0.0s
```

If we do have a test failure, only the details for the failed test sets will be shown:
Expand Down Expand Up @@ -207,6 +203,66 @@ Foo Tests | 3 1 4 0.0s
ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored, 0 broken.
```

You can use the `verbose` option to show results even for test sets that pass.
`verbose=true` (or equivalently `verbose=1`) displays results only for the top level of
nested sets:

```julia-repl; filter = r"[0-9\.]+s"
julia> @testset "Foo Tests" verbose = true begin
@testset "Animals" begin
@testset "Felines" begin
@test foo("cat") == 9
end
@testset "Canines" begin
@test foo("dog") == 9
end
end
@testset "Arrays" begin
@testset "zeros" begin
@test foo(zeros(2)) == 4
end
@testset "fill" begin
@test foo(fill(1.0, 4)) == 16
end
end
end;
Test Summary: | Pass Total Time
Foo Tests | 4 4 0.0s
Animals | 2 2 0.0s
Arrays | 2 2 0.0s
```

whereas `verbose=2` will apply to all nested test sets:

```julia-repl; filter = r"[0-9\.]+s"
julia> @testset "Foo Tests" verbose = 2 begin
@testset "Animals" begin
@testset "Felines" begin
@test foo("cat") == 9
end
@testset "Canines" begin
@test foo("dog") == 9
end
end
@testset "Arrays" begin
@testset "zeros" begin
@test foo(zeros(2)) == 4
end
@testset "fill" begin
@test foo(fill(1.0, 4)) == 16
end
end
end;
Test Summary: | Pass Total Time
Foo Tests | 4 4 0.0s
Animals | 2 2 0.0s
Felines | 1 1 0.0s
Canines | 1 1 0.0s
Arrays | 2 2 0.0s
zeros | 1 1 0.0s
fill | 1 1 0.0s
```

## Testing Log Statements

One can use the [`@test_logs`](@ref) macro to test log statements, or use a [`TestLogger`](@ref).
Expand Down
34 changes: 30 additions & 4 deletions stdlib/Test/src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1053,6 +1053,21 @@ end

#-----------------------------------------------------------------------

"""
VerbosityLevel

Different possible levels of verbosity for test output as reported by DefaultTestSet.

Options are `only_failures`, `this_testset`, and `this_testset_and_children`.
"""
@enum VerbosityLevel only_failures=0 this_testset=1 this_testset_and_children=2

# The established user interface for setting verbosity level is `verbose = true/false`,
# hence the below conversion. Note that by virtue of being an Enum, we can also use the Ints
# 0, 1, 2 to construct a `VerbosityLevel`.
VerbosityLevel(x::Bool) = x ? this_testset : only_failures


"""
DefaultTestSet

Expand All @@ -1065,15 +1080,26 @@ mutable struct DefaultTestSet <: AbstractTestSet
results::Vector{Any}
n_passed::Int
anynonpass::Bool
verbose::Bool
verbose::VerbosityLevel
showtiming::Bool
time_start::Float64
time_end::Union{Float64,Nothing}
failfast::Bool
file::Union{String,Nothing}
rng::Union{Nothing,AbstractRNG}
end
function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing, rng = nothing)
function DefaultTestSet(desc::AbstractString; verbose = nothing, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing, rng = nothing)
if isnothing(verbose)
parent_ts = get_testset()
if parent_ts isa DefaultTestSet && (parent_ts.verbose == this_testset_and_children)
verbose = this_testset_and_children
else
verbose = only_failures
end
elseif !(verbose isa VerbosityLevel)
verbose = VerbosityLevel(verbose)
end

if isnothing(failfast)
# pass failfast state into child testsets
parent_ts = get_testset()
Expand Down Expand Up @@ -1133,7 +1159,7 @@ Defaults to the empty tuple.
"""
function results end

print_verbose(ts::DefaultTestSet) = ts.verbose
print_verbose(ts::DefaultTestSet) = ts.verbose > only_failures
results(ts::DefaultTestSet) = ts.results

# When a DefaultTestSet finishes, it records itself to its parent
Expand Down Expand Up @@ -1275,7 +1301,7 @@ function get_alignment(ts::DefaultTestSet, depth::Int)
# The minimum width at this depth is
ts_width = 2*depth + length(ts.description)
# If not verbose and all passing, no need to look at children
!ts.verbose && !ts.anynonpass && return ts_width
(ts.verbose == only_failures) && !ts.anynonpass && return ts_width
# Return the maximum of this width and the minimum width
# for all children (if they exist)
isempty(ts.results) && return ts_width
Expand Down
179 changes: 128 additions & 51 deletions stdlib/Test/test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1252,72 +1252,149 @@ let ex = :(something_complex + [1, 2, 3])
end

@testset "verbose option" begin
expected = r"""
Test Summary: | Pass Total Time
Parent | 9 9 \s*\d*.\ds
Child 1 | 3 3 \s*\d*.\ds
Child 1.1 (long name) | 1 1 \s*\d*.\ds
Child 1.2 | 1 1 \s*\d*.\ds
Child 1.3 | 1 1 \s*\d*.\ds
Child 2 | 3 3 \s*\d*.\ds
Child 3 | 3 3 \s*\d*.\ds
Child 3.1 | 1 1 \s*\d*.\ds
Child 3.2 | 1 1 \s*\d*.\ds
Child 3.3 | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
@testset "verbose = true" begin
expected = r"""
Test Summary: | Pass Total Time
Parent | 9 9 \s*\d*.\ds
Child 1 | 3 3 \s*\d*.\ds
Child 1.1 (long name) | 1 1 \s*\d*.\ds
Child 1.2 | 1 1 \s*\d*.\ds
Child 1.3 | 1 1 \s*\d*.\ds
Child 2 | 3 3 \s*\d*.\ds
Child 3 | 3 3 \s*\d*.\ds
Child 3.1 | 1 1 \s*\d*.\ds
Child 3.2 | 1 1 \s*\d*.\ds
Child 3.3 | 1 1 \s*\d*.\ds
"""
using Test

@testset "Parent" verbose = true begin
@testset "Child 1" verbose = true begin
@testset "Child 1.1 (long name)" begin
@test 1 == 1
end
mktemp() do f, _
write(f,
"""
using Test

@testset "Child 1.2" begin
@test 1 == 1
end
@testset "Parent" verbose = true begin
@testset "Child 1" verbose = true begin
@testset "Child 1.1 (long name)" begin
@test 1 == 1
end

@testset "Child 1.3" begin
@test 1 == 1
end
end
@testset "Child 1.2" begin
@test 1 == 1
end

@testset "Child 2" begin
@testset "Child 2.1" begin
@test 1 == 1
@testset "Child 1.3" begin
@test 1 == 1
end
end

@testset "Child 2.2" begin
@test 1 == 1
end
@testset "Child 2" begin
@testset "Child 2.1" begin
@test 1 == 1
end

@testset "Child 2.3" begin
@test 1 == 1
end
end
@testset "Child 2.2" begin
@test 1 == 1
end

@testset "Child 3" verbose = true begin
@testset "Child 3.1" begin
@test 1 == 1
@testset "Child 2.3" begin
@test 1 == 1
end
end

@testset "Child 3.2" begin
@test 1 == 1
end
@testset "Child 3" verbose = true begin
@testset "Child 3.1" begin
@test 1 == 1
end

@testset "Child 3.2" begin
@test 1 == 1
end

@testset "Child 3.3" begin
@test 1 == 1
@testset "Child 3.3" begin
@test 1 == 1
end
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end

@testset "verbose = 2" begin
expected = r"""
Test Summary: \| Pass Total Time
Parent \| 9 9 \s*\d*\.\ds
Child 1 \| 3 3 \s*\d*\.\ds
Child 1\.1 \(long name\) \| 2 2 \s*\d*\.\ds
Child 1\.2 \| 1 1 \s*\d*\.\ds
Child 2 \| 3 3 \s*\d*\.\ds
Child 2\.1 \| 1 1 \s*\d*\.\ds
Child 2\.1\.1 \| 1 1 \s*\d*\.\ds
Child 2\.2 \| 1 1 \s*\d*\.\ds
Child 2\.3 \| 1 1 \s*\d*\.\ds
Child 3 \| 3 3 \s*\d*\.\ds
"""

mktemp() do f, _
write(f,
"""
using Test

@testset "Parent" verbose = 2 begin
# Setting the verbosity to just 1 overrides the inherited, higher level of 2.
@testset "Child 1" verbose = 1 begin
@testset "Child 1.1 (long name)" begin
@testset "Child 1.1.1" begin
@test 1 == 1
end

@testset "Child 1.1.2" begin
@test 1 == 1
end
end

@testset "Child 1.2" begin
@test 1 == 1
end
end

@testset "Child 2" begin
@testset "Child 2.1" begin
@testset "Child 2.1.1" begin
@test 1 == 1
end
end

@testset "Child 2.2" begin
@test 1 == 1
end

@testset "Child 2.3" begin
@test 1 == 1
end
end

@testset "Child 3" verbose = false begin
@testset "Child 3.1" begin
@test 1 == 1
end

@testset "Child 3.2" begin
@test 1 == 1
end

@testset "Child 3.3" begin
@test 1 == 1
end
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end

Expand Down
Loading