Skip to content
This repository was archived by the owner on Jan 18, 2025. It is now read-only.

Add option to make verbose = true apply recursively to nested test sets #3

Closed
wants to merge 1 commit into from
Closed

Add option to make verbose = true apply recursively to nested test sets #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 62 additions & 6 deletions docs/src/index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,10 @@ This can be used to allow for factorization of test sets, making it easier to ru
test sets by running the associated functions instead.
Note that in the case of functions, the test set will be given the name of the called function.
In the event that a nested test set has no failures, as happened here, it will be hidden in the
summary, unless the `verbose=true` option is passed:
summary:

```jldoctest testfoo; filter = r"[0-9\.]+s"
julia> @testset verbose = true "Foo Tests" begin
julia> @testset "Foo Tests" begin
@testset "Animals" begin
@test foo("cat") == 9
@test foo("dog") == foo("cat")
Expand All @@ -166,10 +166,6 @@ julia> @testset verbose = true "Foo Tests" begin
end;
Test Summary: | Pass Total Time
Foo Tests | 8 8 0.0s
Animals | 2 2 0.0s
Arrays 1 | 2 2 0.0s
Arrays 2 | 2 2 0.0s
Arrays 3 | 2 2 0.0s
```

If we do have a test failure, only the details for the failed test sets will be shown:
Expand Down Expand Up @@ -201,6 +197,66 @@ Foo Tests | 3 1 4 0.0s
ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored, 0 broken.
```

You can use the `verbose` option to show results even for test sets that pass.
`verbose=true` (or equivalently `verbose=1`) displays results only for the top level of
nested sets:

```julia-repl; filter = r"[0-9\.]+s"
julia> @testset "Foo Tests" verbose = true begin
@testset "Animals" begin
@testset "Felines" begin
@test foo("cat") == 9
end
@testset "Canines" begin
@test foo("dog") == 9
end
end
@testset "Arrays" begin
@testset "zeros" begin
@test foo(zeros(2)) == 4
end
@testset "fill" begin
@test foo(fill(1.0, 4)) == 16
end
end
end;
Test Summary: | Pass Total Time
Foo Tests | 4 4 0.0s
Animals | 2 2 0.0s
Arrays | 2 2 0.0s
```

whereas `verbose=2` will apply to all nested test sets:

```julia-repl; filter = r"[0-9\.]+s"
julia> @testset "Foo Tests" verbose = 2 begin
@testset "Animals" begin
@testset "Felines" begin
@test foo("cat") == 9
end
@testset "Canines" begin
@test foo("dog") == 9
end
end
@testset "Arrays" begin
@testset "zeros" begin
@test foo(zeros(2)) == 4
end
@testset "fill" begin
@test foo(fill(1.0, 4)) == 16
end
end
end;
Test Summary: | Pass Total Time
Foo Tests | 4 4 0.0s
Animals | 2 2 0.0s
Felines | 1 1 0.0s
Canines | 1 1 0.0s
Arrays | 2 2 0.0s
zeros | 1 1 0.0s
fill | 1 1 0.0s
```

## Testing Log Statements

One can use the [`@test_logs`](@ref) macro to test log statements, or use a [`TestLogger`](@ref).
Expand Down
33 changes: 29 additions & 4 deletions src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1053,6 +1053,20 @@ end

#-----------------------------------------------------------------------

"""
VerbosityLevel

Different possible levels of verbosity for test output as reported by DefaultTestSet.

Options are `only_failures`, `this_testset`, and `this_testset_and_children`.
"""
@enum VerbosityLevel only_failures=0 this_testset=1 this_testset_and_children=2

# The established user interface for setting verbosity level is `verbose = true/false`,
# hence the below conversion. Note that by virtue of being an Enum, we can also use the Ints
# 0, 1, 2 to construct a `VerbosityLevel`.
VerbosityLevel(x::Bool) = x ? this_testset : only_failures

"""
DefaultTestSet

Expand All @@ -1065,14 +1079,25 @@ mutable struct DefaultTestSet <: AbstractTestSet
results::Vector{Any}
n_passed::Int
anynonpass::Bool
verbose::Bool
verbose::VerbosityLevel
showtiming::Bool
time_start::Float64
time_end::Union{Float64,Nothing}
failfast::Bool
file::Union{String,Nothing}
end
function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing)
function DefaultTestSet(desc::AbstractString; verbose = nothing, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing, source = nothing)
if isnothing(verbose)
parent_ts = get_testset()
if parent_ts isa DefaultTestSet && (parent_ts.verbose == this_testset_and_children)
verbose = this_testset_and_children
else
verbose = only_failures
end
elseif !(verbose isa VerbosityLevel)
verbose = VerbosityLevel(verbose)
end

if isnothing(failfast)
# pass failfast state into child testsets
parent_ts = get_testset()
Expand Down Expand Up @@ -1230,7 +1255,7 @@ function get_alignment(ts::DefaultTestSet, depth::Int)
# The minimum width at this depth is
ts_width = 2*depth + length(ts.description)
# If not verbose and all passing, no need to look at children
!ts.verbose && !ts.anynonpass && return ts_width
(ts.verbose == only_failures) && !ts.anynonpass && return ts_width
# Return the maximum of this width and the minimum width
# for all children (if they exist)
isempty(ts.results) && return ts_width
Expand Down Expand Up @@ -1344,7 +1369,7 @@ function print_counts(ts::DefaultTestSet, depth, align,

# Only print results at lower levels if we had failures or if the user
# wants.
if (np + nb != subtotal) || (ts.verbose)
if (np + nb != subtotal) || (ts.verbose >= this_testset)
for t in ts.results
if isa(t, DefaultTestSet)
print_counts(t, depth + 1, align,
Expand Down
171 changes: 124 additions & 47 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1200,72 +1200,149 @@ let ex = :(something_complex + [1, 2, 3])
end

@testset "verbose option" begin
expected = r"""
Test Summary: | Pass Total Time
Parent | 9 9 \s*\d*.\ds
Child 1 | 3 3 \s*\d*.\ds
Child 1.1 (long name) | 1 1 \s*\d*.\ds
Child 1.2 | 1 1 \s*\d*.\ds
Child 1.3 | 1 1 \s*\d*.\ds
Child 2 | 3 3 \s*\d*.\ds
Child 3 | 3 3 \s*\d*.\ds
Child 3.1 | 1 1 \s*\d*.\ds
Child 3.2 | 1 1 \s*\d*.\ds
Child 3.3 | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
@testset "verbose = true" begin
expected = r"""
Test Summary: | Pass Total Time
Parent | 9 9 \s*\d*.\ds
Child 1 | 3 3 \s*\d*.\ds
Child 1.1 (long name) | 1 1 \s*\d*.\ds
Child 1.2 | 1 1 \s*\d*.\ds
Child 1.3 | 1 1 \s*\d*.\ds
Child 2 | 3 3 \s*\d*.\ds
Child 3 | 3 3 \s*\d*.\ds
Child 3.1 | 1 1 \s*\d*.\ds
Child 3.2 | 1 1 \s*\d*.\ds
Child 3.3 | 1 1 \s*\d*.\ds
"""
using Test

@testset "Parent" verbose = true begin
@testset "Child 1" verbose = true begin
@testset "Child 1.1 (long name)" begin
@test 1 == 1
end
mktemp() do f, _
write(f,
"""
using Test

@testset "Child 1.2" begin
@test 1 == 1
end
@testset "Parent" verbose = true begin
@testset "Child 1" verbose = 1 begin
@testset "Child 1.1 (long name)" begin
@test 1 == 1
end

@testset "Child 1.3" begin
@test 1 == 1
end
end
@testset "Child 1.2" begin
@test 1 == 1
end

@testset "Child 2" begin
@testset "Child 2.1" begin
@test 1 == 1
@testset "Child 1.3" begin
@test 1 == 1
end
end

@testset "Child 2.2" begin
@test 1 == 1
@testset "Child 2" begin
@testset "Child 2.1" begin
@test 1 == 1
end

@testset "Child 2.2" begin
@test 1 == 1
end

@testset "Child 2.3" begin
@test 1 == 1
end
end

@testset "Child 2.3" begin
@test 1 == 1
@testset "Child 3" verbose = true begin
@testset "Child 3.1" begin
@test 1 == 1
end

@testset "Child 3.2" begin
@test 1 == 1
end

@testset "Child 3.3" begin
@test 1 == 1
end
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end

@testset "verbose = 2" begin
expected = r"""
Test Summary: \| Pass Total Time
Parent \| 9 9 \s*\d*\.\ds
Child 1 \| 3 3 \s*\d*\.\ds
Child 1\.1 \(long name\) \| 2 2 \s*\d*\.\ds
Child 1\.2 \| 1 1 \s*\d*\.\ds
Child 2 \| 3 3 \s*\d*\.\ds
Child 2\.1 \| 1 1 \s*\d*\.\ds
Child 2\.1\.1 \| 1 1 \s*\d*\.\ds
Child 2\.2 \| 1 1 \s*\d*\.\ds
Child 2\.3 \| 1 1 \s*\d*\.\ds
Child 3 \| 3 3 \s*\d*\.\ds
"""

mktemp() do f, _
write(f,
"""
using Test

@testset "Child 3" verbose = true begin
@testset "Child 3.1" begin
@test 1 == 1
@testset "Parent" verbose = 2 begin
# Setting the verbosity to just 1 overrides the inherited, higher level of 2.
@testset "Child 1" verbose = 1 begin
@testset "Child 1.1 (long name)" begin
@testset "Child 1.1.1" begin
@test 1 == 1
end

@testset "Child 1.1.2" begin
@test 1 == 1
end
end

@testset "Child 1.2" begin
@test 1 == 1
end
end

@testset "Child 3.2" begin
@test 1 == 1
@testset "Child 2" begin
@testset "Child 2.1" begin
@testset "Child 2.1.1" begin
@test 1 == 1
end
end

@testset "Child 2.2" begin
@test 1 == 1
end

@testset "Child 2.3" begin
@test 1 == 1
end
end

@testset "Child 3.3" begin
@test 1 == 1
@testset "Child 3" verbose = false begin
@testset "Child 3.1" begin
@test 1 == 1
end

@testset "Child 3.2" begin
@test 1 == 1
end

@testset "Child 3.3" begin
@test 1 == 1
end
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end

Expand Down