Skip to content

Commit

Permalink
Run testitem if not in runtests call (#61)
Browse files Browse the repository at this point in the history 
* Run testitem if not in `runtests` call

* add test

* Gensym name in testitem macro to be safe

* Test we see expected log messages

* Test runtestitem defaults to eager logs and verbose results

* Bump version

* interpolate `nothing`
  • Loading branch information
@nickrobinson251
nickrobinson251 authored May 26, 2023
1 parent f00d702 commit b91b461
Show file tree
Hide file tree
Showing 9 changed files with 181 additions and 65 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "ReTestItems"
uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
version = "1.4.3"
version = "1.5.0"

[deps]
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Expand Down
9 changes: 7 additions & 2 deletions src/ReTestItems.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ function runtests(
tags::Union{Symbol,AbstractVector{Symbol},Nothing}=nothing,
report::Bool=parse(Bool, get(ENV, "RETESTITEMS_REPORT", "false")),
logs::Symbol=default_log_display_mode(report, nworkers),
verbose_results::Bool=logs!=:issues && isinteractive(),
verbose_results::Bool=(logs !== :issues && isinteractive())
)
nworker_threads = _validated_nworker_threads(nworker_threads)
paths′ = filter(paths) do p
Expand Down Expand Up @@ -690,7 +690,12 @@ function runtestitem(ti::TestItem; kw...)
runtestitem(ti, GLOBAL_TEST_CONTEXT_FOR_TESTING; kw...)
end

function runtestitem(ti::TestItem, ctx::TestContext; verbose_results::Bool=false, finish_test::Bool=true, logs::Symbol=:eager)
# Default to verbose output for running an individual test-item by itself, i.e.
# when `runtestitem` called directly or `@testitem` called outside of `runtests`.
function runtestitem(
ti::TestItem, ctx::TestContext;
logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true,
)
name = ti.name
log_testitem_start(ti, ctx.ntestitems)
ts = DefaultTestSet(name; verbose=verbose_results)
Expand Down
4 changes: 2 additions & 2 deletions src/log_capture.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ end

# Marks the start of each test item
function log_testitem_start(ti::TestItem, ntestitems=0)
io = IOContext(IOBuffer(), :color=>Base.get_have_color())
io = IOContext(IOBuffer(), :color => get(DEFAULT_STDOUT[], :color, false)::Bool)
interactive = parse(Bool, get(ENV, "RETESTITEMS_INTERACTIVE", string(Base.isinteractive())))
print(io, format(now(), "HH:MM:SS | "))
!interactive && print(io, _mem_watermark())
Expand All @@ -231,7 +231,7 @@ end

# mostly copied from timing.jl
function log_testitem_done(ti::TestItem, ntestitems=0)
io = IOContext(IOBuffer(), :color=>Base.get_have_color())
io = IOContext(IOBuffer(), :color => get(DEFAULT_STDOUT[], :color, false)::Bool)
interactive = parse(Bool, get(ENV, "RETESTITEMS_INTERACTIVE", string(Base.isinteractive())))
print(io, format(now(), "HH:MM:SS | "))
!interactive && print(io, _mem_watermark())
Expand Down
43 changes: 30 additions & 13 deletions src/macros.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ gettls(k, d) = get(task_local_storage(), k, d)
###
### testsetup
###

"""
TestSetup(name, code)
Expand Down Expand Up @@ -128,6 +129,17 @@ struct TestItem
stats::Vector{PerfStats} # populated when the test item is finished evaluating
scheduled_for_evaluation::ScheduledForEvaluation # to keep track of whether the test item has been scheduled for evaluation
end
function TestItem(id, name, tags, default_imports, setups, retries, file, line, project_root, code)
return TestItem(
id, name, tags, default_imports, setups, retries, file, line, project_root, code,
TestSetup[],
Ref{Int}(0),
DefaultTestSet[],
Ref{Int}(0),
PerfStats[],
ScheduledForEvaluation(),
)
end

"""
@testitem "name" [tags=[] setup=[] retries=0 default_imports=true] begin
Expand Down Expand Up @@ -211,6 +223,7 @@ macro testitem(nm, exs...)
retries = 0
tags = Symbol[]
setup = Any[]
_run = true # useful for testing `@testitem` itself
if length(exs) > 1
kw_seen = Set{Symbol}()
for ex in exs[1:end-1]
Expand All @@ -231,6 +244,9 @@ macro testitem(nm, exs...)
elseif kw == :retries
retries = ex.args[2]
@assert retries isa Integer "`default_imports` keyword must be passed an `Integer`"
elseif kw == :_run
_run = ex.args[2]
@assert _run isa Bool "`_run` keyword must be passed a `Bool`"
else
error("unknown `@testitem` keyword arg `$(ex.args[1])`")
end
Expand All @@ -240,21 +256,22 @@ macro testitem(nm, exs...)
error("expected `@testitem` to have a body")
end
q = QuoteNode(exs[end])
ti = gensym(:ti)
esc(quote
$store_test_item_setup(
$TestItem(
Ref(0), $nm, $tags, $default_imports, $setup, $retries,
$(String(__source__.file)), $(__source__.line),
$gettls(:__RE_TEST_PROJECT__, "."),
$q,
$TestSetup[],
Ref{Int}(0),
$DefaultTestSet[],
Ref{Int}(0),
$PerfStats[],
$ScheduledForEvaluation()
)
let $ti = $TestItem(
Ref(0), $nm, $tags, $default_imports, $setup, $retries,
$(String(__source__.file)), $(__source__.line),
$gettls(:__RE_TEST_PROJECT__, "."),
$q,
)
if !$_run || $gettls(:__RE_TEST_RUNNING__, false)::$Bool
$store_test_item_setup($ti)
$ti
else # We are not in a `runtests` call, so we run the testitem immediately.
$runtestitem($ti)
$nothing
end
end
end)
end

Expand Down
1 change: 1 addition & 0 deletions src/testcontext.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ function get_starting_testitems(ti::TestItems, n)
len = length(ti.testitems)
step = max(1, len / n)
testitems = [ti.testitems[round(Int, i)] for i in 1:step:len]
@debugv 2 "get_starting_testitems" len n allunique(testitems)
@assert length(testitems) == min(n, len) && allunique(testitems)
for (i, t) in enumerate(testitems)
@atomic t.scheduled_for_evaluation.value = true
Expand Down
40 changes: 20 additions & 20 deletions test/_test_log_capture.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
@testset "log capture logs=$(repr(log_display))" begin
@testset "TestItem" begin
@testset "log capture for println" begin
ti = @testitem "uses println" begin
ti = @testitem "uses println" _run=false begin
println("println msg")
end
if log_display == :eager
Expand All @@ -18,7 +18,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "log capture for println(stderr, ...)" begin
ti = @testitem "uses println" begin
ti = @testitem "uses println" _run=false begin
println(stderr, "println msg")
end
if log_display == :eager
Expand All @@ -31,7 +31,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "log capture for printstyled" begin
ti = @testitem "uses printstyled" begin
ti = @testitem "uses printstyled" _run=false begin
printstyled("printstyled msg red", color=:red)
end
if log_display == :eager
Expand All @@ -44,7 +44,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "log capture for @error" begin
ti = @testitem "uses @error" begin
ti = @testitem "uses @error" _run=false begin
@error("@error msg")
end
if log_display == :eager
Expand All @@ -57,7 +57,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "@test_logs @info works within log capture" begin
ti = @testitem "uses @test_logs @info" begin
ti = @testitem "uses @test_logs @info" _run=false begin
@test_logs (:info, "Look ma, I'm logging") begin
@info "Look ma, I'm logging"
end
Expand All @@ -72,7 +72,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "@test_logs @error works within log capture" begin
ti = @testitem "uses @test_logs @error" begin
ti = @testitem "uses @test_logs @error" _run=false begin
@test_logs (:error, "Look ma, I'm logging") begin
@error "Look ma, I'm logging"
end
Expand All @@ -87,7 +87,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "redirect_stdout works within log capture" begin
ti = @testitem "uses redirect_stdout" begin
ti = @testitem "uses redirect_stdout" _run=false begin
mktemp() do tmp_path, tmp_io
redirect_stdout(tmp_io) do
print("This should not be visible to log capture")
Expand All @@ -106,7 +106,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "redirect_stderr works within log capture" begin
ti = @testitem "uses redirect_stderr" begin
ti = @testitem "uses redirect_stderr" _run=false begin
mktemp() do tmp_path, tmp_io
redirect_stderr(tmp_io) do
print(stderr, "This should not be visible to log capture")
Expand All @@ -125,7 +125,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "with_logger works within log capture (redirect to a file)" begin
ti = @testitem "uses with_logger (redirect to a file)" begin
ti = @testitem "uses with_logger (redirect to a file)" _run=false begin
using Logging
mktemp() do tmp_path, tmp_io
logger = SimpleLogger(tmp_io)
Expand All @@ -146,7 +146,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "with_logger works within log capture" begin
ti = @testitem "uses with_logger" begin
ti = @testitem "uses with_logger" _run=false begin
using Logging
logger = SimpleLogger()
with_logger(logger) do
Expand All @@ -163,7 +163,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end

@testset "log capture for display" begin
ti = @testitem "uses display" begin
ti = @testitem "uses display" _run=false begin
display("display msg")
end
if log_display == :eager
Expand All @@ -181,7 +181,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
setup = @testsetup module LoggingTestSetup
println("println msg")
end
ti = @testitem "setup uses println" setup=[LoggingTestSetup] begin end
ti = @testitem "setup uses println" setup=[LoggingTestSetup] _run=false begin end

if log_display == :eager
logs = IOCapture.capture(()->ReTestItems.runtestitem(ti; logs=log_display), color=true).output
Expand All @@ -196,7 +196,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
setup = @testsetup module LoggingTestSetup
println(stderr, "println msg")
end
ti = @testitem "setup uses println" setup=[LoggingTestSetup] begin end
ti = @testitem "setup uses println" setup=[LoggingTestSetup] _run=false begin end
if log_display == :eager
logs = IOCapture.capture(()->ReTestItems.runtestitem(ti; logs=log_display), color=true).output
else
Expand All @@ -211,7 +211,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
setup = @testsetup module LoggingTestSetup
printstyled("printstyled msg red", color=:red)
end
ti = @testitem "setup uses printstyled" setup=[LoggingTestSetup] begin end
ti = @testitem "setup uses printstyled" setup=[LoggingTestSetup] _run=false begin end

if log_display == :eager
logs = IOCapture.capture(()->ReTestItems.runtestitem(ti; logs=log_display), color=true).output
Expand All @@ -226,7 +226,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
setup = @testsetup module LoggingTestSetup
@error("@error msg")
end
ti = @testitem "setup uses @error" setup=[LoggingTestSetup] begin end
ti = @testitem "setup uses @error" setup=[LoggingTestSetup] _run=false begin end

if log_display == :eager
logs = IOCapture.capture(()->ReTestItems.runtestitem(ti; logs=log_display), color=true).output
Expand All @@ -245,7 +245,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end
flush(tmp_io)
end
ti = @testitem "setup uses redirect_stdout" setup=[LoggingTestSetup] begin
ti = @testitem "setup uses redirect_stdout" setup=[LoggingTestSetup] _run=false begin
tmp_path = LoggingTestSetup.tmp_path
@test read(tmp_path, String) == "This should not be visible to log capture"
end
Expand All @@ -267,7 +267,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end
flush(tmp_io)
end
ti = @testitem "setup uses redirect_stderr" setup=[LoggingTestSetup] begin
ti = @testitem "setup uses redirect_stderr" setup=[LoggingTestSetup] _run=false begin
tmp_path = LoggingTestSetup.tmp_path
@test read(tmp_path, String) == "This should not be visible to log capture"
end
Expand All @@ -290,7 +290,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
end
flush(tmp_io)
end
ti = @testitem "setup uses with_logger (redirect to a file)" setup=[LoggingTestSetup] begin
ti = @testitem "setup uses with_logger (redirect to a file)" setup=[LoggingTestSetup] _run=false begin
tmp_path = LoggingTestSetup.tmp_path
@test startswith(read(tmp_path, String), "┌ Info: This should not be visible to log capture\n└ @ ")
end
Expand All @@ -312,7 +312,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
@info "This should be visible to log capture"
end
end
ti = @testitem "setup uses with_logger" setup=[LoggingTestSetup] begin end
ti = @testitem "setup uses with_logger" setup=[LoggingTestSetup] _run=false begin end

if log_display == :eager
logs = IOCapture.capture(()->ReTestItems.runtestitem(ti; logs=log_display), color=true).output
Expand All @@ -327,7 +327,7 @@ const log_display = Symbol(ENV["LOG_DISPLAY"])
setup = @testsetup module LoggingTestSetup
display("display msg")
end
ti = @testitem "setup uses display" setup=[LoggingTestSetup] begin end
ti = @testitem "setup uses display" setup=[LoggingTestSetup] _run=false begin end

if log_display == :eager
logs = IOCapture.capture(()->ReTestItems.runtestitem(ti; logs=log_display), color=true).output
Expand Down
4 changes: 2 additions & 2 deletions test/internals.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ using ReTestItems
# let's test this exhaustively for 1-10 testitems across 1-10 workers.
for nworkers in 1:10
for nitems in 1:10
testitems = [@testitem "ti-$i" begin; end; for i in 1:nitems]
testitems = [@testitem "ti-$i" _run=false begin; end; for i in 1:nitems]
starts = get_starting_testitems(TestItems(graph, testitems, 0), nworkers)
startitems = [x for x in starts if !isnothing(x)]
@test length(starts) == nworkers
Expand Down Expand Up @@ -167,7 +167,7 @@ end # `include_testfiles!` testset
@testset "report_empty_testsets" begin
using ReTestItems: TestItem, report_empty_testsets, PerfStats, ScheduledForEvaluation
using Test: DefaultTestSet, Fail, Error
ti = TestItem(Ref(42), "Dummy TestItem", [], false, [], 0, "source/path", 42, ".", nothing, [], Ref{Int}(), Test.DefaultTestSet[], Ref{Int}(0), PerfStats[], ScheduledForEvaluation())
ti = TestItem(Ref(42), "Dummy TestItem", [], false, [], 0, "source/path", 42, ".", nothing)

ts = DefaultTestSet("Empty testset")
report_empty_testsets(ti, ts)
Expand Down
2 changes: 1 addition & 1 deletion test/log_capture.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ end
@testset "log capture -- reporting" begin
setup1 = @testsetup module TheTestSetup1 end
setup2 = @testsetup module TheTestSetup2 end
ti = @testitem "TheTestItem" setup=[TheTestSetup1, TheTestSetup2] begin end
ti = TestItem(Ref(42), "TheTestItem", [], false, [], 0, "source/path", 42, ".", nothing)
push!(ti.testsetups, setup1)
push!(ti.testsetups, setup2)
push!(ti.testsets, Test.DefaultTestSet("dummy"))
Expand Down
Loading

2 comments on commit b91b461

@nickrobinson251
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/84315

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v1.5.0 -m "<description of version>" b91b4612404954600fa97e7752c0ab64e77153e9
git push origin v1.5.0

Please sign in to comment.