diff --git a/Project.toml b/Project.toml index 7d97505..4cbc8b5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "TestReports" uuid = "dcd651b4-b50a-5b6b-8f22-87e9f253a252" -version = "0.5.4" +version = "0.5.5" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/docs/src/manual.md b/docs/src/manual.md index 507302c..938bf22 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -206,9 +206,12 @@ Each test is recorded in a separate testsuite with the name showing the original `TestReports.jl` has not been tested significantly with custom `TestSet` types, please raise an issue if you find any problems/have a request. -However at a minimum, for a custom `TestSet` type to work with `TestReports` it must: -- Push itself onto its parent when finishing, if it is not at the top level -- Have `description` and `results` fields as per a `DefaultTestSet` +However at a minimum, for a custom `TestSet` type to work with `TestReports`, it must: +- Push itself onto its parent when finishing, if it is not at the top level. +- Have a `description` fields as per a `DefaultTestSet` or have a method defined for + `TestReports.get_description` that returns the description from the `TestSet`. +- Have a `results` fields as per a `DefaultTestSet` or have a method defined for + `TestReports.get_results` that returns the results from the `TestSet`. The following information in a JUnit XML relies on the functionality of `ReportingTestSet`s but can be added to your own custom `TestSet` as described in the table. diff --git a/src/testsets.jl b/src/testsets.jl index 8aabb0c..8dae50b 100644 --- a/src/testsets.jl +++ b/src/testsets.jl @@ -67,12 +67,12 @@ end ReportingTestSet(desc) = ReportingTestSet(desc, [], Dict(), now(), Millisecond(0), now(), gethostname()) function record(ts::ReportingTestSet, t::Result) - push!(ts.results, ReportingResult(t, now()-ts.last_record_time)) + push!(get_results(ts), ReportingResult(t, now()-ts.last_record_time)) ts.last_record_time = now() t end function record(ts::ReportingTestSet, t::AbstractTestSet) - push!(ts.results, t) + push!(get_results(ts), t) ts.last_record_time = now() t end @@ -149,7 +149,7 @@ hostname(ts::AbstractTestSet) = gethostname() set_time_taken!(ts::AbstractTestSet, time_taken) Sets the time taken field of a `ReportingTestSet`. This is used when flattening -`ReportingTestSet`s for report generation and an be extended for custom `TestSet`s. +`ReportingTestSet`s for report generation and can be extended for custom `TestSet`s. """ set_time_taken!(ts::ReportingTestSet, time_taken::Millisecond) = ts.time_taken = time_taken set_time_taken!(ts::AbstractTestSet, time_taken::Millisecond) = nothing @@ -159,11 +159,56 @@ set_time_taken!(ts::AbstractTestSet, time_taken::Millisecond) = nothing set_start_time!(ts::AbstractTestSet, start_time) Sets the start time field of a `ReportingTestSet`. This is used when flattening -`ReportingTestSet`s for report generation and an be extended for custom `TestSet`s. +`ReportingTestSet`s for report generation and can be extended for custom `TestSet`s. """ set_start_time!(ts::ReportingTestSet, start_time::DateTime) = ts.start_time = start_time set_start_time!(ts::AbstractTestSet, start_time::DateTime) = nothing +""" + get_description(ts::ReportingTestSet) + get_description(ts::AbstractTestSet) + +Get the description of a `ReportingTestSet`, returns `ts.description` +for an `AbstractTestSet`. Can be extended for custom `TestSet`s, +must return a `String`. +""" +get_description(ts::ReportingTestSet) = ts.description +get_description(ts::AbstractTestSet) = ts.description + +""" + get_results(ts::ReportingTestSet) + get_results(ts::AbstractTestSet) + +Get the results of a `ReportingTestSet`, returns `ts.results` +for an `AbstractTestSet`. Can be extended for custom `TestSet`s, +must return a `Vector` containing `Result`s and/or +`AbstractTestSet`s. +""" +get_results(ts::ReportingTestSet) = ts.results +get_results(ts::AbstractTestSet) = ts.results + +""" + set_description!(ts::ReportingTestSet, description) + set_description!(ts::AbstractTestSet, description) + +Sets the description field of a `ReportingTestSet` or an `AbstractTestSet`. +This can be extended for custom `TestSet`s if the description is stored +differently. +""" +set_description!(ts::AbstractTestSet, description) = ts.description = description +set_description!(ts::ReportingTestSet, description) = ts.description = description + +""" + set_results!(ts::ReportingTestSet, results::AbstractVector) + set_results!(ts::AbstractTestSet, results::AbstractVector) + +Sets the results field of a `ReportingTestSet` or an `AbstractTestSet`. +This can be extended for custom `TestSet`s if the results are stored +differently. +""" +set_results!(ts::ReportingTestSet, results::AbstractVector) = ts.results = results +set_results!(ts::AbstractTestSet, results::AbstractVector) = ts.results = results + ############ # Checking # ############ @@ -175,7 +220,7 @@ Note that unlike the `DefaultTestSet`, the `ReportingTestSet` does not throw an exception on a failure. Thus to set the exit code from the runner code, we check it using `exit(any_problems(top_level_testset))`. """ -any_problems(ts::AbstractTestSet) = any(any_problems.(ts.results)) +any_problems(ts::AbstractTestSet) = any(any_problems.(get_results(ts))) any_problems(rs::ReportingResult) = any_problems(rs.result) any_problems(::Pass) = false any_problems(::Fail) = true @@ -194,15 +239,15 @@ for writing a report, as a JUnit XML does not allow one testsuite to be nested i The top level `TestSet` becomes the testsuites element, and the middle level `TestSet`s become individual testsuite elements, and the `Result`s become the testcase elements. -If `ts.results` contains any `Result`s, these are added into a new `TestSet` with the -description "Top level tests", which then replaces them in `ts.results`. +If `get_results(ts)` contains any `Result`s, these are added into a new `TestSet` with the +description "Top level tests", which then replaces them using `set_results!(ts, results)`. """ function flatten_results!(ts::AbstractTestSet) # Add any top level Results to their own TestSet handle_top_level_results!(ts) # Flatten all results of top level testset, which should all be testsets now - ts.results = vcat(_flatten_results!.(ts.results)...) + set_results!(ts, vcat(_flatten_results!.(get_results(ts))...)) return ts end @@ -212,7 +257,7 @@ end Recursively flatten `ts` to a vector of `TestSet`s. """ function _flatten_results!(ts::AbstractTestSet)::Vector{<:AbstractTestSet} - original_results = ts.results + original_results = get_results(ts) flattened_results = AbstractTestSet[] # Track results that are a Result so that if there are any, they can be added # in their own testset to flattened_results @@ -226,7 +271,7 @@ function _flatten_results!(ts::AbstractTestSet)::Vector{<:AbstractTestSet} function inner!(childts::AbstractTestSet) # Make it a sibling update_testset_properties!(childts, ts) - childts.description = ts.description * "/" * childts.description + set_description!(childts, get_description(ts) * "/" * get_description(childts)) push!(flattened_results, childts) end @@ -238,10 +283,10 @@ function _flatten_results!(ts::AbstractTestSet)::Vector{<:AbstractTestSet} end end - # results will be empty if ts.results only contains testsets + # results will be empty if get_results(ts) only contains testsets if !isempty(results) # Use same ts to preserve description - ts.results = results + set_results!(ts, results) push!(flattened_results, ts) end return flattened_results @@ -272,7 +317,7 @@ See also: [`properties`](@ref) """ function update_testset_properties!(childts::AbstractTestSet, ts::AbstractTestSet) if isnothing(properties(childts)) && !isnothing(properties(ts)) && !isempty(properties(ts)) - @warn "Properties of testset $(ts.description) can not be added to child testset $(childts.description) as it does not have a TestReports.properties method defined." + @warn "Properties of testset $(get_description(ts)) can not be added to child testset $(get_description(childts)) as it does not have a TestReports.properties method defined." # No need to check if childts is has properties defined and ts doesn't as if this is the case # ts has no properties to add to that of childts. elseif !isnothing(properties(ts)) @@ -281,7 +326,7 @@ function update_testset_properties!(childts::AbstractTestSet, ts::AbstractTestSe # Loop through keys so that warnings can be issued for any duplicates for key in parent_keys if key in child_keys - @warn "Property $key in testest $(ts.description) overwritten by child testset $(childts.description)" + @warn "Property $key in testest $(get_description(ts)) overwritten by child testset $(get_description(childts))" else properties(childts)[key] = properties(ts)[key] end @@ -293,25 +338,26 @@ end """ handle_top_level_results!(ts::AbstractTestSet) -If `ts.results` contains any `Result`s, these are removed from `ts.results` and -added to a new `ReportingTestSet`, which in turn is added to `ts.results`. This -leaves `ts.results` only containing `AbstractTestSet`s. +If `get_results(ts)` contains any `Result`s, these are removed and +added to a new `ReportingTestSet`, which in turn is added to the testsets +results. This leaves `get_results(ts)` only containing `AbstractTestSet`s. The `time_taken` field of the new `ReportingTestSet` is calculated by summing the time taken by the individual results, and the `start_time` field is set to the `start_time` field of `ts`. """ function handle_top_level_results!(ts::AbstractTestSet) - isa_Result = isa.(ts.results, Result) + isa_Result = isa.(get_results(ts), Result) if any(isa_Result) - original_results = ts.results - ts.results = AbstractTestSet[] + original_results = get_results(ts) + new_results = AbstractTestSet[] ts_nested = ReportingTestSet("Top level tests") - ts_nested.results = original_results[isa_Result] - set_time_taken!(ts_nested, sum(x -> time_taken(x)::Millisecond, ts_nested.results)) + set_results!(ts_nested, original_results[isa_Result]) + set_time_taken!(ts_nested, sum(x -> time_taken(x)::Millisecond, get_results(ts_nested))) set_start_time!(ts_nested, start_time(ts)::DateTime) - push!(ts.results, ts_nested) - append!(ts.results, original_results[.!isa_Result]) + push!(new_results, ts_nested) + append!(new_results, original_results[.!isa_Result]) + set_results!(ts, new_results) end return ts end @@ -325,11 +371,11 @@ Displays the test output in the same format as `Pkg.test` by using a function display_reporting_testset(ts::ReportingTestSet) # Create top level default testset to hold all results ts_default = DefaultTestSet("") - add_to_ts_default!.(Ref(ts_default), ts.results) + add_to_ts_default!.(Ref(ts_default), get_results(ts)) try # Finish each of the results of the top level testset, to mimick the # output from Pkg.test() - finish.(ts_default.results) + finish.(get_results(ts_default)) catch TestSetException # Don't want to error here if a test fails or errors. This is handled elswhere. end @@ -353,7 +399,7 @@ add_to_ts_default!(ts_default::DefaultTestSet, result::ReportingResult) = add_to add_to_ts_default!(ts_default::DefaultTestSet, result::Result) = record(ts_default, result) add_to_ts_default!(ts_default::DefaultTestSet, ts::AbstractTestSet) = record(ts_default, ts) function add_to_ts_default!(ts_default::DefaultTestSet, ts::ReportingTestSet) - sub_ts = DefaultTestSet(ts.description) - add_to_ts_default!.(Ref(sub_ts), ts.results) - push!(ts_default.results, sub_ts) + sub_ts = DefaultTestSet(get_description(ts)) + add_to_ts_default!.(Ref(sub_ts), get_results(ts)) + push!(get_results(ts_default), sub_ts) end diff --git a/src/to_xml.jl b/src/to_xml.jl index 8b351d2..5ff92cc 100644 --- a/src/to_xml.jl +++ b/src/to_xml.jl @@ -143,7 +143,7 @@ function report(ts::AbstractTestSet) total_nfails = 0 total_nerrors = 0 testsuiteid = 0 # ID increments from 0 - x_testsuites = map(ts.results) do result + x_testsuites = map(get_results(ts)) do result x_testsuite, ntests, nfails, nerrors = to_xml(result) total_ntests += ntests total_nfails += nfails @@ -171,10 +171,10 @@ the results of `ts` do not have both `description` or `results` fields. See also: [`report`](@ref) """ function check_ts(ts::AbstractTestSet) - !all(isa.(ts.results, AbstractTestSet)) && throw(ArgumentError("Results of ts must all be AbstractTestSets. See documentation for `report`.")) - for results_ts in ts.results - !isa(results_ts.description, AbstractString) && throw(ArgumentError("description field of $(typeof(results_ts)) must be an AbstractString.")) - !all(isa.(results_ts.results, Result)) && throw(ArgumentError("Results of each AbstractTestSet in ts.results must all be Results. See documentation for `report`.")) + !all(isa.(get_results(ts), AbstractTestSet)) && throw(ArgumentError("Results of ts must all be AbstractTestSets. See documentation for `report`.")) + for results_ts in get_results(ts) + !isa(get_description(results_ts), AbstractString) && throw(ArgumentError("description field of $(typeof(results_ts)) must be an AbstractString.")) + !all(isa.(get_results(results_ts), Result)) && throw(ArgumentError("Results of each AbstractTestSet in get_results(ts) must all be Results. See documentation for `report`.")) end end @@ -182,7 +182,7 @@ end to_xml(ts::AbstractTestSet) Create a testsuite node from a `AbstractTestSet`, by creating nodes for each result -in `ts.results`. For creating a JUnit XML, all results must be `AbstractResult`s, that is +in `get_results(ts)`. For creating a JUnit XML, all results must be `AbstractResult`s, that is they cannot be `AbstractTestSet`s, as the XML cannot have one testsuite nested inside another. """ @@ -190,20 +190,20 @@ function to_xml(ts::AbstractTestSet) total_ntests = 0 total_nfails = 0 total_nerrors = 0 - x_testcases = map(ts.results) do result + x_testcases = map(get_results(ts)) do result x_testcase, ntests, nfails, nerrors = to_xml(result) total_ntests += ntests total_nfails += nfails total_nerrors += nerrors # Set attributes which are common across result types - set_attribute!(x_testcase, "classname", ts.description) + set_attribute!(x_testcase, "classname", get_description(ts)) set_attribute!(x_testcase, "time", time_taken(result)::Millisecond) # Set attributes which require variables in this scope (result isa Pass || result isa ReportingResult{Pass}) && set_attribute!(x_testcase, "name", x_testcase["name"] * " (Test $total_ntests)") x_testcase end - x_testsuite = testsuite_xml(ts.description, total_ntests, total_nfails, total_nerrors, x_testcases, time_taken(ts)::Millisecond, start_time(ts)::DateTime, hostname(ts)) + x_testsuite = testsuite_xml(get_description(ts), total_ntests, total_nfails, total_nerrors, x_testcases, time_taken(ts)::Millisecond, start_time(ts)::DateTime, hostname(ts)) add_testsuite_properties!(x_testsuite, ts) x_testsuite, total_ntests, total_nfails, total_nerrors end