From 78a3a5fc968aa1b09b749ed9a0afe4c0970d09f0 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Wed, 24 Jun 2026 11:55:07 -0300
Subject: [PATCH 01/10] Run previously failed tests first

---
 src/ParallelTestRunner.jl | 30 ++++++++----
 test/runtests.jl          | 98 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 115 insertions(+), 13 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 2704bf3..fde1836 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -515,6 +515,18 @@ function default_njobs(; cpu_threads = Sys.CPU_THREADS, free_memory = available_
 end
 
 # Historical test duration database
+struct TestHistoryEntry <: AbstractFloat
+    duration::Float64
+    failed::Bool
+end
+# successful tests are sorted before failed ones, so they always run first
+Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
+Base.promote_rule(::Type{T}, ::Type{TestHistoryEntry}) where {T} = promote_type(T, Float64)
+Base.promote_rule(::Type{TestHistoryEntry}, ::Type{T}) where {T} = promote_type(Float64,T)
+# for compatibility with older versions of ParallelTestRunner
+Base.convert(::Type{TestHistoryEntry}, duration::Number) = TestHistoryEntry(duration, false)
+Base.convert(::Type{Float64}, entry::TestHistoryEntry) = entry.duration
+
 function get_history_file(mod::Module)
     scratch_dir = @get_scratch!("durations")
     return joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)", "$(nameof(mod)).jls")
@@ -523,16 +535,17 @@ function load_test_history(mod::Module)
     history_file = get_history_file(mod)
     if isfile(history_file)
         try
-            return deserialize(history_file)
+            hist::Dict{String, TestHistoryEntry} = deserialize(history_file)
+            return hist
         catch e
             @warn "Failed to load test history from $history_file" exception=e
-            return Dict{String, Float64}()
+            return Dict{String, TestHistoryEntry}()
         end
     else
-        return Dict{String, Float64}()
+        return Dict{String, TestHistoryEntry}()
     end
 end
-function save_test_history(mod::Module, history::Dict{String, Float64})
+function save_test_history(mod::Module, history::Dict{String, TestHistoryEntry})
     history_file = get_history_file(mod)
     try
         mkpath(dirname(history_file))
@@ -992,7 +1005,7 @@ function runtests(mod::Module, args::ParsedArgs;
     tests = collect(keys(testsuite))
     Random.shuffle!(tests)
     historical_durations = load_test_history(mod)
-    sort!(tests, by = x -> -get(historical_durations, x, Inf))
+    sort!(tests, by = x -> get(historical_durations, x, TestHistoryEntry(Inf, false)), rev = true)
 
     return _runtests(
         mod, args;
@@ -1019,7 +1032,7 @@ end
 function _runtests(mod::Module, args::ParsedArgs;
                    testsuite::Dict{String,Expr} = find_tests(pwd()),
                    tests::Vector{String},
-                   historical_durations::Dict{String, Float64},
+                   historical_durations::Dict{String, TestHistoryEntry},
                    init_code = :(),
                    init_worker_code = :(),
                    test_worker = Returns(nothing),
@@ -1157,7 +1170,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             ## currently-running
             for (test, start_time) in running_snapshot
                 elapsed = time() - start_time
-                duration = get(historical_durations, test, est_per_test)
+                duration::Float64 = get(historical_durations, test, est_per_test)
                 est_remaining += max(0.0, duration - elapsed)
             end
             ## yet-to-run
@@ -1520,7 +1533,7 @@ function _runtests(mod::Module, args::ParsedArgs;
 
                 if result isa AbstractTestRecord
                     testset = result[]::DefaultTestSet
-                    historical_durations[testname] = stop - start
+                    historical_durations[testname] = TestHistoryEntry(stop - start, anynonpass(testset))
                 else
                     # If this test raised an exception that means the test runner itself had some problem,
                     # so we may have hit a segfault, deserialization errors or something similar.
@@ -1529,6 +1542,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                     @assert result isa Exception
                     testset = create_testset(testname; start, stop)
                     Test.record(testset, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(NamedTuple[(;exception = result, backtrace = Union{Ptr{Nothing}, Base.InterpreterIP}[])]), LineNumberNode(1)))
+                    historical_durations[testname] = TestHistoryEntry(Inf, true)
                 end
 
                 with_testset(testset) do
diff --git a/test/runtests.jl b/test/runtests.jl
index 87e29a8..c3c9b3d 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,4 +1,5 @@
 using ParallelTestRunner
+using ParallelTestRunner: TestHistoryEntry
 using Test
 
 # Helper macro to show output of tests in case they fail.  Useful for debugging.
@@ -102,7 +103,7 @@ end
         ParallelTestRunner, parse_args(["--jobs=1", "--verbose"]);
         testsuite,
         tests,
-        historical_durations=Dict{String, Float64}(),
+        historical_durations=Dict{String, TestHistoryEntry}(),
         stdout=io,
         stderr=io,
     )
@@ -827,6 +828,37 @@ end
     @test any(contains("--color=yes"), exe.exec)
 end
 
+@testset "TestHistoryEntry" begin
+    flow = TestHistoryEntry(1,true)
+    fhigh = TestHistoryEntry(10,true)
+
+    slow = TestHistoryEntry(1,false)
+    shigh = TestHistoryEntry(10,false)
+
+    # irreflexive: lt(x, x) always yields false
+    @test !isless(flow, flow)
+    @test !isless(slow, slow)
+    @test !isless(fhigh, fhigh)
+    @test !isless(shigh, shigh)
+
+    # asymmetric: if lt(x, y) yields true then lt(y, x) yields false
+    @test isless(flow, fhigh)
+    @test !isless(fhigh, flow)
+    @test isless(shigh, flow)
+    @test !isless(flow, shigh)
+
+    # transitive: lt(x, y) && lt(y, z) implies lt(x, z)
+    @test isless(slow, shigh)
+    @test isless(shigh, flow)
+    @test isless(slow, flow)
+
+    # addition/subtraction with Float64
+    @test TestHistoryEntry(1, true) + 1.0 == 2.0
+    @test TestHistoryEntry(3, false) - 1.0 == 2.0
+    @test TestHistoryEntry(1, false) + 1.0 == 2.0
+    @test TestHistoryEntry(3, true) - 1.0 == 2.0
+end
+
 # ── Integration tests ────────────────────────────────────────────────────────
 
 @testset "non-verbose mode" begin
@@ -862,7 +894,7 @@ end
             ParallelTestRunner, parse_args(["--quickfail", "--verbose", "--jobs=1"]);
             testsuite,
             tests=["fail-test", "pass-test1", "pass-test2", "pass-test3", "pass-test4", "pass-test5"],
-            historical_durations=Dict{String, Float64}(),
+            historical_durations=Dict{String, TestHistoryEntry}(),
             stdout=io,
             stderr=io,
         )
@@ -1183,7 +1215,7 @@ end
                 testsuite,
                 tests=["fail-serial", "pass-serial1", "pass-serial2",
                        "pass-parallel1", "pass-parallel2", "pass-parallel3"],
-                historical_durations=Dict{String, Float64}(),
+                historical_durations=Dict{String, TestHistoryEntry}(),
                 serial=["fail-serial", "pass-serial1", "pass-serial2"],
                 stdout=io,
                 stderr=io,
@@ -1218,7 +1250,7 @@ end
                 testsuite,
                 tests=["fail-parallel", "pass-parallel1", "pass-parallel2",
                        "pass-serial1", "pass-serial2"],
-                historical_durations=Dict{String, Float64}(),
+                historical_durations=Dict{String, TestHistoryEntry}(),
                 serial=["pass-serial1", "pass-serial2"],
                 serial_position=:after,
                 stdout=io,
@@ -1253,7 +1285,7 @@ end
                 testsuite,
                 tests=["pass-parallel1", "pass-parallel2",
                        "fail-serial", "pass-serial1", "pass-serial2"],
-                historical_durations=Dict{String, Float64}(),
+                historical_durations=Dict{String, TestHistoryEntry}(),
                 serial=["fail-serial", "pass-serial1", "pass-serial2"],
                 serial_position=:after,
                 stdout=io,
@@ -1271,6 +1303,62 @@ end
         # The serial tests after `fail-serial` are never started.
         @test !contains(str, r"pass-serial[12] .+ started at")
     end
+
+    @testset "run previously failed tests first" begin
+        # a previously-failed test must start before previously-passing tests,
+        # even if it has a much shorter historical duration than they do
+        mod = @eval(Main, module $(gensym(:failfirstserial)) end)
+
+        testsuite = Dict(
+            "long-serial-pass" => :(@test true),
+            "mid-omitted-serial-pass" => :(@test true),
+            "short-serial-fail" => :(@test true),
+            "long-pass" => :(@test true),
+            "mid-omitted-pass" => :(@test true),
+            "mid-pass" => :(@test true),
+            "short-fail" => :(@test true),
+        )
+        serial = filter(collect(keys(testsuite))) do k
+            contains(k, "serial")
+        end
+
+        ParallelTestRunner.save_test_history(mod, Dict(
+            "long-serial-pass" => TestHistoryEntry(10.0, false),
+            "short-serial-fail" => TestHistoryEntry(1.0, true),
+            "long-pass" => TestHistoryEntry(10.0, false),
+            "mid-pass" => TestHistoryEntry(5.0, false),
+            "short-fail" => TestHistoryEntry(1.0, true),
+        ))
+
+        io = IOBuffer()
+        runtests(
+            mod, parse_args(["--jobs=1", "--verbose"]);
+            testsuite,
+            stdout=io,
+            stderr=io,
+            serial
+        )
+
+        str = String(take!(io)); print(str)
+        @test contains(str, "SUCCESS")
+
+        # create a mapping of test names to the character offset of their start times (lower is earlier)
+        started_at = Dict(
+            name => (m = match(Regex("$(name) .+ started at"), str); @test m !== nothing; m === nothing ? typemax(Int) : m.offset)
+            for name in keys(testsuite)
+        )
+
+        # normal
+        @test started_at["short-fail"] < started_at["mid-pass"]
+        @test started_at["short-fail"] < started_at["long-pass"]
+        @test started_at["short-fail"] < started_at["mid-omitted-pass"]
+        # among the remaining (previously-passing) tests, longer ones still run first
+        @test started_at["long-pass"] < started_at["mid-pass"]
+
+        # serial
+        @test started_at["short-serial-fail"] < started_at["long-serial-pass"]
+        @test started_at["short-serial-fail"] < started_at["mid-omitted-serial-pass"]
+    end
 end
 
 # This testset should always be the last one, don't add anything after this.

From 5e85713176ddb8519dd362f15f2cc76823df8de9 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:28:22 -0300
Subject: [PATCH 02/10] Make more compatible with older versions

---
 src/ParallelTestRunner.jl | 67 +++++++++++++++++++++++++--------------
 test/runtests.jl          | 44 ++++++++++++-------------
 2 files changed, 65 insertions(+), 46 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index fde1836..5ff3c0d 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -514,45 +514,58 @@ function default_njobs(; cpu_threads = Sys.CPU_THREADS, free_memory = available_
     return max(1, min(jobs, memory_jobs))
 end
 
-# Historical test duration database
+# Struct to make sorting test history entries easier
 struct TestHistoryEntry <: AbstractFloat
     duration::Float64
     failed::Bool
 end
 # successful tests are sorted before failed ones, so they always run first
 Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
-Base.promote_rule(::Type{T}, ::Type{TestHistoryEntry}) where {T} = promote_type(T, Float64)
-Base.promote_rule(::Type{TestHistoryEntry}, ::Type{T}) where {T} = promote_type(Float64,T)
-# for compatibility with older versions of ParallelTestRunner
-Base.convert(::Type{TestHistoryEntry}, duration::Number) = TestHistoryEntry(duration, false)
-Base.convert(::Type{Float64}, entry::TestHistoryEntry) = entry.duration
 
-function get_history_file(mod::Module)
+function get_history_files(mod::Module)
     scratch_dir = @get_scratch!("durations")
-    return joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)", "$(nameof(mod)).jls")
+    path_base = joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)")
+    return joinpath(path_base, "$(nameof(mod)).jls"), joinpath(path_base, "$(nameof(mod))_failed.jls")
 end
 function load_test_history(mod::Module)
-    history_file = get_history_file(mod)
-    if isfile(history_file)
+    history_file, failed_history_file = get_history_files(mod)
+
+    hist = if isfile(history_file)
         try
-            hist::Dict{String, TestHistoryEntry} = deserialize(history_file)
-            return hist
+            deserialize(history_file)::Dict{String, Float64}
         catch e
             @warn "Failed to load test history from $history_file" exception=e
-            return Dict{String, TestHistoryEntry}()
+            Dict{String, Float64}()
+        end
+    else
+        Dict{String, Float64}()
+    end
+    failed_hist = if isfile(failed_history_file)
+        try
+            deserialize(failed_history_file)::Set{String}
+        catch e
+            @warn "Failed to load failed test history from $failed_history_file" exception=e
+            Set{String}()
         end
     else
-        return Dict{String, TestHistoryEntry}()
+        Set{String}()
     end
+    return hist, failed_hist
 end
-function save_test_history(mod::Module, history::Dict{String, TestHistoryEntry})
-    history_file = get_history_file(mod)
+function save_test_history(mod::Module, history::Dict{String, Float64}, failed_tests::Set{String})
+    history_file, failed_history_file = get_history_files(mod)
     try
         mkpath(dirname(history_file))
         serialize(history_file, history)
     catch e
         @warn "Failed to save test history to $history_file" exception=e
     end
+    try
+        mkpath(dirname(failed_history_file))
+        serialize(failed_history_file, failed_tests)
+    catch e
+        @warn "Failed to save test failures to $failed_history_file" exception=e
+    end
 end
 
 function test_exe(color::Bool=false)
@@ -1004,14 +1017,16 @@ function runtests(mod::Module, args::ParsedArgs;
     # determine test order
     tests = collect(keys(testsuite))
     Random.shuffle!(tests)
-    historical_durations = load_test_history(mod)
-    sort!(tests, by = x -> get(historical_durations, x, TestHistoryEntry(Inf, false)), rev = true)
+    historical_durations, historical_failures = load_test_history(mod)
+    get_historical_duration(test) = TestHistoryEntry(get(historical_durations, test, Inf), test in historical_failures)
+    sort!(tests, by = x -> get_historical_duration(x), rev = true)
 
     return _runtests(
         mod, args;
         testsuite,
         tests,
         historical_durations,
+        historical_failures,
         init_code,
         init_worker_code,
         test_worker,
@@ -1032,7 +1047,8 @@ end
 function _runtests(mod::Module, args::ParsedArgs;
                    testsuite::Dict{String,Expr} = find_tests(pwd()),
                    tests::Vector{String},
-                   historical_durations::Dict{String, TestHistoryEntry},
+                   historical_durations::Dict{String, Float64},
+                   historical_failures::Set{String},
                    init_code = :(),
                    init_worker_code = :(),
                    test_worker = Returns(nothing),
@@ -1170,7 +1186,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             ## currently-running
             for (test, start_time) in running_snapshot
                 elapsed = time() - start_time
-                duration::Float64 = get(historical_durations, test, est_per_test)
+                duration = get(historical_durations, test, est_per_test)
                 est_remaining += max(0.0, duration - elapsed)
             end
             ## yet-to-run
@@ -1533,7 +1549,12 @@ function _runtests(mod::Module, args::ParsedArgs;
 
                 if result isa AbstractTestRecord
                     testset = result[]::DefaultTestSet
-                    historical_durations[testname] = TestHistoryEntry(stop - start, anynonpass(testset))
+                    historical_durations[testname] = stop - start
+                    if anynonpass(testset)
+                        push!(historical_failures, testname)
+                    else
+                        delete!(historical_failures, testname)
+                    end
                 else
                     # If this test raised an exception that means the test runner itself had some problem,
                     # so we may have hit a segfault, deserialization errors or something similar.
@@ -1542,7 +1563,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                     @assert result isa Exception
                     testset = create_testset(testname; start, stop)
                     Test.record(testset, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(NamedTuple[(;exception = result, backtrace = Union{Ptr{Nothing}, Base.InterpreterIP}[])]), LineNumberNode(1)))
-                    historical_durations[testname] = TestHistoryEntry(Inf, true)
+                    push!(historical_failures, testname)
                 end
 
                 with_testset(testset) do
@@ -1574,7 +1595,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             Test.TESTSET_PRINT_ENABLE[] = old_print_setting
         end
     end
-    save_test_history(mod, historical_durations)
+    save_test_history(mod, historical_durations, historical_failures)
 
     # display the results
     println(io_ctx.stdout)
diff --git a/test/runtests.jl b/test/runtests.jl
index c3c9b3d..3a52c53 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,5 +1,4 @@
 using ParallelTestRunner
-using ParallelTestRunner: TestHistoryEntry
 using Test
 
 # Helper macro to show output of tests in case they fail.  Useful for debugging.
@@ -53,7 +52,7 @@ include(joinpath(@__DIR__, "utils.jl"))
         @test contains(str, "(%)")
     end
 
-    @test isfile(ParallelTestRunner.get_history_file(ParallelTestRunner))
+    @test all(isfile.(ParallelTestRunner.get_history_files(ParallelTestRunner)))
 end
 
 @testset "default njobs" begin
@@ -103,7 +102,8 @@ end
         ParallelTestRunner, parse_args(["--jobs=1", "--verbose"]);
         testsuite,
         tests,
-        historical_durations=Dict{String, TestHistoryEntry}(),
+        historical_durations=Dict{String, Float64}(),
+        historical_failures=Set{String}(),
         stdout=io,
         stderr=io,
     )
@@ -829,11 +829,11 @@ end
 end
 
 @testset "TestHistoryEntry" begin
-    flow = TestHistoryEntry(1,true)
-    fhigh = TestHistoryEntry(10,true)
+    flow = ParallelTestRunner.TestHistoryEntry(1,true)
+    fhigh = ParallelTestRunner.TestHistoryEntry(10,true)
 
-    slow = TestHistoryEntry(1,false)
-    shigh = TestHistoryEntry(10,false)
+    slow = ParallelTestRunner.TestHistoryEntry(1,false)
+    shigh = ParallelTestRunner.TestHistoryEntry(10,false)
 
     # irreflexive: lt(x, x) always yields false
     @test !isless(flow, flow)
@@ -851,12 +851,6 @@ end
     @test isless(slow, shigh)
     @test isless(shigh, flow)
     @test isless(slow, flow)
-
-    # addition/subtraction with Float64
-    @test TestHistoryEntry(1, true) + 1.0 == 2.0
-    @test TestHistoryEntry(3, false) - 1.0 == 2.0
-    @test TestHistoryEntry(1, false) + 1.0 == 2.0
-    @test TestHistoryEntry(3, true) - 1.0 == 2.0
 end
 
 # ── Integration tests ────────────────────────────────────────────────────────
@@ -894,7 +888,8 @@ end
             ParallelTestRunner, parse_args(["--quickfail", "--verbose", "--jobs=1"]);
             testsuite,
             tests=["fail-test", "pass-test1", "pass-test2", "pass-test3", "pass-test4", "pass-test5"],
-            historical_durations=Dict{String, TestHistoryEntry}(),
+            historical_durations=Dict{String, Float64}(),
+            historical_failures=Set{String}(),
             stdout=io,
             stderr=io,
         )
@@ -1215,7 +1210,8 @@ end
                 testsuite,
                 tests=["fail-serial", "pass-serial1", "pass-serial2",
                        "pass-parallel1", "pass-parallel2", "pass-parallel3"],
-                historical_durations=Dict{String, TestHistoryEntry}(),
+                historical_durations=Dict{String, Float64}(),
+                historical_failures=Set{String}(),
                 serial=["fail-serial", "pass-serial1", "pass-serial2"],
                 stdout=io,
                 stderr=io,
@@ -1250,7 +1246,8 @@ end
                 testsuite,
                 tests=["fail-parallel", "pass-parallel1", "pass-parallel2",
                        "pass-serial1", "pass-serial2"],
-                historical_durations=Dict{String, TestHistoryEntry}(),
+                historical_durations=Dict{String, Float64}(),
+                historical_failures=Set{String}(),
                 serial=["pass-serial1", "pass-serial2"],
                 serial_position=:after,
                 stdout=io,
@@ -1285,7 +1282,8 @@ end
                 testsuite,
                 tests=["pass-parallel1", "pass-parallel2",
                        "fail-serial", "pass-serial1", "pass-serial2"],
-                historical_durations=Dict{String, TestHistoryEntry}(),
+                historical_durations=Dict{String, Float64}(),
+                historical_failures=Set{String}(),
                 serial=["fail-serial", "pass-serial1", "pass-serial2"],
                 serial_position=:after,
                 stdout=io,
@@ -1323,12 +1321,12 @@ end
         end
 
         ParallelTestRunner.save_test_history(mod, Dict(
-            "long-serial-pass" => TestHistoryEntry(10.0, false),
-            "short-serial-fail" => TestHistoryEntry(1.0, true),
-            "long-pass" => TestHistoryEntry(10.0, false),
-            "mid-pass" => TestHistoryEntry(5.0, false),
-            "short-fail" => TestHistoryEntry(1.0, true),
-        ))
+            "long-serial-pass" => 10.0,
+            "short-serial-fail" => 1.0,
+            "long-pass" => 10.0,
+            "mid-pass" => 5.0,
+            "short-fail" => 1.0,
+        ), Set(["short-serial-fail", "short-fail"]))
 
         io = IOBuffer()
         runtests(

From fdedad4b15dd5a5c20baa38f4e77ac04065499b4 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 17:00:42 -0300
Subject: [PATCH 03/10] Fewer lines

---
 src/ParallelTestRunner.jl | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 5ff3c0d..5384bc4 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -529,25 +529,16 @@ function get_history_files(mod::Module)
 end
 function load_test_history(mod::Module)
     history_file, failed_history_file = get_history_files(mod)
-
-    hist = if isfile(history_file)
-        try
-            deserialize(history_file)::Dict{String, Float64}
-        catch e
-            @warn "Failed to load test history from $history_file" exception=e
-            Dict{String, Float64}()
-        end
-    else
+    hist = try
+        deserialize(history_file)::Dict{String, Float64}
+    catch e
+        @warn "Failed to load test history from $history_file" exception=e
         Dict{String, Float64}()
     end
-    failed_hist = if isfile(failed_history_file)
-        try
-            deserialize(failed_history_file)::Set{String}
-        catch e
-            @warn "Failed to load failed test history from $failed_history_file" exception=e
-            Set{String}()
-        end
-    else
+    failed_hist = try
+        deserialize(failed_history_file)::Set{String}
+    catch e
+        @warn "Failed to load failed test history from $failed_history_file" exception=e
         Set{String}()
     end
     return hist, failed_hist
@@ -1550,11 +1541,9 @@ function _runtests(mod::Module, args::ParsedArgs;
                 if result isa AbstractTestRecord
                     testset = result[]::DefaultTestSet
                     historical_durations[testname] = stop - start
-                    if anynonpass(testset)
-                        push!(historical_failures, testname)
-                    else
-                        delete!(historical_failures, testname)
-                    end
+                    # push to historical_failures on failure and delete on success
+                    push_or_delete! = anynonpass(testset) ? push! : delete!
+                    push_or_delete!(historical_failures, testname)
                 else
                     # If this test raised an exception that means the test runner itself had some problem,
                     # so we may have hit a segfault, deserialization errors or something similar.

From 2b6b25c1127f63879ecac5abaeb19584ed9b5a8c Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 17:16:50 -0300
Subject: [PATCH 04/10] Add default valuess to history arguments

---
 src/ParallelTestRunner.jl |  4 ++--
 test/runtests.jl          | 10 ----------
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 5384bc4..08be1b8 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1038,8 +1038,8 @@ end
 function _runtests(mod::Module, args::ParsedArgs;
                    testsuite::Dict{String,Expr} = find_tests(pwd()),
                    tests::Vector{String},
-                   historical_durations::Dict{String, Float64},
-                   historical_failures::Set{String},
+                   historical_durations::Dict{String, Float64} = Dict{String, Float64}(),
+                   historical_failures::Set{String} = Set{String}(),
                    init_code = :(),
                    init_worker_code = :(),
                    test_worker = Returns(nothing),
diff --git a/test/runtests.jl b/test/runtests.jl
index 3a52c53..9eb4c92 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -102,8 +102,6 @@ end
         ParallelTestRunner, parse_args(["--jobs=1", "--verbose"]);
         testsuite,
         tests,
-        historical_durations=Dict{String, Float64}(),
-        historical_failures=Set{String}(),
         stdout=io,
         stderr=io,
     )
@@ -888,8 +886,6 @@ end
             ParallelTestRunner, parse_args(["--quickfail", "--verbose", "--jobs=1"]);
             testsuite,
             tests=["fail-test", "pass-test1", "pass-test2", "pass-test3", "pass-test4", "pass-test5"],
-            historical_durations=Dict{String, Float64}(),
-            historical_failures=Set{String}(),
             stdout=io,
             stderr=io,
         )
@@ -1210,8 +1206,6 @@ end
                 testsuite,
                 tests=["fail-serial", "pass-serial1", "pass-serial2",
                        "pass-parallel1", "pass-parallel2", "pass-parallel3"],
-                historical_durations=Dict{String, Float64}(),
-                historical_failures=Set{String}(),
                 serial=["fail-serial", "pass-serial1", "pass-serial2"],
                 stdout=io,
                 stderr=io,
@@ -1246,8 +1240,6 @@ end
                 testsuite,
                 tests=["fail-parallel", "pass-parallel1", "pass-parallel2",
                        "pass-serial1", "pass-serial2"],
-                historical_durations=Dict{String, Float64}(),
-                historical_failures=Set{String}(),
                 serial=["pass-serial1", "pass-serial2"],
                 serial_position=:after,
                 stdout=io,
@@ -1282,8 +1274,6 @@ end
                 testsuite,
                 tests=["pass-parallel1", "pass-parallel2",
                        "fail-serial", "pass-serial1", "pass-serial2"],
-                historical_durations=Dict{String, Float64}(),
-                historical_failures=Set{String}(),
                 serial=["fail-serial", "pass-serial1", "pass-serial2"],
                 serial_position=:after,
                 stdout=io,

From 9cad646bce0ae6d9ce1666ffcb1bcf794ae65f0c Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 17:20:59 -0300
Subject: [PATCH 05/10] Tweak docstrings

---
 src/ParallelTestRunner.jl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 08be1b8..88356c1 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -514,7 +514,7 @@ function default_njobs(; cpu_threads = Sys.CPU_THREADS, free_memory = available_
     return max(1, min(jobs, memory_jobs))
 end
 
-# Struct to make sorting test history entries easier
+# Struct used in runtests to sort failed tests before successful ones
 struct TestHistoryEntry <: AbstractFloat
     duration::Float64
     failed::Bool
@@ -522,6 +522,7 @@ end
 # successful tests are sorted before failed ones, so they always run first
 Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
 
+# Historical test duration database
 function get_history_files(mod::Module)
     scratch_dir = @get_scratch!("durations")
     path_base = joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)")

From 32d72018d6a74677e03f4683bc62855e87d952f2 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 18:10:18 -0300
Subject: [PATCH 06/10] Update test/runtests.jl
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Mosè Giordano <765740+giordano@users.noreply.github.com>
---
 test/runtests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index 9eb4c92..f63cf57 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -52,7 +52,7 @@ include(joinpath(@__DIR__, "utils.jl"))
         @test contains(str, "(%)")
     end
 
-    @test all(isfile.(ParallelTestRunner.get_history_files(ParallelTestRunner)))
+    @test all(isfile, ParallelTestRunner.get_history_files(ParallelTestRunner))
 end
 
 @testset "default njobs" begin

From 08b3ffe105232e97379a89f8f73d7b871c210894 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 18:22:25 -0300
Subject: [PATCH 07/10] Address feedback

---
 src/ParallelTestRunner.jl | 40 +++++++++++++++------------------------
 test/runtests.jl          |  6 +++---
 2 files changed, 18 insertions(+), 28 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 88356c1..680ed29 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -523,41 +523,31 @@ end
 Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
 
 # Historical test duration database
-function get_history_files(mod::Module)
+function get_history_file(mod::Module)
+    # History file version. Change when modifying the history format
+    hist_ver = "v2"
     scratch_dir = @get_scratch!("durations")
-    path_base = joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)")
-    return joinpath(path_base, "$(nameof(mod)).jls"), joinpath(path_base, "$(nameof(mod))_failed.jls")
+    return joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)", hist_ver, "$(nameof(mod)).jls")
 end
 function load_test_history(mod::Module)
-    history_file, failed_history_file = get_history_files(mod)
-    hist = try
-        deserialize(history_file)::Dict{String, Float64}
-    catch e
-        @warn "Failed to load test history from $history_file" exception=e
-        Dict{String, Float64}()
-    end
-    failed_hist = try
-        deserialize(failed_history_file)::Set{String}
-    catch e
-        @warn "Failed to load failed test history from $failed_history_file" exception=e
-        Set{String}()
+    history_file = get_history_file(mod)
+    if isfile(history_file)
+        try
+            return deserialize(history_file)::Tuple{Dict{String, Float64}, Set{String}}
+        catch e
+            @warn "Failed to load test history from $history_file" exception=e
+        end
     end
-    return hist, failed_hist
+    return (Dict{String, Float64}(), Set{String}())
 end
-function save_test_history(mod::Module, history::Dict{String, Float64}, failed_tests::Set{String})
-    history_file, failed_history_file = get_history_files(mod)
+function save_test_history(mod::Module, history::Tuple{Dict{String, Float64}, Set{String}})
+    history_file = get_history_file(mod)
     try
         mkpath(dirname(history_file))
         serialize(history_file, history)
     catch e
         @warn "Failed to save test history to $history_file" exception=e
     end
-    try
-        mkpath(dirname(failed_history_file))
-        serialize(failed_history_file, failed_tests)
-    catch e
-        @warn "Failed to save test failures to $failed_history_file" exception=e
-    end
 end
 
 function test_exe(color::Bool=false)
@@ -1585,7 +1575,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             Test.TESTSET_PRINT_ENABLE[] = old_print_setting
         end
     end
-    save_test_history(mod, historical_durations, historical_failures)
+    save_test_history(mod, (historical_durations, historical_failures))
 
     # display the results
     println(io_ctx.stdout)
diff --git a/test/runtests.jl b/test/runtests.jl
index f63cf57..637df31 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -52,7 +52,7 @@ include(joinpath(@__DIR__, "utils.jl"))
         @test contains(str, "(%)")
     end
 
-    @test all(isfile, ParallelTestRunner.get_history_files(ParallelTestRunner))
+    @test all(isfile, ParallelTestRunner.get_history_file(ParallelTestRunner))
 end
 
 @testset "default njobs" begin
@@ -1310,13 +1310,13 @@ end
             contains(k, "serial")
         end
 
-        ParallelTestRunner.save_test_history(mod, Dict(
+        ParallelTestRunner.save_test_history(mod, (Dict(
             "long-serial-pass" => 10.0,
             "short-serial-fail" => 1.0,
             "long-pass" => 10.0,
             "mid-pass" => 5.0,
             "short-fail" => 1.0,
-        ), Set(["short-serial-fail", "short-fail"]))
+        ), Set(["short-serial-fail", "short-fail"])))
 
         io = IOBuffer()
         runtests(

From f53cf38ce14aaab5aa03499c7d7b7193f7e05146 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 18:52:34 -0300
Subject: [PATCH 08/10] Fix test

---
 test/runtests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index 637df31..8186009 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -52,7 +52,7 @@ include(joinpath(@__DIR__, "utils.jl"))
         @test contains(str, "(%)")
     end
 
-    @test all(isfile, ParallelTestRunner.get_history_file(ParallelTestRunner))
+    @test ParallelTestRunner.get_history_file(ParallelTestRunner)
 end
 
 @testset "default njobs" begin

From 95da78da0cbe138f41c5bd947473c83092206829 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 18:53:38 -0300
Subject: [PATCH 09/10] Actually fix test

---
 test/runtests.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index 8186009..7968f07 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -52,7 +52,7 @@ include(joinpath(@__DIR__, "utils.jl"))
         @test contains(str, "(%)")
     end
 
-    @test ParallelTestRunner.get_history_file(ParallelTestRunner)
+    @test isfile(ParallelTestRunner.get_history_file(ParallelTestRunner))
 end
 
 @testset "default njobs" begin

From 99feec001f285e907b884f5e319488272c992df1 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 3 Aug 2026 19:06:23 -0300
Subject: [PATCH 10/10] Address feedback

---
 src/ParallelTestRunner.jl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 680ed29..1d64b8d 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -515,11 +515,12 @@ function default_njobs(; cpu_threads = Sys.CPU_THREADS, free_memory = available_
 end
 
 # Struct used in runtests to sort failed tests before successful ones
-struct TestHistoryEntry <: AbstractFloat
+struct TestHistoryEntry
     duration::Float64
     failed::Bool
 end
-# successful tests are sorted before failed ones, so they always run first
+# successful tests < failed tests, so when reversing the
+# sort they are also in proper descending order
 Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
 
 # Historical test duration database