Basic bench and chairs using PerfChecker (#50)

* Basic bench and chairs

Update plots

New plots

New plots 2

* FIx format

perf/Project.toml

@@ -0,0 +1,8 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
+ConstraintCommons = "e37357d9-0691-492f-a822-e5ea6a920954"
+ConstraintDomains = "5800fd60-8556-4464-8d61-84ebf7a0bedb"
+Intervals = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
+PerfChecker = "6309bf6b-a531-4b08-891e-8ee981e5c424"

perf/checks.jl

@@ -0,0 +1,264 @@
+using BenchmarkTools
+using CairoMakie
+using Chairmarks
+using PerfChecker
+
+@info "Defining utilities"
+
+d_commons = Dict(
+    :targets => ["ConstraintDomains"],
+    :path => @__DIR__,
+    :pkgs => ("ConstraintDomains", :custom, [v"0.2.5", v"0.3.0", v"0.3.10"], true),
+    :seconds => 100,
+    :samples => 10,
+    :evals => 10,
+)
+
+## SECTION - Utilities
+tags(d) = mapreduce(x -> string(x), (y, z) -> y * "_" * z, d[:tags])
+
+function visu(x, d, ::Val{:allocs})
+    mkpath(joinpath(@__DIR__, "visuals"))
+    c = checkres_to_scatterlines(x, Val(:alloc))
+    save(joinpath(@__DIR__, "visuals", "allocs_evolution_$(tags(d)).png"), c)
+
+    for (name, c2) in checkres_to_pie(x, Val(:alloc))
+        save(joinpath(@__DIR__, "visuals", "allocs_pie_$(name)_$(tags(d)).png"), c2)
+    end
+end
+
+function visu(x, d, ::Val{:benchmark})
+    mkpath(joinpath(d[:path], "visuals"))
+    c = checkres_to_scatterlines(x, Val(:benchmark))
+    save(joinpath(d[:path], "visuals", "bench_evolution_$(tags(d)).png"), c)
+
+    for kwarg in [:times, :gctimes, :memory, :allocs]
+        c2 = checkres_to_boxplots(x, Val(:benchmark); kwarg)
+        save(joinpath(d[:path], "visuals", "bench_boxplots_$(kwarg)_$(tags(d)).png"), c2)
+    end
+end
+
+function visu(x, d, ::Val{:chairmark})
+    mkpath(joinpath(d[:path], "visuals"))
+    c = checkres_to_scatterlines(x, Val(:chairmark))
+    save(joinpath(d[:path], "visuals", "chair_evolution_$(tags(d)).png"), c)
+
+    for kwarg in [:times, :gctimes, :bytes, :allocs]
+        c2 = checkres_to_boxplots(x, Val(:chairmark); kwarg)
+        save(joinpath(d[:path], "visuals", "chair_boxplots_$(kwarg)_$(tags(d)).png"), c2)
+    end
+end
+
+## SECTION - Commons: benchmarks and chairmarks
+
+@info "Running checks: Commons"
+
+d = deepcopy(d_commons)
+d[:tags] = [:common]
+
+x = @check :benchmark d begin
+    using ConstraintDomains
+end begin
+    ed = domain()
+    domain_size(ed) == 0 == length(ed)
+    isempty(ed)
+    π ∉ ed
+end
+
+visu(x, d, Val(:benchmark))
+
+x = @check :chairmark d begin
+    using ConstraintDomains
+end begin
+    ed = domain()
+    domain_size(ed) == 0 == length(ed)
+    isempty(ed)
+    π ∉ ed
+end
+
+visu(x, d, Val(:chairmark))
+
+## SECTION - Continuous: benchmarks and chairmarks
+
+@info "Running checks: Continuous"
+
+d = deepcopy(d_commons)
+d[:tags] = [:continuous]
+
+x = @check :benchmark d begin
+    using ConstraintDomains
+    using Intervals
+end begin
+    if d[:current_version] < v"0.3.0"
+        d1 = domain((1.0, true), (3.15, true))
+        d2 = domain((-42.42, false), (5.0, false))
+    else
+        d1 = domain(1.0 .. 3.15)
+        d2 = domain(Interval{Open,Open}(-42.42, 5.0))
+    end
+    domains = [d1, d2]
+    for d in domains
+        for x in [1, 2.3, π]
+            x ∈ d
+        end
+        for x in [5.1, π^π, Inf]
+            x ∉ d
+        end
+        rand(d) ∈ d
+        rand(d, 1) ∈ d
+        domain_size(d) > 0.0
+    end
+end
+
+visu(x, d, Val(:benchmark))
+
+x = @check :chairmark d begin
+    using ConstraintDomains
+    using Intervals
+end begin
+    if d[:current_version] < v"0.3.0"
+        d1 = domain((1.0, true), (3.15, true))
+        d2 = domain((-42.42, false), (5.0, false))
+    else
+        d1 = domain(1.0 .. 3.15)
+        d2 = domain(Interval{Open,Open}(-42.42, 5.0))
+    end
+    domains = [d1, d2]
+    for d in domains
+        for x in [1, 2.3, π]
+            x ∈ d
+        end
+        for x in [5.1, π^π, Inf]
+            x ∉ d
+        end
+        rand(d) ∈ d
+        rand(d, 1) ∈ d
+        domain_size(d) > 0.0
+    end
+end
+
+visu(x, d, Val(:chairmark))
+
+## SECTION - Discrete: benchmarks and chairmarks
+
+@info "Running checks: Discrete"
+
+d = deepcopy(d_commons)
+d[:tags] = [:discrete]
+
+x = @check :benchmark d begin
+    using ConstraintDomains
+end begin
+    d1 = domain([4, 3, 2, 1])
+    d2 = domain(1)
+    foreach(i -> add!(d2, i), 2:4)
+    domains = [d1, d2]
+    for d in domains
+        for x in [1, 2, 3, 4]
+            x ∈ d
+        end
+        length(d) == 4
+        rand(d) ∈ d
+        add!(d, 5)
+        5 ∈ d
+        delete!(d, 5)
+        5 ∉ d
+        domain_size(d) == 3
+    end
+
+    d3 = domain(1:5)
+    d4 = domain(1:0.5:5)
+    domains2 = [d3, d4]
+    for d in domains2
+        for x in [1, 2, 3, 4, 5]
+            x ∈ d
+        end
+        for x in [42]
+            x ∉ d
+        end
+        rand(d) ∈ d
+    end
+end
+
+visu(x, d, Val(:benchmark))
+
+x = @check :chairmark d begin
+    using ConstraintDomains
+end begin
+    d1 = domain([4, 3, 2, 1])
+    d2 = domain(1)
+    foreach(i -> add!(d2, i), 2:4)
+    domains = [d1, d2]
+    for d in domains
+        for x in [1, 2, 3, 4]
+            x ∈ d
+        end
+        length(d) == 4
+        rand(d) ∈ d
+        add!(d, 5)
+        5 ∈ d
+        delete!(d, 5)
+        5 ∉ d
+        domain_size(d) == 3
+    end
+
+    d3 = domain(1:5)
+    d4 = domain(1:0.5:5)
+    domains2 = [d3, d4]
+    for d in domains2
+        for x in [1, 2, 3, 4, 5]
+            x ∈ d
+        end
+        for x in [42]
+            x ∉ d
+        end
+        rand(d) ∈ d
+    end
+end
+
+visu(x, d, Val(:chairmark))
+
+## SECTION - Explore: benchmarks and chairmarks
+
+@info "Running checks: Explore"
+
+d = deepcopy(d_commons)
+d[:tags] = [:explore]
+d[:pkgs] = ("ConstraintDomains", :custom, [v"0.3.1", v"0.3.10"], true)
+
+# x = @check :allocs d begin
+#     using ConstraintDomains
+# end begin
+#     domains = [domain([1, 2, 3, 4]) for i = 1:4]
+#     X, X̅ = explore(domains, allunique)
+#     length(X) == factorial(4)
+#     length(X̅) == 4^4 - factorial(4)
+# end
+
+# visu(x, d, Val(:allocs))
+
+x = @check :benchmark d begin
+    using ConstraintDomains
+end begin
+    domains = [domain([1, 2, 3, 4]) for i = 1:4]
+    X, X̅ = explore(domains, allunique)
+    length(X) == factorial(4)
+    length(X̅) == 4^4 - factorial(4)
+end
+
+visu(x, d, Val(:benchmark))
+
+x = @check :chairmark d begin
+    using ConstraintDomains
+end begin
+    domains = [domain([1, 2, 3, 4]) for i = 1:4]
+    X, X̅ = explore(domains, allunique)
+    length(X) == factorial(4)
+    length(X̅) == 4^4 - factorial(4)
+end
+
+visu(x, d, Val(:chairmark))
+
+# TODO: add more checks for parameters.jl
+
+@info "All checks have been successfully run!"

perf/metadata/metadata.csv

@@ -0,0 +1,22 @@
+benchmark_ConstraintDomains_v0.2.5_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.0_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.10_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.2.5_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.0_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.10_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.2.5_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.0_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.10_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.2.5_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.0_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.10_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.2.5_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.0_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.10_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.2.5_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.0_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.10_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.1_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
+benchmark_ConstraintDomains_v0.3.10_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.1_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
+chairmark_ConstraintDomains_v0.3.10_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62

perf/output/01d3078e-3e06-59b8-b7a8-13146a1184de.csv

@@ -0,0 +1,11 @@
+times,gctimes,bytes,allocs
+4.7375e-6,0.0,1632.0,19.0
+3.3084000000000004e-6,0.0,1632.0,19.0
+1.9851e-6,0.0,1632.0,19.0
+2.0482e-6,0.0,1632.0,19.0
+1.8231000000000002e-6,0.0,1632.0,19.0
+1.7413000000000002e-6,0.0,1632.0,19.0
+1.7787e-6,0.0,1632.0,19.0
+1.7996e-6,0.0,1632.0,19.0
+1.8254000000000001e-6,0.0,1632.0,19.0
+1.7064e-6,0.0,1632.0,19.0

perf/output/039713ce-acec-5012-a235-f497b94aca3c.csv

@@ -0,0 +1,11 @@
+times,gctimes,memory,allocs
+4785.7,0.0,1632,19
+3461.3,0.0,1632,19
+2710.9,0.0,1632,19
+1916.1,0.0,1632,19
+2028.3,0.0,1632,19
+2050.8,0.0,1632,19
+1767.3,0.0,1632,19
+1749.7,0.0,1632,19
+1824.1,0.0,1632,19
+1808.7,0.0,1632,19

perf/output/06768c48-4ab9-5bb9-a034-980e923d7fac.csv

@@ -0,0 +1,11 @@
+times,gctimes,bytes,allocs
+4.9568000000000006e-6,0.0,864.0,27.0
+2.5383e-6,0.0,864.0,27.0
+1.3226e-6,0.0,864.0,27.0
+1.2310000000000002e-6,0.0,864.0,27.0
+1.1021e-6,0.0,864.0,27.0
+1.2353e-6,0.0,864.0,27.0
+1.1795e-6,0.0,864.0,27.0
+1.052e-6,0.0,864.0,27.0
+1.0814e-6,0.0,864.0,27.0
+1.0822e-6,0.0,864.0,27.0

perf/output/0d9e0caa-8170-543c-9628-2ad63c457e1f.csv

@@ -0,0 +1,11 @@
+times,gctimes,memory,allocs
+8.7,0.0,0,0
+24.0,0.0,0,0
+4.3,0.0,0,0
+4.9,0.0,0,0
+5.4,0.0,0,0
+3.8,0.0,0,0
+3.7,0.0,0,0
+3.8,0.0,0,0
+3.8,0.0,0,0
+4.6,0.0,0,0

perf/output/19661d6c-8c80-5bd8-bfd1-220f92ef21fb.csv

@@ -0,0 +1,11 @@
+times,gctimes,bytes,allocs
+5.193300000000001e-6,0.0,864.0,27.0
+2.4154e-6,0.0,864.0,27.0
+1.0913e-6,0.0,864.0,27.0
+1.1673000000000002e-6,0.0,864.0,27.0
+1.1393e-6,0.0,864.0,27.0
+1.1409e-6,0.0,864.0,27.0
+1.1716000000000001e-6,0.0,864.0,27.0
+1.1899e-6,0.0,864.0,27.0
+1.0990000000000002e-6,0.0,864.0,27.0
+1.1807e-6,0.0,864.0,27.0

perf/output/1c54877a-7fa4-53db-8195-00f829bc16a0.csv

@@ -0,0 +1,11 @@
+times,gctimes,memory,allocs
+40823.0,0.0,41392,309
+42830.1,0.0,41392,309
+38844.4,0.0,41392,309
+37552.7,0.0,41392,309
+36134.9,0.0,41392,309
+36296.1,0.0,41392,309
+35766.0,0.0,41392,309
+36251.5,0.0,41392,309
+35749.3,0.0,41392,309
+35883.3,0.0,41392,309

perf/output/3615ebfa-aa5f-5fc4-b45d-28558621f4bb.csv

@@ -0,0 +1,11 @@
+times,gctimes,bytes,allocs
+2.5900000000000004e-8,0.0,0.0,0.0
+2.49e-8,0.0,0.0,0.0
+5.2e-9,0.0,0.0,0.0
+5.4e-9,0.0,0.0,0.0
+3.9e-9,0.0,0.0,0.0
+3.8e-9,0.0,0.0,0.0
+3.9e-9,0.0,0.0,0.0
+3.8e-9,0.0,0.0,0.0
+3.7e-9,0.0,0.0,0.0
+3.8e-9,0.0,0.0,0.0

perf/output/431f4cb4-beae-56d3-b340-80762efacf8a.csv

@@ -0,0 +1,11 @@
+times,gctimes,bytes,allocs
+1.2562700000000001e-5,0.0,784.0,9.0
+1.14067e-5,0.0,784.0,9.0
+1.04715e-5,0.0,784.0,9.0
+1.04492e-5,0.0,784.0,9.0
+1.0432300000000002e-5,0.0,784.0,9.0
+1.04338e-5,0.0,784.0,9.0
+1.04493e-5,0.0,784.0,9.0
+1.13867e-5,0.0,784.0,9.0
+1.0136800000000001e-5,0.0,784.0,9.0
+1.0012500000000001e-5,0.0,784.0,9.0