Skip to content

Commit 6f80d5b

Browse files
dmbatesdmbates
authored
Update benchmarks to TestData. (#416)
1 parent 183c94a commit 6f80d5b

File tree

1 file changed

+87
-148
lines changed

1 file changed

+87
-148
lines changed

‎benchmark/benchmarks.jl‎

Lines changed: 87 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -1,171 +1,110 @@
1-
using BenchmarkTools, DataFrames, MixedModels, RData, Tables
1+
using BenchmarkTools, MixedModels, StatsModels
2+
using MixedModels: dataset
23

34
const SUITE = BenchmarkGroup()
45

5-
const dat = Dict(Symbol(k) => v for (k, v) in load(joinpath(
6-
dirname(pathof(MixedModels)),
7-
"..",
8-
"test",
9-
"dat.rda",
10-
)));
11-
12-
categorical!(dat[:ml1m], [:G,:H]); # forgot to convert these grouping factors
13-
14-
const mods = Dict{Symbol,Vector{Expr}}(
15-
:Alfalfa => [:(1 + A * B + (1 | G)), :(1 + A + B + (1 | G))],
16-
:Animal => [:(1 + (1 | G) + (1 | H))],
17-
:Arabidopsis => [], # glmm and rename variables
18-
:Assay => [:(1+A+B*C+(1|G)+(1|H))],
19-
:AvgDailyGain => [:(1 + A * U + (1 | G)), :(1 + A + U + (1 | G))],
20-
:BIB => [:(1 + A * U + (1 | G)), :(1 + A + U + (1 | G))],
21-
:Bond => [:(1 + A + (1 | G))],
22-
:Chem97 => [:(1 + (1 | G) + (1 | H)), :(1 + U + (1 | G) + (1 | H))],
23-
:Contraception => [], # glmm and rename variables
24-
:Cultivation => [:(1 + A * B + (1 | G)), :(1 + A + B + (1 | G)), :(1 + A + (1 | G))],
25-
:Demand => [:(1 + U + V + W + X + (1 | G) + (1 | H))],
26-
:Dyestuff => [:(1 + (1 | G))],
27-
:Dyestuff2 => [:(1 + (1 | G))],
28-
:Early => [:(1 + U + U & A + (1 + U | G))],
29-
:Exam => [:(1 + A * U + B + (1 | G)), :(1 + A + B + U + (1 | G))],
30-
:Gasoline => [:(1 + U + (1 | G))],
31-
:Gcsemv => [:(1 + A + (1 | G))], # variables must be renamed
32-
:Genetics => [:(1 + A + (1 | G) + (1 | H))],
33-
:HR => [:(1 + A * U + V + (1 + U | G))],
34-
:Hsb82 => [:(1 + A + B + C + U + (1 | G))],
35-
:IncBlk => [:(1 + A + U + + W + Z + (1 | G))],
36-
:InstEval => [:(1 + A + (1 | G) + (1 | H) + (1 | I)), :(1 + A * I + (1 | G) + (1 | H))],
37-
:KKL => [], # variables must be renamed
38-
:KWDYZ => [], # variables must be renamed
39-
:Mississippi => [:(1 + A + (1 | G))],
40-
:Mmmec => [], # glmm (and offset) and variables renamed
41-
:Multilocation => [:(1 + A + (0 + A | G) + (1 | H))],
42-
:Oxboys => [:(1 + U + (1 + U | G))],
43-
:PBIB => [:(1 + A + (1 | G))],
44-
:Pastes => [:(1 + (1 | G) + (1 | H))],
45-
:Penicillin => [:(1 + (1 | G) + (1 | H))],
46-
:Pixel => [:(1 + U + V + (1 + U | G) + (1 | H))], # variables must be renamed
47-
:Poems => [:(1 + U + V + W + (1 | G) + (1 | H) + (1 | I))],
48-
:Rail => [:(1 + (1 | G))],
49-
:SIMS => [:(1 + U + (1 + U | G))],
50-
:ScotsSec => [:(1 + A + U + V + (1 | G) + (1 | H))],
51-
:Semi2 => [:(1 + A + (1 | G) + (1 | H))],
52-
:Semiconductor => [:(1 + A * B + (1 | G))],
53-
:Socatt => [], # variables must be renamed - binomial glmm?
54-
:TeachingII => [:(1 + A + T + U + V + W + X + Z + (1 | G))],
55-
:VerbAgg => [:(1 + A + B + C + U + (1 | G) + (1 | H))], # Bernoulli glmm and rename variables
56-
:Weights => [:(1 + A * U + (1 + U | G))],
57-
:WWheat => [:(1 + U + (1 + U | G))],
58-
:bdf => [], # rename variables and look up model
59-
:bs10 => [:(1 + U + V + W + ((1 + U + V + W) | G) + ((1 + U + V + W) | H))],
60-
:cake => [:(1 + A * B + (1 | G))],
61-
:cbpp => [:(1 + A + (1 | G))], # Binomial glmm, create and rename variables
6+
const global contrasts = Dict(
7+
:mrk17_exp1 => merge(Dict(n => HelmertCoding() for n in (:F, :P, :Q, :lQ, :lT)),
8+
Dict(n => Grouping() for n in (:item, :subj))),
9+
)
10+
const global fms = Dict(
11+
:dyestuff => [
12+
@formula(yield ~ 1 + (1|batch)),
13+
],
14+
:dyestuff2 => [
15+
@formula(yield ~ 1 + (1|batch)),
16+
],
6217
:d3 => [
63-
:(1 + U + (1 | G) + (1 | H) + (1 | I)),
64-
:(1 + U + (1 + U | G) + (1 + U | H) + (1 + U | I)),
65-
],
66-
:dialectNL => [:(1 + A + T + U + V + W + X + (1 | G) + (1 | H) + (1 | I))],
67-
:egsingle => [:(1 + A + U + V + (1 | G) + (1 | H))],
68-
:epilepsy => [], # unknown origin
69-
:ergoStool => [:(1 + A + (1 | G))],
70-
:gb12 => [:(1 + S + T + U + V + W + X + Z + ((1 + S + U + W) | G) +
71-
((1 + S + T + V) | H))],
72-
:grouseticks => [], # rename variables, glmm needs formula
73-
:guImmun => [], # rename variables, glmm needs formula
74-
:guPrenat => [], # rename variables, glmm needs formula
18+
@formula(y ~ 1 + u + (1+u|g) + (1+u|h) + (1+u|i)),
19+
],
20+
:insteval => [
21+
@formula(y ~ 1 + service + (1|s) + (1|d) + (1|dept)),
22+
@formula(y ~ 1 + service*dept + (1|s) + (1|d)),
23+
],
7524
:kb07 => [
76-
:(1 + S + T + U + V + W + X + Z + ((1 + S + T + U + V + W + X + Z) | G) +
77-
((1 + S + T + U + V + W + X + Z) | H)),
78-
:(1 + S + T + U + V + W + X + Z +
79-
zerocorr((1 + S + T + U + V + W + X + Z) | G) +
80-
zerocorr((1 + S + T + U + V + W + X + Z) | H)),
81-
],
82-
:ml1m => [:(1 + (1 | G) + (1 | H))],
83-
:paulsim => [:(1 + S + T + U + (1 | H) + (1 | G))], # names of H and G should be reversed
84-
:sleepstudy => [:(1 + U + (1 + U | G)), :(1 + U + zerocorr(1 + U | G))],
85-
:s3bbx => [], # probably drop this one
86-
:star => [], # not sure it is worthwhile working with these data
87-
);
25+
@formula(rt_trunc ~ 1+spkr+prec+load+(1|subj)+(1|item)),
26+
@formula(rt_trunc ~ 1+spkr*prec*load+(1|subj)+(1+prec|item)),
27+
@formula(rt_trunc ~ 1+spkr*prec*load+(1+spkr+prec+load|subj)+(1+spkr+prec+load|item)),
28+
],
29+
:machines => [
30+
@formula(score ~ 1 + (1|Worker) + (1|Machine)),
31+
],
32+
:ml1m => [
33+
@formula(y ~ 1 + (1|g) + (1|h)),
34+
],
35+
:mrk17_exp1 => [
36+
@formula(1000/rt ~ 1+F*P*Q*lQ*lT + (1|item) + (1|subj)),
37+
@formula(1000/rt ~ 1+F*P*Q*lQ*lT + (1+P+Q+lQ+lT|item) + (1+F+P+Q+lQ+lT|subj)),
38+
],
39+
:pastes => [
40+
@formula(strength ~ 1 + (1|batch&cask)),
41+
@formula(strength ~ 1 + (1|batch/cask)),
42+
],
43+
:penicillin => [
44+
@formula(diameter ~ 1 + (1|plate) + (1|sample)),
45+
],
46+
:sleepstudy => [
47+
@formula(reaction ~ 1 + days + (1|subj)),
48+
@formula(reaction ~ 1 + days + zerocorr(1+days|subj)),
49+
@formula(reaction ~ 1 + days + (1|subj) + (0+days|subj)),
50+
@formula(reaction ~ 1 + days + (1+days|subj)),
51+
],
52+
)
8853

89-
fitbobyqa(rhs::Expr, dsname::Symbol) =
90-
fit(MixedModel, @eval(@formula(Y ~ $rhs)), dat[dsname])
91-
compactstr(ds, rhs) = replace(string(ds, ':', rhs), ' ' => "")
54+
function fitbobyqa(dsname::Symbol, index::Integer)
55+
fit(
56+
MixedModel,
57+
fms[dsname][index],
58+
dataset(dsname),
59+
contrasts=get!(contrasts, dsname, Dict{Symbol,StatsModels.AbstractContrasts}()),
60+
)
61+
end
9262

9363
SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"])
94-
for ds in [
95-
:Alfalfa,
96-
:AvgDailyGain,
97-
:BIB,
98-
:Bond,
99-
:cake,
100-
:Cultivation,
101-
:Dyestuff,
102-
:Dyestuff2,
103-
:ergoStool,
104-
:Exam,
105-
:Gasoline,
106-
:Hsb82,
107-
:IncBlk,
108-
:Mississippi,
109-
:PBIB,
110-
:Rail,
111-
:Semiconductor,
112-
:TeachingII,
64+
for (ds, i) in [
65+
(:dyestuff, 1,),
66+
(:dyestuff2, 1,),
67+
(:pastes, 1),
68+
(:sleepstudy, 1,),
11369
]
114-
for rhs in mods[ds]
115-
SUITE["simplescalar"][compactstr(ds, rhs)] = @benchmarkable fitbobyqa(
116-
$(QuoteNode(rhs)),
117-
$(QuoteNode(ds)),
118-
)
119-
end
70+
SUITE["simplescalar"][string(ds, ':', i)] = @benchmarkable fitbobyqa($(QuoteNode(ds)), $(QuoteNode(i)))
12071
end
12172

12273
SUITE["singlevector"] = BenchmarkGroup(["single", "vector"])
123-
for ds in [:Early, :HR, :Oxboys, :SIMS, :sleepstudy, :Weights, :WWheat]
124-
for rhs in mods[ds]
125-
SUITE["singlevector"][compactstr(ds, rhs)] = @benchmarkable fitbobyqa(
126-
$(QuoteNode(rhs)),
127-
$(QuoteNode(ds)),
128-
)
129-
end
74+
for (ds, i) in [
75+
(:sleepstudy, 2,),
76+
(:sleepstudy, 3,),
77+
(:sleepstudy, 4,),
78+
]
79+
SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($(QuoteNode(ds)), $(QuoteNode(i)))
13080
end
13181

13282
SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"])
133-
for ds in [:Animal, :Chem97, :Genetics, :Pastes, :Semi2]
134-
for rhs in mods[ds]
135-
SUITE["nested"][compactstr(ds, rhs)] = @benchmarkable fitbobyqa(
136-
$(QuoteNode(rhs)),
137-
$(QuoteNode(ds)),
138-
)
139-
end
83+
for (ds, i) in [
84+
(:pastes, 2,),
85+
]
86+
SUITE["nested"][string(ds, ':', i)] = @benchmarkable fitbobyqa($(QuoteNode(ds)), $(QuoteNode(i)))
14087
end
14188

14289
SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"])
143-
144-
for ds in [
145-
:Assay,
146-
:Demand,
147-
:InstEval,
148-
:Penicillin,
149-
:ScotsSec,
150-
:dialectNL,
151-
:egsingle,
152-
:ml1m,
153-
:paulsim,
90+
for (ds, i) in [
91+
(:insteval, 1),
92+
(:insteval, 2),
93+
(:kb07, 1),
94+
(:machines, 1),
95+
(:ml1m, 1),
96+
(:mrk17_exp1, 1),
97+
(:penicillin, 1),
15498
]
155-
for rhs in mods[ds]
156-
SUITE["crossed"][compactstr(ds, rhs)] = @benchmarkable fitbobyqa(
157-
$(QuoteNode(rhs)),
158-
$(QuoteNode(ds)),
159-
)
160-
end
99+
SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fitbobyqa($(QuoteNode(ds)), $(QuoteNode(i)))
161100
end
162101

163102
SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"])
164-
for ds in [:bs10, :d3, :gb12, :kb07]
165-
for rhs in mods[ds]
166-
SUITE["crossedvector"][compactstr(ds, rhs)] = @benchmarkable fitbobyqa(
167-
$(QuoteNode(rhs)),
168-
$(QuoteNode(ds)),
169-
)
170-
end
103+
for (ds, i) in [
104+
(:d3, 1),
105+
(:kb07, 2),
106+
(:kb07, 3),
107+
(:mrk17_exp1, 2),
108+
]
109+
SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($(QuoteNode(ds)), $(QuoteNode(i)))
171110
end

0 commit comments

Comments
 (0)