Skip to content

Commit 813375d

Browse files
frankierfrankier
authored
Factor out ResponseExpectation from ExpectationBasedItemCriterion (#71)
* Put expectation related code in its own file * Factor out ResponseExpectation from ExpectationBasedItemCriterion * Update docs introduction of ExpectationBasedItemCriterion * Update benchmarks for addition of ResponseExpectation
1 parent a8786fc commit 813375d

File tree

6 files changed

+135
-118
lines changed

6 files changed

+135
-118
lines changed

‎benchmark/benchmarks.jl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ using ComputerAdaptiveTesting.Aggregators
99
using PsychometricsBazaarBase.Optimizers
1010
using PsychometricsBazaarBase.Integrators: even_grid
1111
using ComputerAdaptiveTesting.NextItemRules: mirtcat_quadpts
12-
using ComputerAdaptiveTesting.NextItemRules: DistributionExpectationBasedItemCriterion,
13-
PointExpectationBasedItemCriterion
12+
using ComputerAdaptiveTesting.NextItemRules: ExpectationBasedItemCriterion,
13+
PointResponseExpectation
1414
using ComputerAdaptiveTesting.NextItemRules
1515
using ComputerAdaptiveTesting.Responses
1616

@@ -40,7 +40,7 @@ function prepare_4pls(group)
4040
for (est_nick, ability_estimator) in ability_estimators
4141
next_item_rule = ItemStrategyNextItemRule(
4242
ExhaustiveSearch1Ply(),
43-
PointExpectationBasedItemCriterion(ability_estimator,
43+
ExpectationBasedItemCriterion(PointResponseExpectation(ability_estimator),
4444
AbilityVarianceStateCriterion(
4545
integrator, distribution_estimator(ability_estimator)))
4646
)

‎docs/src/creating_a_cat.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,12 @@ they are as a candidate for the next item. A typical example is
108108
and the item response function ```irf``` to calculate each item's information
109109
``\frac{irf_θ'^2}{irf_θ * (1 - irf_θ)}``.
110110

111-
Within this, there are two types of `ExpectationBasedItemCriterion` which act
112-
as adapters: `PointExpectationBasedItemCriterion` and
113-
`DistributionExpectationBasedItemCriterion`. These take a `StateCriterion`,
114-
which evaluates how good a particular state is in terms getting a good estimate
115-
of the test takers ability. They look one ply ahead to get the expected value
116-
of the ``StateCriterion`` after selecting the given item. The
111+
Within this, you can use `ExpectationBasedItemCriterion` as an adapter. It
112+
takes a `ResponseExpectation`: either `PointResponseExpectation` or
113+
`DistributionResponseExpectation` and a a `StateCriterion`, which evaluates how
114+
good a particular state is in terms getting a good estimate of the test takers
115+
ability. They look one ply ahead to get the expected value of the
116+
``StateCriterion`` after selecting the given item. The
117117
`AbilityVarianceStateCriterion` looks at the variance of the ability ``\theta``
118118
estimate at that state.
119119

‎src/next_item_rules/NextItemRules.jl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export ExhaustiveSearch1Ply
3939
export catr_next_item_aliases
4040
export preallocate
4141
export compute_criteria
42+
export PointResponseExpectation, DistributionResponseExpectation
4243

4344
"""
4445
$(TYPEDEF)
@@ -68,6 +69,7 @@ end
6869
include("./random.jl")
6970
include("./information.jl")
7071
include("./objective_function.jl")
72+
include("./expectation.jl")
7173

7274
const default_prior = IntegralCoeffs.Prior(Cauchy(5, 2))
7375

‎src/next_item_rules/expectation.jl‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#=
2+
This file implements next item combinators which take a state criterion
3+
or item criterion and look at its value in expectation with regards to
4+
a particular item.
5+
=#
6+
7+
abstract type ResponseExpectation end
8+
9+
function ResponseExpectation(ability_estimator::PointAbilityEstimator,
10+
bits...)
11+
PointResponseExpectation(ability_estimator)
12+
end
13+
14+
function ResponseExpectation(ability_estimator::DistributionAbilityEstimator,
15+
bits...)
16+
@returnsome Integrator(bits...) integrator->DistributionResponseExpectation(
17+
ability_estimator,
18+
integrator)
19+
end
20+
21+
"""
22+
$(TYPEDEF)
23+
$(TYPEDFIELDS)
24+
25+
This `ResponseExpectation` gets expected outcomes based on a point ability estimates.
26+
"""
27+
struct PointResponseExpectation{
28+
PointAbilityEstimatorT <: PointAbilityEstimator,
29+
} <: ResponseExpectation
30+
ability_estimator::PointAbilityEstimatorT
31+
end
32+
33+
function Aggregators.response_expectation(
34+
point_response_expectation::PointResponseExpectation,
35+
tracked_responses,
36+
item_idx)
37+
response_expectation(point_response_expectation.ability_estimator,
38+
tracked_responses,
39+
item_idx)
40+
end
41+
42+
struct DistributionResponseExpectation{
43+
DistributionAbilityEstimatorT <: DistributionAbilityEstimator,
44+
AbilityIntegratorT <: AbilityIntegrator
45+
} <: ResponseExpectation
46+
ability_estimator::DistributionAbilityEstimatorT
47+
integrator::AbilityIntegratorT
48+
end
49+
50+
function Aggregators.response_expectation(
51+
dist_response_expectation::DistributionResponseExpectation,
52+
tracked_responses,
53+
item_idx)
54+
response_expectation(dist_response_expectation.ability_estimator,
55+
dist_response_expectation.integrator,
56+
tracked_responses,
57+
item_idx)
58+
end
59+
60+
"""
61+
$(TYPEDEF)
62+
$(TYPEDFIELDS)
63+
64+
This ItemCriterion wraps a a `ResponseExpectation` and a `StateCriterion` or
65+
`ItemCriterion` to look at the criterion's expected value for a particular
66+
item 1-ply ahead.
67+
"""
68+
struct ExpectationBasedItemCriterion{
69+
ResponseExpectationT <: ResponseExpectation,
70+
CriterionT <: Union{StateCriterion, ItemCriterion}
71+
} <: ItemCriterion
72+
response_expectation::ResponseExpectationT
73+
criterion::CriterionT
74+
end
75+
76+
function _get_some_criterion(bits...; kwargs...)
77+
@returnsome StateCriterion(bits...; kwargs...)
78+
@returnsome ItemCriterion(bits...; kwargs...)
79+
end
80+
81+
function ExpectationBasedItemCriterion(bits...;
82+
ability_estimator = nothing,
83+
ability_tracker = nothing)
84+
@requiresome criterion = _get_some_criterion(
85+
bits...; ability_estimator = ability_estimator,
86+
ability_tracker = ability_tracker)
87+
@requiresome ability_estimator = AbilityEstimator(bits...,
88+
ability_estimator = ability_estimator,
89+
ability_tracker = ability_tracker)
90+
response_expectation = ResponseExpectation(ability_estimator, bits...)
91+
ExpectationBasedItemCriterion(response_expectation, criterion)
92+
end
93+
94+
function init_thread(::ExpectationBasedItemCriterion, responses::TrackedResponses)
95+
Speculator(responses, 1)
96+
end
97+
98+
function _generic_criterion(criterion::StateCriterion, tracked_responses, item_idx)
99+
criterion(tracked_responses)
100+
end
101+
# TODO: Support init_thread for wrapped ItemCriterion
102+
function _generic_criterion(criterion::ItemCriterion, tracked_responses, item_idx)
103+
criterion(tracked_responses, item_idx)
104+
end
105+
106+
function (item_criterion::ExpectationBasedItemCriterion)(speculator::Speculator,
107+
tracked_responses::TrackedResponses,
108+
item_idx)
109+
exp_resp = Aggregators.response_expectation(item_criterion.response_expectation,
110+
tracked_responses,
111+
item_idx)
112+
possible_responses = responses(ItemResponse(tracked_responses.item_bank, item_idx))
113+
res = 0.0
114+
for (prob, possible_response) in zip(exp_resp, possible_responses)
115+
replace_speculation!(speculator, SVector(item_idx), SVector(possible_response))
116+
res += prob *
117+
_generic_criterion(item_criterion.criterion, speculator.responses, item_idx)
118+
end
119+
res
120+
end

‎src/next_item_rules/ka.jl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@ function (
128128
rule_config::RuleConfigT where {
129129
RuleConfigT <: ItemStrategyNextItemRule{
130130
<:KernelAbstractionsExhaustiveSearchConfig,
131-
<:PointExpectationBasedItemCriterion{
132-
<:PointAbilityEstimator, <:AbilityVarianceStateCriterion}
131+
<:ExpectationBasedItemCriterion{
132+
<:PointResponseExpectation, <:AbilityVarianceStateCriterion}
133133
}
134134
}
135135
)(responses, items::DichotomousPointsWithLogsItemBank)
@@ -151,8 +151,8 @@ function (
151151
rule::RuleT where {
152152
RuleT <: ItemStrategyNextItemRule{
153153
<:KernelAbstractionsExhaustiveSearch,
154-
<:PointExpectationBasedItemCriterion{
155-
<:PointAbilityEstimator, <:AbilityVarianceStateCriterion}
154+
<:ExpectationBasedItemCriterion{
155+
<:PointResponseExpectation, <:AbilityVarianceStateCriterion}
156156
}
157157
}
158158
)(tracked_responses, items::DichotomousPointsWithLogsItemBank{})

‎src/next_item_rules/objective_function.jl‎

Lines changed: 0 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -109,111 +109,6 @@ function (criterion::AbilityVarianceStateCriterion)(::Vector,
109109
denom)
110110
end
111111

112-
abstract type ExpectationBasedItemCriterion <: ItemCriterion end
113-
114-
function ExpectationBasedItemCriterion(bits...;
115-
ability_estimator = nothing,
116-
ability_tracker = nothing)
117-
criterion = StateCriterion(bits...;
118-
ability_estimator = ability_estimator,
119-
ability_tracker = ability_tracker)
120-
if criterion === nothing
121-
return nothing
122-
end
123-
ability_estimator = AbilityEstimator(bits...,
124-
ability_estimator = ability_estimator,
125-
ability_tracker = ability_tracker)
126-
if ability_estimator === nothing
127-
return nothing
128-
end
129-
ExpectationBasedItemCriterion(ability_estimator, criterion, bits...)
130-
end
131-
132-
function ExpectationBasedItemCriterion(ability_estimator::PointAbilityEstimator,
133-
criterion::StateCriterion,
134-
bits...)
135-
PointExpectationBasedItemCriterion(ability_estimator, criterion)
136-
end
137-
138-
function ExpectationBasedItemCriterion(ability_estimator::DistributionAbilityEstimator,
139-
criterion::StateCriterion,
140-
bits...)
141-
@returnsome Integrator(bits...) integrator->DistributionExpectationBasedItemCriterion(
142-
ability_estimator,
143-
integrator,
144-
criterion)
145-
end
146-
147-
"""
148-
$(TYPEDEF)
149-
$(TYPEDFIELDS)
150-
151-
This ItemCriterion wraps a StateCriterion and looks at its expected value for a
152-
particular item 1-ply ahead based on a point ability estimate.
153-
"""
154-
struct PointExpectationBasedItemCriterion{
155-
PointAbilityEstimatorT <: PointAbilityEstimator,
156-
StateCriterionT <: StateCriterion
157-
} <: ExpectationBasedItemCriterion
158-
ability_estimator::PointAbilityEstimatorT
159-
state_criterion::StateCriterionT
160-
end
161-
162-
"""
163-
$(TYPEDEF)
164-
$(TYPEDFIELDS)
165-
166-
This ItemCriterion wraps a StateCriterion and looks at its expected value for a
167-
particular item 1-ply ahead by integrating over an ability curve.
168-
"""
169-
struct DistributionExpectationBasedItemCriterion{
170-
DistributionAbilityEstimatorT <: DistributionAbilityEstimator,
171-
AbilityIntegratorT <: AbilityIntegrator,
172-
StateCriterionT <: StateCriterion
173-
} <: ExpectationBasedItemCriterion
174-
ability_estimator::DistributionAbilityEstimatorT
175-
integrator::AbilityIntegratorT
176-
state_criterion::StateCriterionT
177-
end
178-
179-
function init_thread(::ExpectationBasedItemCriterion, responses::TrackedResponses)
180-
Speculator(responses, 1)
181-
end
182-
183-
function Aggregators.response_expectation(
184-
item_criterion::PointExpectationBasedItemCriterion,
185-
tracked_responses,
186-
item_idx)
187-
response_expectation(item_criterion.ability_estimator,
188-
tracked_responses,
189-
item_idx)
190-
end
191-
192-
function Aggregators.response_expectation(
193-
item_criterion::DistributionExpectationBasedItemCriterion,
194-
tracked_responses,
195-
item_idx)
196-
response_expectation(item_criterion.ability_estimator,
197-
item_criterion.integrator,
198-
tracked_responses,
199-
item_idx)
200-
end
201-
202-
function (item_criterion::ExpectationBasedItemCriterion)(speculator::Speculator,
203-
tracked_responses::TrackedResponses,
204-
item_idx)
205-
exp_resp = Aggregators.response_expectation(item_criterion,
206-
tracked_responses,
207-
item_idx)
208-
possible_responses = responses(ItemResponse(tracked_responses.item_bank, item_idx))
209-
res = 0.0
210-
for (prob, possible_response) in zip(exp_resp, possible_responses)
211-
replace_speculation!(speculator, SVector(item_idx), SVector(possible_response))
212-
res += prob * item_criterion.state_criterion(speculator.responses)
213-
end
214-
res
215-
end
216-
217112
"""
218113
$(TYPEDEF)
219114
$(TYPEDFIELDS)

0 commit comments

Comments
 (0)