require 'rails_helper'
describe Agents::HumanTaskAgent do
before do
@checker = Agents::HumanTaskAgent.new(name: 'my human task agent')
@checker.options = @checker.default_options
@checker.user = users(:bob)
@checker.save!
@event = Event.new
@event.agent = agents(:bob_rain_notifier_agent)
@event.payload = { 'foo' => { 'bar' => { 'baz' => 'a2b' } },
'name' => 'Joe' }
@event.id = 345
expect(@checker).to be_valid
end
describe 'validations' do
it "validates that trigger_on is 'schedule' or 'event'" do
@checker.options['trigger_on'] = 'foo'
expect(@checker).not_to be_valid
end
it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do
@checker.options['trigger_on'] = 'event'
@checker.options['expected_receive_period_in_days'] = nil
expect(@checker).not_to be_valid
@checker.options['expected_receive_period_in_days'] = 2
expect(@checker).to be_valid
end
it "requires a positive submission_period when trigger_on is set to 'schedule'" do
@checker.options['trigger_on'] = 'schedule'
@checker.options['submission_period'] = nil
expect(@checker).not_to be_valid
@checker.options['submission_period'] = 2
expect(@checker).to be_valid
end
it 'requires a hit.title' do
@checker.options['hit']['title'] = ''
expect(@checker).not_to be_valid
end
it 'requires a hit.description' do
@checker.options['hit']['description'] = ''
expect(@checker).not_to be_valid
end
it 'requires hit.assignments' do
@checker.options['hit']['assignments'] = ''
expect(@checker).not_to be_valid
@checker.options['hit']['assignments'] = 0
expect(@checker).not_to be_valid
@checker.options['hit']['assignments'] = 'moose'
expect(@checker).not_to be_valid
@checker.options['hit']['assignments'] = '2'
expect(@checker).to be_valid
end
it 'requires hit.questions' do
old_questions = @checker.options['hit']['questions']
@checker.options['hit']['questions'] = nil
expect(@checker).not_to be_valid
@checker.options['hit']['questions'] = []
expect(@checker).not_to be_valid
@checker.options['hit']['questions'] = [old_questions[0]]
expect(@checker).to be_valid
end
it 'requires that all questions have key, name, required, type, and question' do
old_questions = @checker.options['hit']['questions']
@checker.options['hit']['questions'].first['key'] = ''
expect(@checker).not_to be_valid
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['name'] = ''
expect(@checker).not_to be_valid
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['required'] = nil
expect(@checker).not_to be_valid
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['type'] = ''
expect(@checker).not_to be_valid
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['question'] = ''
expect(@checker).not_to be_valid
end
it "requires that all questions of type 'selection' have a selections array with keys and text" do
@checker.options['hit']['questions'][0]['selections'] = []
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][0]['selections'] = [{}]
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => '', 'text' => '' }]
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => '', 'text' => 'hi' }]
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => 'hi', 'text' => '' }]
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => 'hi', 'text' => 'hi' }]
expect(@checker).to be_valid
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => 'hi', 'text' => 'hi' }, {}]
expect(@checker).not_to be_valid
end
it "requires that 'poll_options' be present and populated when 'combination_mode' is set to 'poll'" do
@checker.options['combination_mode'] = 'poll'
expect(@checker).not_to be_valid
@checker.options['poll_options'] = {}
expect(@checker).not_to be_valid
@checker.options['poll_options'] = { 'title' => 'Take a poll about jokes',
'instructions' => 'Rank these by how funny they are',
'assignments' => 3,
'row_template' => '{{joke}}' }
expect(@checker).to be_valid
@checker.options['poll_options'] = { 'instructions' => 'Rank these by how funny they are',
'assignments' => 3,
'row_template' => '{{joke}}' }
expect(@checker).not_to be_valid
@checker.options['poll_options'] = { 'title' => 'Take a poll about jokes',
'assignments' => 3,
'row_template' => '{{joke}}' }
expect(@checker).not_to be_valid
@checker.options['poll_options'] = { 'title' => 'Take a poll about jokes',
'instructions' => 'Rank these by how funny they are',
'row_template' => '{{joke}}' }
expect(@checker).not_to be_valid
@checker.options['poll_options'] = { 'title' => 'Take a poll about jokes',
'instructions' => 'Rank these by how funny they are',
'assignments' => 3 }
expect(@checker).not_to be_valid
end
it "requires that all questions be of type 'selection' when 'combination_mode' is 'take_majority'" do
@checker.options['combination_mode'] = 'take_majority'
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][1]['type'] = 'selection'
@checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
expect(@checker).to be_valid
end
it "accepts 'take_majority': 'true' for legacy support" do
@checker.options['take_majority'] = 'true'
expect(@checker).not_to be_valid
@checker.options['hit']['questions'][1]['type'] = 'selection'
@checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
expect(@checker).to be_valid
end
end
describe "when 'trigger_on' is set to 'schedule'" do
before do
@checker.options['trigger_on'] = 'schedule'
@checker.options['submission_period'] = '2'
@checker.options.delete('expected_receive_period_in_days')
end
it 'should check for reviewable HITs frequently' do
expect(@checker).to receive(:review_hits).twice
expect(@checker).to receive(:create_basic_hit).once
@checker.check
@checker.check
end
it "should create HITs every 'submission_period' hours" do
now = Time.now
allow(Time).to receive(:now) { now }
expect(@checker).to receive(:review_hits).exactly(3).times
expect(@checker).to receive(:create_basic_hit).twice
@checker.check
now += 1 * 60 * 60
@checker.check
now += 1 * 60 * 60
@checker.check
end
it 'should ignore events' do
expect(@checker).not_to receive(:create_basic_hit).with(anything)
@checker.receive([events(:bob_website_agent_event)])
end
end
describe "when 'trigger_on' is set to 'event'" do
it 'should not create HITs during check but should check for reviewable HITs' do
@checker.options['submission_period'] = '2'
now = Time.now
allow(Time).to receive(:now) { now }
expect(@checker).to receive(:review_hits).exactly(3).times
expect(@checker).not_to receive(:create_basic_hit)
@checker.check
now += 1 * 60 * 60
@checker.check
now += 1 * 60 * 60
@checker.check
end
it 'should create HITs based on events' do
expect(@checker).to receive(:create_basic_hit).with(events(:bob_website_agent_event)).once
@checker.receive([events(:bob_website_agent_event)])
end
end
describe 'creating hits' do
it 'can create HITs based on events, interpolating their values' do
@checker.options['hit']['title'] = 'Hi {{name}}'
@checker.options['hit']['description'] = 'Make something for {{name}}'
@checker.options['hit']['questions'][0]['name'] = '{{name}} Question 1'
question_form = nil
hit_interface = double('hit_interface', id: 123, url: 'https://')
allow(hit_interface).to receive(:question_form).with(instance_of(Agents::HumanTaskAgent::AgentQuestionForm)) { |agent_question_form_instance|
question_form = agent_question_form_instance
}
allow(hit_interface).to receive(:max_assignments=).with(@checker.options['hit']['assignments'])
allow(hit_interface).to receive(:description=).with('Make something for Joe')
allow(hit_interface).to receive(:lifetime=)
allow(hit_interface).to receive(:reward=).with(@checker.options['hit']['reward'])
expect(RTurk::Hit).to receive(:create).with(title: 'Hi Joe').and_yield(hit_interface).and_return(hit_interface)
@checker.send :create_basic_hit, @event
xml = question_form.to_xml
expect(xml).to include('
Hi Joe')
expect(xml).to include('Make something for Joe')
expect(xml).to include('Joe Question 1')
expect(@checker.memory['hits'][123]['event_id']).to eq(@event.id)
end
it 'works without an event too' do
@checker.options['hit']['title'] = 'Hi {{name}}'
hit_interface = double('hit_interface', id: 123, url: 'https://')
allow(hit_interface).to receive(:question_form).with(instance_of(Agents::HumanTaskAgent::AgentQuestionForm))
allow(hit_interface).to receive(:max_assignments=).with(@checker.options['hit']['assignments'])
allow(hit_interface).to receive(:description=)
allow(hit_interface).to receive(:lifetime=)
allow(hit_interface).to receive(:reward=).with(@checker.options['hit']['reward'])
expect(RTurk::Hit).to receive(:create).with(title: 'Hi').and_yield(hit_interface).and_return(hit_interface)
@checker.send :create_basic_hit
end
end
describe 'reviewing HITs' do
class FakeHit
def initialize(options = {})
@options = options
end
def assignments
@options[:assignments] || []
end
def max_assignments
@options[:max_assignments] || 1
end
def dispose!
@disposed = true
end
def disposed?
@disposed
end
end
class FakeAssignment
attr_accessor :approved
def initialize(options = {})
@options = options
end
def answers
@options[:answers] || {}
end
def status
@options[:status] || ''
end
def approve!
@approved = true
end
end
it 'should work on multiple HITs' do
event2 = Event.new
event2.agent = agents(:bob_rain_notifier_agent)
event2.payload = { 'foo2' => { 'bar2' => { 'baz2' => 'a2b2' } },
'name2' => 'Joe2' }
event2.id = 3452
# It knows about two HITs from two different events.
@checker.memory['hits'] = {}
@checker.memory['hits']['JH3132836336DHG'] = { 'event_id' => @event.id }
@checker.memory['hits']['JH39AA63836DHG'] = { 'event_id' => event2.id }
hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
expect(RTurk::GetReviewableHITs).to receive(:create) { double(hit_ids:) } # It sees 3 HITs.
# It looksup the two HITs that it owns. Neither are ready yet.
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { FakeHit.new }
expect(RTurk::Hit).to receive(:new).with('JH39AA63836DHG') { FakeHit.new }
@checker.send :review_hits
end
it "shouldn't do anything if an assignment isn't ready" do
@checker.memory['hits'] = { 'JH3132836336DHG' => { 'event_id' => @event.id } }
expect(RTurk::GetReviewableHITs).to receive(:create) {
double(hit_ids: %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345])
}
assignments = [
FakeAssignment.new(status: 'Accepted', answers: {}),
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'happy', 'feedback' => 'Take 2' })
]
hit = FakeHit.new(max_assignments: 2, assignments:)
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { hit }
# One of the assignments isn't set to "Submitted", so this should get skipped for now.
expect_any_instance_of(FakeAssignment).not_to receive(:answers)
@checker.send :review_hits
expect(assignments.all? { |a| a.approved == true }).to be_falsey
expect(@checker.memory['hits']).to eq({ 'JH3132836336DHG' => { 'event_id' => @event.id } })
end
it "shouldn't do anything if an assignment is missing" do
@checker.memory['hits'] = { 'JH3132836336DHG' => { 'event_id' => @event.id } }
expect(RTurk::GetReviewableHITs).to receive(:create) {
double(hit_ids: %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345])
}
assignments = [
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'happy', 'feedback' => 'Take 2' })
]
hit = FakeHit.new(max_assignments: 2, assignments:)
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { hit }
# One of the assignments hasn't shown up yet, so this should get skipped for now.
expect_any_instance_of(FakeAssignment).not_to receive(:answers)
@checker.send :review_hits
expect(assignments.all? { |a| a.approved == true }).to be_falsey
expect(@checker.memory['hits']).to eq({ 'JH3132836336DHG' => { 'event_id' => @event.id } })
end
context 'emitting events' do
before do
@checker.memory['hits'] = { 'JH3132836336DHG' => { 'event_id' => @event.id } }
expect(RTurk::GetReviewableHITs).to receive(:create) {
double(hit_ids: %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345])
}
@assignments = [
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'neutral', 'feedback' => '' }),
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'happy', 'feedback' => 'Take 2' })
]
@hit = FakeHit.new(max_assignments: 2, assignments: @assignments)
expect(@hit).not_to be_disposed
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { @hit }
end
it 'should create events when all assignments are ready' do
expect do
@checker.send :review_hits
end.to change { Event.count }.by(1)
expect(@assignments.all? { |a| a.approved == true }).to be_truthy
expect(@hit).to be_disposed
expect(@checker.events.last.payload['answers']).to eq([
{ 'sentiment' => 'neutral', 'feedback' => '' },
{ 'sentiment' => 'happy', 'feedback' => 'Take 2' }
])
expect(@checker.memory['hits']).to eq({})
end
it 'should emit separate answers when options[:separate_answers] is true' do
@checker.options[:separate_answers] = true
expect do
@checker.send :review_hits
end.to change { Event.count }.by(2)
expect(@assignments.all? { |a| a.approved == true }).to be_truthy
expect(@hit).to be_disposed
event1, event2 = @checker.events.last(2)
expect(event1.payload).not_to have_key('answers')
expect(event2.payload).not_to have_key('answers')
expect(event1.payload['answer']).to eq({ 'sentiment' => 'happy', 'feedback' => 'Take 2' })
expect(event2.payload['answer']).to eq({ 'sentiment' => 'neutral', 'feedback' => '' })
expect(@checker.memory['hits']).to eq({})
end
end
describe 'taking majority votes' do
before do
@checker.options['combination_mode'] = 'take_majority'
@checker.memory['hits'] = { 'JH3132836336DHG' => { 'event_id' => @event.id } }
expect(RTurk::GetReviewableHITs).to receive(:create) {
double(hit_ids: %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345])
}
end
it 'should take the majority votes of all questions' do
@checker.options['hit']['questions'][1] = {
'type' => 'selection',
'key' => 'age_range',
'name' => 'Age Range',
'required' => 'true',
'question' => 'Please select your age range:',
'selections' =>
[
{ 'key' => '<50', 'text' => '50 years old or younger' },
{ 'key' => '>50', 'text' => 'Over 50 years old' }
]
}
assignments = [
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'sad', 'age_range' => '<50' }),
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'neutral', 'age_range' => '>50' }),
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'happy', 'age_range' => '>50' }),
FakeAssignment.new(status: 'Submitted', answers: { 'sentiment' => 'happy', 'age_range' => '>50' })
]
hit = FakeHit.new(max_assignments: 4, assignments:)
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { hit }
expect do
@checker.send :review_hits
end.to change { Event.count }.by(1)
expect(assignments.all? { |a| a.approved == true }).to be_truthy
expect(@checker.events.last.payload['answers']).to eq([
{ 'sentiment' => 'sad', 'age_range' => '<50' },
{ 'sentiment' => 'neutral', 'age_range' => '>50' },
{ 'sentiment' => 'happy', 'age_range' => '>50' },
{ 'sentiment' => 'happy', 'age_range' => '>50' }
])
expect(@checker.events.last.payload['counts']).to eq({
'sentiment' => { 'happy' => 2, 'sad' => 1,
'neutral' => 1 }, 'age_range' => { '>50' => 3, '<50' => 1 }
})
expect(@checker.events.last.payload['majority_answer']).to eq({ 'sentiment' => 'happy', 'age_range' => '>50' })
expect(@checker.events.last.payload).not_to have_key('average_answer')
expect(@checker.memory['hits']).to eq({})
end
it 'should also provide an average answer when all questions are numeric' do
# it should accept 'take_majority': 'true' as well for legacy support. Demonstrating that here.
@checker.options.delete :combination_mode
@checker.options['take_majority'] = 'true'
@checker.options['hit']['questions'] = [
{
'type' => 'selection',
'key' => 'rating',
'name' => 'Rating',
'required' => 'true',
'question' => 'Please select a rating:',
'selections' =>
[
{ 'key' => '1', 'text' => 'One' },
{ 'key' => '2', 'text' => 'Two' },
{ 'key' => '3', 'text' => 'Three' },
{ 'key' => '4', 'text' => 'Four' },
{ 'key' => '5.1', 'text' => 'Five Point One' }
]
}
]
assignments = [
FakeAssignment.new(status: 'Submitted', answers: { 'rating' => '1' }),
FakeAssignment.new(status: 'Submitted', answers: { 'rating' => '3' }),
FakeAssignment.new(status: 'Submitted', answers: { 'rating' => '5.1' }),
FakeAssignment.new(status: 'Submitted', answers: { 'rating' => '2' }),
FakeAssignment.new(status: 'Submitted', answers: { 'rating' => '2' })
]
hit = FakeHit.new(max_assignments: 5, assignments:)
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { hit }
expect do
@checker.send :review_hits
end.to change { Event.count }.by(1)
expect(assignments.all? { |a| a.approved == true }).to be_truthy
expect(@checker.events.last.payload['answers']).to eq([
{ 'rating' => '1' },
{ 'rating' => '3' },
{ 'rating' => '5.1' },
{ 'rating' => '2' },
{ 'rating' => '2' }
])
expect(@checker.events.last.payload['counts']).to eq({ 'rating' => { '1' => 1, '2' => 2, '3' => 1, '4' => 0,
'5.1' => 1 } })
expect(@checker.events.last.payload['majority_answer']).to eq({ 'rating' => '2' })
expect(@checker.events.last.payload['average_answer']).to eq({ 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 })
expect(@checker.memory['hits']).to eq({})
end
end
describe 'creating and reviewing polls' do
before do
@checker.options['combination_mode'] = 'poll'
@checker.options['poll_options'] = {
'title' => 'Hi!',
'instructions' => 'hello!',
'assignments' => 2,
'row_template' => 'This is {{sentiment}}'
}
@event.save!
expect(RTurk::GetReviewableHITs).to receive(:create) {
double(hit_ids: %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345])
}
end
it 'creates a poll using the row_template, message, and correct number of assignments' do
@checker.memory['hits'] = { 'JH3132836336DHG' => { 'event_id' => @event.id } }
# Mock out the HIT's submitted assignments.
assignments = [
FakeAssignment.new(status: 'Submitted',
answers: { 'sentiment' => 'sad',
'feedback' => 'This is my feedback 1' }),
FakeAssignment.new(status: 'Submitted',
answers: { 'sentiment' => 'neutral',
'feedback' => 'This is my feedback 2' }),
FakeAssignment.new(status: 'Submitted',
answers: { 'sentiment' => 'happy',
'feedback' => 'This is my feedback 3' }),
FakeAssignment.new(status: 'Submitted',
answers: { 'sentiment' => 'happy',
'feedback' => 'This is my feedback 4' })
]
hit = FakeHit.new(max_assignments: 4, assignments:)
expect(RTurk::Hit).to receive(:new).with('JH3132836336DHG') { hit }
expect(@checker.memory['hits']['JH3132836336DHG']).to be_present
# Setup mocks for HIT creation
question_form = nil
hit_interface = double('hit_interface', id: 'JH39AA63836DH12345', url: 'https://')
allow(hit_interface).to receive(:question_form).with(instance_of(Agents::HumanTaskAgent::AgentQuestionForm)) { |agent_question_form_instance|
question_form = agent_question_form_instance
}
allow(hit_interface).to receive(:max_assignments=).with(@checker.options['poll_options']['assignments'])
allow(hit_interface).to receive(:description=).with(@checker.options['poll_options']['instructions'])
allow(hit_interface).to receive(:lifetime=)
allow(hit_interface).to receive(:reward=).with(@checker.options['hit']['reward'])
expect(RTurk::Hit).to receive(:create).with(title: 'Hi!').and_yield(hit_interface).and_return(hit_interface)
# And finally, the test.
# it does not emit an event until all poll results are in
expect do
@checker.send :review_hits
end.to change { Event.count }.by(0)
# it approves the existing assignments
expect(assignments.all? { |a| a.approved == true }).to be_truthy
expect(hit).to be_disposed
# it creates a new HIT for the poll
xml = question_form.to_xml
expect(xml).to include('This is happy')
expect(xml).to include('This is neutral')
expect(xml).to include('This is sad')
@checker.save
@checker.reload
expect(@checker.memory['hits']['JH3132836336DHG']).not_to be_present
expect(@checker.memory['hits']['JH39AA63836DH12345']).to be_present
expect(@checker.memory['hits']['JH39AA63836DH12345']['event_id']).to eq(@event.id)
expect(@checker.memory['hits']['JH39AA63836DH12345']['type']).to eq('poll')
expect(@checker.memory['hits']['JH39AA63836DH12345']['original_hit']).to eq('JH3132836336DHG')
expect(@checker.memory['hits']['JH39AA63836DH12345']['answers'].length).to eq(4)
end
it 'emits an event when all poll results are in, containing the data from the best answer, plus all others' do
original_answers = [
{ 'sentiment' => 'sad', 'feedback' => 'This is my feedback 1' },
{ 'sentiment' => 'neutral', 'feedback' => 'This is my feedback 2' },
{ 'sentiment' => 'happy', 'feedback' => 'This is my feedback 3' },
{ 'sentiment' => 'happy', 'feedback' => 'This is my feedback 4' }
]
@checker.memory['hits'] = {
'JH39AA63836DH12345' => {
'type' => 'poll',
'original_hit' => 'JH3132836336DHG',
'answers' => original_answers,
'event_id' => 345
}
}
# Mock out the HIT's submitted assignments.
assignments = [
FakeAssignment.new(status: 'Submitted', answers: { '1' => '2', '2' => '5', '3' => '3', '4' => '2' }),
FakeAssignment.new(status: 'Submitted', answers: { '1' => '3', '2' => '4', '3' => '1', '4' => '4' })
]
hit = FakeHit.new(max_assignments: 2, assignments:)
expect(RTurk::Hit).to receive(:new).with('JH39AA63836DH12345') { hit }
expect(@checker.memory['hits']['JH39AA63836DH12345']).to be_present
expect do
@checker.send :review_hits
end.to change { Event.count }.by(1)
# It emits an event
expect(@checker.events.last.payload['answers']).to eq(original_answers)
expect(@checker.events.last.payload['poll']).to eq([{ '1' => '2', '2' => '5', '3' => '3', '4' => '2' },
{ '1' => '3', '2' => '4', '3' => '1', '4' => '4' }])
expect(@checker.events.last.payload['best_answer']).to eq({ 'sentiment' => 'neutral',
'feedback' => 'This is my feedback 2' })
# it approves the existing assignments
expect(assignments.all? { |a| a.approved == true }).to be_truthy
expect(hit).to be_disposed
expect(@checker.memory['hits']).to be_empty
end
end
end
end