human_task_agent_spec.rb 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615
  1. require 'rails_helper'
  2. describe Agents::HumanTaskAgent do
  3. before do
  4. @checker = Agents::HumanTaskAgent.new(:name => "my human task agent")
  5. @checker.options = @checker.default_options
  6. @checker.user = users(:bob)
  7. @checker.save!
  8. @event = Event.new
  9. @event.agent = agents(:bob_rain_notifier_agent)
  10. @event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" } },
  11. 'name' => "Joe" }
  12. @event.id = 345
  13. expect(@checker).to be_valid
  14. end
  15. describe "validations" do
  16. it "validates that trigger_on is 'schedule' or 'event'" do
  17. @checker.options['trigger_on'] = "foo"
  18. expect(@checker).not_to be_valid
  19. end
  20. it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do
  21. @checker.options['trigger_on'] = "event"
  22. @checker.options['expected_receive_period_in_days'] = nil
  23. expect(@checker).not_to be_valid
  24. @checker.options['expected_receive_period_in_days'] = 2
  25. expect(@checker).to be_valid
  26. end
  27. it "requires a positive submission_period when trigger_on is set to 'schedule'" do
  28. @checker.options['trigger_on'] = "schedule"
  29. @checker.options['submission_period'] = nil
  30. expect(@checker).not_to be_valid
  31. @checker.options['submission_period'] = 2
  32. expect(@checker).to be_valid
  33. end
  34. it "requires a hit.title" do
  35. @checker.options['hit']['title'] = ""
  36. expect(@checker).not_to be_valid
  37. end
  38. it "requires a hit.description" do
  39. @checker.options['hit']['description'] = ""
  40. expect(@checker).not_to be_valid
  41. end
  42. it "requires hit.assignments" do
  43. @checker.options['hit']['assignments'] = ""
  44. expect(@checker).not_to be_valid
  45. @checker.options['hit']['assignments'] = 0
  46. expect(@checker).not_to be_valid
  47. @checker.options['hit']['assignments'] = "moose"
  48. expect(@checker).not_to be_valid
  49. @checker.options['hit']['assignments'] = "2"
  50. expect(@checker).to be_valid
  51. end
  52. it "requires hit.questions" do
  53. old_questions = @checker.options['hit']['questions']
  54. @checker.options['hit']['questions'] = nil
  55. expect(@checker).not_to be_valid
  56. @checker.options['hit']['questions'] = []
  57. expect(@checker).not_to be_valid
  58. @checker.options['hit']['questions'] = [old_questions[0]]
  59. expect(@checker).to be_valid
  60. end
  61. it "requires that all questions have key, name, required, type, and question" do
  62. old_questions = @checker.options['hit']['questions']
  63. @checker.options['hit']['questions'].first['key'] = ""
  64. expect(@checker).not_to be_valid
  65. @checker.options['hit']['questions'] = old_questions
  66. @checker.options['hit']['questions'].first['name'] = ""
  67. expect(@checker).not_to be_valid
  68. @checker.options['hit']['questions'] = old_questions
  69. @checker.options['hit']['questions'].first['required'] = nil
  70. expect(@checker).not_to be_valid
  71. @checker.options['hit']['questions'] = old_questions
  72. @checker.options['hit']['questions'].first['type'] = ""
  73. expect(@checker).not_to be_valid
  74. @checker.options['hit']['questions'] = old_questions
  75. @checker.options['hit']['questions'].first['question'] = ""
  76. expect(@checker).not_to be_valid
  77. end
  78. it "requires that all questions of type 'selection' have a selections array with keys and text" do
  79. @checker.options['hit']['questions'][0]['selections'] = []
  80. expect(@checker).not_to be_valid
  81. @checker.options['hit']['questions'][0]['selections'] = [{}]
  82. expect(@checker).not_to be_valid
  83. @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "" }]
  84. expect(@checker).not_to be_valid
  85. @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "hi" }]
  86. expect(@checker).not_to be_valid
  87. @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "" }]
  88. expect(@checker).not_to be_valid
  89. @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }]
  90. expect(@checker).to be_valid
  91. @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }, {}]
  92. expect(@checker).not_to be_valid
  93. end
  94. it "requires that 'poll_options' be present and populated when 'combination_mode' is set to 'poll'" do
  95. @checker.options['combination_mode'] = "poll"
  96. expect(@checker).not_to be_valid
  97. @checker.options['poll_options'] = {}
  98. expect(@checker).not_to be_valid
  99. @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
  100. 'instructions' => "Rank these by how funny they are",
  101. 'assignments' => 3,
  102. 'row_template' => "{{joke}}" }
  103. expect(@checker).to be_valid
  104. @checker.options['poll_options'] = { 'instructions' => "Rank these by how funny they are",
  105. 'assignments' => 3,
  106. 'row_template' => "{{joke}}" }
  107. expect(@checker).not_to be_valid
  108. @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
  109. 'assignments' => 3,
  110. 'row_template' => "{{joke}}" }
  111. expect(@checker).not_to be_valid
  112. @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
  113. 'instructions' => "Rank these by how funny they are",
  114. 'row_template' => "{{joke}}" }
  115. expect(@checker).not_to be_valid
  116. @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
  117. 'instructions' => "Rank these by how funny they are",
  118. 'assignments' => 3}
  119. expect(@checker).not_to be_valid
  120. end
  121. it "requires that all questions be of type 'selection' when 'combination_mode' is 'take_majority'" do
  122. @checker.options['combination_mode'] = "take_majority"
  123. expect(@checker).not_to be_valid
  124. @checker.options['hit']['questions'][1]['type'] = "selection"
  125. @checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
  126. expect(@checker).to be_valid
  127. end
  128. it "accepts 'take_majority': 'true' for legacy support" do
  129. @checker.options['take_majority'] = "true"
  130. expect(@checker).not_to be_valid
  131. @checker.options['hit']['questions'][1]['type'] = "selection"
  132. @checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
  133. expect(@checker).to be_valid
  134. end
  135. end
  136. describe "when 'trigger_on' is set to 'schedule'" do
  137. before do
  138. @checker.options['trigger_on'] = "schedule"
  139. @checker.options['submission_period'] = "2"
  140. @checker.options.delete('expected_receive_period_in_days')
  141. end
  142. it "should check for reviewable HITs frequently" do
  143. mock(@checker).review_hits.twice
  144. mock(@checker).create_basic_hit.once
  145. @checker.check
  146. @checker.check
  147. end
  148. it "should create HITs every 'submission_period' hours" do
  149. now = Time.now
  150. stub(Time).now { now }
  151. mock(@checker).review_hits.times(3)
  152. mock(@checker).create_basic_hit.twice
  153. @checker.check
  154. now += 1 * 60 * 60
  155. @checker.check
  156. now += 1 * 60 * 60
  157. @checker.check
  158. end
  159. it "should ignore events" do
  160. mock(@checker).create_basic_hit(anything).times(0)
  161. @checker.receive([events(:bob_website_agent_event)])
  162. end
  163. end
  164. describe "when 'trigger_on' is set to 'event'" do
  165. it "should not create HITs during check but should check for reviewable HITs" do
  166. @checker.options['submission_period'] = "2"
  167. now = Time.now
  168. stub(Time).now { now }
  169. mock(@checker).review_hits.times(3)
  170. mock(@checker).create_basic_hit.times(0)
  171. @checker.check
  172. now += 1 * 60 * 60
  173. @checker.check
  174. now += 1 * 60 * 60
  175. @checker.check
  176. end
  177. it "should create HITs based on events" do
  178. mock(@checker).create_basic_hit(events(:bob_website_agent_event)).times(1)
  179. @checker.receive([events(:bob_website_agent_event)])
  180. end
  181. end
  182. describe "creating hits" do
  183. it "can create HITs based on events, interpolating their values" do
  184. @checker.options['hit']['title'] = "Hi {{name}}"
  185. @checker.options['hit']['description'] = "Make something for {{name}}"
  186. @checker.options['hit']['questions'][0]['name'] = "{{name}} Question 1"
  187. question_form = nil
  188. hitInterface = OpenStruct.new
  189. hitInterface.id = 123
  190. mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
  191. mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }
  192. @checker.send :create_basic_hit, @event
  193. expect(hitInterface.max_assignments).to eq(@checker.options['hit']['assignments'])
  194. expect(hitInterface.reward).to eq(@checker.options['hit']['reward'])
  195. expect(hitInterface.description).to eq("Make something for Joe")
  196. xml = question_form.to_xml
  197. expect(xml).to include("<Title>Hi Joe</Title>")
  198. expect(xml).to include("<Text>Make something for Joe</Text>")
  199. expect(xml).to include("<DisplayName>Joe Question 1</DisplayName>")
  200. expect(@checker.memory['hits'][123]['event_id']).to eq(@event.id)
  201. end
  202. it "works without an event too" do
  203. @checker.options['hit']['title'] = "Hi {{name}}"
  204. hitInterface = OpenStruct.new
  205. hitInterface.id = 123
  206. mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
  207. mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
  208. @checker.send :create_basic_hit
  209. expect(hitInterface.max_assignments).to eq(@checker.options['hit']['assignments'])
  210. expect(hitInterface.reward).to eq(@checker.options['hit']['reward'])
  211. end
  212. end
  213. describe "reviewing HITs" do
  214. class FakeHit
  215. def initialize(options = {})
  216. @options = options
  217. end
  218. def assignments
  219. @options[:assignments] || []
  220. end
  221. def max_assignments
  222. @options[:max_assignments] || 1
  223. end
  224. def dispose!
  225. @disposed = true
  226. end
  227. def disposed?
  228. @disposed
  229. end
  230. end
  231. class FakeAssignment
  232. attr_accessor :approved
  233. def initialize(options = {})
  234. @options = options
  235. end
  236. def answers
  237. @options[:answers] || {}
  238. end
  239. def status
  240. @options[:status] || ""
  241. end
  242. def approve!
  243. @approved = true
  244. end
  245. end
  246. it "should work on multiple HITs" do
  247. event2 = Event.new
  248. event2.agent = agents(:bob_rain_notifier_agent)
  249. event2.payload = { 'foo2' => { "bar2" => { 'baz2' => "a2b2" } },
  250. 'name2' => "Joe2" }
  251. event2.id = 3452
  252. # It knows about two HITs from two different events.
  253. @checker.memory['hits'] = {}
  254. @checker.memory['hits']["JH3132836336DHG"] = { 'event_id' => @event.id }
  255. @checker.memory['hits']["JH39AA63836DHG"] = { 'event_id' => event2.id }
  256. hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
  257. mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
  258. # It looksup the two HITs that it owns. Neither are ready yet.
  259. mock(RTurk::Hit).new("JH3132836336DHG") { FakeHit.new }
  260. mock(RTurk::Hit).new("JH39AA63836DHG") { FakeHit.new }
  261. @checker.send :review_hits
  262. end
  263. it "shouldn't do anything if an assignment isn't ready" do
  264. @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
  265. mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
  266. assignments = [
  267. FakeAssignment.new(:status => "Accepted", :answers => {}),
  268. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
  269. ]
  270. hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
  271. mock(RTurk::Hit).new("JH3132836336DHG") { hit }
  272. # One of the assignments isn't set to "Submitted", so this should get skipped for now.
  273. mock.any_instance_of(FakeAssignment).answers.times(0)
  274. @checker.send :review_hits
  275. expect(assignments.all? {|a| a.approved == true }).to be_falsey
  276. expect(@checker.memory['hits']).to eq({ "JH3132836336DHG" => { 'event_id' => @event.id } })
  277. end
  278. it "shouldn't do anything if an assignment is missing" do
  279. @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
  280. mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
  281. assignments = [
  282. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
  283. ]
  284. hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
  285. mock(RTurk::Hit).new("JH3132836336DHG") { hit }
  286. # One of the assignments hasn't shown up yet, so this should get skipped for now.
  287. mock.any_instance_of(FakeAssignment).answers.times(0)
  288. @checker.send :review_hits
  289. expect(assignments.all? {|a| a.approved == true }).to be_falsey
  290. expect(@checker.memory['hits']).to eq({ "JH3132836336DHG" => { 'event_id' => @event.id } })
  291. end
  292. context "emitting events" do
  293. before do
  294. @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
  295. mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
  296. @assignments = [
  297. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
  298. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
  299. ]
  300. @hit = FakeHit.new(:max_assignments => 2, :assignments => @assignments)
  301. expect(@hit).not_to be_disposed
  302. mock(RTurk::Hit).new("JH3132836336DHG") { @hit }
  303. end
  304. it "should create events when all assignments are ready" do
  305. expect {
  306. @checker.send :review_hits
  307. }.to change { Event.count }.by(1)
  308. expect(@assignments.all? {|a| a.approved == true }).to be_truthy
  309. expect(@hit).to be_disposed
  310. expect(@checker.events.last.payload['answers']).to eq([
  311. {'sentiment' => "neutral", 'feedback' => ""},
  312. {'sentiment' => "happy", 'feedback' => "Take 2"}
  313. ])
  314. expect(@checker.memory['hits']).to eq({})
  315. end
  316. it "should emit separate answers when options[:separate_answers] is true" do
  317. @checker.options[:separate_answers] = true
  318. expect {
  319. @checker.send :review_hits
  320. }.to change { Event.count }.by(2)
  321. expect(@assignments.all? {|a| a.approved == true }).to be_truthy
  322. expect(@hit).to be_disposed
  323. event1, event2 = @checker.events.last(2)
  324. expect(event1.payload).not_to have_key('answers')
  325. expect(event2.payload).not_to have_key('answers')
  326. expect(event1.payload['answer']).to eq({ 'sentiment' => "happy", 'feedback' => "Take 2" })
  327. expect(event2.payload['answer']).to eq({ 'sentiment' => "neutral", 'feedback' => "" })
  328. expect(@checker.memory['hits']).to eq({})
  329. end
  330. end
  331. describe "taking majority votes" do
  332. before do
  333. @checker.options['combination_mode'] = "take_majority"
  334. @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
  335. mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
  336. end
  337. it "should take the majority votes of all questions" do
  338. @checker.options['hit']['questions'][1] = {
  339. 'type' => "selection",
  340. 'key' => "age_range",
  341. 'name' => "Age Range",
  342. 'required' => "true",
  343. 'question' => "Please select your age range:",
  344. 'selections' =>
  345. [
  346. { 'key' => "<50", 'text' => "50 years old or younger" },
  347. { 'key' => ">50", 'text' => "Over 50 years old" }
  348. ]
  349. }
  350. assignments = [
  351. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad", "age_range"=>"<50"}),
  352. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "age_range"=>">50"}),
  353. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "age_range"=>">50"}),
  354. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "age_range"=>">50"})
  355. ]
  356. hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
  357. mock(RTurk::Hit).new("JH3132836336DHG") { hit }
  358. expect {
  359. @checker.send :review_hits
  360. }.to change { Event.count }.by(1)
  361. expect(assignments.all? {|a| a.approved == true }).to be_truthy
  362. expect(@checker.events.last.payload['answers']).to eq([
  363. { 'sentiment' => "sad", 'age_range' => "<50" },
  364. { 'sentiment' => "neutral", 'age_range' => ">50" },
  365. { 'sentiment' => "happy", 'age_range' => ">50" },
  366. { 'sentiment' => "happy", 'age_range' => ">50" }
  367. ])
  368. expect(@checker.events.last.payload['counts']).to eq({ 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } })
  369. expect(@checker.events.last.payload['majority_answer']).to eq({ 'sentiment' => "happy", 'age_range' => ">50" })
  370. expect(@checker.events.last.payload).not_to have_key('average_answer')
  371. expect(@checker.memory['hits']).to eq({})
  372. end
  373. it "should also provide an average answer when all questions are numeric" do
  374. # it should accept 'take_majority': 'true' as well for legacy support. Demonstrating that here.
  375. @checker.options.delete :combination_mode
  376. @checker.options['take_majority'] = "true"
  377. @checker.options['hit']['questions'] = [
  378. {
  379. 'type' => "selection",
  380. 'key' => "rating",
  381. 'name' => "Rating",
  382. 'required' => "true",
  383. 'question' => "Please select a rating:",
  384. 'selections' =>
  385. [
  386. { 'key' => "1", 'text' => "One" },
  387. { 'key' => "2", 'text' => "Two" },
  388. { 'key' => "3", 'text' => "Three" },
  389. { 'key' => "4", 'text' => "Four" },
  390. { 'key' => "5.1", 'text' => "Five Point One" }
  391. ]
  392. }
  393. ]
  394. assignments = [
  395. FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"1" }),
  396. FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"3" }),
  397. FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"5.1" }),
  398. FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"2" }),
  399. FakeAssignment.new(:status => "Submitted", :answers => { "rating"=>"2" })
  400. ]
  401. hit = FakeHit.new(:max_assignments => 5, :assignments => assignments)
  402. mock(RTurk::Hit).new("JH3132836336DHG") { hit }
  403. expect {
  404. @checker.send :review_hits
  405. }.to change { Event.count }.by(1)
  406. expect(assignments.all? {|a| a.approved == true }).to be_truthy
  407. expect(@checker.events.last.payload['answers']).to eq([
  408. { 'rating' => "1" },
  409. { 'rating' => "3" },
  410. { 'rating' => "5.1" },
  411. { 'rating' => "2" },
  412. { 'rating' => "2" }
  413. ])
  414. expect(@checker.events.last.payload['counts']).to eq({ 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } })
  415. expect(@checker.events.last.payload['majority_answer']).to eq({ 'rating' => "2" })
  416. expect(@checker.events.last.payload['average_answer']).to eq({ 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 })
  417. expect(@checker.memory['hits']).to eq({})
  418. end
  419. end
  420. describe "creating and reviewing polls" do
  421. before do
  422. @checker.options['combination_mode'] = "poll"
  423. @checker.options['poll_options'] = {
  424. 'title' => "Hi!",
  425. 'instructions' => "hello!",
  426. 'assignments' => 2,
  427. 'row_template' => "This is {{sentiment}}"
  428. }
  429. @event.save!
  430. mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
  431. end
  432. it "creates a poll using the row_template, message, and correct number of assignments" do
  433. @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
  434. # Mock out the HIT's submitted assignments.
  435. assignments = [
  436. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad", "feedback"=>"This is my feedback 1"}),
  437. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>"This is my feedback 2"}),
  438. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"This is my feedback 3"}),
  439. FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"This is my feedback 4"})
  440. ]
  441. hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
  442. mock(RTurk::Hit).new("JH3132836336DHG") { hit }
  443. expect(@checker.memory['hits']["JH3132836336DHG"]).to be_present
  444. # Setup mocks for HIT creation
  445. question_form = nil
  446. hitInterface = OpenStruct.new
  447. hitInterface.id = "JH39AA63836DH12345"
  448. mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
  449. mock(RTurk::Hit).create(:title => "Hi!").yields(hitInterface) { hitInterface }
  450. # And finally, the test.
  451. expect {
  452. @checker.send :review_hits
  453. }.to change { Event.count }.by(0) # it does not emit an event until all poll results are in
  454. # it approves the existing assignments
  455. expect(assignments.all? {|a| a.approved == true }).to be_truthy
  456. expect(hit).to be_disposed
  457. # it creates a new HIT for the poll
  458. expect(hitInterface.max_assignments).to eq(@checker.options['poll_options']['assignments'])
  459. expect(hitInterface.description).to eq(@checker.options['poll_options']['instructions'])
  460. xml = question_form.to_xml
  461. expect(xml).to include("<Text>This is happy</Text>")
  462. expect(xml).to include("<Text>This is neutral</Text>")
  463. expect(xml).to include("<Text>This is sad</Text>")
  464. @checker.save
  465. @checker.reload
  466. expect(@checker.memory['hits']["JH3132836336DHG"]).not_to be_present
  467. expect(@checker.memory['hits']["JH39AA63836DH12345"]).to be_present
  468. expect(@checker.memory['hits']["JH39AA63836DH12345"]['event_id']).to eq(@event.id)
  469. expect(@checker.memory['hits']["JH39AA63836DH12345"]['type']).to eq("poll")
  470. expect(@checker.memory['hits']["JH39AA63836DH12345"]['original_hit']).to eq("JH3132836336DHG")
  471. expect(@checker.memory['hits']["JH39AA63836DH12345"]['answers'].length).to eq(4)
  472. end
  473. it "emits an event when all poll results are in, containing the data from the best answer, plus all others" do
  474. original_answers = [
  475. { 'sentiment' => "sad", 'feedback' => "This is my feedback 1"},
  476. { 'sentiment' => "neutral", 'feedback' => "This is my feedback 2"},
  477. { 'sentiment' => "happy", 'feedback' => "This is my feedback 3"},
  478. { 'sentiment' => "happy", 'feedback' => "This is my feedback 4"}
  479. ]
  480. @checker.memory['hits'] = {
  481. 'JH39AA63836DH12345' => {
  482. 'type' => 'poll',
  483. 'original_hit' => "JH3132836336DHG",
  484. 'answers' => original_answers,
  485. 'event_id' => 345
  486. }
  487. }
  488. # Mock out the HIT's submitted assignments.
  489. assignments = [
  490. FakeAssignment.new(:status => "Submitted", :answers => {"1" => "2", "2" => "5", "3" => "3", "4" => "2"}),
  491. FakeAssignment.new(:status => "Submitted", :answers => {"1" => "3", "2" => "4", "3" => "1", "4" => "4"})
  492. ]
  493. hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
  494. mock(RTurk::Hit).new("JH39AA63836DH12345") { hit }
  495. expect(@checker.memory['hits']["JH39AA63836DH12345"]).to be_present
  496. expect {
  497. @checker.send :review_hits
  498. }.to change { Event.count }.by(1)
  499. # It emits an event
  500. expect(@checker.events.last.payload['answers']).to eq(original_answers)
  501. expect(@checker.events.last.payload['poll']).to eq([{"1" => "2", "2" => "5", "3" => "3", "4" => "2"}, {"1" => "3", "2" => "4", "3" => "1", "4" => "4"}])
  502. expect(@checker.events.last.payload['best_answer']).to eq({'sentiment' => "neutral", 'feedback' => "This is my feedback 2"})
  503. # it approves the existing assignments
  504. expect(assignments.all? {|a| a.approved == true }).to be_truthy
  505. expect(hit).to be_disposed
  506. expect(@checker.memory['hits']).to be_empty
  507. end
  508. end
  509. end
  510. end