{"id":{"repo_id":"uiuc","oai_identifier":"oai:www.ideals.illinois.edu:2142/45484"},"canonical_url":"https://search.dev.ndltd.org/etd/uiuc/oai:www.ideals.illinois.edu:2142/45484","repository":{"repo_id":"uiuc","name":"University of Illinois - Urbana-Champaign","base_url":"https://www.ideals.illinois.edu/oai-pmh"},"display":{"title":"Peer consistency evaluation in human computation","abstract":"Peer consistency evaluation is often used in games with a purpose (GWAP) to evaluate workers using outputs of other workers without using gold standard answers. Despite its popularity, the reliability of peer consistency evaluation has never been systematically tested to show how it can be used as a general evaluation method in human computation systems. We present experimental results that show that human computation systems using peer consistency evaluation can lead to outcomes that are even better than those that evaluate workers using gold standard answers. We also show that even without evaluation, simply telling the workers that their answers will be used as future evaluation standards can significantly enhance the workers' performance. Results have important implication for methods that improve the reliability of human computation systems.","abstract_html":"Peer consistency evaluation is often used in games with a purpose (GWAP) to evaluate workers using outputs of other workers without using gold standard answers. Despite its popularity, the reliability of peer consistency evaluation has never been systematically tested to show how it can be used as a general evaluation method in human computation systems. We present experimental results that show that human computation systems using peer consistency evaluation can lead to outcomes that are even better than those that evaluate workers using gold standard answers. We also show that even without evaluation, simply telling the workers that their answers will be used as future evaluation standards can significantly enhance the workers' performance. Results have important implication for methods that improve the reliability of human computation systems.","abstract_has_math":false,"creators":["Huang, Shih-Wen"],"institution":"University of Illinois at Urbana-Champaign","degree_name":"M.S.","degree_level":"Thesis","degree_discipline":"Computer Science","degree_department":null,"school":null,"contributors":["Fu, Wai-Tat"],"advisors":[],"committee_chairs":[],"committee_members":[],"year":2013,"date_issued":"2013-08-22T16:41:36Z","date_published":"2013-08-22T16:41:36Z","updated_at":"2026-07-22T22:25:36Z","subjects":["crowdsourcing","human computation"],"languages":["en"],"rights":["Copyright 2013 Shih-Wen Huang"],"rights_urls":[],"identifier_entries":[]},"links":{"outbound_url":"http://hdl.handle.net/2142/45484","outbound_label":"Handle","outbound_source":"dc:identifier"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor","label":"Contributor","values":["Fu, Wai-Tat"]},{"key":"dc:creator","label":"Author","values":["Huang, Shih-Wen"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date","label":"Dc Date","values":["2013-08-22T16:41:36Z","2013-08"]},{"key":"dc:type","label":"Dc Type","values":["text"]},{"key":"thesis:degree_discipline","label":"Discipline","values":["Computer Science"]},{"key":"thesis:degree_level","label":"Degree Level","values":["Thesis"]},{"key":"thesis:degree_name","label":"Degree Name","values":["M.S."]},{"key":"thesis:institution_name","label":"Thesis Institution Name","values":["University of Illinois at Urbana-Champaign"]}]},{"id":"subjects_keywords","label":"Subjects and Keywords","entries":[{"key":"dc:subject","label":"Dc Subject","values":["crowdsourcing","human computation"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:language","label":"Dc Language","values":["en"]},{"key":"dc:rights","label":"Dc Rights","values":["Copyright 2013 Shih-Wen Huang"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier","label":"Identifier","values":["http://hdl.handle.net/2142/45484"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description","label":"Description","values":["Peer consistency evaluation is often used in games with a purpose (GWAP) to evaluate workers using outputs of other workers without using gold standard answers. Despite its popularity, the reliability of peer consistency evaluation has never been systematically tested to show how it can be used as a general evaluation method in human computation systems. We present experimental results that show that human computation systems using peer consistency evaluation can lead to outcomes that are even better than those that evaluate workers using gold standard answers. We also show that even without evaluation, simply telling the workers that their answers will be used as future evaluation standards can significantly enhance the workers' performance. Results have important implication for methods that improve the reliability of human computation systems.","Item withdrawn by Mark Zulauf (zulauf@illinois.edu) on 2013-07-18T13:08:20Z Item was in collections: University of Illinois Theses & Dissertations (ID: 1) No. of bitstreams: 1 Huang_Shih-Wen.pdf: 663704 bytes, checksum: e56dfac50e3ed80dce8e92b6276a1b2c (MD5)","Made available in DSpace on 2013-08-22T16:41:36Z (GMT). No. of bitstreams: 2 Shih-Wen_Huang.pdf: 663704 bytes, checksum: e56dfac50e3ed80dce8e92b6276a1b2c (MD5) license.txt: 4064 bytes, checksum: dc66a071322548a9a3d72e111fefc518 (MD5)"]},{"key":"dc:title","label":"Title","values":["Peer consistency evaluation in human computation"]}]}],"canonical_facts":{"dc:contributor":["Fu, Wai-Tat"],"dc:creator":["Huang, Shih-Wen"],"dc:date":["2013-08-22T16:41:36Z","2013-08"],"dc:description":["Peer consistency evaluation is often used in games with a purpose (GWAP) to evaluate workers using outputs of other workers without using gold standard answers. Despite its popularity, the reliability of peer consistency evaluation has never been systematically tested to show how it can be used as a general evaluation method in human computation systems. We present experimental results that show that human computation systems using peer consistency evaluation can lead to outcomes that are even better than those that evaluate workers using gold standard answers. We also show that even without evaluation, simply telling the workers that their answers will be used as future evaluation standards can significantly enhance the workers' performance. Results have important implication for methods that improve the reliability of human computation systems.","Item withdrawn by Mark Zulauf (zulauf@illinois.edu) on 2013-07-18T13:08:20Z Item was in collections: University of Illinois Theses & Dissertations (ID: 1) No. of bitstreams: 1 Huang_Shih-Wen.pdf: 663704 bytes, checksum: e56dfac50e3ed80dce8e92b6276a1b2c (MD5)","Made available in DSpace on 2013-08-22T16:41:36Z (GMT). No. of bitstreams: 2 Shih-Wen_Huang.pdf: 663704 bytes, checksum: e56dfac50e3ed80dce8e92b6276a1b2c (MD5) license.txt: 4064 bytes, checksum: dc66a071322548a9a3d72e111fefc518 (MD5)"],"dc:identifier":["http://hdl.handle.net/2142/45484"],"dc:language":["en"],"dc:rights":["Copyright 2013 Shih-Wen Huang"],"dc:subject":["crowdsourcing","human computation"],"dc:title":["Peer consistency evaluation in human computation"],"dc:type":["text"],"thesis:degree_discipline":["Computer Science"],"thesis:degree_level":["Thesis"],"thesis:degree_name":["M.S."],"thesis:institution_name":["University of Illinois at Urbana-Champaign"]},"updated_at":"2026-07-22T22:25:36Z"}