{"metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:00241663","sets":["1164:5159:11541:11870"]},"path":["11870"],"owner":"44499","recid":"241663","title":["Comparing Likert Scale and Pairwise Comparison for Human Evaluation in Rapport-Building Dialogue Systems"],"pubdate":{"attribute_name":"公開日","attribute_value":"2024-12-05"},"_buckets":{"deposit":"3ad949c7-dbb6-47e9-aa8d-2c35e868ba3e"},"_deposit":{"id":"241663","pid":{"type":"depid","value":"241663","revision_id":0},"owners":[44499],"status":"published","created_by":44499},"item_title":"Comparing Likert Scale and Pairwise Comparison for Human Evaluation in Rapport-Building Dialogue Systems","author_link":["665704","665702","665703","665709","665706","665708","665707","665705"],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"Comparing Likert Scale and Pairwise Comparison for Human Evaluation in Rapport-Building Dialogue Systems"},{"subitem_title":"Comparing Likert Scale and Pairwise Comparison for Human Evaluation in Rapport-Building Dialogue Systems","subitem_title_language":"en"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"話者認識・音声分析","subitem_subject_scheme":"Other"}]},"item_type_id":"4","publish_date":"2024-12-05","item_4_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"Nara Institute of Science and Technology/Guardian Robot Project RIKEN"},{"subitem_text_value":"Guardian Robot Project RIKEN"},{"subitem_text_value":"Guardian Robot Project RIKEN/Nara Institute of Science and Technology"},{"subitem_text_value":"Institute of Science Tokyo/Guardian Robot Project RIKEN/Nara Institute of Science and Technology"}]},"item_4_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"Nara Institute of Science and Technology / Guardian Robot Project RIKEN","subitem_text_language":"en"},{"subitem_text_value":"Guardian Robot Project RIKEN","subitem_text_language":"en"},{"subitem_text_value":"Guardian Robot Project RIKEN / Nara Institute of Science and Technology","subitem_text_language":"en"},{"subitem_text_value":"Institute of Science Tokyo / Guardian Robot Project RIKEN / Nara Institute of Science and Technology","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"eng"}]},"item_publisher":{"attribute_name":"出版者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/241663/files/IPSJ-SLP24154043.pdf","label":"IPSJ-SLP24154043.pdf"},"date":[{"dateType":"Available","dateValue":"2026-12-05"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-SLP24154043.pdf","filesize":[{"value":"2.2 MB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"660","billingrole":"5"},{"tax":["include_tax"],"price":"330","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"22"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"d03646ef-44fa-44d7-a67e-255ddbf5ff70","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2024 by the Information Processing Society of Japan"}]},"item_4_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Muhammad, Yeza Baihaqi"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Angel, García Contreras"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Seiya, Kawano"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Koichiro, Yoshino"}],"nameIdentifiers":[{}]}]},"item_4_creator_6":{"attribute_name":"著者名(英)","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Muhammad, Yeza Baihaqi","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Angel, García Contreras","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Seiya, Kawano","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Koichiro, Yoshino","creatorNameLang":"en"}],"nameIdentifiers":[{}]}]},"item_4_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AN10442647","subitem_source_identifier_type":"NCID"}]},"item_4_textarea_12":{"attribute_name":"Notice","attribute_value_mlt":[{"subitem_textarea_value":"SIG Technical Reports are nonrefereed and hence may later appear in any journals, conferences, symposia, etc."}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_18gh","resourcetype":"technical report"}]},"item_4_source_id_11":{"attribute_name":"ISSN","attribute_value_mlt":[{"subitem_source_identifier":"2188-8663","subitem_source_identifier_type":"ISSN"}]},"item_4_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"Human evaluation plays a critical role in dialogue systems research, especially in non-task-oriented systems such as rapport-building dialogue systems. Current evaluations often rely on Likert scales to assess user experience, but this method introduces challenges such as inconsistent scale perception, inefficiency, and central tendency bias. Moreover, it is difficult to compare the agent's performance across multiple criteria due to the problem of uneven scoring interpretations by participants on the Likert scale. On the other hand, pairwise comparison emphasizes direct item-to-item evaluation based on defined criteria, producing scores that more closely align with participants' preferences and minimizing biases. This paper compares an evaluation framework for rapport-building dialogue systems using pairwise comparison with a conventional Likert scale system. These approaches are tested through dialogue experiments involving six participants and four dialogue systems embedded in a conversational robot: CommA, CommI, CommO, and CommE, to measure human-agent rapport. Our experimental results indicated that the pairwise comparison method better represented systems' overall performance compared to the Likert scale. It also demonstrated lower variability, higher reliability, and a shorter completion time.","subitem_description_type":"Other"}]},"item_4_description_8":{"attribute_name":"論文抄録(英)","attribute_value_mlt":[{"subitem_description":"Human evaluation plays a critical role in dialogue systems research, especially in non-task-oriented systems such as rapport-building dialogue systems. Current evaluations often rely on Likert scales to assess user experience, but this method introduces challenges such as inconsistent scale perception, inefficiency, and central tendency bias. Moreover, it is difficult to compare the agent's performance across multiple criteria due to the problem of uneven scoring interpretations by participants on the Likert scale. On the other hand, pairwise comparison emphasizes direct item-to-item evaluation based on defined criteria, producing scores that more closely align with participants' preferences and minimizing biases. This paper compares an evaluation framework for rapport-building dialogue systems using pairwise comparison with a conventional Likert scale system. These approaches are tested through dialogue experiments involving six participants and four dialogue systems embedded in a conversational robot: CommA, CommI, CommO, and CommE, to measure human-agent rapport. Our experimental results indicated that the pairwise comparison method better represented systems' overall performance compared to the Likert scale. It also demonstrated lower variability, higher reliability, and a shorter completion time.","subitem_description_type":"Other"}]},"item_4_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicPageEnd":"5","bibliographic_titles":[{"bibliographic_title":"研究報告音声言語情報処理(SLP)"}],"bibliographicPageStart":"1","bibliographicIssueDates":{"bibliographicIssueDate":"2024-12-05","bibliographicIssueDateType":"Issued"},"bibliographicIssueNumber":"43","bibliographicVolumeNumber":"2024-SLP-154"}]},"relation_version_is_last":true,"weko_creator_id":"44499"},"id":241663,"updated":"2025-01-19T07:35:15.080089+00:00","links":{},"created":"2025-01-19T01:46:23.990485+00:00"}