{"links":{},"id":66516,"metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:00066516","sets":["5471:5472"]},"path":["5472"],"owner":"11","recid":"66516","title":["On the Robustness of Information Retrieval Metrics to Biased Relevance Assessments"],"pubdate":{"attribute_name":"公開日","attribute_value":"2009-04-08"},"_buckets":{"deposit":"f1a75bf2-0d10-4301-b431-60d1a6afdfca"},"_deposit":{"id":"66516","pid":{"type":"depid","value":"66516","revision_id":0},"owners":[11],"status":"published","created_by":11},"item_title":"On the Robustness of Information Retrieval Metrics to Biased Relevance Assessments","author_link":["0","0"],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"On the Robustness of Information Retrieval Metrics to Biased Relevance Assessments"},{"subitem_title":"On the Robustness of Information Retrieval Metrics to Biased Relevance Assessments","subitem_title_language":"en"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"Regular Paper","subitem_subject_scheme":"Other"}]},"item_type_id":"5","publish_date":"2009-04-08","item_5_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"NewsWatch, Inc."}]},"item_5_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"NewsWatch, Inc.","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"eng"}]},"item_publisher":{"attribute_name":"出版者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/66516/files/IPSJ-JIP1700013.pdf"},"date":[{"dateType":"Available","dateValue":"2011-04-08"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-JIP1700013.pdf","filesize":[{"value":"216.7 kB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"0","billingrole":"5"},{"tax":["include_tax"],"price":"0","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"598cc58d-0edb-4cbb-90cd-fcd3069a221a","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2009 by the Information Processing Society of Japan"}]},"item_5_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Tetsuya, Sakai"}],"nameIdentifiers":[{}]}]},"item_5_creator_6":{"attribute_name":"著者名(英)","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Tetsuya, Sakai","creatorNameLang":"en"}],"nameIdentifiers":[{}]}]},"item_5_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AA00700121","subitem_source_identifier_type":"NCID"}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_6501","resourcetype":"journal article"}]},"item_5_source_id_11":{"attribute_name":"ISSN","attribute_value_mlt":[{"subitem_source_identifier":"1882-6652","subitem_source_identifier_type":"ISSN"}]},"item_5_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"Information Retrieval (IR) test collections are growing larger, and relevance data constructed through <i>pooling</i> are suspected of becoming more and more <i>incomplete</i> and <i>biased</i>. Several studies have used IR evaluation metrics specifically designed to handle this problem, but most of them have only examined the metrics under <i>incomplete but unbiased</i> conditions, using random samples of the original relevance data. This paper examines nine metrics in more realistic settings, by reducing the number of pooled systems and the number of pooled documents. Even though previous studies have shown that metrics based on a <i>condensed list</i>, obtained by removing all unjudged documents from the original ranked list, are effective for handling very incomplete but unbiased relevance data, we show that these results do not hold when the relevance data are biased towards particular systems or towards the top of the pools. More specifically, we show that the condensed-list versions of <i>Average Precision</i>, <i>Qmeasure</i> and <i>normalised Discounted Cumulative Gain</i>, which we denote as AP', Q' and nDCG', are not necessarily superior to the original metrics for handling biases. Nevertheless, AP' and Q' <i>are</i> generally superior to <i>bpref</i>, <i>Rank-Biased Precision</i> and its condensed-list version even in the presence of biases.","subitem_description_type":"Other"}]},"item_5_description_8":{"attribute_name":"論文抄録(英)","attribute_value_mlt":[{"subitem_description":"Information Retrieval (IR) test collections are growing larger, and relevance data constructed through <i>pooling</i> are suspected of becoming more and more <i>incomplete</i> and <i>biased</i>. Several studies have used IR evaluation metrics specifically designed to handle this problem, but most of them have only examined the metrics under <i>incomplete but unbiased</i> conditions, using random samples of the original relevance data. This paper examines nine metrics in more realistic settings, by reducing the number of pooled systems and the number of pooled documents. Even though previous studies have shown that metrics based on a <i>condensed list</i>, obtained by removing all unjudged documents from the original ranked list, are effective for handling very incomplete but unbiased relevance data, we show that these results do not hold when the relevance data are biased towards particular systems or towards the top of the pools. More specifically, we show that the condensed-list versions of <i>Average Precision</i>, <i>Qmeasure</i> and <i>normalised Discounted Cumulative Gain</i>, which we denote as AP', Q' and nDCG', are not necessarily superior to the original metrics for handling biases. Nevertheless, AP' and Q' <i>are</i> generally superior to <i>bpref</i>, <i>Rank-Biased Precision</i> and its condensed-list version even in the presence of biases.","subitem_description_type":"Other"}]},"item_5_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicPageEnd":"166","bibliographic_titles":[{"bibliographic_title":"Journal of information processing"}],"bibliographicPageStart":"156","bibliographicIssueDates":{"bibliographicIssueDate":"2009-04-08","bibliographicIssueDateType":"Issued"},"bibliographicVolumeNumber":"17"}]},"relation_version_is_last":true,"item_5_alternative_title_2":{"attribute_name":"その他タイトル","attribute_value_mlt":[{"subitem_alternative_title":"Information Retrieval"}]},"weko_creator_id":"11"},"created":"2025-01-18T23:27:15.586791+00:00","updated":"2025-01-22T01:00:49.820950+00:00"}