{"created":"2025-01-19T01:35:25.481249+00:00","updated":"2025-01-19T09:58:18.369208+00:00","metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:00233823","sets":["934:1022:11484:11595"]},"path":["11595"],"owner":"44499","recid":"233823","title":["Provide Interpretability of Document Classification by Large Language Models Based on Word Masking"],"pubdate":{"attribute_name":"公開日","attribute_value":"2024-04-23"},"_buckets":{"deposit":"ff201857-d3e5-45e3-b1eb-a44f73ef25e4"},"_deposit":{"id":"233823","pid":{"type":"depid","value":"233823","revision_id":0},"owners":[44499],"status":"published","created_by":44499},"item_title":"Provide Interpretability of Document Classification by Large Language Models Based on Word Masking","author_link":["636000","635997","635999","635998"],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"Provide Interpretability of Document Classification by Large Language Models Based on Word Masking"},{"subitem_title":"Provide Interpretability of Document Classification by Large Language Models Based on Word Masking","subitem_title_language":"en"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"[テクニカルノート] deep learning, news documents classification, LLM, BERT, Attention, word masking","subitem_subject_scheme":"Other"}]},"item_type_id":"3","publish_date":"2024-04-23","item_3_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"Kogakuin University"},{"subitem_text_value":"Kogakuin University"}]},"item_3_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"Kogakuin University","subitem_text_language":"en"},{"subitem_text_value":"Kogakuin University","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"eng"}]},"item_publisher":{"attribute_name":"出版者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/233823/files/IPSJ-TOD1702002.pdf","label":"IPSJ-TOD1702002.pdf"},"date":[{"dateType":"Available","dateValue":"2026-04-23"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-TOD1702002.pdf","filesize":[{"value":"5.3 MB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"0","billingrole":"5"},{"tax":["include_tax"],"price":"0","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"13"},{"tax":["include_tax"],"price":"0","billingrole":"39"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"44e888e2-533d-4581-973b-82167dc9a30a","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2024 by the Information Processing Society of Japan"}]},"item_3_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Atsuki, Tamekuri"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Saneyasu, Yamaguchi"}],"nameIdentifiers":[{}]}]},"item_3_creator_6":{"attribute_name":"著者名(英)","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Atsuki, Tamekuri","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Saneyasu, Yamaguchi","creatorNameLang":"en"}],"nameIdentifiers":[{}]}]},"item_3_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AA11464847","subitem_source_identifier_type":"NCID"}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_6501","resourcetype":"journal article"}]},"item_3_source_id_11":{"attribute_name":"ISSN","attribute_value_mlt":[{"subitem_source_identifier":"1882-7799","subitem_source_identifier_type":"ISSN"}]},"item_3_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"Deep neural networks have greatly improved natural language processing and text analysis technologies. In particular, pre-trained large language models have achieved significant improvement. However, it has been argued that they are black boxes and that it is important to provide interpretability. In our previous work, we focused on self-attention and proposed methods for providing and evaluating interpretability. However, the work did not use large language models, and the evaluation method used unusual sentences by deleting words. In this paper, we focus on BERT, which is a popular large language model, and its masking function instead of deleting words. We then show a problem of using this masking function to provide interpretability, which is that the mask token is not neutral for decision. We then propose an evaluation method based on this masking function with training to learn that the mask token is neutral.\n------------------------------\nThis is a preprint of an article intended for publication Journal of\nInformation Processing(JIP). This preprint should not be cited. This\narticle should be cited as: Journal of Information Processing Vol.32(2024) (online)\n------------------------------","subitem_description_type":"Other"}]},"item_3_description_8":{"attribute_name":"論文抄録(英)","attribute_value_mlt":[{"subitem_description":"Deep neural networks have greatly improved natural language processing and text analysis technologies. In particular, pre-trained large language models have achieved significant improvement. However, it has been argued that they are black boxes and that it is important to provide interpretability. In our previous work, we focused on self-attention and proposed methods for providing and evaluating interpretability. However, the work did not use large language models, and the evaluation method used unusual sentences by deleting words. In this paper, we focus on BERT, which is a popular large language model, and its masking function instead of deleting words. We then show a problem of using this masking function to provide interpretability, which is that the mask token is not neutral for decision. We then propose an evaluation method based on this masking function with training to learn that the mask token is neutral.\n------------------------------\nThis is a preprint of an article intended for publication Journal of\nInformation Processing(JIP). This preprint should not be cited. This\narticle should be cited as: Journal of Information Processing Vol.32(2024) (online)\n------------------------------","subitem_description_type":"Other"}]},"item_3_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographic_titles":[{"bibliographic_title":"情報処理学会論文誌データベース（TOD）"}],"bibliographicIssueDates":{"bibliographicIssueDate":"2024-04-23","bibliographicIssueDateType":"Issued"},"bibliographicIssueNumber":"2","bibliographicVolumeNumber":"17"}]},"relation_version_is_last":true,"weko_creator_id":"44499"},"id":233823,"links":{}}