{"id":2008641,"created":"2026-03-09T04:27:28.418973+00:00","metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:02008641","sets":["581:1765244505933:1765246177381"]},"path":["1765246177381"],"owner":"80578","recid":"2008641","title":["日本特有のコード生成ベンチマークの開発と評価"],"pubdate":{"attribute_name":"PubDate","attribute_value":"2026-03-15"},"_buckets":{"deposit":"79446736-af7a-465e-a211-40c7bf4700a5"},"_deposit":{"id":"2008641","pid":{"type":"depid","value":"2008641","revision_id":0},"owner":"80578","owners":[80578],"status":"published","created_by":80578},"item_title":"日本特有のコード生成ベンチマークの開発と評価","author_link":[],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"日本特有のコード生成ベンチマークの開発と評価","subitem_title_language":"ja"},{"subitem_title":"Development and Evaluation of a Japan-specific Code Generation Benchmark","subitem_title_language":"en"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"[特集：若手研究者（特選論文）] コード生成，大規模言語モデル，ベンチマーク，日本語性能","subitem_subject_scheme":"Other"}]},"item_type_id":"2","publish_date":"2026-03-15","item_2_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"日本女子大学大学院理学研究科数理・物性構造科学専攻"},{"subitem_text_value":"日本女子大学大学院理学研究科数理・物性構造科学専攻"},{"subitem_text_value":"日本女子大学理学部数理情報科学科"}]},"item_2_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"Graduate School of Science Division of Mathematical and Physical Sciences, Japan Women's University","subitem_text_language":"en"},{"subitem_text_value":"Graduate School of Science Division of Mathematical and Physical Sciences, Japan Women's University","subitem_text_language":"en"},{"subitem_text_value":"Department of Mathematics, Physics, and Computer Science, Japan Women's University","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"jpn"}]},"control_number":"2008641","publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/2008641/files/IPSJ-JNL6703004.pdf","label":"IPSJ-JNL6703004.pdf"},"date":[{"dateType":"Available","dateValue":"2028-03-15"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-JNL6703004.pdf","filesize":[{"value":"1.1 MB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"660","billingrole":"5"},{"tax":["include_tax"],"price":"330","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"8"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"31de71ad-b86d-4c1c-a300-3655cf393c9a","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2026 by the Information Processing Society of Japan"}]},"item_2_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"佐藤,美唯"}]},{"creatorNames":[{"creatorName":"伊東,和香"}]},{"creatorNames":[{"creatorName":"倉光,君郎"}]}]},"item_2_creator_6":{"attribute_name":"著者名(英)","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Miyu Sato","creatorNameLang":"en"}]},{"creatorNames":[{"creatorName":"Waka Ito","creatorNameLang":"en"}]},{"creatorNames":[{"creatorName":"Kimio Kuramitsu","creatorNameLang":"en"}]}]},"item_2_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AN00116647","subitem_source_identifier_type":"NCID"}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_6501","resourcetype":"journal article"}]},"item_2_publisher_15":{"attribute_name":"公開者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"item_2_source_id_11":{"attribute_name":"ISSN","attribute_value_mlt":[{"subitem_source_identifier":"1882-7764","subitem_source_identifier_type":"ISSN"}]},"item_2_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"コード生成ベンチマークは，大規模言語モデル（Large Language Model，LLM）の基礎的なプログラミング能力を評価するうえで不可欠である．我々はこれまでに，英語中心のデータで事前学習されたLLMが，コード生成ベンチマークHumanEvalとその日本語版JHumanEvalにおいて，ほぼ同等の性能を示すことを明らかにしてきた．この結果は，LLMが英語で獲得したコード生成能力を日本語でも活用している，言語間転移が生じている可能性を示唆している．しかし，HumanEvalとJHumanEvalは翻訳関係にあり，入力言語は異なるものの，要求されるコード生成能力は本質的に同一であることから，日本特有の文化的背景や日本語処理を反映したタスクまでは評価できない限界がある．本研究では，英語からの翻訳に依存しない日本特有のコード生成ベンチマークSakuraEvalを開発し，それを用いてLLMのコード生成能力を評価することを目指す．SakuraEvalは日本の文化的背景や日本語処理など日本独自の要件に対応したコード生成タスクで構成され，HumanEvalとJHumanEvalとは異なる観点からコード生成能力を評価する．本論文では，SakuraEvalを紹介するとともに，14種類のLLMを用いた評価実験の結果を報告する．","subitem_description_type":"Other"}]},"item_2_description_8":{"attribute_name":"論文抄録(英)","attribute_value_mlt":[{"subitem_description":"Code generation benchmarks are essential for evaluating the fundamental programming capabilities of LLMs. We have demonstrated that LLMs pre-trained primarily on English data perform similarly on HumanEval and its Japanese version, JHumanEval. This result suggests the possibility of cross-lingual transfer, whereby code generation capabilities learned in English are effectively used in Japanese. However, HumanEval and JHumanEval require the same code generation capabilities, as JHumanEval is translated from HumanEval. These benchmarks are limited in their ability to evaluate tasks that reflect Japan-specific cultural contexts or Japanese language processing. To address this limitation, we introduce SakuraEval, a Japan-specific code generation benchmark that does not rely on translation from English. SakuraEval comprises tasks designed to reflect Japanese cultural background and language-specific requirements, enabling distinct evaluation from two benchmarks. This paper presents SakuraEval and reports the evaluation results of 14 LLMs.","subitem_description_type":"Other"}]},"item_2_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicPageEnd":"545","bibliographic_titles":[{"bibliographic_title":"情報処理学会論文誌"}],"bibliographicPageStart":"537","bibliographicIssueDates":{"bibliographicIssueDate":"2026-03-15","bibliographicIssueDateType":"Issued"},"bibliographicIssueNumber":"3","bibliographicVolumeNumber":"67"}]},"relation_version_is_last":true,"item_2_identifier_registration":{"attribute_name":"ID登録","attribute_value_mlt":[{"subitem_identifier_reg_text":"10.20729/0002008641","subitem_identifier_reg_type":"JaLC"}]},"weko_creator_id":"80578"},"updated":"2026-03-15T06:02:22.123610+00:00","links":{}}