{"metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:00231988","sets":["934:935:11486:11487"]},"path":["11487"],"owner":"44499","recid":"231988","title":["ChatGPTはどのくらいプログラミングを理解しているか？"],"pubdate":{"attribute_name":"公開日","attribute_value":"2024-01-31"},"_buckets":{"deposit":"6f9a9ab5-b39a-4317-bb97-29fd226dd7b6"},"_deposit":{"id":"231988","pid":{"type":"depid","value":"231988","revision_id":0},"owners":[44499],"status":"published","created_by":44499},"item_title":"ChatGPTはどのくらいプログラミングを理解しているか？","author_link":["627607","627609","627606","627610","627605","627603","627608","627604","627598","627602","627599","627601","627597","627600"],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"ChatGPTはどのくらいプログラミングを理解しているか？"},{"subitem_title":"How Well does ChatGPT Understand Programming?","subitem_title_language":"en"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"[発表概要, Unrefereed Presentatin Abstract] ","subitem_subject_scheme":"Other"}]},"item_type_id":"3","publish_date":"2024-01-31","item_3_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"日本女子大学大学院理学研究科数理物性構造科学専攻"},{"subitem_text_value":"日本女子大学大学院理学研究科数理物性構造科学専攻"},{"subitem_text_value":"日本女子大学理学部数理情報科学科"},{"subitem_text_value":"日本女子大学理学部数理情報科学科"},{"subitem_text_value":"日本女子大学理学部数理情報科学科"},{"subitem_text_value":"日本女子大学大学院理学研究科数理物性構造科学専攻"},{"subitem_text_value":"日本女子大学理学部数理情報科学科"}]},"item_3_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"Division of Mathematical and Physical Sciences, Graduate School of Science","subitem_text_language":"en"},{"subitem_text_value":"Division of Mathematical and Physical Sciences, Graduate School of Science","subitem_text_language":"en"},{"subitem_text_value":"Department of Mathematics, Physics, and Computer Science, Japan Women's University","subitem_text_language":"en"},{"subitem_text_value":"Department of Mathematics, Physics, and Computer Science, Japan Women's University","subitem_text_language":"en"},{"subitem_text_value":"Department of Mathematics, Physics, and Computer Science, Japan Women's University","subitem_text_language":"en"},{"subitem_text_value":"Division of Mathematical and Physical Sciences, Graduate School of Science","subitem_text_language":"en"},{"subitem_text_value":"Department of Mathematics, Physics, and Computer Science, Japan Women's University","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"jpn"}]},"item_publisher":{"attribute_name":"出版者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/231988/files/IPSJ-TPRO1701004.pdf","label":"IPSJ-TPRO1701004.pdf"},"date":[{"dateType":"Available","dateValue":"2026-01-31"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-TPRO1701004.pdf","filesize":[{"value":"114.1 kB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"0","billingrole":"5"},{"tax":["include_tax"],"price":"0","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"15"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"8e06f522-49d5-478a-8f8a-9ae3bcb4f21c","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2024 by the Information Processing Society of Japan"}]},"item_3_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"佐藤, 美唯"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"相馬, 菜生"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"伊東, 和香"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"小原, 有以"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"東出, 紗也夏"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"高野, 志歩"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"倉光, 君郎"}],"nameIdentifiers":[{}]}]},"item_3_creator_6":{"attribute_name":"著者名(英)","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Miyu, Sato","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Nao, Souma","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Waka, Ito","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Yui, Obara","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Sayaka, Higashide","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Shiho, Takano","creatorNameLang":"en"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"Kimio, Kuramitsu","creatorNameLang":"en"}],"nameIdentifiers":[{}]}]},"item_3_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AA11464814","subitem_source_identifier_type":"NCID"}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_6501","resourcetype":"journal article"}]},"item_3_source_id_11":{"attribute_name":"ISSN","attribute_value_mlt":[{"subitem_source_identifier":"1882-7802","subitem_source_identifier_type":"ISSN"}]},"item_3_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"近年，大規模言語モデルの発展は目覚ましく，プログラミングを含むソフトウェア開発への応用の期待が高まっている．しかし，大規模言語モデルの原理とコード生成能力の関係には未知な部分も多い．本研究では，大規模言語モデルがプログラミングをどのくらい理解しているかを定量化するベンチマークの開発を提案する．ベンチマークの開発により，コード生成，コード移植，リファクタリング，コード修正など様々なコードタスクの得意・不得意の把握が可能になり，さらなる大規模言語モデルの開発や微調整に役立つことが期待される．本発表では，ベンチマークのタスクセットの設計，データセットの整備手順やZero-shotプロンプトの開発進捗を報告し，ChatGPTを用いたベースライン評価を報告する．","subitem_description_type":"Other"}]},"item_3_description_8":{"attribute_name":"論文抄録(英)","attribute_value_mlt":[{"subitem_description":"In recent years, large-scale language models have made remarkable progress, and there are growing expectations for their application to software development, including programming. However, there are many unknowns in the relationship between the principles of large-scale language models and their ability to generate code. In this study, we propose the development of a benchmark to quantify how well large-scale language models understand programming. The development of benchmarks will enable us to understand the strengths and weaknesses of various code tasks, such as code generation, code porting, refactoring, and code modification, and will be useful for further developing and fine-tuning large-scale language models. In this presentation, we will report on the design of the benchmark task set, the procedure for maintaining the dataset and the progress in developing the Zero-shot prompts, and report on the baseline evaluation using ChatGPT.","subitem_description_type":"Other"}]},"item_3_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicPageEnd":"13","bibliographic_titles":[{"bibliographic_title":"情報処理学会論文誌プログラミング（PRO）"}],"bibliographicPageStart":"13","bibliographicIssueDates":{"bibliographicIssueDate":"2024-01-31","bibliographicIssueDateType":"Issued"},"bibliographicIssueNumber":"1","bibliographicVolumeNumber":"17"}]},"relation_version_is_last":true,"weko_creator_id":"44499"},"id":231988,"updated":"2025-01-19T10:34:36.432086+00:00","links":{},"created":"2025-01-19T01:32:35.178829+00:00"}