{"created":"2025-01-18T23:36:50.907780+00:00","updated":"2025-01-21T18:36:56.603532+00:00","metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:00083285","sets":["1164:2240:6731:6840"]},"path":["6840"],"owner":"11","recid":"83285","title":["MapReduce処理系の「京」での実装"],"pubdate":{"attribute_name":"公開日","attribute_value":"2012-07-25"},"_buckets":{"deposit":"65b3b0dc-6367-4ace-8758-c781661c6a7b"},"_deposit":{"id":"83285","pid":{"type":"depid","value":"83285","revision_id":0},"owners":[11],"status":"published","created_by":11},"item_title":"MapReduce処理系の「京」での実装","author_link":["0","0"],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"MapReduce処理系の「京」での実装"},{"subitem_title":"Implementing MapReduce on K-Computer","subitem_title_language":"en"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"システムソフトウェア","subitem_subject_scheme":"Other"}]},"item_type_id":"4","publish_date":"2012-07-25","item_4_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"理研計算科学研究機構"},{"subitem_text_value":"理研計算科学研究機構"}]},"item_4_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"AICS, Riken","subitem_text_language":"en"},{"subitem_text_value":"AICS, Riken","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"jpn"}]},"item_publisher":{"attribute_name":"出版者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/83285/files/IPSJ-HPC12135006.pdf"},"date":[{"dateType":"Available","dateValue":"2014-07-25"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-HPC12135006.pdf","filesize":[{"value":"108.2 kB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"660","billingrole":"5"},{"tax":["include_tax"],"price":"330","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"14"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"5f3dffed-52a8-49ab-9a3d-359918416b14","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2012 by the Information Processing Society of Japan"}]},"item_4_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"松田, 元彦"},{"creatorName":"丸山, 直也"}],"nameIdentifiers":[{}]}]},"item_4_creator_6":{"attribute_name":"著者名(英)","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"Motohiko, Matsuda","creatorNameLang":"en"},{"creatorName":"Naoya, Maruyama","creatorNameLang":"en"}],"nameIdentifiers":[{}]}]},"item_4_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AN10463942","subitem_source_identifier_type":"NCID"}]},"item_4_textarea_12":{"attribute_name":"Notice","attribute_value_mlt":[{"subitem_textarea_value":"SIG Technical Reports are nonrefereed and hence may later appear in any journals, conferences, symposia, etc."}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_18gh","resourcetype":"technical report"}]},"item_4_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"K 上で MapReduce を使って HPC 計算を行うための高性能な処理系 KMR の実現について述べる．高並列環境では，ノード数が多いので通信には低オーバーヘッドが要求される．また，多数ノードがファイルシステムを共有するのでファイル I/O を効率的に行うことも簡単ではなくなる．そのため，高並列環境に適応した MapReduce として KMR を実装している．K では MPI 拡張としてリモートメモリアクセス （RDMA） が提供されているので，その活用も重要である．まず，基本情報として K のファイル I/O と RDMA の通信オーバーヘッド特性を報告し，それに続いて KMR の実装を紹介する．KMR では，MapReduce の shuffle 操作に必要になる全対全通信に scatter-gather を組合わせた log (N) ステップのアルゴリズムを利用する．また，ファイルの読込みには断片の読込みと allgather による全体の再構成を用いる．","subitem_description_type":"Other"}]},"item_4_description_8":{"attribute_name":"論文抄録(英)","attribute_value_mlt":[{"subitem_description":"KMR is an implementation of a popular MapReduce framework on K-computer, which adapts to a highly parallel environment. There, low overhead communications and efficient file-I/O are naturally required. It is because there are many compute-nodes, where communications are repeated to each of them many times, and also they share a single file system and simply reading files by each node reveals a bottleneck. Thus, a new MapReduce implementation is needed particularly designed for a highly parallel environment. For the basic information, we report file-I/O performance and overheads of RDMA (remote direct memory access) operations, which motivate our design decisions. Following that, some of the KMR implementation is shown. It uses a log (N)-step algorithm for all-to-all communication needed in the shuffle-stage of MapReduce. It is based on the combination of scatter-gather, where the log (N)-step algorithm was not ever necessary but is now necessary for a class of the scale of K-computer. Its file operations are based on reading in chucks and aggregating them using allgather communication.","subitem_description_type":"Other"}]},"item_4_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicPageEnd":"7","bibliographic_titles":[{"bibliographic_title":"研究報告ハイパフォーマンスコンピューティング（HPC）"}],"bibliographicPageStart":"1","bibliographicIssueDates":{"bibliographicIssueDate":"2012-07-25","bibliographicIssueDateType":"Issued"},"bibliographicIssueNumber":"6","bibliographicVolumeNumber":"2012-HPC-135"}]},"relation_version_is_last":true,"weko_creator_id":"11"},"id":83285,"links":{}}