{"created":"2025-01-19T00:50:25.703594+00:00","updated":"2025-01-20T03:54:51.942125+00:00","metadata":{"_oai":{"id":"oai:ipsj.ixsq.nii.ac.jp:00182857","sets":["1164:1867:9088:9218"]},"path":["9218"],"owner":"11","recid":"182857","title":["Apache SparkのSerialize処理最適化による処理速度向上手法"],"pubdate":{"attribute_name":"公開日","attribute_value":"2017-07-19"},"_buckets":{"deposit":"3a2cba4f-8f95-4b0d-81e6-61bff07cdf2a"},"_deposit":{"id":"182857","pid":{"type":"depid","value":"182857","revision_id":0},"owners":[11],"status":"published","created_by":11},"item_title":"Apache SparkのSerialize処理最適化による処理速度向上手法","author_link":["399502","399504","399503"],"item_titles":{"attribute_name":"タイトル","attribute_value_mlt":[{"subitem_title":"Apache SparkのSerialize処理最適化による処理速度向上手法"}]},"item_keyword":{"attribute_name":"キーワード","attribute_value_mlt":[{"subitem_subject":"ビッグデータ","subitem_subject_scheme":"Other"}]},"item_type_id":"4","publish_date":"2017-07-19","item_4_text_3":{"attribute_name":"著者所属","attribute_value_mlt":[{"subitem_text_value":"名古屋工業大学"},{"subitem_text_value":"名古屋工業大学"},{"subitem_text_value":"名古屋工業大学"}]},"item_4_text_4":{"attribute_name":"著者所属(英)","attribute_value_mlt":[{"subitem_text_value":"Nagoya Institute of Technology","subitem_text_language":"en"},{"subitem_text_value":"Nagoya Institute of Technology","subitem_text_language":"en"},{"subitem_text_value":"Nagoya Institute of Technology","subitem_text_language":"en"}]},"item_language":{"attribute_name":"言語","attribute_value_mlt":[{"subitem_language":"jpn"}]},"item_publisher":{"attribute_name":"出版者","attribute_value_mlt":[{"subitem_publisher":"情報処理学会","subitem_publisher_language":"ja"}]},"publish_status":"0","weko_shared_id":-1,"item_file_price":{"attribute_name":"Billing file","attribute_type":"file","attribute_value_mlt":[{"url":{"url":"https://ipsj.ixsq.nii.ac.jp/record/182857/files/IPSJ-OS17141016.pdf","label":"IPSJ-OS17141016.pdf"},"date":[{"dateType":"Available","dateValue":"2019-07-19"}],"format":"application/pdf","billing":["billing_file"],"filename":"IPSJ-OS17141016.pdf","filesize":[{"value":"2.7 MB"}],"mimetype":"application/pdf","priceinfo":[{"tax":["include_tax"],"price":"660","billingrole":"5"},{"tax":["include_tax"],"price":"330","billingrole":"6"},{"tax":["include_tax"],"price":"0","billingrole":"11"},{"tax":["include_tax"],"price":"0","billingrole":"44"}],"accessrole":"open_date","version_id":"a044ca86-1e16-4483-82a5-e8632d1a9913","displaytype":"detail","licensetype":"license_note","license_note":"Copyright (c) 2017 by the Information Processing Society of Japan"}]},"item_4_creator_5":{"attribute_name":"著者名","attribute_type":"creator","attribute_value_mlt":[{"creatorNames":[{"creatorName":"稲垣, 英夫"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"川島, 龍太"}],"nameIdentifiers":[{}]},{"creatorNames":[{"creatorName":"松尾, 啓志"}],"nameIdentifiers":[{}]}]},"item_4_source_id_9":{"attribute_name":"書誌レコードID","attribute_value_mlt":[{"subitem_source_identifier":"AN10444176","subitem_source_identifier_type":"NCID"}]},"item_4_textarea_12":{"attribute_name":"Notice","attribute_value_mlt":[{"subitem_textarea_value":"SIG Technical Reports are nonrefereed and hence may later appear in any journals, conferences, symposia, etc."}]},"item_resource_type":{"attribute_name":"資源タイプ","attribute_value_mlt":[{"resourceuri":"http://purl.org/coar/resource_type/c_18gh","resourcetype":"technical report"}]},"item_4_source_id_11":{"attribute_name":"ISSN","attribute_value_mlt":[{"subitem_source_identifier":"2188-8795","subitem_source_identifier_type":"ISSN"}]},"item_4_description_7":{"attribute_name":"論文抄録","attribute_value_mlt":[{"subitem_description":"ビッグデータの解析手法として，クラスタ環境を用いた並列分散処理が注目されており，並列分散処理を効率的に記述するためのフレームワークとして Apache Spark が普及している．Spark はデータに対して多段な処理が発生する場合でもメモリ上で処理するため，他のフレームワークと比較してディスクアクセスの回数が減少し，パフォーマンスが向上する．しかし，計算機間でのデータ転送を必要とする Shuffle 処理では，大量のデータが転送されるため，処理全体の性能が低下する．Shuffle 処理の際，転送元の計算機において送信データの Serialize が行われ，同様に転送先の計算機において受信データの Deserialize が行われる．Serialize / Deserialize は Word サイズや Byte オーダなどの仕様が異なる計算機間のデータ転送では有用であるが，Spark のプロセスはすべて JVM 上で動作しているため，このような仕様はすべて統一されており，Serialize / Deserialize は簡素化できる．そこで本研究では Serialize / Deserialize 処理を最適化してデータ転送性能の向上を図る．具体的には，送信データに対する Serialize を簡素化して，クラス名などの省略可能な情報を付与せずバイト列変換のみを行い，転送先では受信データをバイト列のままデータとして扱うことで処理速度が向上する．評価から，JavaSerializer と比較した際に Serialize / Deserialize 処理にかかる時間を最大 57.3% 削減，全体の処理時間を最大 19.8% 削減し，手法の有効性を確認した．","subitem_description_type":"Other"}]},"item_4_biblio_info_10":{"attribute_name":"書誌情報","attribute_value_mlt":[{"bibliographicPageEnd":"7","bibliographic_titles":[{"bibliographic_title":"研究報告システムソフトウェアとオペレーティング・システム（OS）"}],"bibliographicPageStart":"1","bibliographicIssueDates":{"bibliographicIssueDate":"2017-07-19","bibliographicIssueDateType":"Issued"},"bibliographicIssueNumber":"16","bibliographicVolumeNumber":"2017-OS-141"}]},"relation_version_is_last":true,"weko_creator_id":"11"},"id":182857,"links":{}}