@techreport{oai:ipsj.ixsq.nii.ac.jp:00218464,
 author = {Chiori, Hori and Chiori, Hori},
 issue = {5},
 month = {Jun},
 note = {Science fiction television and movies have portrayed humanoid robots with human-like capabilities to recognize their surroundings and the context of the situation. While computers have recently become much more capable at many perceptual tasks, they are not yet ready to take the place of a human in many situations. The recent artificial intelligence (AI) boom and intelligent use of data acquired from various sensors has certainly accelerated the development of technologies needed to realize these advanced human-like capabilities in machines. We have developed a new AI system, called Scene-Aware Interaction, that enables machines to translate their perception and understanding of a scene and respond to it using natural language to interact more effectively with humans. To develop such a machine, we have proposed the Audio-Visual Scene-Aware Dialog (AVSD) task, collected an AVSD dataset, developed AVSD technologies, and hosted three-time AVSD challenge track at the Dialog System Technology Challenges (DSTC). We tested the performance of answer generation and temporal reasoning by finding evidence from the video to support each answer. This paper introduces a new system that extends our AV-transformer-based system with attentional multimodal fusion, joint student-teacher learning (JSTL), and model combination techniques, achieving state-of-the-art performances on the AVSD datasets for DSTC7-8,10. We applied the Scene-aware interaction technology to a car navigation system to recognizes contextual objects and events based on multimodal sensing information, such as images and video captured with cameras, audio information recorded with microphones, and localization information measured with LiDAR., Science fiction television and movies have portrayed humanoid robots with human-like capabilities to recognize their surroundings and the context of the situation. While computers have recently become much more capable at many perceptual tasks, they are not yet ready to take the place of a human in many situations. The recent artificial intelligence (AI) boom and intelligent use of data acquired from various sensors has certainly accelerated the development of technologies needed to realize these advanced human-like capabilities in machines. We have developed a new AI system, called Scene-Aware Interaction, that enables machines to translate their perception and understanding of a scene and respond to it using natural language to interact more effectively with humans. To develop such a machine, we have proposed the Audio-Visual Scene-Aware Dialog (AVSD) task, collected an AVSD dataset, developed AVSD technologies, and hosted three-time AVSD challenge track at the Dialog System Technology Challenges (DSTC). We tested the performance of answer generation and temporal reasoning by finding evidence from the video to support each answer. This paper introduces a new system that extends our AV-transformer-based system with attentional multimodal fusion, joint student-teacher learning (JSTL), and model combination techniques, achieving state-of-the-art performances on the AVSD datasets for DSTC7-8,10. We applied the Scene-aware interaction technology to a car navigation system to recognizes contextual objects and events based on multimodal sensing information, such as images and video captured with cameras, audio information recorded with microphones, and localization information measured with LiDAR.},
 title = {Human-Robot Interaction through Multi-modal Semantic Understanding},
 year = {2022}
}