@techreport{oai:ipsj.ixsq.nii.ac.jp:00241690,
 author = {Sameer, Deshmukh and Mingchuan, Lyu and Hiroki, Tokura and Takumi, Honda and Sameer, Deshmukh and Mingchuan, Lyu and Hiroki, Tokura and Takumi, Honda},
 issue = {25},
 month = {Dec},
 note = {Large language models using the transformer architecture require massive computational resources for training to acceptable levels of accuracy. Recent advances have shown that the MLP layers within such models can be pruned to up to 90% sparsity to reduce the computational requirement of training and inference. However, achieving high performance for the sparse matrix multiplication remains a challenge on GPUs. Several approaches have been suggested for improving the performance of sparse matrix multiplication using structured sparsity. In this paper, we first survey and benchmark some of the sparsity structures that can be applied to dense matrices, and then examine the training loss curves of a 162M Mistral model using various structures of sparsity. Our results show promising future directions for research in improving the training time of transformers using sparsity., Large language models using the transformer architecture require massive computational resources for training to acceptable levels of accuracy. Recent advances have shown that the MLP layers within such models can be pruned to up to 90% sparsity to reduce the computational requirement of training and inference. However, achieving high performance for the sparse matrix multiplication remains a challenge on GPUs. Several approaches have been suggested for improving the performance of sparse matrix multiplication using structured sparsity. In this paper, we first survey and benchmark some of the sparsity structures that can be applied to dense matrices, and then examine the training loss curves of a 162M Mistral model using various structures of sparsity. Our results show promising future directions for research in improving the training time of transformers using sparsity.},
 title = {A survey of sparse structures in the multi-layer perceptron of large language models},
 year = {2024}
}