@article{journals-micro24-liuyao,
author = {Dai, Liuyao and Qi, Hao and Chen, Weicong and Lu, Xiaoyi},
title = "{High-Speed Data Communication with Advanced Networks in Large Language Model Training}",
journal = {IEEE Micro},
year = {2024},
volume = {44},
number = {02},
issn = {1937-4143},
pages = {31-40},
abstract = {Large language models (LLMs) like Generative Pre-trained Transformer, Bidirectional Encoder Representations from Transformers, and T5 are pivotal in natural language processing. Their distributed training is influenced by high-speed interconnects. This article characterizes their training performance across various interconnects and communication protocols: TCP/IP, Internet Protocol over InfiniBand, (IPoIB), and Remote Direct Memory Access (RDMA), using data and model parallelism. RDMA-100 Gbps outperforms IPoIB-100 Gbps and TCP/IP-10 Gbps, with average gains of 2.5x and 4.8x in data parallelism, while in model parallelism, the gains were 1.1x and 1.2x. RDMA achieves the highest interconnect utilization (up to 60 Gbps), compared to IPoIB with up to 20 Gbps and TCP/IP with up to 9 Gbps. Larger models demand increased communication bandwidth, with AllReduce in data parallelism consuming up to 91% of training time, and forward receive and back-embedding AllReduce in model parallelism taking up to 90%. The larger-scale experiment confirms that communication predominates iterations. Our findings underscore the significance of communication in distributed LLM training and present opportunities for optimization.},
keywords = {training;parallel processing;data models;computational modeling;decoding;tcpip;synchronization;high-speed networks;large language models},
doi = {10.1109/MM.2024.3360081},
publisher = {IEEE Computer Society},
address = {Los Alamitos, CA, USA},
series = {IEEE Micro'24},
month = {mar}
}