@article{xu_nguyen_le-cong_hoang_liu_kim_gong_niu_wang_le_et al._2023, title={Are We Ready to Embrace Generative AI for Software Q&A?}, ISSN={["1527-1366"]}, DOI={10.1109/ASE56229.2023.00023}, abstractNote={Stack Overflow, the world's largest software Q&A (SQA) website, is facing a significant traffic drop due to the emergence of generative AI techniques. ChatGPT is banned by Stack Overflow after only 6 days from its release. The main reason provided by the official Stack Overflow is that the answers generated by ChatGPT are of low quality. To verify this, we conduct a comparative evaluation of human-written and ChatGPT-generated answers. Our methodology employs both automatic comparison and a manual study. Our results suggest that human-written and ChatGPT-generated answers are semantically similar, however, human-written answers outperform ChatGPT-generated ones consistently across multiple aspects, specifically by 10% on the overall score. We release the data, analysis scripts, and detailed results at https://github.com/maxxbw54/GAI4SQA.}, journal={2023 38TH IEEE/ACM INTERNATIONAL CONFERENCE ON AUTOMATED SOFTWARE ENGINEERING, ASE}, author={Xu, Bowen and Nguyen, Thanh-Dat and Le-Cong, Thanh and Hoang, Thong and Liu, Jiakun and Kim, Kisub and Gong, Chen and Niu, Changan and Wang, Chenyu and Le, Bach and et al.}, year={2023}, pages={1713–1717} }
 @article{zhou_xu_han_yang_he_lo_2023, title={CCBERT: Self-Supervised Code Change Representation Learning}, ISSN={["1063-6773"]}, DOI={10.1109/ICSME58846.2023.00028}, abstractNote={Numerous code changes are made by developers in their daily work, and a superior representation of code changes is desired for effective code change analysis. Recently, Hoang et al. proposed CC2Vec, a neural network-based approach that learns a distributed representation of code changes to capture the semantic intent of the changes. Despite demonstrated effectiveness in multiple tasks, CC2Vec has several limitations: 1) it considers only coarse-grained information about code changes, and 2) it relies on log messages rather than the self-contained content of the code changes. In this work, we propose CCBERT (Code Change BERT), a new Transformer-based pre-trained model that learns a generic representation of code changes based on a large-scale dataset containing massive unlabeled code changes. CCBERT is pre-trained on four proposed self-supervised objectives that are specialized for learning code change representations based on the contents of code changes. CCBERT perceives fine-grained code changes at the token level by learning from the old and new versions of the content, along with the edit actions. Our experiments demonstrate that CCBERT significantly outperforms CC2Vec or the state-of-the-art approaches of the downstream tasks by 7.7%–14.0% in terms of different metrics and tasks. CCBERT consistently outperforms large pre-trained code models, such as CodeBERT, while requiring 6–10× less training time, 5–30× less inference time, and 7.9× less GPU memory.}, journal={2023 IEEE INTERNATIONAL CONFERENCE ON SOFTWARE MAINTENANCE AND EVOLUTION, ICSME}, author={Zhou, Xin and Xu, Bowen and Han, DongGyun and Yang, Zhou and He, Junda and Lo, David}, year={2023}, pages={182–193} }
 @article{zhou_kim_xu_liu_han_lo_2023, title={The Devil is in the Tails: How Long-Tailed Code Distributions Impact Large Language Models}, ISSN={["1527-1366"]}, DOI={10.1109/ASE56229.2023.00157}, abstractNote={Learning-based techniques, especially advanced Large Language Models (LLMs) for code, have gained considerable popularity in various software engineering (SE) tasks. However, most existing works focus on designing better learning-based models and pay less attention to the properties of datasets. Learning-based models, including popular LLMs for code, heavily rely on data, and the data's properties (e.g., data distribution) could significantly affect their behavior. We conducted an exploratory study on the distribution of SE data and found that such data usually follows a skewed distribution (i.e., long-tailed distribution) where a small number of classes have an extensive collection of samples, while a large number of classes have very few samples. We investigate three distinct SE tasks and analyze the impacts of long-tailed distribution on the performance of LLMs for code. Our experimental results reveal that the long-tailed distribution has a substantial impact on the effectiveness of LLMs for code. Specifically, LLMs for code perform between 30.0% and 254.0% worse on data samples associated with infrequent labels compared to data samples of frequent labels. Our study provides a better understanding of the effects of long-tailed distributions on popular LLMs for code and insights for the future development of SE automation.}, journal={2023 38TH IEEE/ACM INTERNATIONAL CONFERENCE ON AUTOMATED SOFTWARE ENGINEERING, ASE}, author={Zhou, Xin and Kim, Kisub and Xu, Bowen and Liu, Jiakun and Han, DongGyun and Lo, David}, year={2023}, pages={40–52} }