@techreport{chang2023_t6a,
    Author = "Cho, Jae-Heung and Park, Yoon-Ah and Kim, Jaewon and Chang, Joon-Hyuk",
    title = "HYU submission for the DCASE 2023 task 6a: automated audio captioning model using AL-MixGen and synonyms substitution",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "This paper presents the automated audio captioning model for participating in the detection and classification of acoustic scenes and events 2023 challenge task 6A. The model consists of two parts: an audio feature extractor and a language model. The audio feature extractor employed in our model is the pre-trained convolutional neural network 14 (CNN14), trained with AudioSet. CNN14 has demonstrated excellent performance in extracting audio features. For the language model, we utilized bidirectional autoregressive transformers model, which has achieved remarkable success in generating the text. We pre-trained the model with WavCaps, AudioCaps and Clotho dataset to manage the limitation of data availability, and then fine-tuned with Clotho dataset. Furthermore, AL-MixGen and synonyms substitution methods were also implemented for data augmentation. To improve the evaluation metric directly, we trained the model with reinforcement learning to optimize the CIDEr score. Finally, we achieved improved performance by adapting an ensemble of higher-performance models, leading to the accomplishment of 0.343 SPIDEr score."
}

@techreport{greeshma2023_t6a,
    Author = "Greeshma, Karanth and Rao, Ninaad and Subramanian, Srikumar and Shah, Ankit",
    title = "DCASE 2023 task 6 automated audio captioning and language-based retrieval",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "The objective of this project is to examine audio signals utilizing natural language to capture their complex characteristics. This initiative is part of Task 6 in the DCASE 2023 Competition and consists of two subtasks. The first subtask is Automated Audio Captioning, which generates text descriptions of audio content. This task involves the intermodal processing of an audio signal as input and a text description as output. Our best-performing model for this uses the PANN architecture [1] with the CNN-14 feature extractor and BART [2] encoder and decoder. The second subtask is called Language-Based Audio Retrieval, where the system retrieves audio signals by searching for their sound content descriptions. The queries for this subtask are human-generated audio captions. In this task, our best-performing model uses CLAP [3] audio embeddings and Roberta text embeddings [4]. This document presents a summary of our work done for this challenge."
}

@techreport{guan2023_t6a,
    Author = "Xiao, Feiyang and Zhu, Qiaoxi and Lan, Haiyan and Wang, Wenwu and Guan, Jian",
    title = "Ensemble systems with contrastive language-audio pretraining and attention-based audio features for audio captioning and retrieval",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "This technical report describes our submission on Task 6 (automated audio captioning and language-based audio retrieval) of the Detection and Classification of Acoustic Scenes and Events (DCASE) 2023 Challenge. The proposed systems in this submission are based on a contrastive language-audio pretraining strategy and the attention-based audio feature representation. Experiments show that our systems can achieve a SPIDEr-FL score of 28.32 on automated audio captioning and an mAP score of 31.18 on language-based audio retrieval."
}

@techreport{kadl\u010D\xEDk2023_t6a,
    Author = "Kadlčı́k, Marek and Hájek, Adam and Kieslich, Jürgen and Winiecki, Radosław",
    title = "A whisper transformer for audio captioning trained with synthetic captions and transfer learning",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "The field of audio captioning has seen significant advancements in recent years, driven by the availability of large-scale audio datasets and advancements in deep learning techniques. In this technical report, we present our approach to audio captioning, focusing on the use of a pretrained speech-to-text Whisper model and pretraining on synthetic captions. We discuss our training procedures and present our experiments’ results, which include model size variations, dataset mixtures, and other hyperparameters. Our findings demonstrate the impact of different training strategies on the performance of the audio captioning model. Our code and trained models are publicly available on GitHub and Hugging Face Hub."
}

@techreport{labbe2023_t6a,
    Author = "Labbé, Etienne and Pellegrini, Thomas and Pinquier, Julien",
    title = "IRIT-UPS DCASE 2023 audio captioning and retrieval system",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = {This technical report provides a concise overview of our systems submitted to the DCASE Challenge 2023 for tasks 6a, "Automated Audio Captioning" (AAC), and 6b, "Language-Based Audio Retrieval" (LBAR). In task 6a, we made four distinct submissions. The first submission employed a standard CNN14 encoder paired with a transformer decoder. In the second submission, we replaced this encoder with a ConvNeXt model to enhance audio representation. The third submission incorporated additional training data. We introduced a new task embedding approach to differentiate between different writing styles and audio types. Finally, in the fourth submission, we employed an ensemble method to combine five models trained on different seeds, aiming to improve the quality of the captions. For task 6b, we use the AAC models and we propose a novel approach to accomplish the LBAR task by leveraging the AAC system loss function without requiring any additional training. Our most successful AAC and LBAR systems achieved a SPIDEr-FL score of 0.320 and an mAP@10 score of 0.269. These results demonstrate relative improvements of 22.6\\% and 21.2\\% compared to the AAC and LBAR baselines, respectively.}
}

@techreport{lee2023_t6a,
    Author = "Sim, Jaeheon and Kim, Eungbeom and Lee, Kyogu",
    title = "Label-refined sequential training with noisy data for automated audio captioning",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "This technical report describes the submission to the Detection and Classification of Acoustic Scenes and Events (DCASE) 2023 Challenge Task 6A: Automated Audio Captioning. We utilize a label-refined sequential training method to leverage the large additional dataset which contains two types of noise including domain shift and label noise. We investigate the usefulness of the additional noisy dataset and observe that the models directly trained on the dataset naively including the additional and target dataset suffer from a poor performance. From this observation, we aim to fully leverage the additional dataset by addressing the two types of noise simultaneously. We sequentially train the model with the prior knowledge about the difference between the target dataset and each of the additional datasets, from the largest to the nearest. We finally train the model on the target dataset, thereby progressively minimizing the domain gap. After this training procedure, we apply a label refinement method which is based on pseudo-labelling from self-training method and repeat the sequential training procedure. The proposed method mitigates the noise in the dataset and achieves the improved performance."
}

@techreport{lim2023_t6a,
    Author = "Chou, Seungmin and Yim, Jaeseung and Lim, Changwon",
    title = "CAU submission to DCASE 2023 task 6a: Audio captioning using wavegrams that contain frequency information",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "This technical report describes an Automated Audio Captioning model for the Detection and Classification of Acoustic Scenes and Events (DCASE) 2023 Challenge, Task 6A. Utilizing wavegram and patchout as proposed in [1] and [2], respectively, we propose audio captioning using Wavegrams that contain frequency information. We use pre-trained models trained using AudioSet data, to make word embedding. Our proposed sequence-to-sequence model consists of CNN14 encoder and a Transformer decoder. Experiments show that the proposed model achieves a SPIDEr score of 0.011."
}

@techreport{schaumloeffel2023_t6a,
    Author = "Schaumlöffel, Timothy and Vilas, Martina G. and Roig, Gemma",
    title = "PEACS: Prefix encoding for auditory caption synthesis",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "This technical report describes an Automated Audio Captioning system for the Detection and Classification of Acoustic Scenes and Events (DCASE) 2023 Challenge, Task 6a (automated audio captioning). Our approach employs an encoder-decoder architecture, with the encoder utilizing a large contrastive pre-trained HTS-AT capable of handling variable-length audio segments. The decoder is based on the GPT2 model. To incorporate audio into the decoding process, we employ a light mapping network that translates audio representations into a prefix, effectively guiding the decoder’s generation process. Given the limited data availability, we pre-train our model on various audio captioning datasets and fine-tune it on Clotho. We reach a SPIDERr-FL score of 29.3 on the evaluation split of the Clotho-v2 dataset."
}

@techreport{wu2023_t6a,
    Author = "Wu, Shih-Lun and Chang, Xuankai and Wichern, Gordon and Jung, Jee-weon and Germain, François and Roux, Jonathan Le and Watanabe, Shinji",
    title = "BEATs-based audio captioning model with INSTRUCTOR embedding supervision and ChatGPT mix-up",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "DCASE 2023 Task 6A, automated audio captioning (AAC), aims at generating informative descriptions for various sounds from nature and/or human activities. Our AAC system follows the sequence-to-sequence (seq2seq) architecture. The audio encoder stack is comprised of a frozen BEAT S Transformer followed by a 2-layer Conformer. The BEAT S module, which has been pretrained on both masked audio token prediction and audio event classification, extracts fine-grained (i.e., ≈ 50 Hz) audio features, while the Conformer downsamples and summarizes the audio features before they are cross-attended by the BART text decoder. Besides the autoregressive negative log-likelihood (NLL) loss computed on decoder outputs, we simultaneously apply an audio-text contrastive loss on our encoder output to infuse language modality knowledge into it. Specifically, we feed ground-truth captions into INSTRUCTOR Transformer, a state-of-the-art text embedding model, and teach our audio encoder to predict the INSTRUCTOR text embeddings through InfoNCE loss. In addition, we leverage ChatGPT to produce caption mix-ups (i.e., grammatical and compact combinations of two captions) which, together with the corresponding audio mixtures, increases not only the amount but also the complexity and diversity of our training data. During inference, we employ nucleus sampling and a hybrid reranking algorithm that considers both likelihood and audio-caption representation similarity. Combining our efforts, our best single model and ensemble system achieve 0.326 and 0.336 SPIDEr-FL scores, respectively, on the Clotho (V2) evaluation split."
}

@techreport{yan2023_t6a,
    Author = "Sun, Haoran and Yan, Zhiyong and Wang, Yongqing and Dinkel, Heinrich and Zhang, Junbo and Wang, Yujun",
    title = "Leveraging multi-task training and image retrieval with CLAP for audio captioning",
    institution = "DCASE2023 Challenge",
    year = "2023",
    month = "May",
    abstract = "This technical report serves as our submission to Task 6 of the Detection and Classification of Acoustic Scenes and Events (DCASE) 2023 challenge. Our system, as described in this report, consists of two sub-systems designed for the respective sub-tasks: automated audio captioning (task A) and text-to-audio retrieval (task B). The text-to-audio retrieval system employs a tri-encoder architecture, where pre-trained audio and text encoders are trained to establish relationships. Additionally, an extra pre-trained image encoder is utilized to enhance the connections between these encoders. Through this retrieval process, the audio encoder can be considered a pre-trained encoder for task A. Furthermore, we employ multi-task training with audio tagging during the retrieval phase to strengthen the encoder for audio captioning. Pre-training is conducted using AudioCaps and a portion of WavCaps datasets, and both sub-systems are subsequently finetuned on Clotho dataset. Experimental results demonstrate that our model achieves a SPIDEr score of 0.305 and a SPIDEr-FL score of 0.294 for captioning, as well as an mAP (mean Average Precision) of 0.321 for text-to-audio retrieval."
}