diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index d4cfe1711a5e..e62a3d0524be 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -693,8 +693,6 @@ title: UL2 - local: model_doc/umt5 title: UMT5 - - local: model_doc/xcodec - title: X-CODEC - local: model_doc/xmod title: X-MOD - local: model_doc/xglm @@ -945,6 +943,8 @@ title: WavLM - local: model_doc/whisper title: Whisper + - local: model_doc/xcodec + title: X-Codec - local: model_doc/xls_r title: XLS-R - local: model_doc/xlsr_wav2vec2 diff --git a/docs/source/en/model_doc/xcodec.md b/docs/source/en/model_doc/xcodec.md index 959c6b71207a..c4a0b92a26f6 100644 --- a/docs/source/en/model_doc/xcodec.md +++ b/docs/source/en/model_doc/xcodec.md @@ -1,4 +1,4 @@ - +*This model was released on 2024-08-30 and added to Hugging Face Transformers on 2025-08-15.* # X-Codec @@ -22,23 +23,28 @@ rendered properly in your Markdown viewer. ## Overview -The X-Codec model was proposed in [Codec Does Matter: Exploring the Semantic Shortcoming of Codec for Audio Language Model](https://arxiv.org/abs/2408.17175) by Zhen Ye, Peiwen Sun, Jiahe Lei, Hongzhan Lin, Xu Tan, Zheqi Dai, Qiuqiang Kong, Jianyi Chen, Jiahao Pan, Qifeng Liu, Yike Guo, Wei Xue +The X-Codec model was proposed in [Codec Does Matter: Exploring the Semantic Shortcoming of Codec for Audio Language Model](https://huggingface.co/papers/2408.17175) by Zhen Ye, Peiwen Sun, Jiahe Lei, Hongzhan Lin, Xu Tan, Zheqi Dai, Qiuqiang Kong, Jianyi Chen, Jiahao Pan, Qifeng Liu, Yike Guo, Wei Xue. -The X-Codec model is a neural audio codec that integrates semantic information from self-supervised models (e.g., HuBERT) alongside traditional acoustic information. This enables : +The X-Codec model is a neural audio codec that integrates semantic information from self-supervised models (e.g., HuBERT) alongside traditional acoustic information. This enables: -- **Music continuation** : Better modeling of musical semantics yields more coherent continuations. -- **Text-to-Sound Synthesis** : X-Codec captures semantic alignment between text prompts and generated audio. +- **Music continuation**: Better modeling of musical semantics yields more coherent continuations. +- **Text-to-Sound Synthesis**: X-Codec captures semantic alignment between text prompts and generated audio. - **Semantic aware audio tokenization**: X-Codec is used as an audio tokenizer in the YuE lyrics to song generation model. The abstract of the paper states the following: *Recent advancements in audio generation have been significantly propelled by the capabilities of Large Language Models (LLMs). The existing research on audio LLM has primarily focused on enhancing the architecture and scale of audio language models, as well as leveraging larger datasets, and generally, acoustic codecs, such as EnCodec, are used for audio tokenization. However, these codecs were originally designed for audio compression, which may lead to suboptimal performance in the context of audio LLM. Our research aims to address the shortcomings of current audio LLM codecs, particularly their challenges in maintaining semantic integrity in generated audio. For instance, existing methods like VALL-E, which condition acoustic token generation on text transcriptions, often suffer from content inaccuracies and elevated word error rates (WER) due to semantic misinterpretations of acoustic tokens, resulting in word skipping and errors. To overcome these issues, we propose a straightforward yet effective approach called X-Codec. X-Codec incorporates semantic features from a pre-trained semantic encoder before the Residual Vector Quantization (RVQ) stage and introduces a semantic reconstruction loss after RVQ. By enhancing the semantic ability of the codec, X-Codec significantly reduces WER in speech synthesis tasks and extends these benefits to non-speech applications, including music and sound generation. Our experiments in text-to-speech, music continuation, and text-to-sound tasks demonstrate that integrating semantic information substantially improves the overall performance of language models in audio generation.* -Demos can be found in this [post](https://x-codec-audio.github.io/). +Model cards: +- [xcodec-hubert-librispeech](https://huggingface.co/hf-audio/xcodec-hubert-librispeech) (for speech) +- [xcodec-wavlm-mls](https://huggingface.co/hf-audio/xcodec-wavlm-mls) (for speech) +- [xcodec-wavlm-more-data](https://huggingface.co/hf-audio/xcodec-wavlm-more-data) (for speech) +- [xcodec-hubert-general](https://huggingface.co/hf-audio/xcodec-hubert-general) (for general audio) +- [xcodec-hubert-general-balanced](https://huggingface.co/hf-audio/xcodec-hubert-general-balanced) (for general audio) +This model was contributed by [Manal El Aidouni](https://huggingface.co/Manel). The original code can be found [here](https://github.com/zhenye234/xcodec) and original checkpoints for the five different models [here](https://github.com/zhenye234/xcodec?tab=readme-ov-file#available-models). -This model was contributed by [Manal El Aidouni](https://huggingface.co/Manel). The original code can be found [here](https://github.com/zhenye234/xcodec) and original checkpoint [here](https://huggingface.co/ZhenYe234/xcodec/blob/main/xcodec_speech_hubert_librispeech.pth). - +Demos can be found on this [page](https://x-codec-audio.github.io/). ## Usage example @@ -51,13 +57,16 @@ from transformers import XcodecModel, AutoFeatureExtractor dummy_dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") # load model and feature extractor -model = XcodecModel.from_pretrained("Manel/X-Codec") -feature_extractor = AutoFeatureExtractor.from_pretrained("Manel/X-Codec") +model_id = "hf-audio/xcodec-hubert-librispeech" +model = XcodecModel.from_pretrained(model_id) +feature_extractor = AutoFeatureExtractor.from_pretrained(model_id) + # load audio sample dummy_dataset = dummy_dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate)) audio_sample = dummy_dataset[-1]["audio"]["array"] inputs = feature_extractor(raw_audio=audio_sample, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt") +# encode and decode encoder_outputs = model.encode(inputs["input_values"]) decoder_outputs = model.decode(encoder_outputs.audio_codes) audio_values = decoder_outputs.audio_values diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index f4666c7c56f9..0307aeba077f 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -115,7 +115,7 @@ ("wavlm", "Wav2Vec2FeatureExtractor"), ("whisper", "WhisperFeatureExtractor"), ("xclip", "CLIPFeatureExtractor"), - ("xcodec", "EncodecFeatureExtractor"), + ("xcodec", "DacFeatureExtractor"), ("yolos", "YolosFeatureExtractor"), ] ) diff --git a/src/transformers/models/xcodec/__init__.py b/src/transformers/models/xcodec/__init__.py index 45e7620f54d7..17c79e8aeeed 100644 --- a/src/transformers/models/xcodec/__init__.py +++ b/src/transformers/models/xcodec/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. +# Copyright 2025 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/transformers/models/xcodec/configuration_xcodec.py b/src/transformers/models/xcodec/configuration_xcodec.py index 45bd1077af2e..959838cc1d57 100644 --- a/src/transformers/models/xcodec/configuration_xcodec.py +++ b/src/transformers/models/xcodec/configuration_xcodec.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2024 Descript and The HuggingFace Inc. team. All rights reserved. +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ import numpy as np -from transformers import DacConfig, HubertConfig +from transformers import AutoConfig, DacConfig, HubertConfig, WavLMConfig from ...configuration_utils import PretrainedConfig from ...utils import logging @@ -41,14 +41,8 @@ class XcodecConfig(PretrainedConfig): Args: target_bandwidths (`List[float]`, *optional*, defaults to `[0.5, 1, 1.5, 2, 4]`): The range of different bandwidths (in kbps) the model can encode audio with. - audio_channels (`int`, *optional*, defaults to 1): - Number of channels in the audio data. Either 1 for mono or 2 for stereo. sample_rate (`int`, *optional*, defaults to 16000): The sampling rate at which the audio waveform should be digitalized, in hertz (Hz). - input_channels (`int`, *optional*, defaults to 768): - Number of channels of the input to the first convolution in the semantic encoder. - encoder_channels (`int`, *optional*, defaults to 768): - Number of hidden channels in each semantic encoder block. kernel_size (`int`, *optional*, defaults to 3): Kernel size for the initial semantic convolution. channel_ratios (`List[float]`, *optional*, defaults to `[1, 1]`): @@ -59,27 +53,15 @@ class XcodecConfig(PretrainedConfig): Dilation factors for the residual units in semantic blocks. unit_kernel_size (`int`, *optional*, defaults to 3): Kernel size inside each ResidualUnit in semantic blocks. - decoder_channels (`int`, *optional*, defaults to 768): - Number of hidden channels in each semantic decoder block. - output_channels (`int`, *optional*, defaults to 768): - Number of output channels in the semantic decoder. codebook_size (`int`, *optional*, defaults to 1024): - Number of entries in each residual quantizer’s codebook. - num_quantizers (`int`, *optional*, defaults to 8): - Number of sequential quantizers (codebooks) in the RVQ stack. - codebook_dim (`int`, *optional*, defaults to 1024): - Dimensionality of each codebook vector. + Number of entries in each residual quantizer's codebook. + codebook_dim (`int`, *optional*): + Dimensionality of each codebook vector. Defaults to sum of hidden size of acoustic and semantic models. initializer_range (`float`, *optional*, defaults to 0.02): Standard deviation of the truncated normal initializer for all weight matrices. - hidden_dim (`int`, *optional*, defaults to 1024): - Dimensionality of the joint acoustic+semantic FC layer. - intermediate_dim (`int`, *optional*, defaults to 768): - Dimensionality of the next FC layer in the decoder path. - output_dim (`int`, *optional*, defaults to 256): - Dimensionality of the final FC layer before feeding into the acoustic decoder. acoustic_model_config (`Union[Dict, DacConfig]`, *optional*): An instance of the configuration for the acoustic (DAC) model. - semantic_model_config (`Union[Dict, HubertConfig]`, *optional*): + semantic_model_config (`Union[Dict, HubertConfig, WavLMConfig]`, *optional*): An instance of the configuration object for the semantic (HuBERT) model. Example: @@ -87,10 +69,10 @@ class XcodecConfig(PretrainedConfig): ```python >>> from transformers import XcodecModel, XcodecConfig - >>> # Initializing a " " style configuration + >>> # Initializing configuration >>> configuration = XcodecConfig() - >>> # Initializing a model (with random weights) from the " " style configuration + >>> # Initializing a model (with random weights) from the configuration >>> model = XcodecModel(configuration) >>> # Accessing the model configuration @@ -101,30 +83,21 @@ class XcodecConfig(PretrainedConfig): sub_configs = { "acoustic_model_config": DacConfig, - "semantic_model_config": HubertConfig, + "semantic_model_config": AutoConfig, } def __init__( self, target_bandwidths: Optional[list[float]] = None, - audio_channels: int = 1, sample_rate: int = 16000, - input_channels: int = 768, - encoder_channels: int = 768, kernel_size: int = 3, channel_ratios: list[float] = [1, 1], strides: list[int] = [1, 1], block_dilations: list[int] = [1, 1], unit_kernel_size: int = 3, - decoder_channels: int = 768, - output_channels: int = 768, codebook_size: int = 1024, - num_quantizers: int = 8, - codebook_dim: int = 1024, + codebook_dim: Optional[int] = None, initializer_range: float = 0.02, - hidden_dim: int = 1024, - intermediate_dim: int = 768, - output_dim: int = 256, acoustic_model_config: Union[dict, DacConfig] = None, semantic_model_config: Union[dict, HubertConfig] = None, **kwargs, @@ -134,6 +107,8 @@ def __init__( if acoustic_model_config is None: self.acoustic_model_config = DacConfig( encoder_hidden_size=64, + # NOTE: original DAC uses [2, 4, 8, 8] `downsampling ratios`, namely reverse of `upsampling_ratios` + # (not sure if intentional by Xcodec but we keep it) downsampling_ratios=[8, 5, 4, 2], decoder_hidden_size=1024, upsampling_ratios=[8, 5, 4, 2], @@ -143,44 +118,69 @@ def __init__( self.acoustic_model_config = DacConfig(**acoustic_model_config) elif isinstance(acoustic_model_config, DacConfig): self.acoustic_model_config = acoustic_model_config + else: + raise ValueError( + f"acoustic_model_config must be a dict or DacConfig instance, but got {type(acoustic_model_config)}" + ) if semantic_model_config is None: self.semantic_model_config = HubertConfig() elif isinstance(semantic_model_config, dict): - self.semantic_model_config = HubertConfig(**semantic_model_config) - elif isinstance(semantic_model_config, HubertConfig): + if "_name_or_path" in semantic_model_config: + # If the config is a path, load it using AutoConfig + self.semantic_model_config = AutoConfig.from_pretrained(semantic_model_config["_name_or_path"]) + else: + # assume HubertConfig as probably created from scratch + logger.warning( + "Could not determine semantic model type from config architecture. Defaulting to `HubertConfig`." + ) + self.semantic_model_config = HubertConfig(**semantic_model_config) + elif isinstance(semantic_model_config, WavLMConfig) or isinstance(semantic_model_config, HubertConfig): self.semantic_model_config = semantic_model_config + else: + raise ValueError( + f"semantic_model_config must be a dict, HubertConfig, or WavLMConfig instance, but got {type(semantic_model_config)}" + ) if target_bandwidths is None: target_bandwidths = [0.5, 1, 1.5, 2, 4] self.target_bandwidths = target_bandwidths - self.audio_channels = audio_channels self.sample_rate = sample_rate - self.input_channels = input_channels - self.encoder_channels = encoder_channels self.kernel_size = kernel_size self.channel_ratios = channel_ratios self.strides = strides self.block_dilations = block_dilations self.unit_kernel_size = unit_kernel_size - self.decoder_channels = decoder_channels - self.output_channels = output_channels self.codebook_size = codebook_size - self.num_quantizers = num_quantizers - self.codebook_dim = codebook_dim self.initializer_range = initializer_range - self.hidden_dim = hidden_dim - self.intermediate_dim = intermediate_dim - self.output_dim = output_dim + if codebook_dim is None: + codebook_dim = self.acoustic_model_config.hidden_size + self.semantic_model_config.hidden_size + self.codebook_dim = codebook_dim @property def frame_rate(self) -> int: - return math.ceil(self.sample_rate / np.prod(self.acoustic_model_config.upsampling_ratios)) + return math.ceil(self.sample_rate / self.hop_length) + + @property + def semantic_hidden_size(self) -> int: + return self.semantic_model_config.hidden_size @property def hop_length(self) -> int: return int(np.prod(self.acoustic_model_config.downsampling_ratios)) + @property + def codebook_nbits(self) -> int: + return math.ceil(math.log2(self.codebook_size)) + + @property + def hidden_size(self) -> int: + return self.acoustic_model_config.hidden_size + self.semantic_model_config.hidden_size + + @property + def num_quantizers(self) -> int: + return int(1000 * self.target_bandwidths[-1] // (self.frame_rate * self.codebook_nbits)) + __all__ = ["XcodecConfig"] diff --git a/src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py b/src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py index 30783234fc2b..e0ea399cb4af 100644 --- a/src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py +++ b/src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2024 Descript and The HuggingFace Inc. team. All rights reserved. +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,9 +17,11 @@ import re import torch +import yaml from transformers import ( - EncodecFeatureExtractor, + AutoConfig, + DacFeatureExtractor, XcodecConfig, XcodecModel, logging, @@ -178,15 +180,50 @@ def convert_old_keys_to_new_keys(original_state_dict: dict[str, torch.Tensor]) - return converted_checkpoint +# for reference, see original implementation: https://github.com/zhenye234/xcodec/blob/main/models/soundstream_semantic.py#L24 @torch.no_grad() -def convert_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=None, push_to_hub=None): - if config_path is not None: - config = XcodecConfig.from_pretrained(config_path) +def convert_checkpoint(checkpoint_path, config_path, pytorch_dump_folder_path=None, push_to_hub=None): + # load config yaml file + with open(config_path, "r") as f: + model_config = yaml.safe_load(f) + + # extra relevant parameters + ratios = model_config["generator"]["config"]["ratios"] + target_bandwidths = model_config["generator"]["config"]["target_bandwidths"] + sample_rate = model_config["generator"]["config"]["sample_rate"] + acoustic_model_config = { + "encoder_hidden_size": 64, + "decoder_hidden_size": 1024, + # NOTE: original DAC uses [2, 4, 8, 8] `downsampling ratios`, namely reverse of `upsampling_ratios` + # (not sure if intentional by Xcodec but we keep it) + "downsampling_ratios": ratios, + "upsampling_ratios": ratios, + "sampling_rate": sample_rate, + "hidden_size": model_config["generator"]["config"]["D"], + } + semantic_model = model_config["generator"]["config"]["semantic_techer"] + if semantic_model == "hubert_base": + semantic_model_config = AutoConfig.from_pretrained("facebook/hubert-base-ls960") + elif semantic_model == "wavlm_base_plus": + semantic_model_config = AutoConfig.from_pretrained("microsoft/wavlm-base-plus") + elif semantic_model == "hubert_base_general": + semantic_model_config = AutoConfig.from_pretrained("ZhenYe234/hubert_base_general_audio") else: - config = XcodecConfig() + raise ValueError(f"Unknown semantic model: {semantic_model}") + + config = XcodecConfig( + target_bandwidths=target_bandwidths, + acoustic_model_config=acoustic_model_config, + semantic_model_config=semantic_model_config, + sample_rate=sample_rate, + codebook_size=model_config["generator"]["config"]["bins"], + ) - with torch.device("meta"): - model = XcodecModel(config) + # create model + if not torch.cuda.is_available(): + raise ValueError("Run this script on a machine with a GPU for weight norm layers to be correctly copied.") + torch_device = "cuda" + model = XcodecModel(config).to(torch_device) logger.info("Loading original checkpoint ...") @@ -208,12 +245,15 @@ def convert_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=No raise ValueError(f"missing keys found: {missing_keys}") model.remove_weight_norm() + if pytorch_dump_folder_path is not None: + model.save_pretrained(pytorch_dump_folder_path) - model.save_pretrained(pytorch_dump_folder_path) - - feature_extractor = EncodecFeatureExtractor(feature_size=config.audio_channels, sampling_rate=config.sample_rate) - - feature_extractor.save_pretrained(pytorch_dump_folder_path) + feature_extractor = DacFeatureExtractor( + sampling_rate=config.sample_rate, + hop_length=config.acoustic_model_config.hop_length, + ) + if pytorch_dump_folder_path is not None: + feature_extractor.save_pretrained(pytorch_dump_folder_path) if push_to_hub: print("Pushing to the hub...") @@ -221,13 +261,76 @@ def convert_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=No model.push_to_hub(push_to_hub) +""" +Models checkpoints can be downloaded from here: +https://github.com/zhenye234/xcodec?tab=readme-ov-file#available-models + +1) `xcodec_hubert_librispeech`: +``` +# Download config and checkpoint files +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/config_hubert.yaml -P /raid/eric/xcodec_original +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/xcodec_speech_hubert_librispeech.pth -P /raid/eric/xcodec_original +# Run conversion: +python src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py \ + --checkpoint_path /raid/eric/xcodec_original/xcodec_speech_hubert_librispeech.pth \ + --config_path /raid/eric/xcodec_original/config_hubert.yaml \ + --push_to_hub hf-audio/xcodec-hubert-librispeech +``` + +2) `xcodec_hubert_general_audio`: +``` +# Download config and checkpoint files +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/config_hubert_general.yaml -P /raid/eric/xcodec_original +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/xcodec_hubert_general_audio.pth -P /raid/eric/xcodec_original +# Run conversion: +python src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py \ + --checkpoint_path /raid/eric/xcodec_original/xcodec_hubert_general_audio.pth \ + --config_path /raid/eric/xcodec_original/config_hubert_general.yaml \ + --push_to_hub hf-audio/xcodec-hubert-general +``` + +3) `xcodec_hubert_general_audio_more_data` (more balanced dataset): +``` +# Download config and checkpoint files +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/config_hubert_general.yaml -P /raid/eric/xcodec_original +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/xcodec_hubert_general_audio_v2.pth -P /raid/eric/xcodec_original +# Run conversion: +python src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py \ + --checkpoint_path /raid/eric/xcodec_original/xcodec_hubert_general_audio_v2.pth \ + --config_path /raid/eric/xcodec_original/config_hubert_general.yaml \ + --push_to_hub hf-audio/xcodec-hubert-general-balanced +``` + +4) `xcodec_wavlm_mls`: +``` +# Download config and checkpoint files +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/config_wavlm.yaml -P /raid/eric/xcodec_original +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/xcodec_speech_wavlm_mls.pth -P /raid/eric/xcodec_original +# Run conversion: +python src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py \ + --checkpoint_path /raid/eric/xcodec_original/xcodec_speech_wavlm_mls.pth \ + --config_path /raid/eric/xcodec_original/config_wavlm.yaml \ + --push_to_hub hf-audio/xcodec-wavlm-mls +``` + +5) `xcodec_wavlm_more_data`: +``` +# Download config and checkpoint files +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/config_wavlm.yaml -P /raid/eric/xcodec_original +wget https://huggingface.co/ZhenYe234/xcodec/resolve/main/xcodec_speech_wavlm_more_data.pth -P /raid/eric/xcodec_original +# Run conversion: +python src/transformers/models/xcodec/convert_xcodec_weights_to_hf.py \ + --checkpoint_path /raid/eric/xcodec_original/xcodec_speech_wavlm_more_data.pth \ + --config_path /raid/eric/xcodec_original/config_wavlm.yaml \ + --push_to_hub hf-audio/xcodec-wavlm-more-data +""" if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--checkpoint_path", required=True, default=None, type=str, help="Path to original checkpoint") - parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert") parser.add_argument( - "--pytorch_dump_folder_path", required=True, default=None, type=str, help="Path to the output PyTorch model." + "--config_path", required=True, default=None, type=str, help="Path to hf config.yaml of model to convert" ) + parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") parser.add_argument( "--push_to_hub", default=None, type=str, help="Where to upload the converted model on the 🤗 hub." ) @@ -235,7 +338,7 @@ def convert_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=No args = parser.parse_args() convert_checkpoint( args.checkpoint_path, - args.pytorch_dump_folder_path, args.config_path, + args.pytorch_dump_folder_path, args.push_to_hub, ) diff --git a/src/transformers/models/xcodec/modeling_xcodec.py b/src/transformers/models/xcodec/modeling_xcodec.py index a62d0d2952bf..71064c7f1d3f 100644 --- a/src/transformers/models/xcodec/modeling_xcodec.py +++ b/src/transformers/models/xcodec/modeling_xcodec.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2024 Descript and The HuggingFace Inc. team. All rights reserved. +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -115,15 +115,19 @@ def __init__(self, config): super().__init__() if len(config.strides) != len(config.channel_ratios): raise ValueError("Number of strides must match the number of channel_ratios.") - self.conv = nn.Conv1d( - config.input_channels, config.encoder_channels, config.kernel_size, 1, config.kernel_size // 2, bias=False + config.semantic_hidden_size, + config.semantic_hidden_size, + config.kernel_size, + 1, + config.kernel_size // 2, + bias=False, ) - in_channels = config.encoder_channels + in_channels = config.semantic_hidden_size conv_blocks = [] for i, stride in enumerate(config.strides): - out_channels = int(config.encoder_channels * config.channel_ratios[i]) + out_channels = int(config.semantic_hidden_size * config.channel_ratios[i]) conv_blocks += [SemanticEncoderBlock(config, in_channels, out_channels, stride)] in_channels = out_channels @@ -171,8 +175,8 @@ class SemanticDecoder(nn.Module): def __init__(self, config): super().__init__() self.conv1 = nn.Conv1d( - in_channels=config.decoder_channels, - out_channels=int(config.decoder_channels * config.channel_ratios[0]), + in_channels=config.semantic_hidden_size, + out_channels=int(config.semantic_hidden_size * config.channel_ratios[0]), kernel_size=config.kernel_size, stride=1, padding=config.kernel_size // 2, @@ -180,19 +184,19 @@ def __init__(self, config): ) conv_blocks = [] for i, stride in enumerate(config.strides): - in_channels = int(config.decoder_channels * config.channel_ratios[i]) + in_channels = int(config.semantic_hidden_size * config.channel_ratios[i]) if i < (len(config.channel_ratios) - 1): - out_channels = int(config.decoder_channels * config.channel_ratios[i + 1]) + out_channels = int(config.semantic_hidden_size * config.channel_ratios[i + 1]) else: - out_channels = config.decoder_channels + out_channels = config.semantic_hidden_size conv_blocks += [SemanticDecoderBlock(config, in_channels, out_channels, stride)] self.conv_blocks = nn.ModuleList(conv_blocks) self.conv2 = nn.Conv1d( - config.decoder_channels, - config.output_channels, + config.semantic_hidden_size, + config.semantic_hidden_size, config.kernel_size, stride=1, padding=config.kernel_size // 2, @@ -381,16 +385,16 @@ def __init__(self, config): super().__init__(config) self.config = config self.pad = config.hop_length // 2 - dac = AutoModel.from_config(config.acoustic_model_config) - self.acoustic_encoder = dac.encoder - self.acoustic_decoder = dac.decoder + acoustic_model = AutoModel.from_config(config.acoustic_model_config) + self.acoustic_encoder = acoustic_model.encoder + self.acoustic_decoder = acoustic_model.decoder self._adjust_dac_decoder(self.acoustic_decoder) self.encoder_semantic = SemanticEncoder(config) self.decoder_semantic = SemanticDecoder(config) - self.semantic_model = AutoModel.from_config(config.semantic_model_config) - self.fc = nn.Linear(config.hidden_dim, config.hidden_dim) - self.fc1 = nn.Linear(config.hidden_dim, config.intermediate_dim) - self.fc2 = nn.Linear(config.hidden_dim, config.output_dim) + self.semantic_model = AutoModel.from_config(config.semantic_model_config).eval() + self.fc = nn.Linear(config.hidden_size, config.hidden_size) + self.fc1 = nn.Linear(config.hidden_size, config.semantic_model_config.hidden_size) + self.fc2 = nn.Linear(config.hidden_size, config.acoustic_model_config.hidden_size) self.quantizer = XcodecResidualVectorQuantization(config) @staticmethod @@ -524,16 +528,16 @@ def forward( return_dict: Optional[bool] = None, **kwargs, ) -> Union[tuple[torch.Tensor, torch.Tensor], XcodecOutput]: - r""" + """ Encodes and quantizes the input audio into discrete codes, then decodes those codes back into an audio waveform. + Args: input_values (`torch.FloatTensor` of shape `(batch_size, channels, num_samples)`): The raw float values of the input audio waveform. audio_codes (`torch.LongTensor` of shape `(batch_size, num_quantizers, codes_length)`: Discrete code indices computed using `model.encode`. bandwidth (`float`, *optional*): - Target bandwidth in kbps. Must be one of `config.target_bandwidths`. - Defaults to the highest available bandwidth. + Target bandwidth in kbps. Must be one of `config.target_bandwidths`. Defaults to the highest available bandwidth. return_dict (`bool`, *optional*): Whether to return a [`XcodecOutput`] instead of a plain tuple. @@ -548,13 +552,14 @@ def forward( >>> from datasets import load_dataset >>> from transformers import AutoFeatureExtractor, XcodecModel - >>> dataset = load_dataset("hf-internal-testing/ashraq-esc50-1-dog-example") - >>> audio_sample = dataset["train"]["audio"][0]["array"] - - >>> model_id = "Manel/X-Codec" + >>> model_id = "hf-audio/xcodec-hubert-librispeech" >>> model = XcodecModel.from_pretrained(model_id) >>> feature_extractor = AutoFeatureExtractor.from_pretrained(model_id) + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> dataset = dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate)) + >>> audio_sample = dataset[0]['audio']['array'] + >>> inputs = feature_extractor(raw_audio=audio_sample, return_tensors="pt") >>> outputs = model(**inputs) diff --git a/tests/fixtures/xcodec/integration_tests.json b/tests/fixtures/xcodec/integration_tests.json new file mode 100644 index 000000000000..6c4e436350f2 --- /dev/null +++ b/tests/fixtures/xcodec/integration_tests.json @@ -0,0 +1,2150 @@ +[ + { + "repo_id": "hf-audio/xcodec-hubert-librispeech", + "bandwidth": 0.5, + "codes": [ + [ + [ + 590, + 590, + 306, + 306, + 590, + 1006, + 826, + 916, + 590, + 826, + 826, + 844, + 844, + 746, + 392 + ] + ] + ], + "decoded": [ + [ + [ + -0.000644786050543189, + 0.00013058912009000778, + 0.0006741542601957917, + 0.00034384033642709255, + -0.00010847719386219978, + 0.00014182832092046738, + 6.248801946640015e-05, + -0.0003324758727103472, + -0.0005267199594527483, + -0.0010175877250730991, + -0.0011251929681748152, + -0.0006446582265198231, + -0.00047831423580646515, + -0.00023265788331627846, + -0.0001271876972168684, + -1.8235761672258377e-05, + -5.327444523572922e-05, + -0.0001765203196555376, + -0.0004240723792463541, + -0.0002993491943925619, + -0.0002642744220793247, + -0.0003135430160909891, + -0.0002605144400149584, + -0.00026806839741766453, + -2.2151507437229156e-06, + 0.00020786607638001442, + 0.00017593754455447197, + -0.00016461312770843506, + -0.0003985136281698942, + -0.00025607552379369736, + -0.00013757869601249695, + -6.365473382174969e-05, + -0.00020918482914566994, + -0.0002473779022693634, + -0.00014893501065671444, + 0.0001332222018390894, + 0.00040268548764288425, + 0.0002646979410201311, + 7.32732005417347e-05, + -0.00025837705470621586, + -0.0007715015672147274, + -0.0012102078180760145, + -0.0009501995518803596, + -0.00017246650531888008, + 0.0003386975731700659, + 0.0003189577255398035, + 0.00021003605797886848, + 0.0001099675428122282, + 0.00024433922953903675, + 3.217277117073536e-05 + ] + ] + ], + "codec_error": 0.005472081247717142, + "codec_tol": 1e-05, + "dec_tol": 0.001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-librispeech", + "bandwidth": 1.0, + "codes": [ + [ + [ + 590, + 590, + 306, + 306, + 590, + 1006, + 826, + 916, + 590, + 826, + 826, + 844, + 844, + 746, + 392 + ], + [ + 748, + 375, + 156, + 327, + 327, + 156, + 448, + 448, + 327, + 346, + 1023, + 346, + 346, + 1023, + 1023 + ] + ] + ], + "decoded": [ + [ + [ + -0.00022301427088677883, + 0.00018040905706584454, + 0.00034872093237936497, + 0.00010567391291260719, + -0.0002222477924078703, + -0.00012606964446604252, + -7.731537334620953e-05, + -0.00012137391604483128, + -0.0001762432511895895, + -0.0007736883126199245, + -0.0010079797357320786, + -0.00046425615437328815, + -7.964251562952995e-05, + -3.2845884561538696e-05, + -1.0334886610507965e-05, + 8.499552495777607e-05, + -0.00010045897215604782, + -0.00029115937650203705, + -0.00044626137241721153, + -0.00014261016622185707, + -0.00036265188828110695, + -0.0005839171353727579, + -0.00024048984050750732, + 1.8859747797250748e-05, + 5.6158751249313354e-06, + 0.00017515220679342747, + 0.0004101542290300131, + 0.0001747957430779934, + -0.0003047185018658638, + -0.00032516056671738625, + -0.0002171192318201065, + -0.0002566229086369276, + -0.00022618728689849377, + 8.058198727667332e-05, + 0.00029059057123959064, + 0.000731412204913795, + 0.0009413085645064712, + 0.0005381645169109106, + -7.746554911136627e-05, + -0.0005039803218096495, + -0.0008559448178857565, + -0.0010777676943689585, + -0.0005720553454011679, + 0.00046507292427122593, + 0.0009388792095705867, + 0.0007773893885314465, + 0.000560535816475749, + 0.00030490709468722343, + 0.00010391278192400932, + -0.0004857585299760103 + ] + ] + ], + "codec_error": 0.005680468864738941, + "codec_tol": 1e-05, + "dec_tol": 0.001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-librispeech", + "bandwidth": 1.5, + "codes": [ + [ + [ + 590, + 590, + 306, + 306, + 590, + 1006, + 826, + 916, + 590, + 826, + 826, + 844, + 844, + 746, + 392 + ], + [ + 748, + 375, + 156, + 327, + 327, + 156, + 448, + 448, + 327, + 346, + 1023, + 346, + 346, + 1023, + 1023 + ], + [ + 351, + 835, + 351, + 992, + 992, + 817, + 127, + 832, + 855, + 177, + 817, + 177, + 177, + 97, + 472 + ] + ] + ], + "decoded": [ + [ + [ + -8.131377398967743e-05, + 0.0003722899127751589, + 0.0005264813080430031, + 0.00024720048531889915, + -0.00012781796976923943, + -6.020977161824703e-05, + 2.0435545593500137e-06, + -9.971903637051582e-06, + -5.366327241063118e-05, + -0.0007150094024837017, + -0.0009912890382111073, + -0.00038447463884949684, + 5.855737254023552e-05, + 0.00012949109077453613, + 0.00017356942407786846, + 0.0002417820505797863, + -4.827766679227352e-05, + -0.00032059219665825367, + -0.0004915285389870405, + -0.00013681501150131226, + -0.00028307782486081123, + -0.00039494410157203674, + 5.228118970990181e-05, + 0.0003737024962902069, + 0.0003404058516025543, + 0.0004928638227283955, + 0.000665064319036901, + 0.00023967307060956955, + -0.000433565117418766, + -0.0005353728774935007, + -0.00038189860060811043, + -0.00032747630029916763, + -0.00019068107940256596, + 0.0002735431771725416, + 0.0006756627699360251, + 0.0012527555227279663, + 0.0013769968645647168, + 0.000728685175999999, + -0.0001932412851601839, + -0.0009031407535076141, + -0.0014542911667376757, + -0.0017141078133136034, + -0.001031694933772087, + 0.00032188743352890015, + 0.0010538509814068675, + 0.0010152260074391961, + 0.0008743192302063107, + 0.0006233796011656523, + 0.0002936923410743475, + -0.0004914039745926857 + ] + ] + ], + "codec_error": 0.005860992707312107, + "codec_tol": 1e-05, + "dec_tol": 0.001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-librispeech", + "bandwidth": 2.0, + "codes": [ + [ + [ + 590, + 590, + 306, + 306, + 590, + 1006, + 826, + 916, + 590, + 826, + 826, + 844, + 844, + 746, + 392 + ], + [ + 748, + 375, + 156, + 327, + 327, + 156, + 448, + 448, + 327, + 346, + 1023, + 346, + 346, + 1023, + 1023 + ], + [ + 351, + 835, + 351, + 992, + 992, + 817, + 127, + 832, + 855, + 177, + 817, + 177, + 177, + 97, + 472 + ], + [ + 156, + 1008, + 546, + 266, + 856, + 116, + 173, + 531, + 894, + 938, + 87, + 531, + 745, + 747, + 747 + ] + ] + ], + "decoded": [ + [ + [ + 6.87665306031704e-05, + 0.0005173773970454931, + 0.0006601555505767465, + 0.0004073025193065405, + 2.4633249267935753e-05, + 6.317137740552425e-05, + 0.00011769519187510014, + 0.00010632933117449284, + 6.606383249163628e-05, + -0.0006399150006473064, + -0.0008867487777024508, + -0.00026029557920992374, + 0.00015681260265409946, + 0.00021465751342475414, + 0.00026818737387657166, + 0.00039722840301692486, + 0.00010386388748884201, + -0.0001172386109828949, + -0.0002974425442516804, + 6.535928696393967e-05, + -0.00014087161980569363, + -0.0002665710635483265, + 0.00022345990873873234, + 0.0005574806127697229, + 0.0005362457595765591, + 0.0006526130018755794, + 0.0007966818520799279, + 0.0003330092877149582, + -0.00044982251711189747, + -0.0006805681623518467, + -0.0005900394171476364, + -0.0005267017986625433, + -0.00036593549884855747, + 0.00016947905533015728, + 0.0006881445879116654, + 0.001375970197841525, + 0.0014863089891150594, + 0.0008116817334666848, + -8.753198198974133e-05, + -0.0008385318797081709, + -0.0014865880366414785, + -0.0017615130636841059, + -0.0009944913908839226, + 0.00041615008376538754, + 0.0011417579371482134, + 0.001129985903389752, + 0.0010677824029698968, + 0.0007883547805249691, + 0.00044182222336530685, + -0.0003320761024951935 + ] + ] + ], + "codec_error": 0.005977352149784565, + "codec_tol": 1e-05, + "dec_tol": 0.001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-librispeech", + "bandwidth": 4.0, + "codes": null, + "decoded": [ + [ + [ + 4.919269122183323e-05, + 0.00044998968951404095, + 0.0005170812364667654, + 0.00038593681529164314, + 0.00012237182818353176, + 6.045890040695667e-05, + 7.303990423679352e-05, + 0.00023534847423434258, + 0.00028711999766528606, + -0.0004980487283319235, + -0.0007807251531630754, + -0.0001900549978017807, + 0.00020040222443640232, + 0.00022384990006685257, + 0.00028798868879675865, + 0.0005522638093680143, + 0.0003211870789527893, + 5.3691910579800606e-05, + -0.0002666471991688013, + 3.6196084693074226e-05, + -0.00030040694400668144, + -0.0006298250518739223, + -0.00017189816571772099, + 0.0004776653368026018, + 0.0007132112514227629, + 0.0007824968779459596, + 0.0010266141034662724, + 0.0007283660816028714, + -0.00011806422844529152, + -0.0006026057526469231, + -0.0005629444494843483, + -0.00034018163569271564, + -0.0001810593530535698, + 0.00023290864191949368, + 0.0007565215928480029, + 0.001519701792858541, + 0.0016510691493749619, + 0.0009673768654465675, + 0.00013116677291691303, + -0.0004444648511707783, + -0.001082052243873477, + -0.001428384566679597, + -0.0006310269236564636, + 0.0007636576192453504, + 0.0013155697379261255, + 0.001216168631799519, + 0.0012932788813486695, + 0.0010688184993341565, + 0.0005435235798358917, + -0.00032686395570635796 + ] + ] + ], + "codec_error": 0.00608655484393239, + "codec_tol": 1e-05, + "dec_tol": 0.001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general", + "bandwidth": 0.5, + "codes": [ + [ + [ + 935, + 433, + 126, + 803, + 850, + 448, + 917, + 387, + 387, + 592, + 592, + 855, + 917, + 572, + 28 + ] + ] + ], + "decoded": [ + [ + [ + -0.00021017552353441715, + 0.0004965162370353937, + 0.0001386473886668682, + -0.00020841951481997967, + -8.969777263700962e-05, + -0.0002771597355604172, + -0.00027932715602219105, + 5.5042095482349396e-05, + 0.00026329560205340385, + 0.00017548329196870327, + 0.00038894289173185825, + 0.0005539481062442064, + 0.00014357827603816986, + -0.00031660543754696846, + -0.0005882629193365574, + -0.00044182687997817993, + -0.0001490083523094654, + -5.8116158470511436e-05, + 4.0457816794514656e-05, + -7.796147838234901e-05, + 3.5779085010290146e-06, + 0.00017821392975747585, + 9.860633872449398e-05, + 0.00010030693374574184, + 1.2427102774381638e-05, + -0.0001045640092343092, + -9.76896844804287e-05, + -5.82672655582428e-05, + 0.00017144251614809036, + 7.190275937318802e-06, + -0.00019244407303631306, + 5.330890417098999e-05, + 0.0001915388274937868, + 2.933409996330738e-05, + -9.671831503510475e-05, + -0.00038430141285061836, + -0.000596470432356, + -0.00054143276065588, + -0.0003454047255218029, + -0.0001294529065489769, + -0.00019761803559958935, + -0.00011440459638834, + 1.8423888832330704e-05, + 5.5767595767974854e-05, + 0.00012666056863963604, + -9.990530088543892e-06, + -8.841603994369507e-05, + -7.79847614467144e-05, + -0.0001224260777235031, + -9.860680438578129e-05 + ] + ] + ], + "codec_error": 0.0038952771574258804, + "codec_tol": 1e-06, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general", + "bandwidth": 1.0, + "codes": [ + [ + [ + 935, + 433, + 126, + 803, + 850, + 448, + 917, + 387, + 387, + 592, + 592, + 855, + 917, + 572, + 28 + ], + [ + 739, + 882, + 882, + 49, + 459, + 64, + 189, + 459, + 459, + 459, + 143, + 551, + 550, + 760, + 808 + ] + ] + ], + "decoded": [ + [ + [ + -0.0002046455629169941, + 0.00046922592446208, + 0.00011031189933419228, + -0.0002996474504470825, + -0.00021691364236176014, + -0.00039181788451969624, + -0.0003864085301756859, + -4.2735831812024117e-05, + 0.00022559752687811852, + 0.00019318307749927044, + 0.00036341347731649876, + 0.00047189113683998585, + 5.879811942577362e-05, + -0.00035421736538410187, + -0.0005878964439034462, + -0.00047568860463798046, + -0.00024429219774901867, + -8.333590812981129e-05, + 9.222817607223988e-05, + -2.709287218749523e-05, + 2.5267712771892548e-05, + 0.00020193355157971382, + 0.00010740640573203564, + 1.998012885451317e-05, + -0.00012808036990463734, + -0.00025810301303863525, + -0.00024647475220263004, + -0.0001851061824709177, + 6.030406802892685e-05, + -3.3717602491378784e-05, + -0.00014680484309792519, + 5.045789293944836e-05, + 3.881426528096199e-05, + -0.00020871451124548912, + -0.0002821478992700577, + -0.0005010757595300674, + -0.0007123809773474932, + -0.0006850638892501593, + -0.0005063675343990326, + -0.00030805286951363087, + -0.0003586956299841404, + -0.00024597207084298134, + -0.0001272936351597309, + -0.0001404962968081236, + -9.55783762037754e-05, + -0.00024282699450850487, + -0.0003365979064255953, + -0.000235437648370862, + -0.00011085602454841137, + -1.4352379366755486e-05 + ] + ] + ], + "codec_error": 0.003511926392093301, + "codec_tol": 1e-06, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general", + "bandwidth": 1.5, + "codes": [ + [ + [ + 935, + 433, + 126, + 803, + 850, + 448, + 917, + 387, + 387, + 592, + 592, + 855, + 917, + 572, + 28 + ], + [ + 739, + 882, + 882, + 49, + 459, + 64, + 189, + 459, + 459, + 459, + 143, + 551, + 550, + 760, + 808 + ], + [ + 710, + 536, + 176, + 531, + 623, + 626, + 100, + 833, + 796, + 311, + 457, + 382, + 360, + 176, + 410 + ] + ] + ], + "decoded": [ + [ + [ + -0.0001305723562836647, + 0.0007112827152013779, + 0.0003338572569191456, + -6.938702426850796e-05, + 0.0001109603326767683, + -7.854425348341465e-05, + -5.859602242708206e-05, + 0.00043661193922162056, + 0.0007921892683953047, + 0.0007424359209835529, + 0.0009273507166653872, + 0.0010653517674654722, + 0.0005372778978198767, + -4.121800884604454e-05, + -0.0003301908727735281, + -0.00012553343549370766, + 0.00018500350415706635, + 0.000404976774007082, + 0.0006721802055835724, + 0.0005569383502006531, + 0.0005787969566881657, + 0.0007312376983463764, + 0.0005475706420838833, + 0.00038789398968219757, + 0.00024302303791046143, + 0.0001421694178134203, + 0.00014890613965690136, + 0.000260709086433053, + 0.0005868605803698301, + 0.0004271466750651598, + 0.00019052578136324883, + 0.000402234960347414, + 0.00040456512942910194, + 8.718366734683514e-05, + 8.386559784412384e-07, + -0.00021422700956463814, + -0.00043249805457890034, + -0.00034649111330509186, + -9.317276999354362e-05, + 0.00012367917224764824, + -5.164183676242828e-06, + 4.87668439745903e-05, + 0.0001070518046617508, + 2.7181115001440048e-05, + 0.0001189960166811943, + 4.686368629336357e-05, + 7.493654265999794e-06, + 0.00016385293565690517, + 0.0002651643007993698, + 0.0002767094410955906 + ] + ] + ], + "codec_error": 0.0034364312887191772, + "codec_tol": 1e-06, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general", + "bandwidth": 2.0, + "codes": [ + [ + [ + 935, + 433, + 126, + 803, + 850, + 448, + 917, + 387, + 387, + 592, + 592, + 855, + 917, + 572, + 28 + ], + [ + 739, + 882, + 882, + 49, + 459, + 64, + 189, + 459, + 459, + 459, + 143, + 551, + 550, + 760, + 808 + ], + [ + 710, + 536, + 176, + 531, + 623, + 626, + 100, + 833, + 796, + 311, + 457, + 382, + 360, + 176, + 410 + ], + [ + 791, + 515, + 953, + 596, + 454, + 753, + 295, + 454, + 454, + 115, + 515, + 816, + 36, + 75, + 226 + ] + ] + ], + "decoded": [ + [ + [ + -0.00014006253331899643, + 0.0007989001460373402, + 0.00046538468450307846, + 8.566956967115402e-05, + 0.0002664625644683838, + 5.368725396692753e-05, + 4.951213486492634e-05, + 0.000605646288022399, + 0.001051453989930451, + 0.0011005131527781487, + 0.001340654562227428, + 0.0014765722444280982, + 0.0008794022724032402, + 0.00019451347179710865, + -0.0001761973835527897, + 1.8655788153409958e-05, + 0.0003523188643157482, + 0.0006215828470885754, + 0.0009482598397880793, + 0.0008765892125666142, + 0.0009128025267273188, + 0.0010658367536962032, + 0.0008650952950119972, + 0.0006931314710527658, + 0.0005274452269077301, + 0.00040836725383996964, + 0.0003828315529972315, + 0.00045729358680546284, + 0.0008139281999319792, + 0.0007075911853462458, + 0.0004986736457794905, + 0.0007380517199635506, + 0.0007470641285181046, + 0.0003727404400706291, + 0.00020861998200416565, + -4.448019899427891e-05, + -0.00026486185379326344, + -0.00014658598229289055, + 0.00014992570504546165, + 0.0003946979995816946, + 0.00024374201893806458, + 0.000254632206633687, + 0.00027234642766416073, + 0.00017000176012516022, + 0.0002736307214945555, + 0.00022304686717689037, + 0.00018746289424598217, + 0.0003647569101303816, + 0.00047829560935497284, + 0.0004925637040287256 + ] + ] + ], + "codec_error": 0.003246477572247386, + "codec_tol": 1e-06, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general", + "bandwidth": 4.0, + "codes": null, + "decoded": [ + [ + [ + -0.0001746553461998701, + 0.0008285697549581528, + 0.0005095258820801973, + 0.00022105872631072998, + 0.00047804228961467743, + 0.0002920464612543583, + 0.00020101177506148815, + 0.0006921763997524977, + 0.001189485308714211, + 0.001171361654996872, + 0.001430683652870357, + 0.0016735134413465858, + 0.0011466527357697487, + 0.0004955567419528961, + 0.00013427832163870335, + 0.0003044463228434324, + 0.0005627109203487635, + 0.0007822064217180014, + 0.001134557300247252, + 0.0010599115630611777, + 0.0011251196265220642, + 0.001351013546809554, + 0.001148079987615347, + 0.0009455143008381128, + 0.000764030497521162, + 0.0005778523627668619, + 0.00042620301246643066, + 0.0005069044418632984, + 0.0009447732008993626, + 0.0009011845104396343, + 0.0007778429426252842, + 0.0010773082030937076, + 0.0009713636245578527, + 0.00035730027593672276, + 0.00012435344979166985, + -0.0001019404735416174, + -0.0003732512705028057, + -0.00022046314552426338, + 0.0002396036870777607, + 0.0005130092613399029, + 0.00034346291795372963, + 0.00045024044811725616, + 0.000551396980881691, + 0.00038504088297486305, + 0.00042170868255198, + 0.000304151326417923, + 0.00014883745461702347, + 0.00035709445364773273, + 0.0005582491867244244, + 0.0006064062472432852 + ] + ] + ], + "codec_error": 0.0029708717484027147, + "codec_tol": 1e-06, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general-balanced", + "bandwidth": 0.5, + "codes": [ + [ + [ + 361, + 327, + 361, + 220, + 296, + 448, + 794, + 794, + 220, + 215, + 215, + 523, + 794, + 572, + 837 + ] + ] + ], + "decoded": [ + [ + [ + -0.00034493682323955, + 0.0004888690891675651, + 5.2613399020629004e-05, + -0.0003154197765979916, + -0.000374013528926298, + -0.0006679014186374843, + -0.0004468882689252496, + 0.00014481281687039882, + 0.0004187110753264278, + 0.0002444020356051624, + 0.0005527980392798781, + 0.0007324246689677238, + 0.0001953293103724718, + -0.0001868946710601449, + -0.00039330992149189115, + -1.5875579265411943e-05, + 0.0003087812219746411, + 0.00023550353944301605, + 0.00026388472178950906, + -1.399356551701203e-05, + 5.0710201321635395e-05, + 0.00025053517310880125, + 0.0001787259243428707, + 0.0004594939819071442, + 0.000588070775847882, + 0.00020284971105866134, + -0.0001789979578461498, + -0.0004223345313221216, + -0.0005676769069395959, + -0.0009293632465414703, + -0.0010678550461307168, + -0.0005041397525928915, + 4.17735063820146e-05, + 0.0003083720221184194, + 0.00022762120352126658, + -0.0001844105718191713, + -0.0003540224861353636, + -0.0004843481001444161, + -0.0005098123801872134, + -4.7943081881385297e-05, + 0.0003332827764097601, + 0.0006624205270782113, + 0.0007267107721418142, + 0.0005331048159860075, + 0.0005102302529849112, + 0.00026525993598625064, + -0.00011996129614999518, + -0.0003491249808575958, + -0.00035246959305368364, + -0.00011564368469407782 + ] + ] + ], + "codec_error": 0.0033109495416283607, + "codec_tol": 0.0001, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general-balanced", + "bandwidth": 1.0, + "codes": [ + [ + [ + 361, + 327, + 361, + 220, + 296, + 448, + 794, + 794, + 220, + 215, + 215, + 523, + 794, + 572, + 837 + ], + [ + 558, + 561, + 17, + 689, + 341, + 17, + 17, + 746, + 995, + 203, + 203, + 626, + 44, + 930, + 137 + ] + ] + ], + "decoded": [ + [ + [ + 0.00030808174051344395, + 0.001045881537720561, + 0.0007541970699094236, + 0.00033998070284724236, + 0.0003453573735896498, + 9.488919749855995e-05, + 0.00013062538346275687, + 0.0006009942735545337, + 0.0008533764048479497, + 0.0006959228776395321, + 0.0010016924934461713, + 0.0011683015618473291, + 0.0007219507242552936, + 0.0004506743571255356, + 0.0003839473647531122, + 0.0006096045835874975, + 0.0006778741371817887, + 0.0006203189841471612, + 0.0006753152702003717, + 0.0004131025052629411, + 0.000491982267703861, + 0.000795087602455169, + 0.0007778703584335744, + 0.0009378900285810232, + 0.0010353495599702, + 0.0006254438776522875, + 0.000214845611480996, + 0.00012940631131641567, + 0.00015907330089248717, + -7.865422958275303e-05, + -4.9406051402911544e-05, + 0.000535525381565094, + 0.0008429466979578137, + 0.0009230281575582922, + 0.0009178092004731297, + 0.0005792290903627872, + 0.0003179354825988412, + 0.0002350879949517548, + 0.0002593859098851681, + 0.00047500411164946854, + 0.0005929681938141584, + 0.0008235352579504251, + 0.0009488606592640281, + 0.0009453080128878355, + 0.0011052204063162208, + 0.0009675186011008918, + 0.0005994988605380058, + 0.00043946868390776217, + 0.0004515677283052355, + 0.00041880516801029444 + ] + ] + ], + "codec_error": 0.0031247916631400585, + "codec_tol": 0.0001, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general-balanced", + "bandwidth": 1.5, + "codes": [ + [ + [ + 361, + 327, + 361, + 220, + 296, + 448, + 794, + 794, + 220, + 215, + 215, + 523, + 794, + 572, + 837 + ], + [ + 558, + 561, + 17, + 689, + 341, + 17, + 17, + 746, + 995, + 203, + 203, + 626, + 44, + 930, + 137 + ], + [ + 551, + 408, + 315, + 232, + 209, + 733, + 935, + 96, + 644, + 260, + 204, + 1005, + 360, + 220, + 566 + ] + ] + ], + "decoded": [ + [ + [ + 0.000503583112731576, + 0.0012962579494342208, + 0.000953793409280479, + 0.000562680943403393, + 0.0005354040767997503, + 0.0001995227939914912, + 0.00021683776867575943, + 0.0007633829372934997, + 0.0011789660202339292, + 0.0011270726099610329, + 0.0015703362878412008, + 0.0018343244446441531, + 0.0013319269055500627, + 0.000994652509689331, + 0.000904104090295732, + 0.0010642888955771923, + 0.001001085969619453, + 0.0008879639790393412, + 0.0009764981223270297, + 0.0007446073577739298, + 0.0009423579322174191, + 0.0014007777208462358, + 0.0014580122660845518, + 0.001651210943236947, + 0.0017139619449153543, + 0.0011293691350147128, + 0.00046205963008105755, + 0.0002840239612851292, + 0.00037141164648346603, + 0.00018329237354919314, + 0.00032734169508330524, + 0.0011594139505177736, + 0.001596459187567234, + 0.0016294002998620272, + 0.00158004742115736, + 0.0011817252961918712, + 0.0007946674595586956, + 0.0006213290616869926, + 0.0006479363073594868, + 0.0008777808980084956, + 0.0010338183492422104, + 0.001345760771073401, + 0.0015597327146679163, + 0.0016180119710043073, + 0.0018115403363481164, + 0.001682001631706953, + 0.0012854461092501879, + 0.001027835882268846, + 0.0009527513175271451, + 0.000887712521944195 + ] + ] + ], + "codec_error": 0.0029652512166649103, + "codec_tol": 0.0001, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general-balanced", + "bandwidth": 2.0, + "codes": [ + [ + [ + 361, + 327, + 361, + 220, + 296, + 448, + 794, + 794, + 220, + 215, + 215, + 523, + 794, + 572, + 837 + ], + [ + 558, + 561, + 17, + 689, + 341, + 17, + 17, + 746, + 995, + 203, + 203, + 626, + 44, + 930, + 137 + ], + [ + 551, + 408, + 315, + 232, + 209, + 733, + 935, + 96, + 644, + 260, + 204, + 1005, + 360, + 220, + 566 + ], + [ + 14, + 14, + 407, + 656, + 472, + 407, + 472, + 365, + 444, + 521, + 162, + 128, + 575, + 340, + 407 + ] + ] + ], + "decoded": [ + [ + [ + 0.0005418736836872995, + 0.001448116498067975, + 0.001061455113813281, + 0.0007103311945684254, + 0.0007390136015601456, + 0.00034474380663596094, + 0.0003522688348311931, + 0.0008343191584572196, + 0.001150410040281713, + 0.0010256256209686399, + 0.001495253061875701, + 0.001773528871126473, + 0.0012762193800881505, + 0.0009920906741172075, + 0.0009303519036620855, + 0.0011235735146328807, + 0.0010679269907996058, + 0.0009568841778673232, + 0.0010723505401983857, + 0.0007524813408963382, + 0.0009166915551759303, + 0.0013677531387656927, + 0.0013341642916202545, + 0.0015379394171759486, + 0.0016818979056552052, + 0.001159622217528522, + 0.0005840237135998905, + 0.00046342622954398394, + 0.0005499363178387284, + 0.00028997453046031296, + 0.0003494007105473429, + 0.00111175118945539, + 0.0014719140017405152, + 0.001518196426331997, + 0.0015718521317467093, + 0.0012554096756502986, + 0.0009631850407458842, + 0.0008293383289128542, + 0.0007663772557862103, + 0.0008835261687636375, + 0.0009167938260361552, + 0.0011743532959371805, + 0.0012979755410924554, + 0.0013041547499597073, + 0.0016306998440995812, + 0.001617942820303142, + 0.0012842006981372833, + 0.001117410254664719, + 0.0010679230326786637, + 0.0009179076296277344 + ] + ] + ], + "codec_error": 0.002776096574962139, + "codec_tol": 0.0001, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-hubert-general-balanced", + "bandwidth": 4.0, + "codes": null, + "decoded": null, + "codec_error": 0.002473212545737624, + "codec_tol": 0.0001, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-mls", + "bandwidth": 0.5, + "codes": null, + "decoded": [ + [ + [ + -0.0002471721963956952, + -0.0007365602068603039, + -0.00029661963344551623, + -0.0001487718109274283, + -0.00025510278646834195, + -8.943512511905283e-06, + 0.0001678174448898062, + 0.0001996321661863476, + -4.0839742723619565e-05, + -0.00035881157964468, + -0.0003721409884747118, + -0.00031694662175141275, + -0.00031707881134934723, + -0.000331440765876323, + -4.38143324572593e-05, + 0.0003987671807408333, + 0.0005647469661198556, + 0.000618462567217648, + 0.00044193019857630134, + -8.946254092734307e-05, + -0.00041933005559258163, + -0.00044410640839487314, + -0.00023401528596878052, + -3.552871567080729e-05, + 0.00014210691733751446, + 0.0002467158483341336, + 0.00015603734937030822, + 3.193183147232048e-05, + -0.00013763393508270383, + -0.00035281648160889745, + -0.000529766664840281, + -0.0006517550209537148, + -0.0006077081197872758, + -0.00020284725178498775, + 0.00034685738501138985, + 0.0006811313796788454, + 0.000679212505929172, + 0.0005077550304122269, + 0.0003816273238044232, + 0.0002144991885870695, + -0.00016964729002211243, + -0.00016414248966611922, + 7.161950634326786e-05, + 0.0002906467125285417, + 0.0007565916166640818, + 0.0010058481711894274, + 0.0012552258558571339, + 0.0010698521509766579, + 0.0009335971553809941, + 0.000667513522785157 + ] + ] + ], + "codec_error": 0.002784556010738015, + "codec_tol": 1e-05, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-mls", + "bandwidth": 1.0, + "codes": null, + "decoded": [ + [ + [ + -0.00021897304395679384, + -0.0006707283901050687, + -0.0003198632912244648, + -0.0002019737585214898, + -0.00031223104451783, + -0.00015073314716573805, + 1.3109893188811839e-05, + 0.00010109333379659802, + -4.4847565732197836e-05, + -0.00026841764338314533, + -0.0002181670570280403, + -0.00029383719083853066, + -0.0005045154830440879, + -0.0005756605532951653, + -0.0002519423433113843, + 0.00017382082296535373, + 0.0003450627264101058, + 0.0004639998951461166, + 0.00032735118293203413, + -0.0002998501295223832, + -0.0007324003963731229, + -0.0006358272512443364, + -0.00019679185061249882, + 9.154963481705636e-05, + 0.00023105139553081244, + 0.0002694898867048323, + 5.3933585149934515e-05, + -0.00017367034160997719, + -0.000370326975826174, + -0.00045389344450086355, + -0.0003742754051927477, + -0.0003183094668202102, + -0.0003107236698269844, + -7.117551285773516e-05, + 0.00032562544220127165, + 0.0005173510289750993, + 0.00035740318708121777, + 0.0001532924798084423, + 8.018290100153536e-05, + -8.011072350200266e-05, + -0.0004129649023525417, + -0.0002168295468436554, + 0.00016200236859731376, + 0.00034917722223326564, + 0.0006240003858692944, + 0.0007102257804945111, + 0.0006860584835521877, + 0.00021981803001835942, + -7.106941484380513e-05, + -0.00018566018843557686 + ] + ] + ], + "codec_error": 0.0025421823374927044, + "codec_tol": 1e-05, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-mls", + "bandwidth": 1.5, + "codes": null, + "decoded": [ + [ + [ + -0.00028943613870069385, + -0.0007279455894604325, + -0.00037402231828309596, + -0.00023792865977156907, + -0.0003411142679397017, + -0.0001595417852513492, + 4.3847772758454084e-05, + 3.897479109582491e-05, + -0.00015776942018419504, + -0.0003887333150487393, + -0.0003272934991400689, + -0.000308241811580956, + -0.0005203168257139623, + -0.0006497218855656683, + -0.00032806317904032767, + 9.003604645840824e-05, + 0.0002507736498955637, + 0.00038551652687601745, + 0.00031532219145447016, + -0.00028256408404558897, + -0.000780107919126749, + -0.0006798981921747327, + -0.000187133060535416, + 9.075377602130175e-05, + 0.0002059313701465726, + 0.000252072379225865, + 8.591249934397638e-05, + -6.903265602886677e-05, + -0.00022177191567607224, + -0.0003164776717312634, + -0.0002896834921557456, + -0.00034714731737039983, + -0.0004501724033616483, + -0.00014487521548289806, + 0.00042451676563359797, + 0.0007103644893504679, + 0.0005320025375112891, + 0.00030199086177162826, + 0.00017571817443240434, + -7.344436016865075e-05, + -0.0004125718551222235, + -0.00022480337065644562, + 0.00010712102812249213, + 0.00021271411969792098, + 0.0004759627627208829, + 0.0007045314996503294, + 0.0008243073243647814, + 0.00042970190406776965, + 0.00021766081044916064, + 3.350689439685084e-05 + ] + ] + ], + "codec_error": 0.0024348432198166847, + "codec_tol": 1e-05, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-mls", + "bandwidth": 2.0, + "codes": null, + "decoded": [ + [ + [ + -0.00010455434676259756, + -0.0006130463443696499, + -0.00019683418213389814, + -0.00010099369683302939, + -0.00021099163859616965, + -2.6220950530841947e-06, + 0.00020871838205493987, + 0.00018668117991182953, + -1.0649124305928126e-05, + -0.00018443060980644077, + -9.16237331693992e-05, + -0.00010015940642915666, + -0.0003710154560394585, + -0.0004892981378361583, + -0.00011564286251086742, + 0.00032124019344337285, + 0.0004669901682063937, + 0.000622692343313247, + 0.0005683908239006996, + -0.0001299505529459566, + -0.0007344742771238089, + -0.0006522545008920133, + -7.369450759142637e-05, + 0.00023016634804662317, + 0.00029706733766943216, + 0.00040724140126258135, + 0.00031984460656531155, + 0.00010261082206852734, + -0.00017221950110979378, + -0.0002610271912999451, + -0.00015819921100046486, + -0.00027220440097153187, + -0.00047697784611955285, + -0.00013687692990060896, + 0.0004770267987623811, + 0.0007236705278046429, + 0.0005755429738201201, + 0.0005305277300067246, + 0.0004971636226400733, + 9.585886436980218e-05, + -0.00044038970372639596, + -0.0002591072116047144, + 8.882302790880203e-05, + 0.00010468350956216455, + 0.0004465210950002074, + 0.0008701237966306508, + 0.001133413054049015, + 0.0006035252590663731, + 0.00032269812072627246, + 0.0001862043427536264 + ] + ] + ], + "codec_error": 0.0022935366723686457, + "codec_tol": 1e-05, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-mls", + "bandwidth": 4.0, + "codes": null, + "decoded": [ + [ + [ + 0.0002609139191918075, + -0.0005568607593886554, + -8.985280146589503e-06, + 8.966325549408793e-05, + -7.479086343664676e-05, + 0.0002076481468975544, + 0.00034824819886125624, + 0.000307059584883973, + 0.00019627390429377556, + -3.6874123907182366e-05, + -7.025599188636988e-06, + -0.00012438457633834332, + -0.00032755275606177747, + -0.000411909946706146, + -8.821926894597709e-05, + 0.0003717693325597793, + 0.0004727449850179255, + 0.0006257848581299186, + 0.0005679113091900945, + -0.00014851021114736795, + -0.0006242442759685218, + -0.0005352640873752534, + -1.3237040548119694e-06, + 0.0003045561315957457, + 0.00041467309347353876, + 0.0005932425847277045, + 0.00044341094326227903, + 0.00023843055532779545, + 8.901742694433779e-05, + -1.0392006515758112e-05, + 0.0001084191317204386, + 0.00021004895097576082, + 8.228524529840797e-05, + 0.0003851399233099073, + 0.0008233294356614351, + 0.0008892629412002861, + 0.000785985728725791, + 0.0006910106167197227, + 0.0005654219421558082, + 0.00034487323137000203, + -0.00012858158152084798, + -3.434516838751733e-05, + 0.0003092561091762036, + 0.0003492308605927974, + 0.0006020109285600483, + 0.0008246254292316735, + 0.0010068734409287572, + 0.000397632597014308, + 0.00017323131032753736, + 0.00012472675007302314 + ] + ] + ], + "codec_error": 0.002119641751050949, + "codec_tol": 1e-05, + "dec_tol": 0.0001 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-more-data", + "bandwidth": 0.5, + "codes": [ + [ + [ + 44, + 881, + 344, + 344, + 344, + 881, + 44, + 881, + 571, + 813, + 107, + 950, + 437, + 950, + 437 + ] + ] + ], + "decoded": [ + [ + [ + 8.60798463691026e-05, + -0.0003060192975681275, + 9.043663885677233e-05, + 0.00014830089639872313, + -2.100023630191572e-05, + 0.00017922179540619254, + 0.00023896087077446282, + 0.00024265481624752283, + 9.400439739692956e-05, + -0.0001393423299305141, + -0.00017324750660918653, + -0.00010024692164734006, + -9.44490238907747e-05, + -0.00022922895732335746, + -1.1478223314043134e-06, + 0.0003733094781637192, + 0.00048060796689242125, + 0.00044257345143705606, + 0.00030929266358725727, + -2.485524964868091e-05, + -0.00015137175796553493, + -4.780884046340361e-05, + 0.00020679077715612948, + 0.00033283786615356803, + 0.0003789166221395135, + 0.0004155742935836315, + 0.00019656820222735405, + -6.525123899336904e-05, + -0.00014414236648008227, + -0.00024506013141945004, + -0.0002763999509625137, + -0.0002528353070374578, + -0.0002784187672659755, + -5.5240401707123965e-05, + 0.00034663392580114305, + 0.0005081702838651836, + 0.0003451743978075683, + 0.00017948850290849805, + 0.0001805563224479556, + 5.415735358837992e-05, + -0.0002908696769736707, + -0.00022546530817635357, + -2.816985215758905e-06, + 8.796519978204742e-05, + 0.00036707063554786146, + 0.0004874078440479934, + 0.0006295923958532512, + 0.00048173684626817703, + 0.0004252410144545138, + 0.00026322310441173613 + ] + ] + ], + "codec_error": 0.002914232201874256, + "codec_tol": 1e-05, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-more-data", + "bandwidth": 1.0, + "codes": [ + [ + [ + 44, + 881, + 344, + 344, + 344, + 881, + 44, + 881, + 571, + 813, + 107, + 950, + 437, + 950, + 437 + ], + [ + 659, + 335, + 335, + 801, + 30, + 726, + 647, + 721, + 562, + 421, + 421, + 797, + 797, + 797, + 797 + ] + ] + ], + "decoded": [ + [ + [ + -0.00013853900600224733, + -0.0006001053261570632, + -0.00014205696061253548, + -5.91947537031956e-06, + -0.00010432626731926575, + 8.372707816306502e-05, + 0.00019299553241580725, + 0.0002358516794629395, + -3.9954891690285876e-05, + -0.0002931125636678189, + -0.00026569172041490674, + -0.00043283088598400354, + -0.0006556544685736299, + -0.0006934062112122774, + -0.00022642247495241463, + 0.0002532481448724866, + 0.0005737473256886005, + 0.0007966211414895952, + 0.000520562578458339, + -0.00015708617866039276, + -0.0004774363187607378, + -0.0003260352532379329, + 6.291552563197911e-05, + 0.0002651271061040461, + 0.00031190537265501916, + 0.0003755779762286693, + 0.00024414766812697053, + 7.252671639434993e-05, + -6.486603524535894e-05, + -0.00016354414401575923, + -3.527149237925187e-05, + -0.00011908992746612057, + -0.0004999941447749734, + -0.0004161799151916057, + 0.0002218327426817268, + 0.0004935782635584474, + 0.0002801110385917127, + 0.00032458442728966475, + 0.0007008450338616967, + 0.0006555632571689785, + 7.318511052289978e-05, + 0.0001435471058357507, + 0.0003743935376405716, + 0.00017988341278396547, + 0.00022015272406861186, + 0.00037314006476663053, + 0.0005442615947686136, + 0.00028906844090670347, + 0.0002551917568780482, + 0.00038736595888622105 + ] + ] + ], + "codec_error": 0.002613937947899103, + "codec_tol": 1e-05, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-more-data", + "bandwidth": 1.5, + "codes": null, + "decoded": [ + [ + [ + 6.279885565163568e-05, + -0.00033689799602143466, + 0.00011250129318796098, + 0.00019457557937130332, + 0.0001285735343117267, + 0.00032528748852200806, + 0.0003915508568752557, + 0.0003026904596481472, + 0.0002715887385420501, + 0.00014779283083043993, + 0.00011752712453017011, + 5.8236309996573254e-05, + -0.00019055884331464767, + -0.0003741715627256781, + 6.279123772401363e-05, + 0.0007348853396251798, + 0.0010394826531410217, + 0.0010661018313840032, + 0.0007755373371765018, + -2.9394934244919568e-05, + -0.000576356309466064, + -0.00027806576690636575, + 0.00037756405072286725, + 0.0006358004757203162, + 0.0006607291288673878, + 0.000555443752091378, + 0.00015076500130817294, + -7.972745515871793e-05, + -7.068378909025341e-05, + -0.00018670226563699543, + -0.00018936209380626678, + -0.00012967083603143692, + -0.0003847008920274675, + -0.0004100532678421587, + 0.0002501893322914839, + 0.0007843074854463339, + 0.000563940207939595, + 0.000467402336653322, + 0.0007548240246251225, + 0.0005277566961012781, + -1.5599987818859518e-05, + 0.0002939583209808916, + 0.0006670067086815834, + 0.0005304471123963594, + 0.000824933813419193, + 0.0012162369675934315, + 0.001134063582867384, + 0.0007296190597116947, + 0.0008042262634262443, + 0.0005679695168510079 + ] + ] + ], + "codec_error": 0.002474759239703417, + "codec_tol": 1e-05, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-more-data", + "bandwidth": 2.0, + "codes": null, + "decoded": [ + [ + [ + 0.00025080336490646005, + -0.00017259249580092728, + 0.00025967558030970395, + 0.000364720297511667, + 0.0002696491137612611, + 0.0004170248284935951, + 0.0005731081473641098, + 0.0005545281455852091, + 0.0004947385750710964, + 0.00031294734799303114, + 0.0002801477094180882, + 0.0001770674716681242, + -0.00011514315701788291, + -0.00026510338648222387, + 0.00021959032164886594, + 0.000816573272459209, + 0.001120111788623035, + 0.0012912392849102616, + 0.001166811096481979, + 0.0005309308762662113, + 8.758961485000327e-06, + 0.00021711227600462735, + 0.0007150850724428892, + 0.0008045516442507505, + 0.0007022423087619245, + 0.0007424060022458434, + 0.0006152429268695414, + 0.0004591939796227962, + 0.0003924915799871087, + 0.00026367188547737896, + 0.00017671581008471549, + 4.0699342207517475e-05, + -0.0001596197544131428, + -7.874160655774176e-05, + 0.0004457739123608917, + 0.0008729892433620989, + 0.0007467817049473524, + 0.0007103694952093065, + 0.0010271944338455796, + 0.0008960479754023254, + 0.0004278179258108139, + 0.0006703450926579535, + 0.0008964851731434464, + 0.0006187634426169097, + 0.0007422066992148757, + 0.0010011898120865226, + 0.0008749665576033294, + 0.00045576749835163355, + 0.0005878594820387661, + 0.00046867664786987007 + ] + ] + ], + "codec_error": 0.0023411870934069157, + "codec_tol": 1e-05, + "dec_tol": 1e-05 + }, + { + "repo_id": "hf-audio/xcodec-wavlm-more-data", + "bandwidth": 4.0, + "codes": null, + "decoded": [ + [ + [ + 0.0005943458527326584, + 0.00017469993326812983, + 0.0007098331698216498, + 0.0008159588905982673, + 0.0008173630340024829, + 0.0010675098747015, + 0.0011202740715816617, + 0.0009797199163585901, + 0.0008500582771375775, + 0.0006013234378769994, + 0.000489078345708549, + 0.0004758498689625412, + 0.0004311382654123008, + 0.00044707287452183664, + 0.0008811730076558888, + 0.0014070728793740273, + 0.0015382873825728893, + 0.0014448357978835702, + 0.001160905696451664, + 0.0004969813744537532, + -1.3974204193800688e-06, + 0.00016932631842792034, + 0.0007671721396036446, + 0.0010725297033786774, + 0.0010861807968467474, + 0.0010590273886919022, + 0.0007575892377644777, + 0.0003892407985404134, + -1.2117772712372243e-05, + -0.00036681993515230715, + -0.0004377248405944556, + -0.00038845473318360746, + -0.00036490376805886626, + -5.0593931518960744e-05, + 0.0005764220259152353, + 0.0010159657103940845, + 0.000954087299760431, + 0.0009282709797844291, + 0.0010461328784003854, + 0.0007925458485260606, + 0.0002687857486307621, + 0.00048614019760861993, + 0.0009145350777544081, + 0.0009633700246922672, + 0.0012744814157485962, + 0.0016483643557876348, + 0.0016050116391852498, + 0.0008498440147377551, + 0.0005446132854558527, + 0.00015282645472325385 + ] + ] + ], + "codec_error": 0.0021570040844380856, + "codec_tol": 1e-05, + "dec_tol": 1e-05 + } +] \ No newline at end of file diff --git a/tests/models/dac/test_modeling_dac.py b/tests/models/dac/test_modeling_dac.py index 0f9d6ea3e469..9ceec8fd61b0 100644 --- a/tests/models/dac/test_modeling_dac.py +++ b/tests/models/dac/test_modeling_dac.py @@ -376,12 +376,14 @@ def test_identity_shortcut(self): self.model_tester.create_and_check_model_forward(config, inputs_dict) +# Copied from transformers.tests.encodec.test_modeling_encodec.normalize def normalize(arr): norm = np.linalg.norm(arr) normalized_arr = arr / norm return normalized_arr +# Copied from transformers.tests.encodec.test_modeling_encodec.compute_rmse def compute_rmse(arr1, arr2): arr1_np = arr1.cpu().numpy().squeeze() arr2_np = arr2.cpu().numpy().squeeze() diff --git a/tests/models/xcodec/test_modeling_xcodec.py b/tests/models/xcodec/test_modeling_xcodec.py index 0d2a2aade593..186ff364c2ce 100644 --- a/tests/models/xcodec/test_modeling_xcodec.py +++ b/tests/models/xcodec/test_modeling_xcodec.py @@ -1,4 +1,4 @@ -# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ """Testing suite for the PyTorch Xcodec model.""" import inspect +import json import math import os import tempfile import unittest +from pathlib import Path import numpy as np from datasets import Audio, load_dataset +from parameterized import parameterized from pytest import mark from tests.test_configuration_common import ConfigTester @@ -52,7 +55,6 @@ def __init__( num_channels=1, sample_rate=16000, codebook_size=1024, - num_quantizers=8, num_samples=400, is_training=False, ): @@ -61,7 +63,6 @@ def __init__( self.num_channels = num_channels self.sample_rate = sample_rate self.codebook_size = codebook_size - self.num_quantizers = num_quantizers self.is_training = is_training self.num_samples = num_samples @@ -79,7 +80,7 @@ def prepare_config_and_inputs_for_model_class(self, model_class): config, inputs_dict = self.prepare_config_and_inputs() codes_length = math.ceil(self.num_samples / config.hop_length) inputs_dict["audio_codes"] = ids_tensor( - [self.batch_size, self.num_quantizers, codes_length], config.codebook_size + [self.batch_size, config.num_quantizers, codes_length], config.codebook_size ) return config, inputs_dict @@ -89,7 +90,6 @@ def get_config(self): sample_rate=self.sample_rate, audio_channels=self.num_channels, codebook_size=self.codebook_size, - num_quantizers=self.num_quantizers, ) def create_and_check_model_forward(self, config, inputs_dict): @@ -121,7 +121,7 @@ def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): def setUp(self): self.model_tester = XcodecModelTester(self) self.config_tester = ConfigTester( - self, config_class=XcodecConfig, hidden_size=37, common_properties=[], has_text_modality=False + self, config_class=XcodecConfig, common_properties=[], has_text_modality=False ) def test_config(self): @@ -417,53 +417,97 @@ def normalize(arr): # Copied from transformers.tests.encodec.test_modeling_encodec.compute_rmse def compute_rmse(arr1, arr2): - arr1_normalized = normalize(arr1) - arr2_normalized = normalize(arr2) + arr1_np = arr1.cpu().numpy().squeeze() + arr2_np = arr2.cpu().numpy().squeeze() + max_length = min(arr1.shape[-1], arr2.shape[-1]) + arr1_np = arr1_np[..., :max_length] + arr2_np = arr2_np[..., :max_length] + arr1_normalized = normalize(arr1_np) + arr2_normalized = normalize(arr2_np) return np.sqrt(((arr1_normalized - arr2_normalized) ** 2).mean()) -# @slow -@require_torch -class XcodecIntegrationTest(unittest.TestCase): - def test_integration(self): - expected_rmse = { - "0.5": 0.0065491, - "4.0": 0.0070978, - } - expected_codesums = { - "0.5": [117262], - "4.0": [926416], - } - - librispeech = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - model_id = "Manel/X-Codec" - model = XcodecModel.from_pretrained(model_id).to(torch_device).eval() - feature_extractor = AutoFeatureExtractor.from_pretrained(model_id) - - librispeech = librispeech.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate)) - audio = librispeech[-1]["audio"]["array"] +""" +Integration tests for XCodec - inputs = feature_extractor( - raw_audio=audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt" - ).to(torch_device) +Code for reproducing expected outputs can be found here: +https://gist.github.com/ebezzam/cdaf8c223e59e7677b2ea6bc2dc8230b - for bandwidth, exp_rmse in expected_rmse.items(): - bandwidth = float(bandwidth) - with torch.no_grad(): - audio_codes = model.encode(inputs["input_values"], bandwidth=bandwidth, return_dict=False) - codesum = int(audio_codes.sum().item()) +One reason for higher tolerances is because of different implementation of `Snake1d` within Transformer version DAC +See here: https://github.com/huggingface/transformers/pull/39793#issue-3277407384 - expected_codesum = expected_codesums[str(bandwidth)][0] - self.assertEqual(codesum, expected_codesum) +""" - input_values_dec = model.decode(audio_codes, return_dict=False) - input_values_enc_dec = model(inputs["input_values"], bandwidth=bandwidth)[1] +RESULTS_PATH = Path(__file__).parent.parent.parent / "fixtures/xcodec/integration_tests.json" - self.assertTrue(torch.allclose(input_values_dec, input_values_enc_dec, atol=1e-3)) +with open(RESULTS_PATH, "r") as f: + raw_data = json.load(f) + +# convert dicts into tuples ordered to match test args +EXPECTED_OUTPUTS_JSON = [ + ( + f"{d['repo_id']}_{d['bandwidth']}", + d["repo_id"], + d["bandwidth"], + d["codes"], + d["decoded"], + d["codec_error"], + d["codec_tol"], + d["dec_tol"], + ) + for d in raw_data +] + + +@slow +@require_torch +class XcodecIntegrationTest(unittest.TestCase): + @parameterized.expand(EXPECTED_OUTPUTS_JSON) + def test_integration( + self, test_name, repo_id, bandwidth, exp_codes, exp_decoded, exp_codec_err, codec_tol, dec_tol + ): + # load model + model = XcodecModel.from_pretrained(repo_id).to(torch_device).eval() + feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id) + + # load audio example + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + librispeech_dummy = librispeech_dummy.cast_column( + "audio", Audio(sampling_rate=feature_extractor.sampling_rate) + ) + audio_array = librispeech_dummy[0]["audio"]["array"] + inputs = feature_extractor( + raw_audio=audio_array, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt" + ).to(torch_device) + x = inputs["input_values"] + + with torch.no_grad(): + ENC_TOL = 0 + audio_codes = model.encode(x, bandwidth=bandwidth, return_dict=False) + if exp_codes is not None: + exp_codes = torch.tensor(exp_codes).to(torch_device) + torch.testing.assert_close( + audio_codes[..., : exp_codes.shape[-1]], + exp_codes, + rtol=ENC_TOL, + atol=ENC_TOL, + ) + + # dec_tol = 1e-5 # increased to 1e-4 for passing on 4 kbps + input_values_dec = model.decode(audio_codes).audio_values + if exp_decoded is not None: + exp_decoded = torch.tensor(exp_decoded).to(torch_device) + torch.testing.assert_close( + input_values_dec[..., : exp_decoded.shape[-1]], + exp_decoded, + rtol=dec_tol, + atol=dec_tol, + ) - self.assertTrue(inputs["input_values"].shape == input_values_enc_dec.shape) + # compute codec error + codec_err = compute_rmse(input_values_dec, x) + torch.testing.assert_close(codec_err, exp_codec_err, rtol=codec_tol, atol=codec_tol) - arr = inputs["input_values"][0].cpu().numpy() - arr_enc_dec = input_values_enc_dec[0].cpu().numpy() - rmse = compute_rmse(arr, arr_enc_dec) - self.assertTrue(np.abs(rmse - exp_rmse) < 1e-5) + # make sure forward and decode gives same result + audio_values_enc_dec = model(x, bandwidth=bandwidth).audio_values + torch.testing.assert_close(input_values_dec, audio_values_enc_dec, rtol=1e-6, atol=1e-6) diff --git a/utils/add_dates.py b/utils/add_dates.py index 9efa831d30a5..1fc03fe71525 100644 --- a/utils/add_dates.py +++ b/utils/add_dates.py @@ -220,7 +220,7 @@ def insert_dates(model_card_list: list[str]): # If the dates info line does not exist, add it else: - paper_link = get_paper_link(path=file_path) + paper_link = get_paper_link(model_card=model_card, path=file_path) release_date = "" if not (paper_link == "No_paper" or paper_link == "blog"):