DASB pipeline implementation for Common Voice

.gitignore

@@ -44,6 +44,8 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
+cache/
+ASR-cv*
 nosetests.xml
 coverage.xml
 *.cover

README.md

@@ -21,7 +21,7 @@ The SpeechBrain Benchmarks currently include the following:
 
 - [MOABB](https://github.com/speechbrain/benchmarks/tree/main/benchmarks/MOABB) - A benchmark designed for evaluating neural models in well-known EEG tasks like motor imagery, P300, and SSVEP.
 
-- [DASB](https://github.com/speechbrain/benchmarks/tree/main/benchmarks/DASB) - A benchmark designed for evaluating discrete audio tokens across a wide range of discriminative
+- [DASB](https://github.com/speechbrain/benchmarks/tree/DASB/benchmarks/DASB) - A benchmark designed for evaluating discrete audio tokens across a wide range of discriminative
 and  generative tasks.
 
 

benchmarks/DASB/LibriSpeech/ASR/LSTM/hparams/train_speech_tokenizer.yaml → benchmarks/DASB/CommonVoice/ASR/LSTM/hparams/train.yaml

@@ -1,6 +1,8 @@
 # ################################
-# Recipe for training an discrete-input ctc ASR system with librispeech.
-# Decoding is performed with ctc greedy or LM-rescored decoder.
+# Script for training an ASR model evaluating an SSL representation
+# model on one language from the CommonVoice dataset. A SentencePiece tokenizer
+# with number of tokens equal to <output_neurons> is learned in a first phase
+# on the considered language.
 #
 # Authors
 # * Pooneh Mousavi 2024
@@ -9,69 +11,100 @@
 # Seed needs to be set at top of yaml, before objects with parameters are made
 seed: 1986
 __set_seed: !apply:torch.manual_seed [!ref <seed>]
-output_folder: !ref results/MP3S-LSTM/speech_tokenizer/<seed>
-output_wer_folder: !ref <output_folder>/
+language: cy # use 'cy' for Welsh  and 'eu' for Basque
+output_folder: !ref results/CommonVoice/speech_tokenizer/<language>/<seed>
+test_wer_file: !ref <output_folder>/wer_test.txt
 save_folder: !ref <output_folder>/save
 train_log: !ref <output_folder>/train_log.txt
-
+cached_data_folder: cache/CommonVoice/<language>/LSTM/speech_tokenizer/<seed>
+run_name: !PLACEHOLDER
 
 # Data files
-data_folder: !PLACEHOLDER # e,g./path/to/LibriSpeech
-# noise/ris dataset will automatically be downloaded
-# data_folder_rirs: !ref <data_folder>
-train_splits: ["train-clean-100"]
-dev_splits: ["dev-clean"]
-test_splits: ["test-clean", "test-other"]
-
-skip_prep: False
-ckpt_interval_minutes: 25 # save checkpoint every N min
-train_csv: !ref <output_folder>/train-clean-100.csv
-valid_csv: !ref <output_folder>/dev-clean.csv
-test_csv:
-   - !ref <output_folder>/test-clean.csv
-   - !ref <output_folder>/test-other.csv
-
+data_folder: !PLACEHOLDER # e.g, /local/cv-corpus-11.0-2022-09-21/<language>
+train_tsv_file: !ref <data_folder>/train.tsv  # Standard CommonVoice .tsv files
+dev_tsv_file: !ref <data_folder>/dev.tsv  # Standard CommonVoice .tsv files
+test_tsv_file: !ref <data_folder>/test.tsv  # Standard CommonVoice .tsv files
+accented_letters: True
+train_csv: !ref <save_folder>/train.csv
+valid_csv: !ref <save_folder>/dev.csv
+test_csv: !ref <save_folder>/test.csv
+skip_prep: False # Skip data preparation
+testing: True # If set to True, the test evlaution is done, otherwise skipped.
+
+tokens_folder: !PLACEHOLDER  # Path to the folder where extracted tokens are saved.
+pretrain_embeddings_folder: non
+
+avoid_if_longer_than: 10.0
 
 # Training parameters
 number_of_epochs: 20
-lr: 0.0002
-sorting: ascending
-precision: fp32
-
-# With data_parallel batch_size is split into N jobs
-# With DDP batch_size is multiplied by N jobs
-# Must be 3 per GPU to fit 32GB of VRAM
-batch_size: 4
+
+batch_size_exponent: 4 # @orion_step1: --batch_size_exponent~"uniform(2, 4,discrete=True)"
+batch_size: !ref 2 ** <batch_size_exponent>
 test_batch_size: 1
+grad_accumulation_factor: 2
+max_grad_norm: 5.0
 
 
-### Config for Tokenizer
-vocab_size: 1024
-num_codebooks: 2
-sample_rate: 16000
+sorting: descending #random
+num_workers: 8
+loss_reduction: batchmean
+precision: fp32 # bf16, fp16 or fp32loss_reduction: batchmean
+valid_search_interval: 1
+avg_checkpoints: 10 # Number of checkpoints to average for evaluation
+cache_size: 1.e+10
+token_type: bpe  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
 
-# Feature parameters
+lr_model: 0.0002 # @orion_step1: --lr_model~"loguniform(0.00001,0.5)"
+
+# Training parameters
+dynamic_batching: True
+max_batch_length_train: 850
+max_batch_len_val: 100
+num_bucket: 200
+shuffle: False # if true re-creates batches at each epoch shuffling examples.
+max_batch_ex: 128
+batch_ordering: random
+
+dynamic_batch_sampler_train:
+   max_batch_length: !ref <max_batch_length_train>
+   num_buckets: !ref <num_bucket>
+   shuffle: !ref <shuffle>
+   batch_ordering: !ref <batch_ordering>
+   max_batch_ex: !ref <max_batch_ex>
+
+dynamic_batch_sampler_val:
+   max_batch_length: !ref <max_batch_len_val>
+   num_buckets: !ref <num_bucket>
+   shuffle: !ref <shuffle>
+   batch_ordering: !ref <batch_ordering>
+   max_batch_ex: !ref <max_batch_ex>
 
-encoder_dim: 1024
 
 # Dataloader options
 train_dataloader_opts:
    batch_size: !ref <batch_size>
+dataloader_options:
+   batch_size: !ref <batch_size>
+   num_workers: 4
+test_dataloader_options:
+   batch_size: !ref <test_batch_size>
+   num_workers: 4
+
 
 valid_dataloader_opts:
    batch_size: !ref <batch_size>
 
-test_dataloader_opts:
-   batch_size: !ref <test_batch_size>
-
 # Model parameters
+
 activation: !name:torch.nn.Sigmoid
 dnn_layers: 1
-dnn_neurons: 1024
+dnn_neurons: 768
 freeze_encoder: True
 
 # Outputs
-output_neurons: 30  # BPE size, index(blank/eos/bos) = 0
+output_neurons: 100  # BPE size, index(blank/eos/bos) = 0
 
 # Decoding parameters
 blank_index: 0
@@ -92,16 +125,20 @@ test_beam_search:
    # If you don't want to use an LM, comment it out or set it to null
    kenlm_model_path: null
 
+### Config for Tokenizer
+vocab_size: 1024
+num_codebooks: 2
+sample_rate: 16000
+
+# Feature parameters
+encoder_dim: 1024
+
 # Functions and classes
 #
 epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
    limit: !ref <number_of_epochs>
 
-# EnCodec model (see https://huggingface.co/docs/transformers/v4.31.0/en/model_doc/encodec)
-# EnCodec model (see https://huggingface.co/docs/transformers/v4.31.0/en/model_doc/encodec)
-codec: !new:speechbrain.lobes.models.discrete.speechtokenizer_interface.SpeechTokenizer_interface
-   source: fnlp/SpeechTokenizer  # Only the 24kHz version supports mono audio
-   save_path: !ref <save_folder>
+# Modules
 discrete_embedding_layer: !new:custom_model.Discrete_EmbeddingLayer
    num_codebooks: !ref <num_codebooks>
    vocab_size: !ref <vocab_size>
@@ -111,6 +148,7 @@ attention_mlp: !new:custom_model.AttentionMLP
    input_dim: !ref <encoder_dim>
    hidden_dim: !ref <encoder_dim>
 
+
 enc: !new:speechbrain.nnet.RNN.LSTM
    input_shape: [Null, Null, !ref <encoder_dim>]
    num_layers: 2
@@ -132,17 +170,16 @@ modules:
    enc: !ref <enc>
    ctc_lin: !ref <ctc_lin>
    attention_mlp: !ref <attention_mlp>
-   codec: !ref <codec>
    discrete_embedding_layer: !ref <discrete_embedding_layer>
 
 model: !new:torch.nn.ModuleList
    - [!ref <enc>, !ref <ctc_lin>, !ref <discrete_embedding_layer>, !ref <attention_mlp>]
 
 model_opt_class: !name:torch.optim.Adam
-   lr: !ref <lr>
+   lr: !ref <lr_model>
 
 lr_annealing_model: !new:speechbrain.nnet.schedulers.NewBobScheduler
-   initial_value: !ref <lr>
+   initial_value: !ref <lr_model>
    improvement_threshold: 0.0025
    annealing_factor: 0.8
    patient: 0
@@ -155,7 +192,6 @@ checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
       model: !ref <model>
       scheduler_model: !ref <lr_annealing_model>
       attention_mlp: !ref <attention_mlp>
-      codec: !ref <codec>
       discrete_embedding_layer: !ref <discrete_embedding_layer>
       counter: !ref <epoch_counter>
       tokenizer: !ref <label_encoder>

benchmarks/DASB/CommonVoice/ASR/LSTM/hparams/train_dac.yaml

@@ -19,6 +19,7 @@ train_log: !ref <output_folder>/train_log.txt
 
 # Data files
 data_folder: !PLACEHOLDER # e.g, /local/cv-corpus-11.0-2022-09-21/<language>
+cached_data_folder: !PLACEHOLDER # e.g., path/to/cache
 train_tsv_file: !ref <data_folder>/train.tsv  # Standard CommonVoice .tsv files
 dev_tsv_file: !ref <data_folder>/dev.tsv  # Standard CommonVoice .tsv files
 test_tsv_file: !ref <data_folder>/test.tsv  # Standard CommonVoice .tsv files
@@ -28,6 +29,9 @@ valid_csv: !ref <save_folder>/dev.csv
 test_csv: !ref <save_folder>/test.csv
 skip_prep: False # Skip data preparation
 
+tokens_folder: !PLACEHOLDER  # Path to the folder where extracted tokens are saved.
+pretrain_embeddings_folder: none  # Optional: If pretrain_embeddings is True, this should be set to the path where the pretrained embeddings are saved.
+
 avoid_if_longer_than: 10.0
 
 # Training parameters
@@ -97,6 +101,8 @@ vocab_size: 1024
 model_bitrate: 8kbps
 num_codebooks: 2  # NOTE: must be smaller or equal to the maximum number of codebooks for the given model type
 sample_rate: 24000
+pretrain_embeddings: False
+freeze_embedding: False
 
 # Feature parameters
 encoder_dim: 1024