Version 0.4.0

openspeech/datasets/aishell/lit_data_module.py

@@ -102,7 +102,7 @@ def _parse_manifest_file(self, manifest_file_path: str) -> Tuple[list, list]:
 
         return audio_paths, transcripts
 
-    def prepare_data(self):
+    def prepare_data(self) -> None:
         r"""
         Prepare AI-Shell manifest file. If there is not exist manifest file, generate manifest file.
 
@@ -119,7 +119,7 @@ def prepare_data(self):
                 raise ValueError("Dataset path is not valid.")
             self._generate_manifest_files(self.configs.dataset.manifest_file_path)
 
-    def setup(self, stage: Optional[str] = None, tokenizer: Tokenizer = None):
+    def setup(self, stage: Optional[str] = None) -> None:
         r"""
         Split `train` and `valid` dataset for training.
 
@@ -150,8 +150,6 @@ def setup(self, stage: Optional[str] = None, tokenizer: Tokenizer = None):
                 dataset_path=self.configs.dataset.dataset_path,
                 audio_paths=audio_paths[stage],
                 transcripts=transcripts[stage],
-                sos_id=tokenizer.sos_id,
-                eos_id=tokenizer.eos_id,
                 apply_spec_augment=self.configs.audio.apply_spec_augment if stage == 'train' else False,
                 del_silence=self.configs.audio.del_silence if stage == 'train' else False,
             )

openspeech/datasets/ksponspeech/lit_data_module.py

@@ -131,7 +131,7 @@ def prepare_data(self):
                 raise FileNotFoundError
             self._generate_manifest_files(self.configs.dataset.manifest_file_path)
 
-    def setup(self, stage: Optional[str] = None, tokenizer: Tokenizer = None):
+    def setup(self, stage: Optional[str] = None) -> None:
         r"""
         Split `train` and `valid` dataset for training.
 
@@ -167,8 +167,6 @@ def setup(self, stage: Optional[str] = None, tokenizer: Tokenizer = None):
                 dataset_path=dataset_path,
                 audio_paths=audio_paths[stage],
                 transcripts=transcripts[stage],
-                sos_id=tokenizer.sos_id,
-                eos_id=tokenizer.eos_id,
                 apply_spec_augment=self.configs.audio.apply_spec_augment if stage == 'train' else False,
                 del_silence=self.configs.audio.del_silence if stage == 'train' else False,
             )

openspeech/datasets/librispeech/lit_data_module.py

@@ -123,7 +123,7 @@ def _download_dataset(self) -> None:
                     os.path.join(self.configs.dataset.dataset_path, train_dir, subfolder),
                 )
 
-    def prepare_data(self) -> Tokenizer:
+    def prepare_data(self) -> None:
         """
         Prepare librispeech data
 
@@ -158,7 +158,7 @@ def prepare_data(self) -> Tokenizer:
                     vocab_path=self.configs.tokenizer.vocab_path,
                 )
 
-    def setup(self, stage: Optional[str] = None, tokenizer: Tokenizer = None) -> None:
+    def setup(self, stage: Optional[str] = None) -> None:
         r""" Split dataset into train, valid, and test. """
         valid_end_idx = self.LIBRISPEECH_TRAIN_NUM + self.LIBRISPEECH_VALID_NUM
         audio_paths, transcripts = self._parse_manifest_file(self.configs.dataset.manifest_file_path)
@@ -180,8 +180,6 @@ def setup(self, stage: Optional[str] = None, tokenizer: Tokenizer = None) -> Non
                 dataset_path=self.configs.dataset.dataset_path,
                 audio_paths=audio_paths[stage],
                 transcripts=transcripts[stage],
-                sos_id=tokenizer.sos_id,
-                eos_id=tokenizer.eos_id,
                 apply_spec_augment=self.configs.audio.apply_spec_augment if stage == 'train' else False,
                 del_silence=self.configs.audio.del_silence if stage == 'train' else False,
             )

openspeech/datasets/librispeech/preprocess/character.py

@@ -35,11 +35,8 @@
 def _generate_character_labels(labels_dest):
     logger.info('create_char_labels started..')
 
-    tokens = ' ABCDEFGHIJKLMNOPQRSTUVWXYZ'
-    tokens = list(tokens)
-
     special_tokens = ['<pad>', '<sos>', '<eos>', '<blank>']
-    tokens.extend(special_tokens)
+    tokens = special_tokens + list(' ABCDEFGHIJKLMNOPQRSTUVWXYZ')
 
     # sort together Using zip
     label = {

openspeech_cli/hydra_train.py

@@ -22,7 +22,6 @@
 
 import os
 import hydra
-import wandb
 import pytorch_lightning as pl
 from omegaconf import DictConfig, OmegaConf
 from pytorch_lightning.utilities import rank_zero_info
@@ -45,13 +44,13 @@ def hydra_main(configs: DictConfig) -> None:
     data_module.prepare_data()
     tokenizer = TOKENIZER_REGISTRY[configs.tokenizer.unit](configs)
 
-    data_module.setup(tokenizer=tokenizer)
+    data_module.setup()
 
     model = MODEL_REGISTRY[configs.model.model_name](configs=configs, tokenizer=tokenizer)
 
     trainer = get_pl_trainer(configs, num_devices, logger)
     trainer.fit(model, data_module)
-    trainer.test()
+    trainer.test(model, data_module)
 
 
 if __name__ == '__main__':

setup.py

@@ -25,12 +25,12 @@
 
 setup(
     name='openspeech-core',
-    version='0.3.0',
+    version='0.4.0',
     description='Open-Source Toolkit for End-to-End Automatic Speech Recognition',
     author='Kim, Soohwan and Ha, Sangchun and Cho, Soyoung',
-    author_email='[email protected], seomk9896@naver.com, [email protected]',
-    url='https://github.com/sooftware/openspeech',
-    download_url='https://github.com/sooftware/openspeech/releases/tag/v0.1.zip',
+    author_email='[email protected], seomk9896@gmail.com, [email protected]',
+    url='https://github.com/openspeech-team/openspeech',
+    download_url='https://github.com/openspeech-team/openspeech/releases/tag/v0.4.zip',
     install_requires=[
         'torch>=1.6.0',
         'python-Levenshtein',