[examples] fix main recipes using new decoding style (#2062)

wenet-e2e · Oct 19, 2023 · a09f9f7 · a09f9f7
1 parent 6e56993
commit a09f9f7
Show file tree

Hide file tree

Showing 10 changed files with 231 additions and 390 deletions.
diff --git a/examples/aishell/paraformer/run.sh b/examples/aishell/paraformer/run.sh
@@ -151,29 +151,24 @@ if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
   decoding_chunk_size=
   ctc_weight=0.5
   reverse_weight=0.0
+  python wenet/bin/recognize.py --gpu 0 \
+    --modes $decode_modes \
+    --config $dir/train.yaml \
+    --data_type $data_type \
+    --test_data data/test/data.list \
+    --checkpoint $decode_checkpoint \
+    --beam_size 10 \
+    --batch_size 32 \
+    --penalty 0.0 \
+    --dict $dict \
+    --ctc_weight $ctc_weight \
+    --reverse_weight $reverse_weight \
+    --result_dir $dir \
+    ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
   for mode in ${decode_modes}; do
-  {
-    test_dir=$dir/test_${mode}
-    mkdir -p $test_dir
-    python3 wenet/bin/recognize.py --gpu 0 \
-      --mode $mode \
-      --config $dir/train.yaml \
-      --data_type $data_type \
-      --test_data data/test/data.list \
-      --checkpoint $decode_checkpoint \
-      --beam_size 10 \
-      --batch_size 1 \
-      --penalty 0.0 \
-      --dict $dict \
-      --ctc_weight $ctc_weight \
-      --reverse_weight $reverse_weight \
-      --result_file $test_dir/text \
-      ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
-    python3 tools/compute-wer.py --char=1 --v=1 \
-      data/test/text $test_dir/text > $test_dir/wer
-  } &
+    python tools/compute-wer.py --char=1 --v=1 \
+      data/test/text $dir/$mode/text > $dir/$mode/wer
   done
-  wait
 fi
 
 
@@ -185,82 +180,3 @@ if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
     --output_file $dir/final.zip \
     --output_quant_file $dir/final_quant.zip
 fi
-
-# Optionally, you can add LM and test it with runtime.
-if [ ${stage} -le 7 ] && [ ${stop_stage} -ge 7 ]; then
-  # 7.1 Prepare dict
-  unit_file=$dict
-  mkdir -p data/local/dict
-  cp $unit_file data/local/dict/units.txt
-  tools/fst/prepare_dict.py $unit_file ${data}/resource_aishell/lexicon.txt \
-    data/local/dict/lexicon.txt
-  # 7.2 Train lm
-  lm=data/local/lm
-  mkdir -p $lm
-  tools/filter_scp.pl data/train/text \
-    $data/data_aishell/transcript/aishell_transcript_v0.8.txt > $lm/text
-  local/aishell_train_lms.sh
-  # 7.3 Build decoding TLG
-  tools/fst/compile_lexicon_token_fst.sh \
-    data/local/dict data/local/tmp data/local/lang
-  tools/fst/make_tlg.sh data/local/lm data/local/lang data/lang_test || exit 1;
-  # 7.4 Decoding with runtime
-  chunk_size=-1
-  ./tools/decode.sh --nj 16 \
-    --beam 15.0 --lattice_beam 7.5 --max_active 7000 \
-    --blank_skip_thresh 0.98 --ctc_weight 0.5 --rescoring_weight 1.0 \
-    --chunk_size $chunk_size \
-    --fst_path data/lang_test/TLG.fst \
-    --dict_path data/lang_test/words.txt \
-    data/test/wav.scp data/test/text $dir/final.zip \
-    data/lang_test/units.txt $dir/lm_with_runtime
-  # Please see $dir/lm_with_runtime for wer
-fi
-
-# Optionally, you can decode with k2 hlg
-if [ ${stage} -le 8 ] && [ ${stop_stage} -ge 8 ]; then
-  if [ ! -f data/local/lm/lm.arpa ]; then
-    echo "Please run prepare dict and train lm in Stage 7" || exit 1;
-  fi
-
-  # 8.1 Build decoding HLG
-  required="data/local/hlg/HLG.pt data/local/hlg/words.txt"
-  for f in $required; do
-    if [ ! -f $f ]; then
-      tools/k2/make_hlg.sh data/local/dict/ data/local/lm/ data/local/hlg
-      break
-    fi
-  done
-
-  # 8.2 Decode using HLG
-  decoding_chunk_size=
-  lm_scale=0.7
-  decoder_scale=0.1
-  r_decoder_scale=0.7
-  for mode in hlg_onebest hlg_rescore; do
-  {
-    test_dir=$dir/test_${mode}
-    mkdir -p $test_dir
-    python3 wenet/bin/recognize.py --gpu 0 \
-      --mode $mode \
-      --config $dir/train.yaml \
-      --data_type $data_type \
-      --test_data data/test/data.list \
-      --checkpoint $decode_checkpoint \
-      --beam_size 10 \
-      --batch_size 16 \
-      --penalty 0.0 \
-      --dict $dict \
-      --word data/local/hlg/words.txt \
-      --hlg data/local/hlg/HLG.pt \
-      --lm_scale $lm_scale \
-      --decoder_scale $decoder_scale \
-      --r_decoder_scale $r_decoder_scale \
-      --result_file $test_dir/text \
-      ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
-    python3 tools/compute-wer.py --char=1 --v=1 \
-      data/test/text $test_dir/text > $test_dir/wer
-  }
-  done
-fi
-
diff --git a/examples/aishell/rnnt/run.sh b/examples/aishell/rnnt/run.sh
@@ -156,31 +156,26 @@ if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
   search_transducer_weight=0.7
 
   reverse_weight=0.0
+  python wenet/bin/recognize.py --gpu 0 \
+    --modes $mode \
+    --config $dir/train.yaml \
+    --data_type $data_type \
+    --test_data data/test/data.list \
+    --checkpoint $decode_checkpoint \
+    --beam_size 10 \
+    --batch_size 32 \
+    --penalty 0.0 \
+    --dict $dict \
+    --ctc_weight $rescore_ctc_weight \
+    --transducer_weight $rescore_transducer_weight \
+    --attn_weight $rescore_attn_weight \
+    --search_ctc_weight $search_ctc_weight \
+    --search_transducer_weight $search_transducer_weight \
+    --reverse_weight $reverse_weight \
+    --result_dir $dir \
+    ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
   for mode in ${decode_modes}; do
-  {
-    test_dir=$dir/test_${mode}
-    mkdir -p $test_dir
-    python wenet/bin/recognize.py --gpu 0 \
-      --mode $mode \
-      --config $dir/train.yaml \
-      --data_type $data_type \
-      --test_data data/test/data.list \
-      --checkpoint $decode_checkpoint \
-      --beam_size 10 \
-      --batch_size 1 \
-      --penalty 0.0 \
-      --dict $dict \
-      --ctc_weight $rescore_ctc_weight \
-      --transducer_weight $rescore_transducer_weight \
-      --attn_weight $rescore_attn_weight \
-      --search_ctc_weight $search_ctc_weight \
-      --search_transducer_weight $search_transducer_weight \
-      --reverse_weight $reverse_weight \
-      --result_file $test_dir/text \
-      ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
     python tools/compute-wer.py --char=1 --v=1 \
-      data/test/text $test_dir/text > $test_dir/wer
-  } &
+      data/test/text $dir/$mode/text > $dir/$mode/wer
   done
-  wait
 fi
diff --git a/examples/aishell/s0/run.sh b/examples/aishell/s0/run.sh
@@ -191,29 +191,24 @@ if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
   decoding_chunk_size=
   ctc_weight=0.3
   reverse_weight=0.5
+  python wenet/bin/recognize.py --gpu 0 \
+    --modes $decode_modes \
+    --config $dir/train.yaml \
+    --data_type $data_type \
+    --test_data data/test/data.list \
+    --checkpoint $decode_checkpoint \
+    --beam_size 10 \
+    --batch_size 32 \
+    --penalty 0.0 \
+    --dict $dict \
+    --ctc_weight $ctc_weight \
+    --reverse_weight $reverse_weight \
+    --result_dir $dir \
+    ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
   for mode in ${decode_modes}; do
-  {
-    test_dir=$dir/test_${mode}
-    mkdir -p $test_dir
-    python wenet/bin/recognize.py --gpu 0 \
-      --mode $mode \
-      --config $dir/train.yaml \
-      --data_type $data_type \
-      --test_data data/test/data.list \
-      --checkpoint $decode_checkpoint \
-      --beam_size 10 \
-      --batch_size 1 \
-      --penalty 0.0 \
-      --dict $dict \
-      --ctc_weight $ctc_weight \
-      --reverse_weight $reverse_weight \
-      --result_file $test_dir/text \
-      ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
     python tools/compute-wer.py --char=1 --v=1 \
-      data/test/text $test_dir/text > $test_dir/wer
-  } &
+      data/test/text $dir/$mode/text > $dir/$mode/wer
   done
-  wait
 fi
 
 
@@ -277,30 +272,27 @@ if [ ${stage} -le 8 ] && [ ${stop_stage} -ge 8 ]; then
   lm_scale=0.7
   decoder_scale=0.1
   r_decoder_scale=0.7
-  for mode in hlg_onebest hlg_rescore; do
-  {
-    test_dir=$dir/test_${mode}
-    mkdir -p $test_dir
-    python wenet/bin/recognize.py --gpu 0 \
-      --mode $mode \
-      --config $dir/train.yaml \
-      --data_type $data_type \
-      --test_data data/test/data.list \
-      --checkpoint $decode_checkpoint \
-      --beam_size 10 \
-      --batch_size 16 \
-      --penalty 0.0 \
-      --dict $dict \
-      --word data/local/hlg/words.txt \
-      --hlg data/local/hlg/HLG.pt \
-      --lm_scale $lm_scale \
-      --decoder_scale $decoder_scale \
-      --r_decoder_scale $r_decoder_scale \
-      --result_file $test_dir/text \
-      ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
+  decode_modes="hlg_onebest hlg_rescore"
+  python wenet/bin/recognize.py --gpu 0 \
+    --modes $decode_modes \
+    --config $dir/train.yaml \
+    --data_type $data_type \
+    --test_data data/test/data.list \
+    --checkpoint $decode_checkpoint \
+    --beam_size 10 \
+    --batch_size 16 \
+    --penalty 0.0 \
+    --dict $dict \
+    --word data/local/hlg/words.txt \
+    --hlg data/local/hlg/HLG.pt \
+    --lm_scale $lm_scale \
+    --decoder_scale $decoder_scale \
+    --r_decoder_scale $r_decoder_scale \
+    --result_dir $dir \
+    ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
+  for mode in ${decode_modes}; do
     python tools/compute-wer.py --char=1 --v=1 \
-      data/test/text $test_dir/text > $test_dir/wer
-  }
+      data/test/text $dir/$mode/text > $dir/$mode/wer
   done
 fi
 

diff --git a/examples/aishell2/rnnt/run.sh b/examples/aishell2/rnnt/run.sh
@@ -140,31 +140,25 @@ if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
             --val_best \
             2>&1 | tee -a $dir/average.log || exit 1;
     fi
-
+    python wenet/bin/recognize.py --gpu 0 \
+        --modes $decode_modes \
+        --config $dir/train.yaml \
+        --data_type raw \
+        --test_data data/test/data.list \
+        --checkpoint $decode_checkpoint \
+        --beam_size 10 \
+        --batch_size 1 \
+        --penalty 0.0 \
+        --dict $dict \
+        --ctc_weight $rescore_ctc_weight \
+        --transducer_weight $rescore_transducer_weight \
+        --attn_weight $rescore_attn_weight \
+        --search_ctc_weight $search_ctc_weight \
+        --search_transducer_weight $search_transducer_weight \
+        --result_dir $dir \
+        ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
     for mode in ${decode_modes}; do
-    {
-        test_dir=$dir/test_${mode}_chunk_${decoding_chunk_size}
-        mkdir -p $test_dir
-        python wenet/bin/recognize.py --gpu 0 \
-            --mode $mode \
-            --config $dir/train.yaml \
-            --data_type raw \
-            --test_data data/test/data.list \
-            --checkpoint $decode_checkpoint \
-            --beam_size 10 \
-            --batch_size 1 \
-            --penalty 0.0 \
-            --dict $dict \
-            --ctc_weight $rescore_ctc_weight \
-            --transducer_weight $rescore_transducer_weight \
-            --attn_weight $rescore_attn_weight \
-            --search_ctc_weight $search_ctc_weight \
-            --search_transducer_weight $search_transducer_weight \
-            --result_file $test_dir/text \
-            ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
-         python tools/compute-wer.py --char=1 --v=1 \
-            data/test/text $test_dir/text > $test_dir/wer
-    } &
+        python tools/compute-wer.py --char=1 --v=1 \
+            data/test/text $dir/$mode/text > $dir/$mode/wer
     done
-    wait
 fi
diff --git a/examples/aishell2/s0/run.sh b/examples/aishell2/s0/run.sh
@@ -134,29 +134,23 @@ if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
     # -1 for full chunk
     decoding_chunk_size=
     ctc_weight=0.5
+    python wenet/bin/recognize.py --gpu 0 \
+      --modes $decode_modes \
+      --config $dir/train.yaml \
+      --data_type $data_type \
+      --test_data data/test/data.list \
+      --checkpoint $decode_checkpoint \
+      --beam_size 10 \
+      --batch_size 32 \
+      --penalty 0.0 \
+      --dict $dict \
+      --ctc_weight $ctc_weight \
+      --result_dir $dir \
+      ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
     for mode in ${decode_modes}; do
-    {
-        test_dir=$dir/test_${mode}
-        mkdir -p $test_dir
-        python wenet/bin/recognize.py --gpu 0 \
-            --mode $mode \
-            --config $dir/train.yaml \
-            --data_type raw \
-            --test_data data/test/data.list \
-            --checkpoint $decode_checkpoint \
-            --beam_size 10 \
-            --batch_size 1 \
-            --penalty 0.0 \
-            --dict $dict \
-            --ctc_weight $ctc_weight \
-            --result_file $test_dir/text \
-            ${decoding_chunk_size:+--decoding_chunk_size $decoding_chunk_size}
-         python tools/compute-wer.py --char=1 --v=1 \
-            data/test/text $test_dir/text > $test_dir/wer
-    } &
+      python tools/compute-wer.py --char=1 --v=1 \
+        data/test/text $dir/$mode/text > $dir/$mode/wer
     done
-    wait
-
 fi
 
 if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then