[Auto Parallel] Support semi-auto trainer and fit Llama2 training (#7885

) * support semi-auto trainer and fit Llama2 training * support shard_dataloader in dynamic semi-auto * rewrite traning loop * refactor traning loop * refine args of auto trainer * broadcast loss * add auto ci cases
PaddlePaddle · Jan 31, 2024 · 3a704ea · 3a704ea
1 parent 44bfeb0
commit 3a704ea
Show file tree

Hide file tree

Showing 12 changed files with 1,139 additions and 369 deletions.
diff --git a/llm/llama/auto_parallel/run_auto.sh b/llm/llama/auto_parallel/run_auto.sh
@@ -68,6 +68,6 @@ python -u -m paddle.distributed.launch \
     --do_eval \
     --device "gpu" \
     --data_impl "mmap" \
-    --parallel_mode "auto"
+    --enable_auto_parallel 1
 
     # --resume_from_checkpoint "output/llama_auto_serial/checkpoint-2" \
diff --git a/llm/llama/auto_parallel/run_auto_sp.sh b/llm/llama/auto_parallel/run_auto_sp.sh
@@ -68,7 +68,7 @@ python -u -m paddle.distributed.launch \
     --do_eval \
     --device "gpu" \
     --data_impl "mmap" \
-    --parallel_mode "auto" \
+    --enable_auto_parallel 1 \
     --sequence_parallel true \
 
     # --resume_from_checkpoint "output/llama_auto_serial/checkpoint-2" \