From 188b9d965d02a5e5e0251a9fe9db00c4eb69fe73 Mon Sep 17 00:00:00 2001 From: "Dr. Alex A. Anderson" Date: Wed, 25 Sep 2024 20:30:51 -0700 Subject: [PATCH] update mutligpu readme --- recipes/quickstart/finetuning/multigpu_finetuning.md | 2 +- src/llama_recipes/finetuning.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/recipes/quickstart/finetuning/multigpu_finetuning.md b/recipes/quickstart/finetuning/multigpu_finetuning.md index 0dbf99b8f..f1a1cba07 100644 --- a/recipes/quickstart/finetuning/multigpu_finetuning.md +++ b/recipes/quickstart/finetuning/multigpu_finetuning.md @@ -23,7 +23,7 @@ Get access to a machine with multiple GPUs (in this case we tested with 4 A100 a This has been tested on 4 H100s GPUs. ```bash - FSDP_CPU_RAM_EFFICIENT_LOADING=1 ACCELERATE_USE_FSDP=1 torchrun --nnodes 1 --nproc_per_node 4 finetuning.py --enable_fsdp --quantization int4 --model_name /path_of_model_folder/70B --mixed_precision False --low_cpu_fsdp --use_peft --peft_method lora --output_dir Path/to/save/PEFT/model + FSDP_CPU_RAM_EFFICIENT_LOADING=1 ACCELERATE_USE_FSDP=1 torchrun --nnodes 1 --nproc_per_node 4 finetuning.py --enable_fsdp --quantization 4bit --model_name /path_of_model_folder/70B --mixed_precision False --low_cpu_fsdp --use_peft --peft_method lora --output_dir Path/to/save/PEFT/model ``` ### With FSDP + PEFT diff --git a/src/llama_recipes/finetuning.py b/src/llama_recipes/finetuning.py index 029b13d5b..1d845e953 100644 --- a/src/llama_recipes/finetuning.py +++ b/src/llama_recipes/finetuning.py @@ -21,11 +21,15 @@ AutoTokenizer, BitsAndBytesConfig, AutoProcessor, - MllamaForConditionalGeneration, - AutoModel, + AutoModel ) from transformers.models.llama.modeling_llama import LlamaDecoderLayer -from transformers.models.mllama.modeling_mllama import MllamaSelfAttentionDecoderLayer,MllamaCrossAttentionDecoderLayer,MllamaVisionEncoderLayer +from transformers.models.mllama.modeling_mllama import ( + MllamaSelfAttentionDecoderLayer, + MllamaCrossAttentionDecoderLayer, + MllamaVisionEncoderLayer, + MllamaForConditionalGeneration +) from llama_recipes.configs import fsdp_config as FSDP_CONFIG from llama_recipes.configs import train_config as TRAIN_CONFIG