falcon7b-fine-tuning-with-qlora.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"All rights is for hugging face","metadata":{}},{"cell_type":"code","source":"!pip install -q -U trl transformers accelerate git+https://github.com/huggingface/peft.git\n!pip install -q datasets bitsandbytes einops wandb","metadata":{"execution":{"iopub.status.busy":"2023-10-17T20:52:18.132652Z","iopub.execute_input":"2023-10-17T20:52:18.132930Z","iopub.status.idle":"2023-10-17T20:53:01.292231Z","shell.execute_reply.started":"2023-10-17T20:52:18.132907Z","shell.execute_reply":"2023-10-17T20:53:01.291018Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"from datasets import load_dataset\n\ndataset = load_dataset(\"timdettmers/openassistant-guanaco\", split=\"train\")","metadata":{"execution":{"iopub.status.busy":"2023-10-17T20:53:01.293692Z","iopub.execute_input":"2023-10-17T20:53:01.294095Z","iopub.status.idle":"2023-10-17T20:53:04.548036Z","shell.execute_reply.started":"2023-10-17T20:53:01.294071Z","shell.execute_reply":"2023-10-17T20:53:04.547317Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Downloading and preparing dataset json/timdettmers--openassistant-guanaco to /root/.cache/huggingface/datasets/json/timdettmers--openassistant-guanaco-c21e85fd8b1a6952/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1b14d0eb8a0f49b8a70fac9e90c5f073"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/20.9M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7533252c7c004110866873a2e896f905"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/1.11M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"401be0055fc4405dad8573d3f12bb62c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"82272be4242c46509e61912b0b1eeed6"}},"metadata":{}},{"name":"stdout","text":"Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/timdettmers--openassistant-guanaco-c21e85fd8b1a6952/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b. Subsequent calls will reuse this data.\n","output_type":"stream"}]},{"cell_type":"code","source":"import torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoTokenizer\n\nmodel_name = \"tiiuae/falcon-7b\"\n\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_name,\n    quantization_config= BitsAndBytesConfig(\n        load_in_4bit=True,\n        bnb_4bit_quant_type=\"nf4\",\n        bnb_4bit_compute_dtype=torch.float16,\n    ),\n    trust_remote_code=True\n)\nmodel.config.use_cache = False","metadata":{"execution":{"iopub.status.busy":"2023-10-17T20:59:11.008469Z","iopub.execute_input":"2023-10-17T20:59:11.009274Z","iopub.status.idle":"2023-10-17T21:01:28.198720Z","shell.execute_reply.started":"2023-10-17T20:59:11.009234Z","shell.execute_reply":"2023-10-17T21:01:28.198026Z"},"trusted":true},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading (…)lve/main/config.json:   0%|          | 0.00/1.05k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7a4b5163d0d74116baca79b20ab1b19a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)figuration_falcon.py:   0%|          | 0.00/7.16k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c4c3aac2dd1540e58b42b6d2d0612607"}},"metadata":{}},{"name":"stderr","text":"A new version of the following files was downloaded from https://huggingface.co/tiiuae/falcon-7b:\n- configuration_falcon.py\n. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading (…)n/modeling_falcon.py:   0%|          | 0.00/56.9k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e6dde34b76ec415c9bae428b4e68df15"}},"metadata":{}},{"name":"stderr","text":"A new version of the following files was downloaded from https://huggingface.co/tiiuae/falcon-7b:\n- modeling_falcon.py\n. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading (…)model.bin.index.json:   0%|          | 0.00/16.9k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"51b2e7bc41884586b3b351f1df858dc9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f18f7742627427cb912af842bbb8b80"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)l-00001-of-00002.bin:   0%|          | 0.00/9.95G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d09288de680401ab911e817d80cd554"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)l-00002-of-00002.bin:   0%|          | 0.00/4.48G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"823726eec72849a2982421332f6883a7"}},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a226bdfe65f442789e3269eb749a5eaf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)neration_config.json:   0%|          | 0.00/117 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c7f745e6aea0437a8ea11dcfa100da46"}},"metadata":{}}]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code = True)\ntokenizer.pad_token = tokenizer.eos_token","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:01:35.638436Z","iopub.execute_input":"2023-10-17T21:01:35.639222Z","iopub.status.idle":"2023-10-17T21:01:36.655176Z","shell.execute_reply.started":"2023-10-17T21:01:35.639187Z","shell.execute_reply":"2023-10-17T21:01:36.654346Z"},"trusted":true},"execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading (…)okenizer_config.json:   0%|          | 0.00/287 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"eb17393633a7479d9e832e0cab9a28cf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)/main/tokenizer.json:   0%|          | 0.00/2.73M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f7c26b94564b461ebc2b8224414508ed"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)cial_tokens_map.json:   0%|          | 0.00/281 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"98219c4067484fb29a7f81bb23761423"}},"metadata":{}}]},{"cell_type":"code","source":"from peft import LoraConfig\n\npeft_config= LoraConfig(\n    lora_alpha= 16,\n    lora_dropout= 0.1,\n    r = 64,\n    bias= \"none\",\n    task_type= \"CAUSAL_LM\",\n    target_modules= [\n        \"query_key_value\",\n        \"dense\",\n        \"dense_h_to_4h\",\n        \"dense_4h_to_h\",\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:03:08.020442Z","iopub.execute_input":"2023-10-17T21:03:08.021046Z","iopub.status.idle":"2023-10-17T21:03:08.069375Z","shell.execute_reply.started":"2023-10-17T21:03:08.021015Z","shell.execute_reply":"2023-10-17T21:03:08.068674Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"from transformers import TrainingArguments\nfrom trl import SFTTrainer\n\ntraining_arguments = TrainingArguments(\n    output_dir= \"./results\",\n    per_device_train_batch_size= 4,\n    gradient_accumulation_steps= 4,\n    optim= \"paged_adamw_32bit\",\n    save_steps= 10,\n    logging_steps= 10,\n    learning_rate= 2e-4,\n    fp16=True,\n    max_grad_norm= 0.3,\n    max_steps= 500,\n    warmup_ratio= 0.03,\n    group_by_length= True,\n    lr_scheduler_type= \"constant\",\n    gradient_checkpointing= True,\n)\n\nmax_seq_length= 512\n\ntrainer= SFTTrainer(\n    model= model,\n    train_dataset= dataset,\n    peft_config= peft_config,\n    dataset_text_field= \"text\",\n    max_seq_length= max_seq_length,\n    tokenizer= tokenizer,\n    args= training_arguments,\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:05:52.594911Z","iopub.execute_input":"2023-10-17T21:05:52.595553Z","iopub.status.idle":"2023-10-17T21:06:13.766316Z","shell.execute_reply.started":"2023-10-17T21:05:52.595525Z","shell.execute_reply":"2023-10-17T21:06:13.765429Z"},"trusted":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1e92ce64efbe48a2a0565cce9ba21101"}},"metadata":{}}]},{"cell_type":"code","source":"for name, module in trainer.model.named_modules():\n    if \"norm\" in name:\n        module = module.to(torch.float32)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T21:06:21.800724Z","iopub.execute_input":"2023-10-17T21:06:21.801152Z","iopub.status.idle":"2023-10-17T21:06:21.811531Z","shell.execute_reply.started":"2023-10-17T21:06:21.801122Z","shell.execute_reply":"2023-10-17T21:06:21.810662Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{},"execution_count":null,"outputs":[]}]}