a(n);\\n for (int i = 0; i < n; ++i) {\\n cin >> a[i];\\n }\\n int max_val = a[0];\\n for (int i = 2; i < n; i += 2) {\\n if (a[i] > max_val) {\\n max_val = a[i];\\n }\\n }\\n cout << max_val << '\\\\n';\\n }\\n return 0;\\n}\\n```<|im_end|>\\n\""
+ ],
+ "application/vnd.google.colaboratory.intrinsic+json": {
+ "type": "string"
+ }
+ },
+ "metadata": {},
+ "execution_count": 7
+ }
+ ],
+ "source": [
+ "tokenizer.apply_chat_template(\n",
+ " ds[\"train\"][\"messages\"][0], tokenize=False\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "id": "vPLgr6sxBmOY"
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "def formatting_prompts_func(example):\n",
+ " output_texts = []\n",
+ " for i in range(len(example['problem'])):\n",
+ " text = f\"### Problem: {example['problem'][i]}\\n ### Response: {example['generation'][i]}\"\n",
+ " output_texts.append(text)\n",
+ " return output_texts\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "idAEIeSQ3xdS"
+ },
+ "source": [
+ "\n",
+ "### Train the model\n",
+ "Now let's use Huggingface TRL's `SFTTrainer`! More docs here: [TRL SFT docs](https://huggingface.co/docs/trl/sft_trainer). We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support TRL's `DPOTrainer`!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 49,
+ "referenced_widgets": [
+ "e92eb62e284743b6af284638a8da591b",
+ "0360fb88da70455f8dee3448741da9f6",
+ "507ba8e73c8c451db278848ee054e130",
+ "c6f3eb06ddb945e78916ae91482ff0ce",
+ "1470b5ec997b42aba820870ae023e169",
+ "85dc6838069f40b4a8db958765eb413c",
+ "c43a088652a34343bdc6899cb7ee48f1",
+ "51794cec056440b4ab0fd9215c2004b5",
+ "271e98a654ba4c02b06083d00bf664d1",
+ "a1525ae5572549e7bb6272d912ba554d",
+ "590c10a5c3da4db6bd1dfb767632fe82"
+ ]
+ },
+ "id": "95_Nn-89DhsL",
+ "outputId": "5e67e3b1-efef-44dd-b027-895c02903b1c"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "e92eb62e284743b6af284638a8da591b",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Unsloth: Tokenizing [\"text\"] (num_proc=2): 0%| | 0/40665 [00:00"
+ ],
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " [60/60 43:58, Epoch 0/1]\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | Step | \n",
+ " Training Loss | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1 | \n",
+ " 1.012300 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0.970100 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 1.070000 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 1.012100 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 0.983100 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 1.033700 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 0.993100 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 1.010000 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 1.060200 | \n",
+ "
\n",
+ " \n",
+ " | 10 | \n",
+ " 0.953000 | \n",
+ "
\n",
+ " \n",
+ " | 11 | \n",
+ " 0.936100 | \n",
+ "
\n",
+ " \n",
+ " | 12 | \n",
+ " 0.949000 | \n",
+ "
\n",
+ " \n",
+ " | 13 | \n",
+ " 0.925800 | \n",
+ "
\n",
+ " \n",
+ " | 14 | \n",
+ " 0.909400 | \n",
+ "
\n",
+ " \n",
+ " | 15 | \n",
+ " 0.949100 | \n",
+ "
\n",
+ " \n",
+ " | 16 | \n",
+ " 0.887300 | \n",
+ "
\n",
+ " \n",
+ " | 17 | \n",
+ " 0.910500 | \n",
+ "
\n",
+ " \n",
+ " | 18 | \n",
+ " 0.907400 | \n",
+ "
\n",
+ " \n",
+ " | 19 | \n",
+ " 0.898400 | \n",
+ "
\n",
+ " \n",
+ " | 20 | \n",
+ " 0.855000 | \n",
+ "
\n",
+ " \n",
+ " | 21 | \n",
+ " 0.872300 | \n",
+ "
\n",
+ " \n",
+ " | 22 | \n",
+ " 0.862700 | \n",
+ "
\n",
+ " \n",
+ " | 23 | \n",
+ " 0.824000 | \n",
+ "
\n",
+ " \n",
+ " | 24 | \n",
+ " 0.868400 | \n",
+ "
\n",
+ " \n",
+ " | 25 | \n",
+ " 0.877200 | \n",
+ "
\n",
+ " \n",
+ " | 26 | \n",
+ " 0.850800 | \n",
+ "
\n",
+ " \n",
+ " | 27 | \n",
+ " 0.838100 | \n",
+ "
\n",
+ " \n",
+ " | 28 | \n",
+ " 0.866100 | \n",
+ "
\n",
+ " \n",
+ " | 29 | \n",
+ " 0.839700 | \n",
+ "
\n",
+ " \n",
+ " | 30 | \n",
+ " 0.847200 | \n",
+ "
\n",
+ " \n",
+ " | 31 | \n",
+ " 0.829900 | \n",
+ "
\n",
+ " \n",
+ " | 32 | \n",
+ " 0.846100 | \n",
+ "
\n",
+ " \n",
+ " | 33 | \n",
+ " 0.857900 | \n",
+ "
\n",
+ " \n",
+ " | 34 | \n",
+ " 0.808500 | \n",
+ "
\n",
+ " \n",
+ " | 35 | \n",
+ " 0.802400 | \n",
+ "
\n",
+ " \n",
+ " | 36 | \n",
+ " 0.814300 | \n",
+ "
\n",
+ " \n",
+ " | 37 | \n",
+ " 0.826800 | \n",
+ "
\n",
+ " \n",
+ " | 38 | \n",
+ " 0.792100 | \n",
+ "
\n",
+ " \n",
+ " | 39 | \n",
+ " 0.806300 | \n",
+ "
\n",
+ " \n",
+ " | 40 | \n",
+ " 0.845900 | \n",
+ "
\n",
+ " \n",
+ " | 41 | \n",
+ " 0.838900 | \n",
+ "
\n",
+ " \n",
+ " | 42 | \n",
+ " 0.782800 | \n",
+ "
\n",
+ " \n",
+ " | 43 | \n",
+ " 0.641500 | \n",
+ "
\n",
+ " \n",
+ " | 44 | \n",
+ " 0.797000 | \n",
+ "
\n",
+ " \n",
+ " | 45 | \n",
+ " 0.817400 | \n",
+ "
\n",
+ " \n",
+ " | 46 | \n",
+ " 0.769200 | \n",
+ "
\n",
+ " \n",
+ " | 47 | \n",
+ " 0.786600 | \n",
+ "
\n",
+ " \n",
+ " | 48 | \n",
+ " 0.791500 | \n",
+ "
\n",
+ " \n",
+ " | 49 | \n",
+ " 0.801300 | \n",
+ "
\n",
+ " \n",
+ " | 50 | \n",
+ " 0.750400 | \n",
+ "
\n",
+ " \n",
+ " | 51 | \n",
+ " 0.767300 | \n",
+ "
\n",
+ " \n",
+ " | 52 | \n",
+ " 0.822300 | \n",
+ "
\n",
+ " \n",
+ " | 53 | \n",
+ " 0.852300 | \n",
+ "
\n",
+ " \n",
+ " | 54 | \n",
+ " 0.788200 | \n",
+ "
\n",
+ " \n",
+ " | 55 | \n",
+ " 0.799200 | \n",
+ "
\n",
+ " \n",
+ " | 56 | \n",
+ " 0.803700 | \n",
+ "
\n",
+ " \n",
+ " | 57 | \n",
+ " 0.781700 | \n",
+ "
\n",
+ " \n",
+ " | 58 | \n",
+ " 0.831200 | \n",
+ "
\n",
+ " \n",
+ " | 59 | \n",
+ " 0.766100 | \n",
+ "
\n",
+ " \n",
+ " | 60 | \n",
+ " 0.710100 | \n",
+ "
\n",
+ " \n",
+ "
"
+ ]
+ },
+ "metadata": {}
+ }
+ ],
+ "source": [
+ "trainer_stats = trainer.train()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "pCqnaKmlO1U9",
+ "outputId": "3005b362-2b86-468a-8f28-8effd9208566"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "462.3942 seconds used for training.\n",
+ "7.71 minutes used for training.\n",
+ "Peak reserved memory = 7.893 GB.\n",
+ "Peak reserved memory for training = 2.129 GB.\n",
+ "Peak reserved memory % of max memory = 53.519 %.\n",
+ "Peak reserved memory for training % of max memory = 14.436 %.\n"
+ ]
+ }
+ ],
+ "source": [
+ "# @title Show final memory and time stats\n",
+ "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n",
+ "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n",
+ "used_percentage = round(used_memory / max_memory * 100, 3)\n",
+ "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n",
+ "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n",
+ "print(\n",
+ " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n",
+ ")\n",
+ "print(f\"Peak reserved memory = {used_memory} GB.\")\n",
+ "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n",
+ "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n",
+ "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ekOmTR1hSNcr"
+ },
+ "source": [
+ "\n",
+ "### Inference\n",
+ "Let's run the model! You can change the instruction and input - leave the output blank!\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "kR3gIAX-SM2q",
+ "outputId": "ef815e2b-2095-4c27-d2af-a6d3007a8f68"
+ },
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ "['A tech company, \"DataGoSlow,\" produces high-performance memory modules. They sell these modules in \\'n\\' different retail outlets. Each outlet sells the modules at a unique price, represented by \\'xi\\' dollars.\\n\\nA large data center plans to purchase these memory modules over \\'q\\' consecutive procurement cycles. For each cycle, the data center has a budget constraint \\'mi\\' dollars. They want to know how many outlets sell the memory modules at a price *strictly less than* their budget.\\n\\n**Constraints:**\\n\\n* Time limit per test: 2.0 seconds\\n* Memory limit per test: 256.0 megabytes\\n\\n**Input Format:**\\n\\n* The first line of the input contains a single integer \\'n\\' (1 ≤ n ≤ 100 000) — the number of retail outlets.\\n* The second line contains \\'n\\' integers \\'xi\\' (1 ≤ xi ≤ 100 000) — prices of the memory modules in each outlet.\\n* The third line contains a single integer \\'q\\' (1 ≤ q ≤ 100 000) — the number of procurement cycles.\\n* Then follow \\'q\\' lines, each containing one integer \\'mi\\' (1 ≤ mi ≤ 109) — the budget for the i-th procurement cycle.\\n\\n**Output Format:**\\n\\n* Print \\'q\\' integers. The i-th of them should be equal to the number of outlets where the memory modules can be purchased at a price *strictly less than* the budget for the i-th cycle.\\n\\n**Example:**\\n\\n```input\\n5\\n3 10 8 6 11\\n4\\n1\\n10\\n3\\n11\\n```\\n\\n```output\\n0\\n2\\n3\\n0\\n```\\n\\n**Note:**\\n\\nIn the first and fourth cases, there are no such outlets.\\n\\nIn the second case, the first and third outlets have prices less than 10.\\n\\nIn the third case, all three outlets have prices less than ']"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 12
+ }
+ ],
+ "source": [
+ "example_problem = \"\"\"A tech company, \"DataGoSlow,\" produces high-performance memory modules. They sell these modules in 'n' different retail outlets. Each outlet sells the modules at a unique price, represented by 'xi' dollars.\n",
+ "\n",
+ "A large data center plans to purchase these memory modules over 'q' consecutive procurement cycles. For each cycle, the data center has a budget constraint 'mi' dollars. They want to know how many outlets sell the memory modules at a price *strictly less than* their budget.\n",
+ "\n",
+ "**Constraints:**\n",
+ "\n",
+ "* Time limit per test: 2.0 seconds\n",
+ "* Memory limit per test: 256.0 megabytes\n",
+ "\n",
+ "**Input Format:**\n",
+ "\n",
+ "* The first line of the input contains a single integer 'n' (1 ≤ n ≤ 100 000) — the number of retail outlets.\n",
+ "* The second line contains 'n' integers 'xi' (1 ≤ xi ≤ 100 000) — prices of the memory modules in each outlet.\n",
+ "* The third line contains a single integer 'q' (1 ≤ q ≤ 100 000) — the number of procurement cycles.\n",
+ "* Then follow 'q' lines, each containing one integer 'mi' (1 ≤ mi ≤ 109) — the budget for the i-th procurement cycle.\n",
+ "\n",
+ "**Output Format:**\n",
+ "\n",
+ "* Print 'q' integers. The i-th of them should be equal to the number of outlets where the memory modules can be purchased at a price *strictly less than* the budget for the i-th cycle.\n",
+ "\n",
+ "**Example:**\n",
+ "\n",
+ "```input\n",
+ "5\n",
+ "3 10 8 6 11\n",
+ "4\n",
+ "1\n",
+ "10\n",
+ "3\n",
+ "11\"\"\"\n",
+ "\n",
+ "FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n",
+ "inputs = tokenizer([example_problem], return_tensors = \"pt\").to(\"cuda\")\n",
+ "\n",
+ "outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)\n",
+ "tokenizer.batch_decode(outputs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "CrSvZObor0lY"
+ },
+ "source": [
+ " You can also use a `TextStreamer` for continuous inference - so you can see the generation token by token, instead of waiting the whole time!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "e2pEuRb1r2Vg",
+ "outputId": "187a7d22-bde7-439a-b1b0-a3e0dc308843"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "Continue the fibonnaci sequence.\n",
+ "\n",
+ "### Input:\n",
+ "1, 1, 2, 3, 5, 8\n",
+ "\n",
+ "### Response:\n",
+ "13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181, 6765, 10946, 17711, 28657, 46368, 75025, 121393, 196418, 317811, \n"
+ ]
+ }
+ ],
+ "source": [
+ "FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n",
+ "\n",
+ "from transformers import TextStreamer\n",
+ "text_streamer = TextStreamer(tokenizer)\n",
+ "_ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "uMuVrWbjAzhc"
+ },
+ "source": [
+ "\n",
+ "### Saving, loading finetuned models\n",
+ "To save the final model as LoRA adapters, either use Huggingface's `push_to_hub` for an online save or `save_pretrained` for a local save.\n",
+ "\n",
+ "**[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "upcOlWe7A1vc",
+ "outputId": "0995f3eb-3eb9-423f-ede2-3b7f2f9f10c9"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "('lora_model/tokenizer_config.json',\n",
+ " 'lora_model/special_tokens_map.json',\n",
+ " 'lora_model/vocab.json',\n",
+ " 'lora_model/merges.txt',\n",
+ " 'lora_model/added_tokens.json',\n",
+ " 'lora_model/tokenizer.json')"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "model.save_pretrained(\"lora_model\") # Local saving\n",
+ "tokenizer.save_pretrained(\"lora_model\")\n",
+ "# model.push_to_hub(\"your_name/lora_model\", token = \"...\") # Online saving\n",
+ "# tokenizer.push_to_hub(\"your_name/lora_model\", token = \"...\") # Online saving"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "AEEcJ4qfC7Lp"
+ },
+ "source": [
+ "Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "MKX_XKs_BNZR",
+ "outputId": "0412171f-a9b4-4018-a4aa-d9725d2b8201"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
+ "\n",
+ "### Instruction:\n",
+ "What is a famous tall tower in Paris?\n",
+ "\n",
+ "### Input:\n",
+ "\n",
+ "\n",
+ "### Response:\n",
+ "One of the most famous tall towers in Paris is the Eiffel Tower. It was built by Gustave Eiffel in 1889 and stands at a height of 324 meters (1,063 feet). It is a symbol of Paris and one of the most recognizable structures in the world.<|endoftext|>\n"
+ ]
+ }
+ ],
+ "source": [
+ "if False:\n",
+ " from unsloth import FastLanguageModel\n",
+ " model, tokenizer = FastLanguageModel.from_pretrained(\n",
+ " model_name = \"lora_model\", # YOUR MODEL YOU USED FOR TRAINING\n",
+ " max_seq_length = max_seq_length,\n",
+ " dtype = dtype,\n",
+ " load_in_4bit = load_in_4bit,\n",
+ " )\n",
+ " FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n",
+ "\n",
+ "# alpaca_prompt = You MUST copy from above!\n",
+ "\n",
+ "inputs = tokenizer(\n",
+ "[\n",
+ " alpaca_prompt.format(\n",
+ " \"What is a famous tall tower in Paris?\", # instruction\n",
+ " \"\", # input\n",
+ " \"\", # output - leave this blank for generation!\n",
+ " )\n",
+ "], return_tensors = \"pt\").to(\"cuda\")\n",
+ "\n",
+ "from transformers import TextStreamer\n",
+ "text_streamer = TextStreamer(tokenizer)\n",
+ "_ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "QQMjaNrjsU5_"
+ },
+ "source": [
+ "You can also use Hugging Face's `AutoModelForPeftCausalLM`. Only use this if you do not have `unsloth` installed. It can be hopelessly slow, since `4bit` model downloading is not supported, and Unsloth's **inference is 2x faster**."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "yFfaXG0WsQuE"
+ },
+ "outputs": [],
+ "source": [
+ "if False:\n",
+ " # I highly do NOT suggest - use Unsloth if possible\n",
+ " from peft import AutoPeftModelForCausalLM\n",
+ " from transformers import AutoTokenizer\n",
+ " model = AutoPeftModelForCausalLM.from_pretrained(\n",
+ " \"lora_model\", # YOUR MODEL YOU USED FOR TRAINING\n",
+ " load_in_4bit = load_in_4bit,\n",
+ " )\n",
+ " tokenizer = AutoTokenizer.from_pretrained(\"lora_model\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "f422JgM9sdVT"
+ },
+ "source": [
+ "### Saving to float16 for VLLM\n",
+ "\n",
+ "We also support saving to `float16` directly. Select `merged_16bit` for float16 or `merged_4bit` for int4. We also allow `lora` adapters as a fallback. Use `push_to_hub_merged` to upload to your Hugging Face account! You can go to https://huggingface.co/settings/tokens for your personal tokens."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "iHjt_SMYsd3P"
+ },
+ "outputs": [],
+ "source": [
+ "# Merge to 16bit\n",
+ "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"merged_16bit\",)\n",
+ "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"merged_16bit\", token = \"\")\n",
+ "\n",
+ "# Merge to 4bit\n",
+ "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"merged_4bit\",)\n",
+ "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"merged_4bit\", token = \"\")\n",
+ "\n",
+ "# Just LoRA adapters\n",
+ "if False: model.save_pretrained_merged(\"model\", tokenizer, save_method = \"lora\",)\n",
+ "if False: model.push_to_hub_merged(\"hf/model\", tokenizer, save_method = \"lora\", token = \"\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TCv4vXHd61i7"
+ },
+ "source": [
+ "### GGUF / llama.cpp Conversion\n",
+ "To save to `GGUF` / `llama.cpp`, we support it natively now! We clone `llama.cpp` and we default save it to `q8_0`. We allow all methods like `q4_k_m`. Use `save_pretrained_gguf` for local saving and `push_to_hub_gguf` for uploading to HF.\n",
+ "\n",
+ "Some supported quant methods (full list on our [Wiki page](https://github.com/unslothai/unsloth/wiki#gguf-quantization-options)):\n",
+ "* `q8_0` - Fast conversion. High resource use, but generally acceptable.\n",
+ "* `q4_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K.\n",
+ "* `q5_k_m` - Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K.\n",
+ "\n",
+ "[**NEW**] To finetune and auto export to Ollama, try our [Ollama notebook](https://colab.research.google.com/drive/1WZDi7APtQ9VsvOrQSSC5DDtxq159j8iZ?usp=sharing)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "FqfebeAdT073"
+ },
+ "outputs": [],
+ "source": [
+ "# Save to 8bit Q8_0\n",
+ "if False: model.save_pretrained_gguf(\"model\", tokenizer,)\n",
+ "# Remember to go to https://huggingface.co/settings/tokens for a token!\n",
+ "# And change hf to your username!\n",
+ "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, token = \"\")\n",
+ "\n",
+ "# Save to 16bit GGUF\n",
+ "if False: model.save_pretrained_gguf(\"model\", tokenizer, quantization_method = \"f16\")\n",
+ "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, quantization_method = \"f16\", token = \"\")\n",
+ "\n",
+ "# Save to q4_k_m GGUF\n",
+ "if False: model.save_pretrained_gguf(\"model\", tokenizer, quantization_method = \"q4_k_m\")\n",
+ "if False: model.push_to_hub_gguf(\"hf/model\", tokenizer, quantization_method = \"q4_k_m\", token = \"\")\n",
+ "\n",
+ "# Save to multiple GGUF options - much faster if you want multiple!\n",
+ "if False:\n",
+ " model.push_to_hub_gguf(\n",
+ " \"hf/model\", # Change hf to your username!\n",
+ " tokenizer,\n",
+ " quantization_method = [\"q4_k_m\", \"q8_0\", \"q5_k_m\",],\n",
+ " token = \"\", # Get a token at https://huggingface.co/settings/tokens\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FyKl34TVyjv-"
+ },
+ "source": [
+ "Now, use the `model-unsloth.gguf` file or `model-unsloth-Q4_K_M.gguf` file in llama.cpp or a UI based system like Jan or Open WebUI. You can install Jan [here](https://github.com/janhq/jan) and Open WebUI [here](https://github.com/open-webui/open-webui)\n",
+ "\n",
+ "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n",
+ "\n",
+ "Some other links:\n",
+ "1. Train your own reasoning model - Llama GRPO notebook [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb)\n",
+ "2. Saving finetunes to Ollama. [Free notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)\n",
+ "3. Llama 3.2 Vision finetuning - Radiography use case. [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb)\n",
+ "6. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://docs.unsloth.ai/get-started/unsloth-notebooks)!\n",
+ "\n",
+ "
\n",
+ "
![](\"https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png\")
\n",
+ "
![](\"https://github.com/unslothai/unsloth/raw/main/images/Discord.png\")
\n",
+ "
![](\"https://github.com/unslothai/unsloth/blob/main/images/documentation%20green%20button.png?raw=true\")
\n",
+ "\n",
+ " Join Discord if you need help + ⭐️
Star us on Github ⭐️\n",
+ "
![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
![](\"https://github.com/unslothai/unsloth/raw/main/images/Discord)