llama_spec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "if not os.path.exists(\"/scratch/sarthak\"):\n",
    "    os.makedirs(\"/scratch/sarthak\")\n",
    "\n",
    "os.environ[\"HF_HOME\"] = \"/scratch/sarthak/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "from typing import List, Tuple, Optional\n",
    "import time\n",
    "import numpy as np\n",
    "import datasets\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_gr_dataset = datasets.load_dataset('wmt16', 'de-en', split='validation')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "notebookRunGroups": {
     "groupValue": "1"
    }
   },
   "source": [
    "# Speculative Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SpeculativeDecoder:\n",
    "    def __init__(\n",
    "        self,\n",
    "        target_model_name = \"meta-llama/Llama-3.2-3B\",\n",
    "        draft_model_name = \"meta-llama/Llama-3.2-1B\",\n",
    "        gamma = 4,\n",
    "        temperature = 1.0\n",
    "    ):\n",
    "        self.gamma = gamma\n",
    "\n",
    "        self.temperature = temperature\n",
    "\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(target_model_name)\n",
    "        self.tokenizer.pad_token = self.tokenizer.eos_token\n",
    "        self.target_model = AutoModelForCausalLM.from_pretrained(target_model_name, device_map='auto')\n",
    "\n",
    "        self.draft_model = AutoModelForCausalLM.from_pretrained(draft_model_name, device_map='auto')\n",
    "\n",
    "        self.target_model.eval()\n",
    "        self.draft_model.eval()\n",
    "\n",
    "    def get_draft_logits(\n",
    "        self,\n",
    "        input_ids: torch.Tensor,\n",
    "        attention_mask: torch.Tensor,\n",
    "        decoder_input_ids: torch.Tensor,\n",
    "        gamma: int\n",
    "    ) -> Tuple[torch.Tensor, torch.Tensor]:\n",
    "        \"\"\"Get draft logits for gamma tokens\"\"\"\n",
    "        draft_tokens = []\n",
    "        draft_probs = []\n",
    "        current_decoder_ids = decoder_input_ids\n",
    "\n",
    "        # Generate gamma tokens from the draft model\n",
    "        for _ in range(gamma):\n",
    "            with torch.no_grad():\n",
    "                outputs = self.draft_model(\n",
    "                    input_ids=current_decoder_ids,\n",
    "                    attention_mask=attention_mask,\n",
    "                    # decoder_input_ids=current_decoder_ids,\n",
    "                    return_dict=True,\n",
    "                )\n",
    "                logits = outputs.logits[:, -1, :]  # Get logits for last position\n",
    "                probs = logits\n",
    "                # probs = F.softmax(logits, dim=-1)\n",
    "\n",
    "                token_id = torch.argmax(probs, dim=-1)\n",
    "                prob = probs.gather(-1, token_id.unsqueeze(-1)).squeeze(-1)\n",
    "\n",
    "                draft_tokens.append(token_id.item())\n",
    "                draft_probs.append(prob.item())\n",
    "\n",
    "                # Update decoder inputs for next iteration\n",
    "                current_decoder_ids = torch.cat(\n",
    "                    [current_decoder_ids, token_id.view(1, 1)],\n",
    "                    dim=1\n",
    "                )\n",
    "\n",
    "                if token_id.item() == self.tokenizer.eos_token_id:\n",
    "                    break\n",
    "\n",
    "        return draft_tokens, draft_probs, outputs.logits.squeeze(0)\n",
    "\n",
    "    def get_target_probs(\n",
    "        self,\n",
    "        input_ids: torch.Tensor,\n",
    "        attention_mask: torch.Tensor,\n",
    "        decoder_input_ids: torch.Tensor,\n",
    "        draft_tokens: torch.Tensor,\n",
    "    ) -> torch.Tensor:\n",
    "        \"\"\"Get target probabilities for the draft tokens in parallel.\"\"\"\n",
    "        with torch.no_grad():\n",
    "            # Add draft tokens to decoder input\n",
    "            full_decoder_ids = torch.cat([decoder_input_ids, draft_tokens.unsqueeze(0)], dim=1)\n",
    "            # print(input_ids_batched.shape, attention_mask_batched.shape, padded_decoder_ids.shape)\n",
    "            # decoder_mask = torch.triu(\n",
    "            #     torch.ones((full_decoder_ids.shape[1], full_decoder_ids.shape[1] + 1))\n",
    "            # )\n",
    "            # decoder_mask = decoder_mask[-(len(draft_tokens) + 1):, :-1]\n",
    "            # decoder_mask = 1 - decoder_mask\n",
    "            \n",
    "            # the shapes that we want to see are:\n",
    "            # torch.Size([11, 12]) torch.Size([1, 12])\n",
    "            # torch.Size([11, 12]) torch.Size([11, 12])\n",
    "            # torch.Size([11, 32128])\n",
    "            # torch.Size([11, 32128]) torch.Size([11, 32128])\n",
    "\n",
    "            # What im getting\n",
    "            # torch.Size([1, 12]) torch.Size([1, 12])\n",
    "            # torch.Size([1, 12]) torch.Size([1, 12])\n",
    "            # torch.Size([1, 11, 32128])\n",
    "\n",
    "\n",
    "            decoder_mask = torch.ones(full_decoder_ids.shape[1])\n",
    "            decoder_mask = decoder_mask.unsqueeze(0)\n",
    "\n",
    "\n",
    "            # print(decoder_mask.shape, full_decoder_ids.shape)\n",
    "\n",
    "            # conver to a batched input\n",
    "            # input_ids = input_ids.repeat(len(draft_tokens) + 1, 1)\n",
    "            # attention_mask = attention_mask.repeat(len(draft_tokens) + 1, 1)\n",
    "            # full_decoder_ids = full_decoder_ids.repeat(len(draft_tokens) + 1, 1)\n",
    "\n",
    "            # print(decoder_mask.shape, full_decoder_ids.shape)\n",
    "\n",
    "\n",
    "            # outputs = self.target_model(\n",
    "            #     input_ids=input_ids,\n",
    "            #     attention_mask=attention_mask,\n",
    "            #     decoder_input_ids=full_decoder_ids,\n",
    "            #     decoder_attention_mask=decoder_mask,\n",
    "            #     return_dict=True\n",
    "            # )\n",
    "            outputs = self.target_model(\n",
    "                input_ids=full_decoder_ids,\n",
    "                attention_mask=decoder_mask,\n",
    "                return_dict=True\n",
    "            )\n",
    "\n",
    "            \n",
    "            # dim_0_indices = torch.arange(len(draft_tokens) + 1)\n",
    "            # dim_1_indices = torch.arange(len(draft_tokens) + 1) + full_decoder_ids.shape[1] - 1 - len(draft_tokens)\n",
    "            # logits = outputs.logits[dim_0_indices, dim_1_indices, :]\n",
    "            logits = outputs.logits[:, -(len(draft_tokens) + 1):, :]\n",
    "            logits = logits.squeeze(0)\n",
    "\n",
    "            # print(logits.shape)\n",
    "            \n",
    "            # Get probabilities for positions before each draft token\n",
    "            # logits = outputs.logits[:, -(len(draft_tokens) + 1):-1, :]\n",
    "            target_probs = F.softmax(logits, dim=-1)\n",
    "\n",
    "            # print(target_probs.shape, target_probs.squeeze(0).shape)\n",
    "            \n",
    "\n",
    "            return target_probs.squeeze(0), logits\n",
    "\n",
    "    def verify_tokens(\n",
    "        self,\n",
    "        target_probs: torch.Tensor,\n",
    "        draft_tokens: torch.Tensor,\n",
    "        draft_probs: torch.Tensor,\n",
    "    ) -> int:\n",
    "        \"\"\"Determine number of accepted tokens\"\"\"\n",
    "        # Get target probabilities for the draft tokens\n",
    "        # get the probabilities of the tokens at the indices of the draft tokens\n",
    "        target_probs_draft_tokens = torch.gather(target_probs, 1, draft_tokens.unsqueeze(0))\n",
    "        # Calculate acceptance ratios\n",
    "        acceptance_ratios = target_probs_draft_tokens / draft_probs.clamp(min=1e-10)\n",
    "\n",
    "        # Sample uniform random numbers \n",
    "        random_nums = torch.rand_like(acceptance_ratios)\n",
    "        acceptance_mask = random_nums <= acceptance_ratios\n",
    "\n",
    "        num_accepted = (acceptance_mask.cumsum(dim=-1) == torch.arange(1, len(acceptance_ratios) + 1)).sum().item()\n",
    "\n",
    "        return num_accepted\n",
    "\n",
    "    def translate(\n",
    "        self,\n",
    "        source_text: str,\n",
    "        max_length: int = 128\n",
    "    ) -> str:\n",
    "        \"\"\"Generate from source text using speculative decoding.\"\"\"\n",
    "        # Encode source text\n",
    "        encoder_inputs = self.tokenizer(\n",
    "            source_text,\n",
    "            return_tensors=\"pt\",\n",
    "            padding=True\n",
    "        )\n",
    "\n",
    "        # Initialize with start token\n",
    "        decoder_input_ids = torch.tensor([[id for id in encoder_inputs.input_ids[0][:5]]])\n",
    "\n",
    "        # output = self.target_model(\n",
    "        #     input_ids=encoder_inputs.input_ids,\n",
    "        #     attention_mask=encoder_inputs.attention_mask,\n",
    "        #     decoder_input_ids=decoder_input_ids,\n",
    "        #     return_dict=True\n",
    "        # )\n",
    "\n",
    "        # probs = output.logits[:, -1, :]\n",
    "                    \n",
    "        # probs = F.softmax(probs / (self.temperature + 1e-13), dim=-1)\n",
    "        # token_id = torch.multinomial(probs, num_samples=1)\n",
    "\n",
    "        # decoder_input_ids = torch.tensor([[self.tokenizer.pad_token_id, token_id.item()]])\n",
    "\n",
    "        total_tokens = 0\n",
    "        accepted_tokens = 0\n",
    "\n",
    "        while decoder_input_ids.shape[1] < max_length:\n",
    "            # Get draft tokens autoregressively\n",
    "            draft_tokens, draft_probs, draft_logits = self.get_draft_logits(\n",
    "                encoder_inputs.input_ids,\n",
    "                encoder_inputs.attention_mask,\n",
    "                decoder_input_ids,\n",
    "                self.gamma\n",
    "            )\n",
    "\n",
    "            draft_tokens = torch.tensor(draft_tokens)\n",
    "            draft_probs = torch.tensor(draft_probs)\n",
    "\n",
    "            # softmax the draft probs\n",
    "            draft_probs = F.softmax(draft_probs / (self.temperature + 1e-13), dim=-1)\n",
    "\n",
    "            if len(draft_tokens) == 0:\n",
    "                raise ValueError(\"Draft tokens not generated.\")\n",
    "\n",
    "            # Get target probabilities in parallel\n",
    "            # start = time.time()\n",
    "            target_probs, target_logits = self.get_target_probs(\n",
    "                encoder_inputs.input_ids,\n",
    "                encoder_inputs.attention_mask,\n",
    "                decoder_input_ids,\n",
    "                draft_tokens\n",
    "            )\n",
    "            # target probs are the logits but with softmax applied\n",
    "\n",
    "            # Verify tokens\n",
    "            n_accepted = self.verify_tokens(target_probs, draft_tokens, draft_probs)\n",
    "            # Accept verified tokens\n",
    "            if n_accepted > 0:\n",
    "                decoder_input_ids = torch.cat([\n",
    "                    decoder_input_ids,\n",
    "                    draft_tokens[:n_accepted].unsqueeze(0)\n",
    "                ], dim=1)\n",
    "                \n",
    "            with torch.no_grad():\n",
    "                # n_rejected = self.gamma - n_accepted\n",
    "                n_rejected = len(draft_tokens) - n_accepted \n",
    "                total_tokens += len(draft_tokens)\n",
    "                accepted_tokens += n_accepted\n",
    "\n",
    "                if n_rejected > 0:\n",
    "                    probs = target_logits[-n_rejected, :] #- draft_logits[1-n_rejected, :]\n",
    "                else:\n",
    "                    probs = target_logits[-1, :]\n",
    "                    \n",
    "                probs = F.softmax(probs / (self.temperature + 1e-13), dim=-1)\n",
    "                # probs /= max(self.temperature, 1e-13)\n",
    "                # probs_max = torch.where(probs > 0, probs, torch.zeros_like(probs))\n",
    "                # probs_max_sum = torch.sum(probs_max)\n",
    "                # probs = probs_max / max(probs_max_sum, 1e-13)\n",
    "                \n",
    "                token_id = torch.multinomial(probs, num_samples=1).unsqueeze(0)\n",
    "\n",
    "                decoder_input_ids = torch.cat([decoder_input_ids, token_id], dim=1)\n",
    "\n",
    "            # Check for end of sequence\n",
    "            if decoder_input_ids[0][-1].item() == self.tokenizer.eos_token_id:\n",
    "                break\n",
    "\n",
    "            # # or if a full stop is generated\n",
    "            # if decoder_input_ids[0][-1].item() == self.tokenizer.convert_tokens_to_ids('.'):\n",
    "            #     break\n",
    "\n",
    "        # Decode translation\n",
    "        translation = self.tokenizer.decode(\n",
    "            decoder_input_ids[0],\n",
    "            skip_special_tokens=True,\n",
    "            clean_up_tokenization_spaces=True\n",
    "        )\n",
    "        perc_accepted = accepted_tokens / total_tokens * 100\n",
    "        return translation, perc_accepted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdc89c02de414f1abbf3e41d62e0d79e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SpeculativeDecoder(target_model_name=\"google-t5/t5-3b\", draft_model_name=\"google-t5/t5-small\")\n",
    "# SpeculativeDecoder(target_model_name=\"google-t5/t5-large\", draft_model_name=\"google-t5/t5-base\")\n",
    "# decoder = SpeculativeDecoder()\n",
    "# decoder.translate(\"Hi, how are you?\", max_length=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 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"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "notebookRunGroups": {
     "groupValue": "1"
    }
   },
   "source": [
    "# Only Large Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NormalDecoder:\n",
    "    def __init__(\n",
    "        self,\n",
    "        model_name: str = \"meta-llama/Llama-3.2-3B\",\n",
    "        device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\",\n",
    "    ):\n",
    "        self.device = device\n",
    "\n",
    "        # Initialize tokenizer and model\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "        # self.model = T5ForConditionalGeneration.from_pretrained(model_name).to(device)\n",
    "        self.model = AutoModelForCausalLM.from_pretrained(model_name, device_map='auto')\n",
    "\n",
    "        self.tokenizer.pad_token = self.tokenizer.eos_token\n",
    "\n",
    "        self.model.eval()\n",
    "\n",
    "    def get_logits(\n",
    "        self,\n",
    "        input_ids: torch.Tensor,\n",
    "        attention_mask: torch.Tensor,\n",
    "        decoder_input_ids: torch.Tensor\n",
    "    ) -> torch.Tensor:\n",
    "        \"\"\"Get logits from model for the last token.\"\"\"\n",
    "        with torch.no_grad():\n",
    "            outputs = self.model(\n",
    "                input_ids=decoder_input_ids,\n",
    "                attention_mask=attention_mask,\n",
    "                # decoder_input_ids=decoder_input_ids,\n",
    "                return_dict=True\n",
    "            )\n",
    "            return outputs.logits[:, -1, :]\n",
    "\n",
    "    def sample_token(self, logits: torch.Tensor, temperature: float = 1.0) -> Tuple[torch.Tensor, torch.Tensor]:\n",
    "        \"\"\"Sample a token from logits using temperature sampling.\"\"\"\n",
    "        if temperature == 0:\n",
    "            # Greedy sampling\n",
    "            token_id = torch.argmax(logits, dim=-1)\n",
    "            prob = torch.ones_like(token_id, dtype=torch.float)\n",
    "        else:\n",
    "            # Temperature sampling\n",
    "            probs = F.softmax(logits / temperature, dim=-1)\n",
    "            token_id = torch.multinomial(probs, num_samples=1).squeeze(-1)\n",
    "            prob = probs.gather(-1, token_id.unsqueeze(-1)).squeeze(-1)\n",
    "        return token_id, prob\n",
    "\n",
    "    def translate(\n",
    "        self,\n",
    "        source_text: str,\n",
    "        max_length: int = 128,\n",
    "        temperature: float = 0.7\n",
    "    ) -> str:\n",
    "        \"\"\"Translate source text using the normal T5 model.\"\"\"\n",
    "        # Encode source text\n",
    "        encoder_inputs = self.tokenizer(\n",
    "            source_text,\n",
    "            return_tensors=\"pt\",\n",
    "            padding=True\n",
    "        ).to(self.device)\n",
    "\n",
    "        # Initialize decoder input with start token\n",
    "        # decoder_input_ids = torch.tensor([[self.tokenizer.pad_token_id]], device=self.device)\n",
    "        decoder_input_ids = torch.tensor([[id for id in encoder_inputs.input_ids[0][:5]]]).to(self.device)\n",
    "\n",
    "        while decoder_input_ids.shape[1] < max_length:\n",
    "            # Generate logits for the next token\n",
    "            logits = self.get_logits(\n",
    "                encoder_inputs.input_ids,\n",
    "                encoder_inputs.attention_mask,\n",
    "                decoder_input_ids\n",
    "            )\n",
    "\n",
    "            # Sample a token\n",
    "            token_id, _ = self.sample_token(logits, temperature)\n",
    "\n",
    "            # Add token to the decoder input\n",
    "            decoder_input_ids = torch.cat(\n",
    "                [decoder_input_ids, token_id.view(1, 1)],\n",
    "                dim=1\n",
    "            )\n",
    "\n",
    "            # Break if end token is generated\n",
    "            if token_id.item() == self.tokenizer.eos_token_id:\n",
    "                break\n",
    "\n",
    "        # Decode and return translation\n",
    "        translation = self.tokenizer.decode(\n",
    "            decoder_input_ids[0],\n",
    "            skip_special_tokens=True,\n",
    "            clean_up_tokenization_spaces=True\n",
    "        )\n",
    "        return translation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4841eca391a45ea81f7f6a2f4fce10d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Starting from v4.46, the `logits` model output will have the same type as the model (except at train time, where it will always be FP32)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speculative Generation: Hi, how are you? I am Ozan, a Turkish guy who loves to travel and meet new people. I am a student and i currently live in Istanbul. It is my dream to visit all countries in the world meet new people and experience new cultures. I am flexible and open minded. YOu can contact me skype:anatolozan91 or:ozan.anatolozan@live.com.:\n",
      "Accepted Tokens: 72.22%\n",
      "Speculative Decoding Time: 8.43s\n"
     ]
    }
   ],
   "source": [
    "decoder = SpeculativeDecoder()\n",
    "begin = time.time()\n",
    "res, perc = decoder.translate(\"Hi, how are you?\", max_length=128)\n",
    "spec_time = time.time() - begin\n",
    "\n",
    "print(f\"Speculative Generation: {res}\")\n",
    "print(f\"Accepted Tokens: {perc:.2f}%\")\n",
    "print(f\"Speculative Decoding Time: {spec_time:.2f}s\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e4bc39cab684646862257c83eb49499",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normal Generation: Hi, how are you? It's been a while since I posted here. I've had a busy summer. I've been cooking all the time, so I've been a bit busy. I've been taking loads of photos, but it's hard to find time to write. Now that I've got a bit of free time, I'm going to share with you some of the dishes I've been cooking. I'm going to start with this one. It's a simple dish, but it's really good. I've been making it for years, and it's always a hit. I've been cooking it for my\n",
      "Normal Decoding Time: 10.69s\n",
      "Speedup: 1.27x\n"
     ]
    }
   ],
   "source": [
    "normal_decoder = NormalDecoder()\n",
    "begin = time.time()\n",
    "res_normal = normal_decoder.translate(\"Hi, how are you?\", max_length=128)\n",
    "normal_time = time.time() - begin\n",
    "\n",
    "print(f\"Normal Generation: {res_normal}\")\n",
    "print(f\"Normal Decoding Time: {normal_time:.2f}s\")\n",
    "print(f\"Speedup: {normal_time / spec_time:.2f}x\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Varying gamma values for T5-3b(target) + T5-small(draft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "notebookRunGroups": {
     "groupValue": "1"
    }
   },
   "outputs": [],
   "source": [
    "def vary_gamma(gamma=3):\n",
    "\n",
    "    speculative_decoder = SpeculativeDecoder(gamma=gamma)\n",
    "    normal_decoder = NormalDecoder()\n",
    "\n",
    "    spec_total_time = 0\n",
    "    normal_total_time = 0\n",
    "    total_iters = 0\n",
    "    total_pc = 0\n",
    "\n",
    "    f = open(\"res/t53b-spec-gamma-{}.txt\".format(gamma), \"w\")\n",
    "\n",
    "    for i in tqdm(en_gr_dataset['translation'][:30]):\n",
    "        source_text = i['en']\n",
    "        target_text = i['de']\n",
    "        \n",
    "        # Time the translation\n",
    "        start_time = time.time()\n",
    "        spec_translation, pc = speculative_decoder.translate(source_text)\n",
    "        end_time = time.time()\n",
    "\n",
    "        spec_time = end_time - start_time\n",
    "\n",
    "        # spec_total_time += spec_time\n",
    "        \n",
    "        start_time = time.time()\n",
    "        normal_translation = normal_decoder.translate(source_text)\n",
    "        end_time = time.time()\n",
    "\n",
    "        normal_time = end_time - start_time\n",
    "\n",
    "        # normal_total_time += normal_time\n",
    "\n",
    "        # print(f\"Source: {source_text}\")\n",
    "        # print(f\"Normal Translation: {normal_translation}\")\n",
    "        # print(f\"Time taken: {normal_time:.2f} seconds\")\n",
    "        # print(f\"Speculative Translation: {spec_translation}\")\n",
    "        # print(f\"Time taken: {spec_time:.2f} seconds\")\n",
    "        # print(f\"Percentage tokens accepted: {pc:.2f}%\")\n",
    "        # print(f\"Target: {target_text}\")\n",
    "        f.write(f\"Source: {source_text}\\n\")\n",
    "        f.write(f\"Normal Translation: {normal_translation}\\n\")\n",
    "        f.write(f\"Time taken: {normal_time:.2f} seconds\\n\")\n",
    "        f.write(f\"Speculative Translation: {spec_translation}\\n\")\n",
    "        f.write(f\"Time taken: {spec_time:.2f} seconds\\n\")\n",
    "        f.write(f\"Percentage tokens accepted: {pc:.2f}%\\n\")\n",
    "        f.write(f\"Target: {target_text}\\n\")\n",
    "        f.write(\"\\n\")\n",
    "\n",
    "        spec_total_time += spec_time\n",
    "        normal_total_time += normal_time\n",
    "        total_pc += pc\n",
    "        total_iters += 1\n",
    "\n",
    "    print(f\"\\nAverage time taken for normal decoding: {normal_total_time / total_iters:.2f} seconds\")\n",
    "    print(f\"Average time taken for speculative decoding: {spec_total_time / total_iters:.2f} seconds\")\n",
    "    print(f\"Average percentage of tokens accepted: {total_pc / total_iters:.2f}%\")\n",
    "    print(f\"Average speedup over {total_iters} iterations: {normal_total_time / spec_total_time:.2f}x\")\n",
    "\n",
    "    f.write(f\"\\nAverage time taken for normal decoding: {normal_total_time / total_iters:.2f} seconds\\n\")\n",
    "    f.write(f\"Average time taken for speculative decoding: {spec_total_time / total_iters:.2f} seconds\\n\")\n",
    "    f.write(f\"Average percentage of tokens accepted: {total_pc / total_iters:.2f}%\\n\")\n",
    "    f.write(f\"Average speedup over {total_iters} iterations: {normal_total_time / spec_total_time:.2f}x\\n\")\n",
    "    f.write(\"\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "notebookRunGroups": {
     "groupValue": "2"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [02:46<00:00,  5.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.17 seconds\n",
      "Average time taken for speculative decoding: 2.37 seconds\n",
      "Average percentage of tokens accepted: 78.73%\n",
      "Average speedup over 30 iterations: 1.34x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_gamma(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [02:31<00:00,  5.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.20 seconds\n",
      "Average time taken for speculative decoding: 1.84 seconds\n",
      "Average percentage of tokens accepted: 78.10%\n",
      "Average speedup over 30 iterations: 1.74x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_gamma(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [02:30<00:00,  5.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.22 seconds\n",
      "Average time taken for speculative decoding: 1.81 seconds\n",
      "Average percentage of tokens accepted: 65.90%\n",
      "Average speedup over 30 iterations: 1.78x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_gamma(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [02:52<00:00,  5.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.59 seconds\n",
      "Average time taken for speculative decoding: 2.17 seconds\n",
      "Average percentage of tokens accepted: 59.11%\n",
      "Average speedup over 30 iterations: 1.66x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_gamma(9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Varying models values for optimal gamma value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vary_models(target_model_name = \"google/t5-3b\", draft_model_name = \"google/t5-small\", gamma=7, verbose=False):\n",
    "\n",
    "    speculative_decoder = SpeculativeDecoder(target_model_name=target_model_name, draft_model_name=draft_model_name, gamma=gamma)\n",
    "    normal_decoder = NormalDecoder(model_name=target_model_name)\n",
    "\n",
    "    spec_total_time = 0\n",
    "    normal_total_time = 0\n",
    "    total_iters = 0\n",
    "    total_pc = 0\n",
    "\n",
    "    target_name = target_model_name.split(\"/\")[1]\n",
    "    draft_name = draft_model_name.split(\"/\")[1]\n",
    "\n",
    "    f = open(f\"res/spec-{target_name}-{draft_name}.txt\", \"w\")\n",
    "\n",
    "    for i in tqdm(en_gr_dataset['translation'][:30]):\n",
    "        source_text = i['en']\n",
    "        target_text = i['de']\n",
    "        \n",
    "        # Time the translation\n",
    "        start_time = time.time()\n",
    "        spec_translation, pc = speculative_decoder.translate(source_text)\n",
    "        end_time = time.time()\n",
    "\n",
    "        spec_time = end_time - start_time\n",
    "\n",
    "        # spec_total_time += spec_time\n",
    "        \n",
    "        start_time = time.time()\n",
    "        normal_translation = normal_decoder.translate(source_text)\n",
    "        end_time = time.time()\n",
    "\n",
    "        normal_time = end_time - start_time\n",
    "\n",
    "        # normal_total_time += normal_time\n",
    "        if verbose:\n",
    "            print(f\"Source: {source_text}\")\n",
    "            print(f\"Normal Translation: {normal_translation}\")\n",
    "            print(f\"Time taken: {normal_time:.2f} seconds\")\n",
    "            print(f\"Speculative Translation: {spec_translation}\")\n",
    "            print(f\"Time taken: {spec_time:.2f} seconds\")\n",
    "            print(f\"Percentage tokens accepted: {pc:.2f}%\")\n",
    "            print(f\"Target: {target_text}\")\n",
    "        f.write(f\"Source: {source_text}\\n\")\n",
    "        f.write(f\"Normal Translation: {normal_translation}\\n\")\n",
    "        f.write(f\"Time taken: {normal_time:.2f} seconds\\n\")\n",
    "        f.write(f\"Speculative Translation: {spec_translation}\\n\")\n",
    "        f.write(f\"Time taken: {spec_time:.2f} seconds\\n\")\n",
    "        f.write(f\"Percentage tokens accepted: {pc:.2f}%\\n\")\n",
    "        f.write(f\"Target: {target_text}\\n\")\n",
    "        f.write(\"\\n\")\n",
    "\n",
    "        spec_total_time += spec_time\n",
    "        normal_total_time += normal_time\n",
    "        total_pc += pc\n",
    "        total_iters += 1\n",
    "\n",
    "    print(f\"\\nAverage time taken for normal decoding: {normal_total_time / total_iters:.2f} seconds\")\n",
    "    print(f\"Average time taken for speculative decoding: {spec_total_time / total_iters:.2f} seconds\")\n",
    "    print(f\"Average percentage of tokens accepted: {total_pc / total_iters:.2f}%\")\n",
    "    print(f\"Average speedup over {total_iters} iterations: {normal_total_time / spec_total_time:.2f}x\")\n",
    "\n",
    "    f.write(f\"\\nAverage time taken for normal decoding: {normal_total_time / total_iters:.2f} seconds\\n\")\n",
    "    f.write(f\"Average time taken for speculative decoding: {spec_total_time / total_iters:.2f} seconds\\n\")\n",
    "    f.write(f\"Average percentage of tokens accepted: {total_pc / total_iters:.2f}%\\n\")\n",
    "    f.write(f\"Average speedup over {total_iters} iterations: {normal_total_time / spec_total_time:.2f}x\\n\")\n",
    "    f.write(\"\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [02:29<00:00,  4.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.14 seconds\n",
      "Average time taken for speculative decoding: 1.85 seconds\n",
      "Average percentage of tokens accepted: 71.06%\n",
      "Average speedup over 30 iterations: 1.69x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_models(\"google-t5/t5-3b\", \"google-t5/t5-small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [03:06<00:00,  6.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.46 seconds\n",
      "Average time taken for speculative decoding: 2.76 seconds\n",
      "Average percentage of tokens accepted: 62.88%\n",
      "Average speedup over 30 iterations: 1.25x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_models(\"google-t5/t5-3b\", \"google-t5/t5-base\", gamma=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vary_models(\"google-t5/t5-3b\", \"google-t5/t5-large\", gamma=7, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [02:26<00:00,  4.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.04 seconds\n",
      "Average time taken for speculative decoding: 1.83 seconds\n",
      "Average percentage of tokens accepted: 70.80%\n",
      "Average speedup over 30 iterations: 1.66x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_models(\"google-t5/t5-large\", \"google-t5/t5-small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [03:05<00:00,  6.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.06 seconds\n",
      "Average time taken for speculative decoding: 3.13 seconds\n",
      "Average percentage of tokens accepted: 65.75%\n",
      "Average speedup over 30 iterations: 0.98x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_models(\"google-t5/t5-large\", \"google-t5/t5-base\", gamma=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 30/30 [01:35<00:00,  3.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 1.60 seconds\n",
      "Average time taken for speculative decoding: 1.57 seconds\n",
      "Average percentage of tokens accepted: 71.93%\n",
      "Average speedup over 30 iterations: 1.02x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_models(\"google-t5/t5-base\", \"google-t5/t5-small\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Varying temperature values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vary_temp(target_model_name = \"google-t5/t5-3b\", draft_model_name = \"google-t5/t5-small\", temperature=0.7):\n",
    "\n",
    "    speculative_decoder = SpeculativeDecoder(target_model_name=target_model_name, draft_model_name=draft_model_name, gamma=7, temperature=temperature)\n",
    "    normal_decoder = NormalDecoder(model_name=target_model_name)\n",
    "\n",
    "    spec_total_time = 0\n",
    "    normal_total_time = 0\n",
    "    total_iters = 0\n",
    "    total_pc = 0\n",
    "\n",
    "    f = open(f\"res/spec-temp-{temperature}.txt\", \"w\")\n",
    "\n",
    "    for i in tqdm(en_gr_dataset['translation'][:30]):\n",
    "        source_text = i['en']\n",
    "        target_text = i['de']\n",
    "        \n",
    "        # Time the translation\n",
    "        start_time = time.time()\n",
    "        spec_translation, pc = speculative_decoder.translate(source_text)\n",
    "        end_time = time.time()\n",
    "\n",
    "        spec_time = end_time - start_time\n",
    "\n",
    "        # spec_total_time += spec_time\n",
    "        \n",
    "        start_time = time.time()\n",
    "        normal_translation = normal_decoder.translate(source_text)\n",
    "        end_time = time.time()\n",
    "\n",
    "        normal_time = end_time - start_time\n",
    "\n",
    "        # normal_total_time += normal_time\n",
    "\n",
    "        # print(f\"Source: {source_text}\")\n",
    "        # print(f\"Normal Translation: {normal_translation}\")\n",
    "        # print(f\"Time taken: {normal_time:.2f} seconds\")\n",
    "        # print(f\"Speculative Translation: {spec_translation}\")\n",
    "        # print(f\"Time taken: {spec_time:.2f} seconds\")\n",
    "        # print(f\"Percentage tokens accepted: {pc:.2f}%\")\n",
    "        # print(f\"Target: {target_text}\")\n",
    "        f.write(f\"Source: {source_text}\\n\")\n",
    "        f.write(f\"Normal Translation: {normal_translation}\\n\")\n",
    "        f.write(f\"Time taken: {normal_time:.2f} seconds\\n\")\n",
    "        f.write(f\"Speculative Translation: {spec_translation}\\n\")\n",
    "        f.write(f\"Time taken: {spec_time:.2f} seconds\\n\")\n",
    "        f.write(f\"Percentage tokens accepted: {pc:.2f}%\\n\")\n",
    "        f.write(f\"Target: {target_text}\\n\")\n",
    "        f.write(\"\\n\")\n",
    "\n",
    "        spec_total_time += spec_time\n",
    "        normal_total_time += normal_time\n",
    "        total_pc += pc\n",
    "        total_iters += 1\n",
    "\n",
    "    print(f\"\\nAverage time taken for normal decoding: {normal_total_time / total_iters:.2f} seconds\")\n",
    "    print(f\"Average time taken for speculative decoding: {spec_total_time / total_iters:.2f} seconds\")\n",
    "    print(f\"Average percentage of tokens accepted: {total_pc / total_iters:.2f}%\")\n",
    "    print(f\"Average speedup over {total_iters} iterations: {normal_total_time / spec_total_time:.2f}x\")\n",
    "\n",
    "    f.write(f\"\\nAverage time taken for normal decoding: {normal_total_time / total_iters:.2f} seconds\\n\")\n",
    "    f.write(f\"Average time taken for speculative decoding: {spec_total_time / total_iters:.2f} seconds\\n\")\n",
    "    f.write(f\"Average percentage of tokens accepted: {total_pc / total_iters:.2f}%\\n\")\n",
    "    f.write(f\"Average speedup over {total_iters} iterations: {normal_total_time / spec_total_time:.2f}x\\n\")\n",
    "    f.write(\"\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [03:22<00:00,  6.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.19 seconds\n",
      "Average time taken for speculative decoding: 3.57 seconds\n",
      "Average percentage of tokens accepted: 17.77%\n",
      "Average speedup over 30 iterations: 0.90x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_temp(temperature=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [02:53<00:00,  5.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.18 seconds\n",
      "Average time taken for speculative decoding: 2.60 seconds\n",
      "Average percentage of tokens accepted: 35.19%\n",
      "Average speedup over 30 iterations: 1.22x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_temp(temperature=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [02:40<00:00,  5.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.41 seconds\n",
      "Average time taken for speculative decoding: 1.96 seconds\n",
      "Average percentage of tokens accepted: 64.49%\n",
      "Average speedup over 30 iterations: 1.74x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_temp(temperature=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "100%|██████████| 30/30 [03:05<00:00,  6.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average time taken for normal decoding: 3.88 seconds\n",
      "Average time taken for speculative decoding: 2.31 seconds\n",
      "Average percentage of tokens accepted: 67.20%\n",
      "Average speedup over 30 iterations: 1.68x\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vary_temp(temperature=1.0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}