Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flags for varying NIM seed and temperature every call #808

Merged
merged 8 commits into from
Jul 30, 2024
Merged

Flags for varying NIM seed and temperature every call #808

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
79aaa7b
add support for configured random seed (as per API spec)
leondz Jul 30, 2024
430a6a5
add support for varying temp, seed per call - enabled by default
leondz Jul 30, 2024
3fb69a3
use instance-based seed, set from config in constructor; rely on None…
leondz Jul 30, 2024
a9d2d98
prune unused import
leondz Jul 30, 2024
e9be28a
let config do its work instead of accessing config_root directly
leondz Jul 30, 2024
2779ede
prefer `hasattr` to `dir`
jmartin-tech Jul 30, 2024
c5445ba
default `seed` to `None`
jmartin-tech Jul 30, 2024
281c31e
Merge branch 'main' into feature/nim_vary_seed_temp
leondz Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions garak/generators/nim.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

"""NVIDIA Inference Microservice LLM interface"""

import random
from typing import List, Union

import openai
Expand Down Expand Up @@ -38,6 +39,8 @@ class NVOpenAIChat(OpenAICompatible):
"top_p": 0.7,
"top_k": 0, # top_k is hard set to zero as of 24.04.30
"uri": "https://integrate.api.nvidia.com/v1/",
"vary_seed_each_call": True, # encourage variation when generations>1. not respected by all NIMs
"vary_temp_each_call": True, # encourage variation when generations>1. not respected by all NIMs
"suppressed_params": {"n", "frequency_penalty", "presence_penalty"},
}
active = True
Expand Down Expand Up @@ -67,6 +70,13 @@ def _call_model(
assert (
generations_this_call == 1
), "generations_per_call / n > 1 is not supported"

if self.vary_seed_each_call:
self.seed = random.randint(0, 65535)

if self.vary_temp_each_call:
self.temperature = random.random()

return super()._call_model(prompt, generations_this_call)


Expand Down
9 changes: 6 additions & 3 deletions garak/generators/openai.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
# lists derived from https://platform.openai.com/docs/models
chat_models = (
"gpt-4", # links to latest version
"gpt-4-turbo", # links to latest version
"gpt-4o", # links to latest version
"gpt-4o-mini", # links to latest version
"gpt-4-turbo", # links to latest version
"gpt-4o", # links to latest version
"gpt-4o-mini", # links to latest version
"gpt-4-turbo-preview",
"gpt-3.5-turbo", # links to latest version
"gpt-4-32k",
Expand Down Expand Up @@ -135,6 +135,8 @@ def __init__(self, name="", generations=10, config_root=_config):
self._load_config(config_root)
self.fullname = f"{self.generator_family_name} {self.name}"
self.key_env_var = self.ENV_VAR
if not hasattr(self, "seed"):
self.seed = None

self._load_client()

Expand Down Expand Up @@ -183,6 +185,7 @@ def _call_model(
"frequency_penalty": self.frequency_penalty,
"presence_penalty": self.presence_penalty,
"stop": self.stop,
"seed": self.seed,
}

create_args = {
Expand Down
Loading