Merge pull request #41 from marclop/f/add-NumCtx-option

api: add `NumCtx` option
parakeet-nest · Dec 21, 2024 · c4d41d8 · c4d41d8
2 parents ea5fe9d + e7d9aa5
commit c4d41d8
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/enums/option/option.go b/enums/option/option.go
@@ -1,14 +1,15 @@
-package option 
+package option
 
 const (
-	RepeatLastN    = "RepeatLastN"
+	RepeatLastN   = "RepeatLastN"
 	Temperature   = "Temperature"
 	Seed          = "Seed"
 	RepeatPenalty = "RepeatPenalty"
 	Stop          = "Stop"
 
 	NumKeep          = "NumKeep"
 	NumPredict       = "NumPredict"
+	NumCtx           = "NumCtx"
 	TopK             = "TopK"
 	TopP             = "TopP"
 	TFSZ             = "TFSZ"
@@ -21,4 +22,4 @@ const (
 	PenalizeNewline  = "PenalizeNewline"
 
 	Verbose = "Verbose"
-)
+)
diff --git a/llm/types.go b/llm/types.go
@@ -102,6 +102,8 @@ type AnswerChat struct {
 */
 
 type Options struct {
+	NumCtx int `json:"num_ctx,omitempty"`
+
 	RepeatLastN   int      `json:"repeat_last_n"`
 	Temperature   float64  `json:"temperature"`
 	Seed          int      `json:"seed"`
@@ -154,6 +156,8 @@ func SetOptions(options map[string]interface{}) Options {
 	defaultOptions := DefaultOptions()
 	for key, value := range options {
 		switch key {
+		case "NumCtx":
+			defaultOptions.NumCtx = value.(int)
 		case "NumPredict":
 			defaultOptions.NumPredict = value.(int)
 		case "NumKeep":