Fixing issues after rebasing

build/builder.py

@@ -141,19 +141,16 @@ class TokenizerArgs:
     def from_args(cls, args):  # -> TokenizerArgs:
         is_SentencePiece = True
         is_TikToken = False
-        checkpoint_dir = args.checkpoint_dir
-
-        if args.model:  # Using a named, well-known model
-            model_config = resolve_model_config(args.model)
-
-            checkpoint_dir = Path(args.model_directory) / model_config.name
 
         if args.tokenizer_path:
             tokenizer_path = args.tokenizer_path
+        elif args.model:  # Using a named, well-known model
+            model_config = resolve_model_config(args.model)
+            tokenizer_path = Path(args.model_directory) / model_config.name / "tokenizer.model"
         elif args.checkpoint_path:
             tokenizer_path = args.checkpoint_path.parent / "tokenizer.model"
-        elif checkpoint_dir:
-            tokenizer_path = checkpoint_dir / "tokenizer.model"
+        elif hasattr(args, "checkpoint_dir") and args.checkpoint_dir:
+            tokenizer_path = args.checkpoint_dir / "tokenizer.model"
         else:
             raise RuntimeError("cannot find tokenizer model")
 

cli.py

@@ -50,36 +50,6 @@ def _add_arguments_common(parser):
         default=None,
         help="Model name for well-known models.",
     )
-    parser.add_argument(
-        "--checkpoint-path",
-        type=Path,
-        default=None,
-        help="Model checkpoint path.",
-    )
-    parser.add_argument(
-        "--checkpoint-dir",
-        type=Path,
-        default=None,
-        help="Model checkpoint directory.",
-    )
-    parser.add_argument(
-        "--params-path",
-        type=Path,
-        default=None,
-        help="Parameter file path.",
-    )
-    parser.add_argument(
-        "--gguf-path",
-        type=Path,
-        default=None,
-        help="GGUF file path.",
-    )
-    parser.add_argument(
-        "--tokenizer-path",
-        type=Path,
-        default=None,
-        help="Model tokenizer path.",
-    )
 
     # TODO: Refactor this so that only common options are here
     # and subcommand-specific options are inside individual