Update Training Container (#18)

* fixed merge

* update to pipeline version v0.0.5 bfore release

* updates

* log incremental

* fix incremental

Author: tybritten

bring-your-own-model/PDK_implementation/pipelines/_on_prem_training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir bring-your-own-model/PDK_implementation/experiment --config const.yaml --repo customer-churn-data --model customer-churn --project pdk-customer-churn"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

bring-your-own-model/PDK_implementation/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir bring-your-own-model/PDK_implementation/experiment --config const.yaml --repo customer-churn-data --model customer-churn --project pdk-customer-churn"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/brain-mri/pipelines/_on_prem_training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/brain-mri/experiment --config const.yaml --repo brain-mri-data --model brain-mri  --project pdk-brain-mri"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/brain-mri/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/brain-mri/experiment --config const.yaml --repo brain-mri-data --model brain-mri --project pdk-brain-mri"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/dog-cat/pipelines/_on_prem_training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/dog-cat/experiment --config const.yaml --repo dogs-and-cats-data --model dogs-and-cats --project pdk-dogs-and-cats"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/dog-cat/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/dog-cat/experiment --config const.yaml --repo dogs-and-cats-data --model dogs-and-cats --project pdk-dogs-and-cats"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/object-detection/pipelines/_on_prem_training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/object-detection/experiment --config const.yaml --repo object-detection-data --model object-detection --project pdk-object-detection --incremental false"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/object-detection/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/object-detection/experiment --config const.yaml --repo object-detection-data --model object-detection --project pdk-object-detection --incremental false"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/sentiment-analysis/pipelines/_on_prem_training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/sentiment-analysis/experiment --config const.yaml --repo finbert-prep --model finbert --project pdk-finbert"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/sentiment-analysis/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/sentiment-analysis/experiment --config const.yaml --repo finbert-prep --model finbert --project pdk-finbert"
     ],
-    "image": "pachyderm/pdk:train-v0.0.4",
+    "image": "pachyderm/pdk:train-v0.0.5",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/training_container/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.10
 
 WORKDIR /app
 

examples/training_container/requirements.txt

@@ -1,3 +1,4 @@
-determined==0.26.7
-PyYAML==5.4.1
-ruamel.yaml==0.17.40
+
+determined==0.27.0
+PyYAML==6.0.1
+ruamel.yaml==0.18.5

examples/training_container/train.py

@@ -83,6 +83,7 @@ def parse_args():
         "--incremental",
         type=bool,
         default=True,
+        action=argparse.BooleanOptionalAction,
         help="Send previous commit to download only the diff",
     )
     return parser.parse_args()
@@ -191,6 +192,7 @@ def run_experiment(client, configfile, code_path, model, incremental):
     version = None
     if incremental:
         version = model.get_version()
+        print("Incremental Training enabled")
 
     if version is None:
         print("Creating a new experiment on DeterminedAI...")
@@ -207,7 +209,7 @@ def run_experiment(client, configfile, code_path, model, incremental):
 
 def get_checkpoint(exp):
     try:
-        return exp.top_checkpoint()
+        return exp.list_checkpoints(max_results=1)[0]
     except AssertionError:
         return None
 
@@ -216,11 +218,11 @@ def get_checkpoint(exp):
 
 
 def get_or_create_model(client, model_name, pipeline, repo):
-    models = client.get_models(name=model_name)
+    models = client.list_models(name=model_name)
 
     if len(models) > 0:
         print(f"Model already present. Updating it : {model_name}")
-        model = client.get_models(name=model_name)[0]
+        model = client.list_models(name=model_name)[0]
     else:
         print(f"Creating a new model : {model_name}")
         model = client.create_model(
@@ -277,7 +279,6 @@ def main():
     print(
         f"Starting pipeline: name='{pipeline}', repo='{args.repo}', job_id='{job_id}'"
     )
-
     # --- Download code repository
 
     local_repo = os.path.join(os.getcwd(), "code-repository")