Merge pull request #16 from determined-ai/mldm_28_mlde_267

fixed training images and BYOM

Author: denisabrantes

bring-your-own-model/PDK_implementation/container/deploy/customer_churn_handler.py

@@ -24,17 +24,17 @@ class CustomerChurnHandler(BaseHandler):
 
     def __init__(self):
         super(CustomerChurnHandler, self).__init__()
-        
+
         f = open("numscale.json")
         self.scale_dict = json.load(f)
         f.close()
 
     def scale_data(self, df):
         for col in self.scale_dict:
             df[col] = (df[col] - self.scale_dict[col]["mean"]) / self.scale_dict[col]["std"]
-        
+
         return df
-    
+
     def encode_categories(self, df):
         expected_categories = {}
         expected_categories["new_cell"] = ['U','Y','N']
@@ -56,16 +56,16 @@ def encode_categories(self, df):
         expected_categories["kid11_15"] = ['U','Y']
         expected_categories["kid16_17"] = ['U','Y']
         expected_categories["creditcd"] = ['Y','N']
-        
+
         for col in expected_categories:
             categorical_col = pd.Categorical(df[col], categories=expected_categories[col], ordered=False)
             one_hot_cols = pd.get_dummies(categorical_col, prefix=col)
             df.drop(col, axis=1, inplace=True)
             df = pd.concat([df, one_hot_cols], axis=1)
-        
+
         return df
 
-    def preprocess(self, requests):
+    def preprocess(self, data):
         """
         Get the data from the JSON request in a dictionary, convert it to a pandas DataFrame.
         Then scale its numerical features using values from numscale.json, encode its categorical features,
@@ -77,25 +77,28 @@ def preprocess(self, requests):
         """
 
         # unpack the data
-        data = requests[0].get('body')
-        if data is None:
-            data = requests[0].get('data')
-            
-        df = pd.DataFrame.from_dict(data).reset_index(drop=True)
-        logger.info('Successfully converted json/dict back to pandas DataFrame')                             
-        
+        df_data = data[0]['data']
+        df = pd.DataFrame.from_dict(df_data).reset_index(drop=True)
+        logger.info('Successfully converted json/dict back to pandas DataFrame')
+
         df = self.scale_data(df)
         logger.info('Numerical features successfully scaled')
-        
+
         df = self.encode_categories(df)
         logger.info('Categorical features successfully encoded')
-        
+
         feature_cols = list(df.columns)
         label_col = "churn"
         if label_col in feature_cols:
             feature_cols.remove(label_col)
-        
-        input_tensor = torch.Tensor(df[feature_cols].values)
+
+        feature_values = df[feature_cols].values
+        x = []
+        for feature in feature_values:
+            x.append(feature)
+
+        input_tensor = torch.Tensor(x)
+        #input_tensor = torch.Tensor(df[feature_cols].values)
         logger.info('Dataframe successfully converted to tensor')
 
         return input_tensor

bring-your-own-model/PDK_implementation/container/deploy/deploy.py

@@ -70,7 +70,7 @@ def create_mar_file(model_name, model_version):
 
 def create_properties_file(model_name, model_version):
     config_properties = """inference_address=http://0.0.0.0:8085
-management_address=http://0.0.0.0:8081
+management_address=http://0.0.0.0:8083
 metrics_address=http://0.0.0.0:8082
 grpc_inference_port=7070
 grpc_management_port=7071

bring-your-own-model/PDK_implementation/container/deploy/requirements.txt

@@ -8,4 +8,4 @@ kserve==0.9.0
 determined==0.22.0
 torch==1.13.1
 pandas==1.5.2
-scikit-learn
+scikit-learn

bring-your-own-model/PDK_implementation/pipelines/_on_prem_deployment-pipeline.json

@@ -18,7 +18,7 @@
     "stdin": [
       "python deploy.py --k8s-config-file /determined_shared_fs/k8s.config --deployment-name customer-churn --service-account-name pach-deploy --resource-requests cpu=2,memory=4Gi --resource-limits cpu=10,memory=8Gi"
     ],
-    "image": "pachyderm/pdk:byom-deploy-v0.0.1",
+    "image": "pachyderm/pdk:byom-deploy-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

bring-your-own-model/PDK_implementation/pipelines/_on_prem_training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir bring-your-own-model/PDK_implementation/experiment --config const.yaml --repo customer-churn-data --model customer-churn --project pdk-customer-churn"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

bring-your-own-model/PDK_implementation/pipelines/deployment-pipeline.json

@@ -18,7 +18,7 @@
     "stdin": [
       "python deploy.py --deployment-name customer-churn --cloud-model-host gcp --cloud-model-bucket pdk-repo-models --resource-requests cpu=2,memory=4Gi --resource-limits cpu=10,memory=8Gi"
     ],
-    "image": "us-central1-docker.pkg.dev/dai-dev-554/pdk-registry/pdk_customer_churn_deploy:2.2",
+    "image": "pachyderm/pdk:byom-deploy-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

bring-your-own-model/PDK_implementation/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir bring-your-own-model/PDK_implementation/experiment --config const.yaml --repo customer-churn-data --model customer-churn --project pdk-customer-churn"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

bring-your-own-model/readme.md

@@ -4,7 +4,7 @@
 
 # PDK - Pachyderm | Determined | KServe
 ## Bringing Your Model to PDK
-**Date/Revision:** August 30, 2023
+**Date/Revision:** January 02, 2024
 
 In this section, we will train and deploy a simple customer churn model on PDK.
 
@@ -72,25 +72,25 @@ data:
  
 * Additionally, if the original experiment had a training length specified in number of epochs, it may be convenient to **define training length in number of batches instead** (the same applies for **min_validation_period**).
   * Indeed, the number of samples in the training set will now vary as new data gets committed to the MLDM repository, and knowing that number of samples is mandatory to define training length in number of epochs.
-  * Note that the training pipeline image could be modified to deal with that issue, but specifying the training length in batches is a simple solution.
+  * Note that the training pipeline image could be modified to deal with that issue, but specifying the training length in batches is a simpler solution.
 * Depending on the organization of the MLDE cluster where these automatically triggered experiments are expected to run, it may also be a good idea to **edit the workspace and project fields accordingly**.
 
  
 
 ### Step 1-2: Add code to download data from MLDM
-* In **startup-hook.sh**, install python-pachyderm.
-* In **data.py**, add the imports (_os_, _shutil_, _python-pachyderm_) that are required to define the two new functions to add: _safe_open_wb_, and _download_pach_repo_. The later one being used to download data from the MLDM repository.
-  * **Note:** In this example, _download_pach_repo_ will only download files corresponding to the difference between current and last commit on the MLDM repository. It won't redownload and retrain on the initial *data_part1* if *data_part2* has been committed afterwards. You can change that behaviour by editing the _download_pach_repo_ function.
+* In **startup-hook.sh**, install `pachyderm-sdk`.
+* In **data.py**, add the imports (`os`, `shutil`, `python-pachyderm`) that are required to define the two new functions to add: `safe_open_wb`, and `download_pach_repo`. The later one being used to download data from the MLDM repository.
+  * **Note:** In this example, `download_pach_repo` will only download files corresponding to the difference between current and last commit on the MLDM repository. It won't redownload and retrain on the initial *data_part1* if *data_part2* has been committed afterwards. You can change that behaviour by editing the `download_pach_repo` function.
 * In **model_def.py**:
-  * Add _os_, _logging_ and _download_pach_repo_ as imports
+  * Add `os`, `logging` and `download_pach_repo` as imports
   * In \_\__init___, check if the model is expected to be trained (which would require downloading data from the MLDM repository, building the training set and building the validation sets) or not.
-  * Add the _download_data_ function, that will call the _download_pach_repo_ function to download files from the MLDM repository and return the list of those files.
+  * Add the `download_data` function, that will call the `download_pach_repo` function to download files from the MLDM repository and return the list of those files.
 
 ### Step 1-3: Make sure the code handles the output of the _download_data_ function
 
-The original code may not handle a list of files, as output by the _download_data_ function. In this example, in the base experiment, a single csv data file was expected, while a list of files can be expected with the PDK experiment. Depending on your original code, and how you expect your data to be committed to MLDM, this may or may not require changes.
+The original code may not handle a list of files, as output by the `download_data` function. In this example, in the base experiment, a single csv data file was expected, while a list of files can be expected with the PDK experiment. Depending on your original code, and how you expect your data to be committed to MLDM, this may or may not require changes.
 
-In this example, the _get_train_and_validation_datasets_ function from **data.py** has been changed to concatenate a list of csv files into a single pandas DataFrame.
+In this example, the `get_train_and_validation_datasets` function from **data.py** has been changed to concatenate a list of csv files into a single pandas DataFrame.
 
 ## Step 2: Preparing MLDM and MLDE
 
@@ -115,6 +115,18 @@ By default, we are using the same Workspace that was created in the deployment t
 det p create "PDK Demos" pdk-customer-churn
 ```
 
+### Step 2-3: Create the storage bucket folders
+
+Create the following folder structure in the storage bucket (can be skipped for vanilla kubernetes deployments):
+
+```bash
+customer-churn
+customer-churn/config
+customer-churn/model-store
+```
+
+ 
+
 
  
 ## Step 3: Create the training pipeline
@@ -132,7 +144,7 @@ In case this is not the case or if you want to dig deeper into the details, all
 * Name this MLDM pipeline by changing the _pipeline.name_.
 * Make sure the input repo matches the MLDM repository where data is expected to be committed.
 * Under _transform_:
-  * Define the image to be used. The current image corresponds to files in the **container/train** folder and should work well as it is.
+  * Define the image to be used. The current image configured in the pipeline should work well as it is.
   * _stdin_ command will be run when the pipeline is triggered. Make sure to change all the relevant options, in particular:
     * _--git-url_ to point to the Git URL containing the model code, since you probably want to change details in the experiment files.
     * _--sub-dir_ if the file structure of your git repository is different to this one.

deploy/README.md

@@ -152,9 +152,7 @@ Also, a Worspace and Project were configured for this experiment. You can change
 
  
 
-**Important**: The default setting for the examples included here is to run on the *gpu-pool* resource pool. If your MLDE instance does not have a resource pool called *gpu-pool*, the experiments will fail to run. Make sure to modify the experiment files as needed.
-
-Also, don't forget to create a Workspace and a Project in MLDE with the same name as configured in the file; otherwise, the experiment will fail to run. This can be done in the Workspaces page in the UI.
+Don't forget to create a Workspace and a Project in MLDE with the same name as configured in the file; otherwise, the experiment will fail to run. This can be done in the Workspaces page in the UI.
 
 ![alt text][github_03_workspaces]
 
@@ -192,8 +190,6 @@ A brief description of the Experiment files:
 
 The experiment files don't need to be modified, except for the Workspace and Project name in the `const.yaml` file. Do keep in mind that, at runtime, the pipeline will pull this code from Github. Any changes to any of the files need to be uploaded to your repository.
 
-
-
  
 
 ### MLDM Images
@@ -375,7 +371,7 @@ In the Training pipeline file, change the command line to point to your github r
 "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/dog-cat/experiment --config const.yaml --repo dogs-and-cats-data --model dogs-and-cats --project pdk-dogs-and-cats"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.3",
 ```
 
 
@@ -438,7 +434,7 @@ Also, replace the path to your image, or use the default value.
  "stdin": [
       "python deploy.py --deployment-name dog-cat --cloud-model-host gcp --cloud-model-bucket pdk-repo-models --resource-requests cpu=2,memory=8Gi --resource-limits cpu=10,memory=8Gi"
     ],
-    "image": "pachyderm/pdk:dog-cat-deploy-v0.0.1",
+    "image": "pachyderm/pdk:dog-cat-deploy-v0.0.3",
 ```
  
 

deploy/deploy_aws.md

@@ -654,6 +654,14 @@ provisioner: kubernetes.io/aws-ebs
 parameters:
   type: gp3
   fsType: ext4
+volumeBindingMode: WaitForFirstConsumer
+allowedTopologies:
+- matchLabelExpressions:
+  - key: failure-domain.beta.kubernetes.io/zone
+    values:
+    - ${AWS_AVAILABILITY_ZONE_1}
+    - ${AWS_AVAILABILITY_ZONE_2}
+    - ${AWS_AVAILABILITY_ZONE_3}    
 EOF
 ```
 
@@ -946,7 +954,8 @@ To create the databases using the psql pod, use these commands:
 ```bash
 kubectl run psql -it --rm=true --image=postgres:13 --command -- psql -h ${RDS_CONNECTION_URL} -U postgres postgres
 
-# The prompt will freeze as it waits for the password. Type the password and press enter.
+# The prompt will freeze as it loads the pod. Wait for the message "If you don't see a command prompt, try pressing enter".
+# Then, type the password and press enter.
 
 postgres=> CREATE DATABASE pachyderm;
 
@@ -1041,7 +1050,7 @@ After running this command, wait about 10 minutes for all the services to be pro
 
 As of MLDM version 2.8.1, a single Helm chart can be used to deploy both MLDM and MDLE.
 
-Because we're using the AWS buckets, there are 2 service accounts in the MLDM namespace that will need access to S3: the main MLDM service account and the `worker` MLDM service account, which runs the pipeline code.
+Because we're using the AWS buckets, there are 2 service accounts that will need access to S3: the main MLDM service account and the `worker` MLDM service account, which runs the pipeline code.
 
 The EKS installation command created the necessary roles with the right permissions, all we need to do is configure the service account to leverage those roles. Run these commands to set the proper ARNs for the roles:
 
@@ -1375,8 +1384,6 @@ pachctl list commit images
 
 pachctl create pipeline -f https://raw.githubusercontent.com/pachyderm/pachyderm/2.6.x/examples/opencv/edges.json
 
-
-
 wget http://imgur.com/8MN9Kg0.png
 
 pachctl put file images@master:AT-AT.png -f 8MN9Kg0.png
@@ -1394,6 +1401,8 @@ pachctl list job
 
  
 
+PS: If you used the default image size for the CPU nodes, the new pipelines may fail at first due to lack of available CPUs. In this case, the autoscaler should automatically add a new node to the CPU node group. Once the new CPUs are available, the pipeline should start automatically.
+
 At this time, you should see the OpenCV project and pipeline in the MLDM UI:
 
 
@@ -1649,7 +1658,7 @@ A more detailed explanation of these attributes:
 
  
 
-This secret needs to be created in the MLDM namespace, as it will be used by the pipelines (that will then map the variables to the MLDE experiment):
+This secret will be used by the pipelines, to map the variables for the MLDE experiments:
 
 ```bash
 kubectl apply -f pipeline-secret.yaml

examples/brain-mri/experiment/const.yaml

@@ -32,9 +32,6 @@ searcher:
     epochs: 4
 min_validation_period:
   epochs: 1
-resources:
-  resource_pool: gpu-pool
-  slots_per_trial: 1
 max_restarts: 0
 entrypoint: model_def:MRIUnetTrial
 profiling:

examples/brain-mri/pipelines/_on_prem_deployment-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python deploy.py --deployment-name brain-mri-deploy --service-account-name pach-deploy --resource-requests cpu=2,memory=4Gi --resource-limits cpu=4,memory=4Gi"
     ],
-    "image": "us-central1-docker.pkg.dev/dai-dev-554/pdk-registry/pdk_brain_mri_deploy:3.2",
+    "image": "pachyderm/pdk:brain-deploy-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/brain-mri/pipelines/_on_prem_training-pipeline.json

@@ -20,7 +20,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/brain-mri/experiment --config const.yaml --repo brain-mri-data --model brain-mri  --project pdk-brain-mri"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/brain-mri/pipelines/deployment-pipeline.json

@@ -18,7 +18,7 @@
     "stdin": [
       "python deploy.py --deployment-name brain-mri-deploy --cloud-model-host gcp --cloud-model-bucket pdk-repo-models --resource-requests cpu=2,memory=8Gi --resource-limits cpu=10,memory=8Gi"
     ],
-    "image": "us-central1-docker.pkg.dev/dai-dev-554/pdk-registry/pdk_brain_mri_deploy:3.2",
+    "image": "pachyderm/pdk:brain-deploy-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/brain-mri/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/brain-mri/experiment --config const.yaml --repo brain-mri-data --model brain-mri --project pdk-brain-mri"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/dog-cat/pipelines/_on_prem_deployment-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python deploy.py --deployment-name dogcat-deploy --service-account-name pach-deploy --resource-requests cpu=2,memory=4Gi --resource-limits cpu=4,memory=4Gi"
     ],
-    "image": "us-central1-docker.pkg.dev/dai-dev-554/pdk-registry/pdk_cats_dogs_deploy:2.1",
+    "image": "pachyderm/pdk:dog-cat-deploy-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/dog-cat/pipelines/_on_prem_training-pipeline.json

@@ -20,7 +20,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/dog-cat/experiment --config const.yaml --repo dogs-and-cats-data --model dogs-and-cats --project pdk-dogs-and-cats"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/dog-cat/pipelines/deployment-pipeline.json

@@ -18,7 +18,7 @@
     "stdin": [
       "python deploy.py --deployment-name dog-cat --cloud-model-host gcp --cloud-model-bucket pdk-repo-models --resource-requests cpu=2,memory=8Gi --resource-limits cpu=10,memory=8Gi"
     ],
-    "image": "us-central1-docker.pkg.dev/dai-dev-554/pdk-registry/pdk_cats_dogs_deploy:2.1",
+    "image": "pachyderm/pdk:dog-cat-deploy-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/dog-cat/pipelines/training-pipeline.json

@@ -19,7 +19,7 @@
     "stdin": [
       "python train.py --git-url https://[email protected]:/determined-ai/pdk.git --git-ref main --sub-dir examples/dog-cat/experiment --config const.yaml --repo dogs-and-cats-data --model dogs-and-cats --project pdk-dogs-and-cats"
     ],
-    "image": "pachyderm/pdk:train-v0.0.1",
+    "image": "pachyderm/pdk:train-v0.0.4",
     "secrets": [
       {
         "name": "pipeline-secret",

examples/object-detection/experiment/const-distributed-search.yaml

@@ -51,7 +51,6 @@ searcher:
     batches: 51520 # 50*1288 == 51520# Real Training
 records_per_epoch: 1288
 resources:
-    resource_pool: gpu-pool
     slots_per_trial: 8
     shm_size: 2000000000
 max_restarts: 0