(PXP-6861): configure Metadata Service creds in kube-setup-ssjdispatcher

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$|^./.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2020-10-09T17:32:47Z",
+  "generated_at": "2020-10-13T16:27:08Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -436,15 +436,21 @@
     ],
     "gen3/bin/kube-setup-ssjdispatcher.sh": [
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
         "is_verified": false,
-        "line_number": 115,
+        "line_number": 117,
         "type": "Secret Keyword"
       },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93",
         "is_verified": false,
-        "line_number": 117,
+        "line_number": 184,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_verified": false,
+        "line_number": 197,
         "type": "Secret Keyword"
       }
     ],
@@ -560,13 +566,13 @@
       {
         "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
         "is_verified": false,
-        "line_number": 155,
+        "line_number": 154,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c",
         "is_verified": false,
-        "line_number": 155,
+        "line_number": 154,
         "type": "Secret Keyword"
       }
     ],

doc/kube-setup-ssjdispatcher.md

@@ -12,6 +12,7 @@ in `creds.json`.
 * create an upload bucket
 * create sns and sqs
 * setup indexd creds
+* setup metadata service creds if they are present
 
 If `auto` is provided as the bucket name, then the script constructs a
 safe name for the bucket.  The caller must configure the `DATA_UPLOAD_BUCKET` in `fence-config-public`.

gen3/bin/kube-setup-ssjdispatcher.sh

@@ -151,6 +151,85 @@ EOM
   fi
 }
 
+setupMDSConfig() {
+  local ssjCredsFile
+  ssjCredsFile="$(gen3_secrets_folder)/creds.json"
+  # don't log nonexistence of $ssjCredsFile since that would have already been logged in setupSsjInfra function
+  [[ -f "$ssjCredsFile" ]] || return 0
+  if ! g3k_manifest_lookup .versions.metadata > /dev/null 2>&1; then
+    gen3_log_info "skipping verifying or syncing metadata service creds because metadata service not in manifest"
+    return 0
+  fi
+
+  mdsCredsFile="$(gen3_secrets_folder)/g3auto/metadata/metadata.env"
+  if [[ ! -f "$mdsCredsFile" ]]; then
+    gen3_log_warn "skipping verifying or syncing metadata service creds because metadata service creds file could not be found"
+    return 0
+  fi
+
+  local jobImageConfig
+  if ! jobImageConfig="$(jq -r -e '.ssjdispatcher.JOBS[] | select(.name == "indexing").imageConfig' < "$ssjCredsFile" 2> /dev/null)"; then
+    gen3_log_warn "skipping verifying or syncing metadata service creds because an \"indexing\" job image configuration could not be found in $ssjCredsFile"
+    return 0
+  fi
+
+  local mdsCreds
+  local mdsUsername
+  local mdsPassword
+  # [[ $? == 1 ]] added here so that if `set -e -o pipefail` were used in the
+  # future and grep can't find 'ADMIN_LOGINS=', kube-setup-ssjdispatcher won't
+  # exit with an error code, but will instead log a warning and exit with 0
+  mdsCreds="$( (grep 'ADMIN_LOGINS=' "$mdsCredsFile" 2> /dev/null || [[ $? == 1 ]]) | cut -s -d '=' -f 2- 2> /dev/null )"
+  mdsUsername="$(cut -s -d ':' -f 1 <<< "$mdsCreds" 2> /dev/null)"
+  mdsPassword="$(cut -s -d ':' -f 2- <<< "$mdsCreds" 2> /dev/null)"
+
+  if [[ -z "$mdsCreds" || -z "$mdsUsername" || -z "$mdsPassword" ]]; then
+    gen3_log_warn "could not parse metadata service basic auth creds from $mdsCredsFile"
+    return 0
+  fi
+
+  local ssjMdsCreds
+  # check that metadata service creds match those configured for ssjdispatcher
+  if ssjMdsCreds="$(jq -r -e '.metadataService' <<< "$jobImageConfig" 2> /dev/null)"; then
+    local ssjMdsUsername
+    local ssjMdsPassword
+    ssjMdsUsername="$(jq -r -e '.username' <<< "$ssjMdsCreds" 2> /dev/null)"
+    ssjMdsPassword="$(jq -r -e '.password' <<< "$ssjMdsCreds" 2> /dev/null)"
+    if [[ "$ssjMdsUsername" != "$mdsUsername" || "$ssjMdsPassword" != "$mdsPassword" ]]; then
+      if [[ -n "$JENKINS_HOME" ]]; then
+        gen3_log_err "metadata service creds already configured for ssjdispatcher are not up-to-date with the metadata service: $ssjCredsFile"
+        return 1
+      fi
+      gen3_log_warn "metadata service creds already configured for ssjdispatcher were not up-to-date with the metadata service before running kube-setup-ssjdispatcher: $ssjCredsFile"
+    else
+      gen3_log_info "metadata service creds configured for ssjdispatcher were verified to already be up-to-date with the metadata service"
+      return 0
+    fi
+  fi
+
+  if [[ -n "$JENKINS_HOME" ]]; then
+    gen3_log_info "running in jenkins, skipping setting up metadata service creds"
+    return 0
+  fi
+
+  gen3_log_info "setting up metadata service creds"
+  local mdsConfig
+  mdsConfig="$(cat - <<EOM
+{
+  "url": "http://revproxy-service/mds",
+  "username": "${mdsUsername}",
+  "password": "${mdsPassword}"
+}
+EOM
+  )"
+  local credsBak
+  credsBak="$(mktemp "$XDG_RUNTIME_DIR/creds.json_XXXXXX")"
+  cp "$ssjCredsFile" "$credsBak"
+  jq -r -e --argjson mdsConfig "$mdsConfig" '(.ssjdispatcher.JOBS[] | select(.name == "indexing") | .imageConfig.metadataService)=$mdsConfig' < "$credsBak" > "$ssjCredsFile"
+  /bin/rm "$credsBak"
+
+  gen3 secrets sync "chore(ssjdispatcher): set up metadata service creds"
+}
 
 # main -------------------
 
@@ -162,6 +241,7 @@ fi
 [[ -z "$GEN3_ROLL_ALL" ]] && gen3 kube-setup-secrets
 
 setupSsjInfra "$@"
+setupMDSConfig
 gen3 roll ssjdispatcher
 g3kubectl apply -f "${GEN3_HOME}/kube/services/ssjdispatcher/ssjdispatcher-service.yaml"