ci: Update loki-release workflows to build Docker images "natively"

.github/jsonnetfile.json

@@ -8,7 +8,7 @@
           "subdir": "workflows"
         }
       },
-      "version": "cfa24256090828f566f1ba59292ce65d8db4a4ae"
+      "version": "5343bc71d96dc4247021a66c3da8fd5cd4c957dd"
     }
   ],
   "legacyImports": true

.github/jsonnetfile.lock.json

@@ -8,8 +8,8 @@
           "subdir": "workflows"
         }
       },
-      "version": "cfa24256090828f566f1ba59292ce65d8db4a4ae",
-      "sum": "tml1dcFlo15kEE6JvN/nPY2xkhfeF3ERZjAyFbnguHA="
+      "version": "5343bc71d96dc4247021a66c3da8fd5cd4c957dd",
+      "sum": "/+ozeV2rndtz8N3cZmrWxbNJFI7fkwoDzhECMHG1RoA="
     }
   ],
   "legacyImports": false

.github/release-workflows.jsonnet

@@ -1,33 +1,43 @@
-local lokiRelease = import 'workflows/main.jsonnet';
-
-local build = lokiRelease.build;
+local lokiRelease = import 'workflows/main.jsonnet',
+      job = lokiRelease.job,
+      step = lokiRelease.step,
+      build = lokiRelease.build;
 local releaseLibRef = 'main';
 local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef;
 local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION');
+local goVersion = std.extVar('GO_VERSION');
 local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion;
 local golangCiLintVersion = 'v1.60.3';
 local imageBuildTimeoutMin = 60;
 local imagePrefix = 'grafana';
 local dockerPluginDir = 'clients/cmd/docker-driver';
+local runner = import 'workflows/runner.libsonnet',
+      r = runner.withDefaultMapping();  // Do we need a different mapping?
+
+local platforms = {
+  amd: [r.forPlatform('linux/amd64')],
+  arm: [r.forPlatform('linux/arm64'), r.forPlatform('linux/arm')],
+  all: self.amd + self.arm,
+};
 
 local imageJobs = {
-  loki: build.image('loki', 'cmd/loki'),
-  fluentd: build.image('fluent-plugin-loki', 'clients/cmd/fluentd', platform=['linux/amd64']),
-  'fluent-bit': build.image('fluent-bit-plugin-loki', 'clients/cmd/fluent-bit', platform=['linux/amd64']),
-  logstash: build.image('logstash-output-loki', 'clients/cmd/logstash', platform=['linux/amd64']),
-  logcli: build.image('logcli', 'cmd/logcli'),
-  'loki-canary': build.image('loki-canary', 'cmd/loki-canary'),
-  'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'),
-  promtail: build.image('promtail', 'clients/cmd/promtail'),
-  querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']),
-  'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']),
+  loki: build.image('loki', 'cmd/loki', platform=platforms.all),
+  fluentd: build.image('fluent-plugin-loki', 'clients/cmd/fluentd', platform=platforms.amd),
+  'fluent-bit': build.image('fluent-bit-plugin-loki', 'clients/cmd/fluent-bit', platform=platforms.amd),
+  logstash: build.image('logstash-output-loki', 'clients/cmd/logstash', platform=platforms.amd),
+  logcli: build.image('logcli', 'cmd/logcli', platform=platforms.all),
+  'loki-canary': build.image('loki-canary', 'cmd/loki-canary', platform=platforms.all),
+  'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto', platform=platforms.all),
+  promtail: build.image('promtail', 'clients/cmd/promtail', platform=platforms.all),
+  querytee: build.image('loki-query-tee', 'cmd/querytee', platform=platforms.amd),
+  'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=platforms.all),
 };
 
 local weeklyImageJobs = {
-  loki: build.weeklyImage('loki', 'cmd/loki'),
-  'loki-canary': build.weeklyImage('loki-canary', 'cmd/loki-canary'),
-  'loki-canary-boringcrypto': build.weeklyImage('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'),
-  promtail: build.weeklyImage('promtail', 'clients/cmd/promtail'),
+  loki: build.weeklyImage('loki', 'cmd/loki', platform=platforms.all),
+  'loki-canary': build.weeklyImage('loki-canary', 'cmd/loki-canary', platform=platforms.all),
+  'loki-canary-boringcrypto': build.weeklyImage('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto', platform=platforms.all),
+  promtail: build.weeklyImage('promtail', 'clients/cmd/promtail', platform=platforms.all),
 };
 
 {
@@ -102,14 +112,15 @@ local weeklyImageJobs = {
     },
   }),
   'images.yml': std.manifestYamlDoc({
-    name: 'publish images',
+    name: 'Publish images',
     on: {
       push: {
         branches: [
           'k[0-9]+*',  // This is a weird glob pattern, not a regexp, do not use ".*", see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
           'main',
         ],
       },
+      workflow_dispatch: {},
     },
     permissions: {
       'id-token': 'write',
@@ -127,16 +138,44 @@ local weeklyImageJobs = {
           use_github_app_token: true,
         },
       },
-    } + std.mapWithKey(function(name, job)
-      job
-      + lokiRelease.job.withNeeds(['check'])
-      + {
-        env: {
+    } + {
+      ['%s-image' % name]:
+        weeklyImageJobs[name]
+        + job.withNeeds(['check'])
+        + job.withEnv({
           BUILD_TIMEOUT: imageBuildTimeoutMin,
           RELEASE_REPO: 'grafana/loki',
           RELEASE_LIB_REF: releaseLibRef,
           IMAGE_PREFIX: imagePrefix,
-        },
-      }, weeklyImageJobs),
+          GO_VERSION: goVersion,
+        })
+      for name in std.objectFields(weeklyImageJobs)
+    } + {
+      ['%s-manifest' % name]:
+        job.new()
+        + job.withNeeds(['%s-image' % name])
+        + job.withEnv({
+          BUILD_TIMEOUT: imageBuildTimeoutMin,
+        })
+        + job.withSteps([
+          step.new('Set up Docker buildx', 'docker/setup-buildx-action@v3'),
+          step.new('Login to DockerHub (from Vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'),
+          step.new('Publish multi-arch manifest')
+          + step.withRun(|||
+            # Unfortunately there is no better way atm than having a separate named output for each digest
+            echo 'linux/arm64 ${{ needs.%(name)s.outputs.image_digest_linux_amd64 }}'
+            echo 'linux/amd64 ${{ needs.%(name)s.outputs.image_digest_linux_arm64 }}'
+            echo 'linux/arm   ${{ needs.%(name)s.outputs.image_digest_linux_arm }}'
+            IMAGE=${{ needs.%(name)s.outputs.image_name }}:${{ needs.%(name)s.outputs.image_tag }}
+            echo "Create multi-arch manifest for $IMAGE"
+            docker buildx imagetools create -t $IMAGE \
+              ${{ needs.%(name)s.outputs.image_name }}@${{ needs.%(name)s.outputs.image_digest_linux_amd64 }} \
+              ${{ needs.%(name)s.outputs.image_name }}@${{ needs.%(name)s.outputs.image_digest_linux_arm64 }} \
+              ${{ needs.%(name)s.outputs.image_name }}@${{ needs.%(name)s.outputs.image_digest_linux_arm }}
+            docker buildx imagetools inspect $IMAGE
+          ||| % { name: '%s-image' % name }),
+        ])
+      for name in std.objectFields(weeklyImageJobs)
+    },
   }),
 }