From 1d97cef66dc76a8faa617d6bab6ea034d1821166 Mon Sep 17 00:00:00 2001 From: nate nowack Date: Mon, 24 Feb 2025 16:28:52 -0600 Subject: [PATCH 1/4] Add `prefect-kubernetes` `v0.5.5` to block records --- .../prefect-kubernetes/blocks/v0.5.5.json | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 collections/prefect-kubernetes/blocks/v0.5.5.json diff --git a/collections/prefect-kubernetes/blocks/v0.5.5.json b/collections/prefect-kubernetes/blocks/v0.5.5.json new file mode 100644 index 00000000..bdfd9942 --- /dev/null +++ b/collections/prefect-kubernetes/blocks/v0.5.5.json @@ -0,0 +1,71 @@ +{ + "prefect-kubernetes": { + "block_types": { + "kubernetes-credentials": { + "name": "Kubernetes Credentials", + "slug": "kubernetes-credentials", + "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/2d0b896006ad463b49c28aaac14f31e00e32cfab-250x250.png", + "documentation_url": "https://prefecthq.github.io/prefect-kubernetes/credentials/#prefect_kubernetes.credentials.KubernetesCredentials", + "description": "Credentials block for generating configured Kubernetes API clients. This block is part of the prefect-kubernetes collection. Install prefect-kubernetes with `pip install prefect-kubernetes` to use this block.", + "code_example": "Load stored Kubernetes credentials:\n```python\nfrom prefect_kubernetes.credentials import KubernetesCredentials\n\nkubernetes_credentials = KubernetesCredentials.load(\"BLOCK_NAME\")\n```", + "block_schema": { + "checksum": "sha256:205323083b1cc3076ead5dc1bf6f6cd5f94d6644e54c05bcd0c9789c20915256", + "fields": { + "block_schema_references": { + "cluster_config": { + "block_schema_checksum": "sha256:90d421e948bfbe4cdc98b124995f0edd0f84b0837549ad1390423bad8e31cf3b", + "block_type_slug": "kubernetes-cluster-config" + } + }, + "block_type_slug": "kubernetes-credentials", + "description": "Credentials block for generating configured Kubernetes API clients.", + "properties": { + "cluster_config": { + "anyOf": [ + { + "$ref": "#/definitions/KubernetesClusterConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "secret_fields": [], + "title": "KubernetesCredentials", + "type": "object", + "definitions": { + "KubernetesClusterConfig": { + "block_schema_references": {}, + "block_type_slug": "kubernetes-cluster-config", + "description": "Stores configuration for interaction with Kubernetes clusters.\n\nSee `from_file` for creation.", + "properties": { + "config": { + "description": "The entire contents of a kubectl config file.", + "title": "Config", + "type": "object" + }, + "context_name": { + "description": "The name of the kubectl context to use.", + "title": "Context Name", + "type": "string" + } + }, + "required": [ + "config", + "context_name" + ], + "secret_fields": [], + "title": "KubernetesClusterConfig", + "type": "object" + } + } + }, + "capabilities": [], + "version": "0.5.3" + } + } + } + } +} \ No newline at end of file From 30696dd6037936224256a065d6c6c89562025629 Mon Sep 17 00:00:00 2001 From: nate nowack Date: Mon, 24 Feb 2025 16:28:53 -0600 Subject: [PATCH 2/4] Update aggregate block metadata with `prefect-kubernetes` `v0.5.5` --- views/aggregate-block-metadata.json | 42 +++++++++++++++++------------ 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/views/aggregate-block-metadata.json b/views/aggregate-block-metadata.json index 10617750..7d901bab 100644 --- a/views/aggregate-block-metadata.json +++ b/views/aggregate-block-metadata.json @@ -6649,38 +6649,46 @@ "description": "Credentials block for generating configured Kubernetes API clients. This block is part of the prefect-kubernetes collection. Install prefect-kubernetes with `pip install prefect-kubernetes` to use this block.", "code_example": "Load stored Kubernetes credentials:\n```python\nfrom prefect_kubernetes.credentials import KubernetesCredentials\n\nkubernetes_credentials = KubernetesCredentials.load(\"BLOCK_NAME\")\n```", "block_schema": { - "checksum": "sha256:957fa8dca90bd1b5fb9c575ee09e80b454116c0b134287fbc2eff47a72564c3b", + "checksum": "sha256:205323083b1cc3076ead5dc1bf6f6cd5f94d6644e54c05bcd0c9789c20915256", "fields": { - "title": "KubernetesCredentials", - "description": "Credentials block for generating configured Kubernetes API clients.", - "type": "object", - "properties": { + "block_schema_references": { "cluster_config": { - "$ref": "#/definitions/KubernetesClusterConfig" + "block_schema_checksum": "sha256:90d421e948bfbe4cdc98b124995f0edd0f84b0837549ad1390423bad8e31cf3b", + "block_type_slug": "kubernetes-cluster-config" } }, "block_type_slug": "kubernetes-credentials", - "secret_fields": [], - "block_schema_references": { + "description": "Credentials block for generating configured Kubernetes API clients.", + "properties": { "cluster_config": { - "block_type_slug": "kubernetes-cluster-config", - "block_schema_checksum": "sha256:90d421e948bfbe4cdc98b124995f0edd0f84b0837549ad1390423bad8e31cf3b" + "anyOf": [ + { + "$ref": "#/definitions/KubernetesClusterConfig" + }, + { + "type": "null" + } + ], + "default": null } }, + "secret_fields": [], + "title": "KubernetesCredentials", + "type": "object", "definitions": { "KubernetesClusterConfig": { - "title": "KubernetesClusterConfig", + "block_schema_references": {}, + "block_type_slug": "kubernetes-cluster-config", "description": "Stores configuration for interaction with Kubernetes clusters.\n\nSee `from_file` for creation.", - "type": "object", "properties": { "config": { - "title": "Config", "description": "The entire contents of a kubectl config file.", + "title": "Config", "type": "object" }, "context_name": { - "title": "Context Name", "description": "The name of the kubectl context to use.", + "title": "Context Name", "type": "string" } }, @@ -6688,14 +6696,14 @@ "config", "context_name" ], - "block_type_slug": "kubernetes-cluster-config", "secret_fields": [], - "block_schema_references": {} + "title": "KubernetesClusterConfig", + "type": "object" } } }, "capabilities": [], - "version": "0.3.7" + "version": "0.5.3" } } } From 4505a6df0d18fe8da26b16f314db0dd142e7e127 Mon Sep 17 00:00:00 2001 From: nate nowack Date: Mon, 24 Feb 2025 16:28:55 -0600 Subject: [PATCH 3/4] Add `prefect-kubernetes` `v0.5.5` to worker records --- .../prefect-kubernetes/workers/v0.5.5.json | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 collections/prefect-kubernetes/workers/v0.5.5.json diff --git a/collections/prefect-kubernetes/workers/v0.5.5.json b/collections/prefect-kubernetes/workers/v0.5.5.json new file mode 100644 index 00000000..16a0ce98 --- /dev/null +++ b/collections/prefect-kubernetes/workers/v0.5.5.json @@ -0,0 +1,233 @@ +{ + "prefect-kubernetes": { + "kubernetes": { + "default_base_job_configuration": { + "job_configuration": { + "command": "{{ command }}", + "env": "{{ env }}", + "labels": "{{ labels }}", + "name": "{{ name }}", + "namespace": "{{ namespace }}", + "job_manifest": { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "generateName": "{{ name }}-", + "labels": "{{ labels }}", + "namespace": "{{ namespace }}" + }, + "spec": { + "backoffLimit": 0, + "template": { + "spec": { + "completions": 1, + "containers": [ + { + "args": "{{ command }}", + "env": "{{ env }}", + "image": "{{ image }}", + "imagePullPolicy": "{{ image_pull_policy }}", + "name": "prefect-job" + } + ], + "parallelism": 1, + "restartPolicy": "Never", + "serviceAccountName": "{{ service_account_name }}" + } + }, + "ttlSecondsAfterFinished": "{{ finished_job_ttl }}" + } + }, + "cluster_config": "{{ cluster_config }}", + "job_watch_timeout_seconds": "{{ job_watch_timeout_seconds }}", + "pod_watch_timeout_seconds": "{{ pod_watch_timeout_seconds }}", + "stream_output": "{{ stream_output }}" + }, + "variables": { + "description": "Default variables for the Kubernetes worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.", + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name given to infrastructure created by a worker.", + "title": "Name" + }, + "env": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "description": "Environment variables to set when starting a flow run.", + "title": "Environment Variables", + "type": "object" + }, + "labels": { + "additionalProperties": { + "type": "string" + }, + "description": "Labels applied to infrastructure created by a worker.", + "title": "Labels", + "type": "object" + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", + "title": "Command" + }, + "namespace": { + "default": "default", + "description": "The Kubernetes namespace to create jobs within.", + "title": "Namespace", + "type": "string" + }, + "image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.", + "examples": [ + "docker.io/prefecthq/prefect:3-latest" + ], + "title": "Image" + }, + "service_account_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The Kubernetes service account to use for job creation.", + "title": "Service Account Name" + }, + "image_pull_policy": { + "default": "IfNotPresent", + "description": "The Kubernetes image pull policy to use for job containers.", + "enum": [ + "IfNotPresent", + "Always", + "Never" + ], + "title": "Image Pull Policy", + "type": "string" + }, + "finished_job_ttl": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The number of seconds to retain jobs after completion. If set, finished jobs will be cleaned up by Kubernetes after the given delay. If not set, jobs will be retained indefinitely.", + "title": "Finished Job TTL" + }, + "job_watch_timeout_seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number of seconds to wait for each event emitted by a job before timing out. If not set, the worker will wait for each event indefinitely.", + "title": "Job Watch Timeout Seconds" + }, + "pod_watch_timeout_seconds": { + "default": 60, + "description": "Number of seconds to watch for pod creation before timing out.", + "title": "Pod Watch Timeout Seconds", + "type": "integer" + }, + "stream_output": { + "default": true, + "description": "If set, output will be streamed from the job to local standard output.", + "title": "Stream Output", + "type": "boolean" + }, + "cluster_config": { + "anyOf": [ + { + "$ref": "#/definitions/KubernetesClusterConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The Kubernetes cluster config to use for job creation." + } + }, + "type": "object", + "definitions": { + "KubernetesClusterConfig": { + "block_schema_references": {}, + "block_type_slug": "kubernetes-cluster-config", + "description": "Stores configuration for interaction with Kubernetes clusters.\n\nSee `from_file` for creation.", + "properties": { + "config": { + "description": "The entire contents of a kubectl config file.", + "title": "Config", + "type": "object" + }, + "context_name": { + "description": "The name of the kubectl context to use.", + "title": "Context Name", + "type": "string" + } + }, + "required": [ + "config", + "context_name" + ], + "secret_fields": [], + "title": "KubernetesClusterConfig", + "type": "object" + } + } + } + }, + "description": "Execute flow runs within jobs scheduled on a Kubernetes cluster. Requires a Kubernetes cluster.", + "display_name": "Kubernetes", + "documentation_url": "https://prefecthq.github.io/prefect-kubernetes/worker/", + "install_command": "pip install prefect-kubernetes", + "is_beta": false, + "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/2d0b896006ad463b49c28aaac14f31e00e32cfab-250x250.png", + "type": "kubernetes" + } + } +} \ No newline at end of file From fb016d2af4aeae9787a4c8e781ec5e2ee52333ed Mon Sep 17 00:00:00 2001 From: nate nowack Date: Mon, 24 Feb 2025 16:28:56 -0600 Subject: [PATCH 4/4] Update aggregate worker metadata with `prefect-kubernetes` `v0.5.5` --- views/aggregate-worker-metadata.json | 162 ++++++++++++++++++--------- 1 file changed, 111 insertions(+), 51 deletions(-) diff --git a/views/aggregate-worker-metadata.json b/views/aggregate-worker-metadata.json index ac12dad6..8afa1c64 100644 --- a/views/aggregate-worker-metadata.json +++ b/views/aggregate-worker-metadata.json @@ -1561,30 +1561,30 @@ "apiVersion": "batch/v1", "kind": "Job", "metadata": { + "generateName": "{{ name }}-", "labels": "{{ labels }}", - "namespace": "{{ namespace }}", - "generateName": "{{ name }}-" + "namespace": "{{ namespace }}" }, "spec": { "backoffLimit": 0, - "ttlSecondsAfterFinished": "{{ finished_job_ttl }}", "template": { "spec": { - "parallelism": 1, "completions": 1, - "restartPolicy": "Never", - "serviceAccountName": "{{ service_account_name }}", "containers": [ { - "name": "prefect-job", + "args": "{{ command }}", "env": "{{ env }}", "image": "{{ image }}", "imagePullPolicy": "{{ image_pull_policy }}", - "args": "{{ command }}" + "name": "prefect-job" } - ] + ], + "parallelism": 1, + "restartPolicy": "Never", + "serviceAccountName": "{{ service_account_name }}" } - } + }, + "ttlSecondsAfterFinished": "{{ finished_job_ttl }}" } }, "cluster_config": "{{ cluster_config }}", @@ -1594,108 +1594,168 @@ }, "variables": { "description": "Default variables for the Kubernetes worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.", - "type": "object", "properties": { "name": { - "title": "Name", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "Name given to infrastructure created by a worker.", - "type": "string" + "title": "Name" }, "env": { - "title": "Environment Variables", - "description": "Environment variables to set when starting a flow run.", - "type": "object", "additionalProperties": { - "type": "string" - } + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "description": "Environment variables to set when starting a flow run.", + "title": "Environment Variables", + "type": "object" }, "labels": { - "title": "Labels", - "description": "Labels applied to infrastructure created by a worker.", - "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Labels applied to infrastructure created by a worker.", + "title": "Labels", + "type": "object" }, "command": { - "title": "Command", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", - "type": "string" + "title": "Command" }, "namespace": { - "title": "Namespace", - "description": "The Kubernetes namespace to create jobs within.", "default": "default", + "description": "The Kubernetes namespace to create jobs within.", + "title": "Namespace", "type": "string" }, "image": { - "title": "Image", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.", - "example": "docker.io/prefecthq/prefect:2-latest", - "type": "string" + "examples": [ + "docker.io/prefecthq/prefect:3-latest" + ], + "title": "Image" }, "service_account_name": { - "title": "Service Account Name", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The Kubernetes service account to use for job creation.", - "type": "string" + "title": "Service Account Name" }, "image_pull_policy": { - "title": "Image Pull Policy", - "description": "The Kubernetes image pull policy to use for job containers.", "default": "IfNotPresent", + "description": "The Kubernetes image pull policy to use for job containers.", "enum": [ "IfNotPresent", "Always", "Never" ], + "title": "Image Pull Policy", "type": "string" }, "finished_job_ttl": { - "title": "Finished Job TTL", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, "description": "The number of seconds to retain jobs after completion. If set, finished jobs will be cleaned up by Kubernetes after the given delay. If not set, jobs will be retained indefinitely.", - "type": "integer" + "title": "Finished Job TTL" }, "job_watch_timeout_seconds": { - "title": "Job Watch Timeout Seconds", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, "description": "Number of seconds to wait for each event emitted by a job before timing out. If not set, the worker will wait for each event indefinitely.", - "type": "integer" + "title": "Job Watch Timeout Seconds" }, "pod_watch_timeout_seconds": { - "title": "Pod Watch Timeout Seconds", - "description": "Number of seconds to watch for pod creation before timing out.", "default": 60, + "description": "Number of seconds to watch for pod creation before timing out.", + "title": "Pod Watch Timeout Seconds", "type": "integer" }, "stream_output": { - "title": "Stream Output", - "description": "If set, output will be streamed from the job to local standard output.", "default": true, + "description": "If set, output will be streamed from the job to local standard output.", + "title": "Stream Output", "type": "boolean" }, "cluster_config": { - "title": "Cluster Config", - "description": "The Kubernetes cluster config to use for job creation.", - "allOf": [ + "anyOf": [ { "$ref": "#/definitions/KubernetesClusterConfig" + }, + { + "type": "null" } - ] + ], + "default": null, + "description": "The Kubernetes cluster config to use for job creation." } }, + "type": "object", "definitions": { "KubernetesClusterConfig": { - "title": "KubernetesClusterConfig", + "block_schema_references": {}, + "block_type_slug": "kubernetes-cluster-config", "description": "Stores configuration for interaction with Kubernetes clusters.\n\nSee `from_file` for creation.", - "type": "object", "properties": { "config": { - "title": "Config", "description": "The entire contents of a kubectl config file.", + "title": "Config", "type": "object" }, "context_name": { - "title": "Context Name", "description": "The name of the kubectl context to use.", + "title": "Context Name", "type": "string" } }, @@ -1703,9 +1763,9 @@ "config", "context_name" ], - "block_type_slug": "kubernetes-cluster-config", "secret_fields": [], - "block_schema_references": {} + "title": "KubernetesClusterConfig", + "type": "object" } } }