Skip to content

Commit

Permalink
Cherry pick agents protocol refactor (#126)
Browse files Browse the repository at this point in the history
* Agents protocol refactor (#4874)

Signed-off-by: Haytham Abuelfutuh <[email protected]>
Signed-off-by: Kevin Su <[email protected]>
Signed-off-by: Kevin Su <[email protected]>
Signed-off-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Haytham Abuelfutuh <[email protected]>
Co-authored-by: Flyte Bot <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>
  • Loading branch information
5 people authored Mar 11, 2024
1 parent 036d020 commit 8535a62
Show file tree
Hide file tree
Showing 70 changed files with 9,410 additions and 1,917 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG/CHANGELOG-v1.11.0-b1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Flyte v1.11.0-b1

Second beta release for 1.11.0.
2 changes: 1 addition & 1 deletion charts/flyte-binary/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
| configuration.auth.oidc.clientId | string | `""` | |
| configuration.auth.oidc.clientSecret | string | `""` | |
| configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` | |
| configuration.co-pilot.image.tag | string | `"v1.10.7"` | |
| configuration.co-pilot.image.tag | string | `"v1.11.0-b1"` | |
| configuration.database.dbname | string | `"flyte"` | |
| configuration.database.host | string | `"127.0.0.1"` | |
| configuration.database.options | string | `"sslmode=disable"` | |
Expand Down
2 changes: 1 addition & 1 deletion charts/flyte-binary/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ configuration:
# repository CoPilot sidecar image repository
repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE
# tag CoPilot sidecar image tag
tag: v1.10.7 # FLYTECOPILOT_TAG
tag: v1.11.0-b1 # FLYTECOPILOT_TAG
# agentService Flyte Agent configuration
agentService:
defaultAgent:
Expand Down
27 changes: 19 additions & 8 deletions charts/flyte-core/README.md

Large diffs are not rendered by default.

66 changes: 60 additions & 6 deletions charts/flyte-core/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ flyteadmin:
image:
# -- Docker image for Flyteadmin deployment
repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE
tag: v1.10.7 # FLYTEADMIN_TAG
tag: v1.11.0-b1 # FLYTEADMIN_TAG
pullPolicy: IfNotPresent
# -- Additional flyteadmin container environment variables
#
Expand Down Expand Up @@ -111,6 +111,14 @@ flyteadmin:
extraArgs: {}
# -- Sets priorityClassName for flyteadmin pod(s).
priorityClassName: ""
# -- Sets securityContext for flyteadmin pod(s).
securityContext:
runAsNonRoot: true
fsGroup: 65534
runAsUser: 1001
fsGroupChangePolicy: "Always"
seLinuxOptions:
type: spc_t

# -- Settings for flyteadmin service monitor
serviceMonitor:
Expand All @@ -134,7 +142,7 @@ flytescheduler:
# -- Docker image for Flytescheduler deployment
repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE
# -- Docker image tag
tag: v1.10.7 # FLYTESCHEDULER_TAG
tag: v1.11.0-b1 # FLYTESCHEDULER_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Flytescheduler deployment
Expand Down Expand Up @@ -179,6 +187,14 @@ flytescheduler:
additionalContainers: []
# -- Sets priorityClassName for flyte scheduler pod(s).
priorityClassName: ""
# -- Sets securityContext for flytescheduler pod(s).
securityContext:
runAsNonRoot: true
fsGroup: 65534
runAsUser: 1001
fsGroupChangePolicy: "Always"
seLinuxOptions:
type: spc_t

#
# DATACATALOG SETTINGS
Expand All @@ -192,7 +208,7 @@ datacatalog:
# -- Docker image for Datacatalog deployment
repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE
# -- Docker image tag
tag: v1.10.7 # DATACATALOG_TAG
tag: v1.11.0-b1 # DATACATALOG_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Datacatalog deployment
Expand Down Expand Up @@ -242,6 +258,14 @@ datacatalog:
extraArgs: {}
# -- Sets priorityClassName for datacatalog pod(s).
priorityClassName: ""
# -- Sets securityContext for datacatalog pod(s).
securityContext:
runAsNonRoot: true
fsGroup: 1001
runAsUser: 1001
fsGroupChangePolicy: "OnRootMismatch"
seLinuxOptions:
type: spc_t

#
# FLYTE_AGENT SETTINGS
Expand All @@ -255,6 +279,8 @@ flyteagent:
defaultAgent:
endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000"
insecure: true
# -- Labels for flyteagent pods
podLabels: {}

#
# FLYTEPROPELLER SETTINGS
Expand All @@ -270,7 +296,7 @@ flytepropeller:
image:
# -- Docker image for Flytepropeller deployment
repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE
tag: v1.10.7 # FLYTEPROPELLER_TAG
tag: v1.11.0-b1 # FLYTEPROPELLER_TAG
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Flytepropeller deployment
resources:
Expand Down Expand Up @@ -320,6 +346,11 @@ flytepropeller:
clusterName: ""
# -- Sets priorityClassName for propeller pod(s).
priorityClassName: ""
# -- Sets securityContext for flytepropeller pod(s).
securityContext:
fsGroup: 65534
runAsUser: 1001
fsGroupChangePolicy: "Always"

# -- Settings for flytepropeller service
service:
Expand Down Expand Up @@ -348,7 +379,7 @@ flyteconsole:
image:
# -- Docker image for Flyteconsole deployment
repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE
tag: v1.10.2 # FLYTECONSOLE_TAG
tag: v1.11.0 # FLYTECONSOLE_TAG
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Flyteconsole deployment
resources:
Expand Down Expand Up @@ -382,6 +413,13 @@ flyteconsole:
priorityClassName: ""
# -- ImagePullSecrets to assign to the Flyteconsole deployment
imagePullSecrets: []
# -- Sets securityContext for flyteconsole pod(s).
securityContext:
runAsNonRoot: true
runAsUser: 1000
fsGroupChangePolicy: "OnRootMismatch"
seLinuxOptions:
type: spc_t

# It will enable the redoc route in ingress
deployRedoc: false
Expand Down Expand Up @@ -419,6 +457,20 @@ webhook:
annotations:
projectcontour.io/upstream-protocol.h2c: grpc
type: ClusterIP
# -- Sets securityContext for webhook pod(s).
securityContext:
fsGroup: 65534
runAsNonRoot: true
runAsUser: 1001
fsGroupChangePolicy: "Always"
seLinuxOptions:
type: spc_t

resources:
requests:
cpu: 200m
ephemeral-storage: 500Mi
memory: 500Mi

# ------------------------------------------------
#
Expand Down Expand Up @@ -679,7 +731,7 @@ configmap:
# -- Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig)
co-pilot:
name: flyte-copilot-
image: cr.flyte.org/flyteorg/flytecopilot:v1.10.7 # FLYTECOPILOT_IMAGE
image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 # FLYTECOPILOT_IMAGE
start-timeout: 30s

# -- Core propeller configuration
Expand Down Expand Up @@ -850,6 +902,8 @@ cluster_resource_manager:
podLabels: {}
# -- nodeSelector for ClusterResource deployment
nodeSelector: {}
# -- Resources for ClusterResource deployment
resources: {}
# -- Configmap for ClusterResource parameters
config:
# -- ClusterResource parameters
Expand Down
18 changes: 9 additions & 9 deletions charts/flyte/README.md

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions charts/flyte/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ flyte:
# -- Docker image for Flyteadmin deployment
repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE
# -- Docker image tag
tag: v1.10.7 # FLYTEADMIN_TAG
tag: v1.11.0-b1 # FLYTEADMIN_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Additional flyteadmin container environment variables
Expand Down Expand Up @@ -84,7 +84,7 @@ flyte:
# -- Docker image for Flytescheduler deployment
repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE
# -- Docker image tag
tag: v1.10.7 # FLYTESCHEDULER_TAG
tag: v1.11.0-b1 # FLYTESCHEDULER_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Flytescheduler deployment
Expand Down Expand Up @@ -129,7 +129,7 @@ flyte:
# -- Docker image for Datacatalog deployment
repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE
# -- Docker image tag
tag: v1.10.7 # DATACATALOG_TAG
tag: v1.11.0-b1 # DATACATALOG_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Datacatalog deployment
Expand Down Expand Up @@ -178,7 +178,7 @@ flyte:
# -- Docker image for Flytepropeller deployment
repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE
# -- Docker image tag
tag: v1.10.7 # FLYTEPROPELLER_TAG
tag: v1.11.0-b1 # FLYTEPROPELLER_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Flytepropeller deployment
Expand Down Expand Up @@ -223,7 +223,7 @@ flyte:
# -- Docker image for Flyteconsole deployment
repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE
# -- Docker image tag
tag: v1.10.2 # FLYTECONSOLE_TAG
tag: v1.11.0 # FLYTECONSOLE_TAG
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Default resources requests and limits for Flyteconsole deployment
Expand Down Expand Up @@ -471,7 +471,7 @@ flyte:
# -- Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig)
co-pilot:
name: flyte-copilot-
image: cr.flyte.org/flyteorg/flytecopilot:v1.10.7 # FLYTECOPILOT_IMAGE
image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 # FLYTECOPILOT_IMAGE
start-timeout: 30s

# -- Core propeller configuration
Expand Down
32 changes: 16 additions & 16 deletions deployment/eks/flyte_aws_scheduler_helm_generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,7 @@ data:
plugins:
k8s:
co-pilot:
image: cr.flyte.org/flyteorg/flytecopilot:v1.10.7
image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1
name: flyte-copilot-
start-timeout: 30s
core.yaml: |
Expand Down Expand Up @@ -868,7 +868,7 @@ spec:
- /etc/flyte/config/*.yaml
- migrate
- run
image: "cr.flyte.org/flyteorg/flyteadmin:v1.10.7"
image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: run-migrations
securityContext:
Expand All @@ -889,7 +889,7 @@ spec:
- flytesnacks
- flytetester
- flyteexamples
image: "cr.flyte.org/flyteorg/flyteadmin:v1.10.7"
image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: seed-projects
securityContext:
Expand All @@ -907,7 +907,7 @@ spec:
- /etc/flyte/config/*.yaml
- clusterresource
- sync
image: "cr.flyte.org/flyteorg/flyteadmin:v1.10.7"
image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: sync-cluster-resources
securityContext:
Expand All @@ -924,7 +924,7 @@ spec:
- mountPath: /etc/secrets/
name: admin-secrets
- name: generate-secrets
image: "cr.flyte.org/flyteorg/flyteadmin:v1.10.7"
image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
command: ["/bin/sh", "-c"]
args:
Expand All @@ -951,7 +951,7 @@ spec:
- --config
- /etc/flyte/config/*.yaml
- serve
image: "cr.flyte.org/flyteorg/flyteadmin:v1.10.7"
image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: flyteadmin
ports:
Expand Down Expand Up @@ -1056,7 +1056,7 @@ spec:
- /etc/flyte/config/*.yaml
- clusterresource
- run
image: "cr.flyte.org/flyteorg/flyteadmin:v1.10.7"
image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: sync-cluster-resources
volumeMounts:
Expand Down Expand Up @@ -1112,7 +1112,7 @@ spec:
seLinuxOptions:
type: spc_t
containers:
- image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.2"
- image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0"
imagePullPolicy: "IfNotPresent"
name: flyteconsole
envFrom:
Expand Down Expand Up @@ -1186,7 +1186,7 @@ spec:
- /etc/datacatalog/config/*.yaml
- migrate
- run
image: "cr.flyte.org/flyteorg/datacatalog:v1.10.7"
image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: run-migrations
volumeMounts:
Expand All @@ -1204,7 +1204,7 @@ spec:
- --config
- /etc/datacatalog/config/*.yaml
- serve
image: "cr.flyte.org/flyteorg/datacatalog:v1.10.7"
image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: datacatalog
ports:
Expand Down Expand Up @@ -1267,7 +1267,7 @@ spec:
template:
metadata:
annotations:
configChecksum: "305d6f36301e10e952435f597dbe6700381a43c496a3be2cca60c175439fc9a"
configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212"
labels:
app.kubernetes.io/name: flytepropeller
app.kubernetes.io/instance: flyte
Expand All @@ -1293,7 +1293,7 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: "cr.flyte.org/flyteorg/flytepropeller:v1.10.7"
image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
name: flytepropeller
ports:
Expand Down Expand Up @@ -1347,9 +1347,9 @@ spec:
labels:
app: flyte-pod-webhook
app.kubernetes.io/name: flyte-pod-webhook
app.kubernetes.io/version: v1.10.7
app.kubernetes.io/version: v1.11.0-b1
annotations:
configChecksum: "305d6f36301e10e952435f597dbe6700381a43c496a3be2cca60c175439fc9a"
configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212"
spec:
securityContext:
fsGroup: 65534
Expand All @@ -1361,7 +1361,7 @@ spec:
serviceAccountName: flyte-pod-webhook
initContainers:
- name: generate-secrets
image: "cr.flyte.org/flyteorg/flytepropeller:v1.10.7"
image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
command:
- flytepropeller
Expand All @@ -1388,7 +1388,7 @@ spec:
mountPath: /etc/flyte/config
containers:
- name: webhook
image: "cr.flyte.org/flyteorg/flytepropeller:v1.10.7"
image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1"
imagePullPolicy: "IfNotPresent"
command:
- flytepropeller
Expand Down
Loading

0 comments on commit 8535a62

Please sign in to comment.