Skip to content

Commit 79fa9f9

Browse files
cheina97cheina97
committed
feat: fabric ports setup
1 parent ceca09c commit 79fa9f9

12 files changed

+119
-7
lines changed

‎.github/workflows/integration.yml

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
- master
88
- release
99
- ipam
10+
- frc/fabricport
1011
repository_dispatch:
1112
types:
1213
- test-command

‎cmd/fabric/main.go

+9
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"sigs.k8s.io/controller-runtime/pkg/cache"
3232
"sigs.k8s.io/controller-runtime/pkg/client"
3333
"sigs.k8s.io/controller-runtime/pkg/client/config"
34+
"sigs.k8s.io/controller-runtime/pkg/healthz"
3435
"sigs.k8s.io/controller-runtime/pkg/log"
3536
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
3637

@@ -142,6 +143,14 @@ func run(cmd *cobra.Command, _ []string) error {
142143
return fmt.Errorf("unable to create manager: %w", err)
143144
}
144145

146+
// Register the healthiness probes.
147+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
148+
return fmt.Errorf("unable to set up healthz probe: %w", err)
149+
}
150+
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
151+
return fmt.Errorf("unable to set up readyz probe: %w", err)
152+
}
153+
145154
gwr, err := sourcedetector.NewGatewayReconciler(
146155
mgr.GetClient(),
147156
mgr.GetScheme(),

‎cmd/gateway/geneve/main.go

+9
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"k8s.io/klog/v2"
2626
ctrl "sigs.k8s.io/controller-runtime"
2727
"sigs.k8s.io/controller-runtime/pkg/client/config"
28+
"sigs.k8s.io/controller-runtime/pkg/healthz"
2829
"sigs.k8s.io/controller-runtime/pkg/log"
2930
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
3031

@@ -95,6 +96,14 @@ func run(cmd *cobra.Command, _ []string) error {
9596
return fmt.Errorf("unable to create manager: %w", err)
9697
}
9798

99+
// Register the healthiness probes.
100+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
101+
return fmt.Errorf("unable to set up healthz probe: %w", err)
102+
}
103+
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
104+
return fmt.Errorf("unable to set up readyz probe: %w", err)
105+
}
106+
98107
inr, err := geneve.NewInternalNodeReconciler(
99108
mgr.GetClient(),
100109
mgr.GetScheme(),

‎cmd/gateway/main.go

+9
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
ctrl "sigs.k8s.io/controller-runtime"
3030
"sigs.k8s.io/controller-runtime/pkg/client"
3131
"sigs.k8s.io/controller-runtime/pkg/client/config"
32+
"sigs.k8s.io/controller-runtime/pkg/healthz"
3233
"sigs.k8s.io/controller-runtime/pkg/log"
3334
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
3435

@@ -160,6 +161,14 @@ func run(cmd *cobra.Command, _ []string) error {
160161
return fmt.Errorf("unable to create manager: %w", err)
161162
}
162163

164+
// Register the healthiness probes.
165+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
166+
return fmt.Errorf("unable to set up healthz probe: %w", err)
167+
}
168+
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
169+
return fmt.Errorf("unable to set up readyz probe: %w", err)
170+
}
171+
163172
if connoptions.EnableConnectionController {
164173
// Setup the connection controller.
165174
connr, err := connection.NewConnectionsReconciler(

‎cmd/gateway/wireguard/main.go

+9
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"sigs.k8s.io/controller-runtime/pkg/cache"
3030
"sigs.k8s.io/controller-runtime/pkg/client/config"
3131
"sigs.k8s.io/controller-runtime/pkg/event"
32+
"sigs.k8s.io/controller-runtime/pkg/healthz"
3233
"sigs.k8s.io/controller-runtime/pkg/log"
3334
"sigs.k8s.io/controller-runtime/pkg/metrics"
3435
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -107,6 +108,14 @@ func run(cmd *cobra.Command, _ []string) error {
107108
return fmt.Errorf("unable to create manager: %w", err)
108109
}
109110

111+
// Register the healthiness probes.
112+
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
113+
return fmt.Errorf("unable to set up healthz probe: %w", err)
114+
}
115+
if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
116+
return fmt.Errorf("unable to set up readyz probe: %w", err)
117+
}
118+
110119
// Setup the controller.
111120
pkr, err := wireguard.NewPublicKeysReconciler(
112121
mgr.GetClient(),

‎deployments/liqo/README.md

+2
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@
8686
| networking.enabled | bool | `true` | Use the default Liqo networking module. |
8787
| networking.fabric.config.fullMasquerade | bool | `false` | Enabe/Disable the full masquerade mode for the fabric pod. It means that all traffic will be masquerade using the first external cidr IP, instead of using the pod IP. Full masquerade is useful when the cluster nodeports uses a PodCIDR IP to masqerade the incoming traffic. IMPORTANT: Please consider that enabling this feature will masquerade the source IP of traffic towards a remote cluster, making impossible for a pod that receives the traffic to know the original source IP. |
8888
| networking.fabric.config.gatewayMasqueradeBypass | bool | `false` | Enable/Disable the masquerade bypass for the gateway pods. It means that the packets from gateway pods will not be masqueraded from the host where the pod is scheduled. This is useful in scenarios where CNIs masquerade the traffic from pod to nodes. For example this is required when using the Azure CNI or Kindnet. |
89+
| networking.fabric.config.healthProbeBindAddress | string | `":8081"` | Set the address where the fabric pod will expose the health probe. To disable the health probe, set the address to ":0". |
90+
| networking.fabric.config.metricsAddress | string | `":8082"` | Set the address where the fabric pod will expose the metrics. To disable the metrics, set the address to ":0". |
8991
| networking.fabric.config.nftablesMonitor | bool | `true` | Enable/Disable the nftables monitor for the fabric pod. It means that the fabric pod will monitor the nftables rules and will restore them in case of changes. In some cases (like K3S), this monitor can cause a huge amount of CPU usage. If you are experiencing high CPU usage, you can disable this feature. |
9092
| networking.fabric.image.name | string | `"ghcr.io/liqotech/fabric"` | Image repository for the fabric pod. |
9193
| networking.fabric.image.version | string | `""` | Custom version for the fabric image. If not specified, the global tag is used. |

‎deployments/liqo/templates/liqo-fabric-daemonset.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ spec:
4545
- --podname=$(POD_NAME)
4646
- --nodename=$(NODE_NAME)
4747
- --geneve-port={{ .Values.networking.genevePort }}
48+
- --health-probe-bind-address=:{{ .Values.networking.fabric.config.healthProbeBindAddressPort}}
49+
- --metrics-address=:{{ .Values.networking.fabric.config.metricsAddressPort}}
4850
{{- if not .Values.requirements.kernel.enabled }}
4951
- --disable-kernel-version-check
5052
{{- end }}
@@ -79,6 +81,14 @@ spec:
7981
valueFrom:
8082
fieldRef:
8183
fieldPath: metadata.name
84+
ports:
85+
- name: healthz
86+
containerPort: {{ .Values.networking.fabric.config.healthProbeBindAddressPort }}
87+
protocol: TCP
88+
readinessProbe:
89+
httpGet:
90+
path: /readyz
91+
port: healthz
8292
hostNetwork: true
8393
{{- if .Values.networking.fabric.pod.priorityClassName }}
8494
priorityClassName: {{ .Values.networking.fabric.pod.priorityClassName }}

‎deployments/liqo/templates/liqo-wireguard-gateway-client-template.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ spec:
7878
ports:
7979
- containerPort: 8082
8080
name: gw-metrics
81+
- containerPort: 8083
82+
name: healthz
8183
{{- end }}
8284
env:
8385
- name: NODE_NAME
@@ -94,6 +96,10 @@ spec:
9496
add:
9597
- NET_ADMIN
9698
- NET_RAW
99+
readinessProbe:
100+
httpGet:
101+
path: /readyz
102+
port: healthz
97103
- name: wireguard
98104
image: {{ .Values.networking.gatewayTemplates.container.wireguard.image.name }}{{ include "liqo.suffix" $wireguardConfig }}:{{ include "liqo.version" $wireguardConfig }}
99105
imagePullPolicy: {{ .Values.pullPolicy }}
@@ -116,6 +122,8 @@ spec:
116122
ports:
117123
- containerPort: 8084
118124
name: wg-metrics
125+
- containerPort: 8085
126+
name: healthz
119127
{{- end }}
120128
securityContext:
121129
capabilities:
@@ -130,6 +138,10 @@ spec:
130138
mountPath: /ipc
131139
- name: wireguard-config
132140
mountPath: /etc/wireguard/keys
141+
readinessProbe:
142+
httpGet:
143+
path: /readyz
144+
port: healthz
133145
- name: geneve
134146
image: {{ .Values.networking.gatewayTemplates.container.geneve.image.name }}{{ include "liqo.suffix" $geneveConfig }}:{{ include "liqo.version" $geneveConfig }}
135147
imagePullPolicy: {{ .Values.pullPolicy }}
@@ -154,6 +166,8 @@ spec:
154166
ports:
155167
- containerPort: 8086
156168
name: gv-metrics
169+
- containerPort: 8087
170+
name: healthz
157171
{{- end }}
158172
env:
159173
- name: NODE_NAME
@@ -169,6 +183,10 @@ spec:
169183
add:
170184
- NET_ADMIN
171185
- NET_RAW
186+
readinessProbe:
187+
httpGet:
188+
path: /readyz
189+
port: healthz
172190
# Uncomment to set a priorityClassName
173191
# priorityClassName: ""
174192
volumes:

‎deployments/liqo/templates/liqo-wireguard-gateway-server-template-eks.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,13 @@ spec:
106106
- containerPort: 8082
107107
name: gw-metrics
108108
{{- end }}
109+
ports:
110+
- containerPort: 8083
111+
name: healthz
112+
readinessProbe:
113+
httpGet:
114+
path: /readyz
115+
port: healthz
109116
env:
110117
- name: NODE_NAME
111118
valueFrom:
@@ -143,6 +150,13 @@ spec:
143150
- containerPort: 8084
144151
name: wg-metrics
145152
{{- end }}
153+
ports:
154+
- containerPort: 8085
155+
name: healthz
156+
readinessProbe:
157+
httpGet:
158+
path: /readyz
159+
port: healthz
146160
securityContext:
147161
capabilities:
148162
add:
@@ -181,6 +195,13 @@ spec:
181195
- containerPort: 8086
182196
name: gv-metrics
183197
{{- end }}
198+
ports:
199+
- containerPort: 8087
200+
name: healthz
201+
readinessProbe:
202+
httpGet:
203+
path: /readyz
204+
port: healthz
184205
env:
185206
- name: NODE_NAME
186207
valueFrom:

‎deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml

+24-6
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,9 @@ spec:
8080
{{- include "liqo.concatenateMap" $d | nindent 16 }}
8181
{{- end }}
8282
{{- if .Values.metrics.enabled }}
83-
- --metrics-address=:8084
83+
- --metrics-address=:8082
8484
{{- end }}
85-
- --health-probe-bind-address=:8085
85+
- --health-probe-bind-address=:8083
8686
- --ping-enabled=true
8787
- --ping-loss-threshold={{ .Values.networking.gatewayTemplates.ping.lossThreshold }}
8888
- --ping-interval={{ .Values.networking.gatewayTemplates.ping.interval }}
@@ -96,9 +96,15 @@ spec:
9696
mountPath: /ipc
9797
{{- if .Values.metrics.enabled }}
9898
ports:
99-
- containerPort: 8084
99+
- containerPort: 8082
100100
name: gw-metrics
101+
- containerPort: 8083
102+
name: healthz
101103
{{- end }}
104+
readinessProbe:
105+
httpGet:
106+
path: /readyz
107+
port: healthz
102108
env:
103109
- name: NODE_NAME
104110
valueFrom:
@@ -127,15 +133,21 @@ spec:
127133
- --mtu={{"{{ .Spec.MTU }}"}}
128134
- --listen-port={{"{{ .Spec.Endpoint.Port }}"}}
129135
{{- if .Values.metrics.enabled }}
130-
- --metrics-address=:8082
136+
- --metrics-address=:8084
131137
{{- end }}
132-
- --health-probe-bind-address=:8083
138+
- --health-probe-bind-address=:8085
133139
- --implementation={{ .Values.networking.gatewayTemplates.wireguard.implementation }}
134140
{{- if .Values.metrics.enabled }}
135141
ports:
136-
- containerPort: 8082
142+
- containerPort: 8084
137143
name: wg-metrics
144+
- containerPort: 8085
145+
name: healthz
138146
{{- end }}
147+
readinessProbe:
148+
httpGet:
149+
path: /readyz
150+
port: healthz
139151
securityContext:
140152
capabilities:
141153
add:
@@ -173,7 +185,13 @@ spec:
173185
ports:
174186
- containerPort: 8086
175187
name: gv-metrics
188+
- containerPort: 8087
189+
name: healthz
176190
{{- end }}
191+
readinessProbe:
192+
httpGet:
193+
path: /readyz
194+
port: healthz
177195
env:
178196
- name: NODE_NAME
179197
valueFrom:

‎deployments/liqo/values.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,12 @@ networking:
124124
# In some cases (like K3S), this monitor can cause a huge amount of CPU usage.
125125
# If you are experiencing high CPU usage, you can disable this feature.
126126
nftablesMonitor: true
127+
# -- Set the port where the fabric pod will expose the health probe.
128+
# To disable the health probe, set the port to 0.
129+
healthProbeBindAddressPort: "8081"
130+
# -- Set the port where the fabric pod will expose the metrics.
131+
# To disable the metrics, set the port to 0.
132+
metricsAddressPort: "8087"
127133

128134
authentication:
129135
# -- Enable/Disable the authentication module.

‎pkg/gateway/flags.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ func InitFlags(flagset *pflag.FlagSet, opts *Options) {
108108
"RetryPeriod for the leader election")
109109

110110
flagset.StringVar(&opts.MetricsAddress, FlagNameMetricsAddress.String(), "0", "Address for the metrics endpoint")
111-
flagset.StringVar(&opts.ProbeAddr, FlagNameProbeAddr.String(), ":8081", "Address for the health probe endpoint")
111+
flagset.StringVar(&opts.ProbeAddr, FlagNameProbeAddr.String(), "0", "Address for the health probe endpoint")
112112

113113
flagset.BoolVar(&opts.DisableKernelVersionCheck, FlagNameDisableKernelVersionCheck.String(), false, "Disable the kernel version check")
114114
flagset.Var(&opts.MinimumKernelVersion, FlagNameMinimumKernelVersion.String(), "Minimum kernel version required by Liqo")

0 commit comments

Comments
 (0)