From 6d5f2c2542575965a6ee2ea8b955b3084fd681c2 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Fri, 25 Jun 2021 16:37:20 +0300 Subject: [PATCH] Upgrade calico-vxlan CNI plugin to v3.19.1 Signed-off-by: Artiom Diomin --- addons/calico-vxlan/calico-vxlan.yaml | 517 +++++++++++++++++--------- 1 file changed, 334 insertions(+), 183 deletions(-) diff --git a/addons/calico-vxlan/calico-vxlan.yaml b/addons/calico-vxlan/calico-vxlan.yaml index 06087f7df..f963e2169 100644 --- a/addons/calico-vxlan/calico-vxlan.yaml +++ b/addons/calico-vxlan/calico-vxlan.yaml @@ -11,12 +11,14 @@ data: typha_service_name: "none" # Configure the backend to use. calico_backend: "vxlan" + # Configure the MTU to use for workload interfaces and tunnels. # - If Wireguard is enabled, set to your network MTU - 60 # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50 # - Otherwise, if IPIP is enabled, set to your network MTU - 20 # - Otherwise, if not using any encapsulation, set to your network MTU. - veth_mtu: "1450" + veth_mtu: "" + # veth_mtu: "" # auto-detect MTU # veth_mtu: "8951" # use this if provider is AWS # veth_mtu: "1400" # use this if provider is OpenStack # veth_mtu: "1410" # use this if provider is GCE @@ -31,6 +33,7 @@ data: { "type": "calico", "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, @@ -59,14 +62,9 @@ data: --- # Source: calico/templates/kdd-crds.yaml - ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: bgpconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -102,6 +100,32 @@ spec: 64512]' format: int32 type: integer + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' @@ -110,13 +134,36 @@ spec: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array serviceClusterIPs: description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them. items: - description: ServiceClusterIPBlock represents a single whitelisted - CIDR block for ClusterIPs. + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. properties: cidr: type: string @@ -127,8 +174,20 @@ spec: Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. items: - description: ServiceExternalIPBlock represents a single whitelisted - CIDR External IP block. + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. properties: cidr: type: string @@ -145,15 +204,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: bgppeers.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -187,17 +241,50 @@ spec: description: The AS Number of the peer. format: int32 type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean node: description: The node name identifying the Calico node instance that - is peering with this peer. If this is not set, this represents a - global peer, i.e. a peer that peers with every node in the deployment. + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. type: string nodeSelector: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object peerIP: - description: The IP address of the peer. + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. type: string peerSelector: description: Selector for the remote nodes to peer with. When this @@ -205,12 +292,15 @@ spec: peering between the local node and selected remote nodes, we configure an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, or the global default if that is not set. type: string - required: - - asNumber - - peerIP + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string type: object type: object served: true @@ -222,15 +312,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: blockaffinities.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -288,15 +373,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: clusterinformations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -357,15 +437,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: felixconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -396,6 +471,25 @@ spec: spec: description: FelixConfigurationSpec contains the values of the Felix configuration. properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enabled" or "Disabled". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string bpfConnectTimeLoadBalancingEnabled: description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The @@ -405,13 +499,13 @@ spec: true]' type: boolean bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls + description: BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' + named cali...). type: string bpfDisableUnprivileged: description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled @@ -423,6 +517,13 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing intepreted by RPF + check. [Default: 0]' + type: integer bpfExternalServiceMode: description: 'BPFExternalServiceMode in BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) @@ -456,10 +557,10 @@ spec: `tc exec bpf debug`. [Default: Off].' type: string chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s + description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch + since it prevents Calico''s rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' @@ -479,11 +580,11 @@ spec: traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic + iptables "DROP" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]' @@ -511,19 +612,21 @@ spec: type: string type: array failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a comma-delimited list of - UDP/TCP ports that Felix will allow incoming traffic to host endpoints + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. Each - port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all inbound host ports, use the value none. - The default value allows ssh access and DHCP. [Default: tcp:22, + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' items: - description: ProtoPort is combination of protocol and port, both - must be specified. + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. properties: + net: + type: string port: type: integer protocol: @@ -534,21 +637,23 @@ spec: type: object type: array failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a comma-delimited list - of UDP/TCP ports that Felix will allow outgoing traffic from host - endpoints to irrespective of the security policy. This is useful - to avoid accidentally cutting off a host with incorrect configuration. - Each port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all outbound host ports, use the value none. - The default value opens etcd’s standard ports to ensure that Felix - does not get cut off from etcd as well as allowing DHCP and DNS. - [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, - udp:53, udp:67]' + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' items: - description: ProtoPort is combination of protocol and port, both - must be specified. + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. properties: + net: + type: string port: type: integer protocol: @@ -558,6 +663,13 @@ spec: - protocol type: object type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -586,8 +698,13 @@ spec: workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. type: string ipipEnabled: type: boolean @@ -598,7 +715,7 @@ spec: ipsetsRefreshInterval: description: 'IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: 90s]' type: string iptablesBackend: @@ -610,7 +727,7 @@ spec: iptablesLockFilePath: description: 'IptablesLockFilePath is the location of the iptables lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s + its standard location (for example if you have mapped it into Felix''s container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: @@ -642,16 +759,16 @@ spec: description: 'IptablesPostWriteCheckInterval is the period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t + This should only occur if another application on the system doesn''t respect the iptables lock. [Default: 1s]' type: string iptablesRefreshInterval: description: 'IptablesRefreshInterval is the period at which Felix re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico’s rules. Set to 0 to disable IP sets - refresh. Note: the default for this value is lower than the other - refresh intervals as a workaround for a Linux kernel bug that was - fixed in kernel version 4.11. If you are using v4.11 or greater + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. [Default: 10s]' type: string @@ -702,10 +819,15 @@ spec: type: string metadataPort: description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed [Default: 8775].' type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string natOutgoingAddress: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the @@ -776,9 +898,9 @@ spec: status reports. [Default: 90s]' type: string routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks + description: 'RouteRefreshInterval is the period at which Felix re-checks the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. + accidentally broken Calico''s rules. Set to 0 to disable route refresh. [Default: 90s]' type: string routeSource: @@ -799,6 +921,13 @@ spec: - max - min type: object + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string sidecarAccelerationEnabled: description: 'SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false]' @@ -860,8 +989,6 @@ spec: Calico''s BPF maps or attached programs. Set to 0 to disable XDP refresh. [Default: 90s]' type: string - required: - - bpfLogLevel type: object type: object served: true @@ -873,15 +1000,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: globalnetworkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -929,7 +1051,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -1012,9 +1134,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1084,7 +1206,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1113,7 +1235,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1217,9 +1339,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1260,7 +1382,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -1343,9 +1465,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1415,7 +1537,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1444,7 +1566,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -1548,9 +1670,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -1649,15 +1771,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: globalnetworksets.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1707,15 +1824,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: hostendpoints.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1771,7 +1883,7 @@ spec: is empty - through the specific interface that has one of the IPs in ExpectedIPs. Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint + as \"eth0\") are supported here; it isn't possible for a HostEndpoint to protect traffic through a specific local workload interface. \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; initially just pre-DNAT policy. Please check Calico documentation @@ -1820,15 +1932,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamblocks.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1893,7 +2000,6 @@ spec: - allocations - attributes - cidr - - deleted - strictAffinity - unallocated type: object @@ -1907,15 +2013,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamconfigs.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -1948,6 +2049,10 @@ spec: properties: autoAllocateBlocks: type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + type: integer strictAffinity: type: boolean required: @@ -1964,15 +2069,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ipamhandles.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2007,6 +2107,8 @@ spec: additionalProperties: type: integer type: object + deleted: + type: boolean handleID: type: string required: @@ -2023,15 +2125,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: ippools.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2092,7 +2189,7 @@ spec: type: object ipipMode: description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). type: string nat-outgoing: @@ -2112,7 +2209,7 @@ spec: vxlanMode: description: Contains configuration for VXLAN tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunelling is disabled). + tunneling is disabled). type: string required: - cidr @@ -2127,15 +2224,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: kubecontrollersconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2241,6 +2333,10 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer required: - controllers type: object @@ -2341,6 +2437,11 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer required: - controllers type: object @@ -2355,15 +2456,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: networkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2400,7 +2496,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -2483,9 +2579,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2555,7 +2651,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2584,7 +2680,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2688,9 +2784,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2731,7 +2827,7 @@ spec: action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria + a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." @@ -2814,9 +2910,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -2886,7 +2982,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -2915,7 +3011,7 @@ spec: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, + limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: @@ -3019,9 +3115,9 @@ spec: One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector + that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. + Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." @@ -3112,15 +3208,10 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null name: networksets.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3210,6 +3301,7 @@ rules: - create - update - delete + - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3458,8 +3550,13 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: {{ Registry "docker.io" }}/calico/cni:v3.15.1 + image: {{ Registry "docker.io" }}/calico/cni:v3.19.1 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: @@ -3480,8 +3577,13 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: {{ Registry "docker.io" }}/calico/cni:v3.15.1 - command: ["/install-cni.sh"] + image: {{ Registry "docker.io" }}/calico/cni:v3.19.1 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Name of the CNI config file to create. - name: CNI_CONF_NAME @@ -3515,19 +3617,24 @@ spec: privileged: true # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: {{ Registry "docker.io" }}/calico/pod2daemon-flexvol:v3.15.1 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true + # - name: flexvol-driver + # image: {{ Registry "docker.io" }}/calico/pod2daemon-flexvol:v3.19.1 + # volumeMounts: + # - name: flexvol-driver-host + # mountPath: /host/driver + # securityContext: + # privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node - image: {{ Registry "docker.io" }}/calico/node:v3.15.1 + image: {{ Registry "docker.io" }}/calico/node:v3.19.1 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -3576,6 +3683,9 @@ spec: configMapKeyRef: name: calico-config key: veth_mtu + # Disable AWS source-destination check on nodes. + # - name: FELIX_AWSSRCDSTCHECK + # value: Disable # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. @@ -3590,9 +3700,6 @@ spec: # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - name: FELIX_HEALTHENABLED value: "true" securityContext: @@ -3629,6 +3736,16 @@ spec: readOnly: false - name: policysync mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true volumes: # Used by calico-node. - name: lib-modules @@ -3644,6 +3761,10 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate # Used to install CNI. - name: cni-bin-dir hostPath: @@ -3651,6 +3772,10 @@ spec: - name: cni-net-dir hostPath: path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni # Mount in the directory for host-local IPAM allocations. This is # used when upgrading from host-local to calico-ipam, and can be removed # if not using the upgrade-ipam init container. @@ -3663,10 +3788,10 @@ spec: type: DirectoryOrCreate path: /var/run/nodeagent # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + # - name: flexvol-driver-host + # hostPath: + # type: DirectoryOrCreate + # path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- apiVersion: v1 @@ -3712,18 +3837,27 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: {{ Registry "docker.io" }}/calico/kube-controllers:v3.15.1 + image: {{ Registry "docker.io" }}/calico/kube-controllers:v3.19.1 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 readinessProbe: exec: command: - /usr/bin/check-status - -r + periodSeconds: 10 --- @@ -3733,6 +3867,23 @@ metadata: name: calico-kube-controllers namespace: kube-system +--- + +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + --- # Source: calico/templates/calico-etcd-secrets.yaml