From 5a91c1c242682d5a87efb03bef391c721307c7aa Mon Sep 17 00:00:00 2001 From: nold Date: Sat, 25 Feb 2023 17:36:54 +0100 Subject: [PATCH] Revert "feat(cilium): bgp" This reverts commit d173b790137a8cc927904d0470c7225b0cde8dcd. --- projects/core/values/cilium.yaml | 971 +++++++++++++++---------------- 1 file changed, 481 insertions(+), 490 deletions(-) diff --git a/projects/core/values/cilium.yaml b/projects/core/values/cilium.yaml index fc6fb4b1..e8f4debd 100644 --- a/projects/core/values/cilium.yaml +++ b/projects/core/values/cilium.yaml @@ -10,18 +10,7 @@ debug: # -- Enable debug logging enabled: false - # -- Configure verbosity levels for debug logging - # This option is used to enable debug messages for operations related to such - # sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is - # for enabling debug messages emitted per request, message and connection. - # - # Applicable values: - # - flow - # - kvstore - # - envoy - # - datapath - # - policy - verbose: ~ + # verbose: rbac: # -- Enable creation of Resource-Based Access Control configuration. @@ -31,13 +20,9 @@ rbac: imagePullSecrets: # - name: "image-pull-secret" -# -- (string) Kubernetes config path -# @default -- `"~/.kube/config"` -kubeConfigPath: "" -# -- (string) Kubernetes service host -k8sServiceHost: "" -# -- (string) Kubernetes service port -k8sServicePort: "" +# kubeConfigPath: ~/.kube/config +# k8sServiceHost: +# k8sServicePort: cluster: # -- Name of the cluster. Only required for Cluster Mesh. @@ -47,151 +32,69 @@ cluster: # may be 0 if Cluster Mesh is not used. id: 0 +# -- Define serviceAccount names for components. +# @default -- Component's fully qualified name. +serviceAccounts: + cilium: + create: true + name: cilium + annotations: {} + etcd: + create: true + name: cilium-etcd-operator + annotations: {} + operator: + create: true + name: cilium-operator + annotations: {} + preflight: + create: true + name: cilium-pre-flight + annotations: {} + relay: + create: true + name: hubble-relay + annotations: {} + ui: + create: true + name: hubble-ui + annotations: {} + clustermeshApiserver: + create: true + name: clustermesh-apiserver + annotations: {} + # -- Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob + clustermeshcertgen: + create: true + name: clustermesh-apiserver-generate-certs + annotations: {} + # -- Hubblecertgen is used if hubble.tls.auto.method=cronJob + hubblecertgen: + create: true + name: hubble-generate-certs + annotations: {} + +# -- Configure termination grace period for cilium-agent DaemonSet. +terminationGracePeriodSeconds: 1 + +# -- Install the cilium agent resources. +agent: true + +# -- Agent container name. +name: cilium + # -- Roll out cilium agent pods automatically when configmap is updated. rollOutCiliumPods: false # -- Agent container image. image: override: ~ - repository: "quay.io/cilium/cilium" - tag: "v1.13.0" + repository: "quay.io/cilium/cilium-ci" + tag: "v1.13" pullPolicy: "IfNotPresent" # cilium-digest - digest: "sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68" - useDigest: true - -# -- Additional containers added to the cilium DaemonSet. -extraContainers: [] - -# -- Additional agent container arguments. -extraArgs: - - --enable-ipv4-masquerade=true - - --enable-bpf-masquerade=true - -# -- Additional agent container environment variables. -extraEnv: [] - -# -- Additional agent hostPath mounts. -extraHostPathMounts: [] - # - name: host-mnt-data - # mountPath: /host/mnt/data - # hostPath: /mnt/data - # hostPathType: Directory - # readOnly: true - # mountPropagation: HostToContainer - -# -- Additional agent volumes. -extraVolumes: [] - -# -- Additional agent volumeMounts. -extraVolumeMounts: [] - -# -- extraConfig allows you to specify additional configuration parameters to be -# included in the cilium-config configmap. -extraConfig: {} -# my-config-a: "1234" -# my-config-b: |- -# test 1 -# test 2 -# test 3 - -# -- Annotations to be added to agent pods -podAnnotations: {} - -# -- Labels to be added to agent pods -podLabels: {} - -# -- Agent resource limits & requests -# ref: https://kubernetes.io/docs/user-guide/compute-resources/ -resources: {} - # limits: - # cpu: 4000m - # memory: 4Gi - # requests: - # cpu: 100m - # memory: 512Mi - -securityContext: - # -- User to run the pod with - # runAsUser: 0 - # -- Run the pod with elevated privileges - privileged: false - # -- SELinux options for the `cilium-agent` and init containers - seLinuxOptions: - level: 's0' - # Running with spc_t since we have removed the privileged mode. - # Users can change it to a different type as long as they have the - # type available on the system. - type: 'spc_t' - capabilities: - # -- Capabilities for the `cilium-agent` container - ciliumAgent: - # Use to set socket permission - - CHOWN - # Used to terminate envoy child process - - KILL - # Used since cilium modifies routing tables, etc... - - NET_ADMIN - # Used since cilium creates raw sockets, etc... - - NET_RAW - # Used since cilium monitor uses mmap - - IPC_LOCK - # Used in iptables. Consider removing once we are iptables-free - - SYS_MODULE - # We need it for now but might not need it for >= 5.11 specially - # for the 'SYS_RESOURCE'. - # In >= 5.8 there's already BPF and PERMON capabilities - - SYS_ADMIN - # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC - - SYS_RESOURCE - # Both PERFMON and BPF requires kernel 5.8, container runtime - # cri-o >= v1.22.0 or containerd >= v1.5.0. - # If available, SYS_ADMIN can be removed. - #- PERFMON - #- BPF - # Allow discretionary access control (e.g. required for package installation) - - DAC_OVERRIDE - # Allow to set Access Control Lists (ACLs) on arbitrary files (e.g. required for package installation) - - FOWNER - # Allow to execute program that changes GID (e.g. required for package installation) - - SETGID - # Allow to execute program that changes UID (e.g. required for package installation) - - SETUID - # -- Capabilities for the `mount-cgroup` init container - mountCgroup: - # Only used for 'mount' cgroup - - SYS_ADMIN - # Used for nsenter - - SYS_CHROOT - - SYS_PTRACE - # -- capabilities for the `apply-sysctl-overwrites` init container - applySysctlOverwrites: - # Required in order to access host's /etc/sysctl.d dir - - SYS_ADMIN - # Used for nsenter - - SYS_CHROOT - - SYS_PTRACE - # -- Capabilities for the `clean-cilium-state` init container - cleanCiliumState: - # Most of the capabilities here are the same ones used in the - # cilium-agent's container because this container can be used to - # uninstall all Cilium resources, and therefore it is likely that - # will need the same capabilities. - # Used since cilium modifies routing tables, etc... - - NET_ADMIN - # Used in iptables. Consider removing once we are iptables-free - - SYS_MODULE - # We need it for now but might not need it for >= 5.11 specially - # for the 'SYS_RESOURCE'. - # In >= 5.8 there's already BPF and PERMON capabilities - - SYS_ADMIN - # Could be an alternative for the SYS_ADMIN for the RLIMIT_NPROC - - SYS_RESOURCE - # Both PERFMON and BPF requires kernel 5.8, container runtime - # cri-o >= v1.22.0 or containerd >= v1.5.0. - # If available, SYS_ADMIN can be removed. - #- PERFMON - #- BPF + digest: "" + useDigest: false # -- Cilium agent update strategy updateStrategy: @@ -199,15 +102,6 @@ updateStrategy: rollingUpdate: maxUnavailable: 1 -# Configuration Values for cilium-agent - -# -- Enable installation of PodCIDR routes between worker -# nodes if worker nodes share a common L2 network segment. -autoDirectNodeRoutes: false - -# -- Annotate k8s node upon initialization with Cilium's metadata. -annotateK8sNode: false - # -- Enable bandwidth manager to optimize TCP and UDP workloads and allow # for rate-limiting traffic from individual Pods with EDT (Earliest Departure # Time) through the "kubernetes.io/egress-bandwidth" Pod annotation. @@ -232,93 +126,8 @@ bgp: # CiliumBGPPeeringPolicy CRDs. bgpControlPlane: # -- Enables the BGP control plane. - enabled: true - -pmtuDiscovery: - # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to - # the client. enabled: false -bpf: - # -- Configure the mount point for the BPF filesystem - root: /sys/fs/bpf - - # -- Enable BPF clock source probing for more efficient tick retrieval. - clockProbe: false - - # -- Enables pre-allocation of eBPF map values. This increases - # memory usage but can reduce latency. - preallocateMaps: false - - # -- (int) Configure the maximum number of entries in the TCP connection tracking - # table. - # @default -- `524288` - ctTcpMax: ~ - - # -- (int) Configure the maximum number of entries for the non-TCP connection - # tracking table. - # @default -- `262144` - ctAnyMax: ~ - - # -- Configure the maximum number of service entries in the - # load balancer maps. - lbMapMax: 65536 - - # -- (int) Configure the maximum number of entries for the NAT table. - # @default -- `524288` - natMax: ~ - - # -- (int) Configure the maximum number of entries for the neighbor table. - # @default -- `524288` - neighMax: ~ - - # -- Configure the maximum number of entries in endpoint policy map (per endpoint). - policyMapMax: 16384 - - # -- (float64) Configure auto-sizing for all BPF maps based on available memory. - # ref: https://docs.cilium.io/en/stable/concepts/ebpf/maps/#ebpf-maps - # @default -- `0.0025` - mapDynamicSizeRatio: ~ - - # -- Configure the level of aggregation for monitor notifications. - # Valid options are none, low, medium, maximum. - monitorAggregation: medium - - # -- Configure the typical time between monitor notifications for - # active connections. - monitorInterval: "5s" - - # -- Configure which TCP flags trigger notifications when seen for the - # first time in a connection. - monitorFlags: "all" - - # -- Allow cluster external access to ClusterIP services. - lbExternalClusterIP: false - - # -- (bool) Enable native IP masquerade support in eBPF - # @default -- `false` - masquerade: ~ - - # -- (bool) Configure whether direct routing mode should route traffic via - # host stack (true) or directly and more efficiently out of BPF (false) if - # the kernel supports it. The latter has the implication that it will also - # bypass netfilter in the host namespace. - # @default -- `false` - hostLegacyRouting: ~ - - # -- (bool) Configure the eBPF-based TPROXY to reduce reliance on iptables rules - # for implementing Layer 7 policy. - # @default -- `false` - tproxy: ~ - - # -- (list) Configure explicitly allowed VLAN id's for bpf logic bypass. - # [0] will allow all VLAN id's without any filtering. - # @default -- `[]` - vlanBypass: ~ - -# -- Clean all eBPF datapath state from the initContainer of the cilium-agent -# DaemonSet. - cni: # -- Install the CNI configuration and binary files into the filesystem. install: true @@ -331,6 +140,52 @@ cni: # - portmap chainingMode: none + # -- Make Cilium take ownership over the `/etc/cni/net.d` directory on the + # node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. + # This ensures no Pods can be scheduled using other CNI plugins during Cilium + # agent downtime. + exclusive: true + + # -- Configure the log file for CNI logging with retention policy of 7 days. + # Disable CNI file logging by setting this field to empty explicitly. + logFile: /var/run/cilium/cilium-cni.log + + # -- Skip writing of the CNI configuration. This can be used if + # writing of the CNI configuration is performed by external automation. + customConf: false + + # -- Configure the path to the CNI configuration directory on the host. + confPath: /etc/cni/net.d + + # -- Configure the path to the CNI binary directory on the host. + binPath: /opt/cni/bin + + # -- Specify the path to a CNI config to read from on agent start. + # This can be useful if you want to manage your CNI + # configuration outside of a Kubernetes environment. This parameter is + # mutually exclusive with the 'cni.configMap' parameter. + # readCniConf: /host/etc/cni/net.d/05-cilium.conf + + # -- When defined, configMap will mount the provided value as ConfigMap and + # interpret the cniConf variable as CNI configuration file and write it + # when the agent starts up + # configMap: cni-configuration + + # -- Configure the key in the CNI ConfigMap to read the contents of + # the CNI configuration from. + configMapKey: cni-config + + # -- Configure the path to where to mount the ConfigMap inside the agent pod. + confFileMountPath: /tmp/cni-configuration + + # -- Configure the path to where the CNI configuration directory is mounted + # inside the agent pod. + hostConfDirMountPath: /host/etc/cni/net.d + +# -- Configure how frequently garbage collection should occur for the datapath +# connection tracking table. +# conntrackGCInterval: "0s" + # -- Configure container runtime specific integration. containerRuntime: # -- Enables specific integrations for container runtimes. @@ -344,11 +199,22 @@ containerRuntime: # -- Configure the path to the container runtime control socket. # socketPath: /path/to/runtime.sock +# crdWaitTimeout: "" + # -- Tail call hooks for custom eBPF programs. customCalls: # -- Enable tail call hooks for custom eBPF programs. enabled: false +# -- Configure which datapath mode should be used for configuring container +# connectivity. Valid options are "veth" or "ipvlan". Deprecated, to be removed +# in v1.12. +datapathMode: veth + +daemon: + # -- Configure where Cilium runtime state should be stored. + runPath: "/var/run/cilium" + # -- Specify which network interfaces can run the eBPF datapath. This means # that a packet sent from a pod to a destination outside the cluster will be # masqueraded (to an output device IPv4 address), if the output device runs the @@ -388,20 +254,10 @@ ingressController: # This will automatically set enable-envoy-config as well. enabled: false - # -- Default ingress load balancer mode - # Supported values: shared, dedicated - # For granular control, use the following annotations on the ingress resource - # ingress.cilium.io/loadbalancer-mode: shared|dedicated, - loadbalancerMode: dedicated - # -- Enforce https for host having matching TLS host in Ingress. # Incoming traffic to http listener will return 308 http error code with respective location in header. enforceHttps: true - # -- IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate - # from Ingress to the Load Balancer service - ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] - # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. secretsNamespace: # -- Create secrets namespace for Ingress. @@ -414,39 +270,6 @@ ingressController: # If disabled, TLS secrets must be maintained externally. sync: true - # -- Load-balancer service in shared mode. - # This is a single load-balancer service for all Ingress resources. - service: - # -- Service name - name: cilium-ingress - # -- Labels to be added for the shared LB service - labels: {} - # -- Annotations to be added for the shared LB service - annotations: {} - # -- Service type for the shared LB service - type: LoadBalancer - # -- Configure a specific nodePort for insecure HTTP traffic on the shared LB service - insecureNodePort: ~ - # -- Configure a specific nodePort for secure HTTPS traffic on the shared LB service - secureNodePort : ~ - -gatewayAPI: - # -- Enable support for Gateway API in cilium - # This will automatically set enable-envoy-config as well. - enabled: false - - # -- SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. - secretsNamespace: - # -- Create secrets namespace for Gateway API. - create: true - - # -- Name of Gateway API secret namespace. - name: cilium-secrets - - # -- Enable secret sync, which will make sure all TLS secrets used by Ingress are synced to secretsNamespace.name. - # If disabled, TLS secrets must be maintained externally. - sync: true - # -- Enables the fallback compatibility solution for when the xt_socket kernel # module is missing and it is needed for the datapath L7 redirection to work # properly. See documentation for details on when this can be disabled: @@ -457,14 +280,109 @@ encryption: # -- Enable transparent network encryption. enabled: false + # -- Encryption method. Can be either ipsec or wireguard. + type: ipsec + + # -- Enable encryption for pure node to node traffic. + # This option is only effective when encryption.type is set to ipsec. + nodeEncryption: false + + ipsec: + # -- Name of the key file inside the Kubernetes secret configured via secretName. + keyFile: "" + + # -- Path to mount the secret inside the Cilium pod. + mountPath: "" + + # -- Name of the Kubernetes secret containing the encryption keys. + secretName: "" + + # -- The interface to use for encrypted traffic. + interface: "" + + wireguard: + # -- Enables the fallback to the user-space implementation. + userspaceFallback: false + + # -- Deprecated in favor of encryption.ipsec.keyFile. + # Name of the key file inside the Kubernetes secret configured via secretName. + # This option is only effective when encryption.type is set to ipsec. + keyFile: keys + + # -- Deprecated in favor of encryption.ipsec.mountPath. + # Path to mount the secret inside the Cilium pod. + # This option is only effective when encryption.type is set to ipsec. + mountPath: /etc/ipsec + + # -- Deprecated in favor of encryption.ipsec.secretName. + # Name of the Kubernetes secret containing the encryption keys. + # This option is only effective when encryption.type is set to ipsec. + secretName: cilium-ipsec-keys + + # -- Deprecated in favor of encryption.ipsec.interface. + # The interface to use for encrypted traffic. + # This option is only effective when encryption.type is set to ipsec. + interface: "" + +endpointHealthChecking: + # -- Enable connectivity health checking between virtual endpoints. + enabled: true + +# -- Enable endpoint status. +# Status can be: policy, health, controllers, logs and / or state. For 2 or more options use a comma. +endpointStatus: + enabled: false + status: "" + endpointRoutes: # -- Enable use of per endpoint routes instead of routing via # the cilium_host interface. enabled: false +eni: + # -- Enable Elastic Network Interface (ENI) integration. + enabled: false + # -- Update ENI Adapter limits from the EC2 API + updateEC2AdapterLimitViaAPI: false + # -- Release IPs not used from the ENI + awsReleaseExcessIPs: false + # -- Enable ENI prefix delegation + awsEnablePrefixDelegation: false + # -- EC2 API endpoint to use + ec2APIEndpoint: "" + # -- Tags to apply to the newly created ENIs + eniTags: {} + # -- If using IAM role for Service Accounts will not try to + # inject identity values from cilium-aws kubernetes secret. + # Adds annotation to service account if managed by Helm. + # See https://github.com/aws/amazon-eks-pod-identity-webhook + iamRole: "" + # -- Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. + subnetIDsFilter: "" + # -- Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs + # Important note: This requires that each instance has an ENI with a matching subnet attached + # when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, + # use the CNI configuration file settings (cni.customConf) instead. + subnetTagsFilter: "" + # -- Filter via AWS EC2 Instance tags (k=v) which will dictate which AWS EC2 Instances + # are going to be used to create new ENIs + instanceTagsFilter: "" + externalIPs: # -- Enable ExternalIPs service support. - enabled: true + enabled: false + +# fragmentTracking enables IPv4 fragment tracking support in the datapath. +# fragmentTracking: true + +# -- Enable connectivity health checking. +healthChecking: true + +# -- TCP port for the agent health API. This is not the port for cilium-health. +healthPort: 9879 # -- Configure the host firewall. hostFirewall: @@ -473,7 +391,7 @@ hostFirewall: hostPort: # -- Enable hostPort service support. - enabled: true + enabled: false # -- Configure socket LB socketLB: @@ -523,21 +441,19 @@ hubble: # are disabled. # Example: # - # enabled: - # - dns:query;ignoreAAAA - # - drop - # - tcp - # - flow - # - icmp - # - http + enabled: + - dns:query;ignoreAAAA + - drop + - tcp + - flow + - icmp + - http # # You can specify the list of metrics from the helm CLI: # # --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" # enabled: ~ - # -- Enables exporting hubble metrics in OpenMetrics format. - enableOpenMetrics: false # -- Configure the port the hubble metric server listens on. port: 9965 # -- Annotations to be added to hubble-metrics service. @@ -546,27 +462,11 @@ hubble: # -- Create ServiceMonitor resources for Prometheus Operator. # This requires the prometheus CRDs to be available. # ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) - enabled: true + enabled: false # -- Labels to add to ServiceMonitor hubble labels: {} # -- Annotations to add to ServiceMonitor hubble annotations: {} - # -- Interval for scrape metrics. - interval: "10s" - # -- Relabeling configs for the ServiceMonitor hubble - relabelings: - - sourceLabels: - - __meta_kubernetes_pod_node_name - targetLabel: node - replacement: ${1} - # -- Metrics relabeling configs for the ServiceMonitor hubble - metricRelabelings: ~ - dashboards: - enabled: false - label: grafana_dashboard - namespace: ~ - labelValue: "1" - annotations: {} # -- Unix domain socket path to listen to when Hubble is enabled. socketPath: /var/run/cilium/hubble.sock @@ -575,23 +475,15 @@ hubble: # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that # Hubble is listening on port 4244. listenAddress: ":4244" - # -- Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. - preferIpv6: false - # -- (bool) Skip Hubble events with unknown cgroup ids - # @default -- `true` - skipUnknownCGroupIDs: ~ - peerService: # -- Enable a K8s Service for the Peer service, so that it can be accessed - # by a non-local client. This configuration option is deprecated, the peer - # service will be non-optional starting Cilium v1.14. - enabled: false + # by a non-local client + enabled: true # -- Service Port for the Peer service. # If not set, it is dynamically assigned to port 443 if TLS is enabled and to # port 80 if not. # servicePort: 80 - # -- Target Port for the Peer service, must match the hubble.listenAddress' - # port. + # -- Target Port for the Peer service. targetPort: 4244 # -- The cluster domain to use to query the Hubble Peer service. It should # be the local cluster. @@ -601,7 +493,7 @@ hubble: # -- Enable mutual TLS for listenAddress. Setting this value to false is # highly discouraged as the Hubble API provides access to potentially # sensitive network flow metadata and is exposed on the host network. - enabled: false + enabled: true # -- Configure automatic TLS certificates generation. auto: # -- Auto-generate certificates. @@ -616,7 +508,7 @@ hubble: # certificates not provided by the user at installation # time. # - certmanager: This method use cert-manager to generate & rotate certificates. - method: certmanager + method: helm # -- Generated certificates validity duration in days. certValidityDuration: 1095 # -- Schedule for certificates regeneration (regardless of their expiration date). @@ -669,8 +561,8 @@ hubble: repository: "quay.io/cilium/hubble-relay" tag: "v1.13.0" # hubble-relay-digest - digest: "sha256:bc00f086285d2d287dd662a319d3dbe90e57179515ce8649425916aecaa9ac3c" - useDigest: true + digest: "" + useDigest: false pullPolicy: "IfNotPresent" # -- Specifies the resources for the hubble-relay pods @@ -688,12 +580,6 @@ hubble: matchLabels: k8s-app: cilium - # -- Pod topology spread constraints for hubble-relay - topologySpreadConstraints: [] - # - maxSkew: 1 - # topologyKey: topology.kubernetes.io/zone - # whenUnsatisfiable: DoNotSchedule - # -- Node labels for pod assignment # ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: @@ -796,12 +682,12 @@ hubble: # -- Enable prometheus metrics for hubble-relay on the configured port at # /metrics prometheus: - enabled: true + enabled: false port: 9966 serviceMonitor: # -- Enable service monitors. # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) - enabled: true + enabled: false # -- Labels to add to ServiceMonitor hubble-relay labels: {} # -- Annotations to add to ServiceMonitor hubble-relay @@ -811,18 +697,6 @@ hubble: # -- Specify the Kubernetes namespace where Prometheus expects to find # service monitors configured. # namespace: "" - # -- Relabeling configs for the ServiceMonitor hubble-relay - relabelings: ~ - # -- Metrics relabeling configs for the ServiceMonitor hubble-relay - metricRelabelings: ~ - - pprof: - # -- Enable pprof for hubble-relay - enabled: false - # -- Configure pprof listen address for hubble-relay - address: localhost - # -- Configure pprof listen port for hubble-relay - port: 6062 ui: # -- Whether to enable the Hubble UI. @@ -903,10 +777,6 @@ hubble: # requests: # cpu: 100m # memory: 64Mi - server: - # -- Controls server listener for ipv6 - ipv6: - enabled: true # -- The number of replicas of Hubble UI to deploy. replicas: 1 @@ -931,12 +801,6 @@ hubble: # -- Affinity for hubble-ui affinity: {} - # -- Pod topology spread constraints for hubble-ui - topologySpreadConstraints: [] - # - maxSkew: 1 - # topologyKey: topology.kubernetes.io/zone - # whenUnsatisfiable: DoNotSchedule - # -- Node labels for pod assignment # ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: @@ -975,23 +839,29 @@ hubble: # -- hubble-ui ingress configuration. ingress: enabled: true + className: ingress-internal-traefik annotations: cert-manager.io/cluster-issuer: vault-issuer traefik.ingress.kubernetes.io/router.tls: 'true' - className: ingress-internal-traefik hosts: - cilium.dc tls: - - secretName: cilium-dc-tls - hosts: - - cilium.dc + - secretName: cilium-dc-tls + hosts: + - cilium.dc # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" -# -- (string) Time to wait before using new identity on endpoint identity change. -# @default -- `"5s"` -identityChangeGracePeriod: "" +# -- Time to wait before using new identity on endpoint identity change. +# identityChangeGracePeriod: "5s" + +# -- GC interval for security identities. +# identityGCInterval: "15m0s" + +# -- Timeout after which identity expires on lack of heartbeat. +# identityHeartbeatTimeout: "30m0s" + # -- Configure whether to install iptables rules to allow for TPROXY # (L7 proxy injection), iptables-based masquerading and compatibility @@ -1023,23 +893,10 @@ ipam: clusterPoolIPv6PodCIDRList: [] # -- IPv6 CIDR mask size to delegate to individual nodes for IPAM. clusterPoolIPv6MaskSize: 120 - # -- The maximum burst size when rate limiting access to external APIs. - # Also known as the token bucket capacity. - # @default -- `20` - externalAPILimitBurstSize: ~ - # -- The maximum queries per second when rate limiting access to - # external APIs. Also known as the bucket refill rate, which is used to - # refill the bucket up to the burst size capacity. - # @default -- `4.0` - externalAPILimitQPS: ~ # -- Configure the eBPF-based ip-masq-agent ipMasqAgent: - enabled: true -# the config of nonMasqueradeCIDRs -# config: - # nonMasqueradeCIDRs: [] - # masqLinkLocal: false + enabled: false # iptablesLockTimeout defines the iptables "--wait" option when invoked from Cilium. # iptablesLockTimeout: "5s" @@ -1098,7 +955,7 @@ kubeProxyReplacementHealthzBindAddr: "" l2NeighDiscovery: # -- Enable L2 neighbor discovery in the agent - enabled: false + enabled: true # -- Override the agent's default neighbor resolution refresh period. refreshPeriod: "30s" @@ -1131,9 +988,6 @@ maglev: {} # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. enableIPv4Masquerade: true -# -- Enables IPv6 BIG TCP support which increases maximum GSO/GRO limits for nodes and pods -enableIPv6BIGTCP: false - # -- Enables masquerading of IPv6 traffic leaving the node from endpoints. enableIPv6Masquerade: true @@ -1159,7 +1013,7 @@ vtep: # -- A space separated list of VTEP device MAC addresses (VTEP MAC), for example "x:x:x:x:x:x y:y:y:y:y:y:y" mac: "" -# -- (string) Allows to explicitly specify the IPv4 CIDR for native routing. +# -- Allows to explicitly specify the IPv4 CIDR for native routing. # When specified, Cilium assumes networking for this CIDR is preconfigured and # hands traffic destined for that range to the Linux network stack without # applying any SNAT. @@ -1169,9 +1023,9 @@ vtep: # direct routing and the Kubernetes CIDR is included in the native routing CIDR, # the user must configure the routes to reach pods, either manually or by # setting the auto-direct-node-routes flag. -ipv4NativeRoutingCIDR: "10.0.0.0/8" +# ipv4NativeRoutingCIDR: -# -- (string) Allows to explicitly specify the IPv6 CIDR for native routing. +# -- Allows to explicitly specify the IPv6 CIDR for native routing. # When specified, Cilium assumes networking for this CIDR is preconfigured and # hands traffic destined for that range to the Linux network stack without # applying any SNAT. @@ -1181,7 +1035,7 @@ ipv4NativeRoutingCIDR: "10.0.0.0/8" # direct routing and the Kubernetes CIDR is included in the native routing CIDR, # the user must configure the routes to reach pods, either manually or by # setting the auto-direct-node-routes flag. -ipv6NativeRoutingCIDR: "" +# ipv6NativeRoutingCIDR: # -- cilium-monitor sidecar. monitor: @@ -1189,7 +1043,7 @@ monitor: enabled: false # -- Configure service load balancing -loadBalancer: +# loadBalancer: # -- standalone enables the standalone L4LB which does not connect to # kube-apiserver. # standalone: false @@ -1214,31 +1068,10 @@ loadBalancer: # endpoints filtering # serviceTopology: false - # -- L7 LoadBalancer - l7: - # -- Enable L7 service load balancing via envoy proxy. - # The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, - # will be forwarded to the local backend proxy to be load balanced to the service endpoints. - # Please refer to docs for supported annotations for more configuration. - # - # Applicable values: - # - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - # - disabled: Disable L7 load balancing. - backend: disabled - # -- List of ports from service to be automatically redirected to above backend. - # Any service exposing one of these ports will be automatically redirected. - # Fine-grained control can be achieved by using the service annotation. - ports: [] - # -- Default LB algorithm - # The default LB algorithm to be used for services, which can be overridden by the - # service annotation (e.g. service.cilium.io/lb-l7-algorithm) - # Applicable values: round_robin, least_request, random - algorithm: round_robin - # -- Configure N-S k8s service loadbalancing nodePort: # -- Enable the Cilium NodePort service implementation. - enabled: true + enabled: false # -- Port range to use for NodePort services. # range: "30000,32767" @@ -1261,38 +1094,24 @@ nodePort: policyEnforcementMode: "default" pprof: - # -- Enable pprof for cilium-agent + # -- Enable Go pprof debugging enabled: false - # -- Configure pprof listen address for cilium-agent - address: localhost - # -- Configure pprof listen port for cilium-agent - port: 6060 # -- Configure prometheus metrics on the configured port at /metrics prometheus: - enabled: true + enabled: false port: 9962 serviceMonitor: # -- Enable service monitors. # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) - enabled: true + enabled: false # -- Labels to add to ServiceMonitor cilium-agent labels: {} # -- Annotations to add to ServiceMonitor cilium-agent annotations: {} - # -- Interval for scrape metrics. - interval: "10s" # -- Specify the Kubernetes namespace where Prometheus expects to find # service monitors configured. # namespace: "" - # -- Relabeling configs for the ServiceMonitor cilium-agent - relabelings: - - sourceLabels: - - __meta_kubernetes_pod_node_name - targetLabel: node - replacement: ${1} - # -- Metrics relabeling configs for the ServiceMonitor cilium-agent - metricRelabelings: ~ # -- Metrics that should be enabled or disabled from the default metric # list. (+metric_foo to enable metric_foo , -metric_bar to disable # metric_bar). @@ -1372,14 +1191,7 @@ tls: # - disabled # - vxlan (default) # - geneve -tunnel: "disabled" - -# -- Configure VXLAN and Geneve tunnel port. -# @default -- Port 8472 for VXLAN, Port 6081 for Geneve -tunnelPort: 0 - -# -- Configure the underlying network MTU to overwrite auto-detected MTU. -MTU: 0 +tunnel: "vxlan" # -- Disable the usage of CiliumEndpoint CRD. disableEndpointCRD: "false" @@ -1414,12 +1226,6 @@ etcd: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - # -- Pod topology spread constraints for cilium-etcd-operator - topologySpreadConstraints: [] - # - maxSkew: 1 - # topologyKey: topology.kubernetes.io/zone - # whenUnsatisfiable: DoNotSchedule - # -- Node labels for cilium-etcd-operator pod assignment # ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: @@ -1492,14 +1298,27 @@ operator: repository: "quay.io/cilium/operator" tag: "v1.13.0" # operator-generic-digest - genericDigest: "sha256:4b58d5b33e53378355f6e8ceb525ccf938b7b6f5384b35373f1f46787467ebf5" - useDigest: true + genericDigest: "" + # operator-azure-digest + azureDigest: "" + # operator-aws-digest + awsDigest: "" + # operator-alibabacloud-digest + alibabacloudDigest: "" + useDigest: false pullPolicy: "IfNotPresent" suffix: "" # -- Number of replicas to run for the cilium-operator deployment replicas: 1 + # -- The priority class to use for cilium-operator + priorityClassName: "" + + # -- DNS policy for Cilium operator pods. + # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: "" + # -- cilium-operator update strategy updateStrategy: type: RollingUpdate @@ -1507,11 +1326,14 @@ operator: maxSurge: 1 maxUnavailable: 1 - # -- Pod topology spread constraints for cilium-operator - topologySpreadConstraints: [] - # - maxSkew: 1 - # topologyKey: topology.kubernetes.io/zone - # whenUnsatisfiable: DoNotSchedule + # -- Affinity for cilium-operator + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator # -- Node labels for cilium-operator pod assignment # ref: https://kubernetes.io/docs/user-guide/node-selection/ @@ -1586,23 +1408,12 @@ operator: # -- Interval for cilium node garbage collection. nodeGCInterval: "5m0s" - # -- Skip CNP node status clean up at operator startup. - skipCNPStatusStartupClean: false - # -- Interval for identity garbage collection. identityGCInterval: "15m0s" # -- Timeout for identity heartbeats. identityHeartbeatTimeout: "30m0s" - pprof: - # -- Enable pprof for cilium-operator - enabled: false - # -- Configure pprof listen address for cilium-operator - address: localhost - # -- Configure pprof listen port for cilium-operator - port: 6061 - # -- Enable prometheus metrics for cilium-operator on the configured port at # /metrics prometheus: @@ -1616,12 +1427,6 @@ operator: labels: {} # -- Annotations to add to ServiceMonitor cilium-operator annotations: {} - # -- Interval for scrape metrics. - interval: "10s" - # -- Relabeling configs for the ServiceMonitor cilium-operator - relabelings: ~ - # -- Metrics relabeling configs for the ServiceMonitor cilium-operator - metricRelabelings: ~ # -- Skip CRDs creation for cilium-operator skipCRDCreation: false @@ -1726,8 +1531,8 @@ preflight: repository: "quay.io/cilium/cilium" tag: "v1.13.0" # cilium-digest - digest: "sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68" - useDigest: true + digest: "" + useDigest: false pullPolicy: "IfNotPresent" # -- The priority class to use for the preflight pod. @@ -1762,8 +1567,6 @@ preflight: effect: NoSchedule - key: node-role.kubernetes.io/master effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized effect: NoSchedule value: "true" @@ -1828,8 +1631,209 @@ enableCriticalPriorityClass: true #disableEnvoyVersionCheck: false clustermesh: + # -- Deploy clustermesh-apiserver for clustermesh useAPIServer: false + # -- Clustermesh explicit configuration. + config: + # -- Enable the Clustermesh explicit configuration. + enabled: false + # -- Default dns domain for the Clustermesh API servers + # This is used in the case cluster addresses are not provided + # and IPs are used. + domain: mesh.cilium.io + # -- List of clusters to be peered in the mesh. + clusters: [] + # clusters: + # # -- Name of the cluster + # - name: cluster1 + # # -- Address of the cluster, use this if you created DNS records for + # # the cluster Clustermesh API server. + # address: cluster1.mesh.cilium.io + # # -- Port of the cluster Clustermesh API server. + # port: 2379 + # # -- IPs of the cluster Clustermesh API server, use multiple ones when + # # you have multiple IPs to access the Clustermesh API server. + # ips: + # - 172.18.255.201 + # # -- base64 encoded PEM values for the cluster client certificate, private key and certificate authority. + # tls: + # cert: "" + # key: "" + + apiserver: + # -- Clustermesh API server image. + image: + override: ~ + repository: "quay.io/cilium/clustermesh-apiserver" + tag: "v1.13.0" + # clustermesh-apiserver-digest + digest: "" + useDigest: false + pullPolicy: "IfNotPresent" + + etcd: + # -- Clustermesh API server etcd image. + image: + override: ~ + repository: "quay.io/coreos/etcd" + tag: "v3.5.7@sha256:7238b08a6bad494e84ed1c632a62d39bdeed1f929950a05c1a32b6d4490a0047" + pullPolicy: "IfNotPresent" + + service: + # -- The type of service used for apiserver access. + type: NodePort + # -- Optional port to use as the node port for apiserver access. + nodePort: 32379 + # -- Optional loadBalancer IP address to use with type LoadBalancer. + # loadBalancerIP: + + # -- Annotations for the clustermesh-apiserver + # For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" + # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 + annotations: {} + + # -- Number of replicas run for the clustermesh-apiserver deployment. + replicas: 1 + + # -- Additional clustermesh-apiserver environment variables. + extraEnv: [] + + # -- Annotations to be added to clustermesh-apiserver pods + podAnnotations: {} + + # -- Labels to be added to clustermesh-apiserver pods + podLabels: {} + + # PodDisruptionBudget settings + podDisruptionBudget: + # -- enable PodDisruptionBudget + # ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ + enabled: false + # -- Minimum number/percentage of pods that should remain scheduled. + # When it's set, maxUnavailable must be disabled by `maxUnavailable: null` + minAvailable: null + # -- Maximum number/percentage of pods that may be made unavailable + maxUnavailable: 1 + + # -- Resource requests and limits for the clustermesh-apiserver container of the clustermesh-apiserver deployment, such as + # resources: + # limits: + # cpu: 1000m + # memory: 1024M + # requests: + # cpu: 100m + # memory: 64Mi + # -- Resource requests and limits for the clustermesh-apiserver + resources: {} + # requests: + # cpu: 100m + # memory: 64Mi + # limits: + # cpu: 1000m + # memory: 1024M + + # -- Affinity for clustermesh.apiserver + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: clustermesh-apiserver + + # -- Node labels for pod assignment + # ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: + kubernetes.io/os: linux + + # -- Node tolerations for pod assignment on nodes with taints + # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + tolerations: [] + + # -- clustermesh-apiserver update strategy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + # -- The priority class to use for clustermesh-apiserver + priorityClassName: "" + + tls: + # -- Configure automatic TLS certificates generation. + # A Kubernetes CronJob is used the generate any + # certificates not provided by the user at installation + # time. + auto: + # -- When set to true, automatically generate a CA and certificates to + # enable mTLS between clustermesh-apiserver and external workload instances. + # If set to false, the certs to be provided by setting appropriate values below. + enabled: true + # Sets the method to auto-generate certificates. Supported values: + # - helm: This method uses Helm to generate all certificates. + # - cronJob: This method uses a Kubernetes CronJob the generate any + # certificates not provided by the user at installation + # time. + # - certmanager: This method use cert-manager to generate & rotate certificates. + method: helm + # -- Generated certificates validity duration in days. + certValidityDuration: 1095 + # -- Schedule for certificates regeneration (regardless of their expiration date). + # Only used if method is "cronJob". If nil, then no recurring job will be created. + # Instead, only the one-shot job is deployed to generate the certificates at + # installation time. + # + # Due to the out-of-band distribution of client certs to external workloads the + # CA is (re)regenerated only if it is not provided as a helm value and the k8s + # secret is manually deleted. + # + # Defaults to none. Commented syntax gives midnight of the first day of every + # fourth month. For syntax, see + # https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule + # schedule: "0 0 1 */4 *" + + # [Example] + # certManagerIssuerRef: + # group: cert-manager.io + # kind: ClusterIssuer + # name: ca-issuer + # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. + # If not specified, a CA issuer will be created. + certManagerIssuerRef: {} + # -- base64 encoded PEM values for the ExternalWorkload CA certificate and private key. + ca: + # -- Optional CA cert. If it is provided, it will be used by the 'cronJob' method to + # generate all other certificates. Otherwise, an ephemeral CA is generated. + cert: "" + # -- Optional CA private key. If it is provided, it will be used by the 'cronJob' method to + # generate all other certificates. Otherwise, an ephemeral CA is generated. + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. + # Used if 'auto' is not enabled. + server: + cert: "" + key: "" + # -- Extra DNS names added to certificate when it's auto generated + extraDnsNames: [] + # -- Extra IP addresses added to certificate when it's auto generated + extraIpAddresses: [] + # -- base64 encoded PEM values for the clustermesh-apiserver admin certificate and private key. + # Used if 'auto' is not enabled. + admin: + cert: "" + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver client certificate and private key. + # Used if 'auto' is not enabled. + client: + cert: "" + key: "" + # -- base64 encoded PEM values for the clustermesh-apiserver remote cluster certificate and private key. + # Used if 'auto' is not enabled. + remote: + cert: "" + key: "" + # -- Configure external workloads support externalWorkloads: # -- Enable support for external workloads, such as VMs (false by default). @@ -1845,14 +1849,6 @@ cgroup: # cgroup2 filesystem at the specified `cgroup.hostRoot` volume, and then the # volume will be mounted inside the cilium agent pod at the same path. enabled: true - # -- Init Container Cgroup Automount resource limits & requests - resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi # -- Configure cgroup root where cgroup2 filesystem is mounted on the host (see also: `cgroup.autoMount`) hostRoot: /run/cilium/cgroupv2 @@ -1886,8 +1882,3 @@ dnsProxy: proxyPort: 0 # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. proxyResponseMaxDelay: 100ms - -# -- SCTP Configuration Values -sctp: - # -- Enable SCTP support. NOTE: Currently, SCTP support does not support rewriting ports or multihoming. - enabled: false