diff --git a/kinder/ci/tools/update-workflows/templates/workflows/instance-config-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/instance-config-tasks.yaml index 6fbd94c5..34a6c7de 100644 --- a/kinder/ci/tools/update-workflows/templates/workflows/instance-config-tasks.yaml +++ b/kinder/ci/tools/update-workflows/templates/workflows/instance-config-tasks.yaml @@ -59,7 +59,7 @@ tasks: - --name={{ .vars.clusterName }} - --loglevel=debug - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} - - --kubeadm-feature-gate="NodeLocalCRISocket=true" + - --kubeadm-feature-gate="NodeLocalCRISocket=false" - --copy-certs=auto timeout: 5m - name: join @@ -84,16 +84,16 @@ tasks: set -x docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' - nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath="{.items[*].metadata.name}") for node in $nodes; do - annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations['kubeadm\.alpha\.kubernetes\.io/cri-socket']}") + annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io\/cri-socket}") if [ -z "$annotation" ]; then echo "Annotation not found on node $node" + exit 1 else echo "Annotation found on node $node: $annotation" - exit 1 fi done ' || { @@ -118,8 +118,35 @@ tasks: ${CMD} bash -c ' if [[ -f "/var/lib/kubelet/instance-config.yaml" ]] && grep -q "containerRuntimeEndpoint:" /var/lib/kubelet/instance-config.yaml; then echo "File exists and contains 'containerRuntimeEndpoint:' field." + exit 1 else echo "File does not exist or does not contain 'containerRuntimeEndpoint:' field." + fi + ' || { + echo "Command failed for $node. Exiting."; + exit 1; + } + done + + exit 0 + timeout: 5m +- name: check-flag + description: | + Check that the container-runtime-endpoint flag does not exist on each node. + cmd: /bin/bash + args: + - -c + - | + set -x + + for node in control-plane-{1..3} worker-{1..2}; do + CMD="docker exec {{ .vars.clusterName }}-$node" + + ${CMD} bash -c ' + if [[ -f "/var/lib/kubelet/kubeadm-flags.env" ]] && grep -q "container-runtime-endpoint" /var/lib/kubelet/kubeadm-flags.env; then + echo "File exists and contains 'container-runtime-endpoint' field." + else + echo "File does not contain 'container-runtime-endpoint' field." exit 1 fi ' || { @@ -157,9 +184,9 @@ tasks: ${CMD} bash -c ' if [[ -f "/var/lib/kubelet/instance-config.yaml" ]] && grep -q "containerRuntimeEndpoint:" /var/lib/kubelet/instance-config.yaml; then echo "File exists and contains 'containerRuntimeEndpoint:' field." + exit 1 else echo "File does not exist or does not contain 'containerRuntimeEndpoint:' field." - exit 1 fi ' || { echo "Command failed for $node. Exiting."; @@ -169,9 +196,9 @@ tasks: exit 0 timeout: 5m -- name: remove-config +- name: check-flag-after-upgrade description: | - Remove the kubelet instance configuration from work node. + Check that the container-runtime-endpoint flag does not exist on each node. cmd: /bin/bash args: - -c @@ -181,7 +208,14 @@ tasks: for node in control-plane-{1..3} worker-{1..2}; do CMD="docker exec {{ .vars.clusterName }}-$node" - ${CMD} bash -c 'rm -f /var/lib/kubelet/instance-config.yaml' || { + ${CMD} bash -c ' + if [[ -f "/var/lib/kubelet/kubeadm-flags.env" ]] && grep -q "container-runtime-endpoint" /var/lib/kubelet/kubeadm-flags.env; then + echo "File exists and contains 'container-runtime-endpoint' field." + else + echo "File does not contain 'container-runtime-endpoint' field." + exit 1 + fi + ' || { echo "Command failed for $node. Exiting."; exit 1; } @@ -189,9 +223,9 @@ tasks: exit 0 timeout: 5m -- name: add-cri-socket-annotation +- name: check-annotation-after-upgrade description: | - Add the cri-socket annotation to each node. + Check that the cri-socket annotation does not exist on each node. cmd: /bin/bash args: - -c @@ -199,18 +233,16 @@ tasks: set -x docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' - ANNOTATION_KEY="kubeadm.alpha.kubernetes.io/cri-socket" - ANNOTATION_VALUE="unix:///var/run/containerd/containerd.sock" - - nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath="{.items[*].metadata.name}") for node in $nodes; do - KUBECONFIG=/etc/kubernetes/admin.conf kubectl annotate node "$node" "$ANNOTATION_KEY=$ANNOTATION_VALUE" --overwrite - if [ $? -eq 0 ]; then - echo "Successfully added annotation to node $node." - else - echo "Failed to add annotation to node $node." + annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io\/cri-socket}") + + if [ -z "$annotation" ]; then + echo "Annotation not found on node $node" exit 1 + else + echo "Annotation found on node $node: $annotation" fi done ' || { @@ -220,33 +252,39 @@ tasks: exit 0 timeout: 5m -- name: check-and-add-flag + +- name: enable-feature description: | - Check and add the container-runtime-endpoint flag to each node's kubeadm-flags.env file + enable the nodeLocalCRISocket Feature gate. cmd: /bin/bash args: - -c - | set -x - for node in control-plane-{1..3} worker-{1..2}; do - CMD="docker exec {{ .vars.clusterName }}-$node" - - ${CMD} bash -c ' - FILE="/var/lib/kubelet/kubeadm-flags.env" - NEW_ARG="--container-runtime-endpoint=unix:///run/containerd/containerd.sock" - - if grep -q "container-runtime-endpoint" "$FILE"; then - echo "flag container-runtime-endpoint already exists" - exit 1 - fi - - sed -i "/^KUBELET_KUBEADM_ARGS=/ s|\"$| $NEW_ARG\"|" "$FILE" - ' || { - echo "Command failed for $node. Exiting."; - exit 1; - } - done + docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' + KUBECONFIG=/etc/kubernetes/admin.conf kubectl get configmap kubeadm-config -n kube-system -o yaml > config.yaml + sed '\''s/^\( *\)\(NodeLocalCRISocket:\) false$/\1\2 true/'\'' config.yaml > updated_config.yaml + + if grep -q "NodeLocalCRISocket: true" updated_config.yaml; then + echo "Updating ConfigMap YAML Successful." + else + echo "Updating ConfigMap YAML failed." + exit 1 + fi + + if KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f updated_config.yaml; then + echo "ConfigMap updated successfully." + else + echo "Failed to apply updated ConfigMap." + exit 1 + fi + + rm config.yaml updated_config.yaml + ' || { + echo "Command failed for $node. Exiting."; + exit 1; + } exit 0 timeout: 5m @@ -299,10 +337,10 @@ tasks: set -x docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' - nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath="{.items[*].metadata.name}") for node in $nodes; do - annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations['kubeadm\.alpha\.kubernetes\.io/cri-socket']}") + annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io\/cri-socket}") if [ -z "$annotation" ]; then echo "Annotation not found on node $node" @@ -315,7 +353,6 @@ tasks: echo "Command failed for $node. Exiting."; exit 1; } - exit 0 timeout: 5m - name: check-flag-after-second-upgrade @@ -334,6 +371,8 @@ tasks: if [[ -f "/var/lib/kubelet/kubeadm-flags.env" ]] && grep -q "container-runtime-endpoint" /var/lib/kubelet/kubeadm-flags.env; then echo "File exists and contains 'container-runtime-endpoint' field." exit 1 + else + echo "File does not contain 'container-runtime-endpoint' field." fi ' || { echo "Command failed for $node. Exiting."; diff --git a/kinder/ci/workflows/instance-config-tasks.yaml b/kinder/ci/workflows/instance-config-tasks.yaml index 24f054fc..29ee320a 100644 --- a/kinder/ci/workflows/instance-config-tasks.yaml +++ b/kinder/ci/workflows/instance-config-tasks.yaml @@ -60,7 +60,7 @@ tasks: - --name={{ .vars.clusterName }} - --loglevel=debug - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} - - --kubeadm-feature-gate="NodeLocalCRISocket=true" + - --kubeadm-feature-gate="NodeLocalCRISocket=false" - --copy-certs=auto timeout: 5m - name: join @@ -85,16 +85,16 @@ tasks: set -x docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' - nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath="{.items[*].metadata.name}") for node in $nodes; do - annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations['kubeadm\.alpha\.kubernetes\.io/cri-socket']}") + annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io\/cri-socket}") if [ -z "$annotation" ]; then echo "Annotation not found on node $node" + exit 1 else echo "Annotation found on node $node: $annotation" - exit 1 fi done ' || { @@ -119,8 +119,35 @@ tasks: ${CMD} bash -c ' if [[ -f "/var/lib/kubelet/instance-config.yaml" ]] && grep -q "containerRuntimeEndpoint:" /var/lib/kubelet/instance-config.yaml; then echo "File exists and contains 'containerRuntimeEndpoint:' field." + exit 1 else echo "File does not exist or does not contain 'containerRuntimeEndpoint:' field." + fi + ' || { + echo "Command failed for $node. Exiting."; + exit 1; + } + done + + exit 0 + timeout: 5m +- name: check-flag + description: | + Check that the container-runtime-endpoint flag does not exist on each node. + cmd: /bin/bash + args: + - -c + - | + set -x + + for node in control-plane-{1..3} worker-{1..2}; do + CMD="docker exec {{ .vars.clusterName }}-$node" + + ${CMD} bash -c ' + if [[ -f "/var/lib/kubelet/kubeadm-flags.env" ]] && grep -q "container-runtime-endpoint" /var/lib/kubelet/kubeadm-flags.env; then + echo "File exists and contains 'container-runtime-endpoint' field." + else + echo "File does not contain 'container-runtime-endpoint' field." exit 1 fi ' || { @@ -158,9 +185,9 @@ tasks: ${CMD} bash -c ' if [[ -f "/var/lib/kubelet/instance-config.yaml" ]] && grep -q "containerRuntimeEndpoint:" /var/lib/kubelet/instance-config.yaml; then echo "File exists and contains 'containerRuntimeEndpoint:' field." + exit 1 else echo "File does not exist or does not contain 'containerRuntimeEndpoint:' field." - exit 1 fi ' || { echo "Command failed for $node. Exiting."; @@ -170,9 +197,9 @@ tasks: exit 0 timeout: 5m -- name: remove-config +- name: check-flag-after-upgrade description: | - Remove the kubelet instance configuration from work node. + Check that the container-runtime-endpoint flag does not exist on each node. cmd: /bin/bash args: - -c @@ -182,7 +209,14 @@ tasks: for node in control-plane-{1..3} worker-{1..2}; do CMD="docker exec {{ .vars.clusterName }}-$node" - ${CMD} bash -c 'rm -f /var/lib/kubelet/instance-config.yaml' || { + ${CMD} bash -c ' + if [[ -f "/var/lib/kubelet/kubeadm-flags.env" ]] && grep -q "container-runtime-endpoint" /var/lib/kubelet/kubeadm-flags.env; then + echo "File exists and contains 'container-runtime-endpoint' field." + else + echo "File does not contain 'container-runtime-endpoint' field." + exit 1 + fi + ' || { echo "Command failed for $node. Exiting."; exit 1; } @@ -190,9 +224,9 @@ tasks: exit 0 timeout: 5m -- name: add-cri-socket-annotation +- name: check-annotation-after-upgrade description: | - Add the cri-socket annotation to each node. + Check that the cri-socket annotation does not exist on each node. cmd: /bin/bash args: - -c @@ -200,18 +234,16 @@ tasks: set -x docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' - ANNOTATION_KEY="kubeadm.alpha.kubernetes.io/cri-socket" - ANNOTATION_VALUE="unix:///var/run/containerd/containerd.sock" - - nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath="{.items[*].metadata.name}") for node in $nodes; do - KUBECONFIG=/etc/kubernetes/admin.conf kubectl annotate node "$node" "$ANNOTATION_KEY=$ANNOTATION_VALUE" --overwrite - if [ $? -eq 0 ]; then - echo "Successfully added annotation to node $node." - else - echo "Failed to add annotation to node $node." + annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io\/cri-socket}") + + if [ -z "$annotation" ]; then + echo "Annotation not found on node $node" exit 1 + else + echo "Annotation found on node $node: $annotation" fi done ' || { @@ -221,33 +253,39 @@ tasks: exit 0 timeout: 5m -- name: check-and-add-flag + +- name: enable-feature description: | - Check and add the container-runtime-endpoint flag to each node's kubeadm-flags.env file + enable the nodeLocalCRISocket Feature gate. cmd: /bin/bash args: - -c - | set -x - for node in control-plane-{1..3} worker-{1..2}; do - CMD="docker exec {{ .vars.clusterName }}-$node" - - ${CMD} bash -c ' - FILE="/var/lib/kubelet/kubeadm-flags.env" - NEW_ARG="--container-runtime-endpoint=unix:///run/containerd/containerd.sock" - - if grep -q "container-runtime-endpoint" "$FILE"; then - echo "flag container-runtime-endpoint already exists" - exit 1 - fi - - sed -i "/^KUBELET_KUBEADM_ARGS=/ s|\"$| $NEW_ARG\"|" "$FILE" - ' || { - echo "Command failed for $node. Exiting."; - exit 1; - } - done + docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' + KUBECONFIG=/etc/kubernetes/admin.conf kubectl get configmap kubeadm-config -n kube-system -o yaml > config.yaml + sed '\''s/^\( *\)\(NodeLocalCRISocket:\) false$/\1\2 true/'\'' config.yaml > updated_config.yaml + + if grep -q "NodeLocalCRISocket: true" updated_config.yaml; then + echo "Updating ConfigMap YAML Successful." + else + echo "Updating ConfigMap YAML failed." + exit 1 + fi + + if KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f updated_config.yaml; then + echo "ConfigMap updated successfully." + else + echo "Failed to apply updated ConfigMap." + exit 1 + fi + + rm config.yaml updated_config.yaml + ' || { + echo "Command failed for $node. Exiting."; + exit 1; + } exit 0 timeout: 5m @@ -300,10 +338,10 @@ tasks: set -x docker exec {{ .vars.clusterName }}-control-plane-1 bash -c ' - nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + nodes=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get nodes -o jsonpath="{.items[*].metadata.name}") for node in $nodes; do - annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations['kubeadm\.alpha\.kubernetes\.io/cri-socket']}") + annotation=$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get node "$node" -o jsonpath="{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io\/cri-socket}") if [ -z "$annotation" ]; then echo "Annotation not found on node $node" @@ -316,7 +354,6 @@ tasks: echo "Command failed for $node. Exiting."; exit 1; } - exit 0 timeout: 5m - name: check-flag-after-second-upgrade @@ -335,6 +372,8 @@ tasks: if [[ -f "/var/lib/kubelet/kubeadm-flags.env" ]] && grep -q "container-runtime-endpoint" /var/lib/kubelet/kubeadm-flags.env; then echo "File exists and contains 'container-runtime-endpoint' field." exit 1 + else + echo "File does not contain 'container-runtime-endpoint' field." fi ' || { echo "Command failed for $node. Exiting.";