Kubelet Event Record QPS Set To Ensure Appropriate Event Capture

#!/usr/bin/env bash
#
# Automation: Configure kubelet eventRecordQPS on every worker node
#
# Usage:
#   1) Put this script on each worker node (e.g. /root/fix-kubelet-event-qps.sh)
#   2) Edit DESIRED_QPS below to your chosen value.
#   3) Run as root:  bash /root/fix-kubelet-event-qps.sh
#
# Idempotent: safe to re-run; it only changes eventRecordQPS if needed.

set -euo pipefail

DESIRED_QPS="50"   # <<< SET THIS TO YOUR APPROPRIATE VALUE

KUBELET_CONFIG="/var/lib/kubelet/config.yaml"
SYSTEMD_DROPIN="/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"

echo "==> Configuring kubelet eventRecordQPS on worker node: $(hostname)"

if [[ "$EUID" -ne 0 ]]; then
  echo "ERROR: Run this script as root." >&2
  exit 1
fi

restart_needed=0

backup_file() {
  local f="$1"
  if [[ -f "$f" ]]; then
    if [[ ! -f "${f}.cis-backup" ]]; then
      cp -p "$f" "${f}.cis-backup"
      echo "Backed up $f to ${f}.cis-backup"
    fi
  fi
}

############################################################
# 1. Prefer kubelet config file at /var/lib/kubelet/config.yaml
############################################################
if [[ -f "$KUBELET_CONFIG" ]]; then
  echo "Detected kubelet config file: $KUBELET_CONFIG"

  backup_file "$KUBELET_CONFIG"

  current_qps=""
  if grep -qE '^[[:space:]]*eventRecordQPS:' "$KUBELET_CONFIG"; then
    current_qps="$(grep -E '^[[:space:]]*eventRecordQPS:' "$KUBELET_CONFIG" | head -n1 | awk '{print $2}')"
  fi

  if [[ "$current_qps" == "$DESIRED_QPS" ]]; then
    echo "eventRecordQPS already set to $DESIRED_QPS in $KUBELET_CONFIG; no change needed."
  else
    if grep -qE '^[[:space:]]*eventRecordQPS:' "$KUBELET_CONFIG"; then
      echo "Updating existing eventRecordQPS from '${current_qps:-unset}' to $DESIRED_QPS in $KUBELET_CONFIG"
      # Replace the line in place
      sed -i -E "s/^[[:space:]]*eventRecordQPS:.*/eventRecordQPS: ${DESIRED_QPS}/" "$KUBELET_CONFIG"
    else
      echo "Adding eventRecordQPS: $DESIRED_QPS to $KUBELET_CONFIG"
      # Append in a simple, valid way at EOF
      printf "\n# CIS 4.2.8: ensure appropriate event capture\neventRecordQPS: %s\n" "$DESIRED_QPS" >> "$KUBELET_CONFIG"
    fi
    restart_needed=1
  fi

else
  ##########################################################
  # 2. Fall back to kubelet systemd drop-in (command-line args)
  ##########################################################
  echo "No kubelet config file at $KUBELET_CONFIG; updating systemd unit arguments."

  if [[ ! -f "$SYSTEMD_DROPIN" ]]; then
    echo "ERROR: $SYSTEMD_DROPIN not found. Manual review of kubelet start method required." >&2
    echo "       See Manual Steps section for guidance." >&2
    exit 1
  fi

  backup_file "$SYSTEMD_DROPIN"

  # We will ensure --event-qps is present or updated in KUBELET_SYSTEM_PODS_ARGS.
  # NOTE: eventRecordQPS in config.yaml corresponds to --event-qps on CLI.
  if grep -q 'KUBELET_SYSTEM_PODS_ARGS' "$SYSTEMD_DROPIN"; then
    # Extract current KUBELET_SYSTEM_PODS_ARGS line(s)
    if grep -q -- '--event-qps' "$SYSTEMD_DROPIN"; then
      echo "Updating existing --event-qps to $DESIRED_QPS in $SYSTEMD_DROPIN"
      sed -i -E "s/(--event-qps=)[0-9]+/\1${DESIRED_QPS}/" "$SYSTEMD_DROPIN"
    else
      echo "Adding --event-qps=$DESIRED_QPS to KUBELET_SYSTEM_PODS_ARGS in $SYSTEMD_DROPIN"
      sed -i -E 's/^(Environment=.*KUBELET_SYSTEM_PODS_ARGS="[^"]*)(".*)$/\1 --event-qps='"$DESIRED_QPS"'\2/' "$SYSTEMD_DROPIN" || {
        echo "WARN: Could not auto-edit KUBELET_SYSTEM_PODS_ARGS. Manual edit required in $SYSTEMD_DROPIN." >&2
        exit 1
      }
    fi
  else
    echo "ERROR: KUBELET_SYSTEM_PODS_ARGS not found in $SYSTEMD_DROPIN. Manual configuration required." >&2
    exit 1
  fi

  restart_needed=1
fi

############################################################
# 3. Restart kubelet if needed
############################################################
if [[ "$restart_needed" -eq 1 ]]; then
  echo "Restarting kubelet to apply changes..."
  systemctl daemon-reload
  systemctl restart kubelet.service
  sleep 5
else
  echo "No kubelet restart required."
fi

############################################################
# 4. Verification (adapted from audit command)
############################################################
echo "==> Verification: kubelet process flags and effective event QPS"

echo "- kubelet process (ps output):"
/bin/ps -fC kubelet || true

if command -v journalctl >/dev/null 2>&1; then
  echo "- Last kubelet start command from systemd journal (if available):"
  journalctl -u kubelet -n 20 --no-pager | grep -E 'kubelet( |$)' || true
fi

if [[ -f "$KUBELET_CONFIG" ]]; then
  echo "- Effective eventRecordQPS in $KUBELET_CONFIG:"
  grep -E '^[[:space:]]*eventRecordQPS:' "$KUBELET_CONFIG" || echo "eventRecordQPS not found in $KUBELET_CONFIG"
else
  echo "- Config file $KUBELET_CONFIG not present; relying on CLI flag --event-qps (see ps output above)."
fi

echo "Done."