Ensure That The --eventRecordQPS Argument Is Set Appropriately

#!/usr/bin/env bash
#
# Remediation: Ensure kubelet eventRecordQPS is set appropriately on every worker node
# Scope: Run on EACH WORKER NODE (as root)
#
# Behavior:
# - Ensures /var/lib/kubelet/config.yaml exists
# - Sets eventRecordQPS to a desired value (default: 0)
# - Idempotent: safe to re-run
# - Restarts kubelet if and only if a change is made
# - Verifies kubelet is running with the expected setting
#

set -euo pipefail

DESIRED_QPS="0"             # adjust if you want a non-zero rate
KUBELET_CONFIG="/var/lib/kubelet/config.yaml"
SYSTEMD_DROPIN="/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"

if ! command -v systemctl >/dev/null 2>&1; then
  echo "systemctl not found; this script assumes systemd-managed kubelet. Exiting." >&2
  exit 1
fi

if [ ! -f "$KUBELET_CONFIG" ]; then
  echo "Kubelet config file $KUBELET_CONFIG not found." >&2
  echo "This node may be using only command-line flags; manually set --eventRecordQPS via $SYSTEMD_DROPIN." >&2
  exit 1
fi

echo "Updating $KUBELET_CONFIG to set eventRecordQPS: $DESIRED_QPS"

tmp_cfg="$(mktemp)"
cp "$KUBELET_CONFIG" "$tmp_cfg"

# If eventRecordQPS key exists (any indent), replace its value.
if grep -Eq '^[[:space:]]*eventRecordQPS:' "$tmp_cfg"; then
  sed -E -i "s/^([[:space:]]*eventRecordQPS:).*/\1 ${DESIRED_QPS}/" "$tmp_cfg"
else
  # Append under a reasonable location; if apiVersion/kind exist, append at end of file.
  {
    echo ""
    echo "eventRecordQPS: ${DESIRED_QPS}"
  } >> "$tmp_cfg"
fi

if cmp -s "$tmp_cfg" "$KUBELET_CONFIG"; then
  echo "No change needed in $KUBELET_CONFIG (eventRecordQPS already ${DESIRED_QPS})."
  rm -f "$tmp_cfg"
else
  echo "Applying updated kubelet configuration."
  cp "$tmp_cfg" "$KUBELET_CONFIG"
  rm -f "$tmp_cfg"

  echo "Reloading systemd and restarting kubelet (this will briefly disrupt workloads on this node)."
  systemctl daemon-reload
  systemctl restart kubelet.service
fi

echo "Verification: checking kubelet process and effective configuration."

# Show kubelet process arguments
/bin/ps -fC kubelet || {
  echo "kubelet process not found after restart; investigate systemctl status kubelet." >&2
  exit 1
}

# Verify setting in the config file
echo "Configured eventRecordQPS in $KUBELET_CONFIG:"
grep -n 'eventRecordQPS' "$KUBELET_CONFIG" || echo "eventRecordQPS not found in $KUBELET_CONFIG"

# If kubelet exposes /configz, query it (optional, best-effort)
if command -v curl >/dev/null 2>&1 && ss -lnt | grep -q ':10250'; then
  echo "Attempting to verify effective kubelet config via /configz (may fail if auth is required)..."
  curl -sSk https://127.0.0.1:10250/configz 2>/dev/null | grep -n 'eventRecordQPS' || \
    echo "Could not verify via /configz; check kubelet logs if needed."
else
  echo "Skipping /configz verification (curl not available or kubelet HTTPS port not listening)."
fi

echo "Done on this worker node."