Kubelet Pod PIDs Limit Is Set

#!/usr/bin/env bash
#
# Automation: Ensure kubelet PodPidsLimit is set on every worker node
#
# Usage:
#   1) Run from an admin machine that has SSH access and sudo on every worker.
#   2) Provide a file with one worker node (hostname or IP) per line:
#        ./set_pod_pids_limit.sh workers.txt
#
# This script:
#   - Connects via SSH to each worker node
#   - Ensures /var/lib/kubelet/config.yaml exists and sets PodPidsLimit
#   - Leaves any existing non-zero PodPidsLimit unchanged (idempotent)
#   - Restarts kubelet if a change was made
#   - Verifies PodPidsLimit via kubelet --version && grep in config
#
# NOTE: This acts on host-level config; it cannot be run via kubectl alone.

set -euo pipefail

WORKER_LIST_FILE="${1:-}"

if [[ -z "$WORKER_LIST_FILE" || ! -f "$WORKER_LIST_FILE" ]]; then
  echo "Usage: $0 <worker-node-list-file>"
  echo "  <worker-node-list-file> must contain one worker hostname/IP per line."
  exit 1
fi

# Desired PodPidsLimit value. Adjust to your operational requirement.
DESIRED_POD_PIDS_LIMIT="4096"

SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10"

run_on_node() {
  local node="$1"
  shift
  ssh $SSH_OPTS "$node" "$@"
}

while IFS= read -r NODE || [[ -n "$NODE" ]]; do
  [[ -z "$NODE" ]] && continue
  echo "==== Processing worker node: $NODE ===="

  # 1) Ensure kubelet config exists and PodPidsLimit is set/updated (runs on every worker node)
  run_on_node "$NODE" "sudo bash -s" <<'EOF'
set -euo pipefail

CONFIG_FILE="/var/lib/kubelet/config.yaml"
DESIRED_POD_PIDS_LIMIT="4096"
CHANGED="false"

if [[ ! -f \"\$CONFIG_FILE\" ]]; then
  echo \"[WARN] \${CONFIG_FILE} does not exist. Cannot set PodPidsLimit automatically.\"
  exit 0
fi

# If PodPidsLimit is already present and non-zero, leave it as-is (idempotent, respects existing tuning)
EXISTING_VAL=\$(sudo awk '/^[[:space:]]*PodPidsLimit:/ {print $2}' \"\$CONFIG_FILE\" | head -n1 || true)
if [[ -n \"\$EXISTING_VAL\" && \"\$EXISTING_VAL\" != \"0\" ]]; then
  echo \"[INFO] PodPidsLimit already set to \${EXISTING_VAL} in \${CONFIG_FILE}; leaving unchanged.\"
else
  echo \"[INFO] Setting PodPidsLimit to \${DESIRED_POD_PIDS_LIMIT} in \${CONFIG_FILE}.\"
  TMP_FILE=\"\${CONFIG_FILE}.tmp.cis_podpids\"

  # If key exists but is 0 or blank, replace it; otherwise append at end
  if grep -qE '^[[:space:]]*PodPidsLimit:' \"\$CONFIG_FILE\"; then
    sudo sed -E \"s|^[[:space:]]*PodPidsLimit:.*|PodPidsLimit: \${DESIRED_POD_PIDS_LIMIT}|\" \"\$CONFIG_FILE\" > \"\$TMP_FILE\"
  else
    sudo cp \"\$CONFIG_FILE\" \"\$TMP_FILE\"
    echo \"PodPidsLimit: \${DESIRED_POD_PIDS_LIMIT}\" | sudo tee -a \"\$TMP_FILE\" >/dev/null
  fi

  sudo mv \"\$TMP_FILE\" \"\$CONFIG_FILE\"
  CHANGED=\"true\"
fi

# 2) If we changed config, restart kubelet so it picks up the new setting
if [[ \"\$CHANGED\" == \"true\" ]]; then
  echo \"[INFO] Restarting kubelet (this will temporarily restart workloads on this node).\"
  if command -v systemctl >/dev/null 2>&1; then
    sudo systemctl restart kubelet
  elif command -v service >/dev/null 2>&1; then
    sudo service kubelet restart
  else
    echo \"[WARN] Could not find systemctl or service to restart kubelet. Please restart kubelet manually.\"
  fi
fi

# 3) Verification: show PodPidsLimit from config and kubelet process status
echo \"[VERIFY] kubelet process (ps -fC kubelet):\"
ps -fC kubelet || echo \"[WARN] kubelet not found in process list.\"

echo \"[VERIFY] PodPidsLimit in \${CONFIG_FILE}:\" 
sudo awk '/^[[:space:]]*PodPidsLimit:/ {print \"PodPidsLimit:\", $2}' \"\$CONFIG_FILE\" || true
EOF

  echo "==== Completed worker node: $NODE ===="
  echo
done < "$WORKER_LIST_FILE"

echo "All nodes processed. Re-run this script safely at any time; it is idempotent."