Cluster, Nodes & Apps Troubleshooting

kubectl cluster-info # Confirms the API server endpoint is reachable.
kubectl get nodes -o wide # Checks Ready status, versions, internal IPs, and node OS.
kubectl get pods -A --sort-by=.metadata.namespace # Finds failing system and app Pods.
kubectl get events -A --sort-by=.lastTimestamp | tail -50 # Recent warnings usually point at the failure.
kubectl top nodes # Requires metrics-server; shows CPU/memory pressure.
kubectl top pods -A --sort-by=memory # Finds noisy workloads.

NODE=worker-1
kubectl describe node "$NODE" # Conditions, taints, allocatable resources, recent node events.
kubectl get pods -A --field-selector spec.nodeName="$NODE" # Workloads placed on this node.

# If you have node shell access:
sudo systemctl status kubelet --no-pager # kubelet health.
sudo journalctl -u kubelet -n 200 --no-pager # kubelet errors: CNI, image pull, cert, eviction.
sudo crictl ps -a # Containers known to the runtime.
sudo crictl logs <container-id> # Logs when kubectl logs is not enough.

POD=my-pod
NS=app
kubectl describe pod "$POD" -n "$NS"
kubectl logs "$POD" -n "$NS" --all-containers=true --tail=120
kubectl logs "$POD" -n "$NS" --all-containers=true --previous
kubectl get pod "$POD" -n "$NS" -o jsonpath='{.spec.containers[*].name}{"\n"}'

NODE=worker-1
kubectl describe node "$NODE"
kubectl get events --field-selector involvedObject.kind=Node,involvedObject.name="$NODE"

kubectl describe pod crashing-pod -n app | grep -A3 -i oom
kubectl top pods -A --containers --sort-by=memory | tail -40

kubectl describe pod my-pod -n app | grep -A8 Events
kubectl get secret registry-creds -n app -o yaml # compare type dockerconfigjson entries

kubectl get pods -n app -o json | jq '..|.reason?|select(.!=null)'
kubectl get events -n app --field-selector reason=Evicted --sort-by=.lastTimestamp

kubectl describe deploy api -n app
kubectl rollout status deploy/api -n app --timeout=5m | cat
kubectl get rs -n app -o wide --show-labels
kubectl get hpa -n app

kubectl describe resourcequota -n app
kubectl describe limitrange -n app
kubectl get pods -n app --field-selector=status.phase=Failed

kubectl auth can-i watch pods --as=system:serviceaccount:app:api-sa -n app
kubectl describe rolebinding -n app
kubectl get role,rolebinding -n app -o wide

kubectl get pvc,pv -n data
kubectl get volumeattachment

kubectl logs deploy/web-api -n app --tail=100 # Logs from one current Pod behind the Deployment.
kubectl logs deploy/web-api -n app --all-containers=true --tail=100 # Include sidecars.
kubectl logs pod/web-api-abc123 -n app -c api --previous # Previous crashed container logs.
kubectl logs -n app -l app=web-api --since=30m --prefix=true # Aggregate by label for recent logs.
kubectl describe pod web-api-abc123 -n app # Events explain scheduling, pull, probe, and mount failures.

kubectl get pods -n kube-system # Managed clusters expose component Pods differently.
kubectl get componentstatuses # Deprecated, but may exist in older clusters.
kubectl logs -n kube-system kube-apiserver-control-plane --tail=100 # kubeadm static Pod example.
kubectl logs -n kube-system kube-controller-manager-control-plane --tail=100
kubectl logs -n kube-system kube-scheduler-control-plane --tail=100
kubectl get --raw='/readyz?verbose' # API server readiness details if your RBAC allows it.

kubectl get cronjobs -A
kubectl get jobs -A --sort-by=.status.startTime | tail

kubectl debug pod/api-XXXX -n app -it --image=busybox --target=api
kubectl debug node/worker-1 -it --image=ubuntu -- chroot /host bash

kubectl cluster-info dump --namespaces kube-system,app --output-directory=/tmp/k8s-dump
tar czf bundle.tgz /tmp/k8s-dump

# Recent warnings scoped to workloads that match a label selector.
kubectl get events -A \
  --field-selector type=Warning \
  --sort-by=.lastTimestamp | tail -n 80

# Follow a single object's timeline (replace kind/name/ns).
kubectl get events --namespace app \
  --field-selector involvedObject.kind=Pod,involvedObject.name=api-aaaa -w

# Group reasons to see dominating failure modes quickly.
kubectl get events -A --sort-by=.lastTimestamp -o yaml | grep ' reason:' | sort | uniq -c | tail

# Correlate DaemonSet turbulence with node maintenance.
kubectl get events -n kube-system --sort-by=.lastTimestamp | grep -i -E 'cni|cilium|calico|flannel|kube-proxy'

kubectl run mtu-probe --rm -it --image=nicolaka/netshoot -- \
  ping -s 8972 -c3 172.31.255.253 # ICMP payload probing for PMTUD black holes