Runbook: Filesystem full - blocks, inodes, and deleted-but-open files

df -h
df -i

# Local filesystems only - skip NFS, which can hang when the server is down
df -hlT -x tmpfs -x devtmpfs

# Files with link count < 1: unlinked but still held open
sudo lsof +L1

# Narrow to the affected filesystem and sort by size
sudo lsof +L1 /var | awk 'NR==1 || $7 > 100000000'

# The gap that confirms it
sudo du -sxh /var
df -h /var

# Preferred: tell the daemon to reopen its logs (no restart, no lost data)
sudo systemctl reload rsyslog
sudo systemctl kill -s HUP nginx.service

# If the daemon has no reopen path, truncate the descriptor in place.
# This frees the blocks immediately and keeps the process running.
# 1234 is the PID, 5 is the fd number, both from the lsof output.
sudo truncate -s 0 /proc/1234/fd/5

# Last resort: restart the process holding it
sudo systemctl restart <unit>

# Top-level offenders, bounded to this filesystem
sudo du -xh --max-depth=1 / 2>/dev/null | sort -rh | head -20

# Then descend into the winner
sudo du -xh --max-depth=1 /var 2>/dev/null | sort -rh | head -20

# The 20 largest individual files on this filesystem
sudo find / -xdev -type f -printf '%s %p\n' 2>/dev/null \
| sort -rn | head -20 | numfmt --field=1 --to=iec

# 1. Journal - bounded, reversible only from backup
sudo journalctl --disk-usage
sudo journalctl --vacuum-size=500M
sudo journalctl --vacuum-time=7d

# 2. Package caches - always safe to rebuild
sudo apt-get clean          # Debian/Ubuntu
sudo dnf clean all          # RHEL family

# 3. Rotated logs older than the retention policy
sudo find /var/log -type f -name '*.gz' -mtime +14 -print   # LOOK FIRST
sudo find /var/log -type f -name '*.gz' -mtime +14 -delete

# 4. Old kernels (Debian) - keep at least the running one and one spare
uname -r
sudo apt-get --purge autoremove

df -i

# Which top-level directory holds the file count
sudo du -x --inodes --max-depth=2 / 2>/dev/null | sort -rn | head -20

# Portable equivalent if du lacks --inodes
for d in /var/*; do printf '%8s %s\n' \
"$(sudo find "$d" -xdev -printf . 2>/dev/null | wc -c)" "$d"; done | sort -rn | head

# Look first
sudo find /var/lib/php/sessions -type f -mtime +2 | wc -l

# Then delete, in batches so the shell does not choke on argument length
sudo find /var/lib/php/sessions -type f -mtime +2 -delete

# Mail spool - inspect before touching; queued mail is data
sudo postqueue -p | tail -1

sudo tune2fs -l /dev/mapper/vg0-var | grep -E 'Reserved block count|Block count|Free blocks'

# Only on a data filesystem, never on / - and reverse it afterwards
sudo tune2fs -m 1 /dev/mapper/vg0-var

df -h /var
df -i /var
sudo lsof +L1 /var | wc -l

# A real write test as the service account
sudo -u www-data sh -c 'echo ok > /var/www/.writetest && rm /var/www/.writetest' \
&& echo 'write OK'

# /etc/systemd/journald.conf
# SystemMaxUse=1G
# SystemKeepFree=2G
sudo systemctl restart systemd-journald
journalctl --disk-usage