casino/ops/tune-kernel.sh

#!/usr/bin/env bash
#
# Kernel and limit tuning for a machine running the arcade.
#
# Every value here was chosen against a measured bottleneck, not copied from a
# listicle. The measurements are in docs/SCALING.md: one instance held 25,000
# concurrent websockets at 586MB, and the limits below are what let it.
#
#   sudo ./ops/tune-kernel.sh apply    write settings and reload
#   ./ops/tune-kernel.sh show          print current values
#   ./ops/tune-kernel.sh check         report values that are below target
#
set -euo pipefail

CONF=/etc/sysctl.d/99-quantum-arcade.conf
LIMITS=/etc/security/limits.d/99-quantum-arcade.conf

log() { printf '%s  %s\n' "$(date -u +%H:%M:%SZ)" "$*"; }

write_sysctl() {
  cat > "$CONF" <<'EOF'
# Quantum Arcade — kernel tuning.
#
# Each setting exists because a specific limit was hit. Do not raise these
# further without a measurement showing the current value is the constraint:
# oversized buffers waste memory that the connection count needs.

# --- connection capacity ---

# Every player holds one websocket, and each socket is a file descriptor.
# 25k connections plus database pool, logs, and headroom.
fs.file-max = 2097152
fs.nr_open = 2097152

# The listen backlog. A crowd arriving at once — a party where everyone opens
# the page after an announcement — bursts far above steady state. The default
# of 4096 drops connections during that burst; they appear to the player as a
# page that will not load.
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.core.netdev_max_backlog = 16384

# --- ephemeral ports ---

# An instance forwarding bets to the game leader opens outbound connections.
# The default range of ~28k ports is exhausted well before the connection
# ceiling is, and the failure looks like random forwarding errors.
net.ipv4.ip_local_port_range = 10240 65535

# Reuse sockets in TIME_WAIT for new outbound connections. Safe for the
# originating side; do not enable tcp_tw_recycle, which is removed in modern
# kernels and broke NAT when it existed.
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 15

# --- websocket idleness ---

# Phones sleep, lose signal, and leave sockets that look alive. Without
# keepalives those accumulate as connections the server is holding buffers for
# and will never hear from again. Detect within ~5 minutes rather than 2 hours.
net.ipv4.tcp_keepalive_time = 240
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 6

# --- buffers ---

# Frames are ~1.8KB and sent a few times a second, so per-socket buffers can
# stay modest. At 25k connections, every extra kilobyte of default buffer is
# another 25MB of RAM that would be better spent on connections.
net.ipv4.tcp_rmem = 4096 87380 6291456
net.ipv4.tcp_wmem = 4096 65536 6291456
net.core.rmem_max = 12582912
net.core.wmem_max = 12582912

# Accept a burst of small writes rather than coalescing them; the frames are
# already batched at 5Hz in the application.
net.ipv4.tcp_slow_start_after_idle = 0

# --- database host ---

# PostgreSQL manages its own caching. Aggressive swapping of a database's
# working set is far worse than reclaiming page cache.
vm.swappiness = 10
vm.overcommit_memory = 1

# Flush dirty pages steadily rather than in large stalls, which show up as
# multi-second latency spikes during settlement.
vm.dirty_background_ratio = 5
vm.dirty_ratio = 15

# --- conntrack ---

# A box behind a firewall tracking 25k connections needs a table that fits
# them, or new connections are dropped with no useful error.
net.netfilter.nf_conntrack_max = 262144
EOF
  log "wrote $CONF"
}

write_limits() {
  cat > "$LIMITS" <<'EOF'
# Quantum Arcade — process limits.
#
# fs.file-max raises the system ceiling; this raises the per-process one.
# Without both, the process hits its own limit long before the kernel's and
# refuses connections while the machine looks idle.
*    soft nofile 1048576
*    hard nofile 1048576
root soft nofile 1048576
root hard nofile 1048576
EOF
  log "wrote $LIMITS"

  # systemd ignores limits.conf for services it starts.
  mkdir -p /etc/systemd/system.conf.d
  cat > /etc/systemd/system.conf.d/99-quantum-arcade.conf <<'EOF'
[Manager]
DefaultLimitNOFILE=1048576
EOF
  log "wrote systemd DefaultLimitNOFILE"
}

cmd_apply() {
  [ "$(id -u)" -eq 0 ] || { echo "must run as root" >&2; exit 1; }
  write_sysctl
  write_limits
  sysctl --system >/dev/null
  log "settings applied; reboot or re-login for limits to take effect"
  cmd_check
}

cmd_show() {
  for k in fs.file-max net.core.somaxconn net.ipv4.ip_local_port_range \
           net.ipv4.tcp_keepalive_time vm.swappiness; do
    printf '  %-34s %s\n' "$k" "$(sysctl -n "$k" 2>/dev/null || echo 'n/a')"
  done
  printf '  %-34s %s\n' "ulimit -n (this shell)" "$(ulimit -n)"
}

# check reports what is below target, so a machine can be inspected without
# changing anything.
cmd_check() {
  local fails=0
  check_min() {
    local key="$1" want="$2" cur
    cur="$(sysctl -n "$key" 2>/dev/null | awk '{print $1}')" || cur=0
    if [ -z "$cur" ] || [ "$cur" -lt "$want" ] 2>/dev/null; then
      printf '  BELOW TARGET  %-30s %s (want >= %s)\n' "$key" "${cur:-unset}" "$want"
      fails=$((fails + 1))
    else
      printf '  ok            %-30s %s\n' "$key" "$cur"
    fi
  }

  check_min fs.file-max 1048576
  check_min net.core.somaxconn 32768
  check_min net.ipv4.tcp_max_syn_backlog 32768

  local nofile
  nofile="$(ulimit -n)"
  if [ "$nofile" -lt 65536 ]; then
    printf '  BELOW TARGET  %-30s %s (want >= 65536)\n' "ulimit -n" "$nofile"
    fails=$((fails + 1))
  else
    printf '  ok            %-30s %s\n' "ulimit -n" "$nofile"
  fi

  if [ "$fails" -gt 0 ]; then
    log "$fails setting(s) below target — run 'sudo $0 apply'"
    return 1
  fi
  log "all checked settings meet target"
}

case "${1:-show}" in
  apply) cmd_apply ;;
  show)  cmd_show ;;
  check) cmd_check ;;
  *)     echo "usage: $0 {apply|show|check}" >&2; exit 2 ;;
esac