Linux Admin : Resources for Linux SysAdmin

$ netstat -an | awk '/tcp/ {print $6}' | sort | uniq -c
     92 ESTABLISHED
      1 FIN_WAIT2
     13 LISTEN
   7979 TIME_WAIT

$ grep processor /proc/cpuinfo | wc -l
4

$ grep -r keep.*alive /etc/
/etc/ufw/sysctl.conf:#net/ipv4/tcp_keepalive_intvl=1800
/etc/nginx/nginx.conf:    keepalive_timeout     5 5;

$ free -m
             total       used       free     shared    buffers     cached
Mem:         14980       1402      13577          0        113        831
-/+ buffers/cache:        458      14521
Swap:

 $ uptime
 02:17:14 up 18:20,  1 user,  load average: 2.77, 2.39, 2.21

$ dstat
You did not select any stats, using -cdngy by default.
----total-cpu-usage---- -dsk/total- -net/total- ---paging-- ---system--
usr sys idl wai hiq siq| read  writ| recv  send|  in   out | int   csw 
 46   2  51   0   0   1|4432B   10k|   0     0 |   0     0 |4346  1870 
 51   3  46   0   0   1|   0    56k|2679k  191k|   0     0 |5130  2318 
 40   3  57   0   0   1|   0     0 |1566k  211k|   0     0 |4825  2141 
 46   2  52   0   0   0|   0     0 |1311k  136k|   0     0 |4606  1997 
 27   2  71   0   0   1|   0     0 | 234k  144k|   0     0 |3278  1693 
 23   2  76   0   0   0|   0   152k| 286k  123k|   0     0 |3094  1683 
 23   2  74   1   0   0|   0    28k| 146k  131k|   0     0 |3103  1576 
 30   2  67   0   0   1|   0     0 | 668k  177k|   0     0 |4023  2020 
 31   2  67   0   0   0|   0     0 | 326k  197k|   0     0 |4330  2273 
 23   2  75   0   0   0|   0     0 | 339k  121k|   0     0 |3020  1428 
 30   2  67   0   0   0|   0     0 |1930k  180k|   0     0 |4487  1947 
 38   3  59   0   0   1|   0    12k| 340k  155k|   0     0 |4403  1994 
 29   2  68   0   0   1|   0     0 | 187k  117k|   0     0 |3449  1729 
 35   4  59   2   0   1|   0     0 | 478k  314k|   0     0 |4415  2338 
 49   4  46   0   0   1|   0     0 |2263k  210k|   0     0 |5153  2289 
 49   2  49   0   0   1|   0    60k|2921k  118k|   0     0 |5063  1532 
 52   2  46   0   0   0|   0    24k|2823k  161k|   0     0 |4842  1740 
 72   2  26   0   0   1|   0     0 |2361k  141k|   0     0 |4715  1600 
 62   3  34   0   0   1|   0     0 |3414k  147k|   0     0 |5487  1863 
 48   2  49   0   0   1|   0     0 |1501k  117k|   0     0 |4211  1722 
 49   4  46   0   0   1|   0     0 |4675k  207k|   0     0 |5660  2286 
 46   2  51   0   0   0|   0     0 | 182k  169k|   0     0 |4178  2373 
 43   1  55   0   0   0|   0    12k| 172k  168k|   0     0 |3407  1843 
 29   2  69   0   0   0|   0     0 | 376k  175k|   0     0 |4013  2216 
 29   2  68   0   0   0|   0     0 | 613k  238k|   0     0 |4885  2628 
 25   2  72   0   0   1|   0     0 | 272k  215k|   0     0 |5105  3126 
 33   3  63   0   0   1|   0     0 |3692k  228k|   0     0 |5978  2397

$ cat /etc/sysctl.conf
# Avoid a smurf attack
net.ipv4.icmp_echo_ignore_broadcasts = 1

# Turn on protection for bad icmp error messages
net.ipv4.icmp_ignore_bogus_error_responses = 1

# Turn on syncookies for SYN flood attack protection
net.ipv4.tcp_syncookies = 1

# Turn on and log spoofed, source routed, and redirect packets
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 1

# No source routed packets here
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# Turn on reverse path filtering
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1

# Make sure no one can alter the routing tables
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0

# Don't act as a router
net.ipv4.ip_forward = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0


# Turn on execshild
kernel.exec-shield = 1
kernel.randomize_va_space = 1

# Tuen IPv6
net.ipv6.conf.default.router_solicitations = 0
net.ipv6.conf.default.accept_ra_rtr_pref = 0
net.ipv6.conf.default.accept_ra_pinfo = 0
net.ipv6.conf.default.accept_ra_defrtr = 0
net.ipv6.conf.default.autoconf = 0
net.ipv6.conf.default.dad_transmits = 0
net.ipv6.conf.default.max_addresses = 1

# Optimization for port usefor LBs
# Increase system file descriptor limit
fs.file-max = 65535

# Allow for more PIDs (to reduce rollover problems); may break some programs 32768
kernel.pid_max = 65536

# Increase system IP port limits
net.ipv4.ip_local_port_range = 2000 65000

# Increase TCP max buffer size setable using setsockopt()
net.ipv4.tcp_rmem = 4096 87380 8388608
net.ipv4.tcp_wmem = 4096 87380 8388608

# Increase Linux auto tuning TCP buffer limits
# min, default, and max number of bytes to use
# set max to at least 4MB, or higher if you use very high BDP paths
# Tcp Windows etc
net.core.rmem_max = 8388608
net.core.wmem_max = 8388608
net.core.netdev_max_backlog = 5000
net.ipv4.tcp_window_scaling = 1

$ 2>/dev/null sysctl -a | grep \
    'tcp_syncookies\|tcp_max_syn_backlog\|tcp_synack_retries'
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 2048

Question: What might cause high number of TIME_WAIT?

Answer:

# This setting allows sockets reusing.
$ echo 'net.ipv4.tcp_tw_recycle = 1' >> /etc/sysctl.conf
$ sysctl -p /etc/sysctl.conf