#!/usr/bin/env python
import os
import datetime
import psutil
 
# This script will dump the memory state and write it to mem.log
# if more than 2880 entries are in the log file, it will start removing 
# old ones in order to keep the log size down.
#
# To use this script first install psutil: sudo pip install psutil
# then run: crontab -e
# add this line to the bottom(adjust the path to where the script is): 
# * * * * * cd /home/madmaze/trash; /usr/bin/python /home/madmaze/trash/memoryLogger.py
#
# This will write add a new entry to mem.log every minute and keep 48hrs of records
 
def getUsage():
    m=psutil.virtual_memory()
    s=psutil.swap_memory()
    memkeys=["total", "available", "percent", "used", "free", "active", "inactive", "buffers", "cached"]
    swapkeys=["total", "used", "free", "perc","sin","sout"]
    info={}
    # MEM
    for n,k in enumerate(m):
        if memkeys[n] not in ["percent","active","inactive"]:
            info["mem_"+memkeys[n]]=str(k/1024/1024)+" MB"
 
    # SWAP
    for n,k in enumerate(s):
        if n < 3:
            info["swap_"+memkeys[n]]=str(k/1024/1024)+" MB"
    
    return info
 
 
logfile="mem.log"
f_in=open(logfile,"r")
lines=f_in.readlines()
f_in.close()
 
totalLen=len(lines)
maxlen=2*24*60	# 2days * 24hr * 60m
 
if totalLen > maxlen+100:
    # Crop file to length
    f_out=open(logfile,"w")
    for n,l in enumerate(lines):
        if (totalLen-n)<maxlen:
            f_out.write(l)
else:
    # just append
    f_out=open(logfile,"a")
 
# Actually append the next line of data
f_out.write(str(datetime.datetime.now())+" "+str(getUsage())+"\n")
f_out.close()

Github Gist: https://gist.github.com/madmaze/560cbcf0392aab824820

Judging by the recorded memory usage pattern, it seems that at 3AM EST a series of database intensive cron jobs kicked off at the same time causing mysql's memory foot print to grow until it crashed. Long story short, I spaced the cron jobs out such that each has plenty of time to complete before the next one begins.