#375 Monit pro

# on server:
sudo apt-get update
sudo apt-get install monit
sudo vim /etc/monit/monitrc
sudo killall nginx
sudo tail /var/log/monit.log

# on local:
cap monit:setup
cap unicorn:setup unicorn:restart monit:setup

load "config/recipes/monit"

namespace :monit do
  desc "Install Monit"
  task :install do
    run "#{sudo} apt-get -y install monit"
  end
  after "deploy:install", "monit:install"

  desc "Setup all Monit configuration"
  task :setup do
    monit_config "monitrc", "/etc/monit/monitrc"
    nginx
    postgresql
    unicorn
    syntax
    reload
  end
  after "deploy:setup", "monit:setup"
  
  task(:nginx, roles: :web) { monit_config "nginx" }
  task(:postgresql, roles: :db) { monit_config "postgresql" }
  task(:unicorn, roles: :app) { monit_config "unicorn" }

  %w[start stop restart syntax reload].each do |command|
    desc "Run Monit #{command} script"
    task command do
      run "#{sudo} service monit #{command}"
    end
  end
end

def monit_config(name, destination = nil)
  destination ||= "/etc/monit/conf.d/#{name}.conf"
  template "monit/#{name}.erb", "/tmp/monit_#{name}"
  run "#{sudo} mv /tmp/monit_#{name} #{destination}"
  run "#{sudo} chown root #{destination}"
  run "#{sudo} chmod 600 #{destination}"
end

set daemon 30

set logfile /var/log/monit.log
set idfile /var/lib/monit/id
set statefile /var/lib/monit/state

set eventqueue
    basedir /var/lib/monit/events
    slots 100

# set mailserver smtp.gmail.com port 587
#     username "foo@example.com" password "secret"
#     using tlsv1
#     with timeout 30 seconds

set alert foo@example.com

set httpd port 2812
    allow admin:"secret"

check system blog_server
  if loadavg(5min) > 2 for 2 cycles then alert
  if memory > 75% for 2 cycles then alert
  if cpu(user) > 75% for 2 cycles then alert

include /etc/monit/conf.d/*

check process nginx with pidfile /var/run/nginx.pid
  start program = "/etc/init.d/nginx start"
  stop program = "/etc/init.d/nginx stop"
  if children > 250 then restart
  if 5 restarts within 5 cycles then timeout

check process postgresql with pidfile <%= postgresql_pid %>
  start program = "/etc/init.d/postgresql start"
  stop program = "/etc/init.d/postgresql stop"
  if failed host localhost port 5432 protocol pgsql then restart
  if 5 restarts within 5 cycles then timeout

check process <%= application %>_unicorn with pidfile <%= unicorn_pid %>
  start program = "/etc/init.d/unicorn_<%= application %> start"
  stop program = "/etc/init.d/unicorn_<%= application %> stop"

<% unicorn_workers.times do |n| %>
  <% pid = unicorn_pid.sub(".pid", ".#{n}.pid") %>
  check process <%= application %>_unicorn_worker_<%= n %> with pidfile <%= pid %>
    start program = "/bin/true"
    stop program = "/usr/bin/test -s <%= pid %> && /bin/kill -QUIT `cat <%= pid %>`"
    if mem > 200.0 MB for 1 cycles then restart
    if cpu > 50% for 3 cycles then restart
    if 5 restarts within 5 cycles then timeout
    alert foo@example.com only on { pid }
    if changed pid 2 times within 60 cycles then alert
<% end %>

after_fork do |server, worker|
  # Start up the database connection again in the worker
  if defined?(ActiveRecord::Base)
    ActiveRecord::Base.establish_connection
  end
  child_pid = server.config[:pid].sub(".pid", ".#{worker.nr}.pid")
  system("echo #{Process.pid} > #{child_pid}")
end

set_default(:postgresql_pid) { "/var/run/postgresql/9.1-main.pid" }

set_default(:unicorn_workers, 2)