Module: God

Defined in:
lib/god.rb,
lib/god/task.rb,
lib/god/watch.rb,
lib/god/logger.rb,
lib/god/driver.rb,
lib/god/metric.rb,
lib/god/errors.rb,
lib/god/socket.rb,
lib/god/process.rb,
lib/god/cli/run.rb,
lib/god/contact.rb,
lib/god/trigger.rb,
lib/god/timeline.rb,
lib/god/registry.rb,
lib/god/behavior.rb,
lib/god/condition.rb,
lib/god/cli/version.rb,
lib/god/cli/command.rb,
lib/god/configurable.rb,
lib/god/simple_logger.rb,
lib/god/event_handler.rb,
lib/god/contacts/prowl.rb,
lib/god/contacts/scout.rb,
lib/god/system/process.rb,
lib/god/contacts/slack.rb,
lib/god/contacts/email.rb,
lib/god/contacts/statsd.rb,
lib/god/contacts/jabber.rb,
lib/god/conditions/tries.rb,
lib/god/contacts/twitter.rb,
lib/god/contacts/webhook.rb,
lib/god/contacts/hipchat.rb,
lib/god/conditions/always.rb,
lib/god/conditions/lambda.rb,
lib/god/contacts/campfire.rb,
lib/god/contacts/airbrake.rb,
lib/god/conditions/complex.rb,
lib/god/conditions/flapping.rb,
lib/god/conditions/cpu_usage.rb,
lib/god/conditions/file_mtime.rb,
lib/god/conditions/disk_usage.rb,
lib/god/system/portable_poller.rb,
lib/god/conditions/memory_usage.rb,
lib/god/conditions/file_touched.rb,
lib/god/behaviors/clean_pid_file.rb,
lib/god/system/slash_proc_poller.rb,
lib/god/conditions/process_exits.rb,
lib/god/conditions/process_running.rb,
lib/god/behaviors/clean_unix_socket.rb,
lib/god/conditions/degrading_lambda.rb,
lib/god/event_handlers/dummy_handler.rb,
lib/god/conditions/socket_responding.rb,
lib/god/conditions/http_response_code.rb,
lib/god/event_handlers/kqueue_handler.rb,
lib/god/behaviors/notify_when_flapping.rb,
lib/god/event_handlers/netlink_handler.rb

Defined Under Namespace

Modules: Behaviors, CLI, Conditions, Configurable, Contacts, System Classes: AbstractMethodNotOverriddenError, Behavior, Condition, Contact, Driver, DriverEvent, DriverEventQueue, DriverOperation, DummyHandler, EventCondition, EventHandler, EventRegistrationFailedError, InvalidCommandError, KQueueHandler, Logger, Metric, NetlinkHandler, NoSuchBehaviorError, NoSuchConditionError, NoSuchContactError, NoSuchWatchError, PollCondition, Process, Registry, SimpleLogger, Socket, Task, TimedEvent, Timeline, Trigger, TriggerCondition, Watch

Constant Summary

VERSION =
'0.13.4'
LOG_BUFFER_SIZE_DEFAULT =
100
PID_FILE_DIRECTORY_DEFAULTS =
['/var/run/god', '~/.god/pids']
DRB_PORT_DEFAULT =
17165
DRB_ALLOW_DEFAULT =
['127.0.0.1']
LOG_LEVEL_DEFAULT =
:info
TERMINATE_TIMEOUT_DEFAULT =
10
STOP_TIMEOUT_DEFAULT =
10
STOP_SIGNAL_DEFAULT =
'TERM'

Class Attribute Summary (collapse)

Class Method Summary (collapse)

Class Attribute Details

+ (Object) contact_groups

Returns the value of attribute contact_groups



209
210
211
# File 'lib/god.rb', line 209

def contact_groups
  @contact_groups
end

+ (Object) contacts

Returns the value of attribute contacts



209
210
211
# File 'lib/god.rb', line 209

def contacts
  @contacts
end

+ (Object) groups

Returns the value of attribute groups



209
210
211
# File 'lib/god.rb', line 209

def groups
  @groups
end

+ (Object) inited

Returns the value of attribute inited



209
210
211
# File 'lib/god.rb', line 209

def inited
  @inited
end

+ (Object) main

Returns the value of attribute main



209
210
211
# File 'lib/god.rb', line 209

def main
  @main
end

+ (Object) pending_watch_states

Returns the value of attribute pending_watch_states



209
210
211
# File 'lib/god.rb', line 209

def pending_watch_states
  @pending_watch_states
end

+ (Object) pending_watches

Returns the value of attribute pending_watches



209
210
211
# File 'lib/god.rb', line 209

def pending_watches
  @pending_watches
end

+ (Object) running

Returns the value of attribute running



209
210
211
# File 'lib/god.rb', line 209

def running
  @running
end

+ (Object) server

Returns the value of attribute server



209
210
211
# File 'lib/god.rb', line 209

def server
  @server
end

+ (Object) watches

Returns the value of attribute watches



209
210
211
# File 'lib/god.rb', line 209

def watches
  @watches
end

Class Method Details

+ (Object) at_exit



737
738
739
740
# File 'lib/god.rb', line 737

def self.at_exit
  self.start
  self.join
end

+ (Object) contact(kind) {|c| ... }

Yields:

  • (c)


372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
# File 'lib/god.rb', line 372

def self.contact(kind)
  # Ensure internal init has run.
  self.internal_init

  # Verify contact has been loaded.
  if CONTACT_LOAD_SUCCESS[kind] == false
    applog(nil, :error, "A required dependency for the #{kind} contact is unavailable.")
    applog(nil, :error, "Run the following commands to install the dependencies:")
    CONTACT_DEPS[kind].each do |d|
      applog(nil, :error, "  [sudo] gem install #{d}")
    end
    abort
  end

  # Create the contact.
  begin
    c = Contact.generate(kind)
  rescue NoSuchContactError => e
    abort e.message
  end

  # Send to block so config can set attributes.
  yield(c) if block_given?

  # Call prepare on the contact.
  c.prepare

  # Remove existing contacts of same name.
  existing_contact = self.contacts[c.name]
  if self.running && existing_contact
    self.uncontact(existing_contact)
  end

  # Warn and noop if the contact has been defined before.
  if self.contacts[c.name] || self.contact_groups[c.name]
    applog(nil, :warn, "Contact name '#{c.name}' already used for a Contact or Contact Group")
    return
  end

  # Abort if the Contact is invalid, the Contact will have printed out its
  # own error messages by now.
  unless Contact.valid?(c) && c.valid?
    abort "Exiting on invalid contact"
  end

  # Add to list of contacts.
  self.contacts[c.name] = c

  # Add to contact group if specified.
  if c.group
    # Ensure group name hasn't been used for a contact already.
    if self.contacts[c.group]
      abort "Contact Group name '#{c.group}' already used for a Contact"
    end

    self.contact_groups[c.group] ||= []
    self.contact_groups[c.group] << c
  end
end

+ (Object) control(name, command)



458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
# File 'lib/god.rb', line 458

def self.control(name, command)
  # Get the list of items.
  items = self.watches_by_name(name)

  jobs = []

  # Do the command.
  case command
    when "start", "monitor"
      items.each { |w| jobs << Thread.new { w.monitor if w.state != :up } }
    when "restart"
      items.each { |w| jobs << Thread.new { w.move(:restart) } }
    when "stop"
      items.each { |w| jobs << Thread.new { w.action(:stop); w.unmonitor if w.state != :unmonitored } }
    when "unmonitor"
      items.each { |w| jobs << Thread.new { w.unmonitor if w.state != :unmonitored } }
    when "remove"
      items.each { |w| self.unwatch(w) }
    else
      raise InvalidCommandError.new
  end

  jobs.each { |j| j.join }

  items.map { |x| x.name }
end

+ (Object) internal_init



237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
# File 'lib/god.rb', line 237

def self.internal_init
  # Only do this once.
  return if self.inited

  # Variable init.
  self.watches = {}
  self.groups = {}
  self.pending_watches = []
  self.pending_watch_states = {}
  self.contacts = {}
  self.contact_groups = {}

  # Set defaults.
  self.log_buffer_size ||= LOG_BUFFER_SIZE_DEFAULT
  self.port ||= DRB_PORT_DEFAULT
  self.allow ||= DRB_ALLOW_DEFAULT
  self.log_level ||= LOG_LEVEL_DEFAULT
  self.terminate_timeout ||= TERMINATE_TIMEOUT_DEFAULT

  # Additional setup.
  self.setup

  # Log level.
  log_level_map = {:debug => Logger::DEBUG,
                   :info => Logger::INFO,
                   :warn => Logger::WARN,
                   :error => Logger::ERROR,
                   :fatal => Logger::FATAL}
  LOG.level = log_level_map[self.log_level]

  # Init has been executed.
  self.inited = true

  # Not yet running.
  self.running = false
end

+ (Object) join



725
726
727
# File 'lib/god.rb', line 725

def self.join
  self.main.join if self.main
end

+ (Object) load(glob)



643
644
645
646
647
# File 'lib/god.rb', line 643

def self.load(glob)
  Dir[glob].each do |f|
    Kernel.load f
  end
end

+ (Object) pattern_match(pattern, list)



759
760
761
762
763
764
765
# File 'lib/god.rb', line 759

def self.pattern_match(pattern, list)
  regex = pattern.split('').join('.*')

  list.select do |item|
    item =~ Regexp.new(regex)
  end.sort_by { |x| x.size }
end

+ (Object) registry



2
3
4
# File 'lib/god/registry.rb', line 2

def self.registry
  @registry ||= Registry.new
end

+ (Object) running_load(code, filename, action = nil)



582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
# File 'lib/god.rb', line 582

def self.running_load(code, filename, action = nil)
  errors = ""
  loaded_watches = []
  unloaded_watches = []
  jobs = []

  begin
    LOG.start_capture

    Gem.clear_paths
    eval(code, root_binding, filename)
    self.pending_watches.each do |w|
      if previous_state = self.pending_watch_states[w.name]
        w.monitor unless previous_state == :unmonitored
      else
        w.monitor if w.autostart?
      end
    end
    loaded_watches = self.pending_watches.map { |w| w.name }
    self.pending_watches.clear
    self.pending_watch_states.clear

    self.watches.each do |name, watch|
      next if loaded_watches.include?(name)

      case action
      when 'stop'
        jobs << Thread.new(watch) { |w| w.action(:stop); self.unwatch(w) }
        unloaded_watches << name
      when 'remove'
        jobs << Thread.new(watch) { |w| self.unwatch(w) }
        unloaded_watches << name
      when 'leave', '', nil
        # Do nothing
      else
        raise InvalidCommandError, "Unknown action: #{action}"
      end
    end

    # Make sure we quit capturing when we're done.
    LOG.finish_capture
  rescue Exception => e
    # Don't ever let running_load take down god.
    errors << LOG.finish_capture

    unless e.instance_of?(SystemExit)
      errors << e.message << "\n"
      errors << e.backtrace.join("\n")
    end
  end

  jobs.each { |t| t.join }

  [loaded_watches, errors, unloaded_watches]
end

+ (Object) running_log(watch_name, since)



555
556
557
558
559
560
561
562
563
# File 'lib/god.rb', line 555

def self.running_log(watch_name, since)
  matches = pattern_match(watch_name, self.watches.keys)

  unless matches.first
    raise NoSuchWatchError.new
  end

  LOG.watch_log_since(matches.first, since)
end

+ (Object) setup



652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
# File 'lib/god.rb', line 652

def self.setup
  if self.pid_file_directory
    # Pid file dir was specified, ensure it is created and writable.
    unless File.exist?(self.pid_file_directory)
      begin
        FileUtils.mkdir_p(self.pid_file_directory)
      rescue Errno::EACCES => e
        abort "Failed to create pid file directory: #{e.message}"
      end
    end

    unless File.writable?(self.pid_file_directory)
      abort "The pid file directory (#{self.pid_file_directory}) is not writable by #{Etc.getlogin}"
    end
  else
    # No pid file dir specified, try defaults.
    PID_FILE_DIRECTORY_DEFAULTS.each do |idir|
      dir = File.expand_path(idir)
      begin
        FileUtils.mkdir_p(dir)
        if File.writable?(dir)
          self.pid_file_directory = dir
          break
        end
      rescue Errno::EACCES => e
      end
    end

    unless self.pid_file_directory
      dirs = PID_FILE_DIRECTORY_DEFAULTS.map { |x| File.expand_path(x) }
      abort "No pid file directory exists, could be created, or is writable at any of #{dirs.join(', ')}"
    end
  end

  if God::Logger.syslog
    LOG.info("Syslog enabled.")
  else
    LOG.info("Syslog disabled.")
  end

  applog(nil, :info, "Using pid file directory: #{self.pid_file_directory}")
end

+ (Object) signal(name, signal)



540
541
542
543
544
545
546
# File 'lib/god.rb', line 540

def self.signal(name, signal)
  items = watches_by_name(name)
  jobs = []
  items.each { |w| jobs << Thread.new { w.signal(signal) } }
  jobs.each { |j| j.join }
  items.map { |x| x.name }
end

+ (Object) start



698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
# File 'lib/god.rb', line 698

def self.start
  self.internal_init

  # Instantiate server.
  self.server = Socket.new(self.port, self.socket_user, self.socket_group, self.socket_perms)

  # Start monitoring any watches set to autostart.
  self.watches.values.each { |w| w.monitor if w.autostart? }

  # Clear pending watches.
  self.pending_watches.clear

  # Mark as running.
  self.running = true

  # Don't exit.
  self.main =
  Thread.new do
    loop do
      sleep 60
    end
  end
end

+ (Object) status



526
527
528
529
530
531
532
# File 'lib/god.rb', line 526

def self.status
  info = {}
  self.watches.map do |name, w|
    info[name] = {:state => w.state, :group => w.group}
  end
  info
end

+ (Object) stop_all



489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
# File 'lib/god.rb', line 489

def self.stop_all
  self.watches.sort.each do |name, w|
    Thread.new do
      w.unmonitor if w.state != :unmonitored
      w.action(:stop) if w.alive?
    end
  end

  terminate_timeout.times do
    return true unless self.watches.map { |name, w| w.alive? }.any?
    sleep 1
  end

  return false
end

+ (Object) task(klass = Task) {|t| ... }

Yields:

  • (t)


288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
# File 'lib/god.rb', line 288

def self.task(klass = Task)
  # Ensure internal init has run.
  self.internal_init

  t = klass.new
  yield(t)

  # Do the post-configuration.
  t.prepare

  # If running, completely remove the watch (if necessary) to prepare for
  # the reload
  existing_watch = self.watches[t.name]
  if self.running && existing_watch
    self.pending_watch_states[existing_watch.name] = existing_watch.state
    self.unwatch(existing_watch)
  end

  # Ensure the new watch has a unique name.
  if self.watches[t.name] || self.groups[t.name]
    abort "Task name '#{t.name}' already used for a Task or Group"
  end

  # Ensure watch is internally valid.
  t.valid? || abort("Task '#{t.name}' is not valid (see above)")

  # Add to list of watches.
  self.watches[t.name] = t

  # Add to pending watches.
  self.pending_watches << t

  # Add to group if specified.
  if t.group
    # Ensure group name hasn't been used for a watch already.
    if self.watches[t.group]
      abort "Group name '#{t.group}' already used for a Task"
    end

    self.groups[t.group] ||= []
    self.groups[t.group] << t
  end

  # Register watch.
  t.register!

  # Log.
  if self.running && existing_watch
    applog(t, :info, "#{t.name} Reloaded config")
  elsif self.running
    applog(t, :info, "#{t.name} Loaded config")
  end
end

+ (Object) terminate



511
512
513
514
515
# File 'lib/god.rb', line 511

def self.terminate
  FileUtils.rm_f(self.pid) if self.pid
  self.server.stop if self.server
  exit!(0)
end

+ (Object) uncontact(contact)



437
438
439
440
441
442
# File 'lib/god.rb', line 437

def self.uncontact(contact)
  self.contacts.delete(contact.name)
  if contact.group
    self.contact_groups[contact.group].delete(contact)
  end
end

+ (Object) unwatch(watch)



347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
# File 'lib/god.rb', line 347

def self.unwatch(watch)
  # Unmonitor.
  watch.unmonitor unless watch.state == :unmonitored

  # Unregister.
  watch.unregister!

  # Remove from watches.
  self.watches.delete(watch.name)

  # Remove from groups.
  if watch.group
    self.groups[watch.group].delete(watch)
  end

  applog(watch, :info, "#{watch.name} unwatched")
end

+ (Object) version



730
731
732
# File 'lib/god.rb', line 730

def self.version
  God::VERSION
end

+ (Object) watch(&block)



279
280
281
# File 'lib/god.rb', line 279

def self.watch(&block)
  self.task(Watch, &block)
end

+ (Object) watches_by_name(name)



444
445
446
447
448
449
# File 'lib/god.rb', line 444

def self.watches_by_name(name)
  case name 
    when "", nil then self.watches.values.dup
    else Array(self.watches[name] || self.groups[name]).dup
  end
end