Module: CloudCrowd

Defined in:

lib/cloud-crowd.rb,
lib/cloud_crowd/node.rb,
lib/cloud_crowd/action.rb,
lib/cloud_crowd/worker.rb,
lib/cloud_crowd/server.rb,
lib/cloud_crowd/models.rb,
lib/cloud_crowd/helpers.rb,
lib/cloud_crowd/inflector.rb,
lib/cloud_crowd/models/job.rb,
lib/cloud_crowd/exceptions.rb,
lib/cloud_crowd/asset_store.rb,
lib/cloud_crowd/command_line.rb,
lib/cloud_crowd/models/work_unit.rb,
lib/cloud_crowd/helpers/resources.rb,
lib/cloud_crowd/models/node_record.rb,
lib/cloud_crowd/asset_store/s3_store.rb,
lib/cloud_crowd/helpers/authorization.rb,
lib/cloud_crowd/asset_store/filesystem_store.rb,
lib/cloud_crowd/asset_store/cloudfiles_store.rb

Defined Under Namespace

Modules: Helpers, Inflector, ModelStatus Classes: Action, AssetStore, CommandLine, Error, Job, Node, NodeRecord, Server, WorkUnit, Worker

Constant Summary

VERSION =

Keep this version in sync with the gemspec.

'0.6.2'

SCHEMA_VERSION =

Increment the schema version when there's a backwards incompatible change.

4

ROOT =

Root directory of the CloudCrowd gem.

File.expand_path(File.dirname(__FILE__) + '/..')

LOG_PATH =

Default folder to log daemonized servers and nodes into.

'log'

PID_PATH =

Default folder to contain the pids of daemonized servers and nodes.

'tmp/pids'

MIN_RETRIES =

Minimum number of attempts per work unit.

1

PROCESSING =

A Job is processing if its WorkUnits are in the queue to be handled by nodes.

1

SUCCEEDED =

A Job has succeeded if all of its WorkUnits have finished successfully.

2

FAILED =

A Job has failed if even a single one of its WorkUnits has failed (they may be attempted multiple times on failure, however).

3

SPLITTING =

A Job is splitting if it's in the process of dividing its inputs up into multiple WorkUnits.

4

MERGING =

A Job is merging if it's busy collecting all of its successful WorkUnits back together into the final result.

5

COMPLETE =

A Job is considered to be complete if it succeeded or if it failed.

[SUCCEEDED, FAILED]

INCOMPLETE =

A Job is considered incomplete if it's being processed, split up or merged.

[PROCESSING, SPLITTING, MERGING]

DISPLAY_STATUS_MAP =

Mapping of statuses to their display strings.

['unknown', 'processing', 'succeeded', 'failed', 'splitting', 'merging']

Class Attribute Summary (collapse)

• + (Object) config readonly

  Returns the value of attribute config.

• + (Object) identity

  Returns the value of attribute identity.

Class Method Summary (collapse)

• + (Object) action_paths

  Retrieve the list of every installed Action for this node or server.

• + (Object) actions

  CloudCrowd::Actions are requested dynamically by name.

• + (Object) central_server

  Get a reference to the central server, including authentication if configured.

• + (Object) client_options

  The standard RestClient options for the central server talking to nodes, as well as the other way around.

• + (Object) configure(config_path)

  Configure CloudCrowd by passing in the path to config.yml.

• + (Object) configure_database(config_path, validate_schema = true)

  Configure the CloudCrowd central database (and connect to it), by passing in a path to database.yml.

• + (Object) display_status(status)

  Return the displayable status name of an internal CloudCrowd status number.

• + (Object) log_path(log_file = nil)

  The path that daemonized servers and nodes will log to.

• + (Boolean) node?

  Or is it a node?.

• + (Object) pid_path(pid_file = nil)

  The path in which daemonized servers and nodes will store their pids.

• + (Boolean) server?

  Is this CloudCrowd instance a server? Useful for avoiding loading unneeded code from actions.

93
94
95

# File 'lib/cloud-crowd.rb', line 93

def config
  @config
end

94
95
96

# File 'lib/cloud-crowd.rb', line 94

def identity
  @identity
end

176
177
178
179
180
181

# File 'lib/cloud-crowd.rb', line 176

def action_paths
  default_actions   = config[:disable_default_actions] ? [] : Dir["#{ROOT}/actions/*.rb"]
  installed_actions = Dir["#{@config_path}/actions/*.rb"]
  custom_actions    = CloudCrowd.config[:actions_path] ? Dir["#{CloudCrowd.config[:actions_path]}/*.rb"] : []
  default_actions + installed_actions + custom_actions
end

162
163
164
165
166
167
168
169
170
171
172
173

# File 'lib/cloud-crowd.rb', line 162

def actions
  return @actions if @actions
  @actions = action_paths.inject({}) do |memo, path|
    name = File.basename(path, File.extname(path))
    require path
    memo[name] = Module.const_get(Inflector.camelize(name))
    memo
  end
rescue NameError => e
  adjusted_message = "One of your actions failed to load. Please ensure that the name of your action class can be deduced from the name of the file. ex: 'word_count.rb' => 'WordCount'\n#{e.message}"
  raise NameError.new(adjusted_message, e.name)
end

119
120
121

# File 'lib/cloud-crowd.rb', line 119

def central_server
  @central_server ||= RestClient::Resource.new(CloudCrowd.config[:central_server], CloudCrowd.client_options)
end

138
139
140
141
142
143
144
145
146
147
148
149

# File 'lib/cloud-crowd.rb', line 138

def client_options
  return @client_options if @client_options
  @client_options = {
    :timeout => (self.server? ? config[:node_timeout] : config[:server_timeout]) || 30,
    :open_timeout => config[:open_timeout] || 5
  }
  if CloudCrowd.config[:http_authentication]
    @client_options[:user]      = CloudCrowd.config[:login]
    @client_options[:password]  = CloudCrowd.config[:password]
  end
  @client_options
end

97
98
99
100
101

# File 'lib/cloud-crowd.rb', line 97

def configure(config_path)
  @config_path = File.expand_path(File.dirname(config_path))
  @config = YAML.load(ERB.new(File.read(config_path)).result)
  @config[:work_unit_retries] ||= MIN_RETRIES
end

106
107
108
109
110
111
112
113
114
115

# File 'lib/cloud-crowd.rb', line 106

def configure_database(config_path, validate_schema=true)
  configuration = YAML.load(ERB.new(File.read(config_path)).result)
  ActiveRecord::Base.establish_connection(configuration)
  if validate_schema
    version = ActiveRecord::Base.connection.select_values('select max(version) from schema_migrations').first.to_i
    return true if version == SCHEMA_VERSION
    puts "Your database schema is out of date. Please use `crowd load_schema` to update it. This will wipe all the tables, so make sure that your jobs have a chance to finish first.\nexiting..."
    exit
  end
end

153
154
155

# File 'lib/cloud-crowd.rb', line 153

def display_status(status)
  DISPLAY_STATUS_MAP[status] || 'unknown'
end

124
125
126
127

# File 'lib/cloud-crowd.rb', line 124

def log_path(log_file=nil)
  @log_path ||= config[:log_path] || LOG_PATH
  log_file ? File.join(@log_path, log_file) : @log_path
end

190
191
192

# File 'lib/cloud-crowd.rb', line 190

def node?
  @identity == :node
end

130
131
132
133

# File 'lib/cloud-crowd.rb', line 130

def pid_path(pid_file=nil)
  @pid_path ||= config[:pid_path] || PID_PATH
  pid_file ? File.join(@pid_path, pid_file) : @pid_path
end

185
186
187

# File 'lib/cloud-crowd.rb', line 185

def server?
  @identity == :server
end