Class: Spider::Model::Storage::BaseStorage

Inherits:
Object
  • Object
show all
Includes:
Logger
Defined in:
lib/spiderfw/model/storage/base_storage.rb

Class Attribute Summary (collapse)

Instance Attribute Summary (collapse)

Class Method Summary (collapse)

Instance Method Summary (collapse)

Methods included from Logger

add, close, close_all, datetime_format, datetime_format=, debug, #debug, debug?, #debug?, enquire_loggers, #error, error, error?, #error?, #fatal, fatal, #fatal?, fatal?, info, #info, info?, #info?, log, #log, method_missing, open, reopen, send_to_loggers, unknown, #unknown, #warn, warn, #warn?, warn?

Constructor Details

- (BaseStorage) initialize(url)



86
87
88
89
90
# File 'lib/spiderfw/model/storage/base_storage.rb', line 86

def initialize(url)
    @url = url
    @configuration = {}
    parse_url(url)
end

Class Attribute Details

+ (Object) capabilities (readonly)

An Hash of storage capabilities. The default for db storages is => false, :sequences => true, :transactions => true (The BaseStorage class provides file sequences in case the subclass does not support them.)



18
19
20
# File 'lib/spiderfw/model/storage/base_storage.rb', line 18

def capabilities
  @capabilities
end

Instance Attribute Details

- (Object) instance_name

Returns the value of attribute instance_name



8
9
10
# File 'lib/spiderfw/model/storage/base_storage.rb', line 8

def instance_name
  @instance_name
end

- (Object) url (readonly)

Returns the value of attribute url



7
8
9
# File 'lib/spiderfw/model/storage/base_storage.rb', line 7

def url
  @url
end

Class Method Details

+ (Object) base_types



28
29
30
# File 'lib/spiderfw/model/storage/base_storage.rb', line 28

def base_types
    Model.base_types
end

+ (Boolean) connection_alive?(conn)

Checks whether a connection is still alive. Must be implemented by subclasses.



75
76
77
# File 'lib/spiderfw/model/storage/base_storage.rb', line 75

def connection_alive?(conn)
    raise "Virtual"
end

+ (Object) connection_attributes



152
153
154
# File 'lib/spiderfw/model/storage/base_storage.rb', line 152

def self.connection_attributes
    @connection_attributes ||= {}
end

+ (Object) connection_pools



46
47
48
# File 'lib/spiderfw/model/storage/base_storage.rb', line 46

def connection_pools
    @pools ||= {}
end

+ (Object) disconnect(conn)



70
71
72
# File 'lib/spiderfw/model/storage/base_storage.rb', line 70

def disconnect(conn)
    raise "Virtual"
end

+ (Object) get_connection(*args)



50
51
52
53
54
# File 'lib/spiderfw/model/storage/base_storage.rb', line 50

def get_connection(*args)
    @pools ||= {}
    @pools[args] ||= ConnectionPool.new(args, self)
    @pools[args].get_connection
end

+ (Object) inherited(subclass)



79
80
81
# File 'lib/spiderfw/model/storage/base_storage.rb', line 79

def inherited(subclass)
    subclass.instance_variable_set("@capabilities", @capabilities)
end

+ (Object) max_connections



42
43
44
# File 'lib/spiderfw/model/storage/base_storage.rb', line 42

def max_connections
    nil
end

+ (Object) new_connection(*args)

Returns a new connection. Must be implemented by the subclasses; args are implementation specific.



38
39
40
# File 'lib/spiderfw/model/storage/base_storage.rb', line 38

def new_connection(*args)
    raise "Unimplemented"
end

+ (Object) release_connection(conn, conn_params)

Frees a connection, relasing it to the pool



57
58
59
60
61
# File 'lib/spiderfw/model/storage/base_storage.rb', line 57

def release_connection(conn, conn_params)
    return unless conn
    return unless @pools && @pools[conn_params]
    @pools[conn_params].release(conn)
end

+ (Object) remove_connection(conn, conn_params)

Removes a connection from the pool.



64
65
66
67
68
# File 'lib/spiderfw/model/storage/base_storage.rb', line 64

def remove_connection(conn, conn_params)
    return unless conn
    return unless @pools && @pools[conn_params]
    @pools[conn_params].remove(conn)
end

+ (Object) sequence_sync



24
25
26
# File 'lib/spiderfw/model/storage/base_storage.rb', line 24

def sequence_sync
    @sequence_sync ||= ::Sync.new
end

+ (Object) storage_type



20
21
22
# File 'lib/spiderfw/model/storage/base_storage.rb', line 20

def storage_type
    :none
end

+ (Boolean) supports?(capability)

True if given named capability is supported by the Storage.



33
34
35
# File 'lib/spiderfw/model/storage/base_storage.rb', line 33

def supports?(capability)
    @capabilities[capability]
end

Instance Method Details

- (Object) ==(storage)



191
192
193
194
195
# File 'lib/spiderfw/model/storage/base_storage.rb', line 191

def ==(storage)
    return false unless self.class == storage.class
    return false unless self.url == storage.url
    return true
end

- (Object) commit

Raises:



236
237
238
239
240
241
# File 'lib/spiderfw/model/storage/base_storage.rb', line 236

def commit
    return false unless transactions_enabled?
    raise StorageException, "Commit without a transaction" unless in_transaction?
    return curr[:savepoints].pop unless curr[:savepoints].empty?
    commit!
end

- (Object) commit!



255
256
257
258
259
260
# File 'lib/spiderfw/model/storage/base_storage.rb', line 255

def commit!
    Spider.logger.debug("#{self.class.name} commit connection #{curr[:conn].object_id}")
    curr[:transaction_nesting] = 0
    do_commit
    release
end

- (Object) commit_or_continue

Raises:



243
244
245
246
247
248
249
250
251
252
253
# File 'lib/spiderfw/model/storage/base_storage.rb', line 243

def commit_or_continue
    return false unless transactions_enabled?
    raise StorageException, "Commit without a transaction" unless in_transaction?
    if curr[:transaction_nesting] == 1
        commit
        curr[:transaction_nesting] = 0
        return true
    else
        curr[:transaction_nesting] -= 1
    end
end

- (Object) configure(conf)



92
93
94
# File 'lib/spiderfw/model/storage/base_storage.rb', line 92

def configure(conf)
    @configuration.merge!(conf.to_hash)
end

- (Object) connect

Instantiates a new connection with current connection params.



128
129
130
131
# File 'lib/spiderfw/model/storage/base_storage.rb', line 128

def connect
    return self.class.get_connection(*@connection_params)
    #Spider::Logger.debug("#{self.class.name} in thread #{Thread.current} acquired connection #{@conn}")
end

- (Boolean) connected?

True if currently connected.



134
135
136
# File 'lib/spiderfw/model/storage/base_storage.rb', line 134

def connected?
    curr[:conn] != nil
end

- (Object) connection

Returns the current connection, or creates a new one. If a block is given, will release the connection after yielding.



141
142
143
144
145
146
147
148
149
150
# File 'lib/spiderfw/model/storage/base_storage.rb', line 141

def connection
    curr[:conn] = connect
    if block_given?
        yield curr[:conn]
        release # unless is_connected
        return true
    else
        return curr[:conn]
    end
end

- (Object) connection_attributes



156
157
158
# File 'lib/spiderfw/model/storage/base_storage.rb', line 156

def connection_attributes
    self.class.connection_attributes[connection] ||= {}
end

- (Object) connection_pool



123
124
125
# File 'lib/spiderfw/model/storage/base_storage.rb', line 123

def connection_pool
    self.class.connection_pools[@connection_params]
end

- (Object) create_sequence(name, start = 1, increment = 1)



308
309
310
# File 'lib/spiderfw/model/storage/base_storage.rb', line 308

def create_sequence(name, start=1, increment=1)
    sequence_next(name, start-1, increment)
end

- (Object) curr



108
109
110
111
112
113
114
115
116
117
118
119
120
121
# File 'lib/spiderfw/model/storage/base_storage.rb', line 108

def curr
    var = nil
    if Spider.conf.get('storage.shared_connection')
        $STORAGES ||= {}
        var = $STORAGES
    else
        var = Thread.current
    end
    var[:storages] ||= {}
    var[:storages][self.class.storage_type] ||= {}
    var[:storages][self.class.storage_type][@connection_params] ||= {
        :transaction_nesting => 0, :savepoints => []
    }
end

- (Object) do_commit

Raises:



262
263
264
# File 'lib/spiderfw/model/storage/base_storage.rb', line 262

def do_commit
    raise StorageException, "The current storage does not support transactions" 
end

- (Object) do_rollback

Raises:



280
281
282
# File 'lib/spiderfw/model/storage/base_storage.rb', line 280

def do_rollback
    raise StorageException, "The current storage does not support transactions" 
end

- (Object) do_start_transaction

May be implemented by subclasses.

Raises:



217
218
219
# File 'lib/spiderfw/model/storage/base_storage.rb', line 217

def do_start_transaction
   raise StorageException, "The current storage does not support transactions" 
end

- (Object) generate_uuid



312
313
314
# File 'lib/spiderfw/model/storage/base_storage.rb', line 312

def generate_uuid
    Spider::DataTypes::UUID.generate
end

- (Object) get_mapper(model)

Raises:



100
101
102
# File 'lib/spiderfw/model/storage/base_storage.rb', line 100

def get_mapper(model)
    raise StorageException, "Unimplemented"
end

- (Object) in_transaction



221
222
223
224
225
226
227
228
229
# File 'lib/spiderfw/model/storage/base_storage.rb', line 221

def in_transaction
    if in_transaction?
        curr[:transaction_nesting] += 1
        return true
    else
        start_transaction
        return false
    end
end

- (Boolean) in_transaction?



231
232
233
# File 'lib/spiderfw/model/storage/base_storage.rb', line 231

def in_transaction?
    return false
end

- (Object) parse_url(url)

Raises:



96
97
98
# File 'lib/spiderfw/model/storage/base_storage.rb', line 96

def parse_url(url)
    raise StorageException, "Unimplemented"
end

- (Object) prepare_value(type, value)



187
188
189
# File 'lib/spiderfw/model/storage/base_storage.rb', line 187

def prepare_value(type, value)
    return value
end

- (Object) release

Releases the current connection to the pool.



161
162
163
164
165
166
167
168
169
170
# File 'lib/spiderfw/model/storage/base_storage.rb', line 161

def release
    # The subclass should check if the connection is alive, and if it is not call remove_connection instead
    c = curr[:conn]
    #Spider.logger.debug("#{self} in thread #{Thread.current} releasing #{curr[:conn]}")
    curr[:conn] = nil
    self.class.release_connection(c, @connection_params)
    #Spider.logger.debug("#{self} in thread #{Thread.current} released #{curr[:conn]}")
    return nil
    #@conn = nil
end

- (Object) rollback



266
267
268
269
270
# File 'lib/spiderfw/model/storage/base_storage.rb', line 266

def rollback
    raise "Can't rollback in a nested transaction" if curr[:transaction_nesting] > 1
    return rollback_savepoint(curr[:savepoints].last) unless curr[:savepoints].empty?
    rollback!
end

- (Object) rollback!



272
273
274
275
276
277
278
# File 'lib/spiderfw/model/storage/base_storage.rb', line 272

def rollback!
    curr[:transaction_nesting] = 0
    Spider.logger.debug("#{self.class.name} rollback")
    do_rollback
    curr[:savepoints] = []
    release
end

- (Object) rollback_savepoint(name = nil)



288
289
290
291
292
293
294
295
# File 'lib/spiderfw/model/storage/base_storage.rb', line 288

def rollback_savepoint(name=nil)
    if name
        curr[:savepoints] = curr[:savepoints][0,(curr[:savepoints].index(name))]
        name
    else
        curr[:savepoints].pop
    end
end

- (Object) savepoint(name)



284
285
286
# File 'lib/spiderfw/model/storage/base_storage.rb', line 284

def savepoint(name)
    curr[:savepoints] << name
end

- (Boolean) sequence_exists?(name)



304
305
306
# File 'lib/spiderfw/model/storage/base_storage.rb', line 304

def sequence_exists?(name)
    File.exist?(sequence_file_path(name))
end

- (Object) sequence_file_path(name)

Utility methods



299
300
301
302
# File 'lib/spiderfw/model/storage/base_storage.rb', line 299

def sequence_file_path(name)
    path = File.join(Spider.paths[:var], 'sequences', name)
    return path
end

- (Object) sequence_next(name, newval = nil, increment = 1)

Increments a named sequence and returns the new value



323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
# File 'lib/spiderfw/model/storage/base_storage.rb', line 323

def sequence_next(name, newval=nil, increment=1)
    path = sequence_file_path(name)
    FileUtils.mkpath(File.dirname(path))
    self.class.sequence_sync.lock(::Sync::EX)
    if newval
        seq = newval
    else
        seq = 0
        File.open(path, 'a+') do |f|
            f.rewind
            f.flock File::LOCK_EX
            cur = f.gets
            if (cur)
                seq, increment_str = cur.split('|')
            else
                seq, increment_str = 0, 1
            end
            seq = seq.to_i
            increment = increment_str.to_i if increment_str
            f.close
        end
        seq += increment
    end
    File.open(path, 'w+') do |f|
        f.print(seq)
        f.print("|#{increment}") if (increment != 1)
        f.flock File::LOCK_UN
        f.close
    end
    self.class.sequence_sync.lock(::Sync::UN)
    return seq
end

- (Object) start_transaction



206
207
208
209
210
211
212
213
214
# File 'lib/spiderfw/model/storage/base_storage.rb', line 206

def start_transaction
    return unless transactions_enabled?
    curr[:transaction_nesting] += 1
    return savepoint("point#{curr[:savepoints].length}") if in_transaction?

    Spider.logger.debug("#{self.class.name} starting transaction for connection #{connection.object_id}")
    do_start_transaction
    return true
end

- (Boolean) supports?(capability)



104
105
106
# File 'lib/spiderfw/model/storage/base_storage.rb', line 104

def supports?(capability)
    self.class.supports?(capability)
end

- (Boolean) supports_transactions?



198
199
200
# File 'lib/spiderfw/model/storage/base_storage.rb', line 198

def supports_transactions?
    return self.class.supports?(:transactions)
end

- (Boolean) transactions_enabled?



202
203
204
# File 'lib/spiderfw/model/storage/base_storage.rb', line 202

def transactions_enabled?
    @configuration['enable_transactions'] && supports_transactions?
end

- (Object) update_sequence(name, val)



317
318
319
320
# File 'lib/spiderfw/model/storage/base_storage.rb', line 317

def update_sequence(name, val)
    # not an alias because the set value behaviour of next_sequence isn't expected in subclasses
    sequence_next(name, val)
end

- (Object) value_for_condition(type, value)

Prepares a value that will be used in a condition.



178
179
180
# File 'lib/spiderfw/model/storage/base_storage.rb', line 178

def value_for_condition(type, value)
    return prepare_value(type, value)
end

- (Object) value_for_save(type, value, save_mode)

Prepares a value for saving.



173
174
175
# File 'lib/spiderfw/model/storage/base_storage.rb', line 173

def value_for_save(type, value, save_mode)
    return prepare_value(type, value)
end

- (Object) value_to_mapper(type, value)



182
183
184
# File 'lib/spiderfw/model/storage/base_storage.rb', line 182

def value_to_mapper(type, value)
    value
end