Class: MartSearch::Controller

Inherits:
Object
  • Object
show all
Includes:
ControllerUtils, Utils, Singleton
Defined in:
lib/martsearch/controller.rb

Overview

Singleton controller class for MartSearch. This is the central contoller for the MartSearch framework - it handles the config file parsing, building up all of the DataSource and Index objects, and managing the search mechanics.

Author:

Constant Summary

USE_CACHE = 
true
USE_OLS = 
true

Instance Attribute Summary (collapse)

Instance Method Summary (collapse)

Methods included from ControllerUtils

#build_datasources, #build_index_builder_conf, #build_index_conf, #build_server_conf, #initialize_cache

Methods included from Utils

#build_http_client, #convert_array_to_hash

Constructor Details

- (Controller) initialize

A new instance of Controller



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/martsearch/controller.rb', line 23

def initialize()
  config_dir = "#{MARTSEARCH_PATH}/config"

  @config = {
    :index         => build_index_conf( config_dir ),
    :datasources   => build_datasources( config_dir ),
    :server        => build_server_conf( "#{config_dir}/server" ),
    :index_builder => build_index_builder_conf( "#{config_dir}/index_builder" )
  }

  @cache             = initialize_cache( @config[:server][:cache] )
  @index             = MartSearch::Index.new( @config[:index] )
  @datasources       = @config[:datasources]
  @datasets          = @config[:server][:datasets]
  @dataviews         = @config[:server][:dataviews]
  @dataviews_by_name = @config[:server][:dataviews_by_name]

  # OLS
  OLS.setup_cache(
    {
      :host => 'web-mei-t87p.internal.sanger.ac.uk',
      :port => 3334,
      :database => 'htgt_ols_cache',
      :user => 'htgt',
      :password => 'htgt'
    }
  ) if USE_OLS

  # Logger
  @logger                 = Logger.new($stdout)
  @logger                 = Logger.new($stderr)
  @logger.datetime_format = "%Y-%m-%d %H:%M:%S "
  @logger.level           = case @config[:server][:log][:level]
    when 'debug' then Logger::DEBUG
    when 'info'  then Logger::INFO
    when 'warn'  then Logger::WARN
    when 'error' then Logger::ERROR
    when 'fatal' then Logger::FATAL
  end

  # Stores for search result data and errors...
  @errors         = { :index => [], :datasets => {} }
  @search_data    = {}
  @search_results = []
end

Instance Attribute Details

- (Object) cache (readonly)

Returns the value of attribute cache



20
21
22
 
# File 'lib/martsearch/controller.rb', line 20

def cache
  @cache
end

- (Object) config (readonly)

Returns the value of attribute config



20
21
22
 
# File 'lib/martsearch/controller.rb', line 20

def config
  @config
end

- (Object) datasets (readonly)

Returns the value of attribute datasets



21
22
23
 
# File 'lib/martsearch/controller.rb', line 21

def datasets
  @datasets
end

- (Object) datasources (readonly)

Returns the value of attribute datasources



21
22
23
 
# File 'lib/martsearch/controller.rb', line 21

def datasources
  @datasources
end

- (Object) dataviews (readonly)

Returns the value of attribute dataviews



21
22
23
 
# File 'lib/martsearch/controller.rb', line 21

def dataviews
  @dataviews
end

- (Object) dataviews_by_name (readonly)

Returns the value of attribute dataviews_by_name



21
22
23
 
# File 'lib/martsearch/controller.rb', line 21

def dataviews_by_name
  @dataviews_by_name
end

- (Object) errors (readonly)

Returns the value of attribute errors



20
21
22
 
# File 'lib/martsearch/controller.rb', line 20

def errors
  @errors
end

- (Object) index (readonly)

Returns the value of attribute index



20
21
22
 
# File 'lib/martsearch/controller.rb', line 20

def index
  @index
end

- (Object) logger (readonly)

Returns the value of attribute logger



20
21
22
 
# File 'lib/martsearch/controller.rb', line 20

def logger
  @logger
end

- (Object) search_data (readonly)

Returns the value of attribute search_data



20
21
22
 
# File 'lib/martsearch/controller.rb', line 20

def search_data
  @search_data
end

- (Object) search_results (readonly)

Returns the value of attribute search_results



21
22
23
 
# File 'lib/martsearch/controller.rb', line 21

def search_results
  @search_results
end

Instance Method Details

- (Object) add_imits_data(mgi_accession_id, displayed_alleles) 



249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/martsearch/controller.rb', line 249

def add_imits_data( mgi_accession_id, displayed_alleles)
  tm1a = nil
  tm1e = nil
  tm1 = nil
  results = get_imits_mart_results( mgi_accession_id )

  #Work out if the starting allele is conditional (tm1a/e) or a deletion (tm1)
  # - if it's conditional, work out if the mouse is also conditional (blank mouse allele)
  # or if the mouse is targeted non-conditional (tm1e)
  results.each do |result|
    #only interested in GC or Cre-excised mice
    next unless ((result["microinjection_status"] == 'Genotype confirmed') or (result["phenotype_status"] == "Cre Excision Complete"))
    new_row = {}

    product = "Mouse"
    pipeline = result["consortium"]

    if(result["microinjection_status"] == 'Genotype confirmed')
      # original ES Cell allele is stored in imits redundantly
      new_row["product"] = product
      new_row["escell_pipeline"] = pipeline

      add_mouse_allele( result, displayed_alleles, new_row)

      #puts "after add_mouse_allele"
      #puts new_row

      add_order_fields(new_row, result)

      displayed_alleles["mouse_1"] = new_row
    end

    if(result["phenotype_status"] == "Cre Excision Complete")
      phenotype_allele_type = result["phenotype_allele_type"]
      allele_strain = displayed_alleles["mouse_1"]["escell_strain"]
      if(phenotype_allele_type)
        final_allele = 'Cre Excised'
        final_allele_id = 'not yet'
        new_row = {
          "product" => product,
          "escell_pipeline" => pipeline,
          "mutation_subtype" => final_allele,
          "escell_strain" => allele_strain,
          "allele_id" => final_allele_id
        }
        displayed_alleles["mouse_2"] = new_row
      end
    end
  end
end

- (Object) add_mouse_allele(result, displayed_alleles, new_row) 



209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/martsearch/controller.rb', line 209

def add_mouse_allele( result, displayed_alleles, new_row)
  starting_allele = result["allele_symbol_superscript"]
  final_allele = result["mouse_allele_symbol_superscript"]
  starting_allele = result["allele_symbol_superscript"]
  final_allele_symbol_superscript = nil
  final_allele = nil
  #puts "adding mouse allele"
  if(/tm\d\w/.match(starting_allele))
    #If there's no override to the input allele, then use it
    #puts "starting allele #{starting_allele}"
    if(final_allele.nil?)
      #puts "final allele is nil"
      final_allele = 'Conditional Ready'
      final_allele_id = displayed_alleles["tm1a"]["allele_id"]
      final_mgi_allele_id = displayed_alleles["tm1a"]["mgi_allele_id"]
      final_allele_symbol_superscript = displayed_alleles["tm1a"]["allele_symbol_superscript"]
    elsif(/tm\de/.match(final_allele))
      final_allele = 'Targeted Non-Conditional'
      final_allele_id = displayed_alleles["tm1e"]["allele_id"]
      final_mgi_allele_id = displayed_alleles["tm1e"]["mgi_allele_id"]
      final_allele_symbol_superscript = displayed_alleles["tm1e"]["allele_symbol_superscript"]
    end
  elsif(/tm\d/.match(starting_allele))
      final_allele = 'Deletion'
      final_allele_id = displayed_alleles["tm1"]["allele_id"]
      final_mgi_allele_id = displayed_alleles["tm1"]["mgi_allele_id"]
      final_allele_symbol_superscript = displayed_alleles["tm1"]["allele_symbol_superscript"]
  end

  allele_type = final_allele
  allele_strain = result["colony_background_strain"]

  new_row["mutation_subtype"] = final_allele
  new_row["escell_strain"] = allele_strain
  new_row["allele_id"] = final_allele_id
  new_row["mgi_allele_id"] = final_mgi_allele_id
  #puts "new row"
  #puts new_row
end

- (Object) add_order_details(result) 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/martsearch/controller.rb', line 93

def add_order_details(result)
  pipeline = result["escell_pipeline"]
  project_id = result["escell_ikmc_project_id"]
  if((pipeline == "EUCOMM") || (pipeline == "EUCOMMTools") || (pipeline == "EUCOMMToolsCre"))
    order_url = "http://www.eummcr.org/order.php"
    order_visual = "EUMMCR"
  elsif ((pipeline == "KOMP-CSD") || (pipeline == "KOMP-Regeneron"))
    if(project_id =~ /VG/)
      order_url = "http://www.komp.org/geneinfo.php?project=#{project_id}"
    else
      order_url = "http://www.komp.org/geneinfo.php?project=CSD#{project_id}"
    end
    order_visual = "KOMP"
  elsif ((pipeline == "MirKO") || (pipeline == "Sanger MGP"))
    order_url = "mailto:mouseinterest@sanger.ac.uk?Subject=Mutant ES Cell line for #{result["marker_symbol"]}"
    order_visual = "WTSI"
  elsif (pipeline == "NorCOMM")
    order_url = "http://www.phenogenomics.ca/services/cmmr/escell_services.html"
    order_visual = "NorCOMM"
  end
  result["order_url"] = order_url
  result["order_visual"] = order_visual
end

- (Object) add_order_fields(new_row, result) 



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/martsearch/controller.rb', line 187

def add_order_fields (new_row, result)
  order_url = nil
  order_visual = nil
  if(result["distribution_centre"]=="UCD")
    project_id = result["report_ikmc_project_id"]
    if(project_id =~ /VG/)
      order_url = "http://www.komp.org/geneinfo.php?project=#{project_id}"
    else
      order_url = "http://www.komp.org/geneinfo.php?project=CSD#{project_id}"
    end
    order_visual = "KOMP"
  elsif(result["emma"] == "1")
    order_url = "http://www.emmanet.org/mutant_types.php?keyword=#{result["marker_symbol"]}"
    order_visual = "EMMA"
  elsif(result["distribution_centre"] == "WTSI")
    order_url = "mailto:mouseinterest@sanger.ac.uk?Subject=Mutant mouse for #{result["marker_symbol"]}"
    order_visual = "WTSI"
  end
  new_row["order_url"] = order_url
  new_row["order_visual"] = order_visual
end

- (Object) add_targ_rep_data(mgi_accession_id, marker_symbol, displayed_alleles) 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/martsearch/controller.rb', line 117

def add_targ_rep_data(mgi_accession_id, marker_symbol, displayed_alleles)
  results = get_targrep_mart_results(mgi_accession_id)

  tm1a = nil
  tm1e = nil
  tm1 = nil
  results.each do |result|
    next unless !(result["parental_cell_line"].nil?)
    result["product"] = 'ES Cell'
    result["marker_symbol"] = marker_symbol

    add_order_details(result)

    if (result["mutation_subtype"] == 'conditional_ready' && tm1a.nil?)
      result["mutation_subtype"] = 'Conditional Ready'
      tm1a = result
    end
    if (result["mutation_subtype"] == 'targeted_non_conditional' && tm1e.nil?)
      result["mutation_subtype"] = 'Targeted NonConditional'
      tm1e = result
    end
    if (result["mutation_subtype"] == 'deletion' && tm1.nil?)
      result["mutation_subtype"] = 'Deletion'
      tm1 = result
    end
  end

  # Display the conditional in preference to the targeted non-conditional
  if(!tm1a.nil?)
    displayed_alleles["tm1a"] = tm1a
  elsif(!tm1e.nil?)
    displayed_alleles["tm1e"] = tm1e
  end

  # Display the deletion if we have it
  if(!tm1.nil?)
    displayed_alleles["tm1"] = tm1
  end
end

- (Hash) browse_counts(use_cache = true)

Function to load in the browsable content config and then query the index for each term and get a count of items returned…

Parameters:

  • use_cache (Boolean) (defaults to: true) —

    Use cached data if available

Returns:

  • (Hash) —

    A hash of all of the browsable content counts



417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
 
# File 'lib/martsearch/controller.rb', line 417

def browse_counts( use_cache=true )
  self.logger.debug("[MartSearch::Controller] ::browse_counts - running browse_counts( '#{use_cache}' )")
  counts = fetch_from_cache( "browse_counts" )
  if counts.nil? || use_cache == false
    all_ok = true
    counts = {}
    @config[:server][:browsable_content].each do |field,field_config|
      counts[field] = {}
      Parallel.each( field_config[:options].keys, :in_threads => 5 ) do |option|
        begin
          option_config         = field_config[:options][option]
          counts[field][option] = @index.count( option_config[:query] )
        rescue MartSearch::IndexSearchError => error
          all_ok                              = false
          counts[field][option] = nil
        end
      end
    end

    write_to_cache( "browse_counts", counts ) if all_ok
  end

  return counts
end

- (Object/nil) fetch_from_cache(key)

Cache interaction helper - fetch data from the cache for a given key.

Parameters:

  • key (String) —

    The cache identifer to look up

Returns:

  • (Object/nil) —

    The deserialized object from the cache, or nil if none found



487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
 
# File 'lib/martsearch/controller.rb', line 487

def fetch_from_cache( key )
  return nil if ! USE_CACHE
  self.logger.debug("[MartSearch::Controller] ::fetch_from_cache - running fetch_from_cache( '#{key}' )")
  cached_data = @cache.fetch( key )

  unless cached_data.nil?
    if @cache.is_a?(MartSearch::MongoCache)
      cached_data = JSON.parse( cached_data['json'], :max_nesting => false )
    else
      cached_data = JSON.parse( cached_data, :max_nesting => false )
    end
    cached_data = cached_data.clean_hash if RUBY_VERSION < '1.9'
    cached_data.recursively_symbolize_keys!
  end
  self.logger.debug("[MartSearch::Controller] ::fetch_from_cache - running fetch_from_cache( '#{key}' ) - DONE")

  return cached_data
end

- (Object) get_imits_mart_results(mgi_accession_id) 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/martsearch/controller.rb', line 157

def get_imits_mart_results(mgi_accession_id)
  imits_mart = datasources[:ikmc-imits'].ds
  error_string  = "Information on Mice is missing"
  results       = handle_biomart_errors( "ikmc-imits", error_string ) do
  imits_mart.search({
      :process_results => true,
      :filters         => {
        'mgi_accession_id' => mgi_accession_id,
      },
      :attributes      => [
        'marker_symbol',
        'consortium',
        'distribution_centre',
        'colony_background_strain',
        'production_centre',
        'distribution_centre',
        'emma',
        'microinjection_status',
        'allele_symbol_superscript',
        'mouse_allele_symbol_superscript',
        'phenotype_allele_type',
        'phenotype_status',
        'phenotype_is_active',
        'distribution_centre',
        'report_ikmc_project_id'
      ].flatten
    })
  end
end

- (Object) get_targrep_mart_results(mgi_accession_id) 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/martsearch/controller.rb', line 70

def get_targrep_mart_results(mgi_accession_id)
  targ_rep_mart = datasources[:ikmc-idcc_targ_rep'].ds
  error_string  = "Information on Targeting Vectors and ES Cells is missing"
  results       = handle_biomart_errors( "ikmc-ikmc-targ_rep", error_string ) do
    targ_rep_mart.search({
      :process_results => true,
      :filters         => { 'mgi_accession_id' => mgi_accession_id},
      :attributes      => [
        'mutation_subtype',
        'parental_cell_line',
        'allele_symbol_superscript',
        'mgi_allele_id',
        'allele_id',
        'parental_cell_line',
        'escell_strain',
        'escell_pipeline',
        'escell_ikmc_project_id'
      ].flatten
    })
  end
  return results
end

- (Object) handle_biomart_errors(data_source, error_string) 



300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/martsearch/controller.rb', line 300

def handle_biomart_errors( data_source, error_string )
  MartSearch::Controller.instance().logger.debug("[MartSearch::ProjectUtils] ::handle_biomart_errors - running handle_biomart_errors( '#{data_source}', '#{error_string}' )")

  results      = { :data => {}, :error => {} }
  error_prefix = "There was a problem querying the '#{data_source}' biomart."
  error_suffix = "Try refreshing your browser or come back in 10 minutes."
  begin
    results[:data] = yield
  rescue Biomart::BiomartError => error
    results[:error] = {
      :text  => error_prefix + " " + error_string + " " + error_suffix,
      :error => error.to_s,
      :type  => error.class
    }
  rescue Timeout::Error => error
    results[:error] = {
      :text  => error_prefix + " " + error_string + " " + error_suffix,
      :error => error.to_s,
      :type  => error.class
    }
  end
end

- (Array) search(query, page = 1, use_cache = true, save_index_data = true)

Function to perform the searches against the index and marts.

Sets up a results stash (@search_data) holding the data in a structure like:

{
  IndexDocUniqueKey => {
    "index"         => {}, # index results for this doc
    "internal_name" => []/{}, # array/hash of sorted biomart data
    "internal_name" => []/{}, # array/hash of sorted biomart data
  }
}

But returns an ordered list of the results/index docs (@search_results)

Parameters:

  • query (String) —

    The query string to pass to the search index

  • page (Integer) (defaults to: 1) —

    The page of results to search for/return

  • use_cache (Boolean) (defaults to: true) —

    Use cached data if available

  • save_index_data (Boolean) (defaults to: true) —

    Try to save the index return to the cache

Returns:

  • (Array) —

    A list of the search results (primary index fields)



361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
 
# File 'lib/martsearch/controller.rb', line 361

def search( query, page=1, use_cache=true, save_index_data=true )
  self.logger.debug("[MartSearch::Controller] ::search - running search( '#{query}', #{page}, #{use_cache}, #{save_index_data} )")
  page = 1 if page == 0
  clear_instance_variables

  cached_index_data = fetch_from_cache( "index:#{query}-page#{page}" )
  if cached_index_data != nil and use_cache
    search_from_cached_index( cached_index_data )
  else
    if search_from_fresh_index( query, page ) and save_index_data
      obj_to_cache    = {
        :search_data           => @search_data,
        :search_results        => @search_results,
        :current_page          => @index.current_page,
        :current_results_total => @index.current_results_total,
        :cache_timestamp       => DateTime.now.to_s
      }
      write_to_cache( "index:#{query}-page#{page}", obj_to_cache )
    end
  end

  unless @search_data.empty?
    fresh_ds_queries_to_do = []

    @search_data.each do |data_key,data|
      cached_dataset_data = fetch_from_cache( "datasets:#{data_key}" )

      if cached_dataset_data != nil and use_cache
        @search_data[data_key] = cached_dataset_data.merge(data)
      else
        fresh_ds_queries_to_do.push(data_key)
      end
    end

    unless fresh_ds_queries_to_do.empty?
      grouped_search_terms = prepare_dataset_search_terms( fresh_ds_queries_to_do )
      if search_from_fresh_datasets( fresh_ds_queries_to_do, grouped_search_terms )
        fresh_ds_queries_to_do.each do |data_key|
          unless @search_data[data_key].nil?
            @search_data[data_key][:cache_timestamp] = DateTime.now.to_s
            write_to_cache( "datasets:#{data_key}", @search_data[data_key] )
          end
        end
      end
    end
  end

  # Return paged_results
  return @search_results
end

- (Object) search_impc(mgi_accession_id) 



323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'lib/martsearch/controller.rb', line 323

def search_impc (mgi_accession_id)
  @displayed_alleles = {}

  index_return = @index.search( mgi_accession_id, 1)
  gene_doc = index_return[mgi_accession_id.to_sym]
  marker_symbol = nil
  if(!gene_doc.nil?)
    marker_symbol = "#{gene_doc[:index][:marker_symbol]}"
  end

  # add the tm1a or tm1e allele AND the tm1 allele if they exist
  add_targ_rep_data( mgi_accession_id, marker_symbol, @displayed_alleles)

  # add the corresponding mouse alleles if they exist
  add_imits_data( mgi_accession_id, @displayed_alleles)

  @displayed_alleles["order"] = ["tm1a","tm1","tm1e","mouse_1","mouse_2"]
  return @displayed_alleles
end

- (Object) write_to_cache(key, value, options = {})

Cache interaction helper - use this to store data in the cache.

Parameters:

  • key (String) —

    The cache identifer to store 'value' under

  • value (Object) —

    The cache 'value' to store

  • options (Hash) (defaults to: {}) —

    An options hash (see #ActiveSupport::Cache for more info)



511
512
513
514
515
516
517
518
519
520
521
 
# File 'lib/martsearch/controller.rb', line 511

def write_to_cache( key, value, options={} )
  return if ! USE_CACHE
  self.logger.debug("[MartSearch::Controller] ::write_to_cache - running write_to_cache( '#{key}', Object, '#{options.inspect}' )")
  @cache.delete( key )
  if @cache.is_a?(MartSearch::MongoCache)
    @cache.write( key, { 'json' => JSON.generate( value, :max_nesting => false ) }, { :expires_in => 36.hours }.merge(options) )
  else
    @cache.write( key, JSON.generate( value, :max_nesting => false ), { :expires_in => 36.hours }.merge(options) )
  end
  self.logger.debug("[MartSearch::Controller] ::write_to_cache - running write_to_cache( '#{key}', Object, '#{options.inspect}' ) - DONE")
end

- (Hash) wtsi_phenotyping_progress_counts(use_cache = true)

Function to calculate the progress of the WTSI Mouse Genetics Project (MGP). This should return counts for three categories:

- Number of genes with lines with Standard Phenotyping (MGP pipeline) done
- Number of genes with lines with Infection Challenge (Citrobacter & Salmonella) done
- Number of genes with lines with Expression (embryo and adult) done

Parameters:

  • use_cache (Boolean) (defaults to: true) —

    Use cached data if available

Returns:

  • (Hash) —

    A hash of the status counts that the MGP wants

Raises:



450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
 
# File 'lib/martsearch/controller.rb', line 450

def wtsi_phenotyping_progress_counts( use_cache=true )
  self.logger.debug("[MartSearch::Controller] ::wtsi_phenotyping_progress_counts - running wtsi_phenotyping_progress_counts( '#{use_cache}' )")
  heatmap_dataset = self.datasets[:wtsi-phenotyping-heatmap']
  raise MartSearch::InvalidConfigError, "MartSearch::Controller.wtsi_phenotyping_progress_counts cannot be called if the 'wtsi-phenotyping-heatmap' dataset is inactive" if heatmap_dataset.nil?

  counts = fetch_from_cache( "wtsi_phenotyping_progress_counts" )
  if counts.nil? || use_cache == false
    heatmap_test_groups_conf = heatmap_dataset.config[:test_groups]
    heatmap_mart             = heatmap_dataset.datasource.ds
    counts                   = {}
    all_ok                   = true

    begin
      counts = {
        :standard_phenotyping => complete_mgp_alleles_count( heatmap_mart, ['haematology_cbc'], ['CompleteInteresting','CompleteNotInteresting'] ),
        :infection_challenge  => complete_mgp_alleles_count( heatmap_mart, ['salmonella_challenge','citrobacter_challenge'], ['CompleteInteresting','CompleteNotInteresting'] ),
        :expression           => complete_mgp_alleles_count( heatmap_mart, ['adult_lac_z_expression','embryo_lac_z_expression'], ['CompleteDataAvailable'] )
      }
    rescue Biomart::BiomartError => error
      all_ok = false
      counts = {
        :standard_phenotyping => counts[:standard_phenotyping]  ? counts[:standard_phenotyping] : '-',
        :infection_challenge  => counts[:infection_challenge]   ? counts[:infection_challenge]  : '-',
        :expression           => counts[:expression]            ? counts[:expression]           : '-',
      }
    end

    write_to_cache( "wtsi_phenotyping_progress_counts", counts, { :expires_in => 12.hours } ) if all_ok
  end

  return counts
end