Module: Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers

Includes:
MigrationHelpers, MigrationHelpers::LooseForeignKeyHelpers, SchemaHelpers
Included in:
Gitlab::Database::PartitioningMigrationHelpers
Defined in:
lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb

Constant Summary collapse

ALLOWED_TABLES = 
%w[group_audit_events project_audit_events instance_audit_events user_audit_events
audit_events web_hook_logs merge_request_diff_files merge_request_diff_commits
ci_runners ci_runner_machines uploads sent_notifications].freeze
ERROR_SCOPE = 
'table partitioning'
MIGRATION_CLASS_NAME = 
"::#{module_parent_name}::BackfillPartitionedTable"
MIGRATION = 
"BackfillPartitionedTable"
BATCH_INTERVAL = 
2.minutes.freeze
BATCH_SIZE = 
50_000
SUB_BATCH_SIZE = 
2_500
PARTITION_BUFFER = 
6
MIN_ID = 
1

Constants included from MigrationHelpers::LooseForeignKeyHelpers

MigrationHelpers::LooseForeignKeyHelpers::INSERT_FUNCTION_NAME, MigrationHelpers::LooseForeignKeyHelpers::INSERT_FUNCTION_NAME_OVERRIDE_TABLE

Constants included from MigrationHelpers

MigrationHelpers::DEFAULT_TIMESTAMP_COLUMNS, MigrationHelpers::ENFORCE_INITIALIZE_ALL_INT_IDS_FROM_MILESTONE, MigrationHelpers::INTEGER_IDS_YET_TO_INITIALIZED_TO_BIGINT_FILE_PATH, MigrationHelpers::PENDING_INT_IDS_ERROR_MSG, MigrationHelpers::TABLE_INT_IDS_YAML_FILE_COMMENT

Constants included from Migrations::RedisHelpers

Migrations::RedisHelpers::SCAN_START_CURSOR

Constants included from Migrations::SidekiqHelpers

Migrations::SidekiqHelpers::DEFAULT_MAX_ATTEMPTS, Migrations::SidekiqHelpers::DEFAULT_TIMES_IN_A_ROW

Constants included from Migrations::ConstraintsHelpers

Migrations::ConstraintsHelpers::MAX_IDENTIFIER_NAME_LENGTH

Constants included from Migrations::BatchedBackgroundMigrationHelpers

Migrations::BatchedBackgroundMigrationHelpers::BATCH_CLASS_NAME, Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_DELAY, Migrations::BatchedBackgroundMigrationHelpers::BATCH_MIN_VALUE, Migrations::BatchedBackgroundMigrationHelpers::EARLY_FINALIZATION_ERROR, Migrations::BatchedBackgroundMigrationHelpers::ENFORCE_EARLY_FINALIZATION_FROM_VERSION, Migrations::BatchedBackgroundMigrationHelpers::MIGRATION_NOT_FOUND_MESSAGE, Migrations::BatchedBackgroundMigrationHelpers::MINIMUM_PAUSE_MS, Migrations::BatchedBackgroundMigrationHelpers::NonExistentMigrationError

Constants included from Migrations::BackgroundMigrationHelpers

Migrations::BackgroundMigrationHelpers::JOB_BUFFER_SIZE

Instance Method Summary collapse

Methods included from MigrationHelpers::LooseForeignKeyHelpers

#has_loose_foreign_key?, #track_record_deletions, #track_record_deletions_override_table_name, #untrack_record_deletions

Methods included from SchemaHelpers

#assert_not_in_transaction_block, #create_comment, #create_trigger, #create_trigger_function, #drop_function, #drop_trigger, #find_all_id_columns_sql, #function_exists?, #object_name, #reset_all_trigger_functions, #reset_trigger_function, #tmp_table_name, #trigger_exists?

Methods included from MigrationHelpers

#add_concurrent_foreign_key, #add_concurrent_index, #add_primary_key_using_index, #add_sequence, #add_timestamps_with_timezone, #backfill_conversion_of_integer_to_bigint, #backfill_iids, #change_column_type_concurrently, #check_trigger_permissions!, #cleanup_concurrent_column_rename, #cleanup_concurrent_column_type_change, #cleanup_conversion_of_integer_to_bigint, #column_for, #concurrent_foreign_key_name, #convert_to_bigint_column, #convert_to_type_column, #copy_foreign_keys, #copy_indexes, #create_or_update_plan_limit, #define_batchable_model, #drop_sequence, #each_batch, #each_batch_range, #ensure_backfill_conversion_of_integer_to_bigint_is_finished, #false_value, #foreign_key_exists?, #foreign_keys_for, #index_exists_by_name?, #index_invalid?, #indexes_for, #initialize_conversion_of_integer_to_bigint, #install_rename_triggers, #install_sharding_key_assignment_trigger, #lock_tables, #postgres_exists_by_name?, #remove_column_default, #remove_concurrent_index, #remove_concurrent_index_by_name, #remove_foreign_key_if_exists, #remove_foreign_key_without_error, #remove_rename_triggers, #remove_sharding_key_assignment_trigger, #remove_timestamps, #rename_column_concurrently, #rename_trigger_name, #replace_sql, #restore_conversion_of_integer_to_bigint, #revert_backfill_conversion_of_integer_to_bigint, #revert_initialize_conversion_of_integer_to_bigint, #swap_primary_key, #table_integer_ids, #true_value, #undo_change_column_type_concurrently, #undo_cleanup_concurrent_column_rename, #undo_cleanup_concurrent_column_type_change, #undo_rename_column_concurrently, #update_column_in_batches, #validate_foreign_key

Methods included from Gitlab::Database::PartitionHelpers

#partition?, #table_partitioned?

Methods included from MigrationHelpers::WraparoundVacuumHelpers

#check_if_wraparound_in_progress

Methods included from AsyncConstraints::MigrationHelpers

#prepare_async_check_constraint_validation, #prepare_async_foreign_key_validation, #prepare_partitioned_async_check_constraint_validation, #prepare_partitioned_async_foreign_key_validation, #unprepare_async_check_constraint_validation, #unprepare_async_foreign_key_validation, #unprepare_partitioned_async_check_constraint_validation, #unprepare_partitioned_async_foreign_key_validation

Methods included from AsyncIndexes::MigrationHelpers

#async_index_creation_available?, #prepare_async_index, #prepare_async_index_from_sql, #prepare_async_index_removal, #unprepare_async_index, #unprepare_async_index_by_name

Methods included from RenameTableHelpers

#finalize_table_rename, #rename_table_safely, #undo_finalize_table_rename, #undo_rename_table_safely

Methods included from MigrationHelpers::FeatureFlagMigratorHelpers

#down_migrate_to_jsonb_setting, #down_migrate_to_setting, #up_migrate_to_jsonb_setting, #up_migrate_to_setting

Methods included from DynamicModelHelpers

define_batchable_model, #each_batch, #each_batch_range

Methods included from Migrations::RedisHelpers

#queue_redis_migration_job

Methods included from Migrations::SidekiqHelpers

#migrate_across_instance, #migrate_within_instance, #sidekiq_queue_migrate, #sidekiq_remove_jobs

Methods included from Migrations::ExtensionHelpers

#create_extension, #drop_extension

Methods included from Migrations::ConstraintsHelpers

#add_check_constraint, #add_multi_column_not_null_constraint, #add_not_null_constraint, #add_text_limit, #check_constraint_exists?, check_constraint_exists?, #check_constraint_name, #check_not_null_constraint_exists?, #check_text_limit_exists?, #copy_check_constraints, #drop_constraint, #remove_check_constraint, #remove_multi_column_not_null_constraint, #remove_not_null_constraint, #remove_text_limit, #rename_constraint, #switch_constraint_names, #text_limit_name, #validate_check_constraint, #validate_check_constraint_name!, #validate_multi_column_not_null_constraint, #validate_not_null_constraint, #validate_text_limit

Methods included from Migrations::TimeoutHelpers

#disable_statement_timeout

Methods included from Migrations::LockRetriesHelpers

#with_lock_retries

Methods included from Migrations::BatchedBackgroundMigrationHelpers

#delete_batched_background_migration, #ensure_batched_background_migration_is_finished, #finalize_batched_background_migration, #gitlab_schema_from_context, #queue_batched_background_migration

Methods included from Migrations::BackgroundMigrationHelpers

#delete_job_tracking, #delete_queued_jobs, #finalize_background_migration, #migrate_in, #queue_background_migration_jobs_by_range_at_intervals, #requeue_background_migration_jobs_by_range_at_intervals

Methods included from Migrations::ReestablishedConnectionStack

#with_restored_connection_stack

Instance Method Details

#cleanup_partitioning_data_migration(table_name, migration = MIGRATION) ⇒ Object

Cleanup a previously enqueued background migration to copy data into a partitioned table. This will not prevent the enqueued jobs from executing, but instead cleans up information in the database used to track the state of the batched background migration. It should be safe to also remove the partitioned table even if the background jobs are still in-progress, as the absence of the table will cause them to safely exit.

Example:

cleanup_partitioning_data_migration :audit_events



253
254
255
256
257
258
259
260
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 253

def cleanup_partitioning_data_migration(table_name, migration = MIGRATION)
  assert_table_is_allowed(table_name)

  partitioned_table_name = make_partitioned_table_name(table_name)
  primary_key = connection.primary_key(table_name)

  delete_batched_background_migration(migration, table_name, primary_key, [partitioned_table_name])
end

#convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, lock_tables: []) ⇒ Object 



411
412
413
414
415
416
417
418
419
420
421
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 411

def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, lock_tables: [])
  validate_not_in_transaction!(:convert_table_to_first_list_partition)

  Gitlab::Database::Partitioning::List::ConvertTable
    .new(migration_context: self,
      table_name: table_name,
      parent_table_name: parent_table_name,
      partitioning_column: partitioning_column,
      zero_partition_value: initial_partitioning_value
    ).partition
end

#create_hash_partitions(table_name, number_of_partitions) ⇒ Object 



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 262

def create_hash_partitions(table_name, number_of_partitions)
  transaction do
    (0..number_of_partitions - 1).each do |partition|
      decimals = Math.log10(number_of_partitions).ceil
      suffix = "%0#{decimals}d" % partition
      partition_name = "#{table_name}_#{suffix}"
      schema = Gitlab::Database::STATIC_PARTITIONS_SCHEMA

      execute(<<~SQL)
        CREATE TABLE #{schema}.#{partition_name}
        PARTITION OF #{table_name}
        FOR VALUES WITH (MODULUS #{number_of_partitions}, REMAINDER #{partition});
      SQL
    end
  end
end

#create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key) ⇒ Object 



373
374
375
376
377
378
379
380
381
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 373

def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key)
  function_name = make_sync_function_name(source_table_name)
  trigger_name = make_sync_trigger_name(source_table_name)

  create_sync_function(function_name, source_table_name, partitioned_table_name, unique_key)
  create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")

  create_sync_trigger(source_table_name, trigger_name, function_name)
end

#drop_nonpartitioned_archive_table(table_name) ⇒ Object 



361
362
363
364
365
366
367
368
369
370
371
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 361

def drop_nonpartitioned_archive_table(table_name)
  assert_table_is_allowed(table_name)

  archived_table_name = make_archived_table_name(table_name)

  with_lock_retries do
    drop_sync_trigger(table_name)
  end

  drop_table(archived_table_name)
end

#drop_partitioned_table_for(table_name) ⇒ Object

Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were used to copy writes to the partitioned table, then removes the partitioned table (also removing partitions).

Example:

drop_partitioned_table_for :audit_events



206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 206

def drop_partitioned_table_for(table_name)
  assert_table_is_allowed(table_name)
  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  with_lock_retries do
    drop_sync_trigger(table_name)
  end

  partitioned_table_name = make_partitioned_table_name(table_name)
  drop_table(partitioned_table_name)
end

#drop_trigger_to_sync_tables(source_table_name) ⇒ Object 



383
384
385
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 383

def drop_trigger_to_sync_tables(source_table_name)
  drop_sync_trigger(source_table_name)
end

#enqueue_partitioning_data_migration(table_name, migration = MIGRATION) ⇒ Object

Enqueue the background jobs that will backfill data in the partitioned table, by batch-copying records from original table. This helper should be called from a post-deploy migration.

Example:

enqueue_partitioning_data_migration :audit_events



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 225

def enqueue_partitioning_data_migration(table_name, migration = MIGRATION)
  assert_table_is_allowed(table_name)

  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  partitioned_table_name = make_partitioned_table_name(table_name)
  primary_key = connection.primary_key(table_name)

  queue_batched_background_migration(
    migration,
    table_name,
    primary_key,
    partitioned_table_name,
    batch_size: BATCH_SIZE,
    sub_batch_size: SUB_BATCH_SIZE,
    job_interval: BATCH_INTERVAL
  )
end

#finalize_backfilling_partitioned_table(table_name) ⇒ Object

Executes jobs from previous BatchedBackgroundMigration to backfill the partitioned table by finishing pending jobs.

NOTE Migrations using this method cannot be scheduled in the same release as the migration that schedules the background migration using the ‘enqueue_partitioning_data_migration` helper, or else the background migration jobs will be force-executed.

Example:

finalize_backfilling_partitioned_table :audit_events



290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 290

def finalize_backfilling_partitioned_table(table_name)
  assert_table_is_allowed(table_name)

  partitioned_table_name = make_partitioned_table_name(table_name)

  unless table_exists?(partitioned_table_name)
    raise "could not find partitioned table for #{table_name}, " \
      "this could indicate the previous partitioning migration has been rolled back."
  end

  ensure_batched_background_migration_is_finished(
    job_class_name: MIGRATION,
    table_name: table_name,
    column_name: connection.primary_key(table_name),
    job_arguments: [partitioned_table_name]
  )

  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
    disable_statement_timeout do
      execute("VACUUM FREEZE ANALYZE #{partitioned_table_name}")
    end
  end
end

#partition_table_by_date(table_name, column_name, min_date: nil, max_date: nil) ⇒ Object

Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. One partition is created per month between the given ‘min_date` and `max_date`. Also installs a trigger on the original table to copy writes into the partitioned table. To copy over historic data from before creation of the partitioned table, use the `enqueue_partitioning_data_migration` helper in a post-deploy migration.

A copy of the original table is required as PG currently does not support partitioning existing tables.

Example:

partition_table_by_date :audit_events, :created_at, min_date: Date.new(2020, 1), max_date: Date.new(2020, 6)

Options are:

:min_date - a date specifying the lower bounds of the partition range
:max_date - a date specifying the upper bounds of the partitioning range, defaults to today + 1 month

Unless min_date is specified explicitly, we default to

  1. The minimum value for the partitioning column in the table

  2. If no data is present yet, the current month



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 95

def partition_table_by_date(table_name, column_name, min_date: nil, max_date: nil)
  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!

  assert_table_is_allowed(table_name)

  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  max_date ||= Date.today + 1.month

  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
    min_date ||= connection.select_one(<<~SQL)['minimum'] || (max_date - 1.month)
      SELECT date_trunc('MONTH', MIN(#{column_name})) AS minimum
      FROM #{table_name}
    SQL
  end

  raise "max_date #{max_date} must be greater than min_date #{min_date}" if min_date >= max_date

  primary_key = connection.primary_key(table_name)
  raise "primary key not defined for #{table_name}" if primary_key.nil?

  partition_column = find_column_definition(table_name, column_name)
  raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil?

  partitioned_table_name = make_partitioned_table_name(table_name)

  transaction do
    create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key)
    create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date)
  end

  with_lock_retries do
    create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key)
  end
end

#partition_table_by_int_range(table_name, column_name, partition_size:, primary_key:) ⇒ Object

Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a int/bigint column. One partition is created per partition_size between 1 and MAX(column_name). Also installs a trigger on the original table to copy writes into the partitioned table. To copy over historic data from before creation of the partitioned table, use the ‘enqueue_partitioning_data_migration` helper in a post-deploy migration. Note: If the original table is empty the system creates 6 partitions in the new table.

A copy of the original table is required as PG currently does not support partitioning existing tables.

Example:

partition_table_by_int_range :merge_request_diff_commits, :merge_request_diff_id, partition_size: 500, primary_key: ['merge_request_diff_id', 'relative_order']

Options are:

:partition_size - a int specifying the partition size
:primary_key - a array specifying the primary query of the new table

Note: The system always adds a buffer of 6 partitions.



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 42

def partition_table_by_int_range(table_name, column_name, partition_size:, primary_key:)
  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!

  assert_table_is_allowed(table_name)

  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  current_primary_key = Array.wrap(connection.primary_key(table_name))
  raise "primary key not defined for #{table_name}" if current_primary_key.blank?

  partition_column = find_column_definition(table_name, column_name)
  raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil?

  primary_key = Array.wrap(primary_key).map(&:to_s)
  raise "the partition column must be part of the primary key" unless primary_key.include?(column_name.to_s)

  primary_key_objects = connection.columns(table_name).select { |column| primary_key.include?(column.name) }

  raise 'partition_size must be greater than 1' unless partition_size > 1

  max_id = Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
    Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection.with_suppressed do
      define_batchable_model(table_name, connection: connection).maximum(column_name) || (partition_size * PARTITION_BUFFER)
    end
  end

  partitioned_table_name = make_partitioned_table_name(table_name)

  with_lock_retries do
    create_range_id_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key_objects)
    create_int_range_partitions(partitioned_table_name, partition_size, MIN_ID, max_id)
    create_trigger_to_sync_tables(table_name, partitioned_table_name, current_primary_key)
  end
end

#partition_table_by_list(table_name, column_name, primary_key:, partition_mappings: nil, partition_name_format: nil, create_partitioned_table_fn: nil, sync_trigger: true) ⇒ Object

Creates a partitioned copy of an existing table, using a LIST partitioning strategy on a int/bigint column. One partition is created per column_name value. Also installs a trigger on the original table to copy writes into the partitioned table. To copy over historic data from before creation of the partitioned table, use the ‘enqueue_partitioning_data_migration` helper in a post-deploy migration.

A copy of the original table is required as PG currently does not support partitioning existing tables.

Example:

partition_table_by_list :ci_runners, :runner_type, primary_key: ['id', 'runner_type'],
  partition_mappings: { instance_type: 1, group_type: 2, project_type: 3 },
  partition_name_format: "%{partition_name}_%{table_name}",
  create_partitioned_table_fn: ->(name) { create_custom_partitioned_table(name) }

Options are:

:primary_key - a array specifying the primary query of the new table
:partition_name_format - the format to be used when naming partitions.
  The %{table_name} and %{partition_name} variables are made available.
  If not specified, a default is generated
:partition_mappings - a hash specifying the mappings between partition name and respective column value(s)
:create_partitioned_table_fn - a lambda allowing a custom function to create the partitioned table
  If not specified, the partitioned table will be created with the same schema as the non-partitioned table



155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 155

def partition_table_by_list(
  table_name, column_name,
  primary_key:, partition_mappings: nil, partition_name_format: nil,
  create_partitioned_table_fn: nil, sync_trigger: true
)
  Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!

  assert_table_is_allowed(table_name)

  assert_not_in_transaction_block(scope: ERROR_SCOPE)

  current_primary_key = Array.wrap(connection.primary_key(table_name))
  raise "primary key not defined for #{table_name}" if current_primary_key.blank?

  partition_column = find_column_definition(table_name, column_name)
  raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil?

  primary_key = Array.wrap(primary_key).map(&:to_s)
  raise "the partition column must be part of the primary key" unless primary_key.include?(column_name.to_s)

  primary_key_objects = connection.columns(table_name).select { |column| primary_key.include?(column.name) }

  if partition_mappings.nil?
    distinct_partitions = Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
      Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection.with_suppressed do
        define_batchable_model(table_name, connection: connection).distinct(column_name).pluck(column_name)
      end
    end

    partition_mappings = distinct_partitions.to_h { |partition_id| [partition_id, partition_id] }
  end

  raise 'partition_mappings must contain more than one partition' unless partition_mappings.count > 1

  partitioned_table_name = make_partitioned_table_name(table_name)

  with_lock_retries do
    create_list_partitioned_copy(
      table_name, partitioned_table_name, partition_column, primary_key_objects, create_partitioned_table_fn)
    create_list_partitions(partitioned_table_name, partition_mappings, partition_name_format)
    create_trigger_to_sync_tables(table_name, partitioned_table_name, current_primary_key) if sync_trigger
  end
end

#prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, async: false) ⇒ Object 



387
388
389
390
391
392
393
394
395
396
397
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 387

def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, async: false)
  validate_not_in_transaction!(:prepare_constraint_for_list_partitioning)

  Gitlab::Database::Partitioning::List::ConvertTable
    .new(migration_context: self,
      table_name: table_name,
      parent_table_name: parent_table_name,
      partitioning_column: partitioning_column,
      zero_partition_value: initial_partitioning_value
    ).prepare_for_partitioning(async: async)
end

#replace_with_partitioned_table(table_name) ⇒ Object

Replaces a non-partitioned table with its partitioned copy. This is the final step in a partitioning migration, which makes the partitioned table ready for use by the application. The partitioned copy should be replaced with the original table in such a way that it appears seamless to any database clients. The replaced table will be renamed to “#replaced_table_archived”. Partitions and primary key constraints will also be renamed to match the naming scheme of the parent table.

NOTE This method should only be used after all other migration steps have completed successfully. There are several limitations to this method that MUST be handled before, or during, the swap migration:

  • Secondary indexes and foreign keys are not automatically recreated on the partitioned table.

  • Some types of constraints (UNIQUE and EXCLUDE) which rely on indexes, will not automatically be recreated on the partitioned table, since the underlying index will not be present.

  • Foreign keys referencing the original non-partitioned table, would also need to be updated to reference the partitioned table, but unfortunately this is not supported in PG11.

  • Views referencing the original table will not be automatically updated to reference the partitioned table.

Example:

replace_with_partitioned_table :audit_events



334
335
336
337
338
339
340
341
342
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 334

def replace_with_partitioned_table(table_name)
  assert_table_is_allowed(table_name)

  partitioned_table_name = make_partitioned_table_name(table_name)
  archived_table_name = make_archived_table_name(table_name)
  primary_key_name = connection.primary_key(table_name)

  replace_table(table_name, partitioned_table_name, archived_table_name, primary_key_name)
end

#revert_converting_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) ⇒ Object 



423
424
425
426
427
428
429
430
431
432
433
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 423

def revert_converting_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:)
  validate_not_in_transaction!(:revert_converting_table_to_first_list_partition)

  Gitlab::Database::Partitioning::List::ConvertTable
    .new(migration_context: self,
      table_name: table_name,
      parent_table_name: parent_table_name,
      partitioning_column: partitioning_column,
      zero_partition_value: initial_partitioning_value
    ).revert_partitioning
end

#revert_preparing_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) ⇒ Object 



399
400
401
402
403
404
405
406
407
408
409
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 399

def revert_preparing_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:)
  validate_not_in_transaction!(:revert_preparing_constraint_for_list_partitioning)

  Gitlab::Database::Partitioning::List::ConvertTable
    .new(migration_context: self,
      table_name: table_name,
      parent_table_name: parent_table_name,
      partitioning_column: partitioning_column,
      zero_partition_value: initial_partitioning_value
    ).revert_preparation_for_partitioning
end

#rollback_replace_with_partitioned_table(table_name) ⇒ Object

Rolls back a migration that replaced a non-partitioned table with its partitioned copy. This can be used to restore the original non-partitioned table in the event of an unexpected issue.

Example:

rollback_replace_with_partitioned_table :audit_events



351
352
353
354
355
356
357
358
359
 
# File 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb', line 351

def rollback_replace_with_partitioned_table(table_name)
  assert_table_is_allowed(table_name)

  partitioned_table_name = make_partitioned_table_name(table_name)
  archived_table_name = make_archived_table_name(table_name)
  primary_key_name = connection.primary_key(archived_table_name)

  replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name)
end