Class: Gitlab::UsageData

Inherits:
Object
  • Object
show all
Extended by:
Gitlab::Usage::TimeFrame, Gitlab::Utils::StrongMemoize, Gitlab::Utils::UsageData
Defined in:
lib/gitlab/usage_data.rb,
lib/gitlab/usage_data/topology.rb

Direct Known Subclasses

UsageDataNonSqlMetrics, UsageDataQueries

Defined Under Namespace

Classes: Topology

Constant Summary collapse

MAX_GENERATION_TIME_FOR_SAAS = 
40.hours
CE_MEMOIZED_VALUES = 
i[
  issue_minimum_id
  issue_maximum_id
  project_minimum_id
  project_maximum_id
  user_minimum_id
  user_maximum_id
  deployment_minimum_id
  deployment_maximum_id
  auth_providers
  recorded_at
].freeze

Constants included from Gitlab::Usage::TimeFrame

Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::DEFAULT_TIMESTAMP_COLUMN, Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME, Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME

Constants included from Gitlab::Utils::UsageData

Gitlab::Utils::UsageData::DISTRIBUTED_HLL_FALLBACK, Gitlab::Utils::UsageData::FALLBACK, Gitlab::Utils::UsageData::HISTOGRAM_FALLBACK, Gitlab::Utils::UsageData::MAX_BUCKET_SIZE

Class Method Summary collapse

Methods included from Gitlab::Usage::TimeFrame

monthly_time_range, monthly_time_range_db_params, weekly_time_range, weekly_time_range_db_params

Methods included from Gitlab::Utils::UsageData

add, add_metric, alt_usage_data, average, count, distinct_count, estimate_batch_distinct_count, histogram, maximum_id, measure_duration, metrics_collection_metadata, minimum_id, redis_usage_data, sum, track_usage_event, with_finished_at, with_metadata, with_prometheus_client

Class Method Details

.components_usage_data ⇒ Object 



147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/gitlab/usage_data.rb', line 147

def components_usage_data
  {
    git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } },
    gitaly: {
      version: alt_usage_data { Gitaly::Server.all.first.server_version },
      servers: alt_usage_data { Gitaly::Server.count },
      clusters: alt_usage_data { Gitaly::Server.gitaly_clusters },
      filesystems: alt_usage_data(fallback: ["-1"]) { Gitaly::Server.filesystems }
    },
    gitlab_pages: {
      enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled },
      version: alt_usage_data { Gitlab::Pages::VERSION }
    },
    container_registry_server: {
      vendor: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_vendor },
      version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version }
    },
    database: {
      # rubocop: disable UsageData/LargeTable
      adapter: alt_usage_data { ApplicationRecord.database.adapter_name },
      version: alt_usage_data { ApplicationRecord.database.version },
      pg_system_id: alt_usage_data { ApplicationRecord.database.system_id },
      flavor: alt_usage_data { ApplicationRecord.database.flavor }
      # rubocop: enable UsageData/LargeTable
    },
    mail: {
      smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
    }
  }
end

.data ⇒ Object 



41
42
43
44
45
46
47
48
 
# File 'lib/gitlab/usage_data.rb', line 41

def data
  clear_memoized

  with_finished_at(:recording_ce_finished_at) do
    { recorded_at: recorded_at }
      .merge(usage_data_metrics)
  end
end

.features_usage_data ⇒ Object 



145
 
# File 'lib/gitlab/usage_data.rb', line 145

def features_usage_data = {}

.object_store_config(component) ⇒ Object 



178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/gitlab/usage_data.rb', line 178

def object_store_config(component)
  config = alt_usage_data(fallback: nil) do
    Settings[component]['object_store']
  end

  if config.present?
    {
      enabled: alt_usage_data { Settings[component]['enabled'] },
      object_store: {
        enabled: alt_usage_data { config['enabled'] },
        direct_upload: alt_usage_data { config['direct_upload'] },
        background_upload: alt_usage_data { false }, # This setting no longer exists
        provider: alt_usage_data { config['connection']['provider'] }
      }
    }
  else
    {
      enabled: alt_usage_data { Settings[component]['enabled'] }
    }
  end
end

.object_store_usage_data ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/gitlab/usage_data.rb', line 200

def object_store_usage_data
  {
    object_store: {
      artifacts: object_store_config('artifacts'),
      external_diffs: object_store_config('external_diffs'),
      lfs: object_store_config('lfs'),
      uploads: object_store_config('uploads'),
      packages: object_store_config('packages')
    }
  }
end

.recorded_at ⇒ Object 



50
51
52
 
# File 'lib/gitlab/usage_data.rb', line 50

def recorded_at
  @recorded_at ||= Time.current
end

.system_usage_data ⇒ Object

rubocop: disable Metrics/AbcSize rubocop: disable CodeReuse/ActiveRecord



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/gitlab/usage_data.rb', line 56

def system_usage_data
  {
    counts: {
      assignee_lists: count(List.assignee),
      ci_external_pipelines: count(::Ci::Pipeline.external),
      ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source),
      ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source),
      ci_triggers: count(::Ci::Trigger),
      ci_pipeline_schedules: count(::Ci::PipelineSchedule),
      auto_devops_enabled: count(::ProjectAutoDevops.enabled),
      auto_devops_disabled: count(::ProjectAutoDevops.disabled),
      deploy_keys: count(DeployKey),
      feature_flags: count(Operations::FeatureFlag),
      environments: count(::Environment),
      clusters: count(::Clusters::Cluster),
      clusters_enabled: count(::Clusters::Cluster.enabled),
      project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type),
      group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type),
      instance_clusters_enabled: count(::Clusters::Cluster.enabled.instance_type),
      clusters_disabled: count(::Clusters::Cluster.disabled),
      project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
      group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
      instance_clusters_disabled: count(::Clusters::Cluster.disabled.instance_type),
      clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
      clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
      clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
      clusters_management_project: count(::Clusters::Cluster.with_management_project),
      clusters_integrations_prometheus: count(::Clusters::Integrations::Prometheus.enabled),
      kubernetes_agents: count(::Clusters::Agent),
      kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id),
      in_review_folder: count(::Environment.in_review_folder),
      groups: count(Group),
      issues: add_metric('CountIssuesMetric', time_frame: 'all'),
      issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
      issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
      issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
      incident_issues: count(::Issue.with_issue_type(:incident), start: minimum_id(Issue), finish: maximum_id(Issue)),
      alert_bot_incident_issues: count(::Issue.authored(::Users::Internal.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)),
      keys: count(Key),
      label_lists: count(List.label),
      lfs_objects: count(LfsObject),
      milestone_lists: count(List.milestone),
      milestones: count(Milestone),
      projects_with_packages: distinct_count(::Packages::Package, :project_id),
      packages: count(::Packages::Package),
      pages_domains: count(PagesDomain),
      pool_repositories: count(PoolRepository),
      projects: count(Project),
      projects_creating_incidents: distinct_count(Issue.with_issue_type(:incident), :project_id),
      projects_imported_from_github: count(Project.where(import_type: 'github')),
      projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
      projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
      projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id),
      projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.of_report_type(:terraform), :project_id),
      projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
      protected_branches: count(ProtectedBranch),
      protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])),
      releases: count(Release),
      remote_mirrors: count(RemoteMirror),
      suggestions: count(Suggestion),
      terraform_reports: count(::Ci::JobArtifact.of_report_type(:terraform)),
      terraform_states: count(::Terraform::State),
      todos: count(Todo),
      uploads: count(Upload),
      web_hooks: count(WebHook),
      labels: count(Label),
      merge_requests: count(MergeRequest),
      notes: count(Note)
    }.merge(
      user_preferences_usage,
      service_desk_counts
    )
  }
end

.system_usage_data_license ⇒ Object

rubocop: enable Metrics/AbcSize rubocop: enable CodeReuse/ActiveRecord



133
134
135
136
137
 
# File 'lib/gitlab/usage_data.rb', line 133

def system_usage_data_license
  {
    license: {}
  }
end

.system_usage_data_weekly ⇒ Object 



139
140
141
142
143
 
# File 'lib/gitlab/usage_data.rb', line 139

def system_usage_data_weekly
  {
    counts_weekly: {}
  }
end

.topology_usage_data ⇒ Object 



212
213
214
 
# File 'lib/gitlab/usage_data.rb', line 212

def topology_usage_data
  Gitlab::UsageData::Topology.new.topology_usage_data
end

.usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) ⇒ Object

Source: gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv



224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'lib/gitlab/usage_data.rb', line 224

def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
  {
    key => {
      configure: usage_activity_by_stage_configure(time_period),
      create: usage_activity_by_stage_create(time_period),
      enablement: usage_activity_by_stage_enablement(time_period),
      manage: usage_activity_by_stage_manage(time_period),
      monitor: usage_activity_by_stage_monitor(time_period),
      package: usage_activity_by_stage_package(time_period),
      plan: usage_activity_by_stage_plan(time_period),
      release: usage_activity_by_stage_release(time_period),
      verify: usage_activity_by_stage_verify(time_period)
    }
  }
end

.usage_activity_by_stage_configure(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord rubocop: disable UsageData/LargeTable



242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
 
# File 'lib/gitlab/usage_data.rb', line 242

def usage_activity_by_stage_configure(time_period)
  {
    clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period),
    clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period),
    clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period),
    clusters_platforms_gke: clusters_user_distinct_count(::Clusters::Cluster.gcp_installed.enabled, time_period),
    clusters_platforms_eks: clusters_user_distinct_count(::Clusters::Cluster.aws_installed.enabled, time_period),
    clusters_platforms_user: clusters_user_distinct_count(::Clusters::Cluster.user_provided.enabled, time_period),
    instance_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.instance_type, time_period),
    instance_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.instance_type, time_period),
    group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period),
    group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period),
    project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period),
    project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period),
    # These two `projects_slack_x` metrics are owned by the Manage stage, but are in this method as their key paths can't change.
    # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123442#note_1427961339.
    projects_slack_notifications_active: distinct_count(::Project.with_slack_integration.where(time_period), :creator_id),
    projects_slack_slash_active: distinct_count(::Project.with_slack_slash_commands_integration.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_create(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



266
267
268
269
270
271
272
273
274
275
 
# File 'lib/gitlab/usage_data.rb', line 266

def usage_activity_by_stage_create(time_period)
  {
    deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id),
    keys: distinct_count(::Key.regular_keys.where(time_period), :user_id),
    projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))),
    projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))),
    remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id),
    snippets: distinct_count(::Snippet.where(time_period), :author_id)
  }
end

.usage_activity_by_stage_enablement(time_period) ⇒ Object

Empty placeholder allows this to match the pattern used by other sections



279
280
281
 
# File 'lib/gitlab/usage_data.rb', line 279

def usage_activity_by_stage_enablement(time_period)
  {}
end

.usage_activity_by_stage_manage(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘campaigns_imported_from_github`, `ldap_group_links` rubocop: disable CodeReuse/ActiveRecord



285
286
287
288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/gitlab/usage_data.rb', line 285

def usage_activity_by_stage_manage(time_period)
  {
    # rubocop: disable UsageData/LargeTable
    events: stage_manage_events(time_period),
    groups: distinct_count(::GroupMember.where(time_period), :user_id),
    users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
    omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
    user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
    bulk_imports: {
      gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab))
    },
    group_imports: group_imports(time_period)
  }
end

.usage_activity_by_stage_monitor(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
 
# File 'lib/gitlab/usage_data.rb', line 302

def usage_activity_by_stage_monitor(time_period)
  # Calculate histogram only for overall as other time periods aren't available/useful here.
  integrations_histogram = time_period.empty? ? histogram(::AlertManagement::HttpIntegration.active, :project_id, buckets: 1..100) : nil

  {
    clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
    clusters_integrations_prometheus: cluster_integrations_user_distinct_count(::Clusters::Integrations::Prometheus, time_period),
    operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
      start: minimum_id(User),
      finish: maximum_id(User)),
    projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
    projects_with_incidents: distinct_count(::Issue.with_issue_type(:incident).where(time_period), :project_id),
    # We are making an assumption here that all alert_management_alerts are associated with an issue of type
    # incident. In reality this is very close to the truth and allows more efficient queries.
    # More info in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121297#note_1416999956
    projects_with_alert_incidents: distinct_count(::AlertManagement::Alert.where(time_period).where.not(issue_id: nil), :project_id),
    projects_with_enabled_alert_integrations_histogram: integrations_histogram
  }.compact
end

.usage_activity_by_stage_package(time_period) ⇒ Object

rubocop: disable CodeReuse/ActiveRecord



324
325
326
327
328
 
# File 'lib/gitlab/usage_data.rb', line 324

def usage_activity_by_stage_package(time_period)
  {
    projects_with_packages: distinct_count(::Project.with_packages.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_plan(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘boards`, `labels`, `milestones`, `uploads` Omitted because too expensive: `epics_deepest_relationship_level` rubocop: disable CodeReuse/ActiveRecord



334
335
336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/gitlab/usage_data.rb', line 334

def usage_activity_by_stage_plan(time_period)
  time_frame = metric_time_period(time_period)
  {
    issues: add_metric('CountUsersCreatingIssuesMetric', time_frame: time_frame),
    notes: distinct_count(::Note.where(time_period), :author_id),
    projects: distinct_count(::Project.where(time_period), :creator_id),
    todos: distinct_count(::Todo.where(time_period), :author_id),
    service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
    service_desk_issues: count(::Issue.service_desk.where(time_period)),
    projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).where(time_period), :creator_id),
    projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_server.where(time_period), :creator_id)
  }
end

.usage_activity_by_stage_release(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘environments`, `feature_flags`, `in_review_folder`, `pages_domains` rubocop: disable CodeReuse/ActiveRecord



351
352
353
354
355
356
357
358
359
360
 
# File 'lib/gitlab/usage_data.rb', line 351

def usage_activity_by_stage_release(time_period)
  time_frame = metric_time_period(time_period)
  {
    deployments: distinct_count(::Deployment.where(time_period), :user_id),
    failed_deployments: distinct_count(::Deployment.failed.where(time_period), :user_id),
    releases: distinct_count(::Release.where(time_period), :author_id),
    successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id),
    releases_with_milestones: add_metric('CountUsersAssociatingMilestonesToReleasesMetric', time_frame: time_frame)
  }
end

.usage_activity_by_stage_verify(time_period) ⇒ Object

Omitted because no user, creator or author associated: ‘ci_runners` rubocop: disable CodeReuse/ActiveRecord



365
366
367
368
369
370
371
372
373
374
375
376
 
# File 'lib/gitlab/usage_data.rb', line 365

def usage_activity_by_stage_verify(time_period)
  {
    ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
    ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
    ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
    ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id)
  }
end

.user_preferences_usage ⇒ Object

augmented in EE



217
218
219
220
221
 
# File 'lib/gitlab/usage_data.rb', line 217

def user_preferences_usage
  {
    user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active))
  }
end

.with_metadata ⇒ Object

rubocop: enable CodeReuse/ActiveRecord



379
380
381
382
383
384
385
386
387
388
389
390
391
 
# File 'lib/gitlab/usage_data.rb', line 379

def with_metadata
  result = nil
  error = nil

  duration = Benchmark.realtime do
    result = yield
  rescue StandardError => e
    error = e
    Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
  end

  ::Gitlab::Usage::ServicePing::LegacyMetricMetadataDecorator.new(result, duration, error: error)
end