usage_data.rb 14.8 KB
Newer Older
1
2
# frozen_string_literal: true

3
4
5
6
7
8
9
# For hardening usage ping and make it easier to add measures there is in place alt_usage_data method
# which handles StandardError and fallbacks into -1
# this way not all measures fail if we encounter one exception
#
# Examples:
#  alt_usage_data { Gitlab::VERSION }
#  alt_usage_data { Gitlab::CurrentSettings.uuid }
10
11
module Gitlab
  class UsageData
12
    BATCH_SIZE = 100
13

14
    class << self
15
      def data(force_refresh: false)
Alex Kalderimis's avatar
Alex Kalderimis committed
16
17
18
        Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
          uncached_data
        end
19
20
21
      end

      def uncached_data
22
23
        license_usage_data
          .merge(system_usage_data)
24
25
26
          .merge(features_usage_data)
          .merge(components_usage_data)
          .merge(cycle_analytics_usage_data)
27
28
      end

29
30
      def to_json(force_refresh: false)
        data(force_refresh: force_refresh).to_json
31
32
      end

33
      def license_usage_data
34
35
36
37
38
        {
          uuid: alt_usage_data { Gitlab::CurrentSettings.uuid },
          hostname: alt_usage_data { Gitlab.config.gitlab.host },
          version: alt_usage_data { Gitlab::VERSION },
          installation_type: alt_usage_data { installation_type },
39
          active_user_count: count(User.active),
40
41
42
43
44
          recorded_at: Time.now,
          edition: 'CE'
        }
      end

45
      # rubocop: disable Metrics/AbcSize
46
      # rubocop: disable CodeReuse/ActiveRecord
47
48
49
      def system_usage_data
        {
          counts: {
50
51
52
53
54
55
56
57
58
59
60
61
62
63
            assignee_lists: count(List.assignee),
            boards: count(Board),
            ci_builds: count(::Ci::Build),
            ci_internal_pipelines: count(::Ci::Pipeline.internal),
            ci_external_pipelines: count(::Ci::Pipeline.external),
            ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source),
            ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source),
            ci_runners: count(::Ci::Runner),
            ci_triggers: count(::Ci::Trigger),
            ci_pipeline_schedules: count(::Ci::PipelineSchedule),
            auto_devops_enabled: count(::ProjectAutoDevops.enabled),
            auto_devops_disabled: count(::ProjectAutoDevops.disabled),
            deploy_keys: count(DeployKey),
            deployments: count(Deployment),
64
65
            successful_deployments: count(Deployment.success),
            failed_deployments: count(Deployment.failed),
66
67
68
            environments: count(::Environment),
            clusters: count(::Clusters::Cluster),
            clusters_enabled: count(::Clusters::Cluster.enabled),
69
70
            project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type),
            group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type),
71
            instance_clusters_enabled: count(::Clusters::Cluster.enabled.instance_type),
72
            clusters_disabled: count(::Clusters::Cluster.disabled),
73
74
            project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type),
            group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type),
75
            instance_clusters_disabled: count(::Clusters::Cluster.disabled.instance_type),
76
77
            clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled),
            clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled),
78
            clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled),
79
80
81
            clusters_applications_helm: count(::Clusters::Applications::Helm.available),
            clusters_applications_ingress: count(::Clusters::Applications::Ingress.available),
            clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available),
82
            clusters_applications_crossplane: count(::Clusters::Applications::Crossplane.available),
83
84
85
            clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available),
            clusters_applications_runner: count(::Clusters::Applications::Runner.available),
            clusters_applications_knative: count(::Clusters::Applications::Knative.available),
86
            clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available),
87
            clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available),
88
            clusters_management_project: count(::Clusters::Cluster.with_management_project),
89
            in_review_folder: count(::Environment.in_review_folder),
90
            grafana_integrated_projects: count(GrafanaIntegration.enabled),
91
92
            groups: count(Group),
            issues: count(Issue),
93
            issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
94
            issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
95
            issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
96
            issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data,
97
            incident_issues: count(::Issue.authored(::User.alert_bot)),
98
99
100
101
102
103
            keys: count(Key),
            label_lists: count(List.label),
            lfs_objects: count(LfsObject),
            milestone_lists: count(List.milestone),
            milestones: count(Milestone),
            pages_domains: count(PagesDomain),
104
            pool_repositories: count(PoolRepository),
105
106
            projects: count(Project),
            projects_imported_from_github: count(Project.where(import_type: 'github')),
107
            projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
Logan King's avatar
Logan King committed
108
            projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
109
            projects_with_alerts_service_enabled: count(AlertsService.active),
110
            projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
111
112
113
114
            protected_branches: count(ProtectedBranch),
            releases: count(Release),
            remote_mirrors: count(RemoteMirror),
            snippets: count(Snippet),
115
116
            suggestions: count(Suggestion),
            todos: count(Todo),
117
            uploads: count(Upload),
118
119
120
121
            web_hooks: count(WebHook),
            labels: count(Label),
            merge_requests: count(MergeRequest),
            notes: count(Note)
122
123
124
          }.merge(
            services_usage,
            usage_counters,
125
126
            user_preferences_usage,
            ingress_modsecurity_usage
127
128
          )
        }
129
      end
130
      # rubocop: enable CodeReuse/ActiveRecord
131
      # rubocop: enable Metrics/AbcSize
132

133
      def cycle_analytics_usage_data
134
        Gitlab::CycleAnalytics::UsageData.new.to_json
135
136
      rescue ActiveRecord::StatementInvalid
        { avg_cycle_analytics: {} }
137
138
      end

139
140
141
142
143
144
145
146
      # rubocop:disable CodeReuse/ActiveRecord
      def grafana_embed_usage_data
        count(Issue.joins('JOIN grafana_integrations USING (project_id)')
          .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'")
          .where(grafana_integrations: { enabled: true }))
      end
      # rubocop: enable CodeReuse/ActiveRecord

147
148
149
150
151
152
      def features_usage_data
        features_usage_data_ce
      end

      def features_usage_data_ce
        {
153
          container_registry_enabled: alt_usage_data { Gitlab.config.registry.enabled },
154
          dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
155
156
157
158
159
160
161
162
163
164
          gitlab_shared_runners_enabled: alt_usage_data { Gitlab.config.gitlab_ci.shared_runners_enabled },
          gravatar_enabled: alt_usage_data { Gitlab::CurrentSettings.gravatar_enabled? },
          influxdb_metrics_enabled: alt_usage_data { Gitlab::Metrics.influx_metrics_enabled? },
          ldap_enabled: alt_usage_data { Gitlab.config.ldap.enabled },
          mattermost_enabled: alt_usage_data { Gitlab.config.mattermost.enabled },
          omniauth_enabled: alt_usage_data { Gitlab::Auth.omniauth_enabled? },
          prometheus_metrics_enabled: alt_usage_data { Gitlab::Metrics.prometheus_metrics_enabled? },
          reply_by_email_enabled: alt_usage_data { Gitlab::IncomingEmail.enabled? },
          signup_enabled: alt_usage_data { Gitlab::CurrentSettings.allow_signup? },
          web_ide_clientside_preview_enabled: alt_usage_data { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
165
          ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity)
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
        }.merge(features_usage_data_container_expiration_policies)
      end

      # rubocop: disable CodeReuse/ActiveRecord
      def features_usage_data_container_expiration_policies
        results = {}
        start = ::Project.minimum(:id)
        finish = ::Project.maximum(:id)

        results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
        base = ::ContainerExpirationPolicy.active
        results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish)

        %i[keep_n cadence older_than].each do |option|
          ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend
            results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish)
          end
        end

        results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish)
        results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish)

        results
189
      end
190
      # rubocop: enable CodeReuse/ActiveRecord
191

Alex Kalderimis's avatar
Alex Kalderimis committed
192
      # @return [Hash<Symbol, Integer>]
193
      def usage_counters
Alex Kalderimis's avatar
Alex Kalderimis committed
194
195
196
197
198
        usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) }
      end

      # @return [Array<#totals>] An array of objects that respond to `#totals`
      def usage_data_counters
199
        [
200
201
202
203
204
205
206
207
208
          Gitlab::UsageDataCounters::WikiPageCounter,
          Gitlab::UsageDataCounters::WebIdeCounter,
          Gitlab::UsageDataCounters::NoteCounter,
          Gitlab::UsageDataCounters::SnippetCounter,
          Gitlab::UsageDataCounters::SearchCounter,
          Gitlab::UsageDataCounters::CycleAnalyticsCounter,
          Gitlab::UsageDataCounters::ProductivityAnalyticsCounter,
          Gitlab::UsageDataCounters::SourceCodeCounter,
          Gitlab::UsageDataCounters::MergeRequestCounter
209
        ]
210
211
      end

212
213
      def components_usage_data
        {
214
215
216
217
218
219
220
221
222
223
224
225
226
227
          git: { version: alt_usage_data { Gitlab::Git.version } },
          gitaly: {
            version: alt_usage_data { Gitaly::Server.all.first.server_version },
            servers: alt_usage_data { Gitaly::Server.count },
            filesystems: alt_usage_data { Gitaly::Server.filesystems }
          },
          gitlab_pages: {
            enabled: alt_usage_data { Gitlab.config.pages.enabled },
            version: alt_usage_data { Gitlab::Pages::VERSION }
          },
          database: {
            adapter: alt_usage_data { Gitlab::Database.adapter_name },
            version: alt_usage_data { Gitlab::Database.version }
          },
228
          app_server: { type: app_server_type }
229
        }
230
      end
231

232
233
234
235
236
237
238
239
      def app_server_type
        Gitlab::Runtime.identify.to_s
      rescue Gitlab::Runtime::IdentificationError => e
        Gitlab::AppLogger.error(e.message)
        Gitlab::ErrorTracking.track_exception(e)
        'unknown_app_server_type'
      end

240
241
242
243
      def ingress_modsecurity_usage
        ::Clusters::Applications::IngressModsecurityUsageService.new.execute
      end

244
      # rubocop: disable CodeReuse/ActiveRecord
245
      def services_usage
246
247
        results = Service.available_services_names.without('jira').each_with_object({}) do |service_name, response|
          response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, type: "#{service_name}_service".camelize))
248
        end
249

250
251
252
253
254
        # Keep old Slack keys for backward compatibility, https://gitlab.com/gitlab-data/analytics/issues/3241
        results[:projects_slack_notifications_active] = results[:projects_slack_active]
        results[:projects_slack_slash_active] = results[:projects_slack_slash_commands_active]

        results.merge(jira_usage)
255
256
257
258
259
260
      end

      def jira_usage
        # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999
        # so we can just check for subdomains of atlassian.net

261
262
263
        results = {
          projects_jira_server_active: 0,
          projects_jira_cloud_active: 0,
264
          projects_jira_active: 0
265
        }
266

267
268
        Service.active
          .by_type(:JiraService)
269
270
271
          .includes(:jira_tracker_data)
          .find_in_batches(batch_size: BATCH_SIZE) do |services|
          counts = services.group_by do |service|
272
            # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
273
274
275
276
277
278
            service_url = service.data_fields&.url || (service.properties && service.properties['url'])
            service_url&.include?('.atlassian.net') ? :cloud : :server
          end

          results[:projects_jira_server_active] += counts[:server].count if counts[:server]
          results[:projects_jira_cloud_active] += counts[:cloud].count if counts[:cloud]
279
          results[:projects_jira_active] += services.size
280
281
282
        end

        results
283
284
      rescue ActiveRecord::StatementInvalid
        { projects_jira_server_active: -1, projects_jira_cloud_active: -1, projects_jira_active: -1 }
285
      end
286
      # rubocop: enable CodeReuse/ActiveRecord
287

288
289
290
291
      def user_preferences_usage
        {} # augmented in EE
      end

292
      def count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil)
293
        if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
294
          Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
295
296
297
298
299
300
301
        else
          relation.count
        end
      rescue ActiveRecord::StatementInvalid
        fallback
      end

302
      def distinct_count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil)
303
        if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
304
          Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish)
305
306
307
        else
          relation.distinct_count_by(column)
        end
308
309
310
      rescue ActiveRecord::StatementInvalid
        fallback
      end
311

312
313
314
315
316
317
318
319
320
321
322
323
      def alt_usage_data(value = nil, fallback: -1, &block)
        if block_given?
          yield
        else
          value
        end
      rescue
        fallback
      end

      private

324
325
326
327
328
329
330
      def installation_type
        if Rails.env.production?
          Gitlab::INSTALLATION_TYPE
        else
          "gitlab-development-kit"
        end
      end
331
332
333
    end
  end
end
334
335

Gitlab::UsageData.prepend_if_ee('EE::Gitlab::UsageData')