Read about our upcoming Code of Conduct on this issue

job_artifact.rb 5.69 KB
Newer Older
1
2
# frozen_string_literal: true

3
module Ci
4
  class JobArtifact < ApplicationRecord
5
    include AfterCommitQueue
6
    include ObjectStorage::BackgroundMove
7
8
    extend Gitlab::Ci::Model

9
10
    NotSupportedAdapterError = Class.new(StandardError)

11
    TEST_REPORT_FILE_TYPES = %w[junit].freeze
12
13
14
15
16
17
    NON_ERASABLE_FILE_TYPES = %w[trace].freeze
    DEFAULT_FILE_NAMES = {
      archive: nil,
      metadata: nil,
      trace: nil,
      junit: 'junit.xml',
18
      codequality: 'gl-code-quality-report.json',
19
20
21
      sast: 'gl-sast-report.json',
      dependency_scanning: 'gl-dependency-scanning-report.json',
      container_scanning: 'gl-container-scanning-report.json',
22
23
24
      dast: 'gl-dast-report.json',
      license_management: 'gl-license-management-report.json',
      performance: 'performance.json'
25
26
27
28
29
30
31
    }.freeze

    TYPE_AND_FORMAT_PAIRS = {
      archive: :zip,
      metadata: :gzip,
      trace: :raw,
      junit: :gzip,
32
33
34
35
36
37
38
39

      # All these file formats use `raw` as we need to store them uncompressed
      # for Frontend to fetch the files and do analysis
      # When they will be only used by backend, they can be `gzipped`.
      codequality: :raw,
      sast: :raw,
      dependency_scanning: :raw,
      container_scanning: :raw,
40
41
42
      dast: :raw,
      license_management: :raw,
      performance: :raw
43
    }.freeze
44

45
    belongs_to :project
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
46
    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
47

48
49
    mount_uploader :file, JobArtifactUploader

50
51
    validates :file_format, presence: true, unless: :trace?, on: :create
    validate :valid_file_format?, unless: :trace?, on: :create
52
    before_save :set_size, if: :file_changed?
53
54
    after_save :update_project_statistics_after_save, if: :size_changed?
    after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed?
55

56
    after_save :update_file_store, if: :file_changed?
57

58
    scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
59

60
61
62
63
64
65
    scope :with_file_types, -> (file_types) do
      types = self.file_types.select { |file_type| file_types.include?(file_type) }.values

      where(file_type: types)
    end

66
    scope :test_reports, -> do
67
68
69
70
71
      with_file_types(TEST_REPORT_FILE_TYPES)
    end

    scope :erasable, -> do
      types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
72
73
74
75

      where(file_type: types)
    end

76
77
    scope :expired, -> (limit) { where('expire_at < ?', Time.now).limit(limit) }

78
    delegate :filename, :exists?, :open, to: :file
79

80
81
    enum file_type: {
      archive: 1,
82
      metadata: 2,
83
      trace: 3,
84
85
86
87
      junit: 4,
      sast: 5, ## EE-specific
      dependency_scanning: 6, ## EE-specific
      container_scanning: 7, ## EE-specific
88
      dast: 8, ## EE-specific
89
90
91
      codequality: 9, ## EE-specific
      license_management: 10, ## EE-specific
      performance: 11 ## EE-specific
92
93
    }

94
95
96
97
98
99
    enum file_format: {
      raw: 1,
      zip: 2,
      gzip: 3
    }

100
101
102
103
104
105
106
107
108
109
110
111
112
113
    # `file_location` indicates where actual files are stored.
    # Ideally, actual files should be stored in the same directory, and use the same
    # convention to generate its path. However, sometimes we can't do so due to backward-compatibility.
    #
    # legacy_path ... The actual file is stored at a path consists of a timestamp
    #                 and raw project/model IDs. Those rows were migrated from
    #                 `ci_builds.artifacts_file` and `ci_builds.artifacts_metadata`
    # hashed_path ... The actual file is stored at a path consists of a SHA2 based on the project ID.
    #                 This is the default value.
    enum file_location: {
      legacy_path: 1,
      hashed_path: 2
    }

114
    FILE_FORMAT_ADAPTERS = {
115
116
      gzip: Gitlab::Ci::Build::Artifacts::Adapters::GzipStream,
      raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream
117
118
    }.freeze

119
120
121
122
123
124
    def valid_file_format?
      unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
        errors.add(:file_format, 'Invalid file format with specified file type')
      end
    end

125
    def update_file_store
126
127
128
      # The file.object_store is set during `uploader.store!`
      # which happens after object is inserted/updated
      self.update_column(:file_store, file.object_store)
129
130
    end

131
132
133
134
    def self.artifacts_size_for(project)
      self.where(project: project).sum(:size)
    end

135
136
137
138
    def local_store?
      [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store)
    end

139
    def hashed_path?
140
141
142
      return true if trace? # ArchiveLegacyTraces background migration might not have `file_location` column

      super || self.file_location.nil?
143
144
    end

145
146
147
148
149
150
151
152
153
154
    def expire_in
      expire_at - Time.now if expire_at
    end

    def expire_in=(value)
      self.expire_at =
        if value
          ChronicDuration.parse(value)&.seconds&.from_now
        end
    end
155

156
157
158
159
160
161
162
163
164
165
    def each_blob(&blk)
      unless file_format_adapter_class
        raise NotSupportedAdapterError, 'This file format requires a dedicated adapter'
      end

      file.open do |stream|
        file_format_adapter_class.new(stream).each_blob(&blk)
      end
    end

166
167
    private

168
169
170
171
    def file_format_adapter_class
      FILE_FORMAT_ADAPTERS[file_format.to_sym]
    end

172
173
174
175
176
177
178
179
180
    def set_size
      self.size = file.size
    end

    def update_project_statistics_after_save
      update_project_statistics(size.to_i - size_was.to_i)
    end

    def update_project_statistics_after_destroy
181
      update_project_statistics(-self.size.to_i)
182
183
184
185
186
187
188
189
190
191
    end

    def update_project_statistics(difference)
      ProjectStatistics.increment_statistic(project_id, :build_artifacts_size, difference)
    end

    def project_destroyed?
      # Use job.project to avoid extra DB query for project
      job.project.pending_delete?
    end
192
193
  end
end