Read about our upcoming Code of Conduct on this issue

relation_factory.rb 10.7 KB
Newer Older
1
2
# frozen_string_literal: true

3
module Gitlab
4
  module ImportExport
5
    class RelationFactory
6
      OVERRIDES = { snippets: :project_snippets,
7
                    ci_pipelines: 'Ci::Pipeline',
8
                    pipelines: 'Ci::Pipeline',
9
                    stages: 'Ci::Stage',
10
11
                    statuses: 'commit_status',
                    triggers: 'Ci::Trigger',
12
                    pipeline_schedules: 'Ci::PipelineSchedule',
13
                    builds: 'Ci::Build',
14
                    runners: 'Ci::Runner',
15
                    hooks: 'ProjectHook',
16
                    merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
James Lopez's avatar
James Lopez committed
17
                    push_access_levels: 'ProtectedBranch::PushAccessLevel',
18
                    create_access_levels: 'ProtectedTag::CreateAccessLevel',
19
                    labels: :project_labels,
20
                    priorities: :label_priorities,
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
21
                    auto_devops: :project_auto_devops,
22
                    label: :project_label,
23
                    custom_attributes: 'ProjectCustomAttribute',
24
                    project_badges: 'Badge',
25
                    metrics: 'MergeRequest::Metrics',
26
                    ci_cd_settings: 'ProjectCiCdSetting',
27
                    error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
28
                    links: 'Releases::Link' }.freeze
29

30
      USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze
31

Kamil Trzciński's avatar
Kamil Trzciński committed
32
      PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
33

James Lopez's avatar
James Lopez committed
34
35
      BUILD_MODELS = %w[Ci::Build commit_status].freeze

36
37
      IMPORTED_OBJECT_MAX_RETRIES = 5.freeze

38
      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
39

40
      TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
41

42
43
44
      def self.create(*args)
        new(*args).create
      end
45

46
47
48
49
50
51
52
53
54
      def self.relation_class(relation_name)
        # There are scenarios where the model is pluralized (e.g.
        # MergeRequest::Metrics), and we don't want to force it to singular
        # with #classify.
        relation_name.to_s.classify.constantize
      rescue NameError
        relation_name.to_s.constantize
      end

55
      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
56
        @relation_name = self.class.overrides[relation_sym] || relation_sym
57
        @relation_hash = relation_hash.except('noteable_id')
58
        @members_mapper = members_mapper
59
        @user = user
60
        @project = project
61
        @imported_object_retries = 0
62

63
64
        @relation_hash['project_id'] = @project.id

65
66
67
68
69
70
71
        # Remove excluded keys from relation_hash
        # We don't do this in the parsed_relation_hash because of the 'transformed attributes'
        # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then,
        # in the create method that attribute is renamed to diff. And because diff is an excluded key,
        # if we clean the excluded keys in the parsed_relation_hash, it will be removed
        # from the object attributes and the export will fail.
        @relation_hash.except!(*excluded_keys)
72
      end
James Lopez's avatar
James Lopez committed
73

74
75
76
77
      # Creates an object from an actual model with name "relation_sym" with params from
      # the relation_hash, updating references with new object IDs, mapping users using
      # the "members_mapper" object, also updating notes if required.
      def create
78
79
        return nil if unknown_service?

80
81
82
83
84
        setup_models

        generate_imported_object
      end

85
86
87
88
      def self.overrides
        OVERRIDES
      end

89
90
91
      private

      def setup_models
micael.bergeron's avatar
micael.bergeron committed
92
93
94
        case @relation_name
        when :merge_request_diff_files       then setup_diff
        when :notes                          then setup_note
95
96
        end

97
98
        update_user_references
        update_project_references
99
        update_group_references
100
        remove_duplicate_assignees
101

James Lopez's avatar
James Lopez committed
102
103
        setup_pipeline if @relation_name == 'Ci::Pipeline'

104
        reset_tokens!
105
        remove_encrypted_attributes!
106
107
      end

108
      def update_user_references
109
        USER_REFERENCES.each do |reference|
110
111
          if @relation_hash[reference]
            @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
112
113
114
115
          end
        end
      end

116
117
118
119
120
121
122
123
      def remove_duplicate_assignees
        return unless @relation_hash['issue_assignees']

        # When an assignee did not exist in the members mapper, the importer is
        # assigned. We only need to assign each user once.
        @relation_hash['issue_assignees'].uniq!(&:user_id)
      end

micael.bergeron's avatar
micael.bergeron committed
124
125
126
127
128
129
      def setup_note
        set_note_author
        # attachment is deprecated and note uploads are handled by Markdown uploader
        @relation_hash['attachment'] = nil
      end

130
131
132
133
134
135
136
      # Sets the author for a note. If the user importing the project
      # has admin access, an actual mapping with new project members
      # will be used. Otherwise, a note stating the original author name
      # is left.
      def set_note_author
        old_author_id = @relation_hash['author_id']
        author = @relation_hash.delete('author')
137

138
        update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
James Lopez's avatar
James Lopez committed
139
140
      end

141
      def has_author?(old_author_id)
142
        admin_user? && @members_mapper.include?(old_author_id)
143
144
145
      end

      def missing_author_note(updated_at, author_name)
146
147
        timestamp = updated_at.split('.').first
        "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
148
149
      end

150
      def generate_imported_object
Kamil Trzciński's avatar
Kamil Trzciński committed
151
152
        if BUILD_MODELS.include?(@relation_name)
          @relation_hash.delete('trace') # old export files have trace
153
          @relation_hash.delete('token')
154
          @relation_hash.delete('commands')
155

156
          imported_object
157
158
        elsif @relation_name == :merge_requests
          MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse!
James Lopez's avatar
James Lopez committed
159
        else
160
          imported_object
James Lopez's avatar
James Lopez committed
161
162
163
        end
      end

164
      def update_project_references
James Lopez's avatar
James Lopez committed
165
166
        # If source and target are the same, populate them with the new project ID.
        if @relation_hash['source_project_id']
James Lopez's avatar
James Lopez committed
167
          @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
James Lopez's avatar
James Lopez committed
168
169
        end

James Lopez's avatar
James Lopez committed
170
171
172
173
174
        @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
      end

      def same_source_and_target?
        @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
James Lopez's avatar
James Lopez committed
175
      end
176

177
178
179
180
181
182
183
      def update_group_references
        return unless EXISTING_OBJECT_CHECK.include?(@relation_name)
        return unless @relation_hash['group_id']

        @relation_hash['group_id'] = @project.group&.id
      end

184
      def reset_tokens!
185
        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
186

187
        # If we import/export a project to the same instance, tokens will have to be reset.
188
189
190
191
        # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
        relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
          @relation_hash[token] = nil
        end
192
      end
193

194
      def remove_encrypted_attributes!
James Lopez's avatar
James Lopez committed
195
        return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
196
197
198
199
200
201

        relation_class.encrypted_attributes.each_key do |key|
          @relation_hash[key.to_s] = nil
        end
      end

202
      def relation_class
203
        @relation_class ||= self.class.relation_class(@relation_name)
204
      end
James Lopez's avatar
James Lopez committed
205

206
      def imported_object
207
208
209
        if existing_or_new_object.respond_to?(:importing)
          existing_or_new_object.importing = true
        end
210

211
212
213
214
215
216
        existing_or_new_object
      rescue ActiveRecord::RecordNotUnique
        # as the operation is not atomic, retry in the unlikely scenario an INSERT is
        # performed on the same object between the SELECT and the INSERT
        @imported_object_retries += 1
        retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
James Lopez's avatar
James Lopez committed
217
      end
218
219
220

      def update_note_for_missing_author(author_name)
        @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
221
        @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}"
222
223
224
      end

      def admin_user?
blackst0ne's avatar
blackst0ne committed
225
        @user.admin?
226
      end
227
228

      def parsed_relation_hash
229
230
        @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash,
                                                                               relation_class: relation_class)
231
      end
James Lopez's avatar
James Lopez committed
232

micael.bergeron's avatar
micael.bergeron committed
233
      def setup_diff
234
235
236
        @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
      end

237
238
239
240
241
242
243
244
      def setup_pipeline
        @relation_hash.fetch('stages').each do |stage|
          stage.statuses.each do |status|
            status.pipeline = imported_object
          end
        end
      end

245
246
247
248
249
      def existing_or_new_object
        # Only find existing records to avoid mapping tables such as milestones
        # Otherwise always create the record, skipping the extra SELECT clause.
        @existing_or_new_object ||= begin
          if EXISTING_OBJECT_CHECK.include?(@relation_name)
250
            attribute_hash = attribute_hash_for(['events'])
251

252
            existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
253

254
255
256
257
258
259
            existing_object
          else
            relation_class.new(parsed_relation_hash)
          end
        end
      end
260

261
262
263
264
265
266
267
      def attribute_hash_for(attributes)
        attributes.inject({}) do |hash, value|
          hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value]
          hash
        end
      end

268
      def existing_object
269
        @existing_object ||= find_or_create_object!
270
      end
271
272
273
274
275

      def unknown_service?
        @relation_name == :services && parsed_relation_hash['type'] &&
          !Object.const_defined?(parsed_relation_hash['type'])
      end
276
277

      def find_or_create_object!
James Lopez's avatar
James Lopez committed
278
279
        return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature

280
        # Can't use IDs as validation exists calling `group` or `project` attributes
281
        finder_hash = parsed_relation_hash.tap do |hash|
James Lopez's avatar
James Lopez committed
282
283
          hash['group'] = @project.group if relation_class.attribute_method?('group_id')
          hash['project'] = @project
284
          hash.delete('project_id')
James Lopez's avatar
James Lopez committed
285
        end
286

James Lopez's avatar
James Lopez committed
287
        GroupProjectObjectBuilder.build(relation_class, finder_hash)
James Lopez's avatar
James Lopez committed
288
      end
289
290
291
    end
  end
end