relation_factory.rb 9.64 KB
Newer Older
1
module Gitlab
2
  module ImportExport
3
    class RelationFactory
4
      OVERRIDES = { snippets: :project_snippets,
5
                    pipelines: 'Ci::Pipeline',
6
                    stages: 'Ci::Stage',
7
8
                    statuses: 'commit_status',
                    triggers: 'Ci::Trigger',
9
                    pipeline_schedules: 'Ci::PipelineSchedule',
10
                    builds: 'Ci::Build',
11
                    hooks: 'ProjectHook',
12
                    merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
James Lopez's avatar
James Lopez committed
13
                    push_access_levels: 'ProtectedBranch::PushAccessLevel',
14
                    create_access_levels: 'ProtectedTag::CreateAccessLevel',
15
                    labels: :project_labels,
16
                    priorities: :label_priorities,
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
17
                    auto_devops: 'ProjectAutoDevops',
18
                    label: :project_label }.freeze
19

blackst0ne's avatar
blackst0ne committed
20
      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze
21

Kamil Trzciński's avatar
Kamil Trzciński committed
22
      PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
23

James Lopez's avatar
James Lopez committed
24
25
      BUILD_MODELS = %w[Ci::Build commit_status].freeze

26
27
      IMPORTED_OBJECT_MAX_RETRIES = 5.freeze

28
      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze
29

James Lopez's avatar
James Lopez committed
30
      TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
31

32
33
34
      def self.create(*args)
        new(*args).create
      end
35

36
      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:)
37
        @relation_name = OVERRIDES[relation_sym] || relation_sym
38
        @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id)
39
        @members_mapper = members_mapper
40
        @user = user
41
        @project = project
42
        @imported_object_retries = 0
43
      end
James Lopez's avatar
James Lopez committed
44

45
46
47
48
      # Creates an object from an actual model with name "relation_sym" with params from
      # the relation_hash, updating references with new object IDs, mapping users using
      # the "members_mapper" object, also updating notes if required.
      def create
49
50
        return nil if unknown_service?

51
52
53
54
55
56
57
58
59
60
61
        setup_models

        generate_imported_object
      end

      private

      def setup_models
        if @relation_name == :notes
          set_note_author

James Lopez's avatar
James Lopez committed
62
          # attachment is deprecated and note uploads are handled by Markdown uploader
63
64
65
          @relation_hash['attachment'] = nil
        end

66
67
        update_user_references
        update_project_references
68
69

        handle_group_label if group_label?
70
        reset_tokens!
71
        remove_encrypted_attributes!
72

73
        @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
74
        set_st_diff_commits if @relation_name == :merge_request_diff
75
        set_diff if @relation_name == :merge_request_diff_files
76
77
      end

78
      def update_user_references
79
        USER_REFERENCES.each do |reference|
80
81
          if @relation_hash[reference]
            @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
82
83
84
85
          end
        end
      end

86
87
88
89
90
91
92
      # Sets the author for a note. If the user importing the project
      # has admin access, an actual mapping with new project members
      # will be used. Otherwise, a note stating the original author name
      # is left.
      def set_note_author
        old_author_id = @relation_hash['author_id']
        author = @relation_hash.delete('author')
93

94
        update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
James Lopez's avatar
James Lopez committed
95
96
      end

97
      def has_author?(old_author_id)
98
        admin_user? && @members_mapper.include?(old_author_id)
99
100
101
      end

      def missing_author_note(updated_at, author_name)
102
103
        timestamp = updated_at.split('.').first
        "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
104
105
      end

106
      def generate_imported_object
Kamil Trzciński's avatar
Kamil Trzciński committed
107
108
        if BUILD_MODELS.include?(@relation_name)
          @relation_hash.delete('trace') # old export files have trace
109
110
          @relation_hash.delete('token')

111
112
          imported_object do |object|
            object.commit_id = nil
James Lopez's avatar
James Lopez committed
113
          end
114
115
        elsif @relation_name == :merge_requests
          MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse!
James Lopez's avatar
James Lopez committed
116
        else
117
          imported_object
James Lopez's avatar
James Lopez committed
118
119
120
        end
      end

121
122
      def update_project_references
        project_id = @relation_hash.delete('project_id')
123

James Lopez's avatar
James Lopez committed
124
125
        # If source and target are the same, populate them with the new project ID.
        if @relation_hash['source_project_id']
126
          @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID
James Lopez's avatar
James Lopez committed
127
128
        end

129
        # project_id may not be part of the export, but we always need to populate it if required.
130
        @relation_hash['project_id'] = project_id
131
        @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
James Lopez's avatar
James Lopez committed
132
      end
133

James Lopez's avatar
James Lopez committed
134
135
      def same_source_and_target?
        @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
136
137
      end

138
139
140
141
142
143
144
      def group_label?
        @relation_hash['type'] == 'GroupLabel'
      end

      def handle_group_label
        # If there's no group, move the label to a project label
        if @relation_hash['group_id']
145
          @relation_hash['project_id'] = nil
146
147
148
149
150
151
          @relation_name = :group_label
        else
          @relation_hash['type'] = 'ProjectLabel'
        end
      end

152
      def reset_tokens!
153
        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
154

155
        # If we import/export a project to the same instance, tokens will have to be reset.
156
157
158
159
        # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
        relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
          @relation_hash[token] = nil
        end
160
      end
161

162
      def remove_encrypted_attributes!
James Lopez's avatar
James Lopez committed
163
        return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
164
165
166
167
168
169

        relation_class.encrypted_attributes.each_key do |key|
          @relation_hash[key.to_s] = nil
        end
      end

170
171
      def relation_class
        @relation_class ||= @relation_name.to_s.classify.constantize
172
      end
James Lopez's avatar
James Lopez committed
173

174
      def imported_object
175
176
        yield(existing_or_new_object) if block_given?
        existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
177

178
179
180
181
182
183
        existing_or_new_object
      rescue ActiveRecord::RecordNotUnique
        # as the operation is not atomic, retry in the unlikely scenario an INSERT is
        # performed on the same object between the SELECT and the INSERT
        @imported_object_retries += 1
        retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
James Lopez's avatar
James Lopez committed
184
      end
185
186
187
188
189
190
191

      def update_note_for_missing_author(author_name)
        @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
        @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
      end

      def admin_user?
blackst0ne's avatar
blackst0ne committed
192
        @user.admin?
193
      end
194
195

      def parsed_relation_hash
196
197
        @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash,
                                                                               relation_class: relation_class)
198
      end
James Lopez's avatar
James Lopez committed
199

200
      def set_st_diff_commits
James Lopez's avatar
James Lopez committed
201
        @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs')
202
203
204

        HashUtil.deep_symbolize_array!(@relation_hash['st_diffs'])
        HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits'])
James Lopez's avatar
James Lopez committed
205
      end
206

207
208
209
210
      def set_diff
        @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
      end

211
212
213
214
215
      def existing_or_new_object
        # Only find existing records to avoid mapping tables such as milestones
        # Otherwise always create the record, skipping the extra SELECT clause.
        @existing_or_new_object ||= begin
          if EXISTING_OBJECT_CHECK.include?(@relation_name)
216
            attribute_hash = attribute_hash_for(['events'])
217

218
            existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
219

220
221
222
223
224
225
            existing_object
          else
            relation_class.new(parsed_relation_hash)
          end
        end
      end
226

227
228
229
230
231
232
233
      def attribute_hash_for(attributes)
        attributes.inject({}) do |hash, value|
          hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value]
          hash
        end
      end

234
235
236
      def existing_object
        @existing_object ||=
          begin
237
238
            existing_object = find_or_create_object!

239
240
            # Done in two steps, as MySQL behaves differently than PostgreSQL using
            # the +find_or_create_by+ method and does not return the ID the second time.
241
            existing_object.update!(parsed_relation_hash)
242
243
244
            existing_object
          end
      end
245
246
247
248
249

      def unknown_service?
        @relation_name == :services && parsed_relation_hash['type'] &&
          !Object.const_defined?(parsed_relation_hash['type'])
      end
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268

      def find_or_create_object!
        finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id]
        finder_hash = parsed_relation_hash.slice(*finder_attributes)

        if label?
          label = relation_class.find_or_initialize_by(finder_hash)
          parsed_relation_hash.delete('priorities') if label.persisted?

          label.save!
          label
        else
          relation_class.find_or_create_by(finder_hash)
        end
      end

      def label?
        @relation_name.to_s.include?('label')
      end
269
270
271
    end
  end
end