Read about our upcoming Code of Conduct on this issue

populate_untracked_uploads_spec.rb 10.1 KB
Newer Older
1
2
require 'spec_helper'

3
4
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180208183958 do
5
  include MigrationsHelpers::TrackUntrackedUploadsHelpers
Michael Kozono's avatar
Michael Kozono committed
6

7
8
  subject { described_class.new }

9
10
11
12
13
14
15
16
  let!(:appearances) { table(:appearances) }
  let!(:namespaces) { table(:namespaces) }
  let!(:notes) { table(:notes) }
  let!(:projects) { table(:projects) }
  let!(:routes) { table(:routes) }
  let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
  let!(:uploads) { table(:uploads) }
  let!(:users) { table(:users) }
17

18
  before do
19
    ensure_temporary_tracking_table_exists
Michael Kozono's avatar
Michael Kozono committed
20
    uploads.delete_all
21
22
  end

23
  context 'with untracked files and tracked files in untracked_files_for_uploads' do
24
25
26
27
28
    let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) }
    let!(:user1) { create_user(avatar: true) }
    let!(:user2) { create_user(avatar: true) }
    let!(:project1) { create_project(avatar: true) }
    let!(:project2) { create_project(avatar: true) }
29

Michael Kozono's avatar
Michael Kozono committed
30
    before do
31
32
      add_markdown_attachment(project1)
      add_markdown_attachment(project2)
33

34
      # File records created by PrepareUntrackedUploads
35
36
37
38
39
40
      untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').first.path)
      untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').last.path)
      untracked_files_for_uploads.create!(path: get_uploads(user1, 'User').first.path)
      untracked_files_for_uploads.create!(path: get_uploads(user2, 'User').first.path)
      untracked_files_for_uploads.create!(path: get_uploads(project1, 'Project').first.path)
      untracked_files_for_uploads.create!(path: get_uploads(project2, 'Project').first.path)
41
42
      untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project1).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project1, 'Project').last.path}")
      untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project2).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project2, 'Project').last.path}")
43

Michael Kozono's avatar
Michael Kozono committed
44
      # Untrack 4 files
45
46
47
      get_uploads(user2, 'User').delete_all
      get_uploads(project2, 'Project').delete_all # 2 files: avatar and a Markdown upload
      get_uploads(appearance, 'Appearance').where("path like '%header_logo%'").delete_all
48
49
50
51
    end

    it 'adds untracked files to the uploads table' do
      expect do
Nick Thomas's avatar
Nick Thomas committed
52
        subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id)
53
54
      end.to change { uploads.count }.from(4).to(8)

55
56
57
      expect(get_uploads(user2, 'User').count).to eq(1)
      expect(get_uploads(project2, 'Project').count).to eq(2)
      expect(get_uploads(appearance, 'Appearance').count).to eq(2)
58
59
    end

Michael Kozono's avatar
Michael Kozono committed
60
    it 'deletes rows after processing them' do
61
62
      expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it

Michael Kozono's avatar
Michael Kozono committed
63
      expect do
64
        subject.perform(1, untracked_files_for_uploads.last.id)
Michael Kozono's avatar
Michael Kozono committed
65
      end.to change { untracked_files_for_uploads.count }.from(8).to(0)
Michael Kozono's avatar
Michael Kozono committed
66
67
    end

68
    it 'does not create duplicate uploads of already tracked files' do
69
      subject.perform(1, untracked_files_for_uploads.last.id)
70

71
72
73
      expect(get_uploads(user1, 'User').count).to eq(1)
      expect(get_uploads(project1, 'Project').count).to eq(2)
      expect(get_uploads(appearance, 'Appearance').count).to eq(2)
74
    end
Michael Kozono's avatar
Michael Kozono committed
75
76

    it 'uses the start and end batch ids [only 1st half]' do
Michael Kozono's avatar
Michael Kozono committed
77
78
79
      ids = untracked_files_for_uploads.all.order(:id).pluck(:id)
      start_id = ids[0]
      end_id = ids[3]
Michael Kozono's avatar
Michael Kozono committed
80
81

      expect do
82
        subject.perform(start_id, end_id)
Michael Kozono's avatar
Michael Kozono committed
83
84
      end.to change { uploads.count }.from(4).to(6)

85
86
87
88
89
      expect(get_uploads(user1, 'User').count).to eq(1)
      expect(get_uploads(user2, 'User').count).to eq(1)
      expect(get_uploads(appearance, 'Appearance').count).to eq(2)
      expect(get_uploads(project1, 'Project').count).to eq(2)
      expect(get_uploads(project2, 'Project').count).to eq(0)
Michael Kozono's avatar
Michael Kozono committed
90
91

      # Only 4 have been either confirmed or added to uploads
Michael Kozono's avatar
Michael Kozono committed
92
      expect(untracked_files_for_uploads.count).to eq(4)
Michael Kozono's avatar
Michael Kozono committed
93
94
95
    end

    it 'uses the start and end batch ids [only 2nd half]' do
Michael Kozono's avatar
Michael Kozono committed
96
97
98
      ids = untracked_files_for_uploads.all.order(:id).pluck(:id)
      start_id = ids[4]
      end_id = ids[7]
Michael Kozono's avatar
Michael Kozono committed
99
100

      expect do
101
        subject.perform(start_id, end_id)
Michael Kozono's avatar
Michael Kozono committed
102
103
      end.to change { uploads.count }.from(4).to(6)

104
105
106
107
108
      expect(get_uploads(user1, 'User').count).to eq(1)
      expect(get_uploads(user2, 'User').count).to eq(0)
      expect(get_uploads(appearance, 'Appearance').count).to eq(1)
      expect(get_uploads(project1, 'Project').count).to eq(2)
      expect(get_uploads(project2, 'Project').count).to eq(2)
Michael Kozono's avatar
Michael Kozono committed
109
110

      # Only 4 have been either confirmed or added to uploads
Michael Kozono's avatar
Michael Kozono committed
111
      expect(untracked_files_for_uploads.count).to eq(4)
Michael Kozono's avatar
Michael Kozono committed
112
    end
113
114
115
116

    it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do
      subject.perform(1, untracked_files_for_uploads.last.id - 1)

117
      expect(ActiveRecord::Base.connection.data_source_exists?(:untracked_files_for_uploads)).to be_truthy
118
119
120
    end

    it 'drops the temporary tracking table after processing the batch, if there are no untracked rows left' do
Michael Kozono's avatar
Michael Kozono committed
121
      expect(subject).to receive(:drop_temp_table_if_finished)
122

Michael Kozono's avatar
Michael Kozono committed
123
      subject.perform(1, untracked_files_for_uploads.last.id)
124
    end
125
126

    it 'does not block a whole batch because of one bad path' do
127
      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a")
128
129
130
131
132
133
134
135
136
137
      expect(untracked_files_for_uploads.count).to eq(9)
      expect(uploads.count).to eq(4)

      subject.perform(1, untracked_files_for_uploads.last.id)

      expect(untracked_files_for_uploads.count).to eq(1)
      expect(uploads.count).to eq(8)
    end

    it 'an unparseable path is shown in error output' do
138
      bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a"
139
140
141
142
143
144
      untracked_files_for_uploads.create!(path: bad_path)

      expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/)

      subject.perform(1, untracked_files_for_uploads.last.id)
    end
145
146
147
148
149
  end

  context 'with no untracked files' do
    it 'does not add to the uploads table (and does not raise error)' do
      expect do
150
        subject.perform(1, 1000)
151
152
153
      end.not_to change { uploads.count }.from(0)
    end
  end
154

155
156
  describe 'upload outcomes for each path pattern' do
    shared_examples_for 'non_markdown_file' do
157
      let!(:expected_upload_attrs) { model_uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
158
      let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
159
160

      before do
161
        model_uploads.delete_all
162
163
164
165
      end

      it 'creates an Upload record' do
        expect do
166
          subject.perform(1, untracked_files_for_uploads.last.id)
167
        end.to change { model_uploads.count }.from(0).to(1)
168

169
        expect(model_uploads.first.attributes).to include(expected_upload_attrs)
170
171
172
      end
    end

Michael Kozono's avatar
Michael Kozono committed
173
    context 'for an appearance logo file path' do
174
175
      let(:model) { create_or_update_appearance(logo: true) }
      let(:model_uploads) { get_uploads(model, 'Appearance') }
176

177
      it_behaves_like 'non_markdown_file'
Michael Kozono's avatar
Michael Kozono committed
178
    end
179

Michael Kozono's avatar
Michael Kozono committed
180
    context 'for an appearance header_logo file path' do
181
182
      let(:model) { create_or_update_appearance(header_logo: true) }
      let(:model_uploads) { get_uploads(model, 'Appearance') }
183

184
      it_behaves_like 'non_markdown_file'
185
186
187
    end

    context 'for a pre-Markdown Note attachment file path' do
188
189
      let(:model) { create_note(attachment: true) }
      let!(:expected_upload_attrs) { get_uploads(model, 'Note').first.attributes.slice('path', 'uploader', 'size', 'checksum') }
190
191
192
      let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }

      before do
193
        get_uploads(model, 'Note').delete_all
194
195
      end

196
197
198
199
      # Can't use the shared example because Note doesn't have an `uploads` association
      it 'creates an Upload record' do
        expect do
          subject.perform(1, untracked_files_for_uploads.last.id)
200
        end.to change { get_uploads(model, 'Note').count }.from(0).to(1)
201

202
        expect(get_uploads(model, 'Note').first.attributes).to include(expected_upload_attrs)
203
      end
204
205
206
    end

    context 'for a user avatar file path' do
207
208
      let(:model) { create_user(avatar: true) }
      let(:model_uploads) { get_uploads(model, 'User') }
209

210
      it_behaves_like 'non_markdown_file'
211
212
213
    end

    context 'for a group avatar file path' do
214
215
      let(:model) { create_group(avatar: true) }
      let(:model_uploads) { get_uploads(model, 'Namespace') }
216

217
      it_behaves_like 'non_markdown_file'
218
219
220
    end

    context 'for a project avatar file path' do
221
222
      let(:model) { create_project(avatar: true) }
      let(:model_uploads) { get_uploads(model, 'Project') }
223

224
      it_behaves_like 'non_markdown_file'
225
226
227
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
228
      let(:model) { create_project }
Michael Kozono's avatar
Michael Kozono committed
229

230
      before do
Michael Kozono's avatar
Michael Kozono committed
231
        # Upload the file
232
        add_markdown_attachment(model)
Michael Kozono's avatar
Michael Kozono committed
233
234

        # Create the untracked_files_for_uploads record
235
        untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(model)}/#{get_uploads(model, 'Project').first.path}")
Michael Kozono's avatar
Michael Kozono committed
236
237

        # Save the expected upload attributes
238
        @expected_upload_attrs = get_uploads(model, 'Project').first.attributes.slice('path', 'uploader', 'size', 'checksum')
Michael Kozono's avatar
Michael Kozono committed
239
240

        # Untrack the file
241
        get_uploads(model, 'Project').delete_all
242
243
244
245
      end

      it 'creates an Upload record' do
        expect do
246
          subject.perform(1, untracked_files_for_uploads.last.id)
247
        end.to change { get_uploads(model, 'Project').count }.from(0).to(1)
248

249
        expect(get_uploads(model, 'Project').first.attributes).to include(@expected_upload_attrs)
250
251
252
      end
    end
  end
253
end