Commit 91719415 authored by Sean McGivern's avatar Sean McGivern
Browse files

Insert at most 1,000 rows at once in MR diff background migration

We were hitting the statement timeout for very large MR diffs. Now we insert at
most 1,000 rows to `merge_request_diff_commits` in a single statement, or 100
rows to `merge_request_diff_files`.
parent d72fd6d18249
......@@ -8,6 +8,7 @@ class MergeRequestDiff < ActiveRecord::Base
end
BUFFER_ROWS = 1000
DIFF_FILE_BUFFER_ROWS = 100
def perform(start_id, stop_id)
merge_request_diffs = MergeRequestDiff
......@@ -26,7 +27,7 @@ def perform(start_id, stop_id)
if diff_ids.length > BUFFER_ROWS ||
commit_rows.length > BUFFER_ROWS ||
file_rows.length > BUFFER_ROWS
file_rows.length > DIFF_FILE_BUFFER_ROWS
flush_buffers!
end
......@@ -46,8 +47,13 @@ def reset_buffers!
def flush_buffers!
if diff_ids.any?
MergeRequestDiff.transaction do
Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows)
Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows)
commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice|
Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows_slice)
end
file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice|
Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows_slice)
end
MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil)
end
......
......@@ -70,8 +70,8 @@ def convert_to_yaml(merge_request_diff_id, commits, diffs)
before do
merge_request.reload_diff(true)
convert_to_yaml(start_id, merge_request_diff.commits, merge_request_diff.diffs)
convert_to_yaml(stop_id, updated_merge_request_diff.commits, updated_merge_request_diff.diffs)
convert_to_yaml(start_id, merge_request_diff.commits, diffs_to_hashes(merge_request_diff.merge_request_diff_files))
convert_to_yaml(stop_id, updated_merge_request_diff.commits, diffs_to_hashes(updated_merge_request_diff.merge_request_diff_files))
MergeRequestDiffCommit.delete_all
MergeRequestDiffFile.delete_all
......@@ -80,6 +80,8 @@ def convert_to_yaml(merge_request_diff_id, commits, diffs)
context 'when BUFFER_ROWS is exceeded' do
before do
stub_const("#{described_class}::BUFFER_ROWS", 1)
allow(Gitlab::Database).to receive(:bulk_insert).and_call_original
end
it 'updates and continues' do
......@@ -87,6 +89,32 @@ def convert_to_yaml(merge_request_diff_id, commits, diffs)
subject.perform(start_id, stop_id)
end
it 'inserts commit rows in chunks of BUFFER_ROWS' do
# There are 29 commits in each diff, so we should have slices of 20 + 9 + 20 + 9.
stub_const("#{described_class}::BUFFER_ROWS", 20)
expect(Gitlab::Database).to receive(:bulk_insert)
.with('merge_request_diff_commits', anything)
.exactly(4)
.times
.and_call_original
subject.perform(start_id, stop_id)
end
it 'inserts diff rows in chunks of DIFF_FILE_BUFFER_ROWS' do
# There are 20 files in each diff, so we should have slices of 20 + 20.
stub_const("#{described_class}::DIFF_FILE_BUFFER_ROWS", 20)
expect(Gitlab::Database).to receive(:bulk_insert)
.with('merge_request_diff_files', anything)
.exactly(2)
.times
.and_call_original
subject.perform(start_id, stop_id)
end
end
context 'when BUFFER_ROWS is not exceeded' do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment