Commit 6071d51f authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Implement efficient destroy of job_trace_chunks

parent 49e3535697aa
......@@ -19,6 +19,30 @@ class JobTraceChunk < ActiveRecord::Base
db: 2
}
def self.delayed_cleanup_blk
ids = all.redis.pluck(:job_id, :chunk_index).map do |data|
"gitlab:ci:trace:#{data.first}:chunks:#{data.second}:data"
end
puts "before cleanup: #{ids.count}"
Proc.new do
puts "after cleanup: #{ids.count}"
Gitlab::Redis::SharedState.with do |redis|
redis.del(ids)
end unless ids.empty?
true
end
end
def self.fast_destroy_all
delayed_cleanup_blk.tap do |cleanup|
delete_all
cleanup.call
end
end
def data
if redis?
redis_data
......
......@@ -209,12 +209,21 @@ class Project < ActiveRecord::Base
has_many :commit_statuses
has_many :pipelines, class_name: 'Ci::Pipeline', inverse_of: :project
# This has to be defined before `has_many :builds, depenedent: :destroy`,
# otherwise we will not delete any data, due to trace chunks
# going through :builds
before_destroy do
puts "destroying all chunks"
self.run_after_commit(&build_trace_chunks.delayed_cleanup_blk)
end
# Ci::Build objects store data on the file system such as artifact files and
# build traces. Currently there's no efficient way of removing this data in
# bulk that doesn't involve loading the rows into memory. As a result we're
# still using `dependent: :destroy` here.
has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName'
has_many :build_trace_chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, through: :builds, source: :chunks
has_many :runner_projects, class_name: 'Ci::RunnerProject'
has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
has_many :variables, class_name: 'Ci::Variable'
......
......@@ -7,6 +7,7 @@ class BuildFinishedWorker
def perform(build_id)
Ci::Build.find_by(id: build_id).try do |build|
# Swap all trace chunks to Database from Redis
# TODO: Do we need that?
build.chunks.redis.map(&:use_database!)
# We execute that in sync as this access the files in order to access local data, and reduce IO
......
......@@ -100,7 +100,7 @@ def erase!
FileUtils.rm(trace_path, force: true)
end
job.chunks.delete_all
job.chunks.fast_destroy_all
job.erase_old_trace!
end
......
......@@ -140,7 +140,7 @@ def truncate(offset)
@size = offset
# remove all next chunks
job_chunks.where('chunk_index > ?', chunk_index).delete_all
job_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
# truncate current chunk
current_chunk.truncate(chunk_offset) if chunk_offset != 0
......@@ -156,8 +156,8 @@ def present?
true
end
def delete!
job_chunks.delete_all
def destroy!
job_chunks.fast_destroy_all
@tell = @size = 0
ensure
invalidate_chunk_cache
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment