Unverified Commit 3ef4f74b authored by Markus Koller's avatar Markus Koller
Browse files

Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.
parent b0d885b53d26
......@@ -85,4 +85,30 @@
it { expect(build.trace_file_path).to eq(build.old_path_to_trace) }
end
end
describe '#update_project_statistics' do
let!(:build) { create(:ci_build, artifacts_size: 23) }
it 'updates project statistics when the artifact size changes' do
expect(ProjectCacheWorker).to receive(:perform_async)
.with(build.project_id, [], [:build_artifacts_size])
build.artifacts_size = 42
build.save!
end
it 'does not update project statistics when the artifact size stays the same' do
expect(ProjectCacheWorker).not_to receive(:perform_async)
build.name = 'changed'
build.save!
end
it 'updates project statistics when the build is destroyed' do
expect(ProjectCacheWorker).to receive(:perform_async)
.with(build.project_id, [], [:build_artifacts_size])
build.destroy
end
end
end
require 'spec_helper'
describe LfsObjectsProject, models: true do
subject { create(:lfs_objects_project, project: project) }
let(:project) { create(:empty_project) }
describe 'associations' do
it { is_expected.to belong_to(:project) }
it { is_expected.to belong_to(:lfs_object) }
end
describe 'validation' do
it { is_expected.to validate_presence_of(:lfs_object_id) }
it { is_expected.to validate_uniqueness_of(:lfs_object_id).scoped_to(:project_id).with_message("already exists in project") }
it { is_expected.to validate_presence_of(:project_id) }
end
describe '#update_project_statistics' do
it 'updates project statistics when the object is added' do
expect(ProjectCacheWorker).to receive(:perform_async)
.with(project.id, [], [:lfs_objects_size])
subject.save!
end
it 'updates project statistics when the object is removed' do
subject.save!
expect(ProjectCacheWorker).to receive(:perform_async)
.with(project.id, [], [:lfs_objects_size])
subject.destroy
end
end
end
......@@ -4,6 +4,7 @@
let!(:namespace) { create(:namespace) }
it { is_expected.to have_many :projects }
it { is_expected.to have_many :project_statistics }
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) }
......@@ -57,6 +58,50 @@
end
end
describe '.with_statistics' do
let(:namespace) { create :namespace }
let(:project1) do
create(:empty_project,
namespace: namespace,
statistics: build(:project_statistics,
storage_size: 606,
repository_size: 101,
lfs_objects_size: 202,
build_artifacts_size: 303))
end
let(:project2) do
create(:empty_project,
namespace: namespace,
statistics: build(:project_statistics,
storage_size: 60,
repository_size: 10,
lfs_objects_size: 20,
build_artifacts_size: 30))
end
it "sums all project storage counters in the namespace" do
project1
project2
statistics = Namespace.with_statistics.find(namespace.id)
expect(statistics.storage_size).to eq 666
expect(statistics.repository_size).to eq 111
expect(statistics.lfs_objects_size).to eq 222
expect(statistics.build_artifacts_size).to eq 333
end
it "correctly handles namespaces without projects" do
statistics = Namespace.with_statistics.find(namespace.id)
expect(statistics.storage_size).to eq 0
expect(statistics.repository_size).to eq 0
expect(statistics.lfs_objects_size).to eq 0
expect(statistics.build_artifacts_size).to eq 0
end
end
describe '#move_dir' do
before do
@namespace = create :namespace
......
......@@ -49,6 +49,7 @@
it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) }
it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
it { is_expected.to have_one(:project_feature).dependent(:destroy) }
it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) }
it { is_expected.to have_one(:last_event).class_name('Event') }
it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
......@@ -1729,6 +1730,26 @@ def create_build(new_pipeline = pipeline, name = 'test')
end
end
describe '#update_project_statistics' do
let(:project) { create(:empty_project) }
it "is called after creation" do
expect(project.statistics).to be_a ProjectStatistics
expect(project.statistics).to be_persisted
end
it "copies the namespace_id" do
expect(project.statistics.namespace_id).to eq project.namespace_id
end
it "updates the namespace_id when changed" do
namespace = create(:namespace)
project.update(namespace: namespace)
expect(project.statistics.namespace_id).to eq namespace.id
end
end
def enable_lfs
allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
end
......
require 'rails_helper'
describe ProjectStatistics, models: true do
let(:project) { create :empty_project }
let(:statistics) { project.statistics }
describe 'constants' do
describe 'STORAGE_COLUMNS' do
it 'is an array of symbols' do
expect(described_class::STORAGE_COLUMNS).to be_kind_of Array
expect(described_class::STORAGE_COLUMNS.map(&:class).uniq).to eq [Symbol]
end
end
describe 'STATISTICS_COLUMNS' do
it 'is an array of symbols' do
expect(described_class::STATISTICS_COLUMNS).to be_kind_of Array
expect(described_class::STATISTICS_COLUMNS.map(&:class).uniq).to eq [Symbol]
end
it 'includes all storage columns' do
expect(described_class::STATISTICS_COLUMNS & described_class::STORAGE_COLUMNS).to eq described_class::STORAGE_COLUMNS
end
end
end
describe 'associations' do
it { is_expected.to belong_to(:project) }
it { is_expected.to belong_to(:namespace) }
end
describe 'statistics columns' do
it "support values up to 8 exabytes" do
statistics.update!(
commit_count: 8.exabytes - 1,
repository_size: 2.exabytes,
lfs_objects_size: 2.exabytes,
build_artifacts_size: 4.exabytes - 1,
)
statistics.reload
expect(statistics.commit_count).to eq(8.exabytes - 1)
expect(statistics.repository_size).to eq(2.exabytes)
expect(statistics.lfs_objects_size).to eq(2.exabytes)
expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
expect(statistics.storage_size).to eq(8.exabytes - 1)
end
end
describe '#total_repository_size' do
it "sums repository and LFS object size" do
statistics.repository_size = 2
statistics.lfs_objects_size = 3
statistics.build_artifacts_size = 4
expect(statistics.total_repository_size).to eq 5
end
end
describe '#refresh!' do
before do
allow(statistics).to receive(:update_commit_count)
allow(statistics).to receive(:update_repository_size)
allow(statistics).to receive(:update_lfs_objects_size)
allow(statistics).to receive(:update_build_artifacts_size)
allow(statistics).to receive(:update_storage_size)
end
context "without arguments" do
before do
statistics.refresh!
end
it "sums all counters" do
expect(statistics).to have_received(:update_commit_count)
expect(statistics).to have_received(:update_repository_size)
expect(statistics).to have_received(:update_lfs_objects_size)
expect(statistics).to have_received(:update_build_artifacts_size)
end
end
context "when passing an only: argument" do
before do
statistics.refresh! only: [:lfs_objects_size]
end
it "only updates the given columns" do
expect(statistics).to have_received(:update_lfs_objects_size)
expect(statistics).not_to have_received(:update_commit_count)
expect(statistics).not_to have_received(:update_repository_size)
expect(statistics).not_to have_received(:update_build_artifacts_size)
end
end
end
describe '#update_commit_count' do
before do
allow(project.repository).to receive(:commit_count).and_return(23)
statistics.update_commit_count
end
it "stores the number of commits in the repository" do
expect(statistics.commit_count).to eq 23
end
end
describe '#update_repository_size' do
before do
allow(project.repository).to receive(:size).and_return(12.megabytes)
statistics.update_repository_size
end
it "stores the size of the repository" do
expect(statistics.repository_size).to eq 12.megabytes
end
end
describe '#update_lfs_objects_size' do
let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
let!(:lfs_objects_project1) { create(:lfs_objects_project, project: project, lfs_object: lfs_object1) }
let!(:lfs_objects_project2) { create(:lfs_objects_project, project: project, lfs_object: lfs_object2) }
before do
statistics.update_lfs_objects_size
end
it "stores the size of related LFS objects" do
expect(statistics.lfs_objects_size).to eq 57.megabytes
end
end
describe '#update_build_artifacts_size' do
let!(:pipeline) { create(:ci_pipeline, project: project) }
let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) }
let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) }
before do
statistics.update_build_artifacts_size
end
it "stores the size of related build artifacts" do
expect(statistics.build_artifacts_size).to eq 101.megabytes
end
end
describe '#update_storage_size' do
it "sums all storage counters" do
statistics.update!(
repository_size: 2,
lfs_objects_size: 3,
)
statistics.reload
expect(statistics.storage_size).to eq 5
end
end
end
......@@ -35,6 +35,14 @@
expect(json_response.length).to eq(1)
expect(json_response.first['name']).to eq(group1.name)
end
it "does not include statistics" do
get api("/groups", user1), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include 'statistics'
end
end
context "when authenticated as admin" do
......@@ -44,6 +52,31 @@
expect(json_response).to be_an Array
expect(json_response.length).to eq(2)
end
it "does not include statistics by default" do
get api("/groups", admin)
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include('statistics')
end
it "includes statistics if requested" do
attributes = {
storage_size: 702,
repository_size: 123,
lfs_objects_size: 234,
build_artifacts_size: 345,
}
project1.statistics.update!(attributes)
get api("/groups", admin), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first['statistics']).to eq attributes.stringify_keys
end
end
context "when using skip_groups in request" do
......
......@@ -49,7 +49,7 @@
end
end
context 'when authenticated' do
context 'when authenticated as regular user' do
it 'returns an array of projects' do
get api('/projects', user)
expect(response).to have_http_status(200)
......@@ -172,6 +172,22 @@
end
end
end
it "does not include statistics by default" do
get api('/projects/all', admin)
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include('statistics')
end
it "includes statistics if requested" do
get api('/projects/all', admin), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).to include 'statistics'
end
end
end
......@@ -196,6 +212,32 @@
expect(json_response.first['name']).to eq(project4.name)
expect(json_response.first['owner']['username']).to eq(user4.username)
end
it "does not include statistics by default" do
get api('/projects/owned', user4)
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first).not_to include('statistics')
end
it "includes statistics if requested" do
attributes = {
commit_count: 23,
storage_size: 702,
repository_size: 123,
lfs_objects_size: 234,
build_artifacts_size: 345,
}
project4.statistics.update!(attributes)
get api('/projects/owned', user4), statistics: true
expect(response).to have_http_status(200)
expect(json_response).to be_an Array
expect(json_response.first['statistics']).to eq attributes.stringify_keys
end
end
end
......
......@@ -583,7 +583,7 @@
service.push_commits = [commit]
expect(ProjectCacheWorker).to receive(:perform_async).
with(project.id, %i(readme))
with(project.id, %i(readme), %i(commit_count repository_size))
service.update_caches
end
......@@ -596,7 +596,7 @@
it 'does not flush any conditional caches' do
expect(ProjectCacheWorker).to receive(:perform_async).
with(project.id, []).
with(project.id, [], %i(commit_count repository_size)).
and_call_original
service.update_caches
......
require 'spec_helper'
describe ProjectCacheWorker do
let(:project) { create(:project) }
let(:worker) { described_class.new }
let(:project) { create(:project) }
let(:statistics) { project.statistics }
describe '#perform' do
before do
......@@ -12,7 +13,7 @@
context 'with a non-existing project' do
it 'does nothing' do
expect(worker).not_to receive(:update_repository_size)
expect(worker).not_to receive(:update_statistics)
worker.perform(-1)
end
......@@ -22,24 +23,19 @@
it 'does nothing' do
allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
expect(worker).not_to receive(:update_repository_size)
expect(worker).not_to receive(:update_statistics)
worker.perform(project.id)
end
end
context 'with an existing project' do
it 'updates the repository size' do
expect(worker).to receive(:update_repository_size).and_call_original
worker.perform(project.id)
end
it 'updates the commit count' do
expect_any_instance_of(Project).to receive(:update_commit_count).
and_call_original
it 'updates the project statistics' do
expect(worker).to receive(:update_statistics)
.with(kind_of(Project), %i(repository_size))
.and_call_original
worker.perform(project.id)
worker.perform(project.id, [], %w(repository_size))
end
it 'refreshes the method caches' do
......@@ -47,33 +43,35 @@
with(%i(readme)).
and_call_original
worker.perform(project.id, %i(readme))
worker.perform(project.id, %w(readme))
end
end
end
describe '#update_repository_size' do
describe '#update_statistics' do
context 'when a lease could not be obtained' do
it 'does not update the repository size' do
allow(worker).to receive(:try_obtain_lease_for).
with(project.id, :update_repository_size).
with(project.id, :update_statistics).
and_return(false)
expect(project).not_to receive(:update_repository_size)
expect(statistics).not_to receive(:refresh!)
worker.update_repository_size(project)
worker.update_statistics(project)
end
end
context 'when a lease could be obtained' do
it 'updates the repository size' do
it 'updates the project statistics' do
allow(worker).to receive(:try_obtain_lease_for).
with(project.id, :update_repository_size).
with(project.id, :update_statistics).
and_return(true)
expect(project).to receive(:update_repository_size).and_call_original
expect(statistics).to receive(:refresh!)
.with(only: %i(repository_size))
.and_call_original
worker.update_repository_size(project)
worker.update_statistics(project, %i(repository_size))
end
end
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment