Read about our upcoming Code of Conduct on this issue

Unverified Commit 6ec53f5d authored by Yorick Peterse's avatar Yorick Peterse
Browse files

Cache the number of open issues and merge requests

Every project page displays a navigation menu that in turn displays the
number of open issues and merge requests. This means that for every
project page we run two COUNT(*) queries, each taking up roughly 30
milliseconds on GitLab.com. By caching these numbers and refreshing them
whenever necessary we can reduce loading times of all these pages by up
to roughly 60 milliseconds.

The number of open issues does not include confidential issues. This is
a trade-off to keep the code simple and to ensure refreshing the data
only needs 2 COUNT(*) queries instead of 3. A downside is that if a
project only has 5 confidential issues the counter will be set to 0.

Because we now have 3 similar counting service classes the code
previously used in Projects::ForksCountService has mostly been moved to
Projects::CountService, which in turn is reused by the various service
classes.

Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/36622
parent c8426d249761
......@@ -53,7 +53,10 @@ class Issue < ActiveRecord::Base
scope :preload_associations, -> { preload(:labels, project: :namespace) }
scope :public_only, -> { where(confidential: false) }
after_save :expire_etag_cache
after_commit :update_project_counter_caches, on: :destroy
attr_spammable :title, spam_title: true
attr_spammable :description, spam_description: true
......@@ -269,6 +272,10 @@ def as_json(options = {})
end
end
def update_project_counter_caches
Projects::OpenIssuesCountService.new(project).refresh_cache
end
private
# Returns `true` if the given User can read the current Issue.
......
......@@ -32,6 +32,7 @@ class MergeRequest < ActiveRecord::Base
after_create :ensure_merge_request_diff, unless: :importing?
after_update :reload_diff_if_branch_changed
after_commit :update_project_counter_caches, on: :destroy
# When this attribute is true some MR validation is ignored
# It allows us to close or modify broken merge requests
......@@ -937,6 +938,10 @@ def mergeable_with_quick_action?(current_user, autocomplete_precheck: false, las
true
end
def update_project_counter_caches
Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache
end
private
def write_ref
......
......@@ -1158,7 +1158,11 @@ def build_timeout_in_minutes=(value)
end
def open_issues_count
issues.opened.count
Projects::OpenIssuesCountService.new(self).count
end
def open_merge_requests_count
Projects::OpenMergeRequestsCountService.new(self).count
end
def visibility_level_allowed_as_fork?(level = self.visibility_level)
......
......@@ -192,6 +192,8 @@ def before_create(issuable)
def after_create(issuable)
# To be overridden by subclasses
issuable.update_project_counter_caches
end
def before_update(issuable)
......@@ -200,6 +202,8 @@ def before_update(issuable)
def after_update(issuable)
# To be overridden by subclasses
issuable.update_project_counter_caches
end
def update(issuable)
......
......@@ -27,6 +27,8 @@ def after_create(issuable)
todo_service.new_issue(issuable, current_user)
user_agent_detail_service.create
resolve_discussions_with_issue(issuable)
super
end
def resolve_discussions_with_issue(issue)
......
......@@ -28,6 +28,8 @@ def after_create(issuable)
todo_service.new_merge_request(issuable, current_user)
issuable.cache_merge_request_closes_issues!(current_user)
update_merge_requests_head_pipeline(issuable)
super
end
private
......
module Projects
# Base class for the various service classes that count project data (e.g.
# issues or forks).
class CountService
def initialize(project)
@project = project
end
def relation_for_count
raise(
NotImplementedError,
'"relation_for_count" must be implemented and return an ActiveRecord::Relation'
)
end
def count
Rails.cache.fetch(cache_key) { uncached_count }
end
def refresh_cache
Rails.cache.write(cache_key, uncached_count)
end
def uncached_count
relation_for_count.count
end
def delete_cache
Rails.cache.delete(cache_key)
end
def cache_key_name
raise(
NotImplementedError,
'"cache_key_name" must be implemented and return a String'
)
end
def cache_key
['projects', @project.id, cache_key_name]
end
end
end
module Projects
# Service class for getting and caching the number of forks of a project.
class ForksCountService
def initialize(project)
@project = project
class ForksCountService < CountService
def relation_for_count
@project.forks
end
def count
Rails.cache.fetch(cache_key) { uncached_count }
end
def refresh_cache
Rails.cache.write(cache_key, uncached_count)
end
def delete_cache
Rails.cache.delete(cache_key)
end
private
def uncached_count
@project.forks.count
end
def cache_key
['projects', @project.id, 'forks_count']
def cache_key_name
'forks_count'
end
end
end
module Projects
# Service class for counting and caching the number of open issues of a
# project.
class OpenIssuesCountService < CountService
def relation_for_count
# We don't include confidential issues in this number since this would
# expose the number of confidential issues to non project members.
@project.issues.opened.public_only
end
def cache_key_name
'open_issues_count'
end
end
end
module Projects
# Service class for counting and caching the number of open merge requests of
# a project.
class OpenMergeRequestsCountService < CountService
def relation_for_count
@project.merge_requests.opened
end
def cache_key_name
'open_merge_requests_count'
end
end
end
......@@ -86,7 +86,8 @@
%span.nav-item-name
Issues
- if @project.issues_enabled?
%span.badge.count.issue_counter= number_with_delimiter(IssuesFinder.new(current_user, project_id: @project.id).execute.opened.count)
%span.badge.count.issue_counter
= number_with_delimiter(@project.open_issues_count)
%ul.sidebar-sub-level-items
= nav_link(controller: :issues) do
......@@ -116,7 +117,8 @@
= custom_icon('mr_bold')
%span.nav-item-name
Merge Requests
%span.badge.count.merge_counter.js-merge-counter= number_with_delimiter(MergeRequestsFinder.new(current_user, project_id: @project.id).execute.opened.count)
%span.badge.count.merge_counter.js-merge-counter
= number_with_delimiter(@project.open_merge_requests_count)
- if project_nav_tab? :pipelines
= nav_link(controller: [:pipelines, :builds, :jobs, :pipeline_schedules, :environments, :artifacts]) do
......
......@@ -28,7 +28,8 @@
%span
Issues
- if @project.issues_enabled?
%span.badge.count.issue_counter= number_with_delimiter(issuables_count_for_state(:issues, :opened, finder: IssuesFinder.new(current_user, project_id: @project.id)))
%span.badge.count.issue_counter
= number_with_delimiter(@project.open_issues_count)
- if project_nav_tab? :merge_requests
- controllers = [:merge_requests, 'projects/merge_requests/conflicts']
......@@ -37,7 +38,8 @@
= link_to project_merge_requests_path(@project), title: 'Merge Requests', class: 'shortcuts-merge_requests' do
%span
Merge Requests
%span.badge.count.merge_counter.js-merge-counter= number_with_delimiter(issuables_count_for_state(:merge_requests, :opened, finder: MergeRequestsFinder.new(current_user, project_id: @project.id)))
%span.badge.count.merge_counter.js-merge-counter
= number_with_delimiter(@project.open_merge_requests_count)
- if project_nav_tab? :pipelines
= nav_link(controller: [:pipelines, :builds, :environments, :artifacts]) do
......
---
title: Cache the number of open issues and merge requests
merge_request:
author:
type: other
......@@ -751,4 +751,22 @@
end
end
end
describe 'removing an issue' do
it 'refreshes the number of open issues of the project' do
project = subject.project
expect { subject.destroy }
.to change { project.open_issues_count }.from(1).to(0)
end
end
describe '.public_only' do
it 'only returns public issues' do
public_issue = create(:issue)
create(:issue, confidential: true)
expect(described_class.public_only).to eq([public_issue])
end
end
end
......@@ -1692,4 +1692,13 @@ def create_pipeline(status)
expect(subject.ref_fetched?).to be_falsey
end
end
describe 'removing a merge request' do
it 'refreshes the number of open merge requests of the target project' do
project = subject.target_project
expect { subject.destroy }
.to change { project.open_merge_requests_count }.from(1).to(0)
end
end
end
......@@ -42,6 +42,11 @@
service.execute(issue)
end
it 'refreshes the number of open issues' do
expect { service.execute(issue) }
.to change { project.open_issues_count }.from(1).to(0)
end
it 'invalidates counter cache for assignees' do
expect_any_instance_of(User).to receive(:invalidate_issue_cache_counts)
......
......@@ -35,6 +35,10 @@
expect(issue.due_date).to eq Date.tomorrow
end
it 'refreshes the number of open issues' do
expect { issue }.to change { project.open_issues_count }.from(0).to(1)
end
context 'when current user cannot admin issues in the project' do
let(:guest) { create(:user) }
......
......@@ -34,6 +34,13 @@
described_class.new(project, user).execute(issue)
end
it 'refreshes the number of opened issues' do
service = described_class.new(project, user)
expect { service.execute(issue) }
.to change { project.open_issues_count }.from(0).to(1)
end
context 'when issue is not confidential' do
it 'executes issue hooks' do
expect(project).to receive(:execute_hooks).with(an_instance_of(Hash), :issue_hooks)
......
......@@ -52,6 +52,13 @@
end
end
it 'refreshes the number of open merge requests for a valid MR' do
service = described_class.new(project, user, {})
expect { service.execute(merge_request) }
.to change { project.open_merge_requests_count }.from(1).to(0)
end
context 'current user is not authorized to close merge request' do
before do
perform_enqueued_jobs do
......
......@@ -18,31 +18,35 @@
end
let(:service) { described_class.new(project, user, opts) }
let(:merge_request) { service.execute }
before do
project.team << [user, :master]
project.team << [assignee, :developer]
allow(service).to receive(:execute_hooks)
@merge_request = service.execute
end
it 'creates an MR' do
expect(@merge_request).to be_valid
expect(@merge_request.title).to eq('Awesome merge_request')
expect(@merge_request.assignee).to be_nil
expect(@merge_request.merge_params['force_remove_source_branch']).to eq('1')
expect(merge_request).to be_valid
expect(merge_request.title).to eq('Awesome merge_request')
expect(merge_request.assignee).to be_nil
expect(merge_request.merge_params['force_remove_source_branch']).to eq('1')
end
it 'executes hooks with default action' do
expect(service).to have_received(:execute_hooks).with(@merge_request)
expect(service).to have_received(:execute_hooks).with(merge_request)
end
it 'refreshes the number of open merge requests' do
expect { service.execute }
.to change { project.open_merge_requests_count }.from(0).to(1)
end
it 'does not creates todos' do
attributes = {
project: project,
target_id: @merge_request.id,
target_type: @merge_request.class.name
target_id: merge_request.id,
target_type: merge_request.class.name
}
expect(Todo.where(attributes).count).to be_zero
......@@ -51,8 +55,8 @@
it 'creates exactly 1 create MR event' do
attributes = {
action: Event::CREATED,
target_id: @merge_request.id,
target_type: @merge_request.class.name
target_id: merge_request.id,
target_type: merge_request.class.name
}
expect(Event.where(attributes).count).to eq(1)
......@@ -69,15 +73,15 @@
}
end
it { expect(@merge_request.assignee).to eq assignee }
it { expect(merge_request.assignee).to eq assignee }
it 'creates a todo for new assignee' do
attributes = {
project: project,
author: user,
user: assignee,
target_id: @merge_request.id,
target_type: @merge_request.class.name,
target_id: merge_request.id,
target_type: merge_request.class.name,
action: Todo::ASSIGNED,
state: :pending
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment