This instance will be upgraded to Heptapod 0.31.0 (final) on 2022-05-24 at 14:00 UTC+2 (a few minutes of down time)

Commit a968d53d authored by Yorick Peterse's avatar Yorick Peterse
Browse files

Track and act upon the number of executed queries

This ensures that we have more visibility in the number of SQL queries
that are executed in web requests. The current threshold is hardcoded to
100 as we will rarely (maybe once or twice) change it.

In production and development we use Sentry if enabled, in the test
environment we raise an error. This feature is also only enabled in
production/staging when running on GitLab.com as it's not very useful to
other users.
parent 456b31523866
class Admin::ServicesController < Admin::ApplicationController
include ServiceParams
before_action :whitelist_query_limiting, only: [:index]
before_action :service, only: [:edit, :update]
def index
......@@ -37,4 +38,8 @@ def services_templates
def service
@service ||= Service.where(id: params[:id], template: true).first
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42430')
end
end
......@@ -2,6 +2,7 @@ module Boards
class IssuesController < Boards::ApplicationController
include BoardsResponses
before_action :whitelist_query_limiting, only: [:index, :update]
before_action :authorize_read_issue, only: [:index]
before_action :authorize_create_issue, only: [:create]
before_action :authorize_update_issue, only: [:update]
......@@ -92,5 +93,10 @@ def serialize_as_json(resource)
}
)
end
def whitelist_query_limiting
# Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42439
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42428')
end
end
end
class Import::GitlabProjectsController < Import::BaseController
before_action :whitelist_query_limiting, only: [:create]
before_action :verify_gitlab_project_import_enabled
def new
......@@ -40,4 +41,8 @@ def project_params
:path, :namespace_id, :file
)
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437')
end
end
......@@ -4,6 +4,7 @@ class Projects::CommitsController < Projects::ApplicationController
include ExtractsPath
include RendersCommits
before_action :whitelist_query_limiting
before_action :require_non_empty_project
before_action :assign_ref_vars
before_action :authorize_download_code!
......@@ -65,4 +66,8 @@ def set_commits
@commits = @commits.with_pipeline_status
@commits = prepare_commits_for_rendering(@commits)
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42330')
end
end
......@@ -3,6 +3,7 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController
include ActionView::Helpers::TextHelper
include CycleAnalyticsParams
before_action :whitelist_query_limiting, only: [:show]
before_action :authorize_read_cycle_analytics!
def show
......@@ -31,4 +32,8 @@ def cycle_analytics_json
permissions: @cycle_analytics.permissions(user: current_user)
}
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42671')
end
end
......@@ -2,6 +2,7 @@ class Projects::ForksController < Projects::ApplicationController
include ContinueParams
# Authorize
before_action :whitelist_query_limiting, only: [:create]
before_action :require_non_empty_project
before_action :authorize_download_code!
before_action :authenticate_user!, only: [:new, :create]
......@@ -54,4 +55,8 @@ def create
render :error
end
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42335')
end
end
......@@ -8,6 +8,7 @@ class Projects::IssuesController < Projects::ApplicationController
prepend_before_action :authenticate_user!, only: [:new]
before_action :whitelist_query_limiting, only: [:create, :create_merge_request, :move, :bulk_update]
before_action :check_issues_available!
before_action :issue, except: [:index, :new, :create, :bulk_update]
before_action :set_issuables_index, only: [:index]
......@@ -247,4 +248,13 @@ def set_issuables_index
@finder_type = IssuesFinder
super
end
def whitelist_query_limiting
# Also see the following issues:
#
# 1. https://gitlab.com/gitlab-org/gitlab-ce/issues/42423
# 2. https://gitlab.com/gitlab-org/gitlab-ce/issues/42424
# 3. https://gitlab.com/gitlab-org/gitlab-ce/issues/42426
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42422')
end
end
......@@ -4,6 +4,7 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap
include RendersCommits
skip_before_action :merge_request
before_action :whitelist_query_limiting, only: [:create]
before_action :authorize_create_merge_request!
before_action :apply_diff_view_cookie!, only: [:diffs, :diff_for_path]
before_action :build_merge_request, except: [:create]
......@@ -125,4 +126,8 @@ def selected_target_project
@project.forked_from_project
end
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42384')
end
end
......@@ -7,6 +7,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
include IssuableCollections
skip_before_action :merge_request, only: [:index, :bulk_update]
before_action :whitelist_query_limiting, only: [:assign_related_issues, :update]
before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort]
before_action :set_issuables_index, only: [:index]
before_action :authenticate_user!, only: [:assign_related_issues]
......@@ -339,4 +340,9 @@ def check_user_can_push_to_source_branch!
access_denied! unless access_check
end
def whitelist_query_limiting
# Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42441
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42438')
end
end
......@@ -2,6 +2,7 @@ class Projects::NetworkController < Projects::ApplicationController
include ExtractsPath
include ApplicationHelper
before_action :whitelist_query_limiting
before_action :require_non_empty_project
before_action :assign_ref_vars
before_action :authorize_download_code!
......@@ -35,4 +36,8 @@ def assign_commit
@options[:extended_sha1] = params[:extended_sha1]
@commit = @repo.commit(@options[:extended_sha1])
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42333')
end
end
......@@ -2,6 +2,7 @@ class Projects::NotesController < Projects::ApplicationController
include NotesActions
include ToggleAwardEmoji
before_action :whitelist_query_limiting, only: [:create]
before_action :authorize_read_note!
before_action :authorize_create_note!, only: [:create]
before_action :authorize_resolve_note!, only: [:resolve, :unresolve]
......@@ -79,4 +80,8 @@ def authorize_create_note!
access_denied! unless can?(current_user, :create_note, noteable)
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42383')
end
end
class Projects::PipelinesController < Projects::ApplicationController
before_action :whitelist_query_limiting, only: [:create, :retry]
before_action :pipeline, except: [:index, :new, :create, :charts]
before_action :commit, only: [:show, :builds, :failures]
before_action :authorize_read_pipeline!
......@@ -166,4 +167,9 @@ def pipeline
def commit
@commit ||= @pipeline.commit
end
def whitelist_query_limiting
# Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42343
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42339')
end
end
......@@ -3,6 +3,7 @@ class ProjectsController < Projects::ApplicationController
include ExtractsPath
include PreviewMarkdown
before_action :whitelist_query_limiting, only: [:create]
before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
before_action :redirect_git_extension, only: [:show]
before_action :project, except: [:index, :new, :create]
......@@ -405,4 +406,8 @@ def redirect_git_extension
#
redirect_to request.original_url.sub(%r{\.git/?\Z}, '') if params[:format] == 'git'
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
end
class RegistrationsController < Devise::RegistrationsController
include Recaptcha::Verify
before_action :whitelist_query_limiting, only: [:destroy]
def new
redirect_to(new_user_session_path)
end
......@@ -83,4 +85,8 @@ def resource
def devise_mapping
@devise_mapping ||= Devise.mappings[:user]
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42380')
end
end
---
title: Track and act upon the number of executed queries
merge_request:
author:
type: added
if Gitlab::QueryLimiting.enable?
require_dependency 'gitlab/query_limiting/active_support_subscriber'
require_dependency 'gitlab/query_limiting/transaction'
require_dependency 'gitlab/query_limiting/middleware'
Gitlab::Application.configure do |config|
config.middleware.use(Gitlab::QueryLimiting::Middleware)
end
end
......@@ -75,6 +75,7 @@ comments: false
- [Ordering table columns](ordering_table_columns.md)
- [Verifying database capabilities](verifying_database_capabilities.md)
- [Database Debugging and Troubleshooting](database_debugging.md)
- [Query Count Limits](query_count_limits.md)
## Testing guides
......
# Query Count Limits
Each controller or API endpoint is allowed to execute up to 100 SQL queries. In
a production environment we'll only log an error in case this threshold is
exceeded, but in a test environment we'll raise an error instead.
## Solving Failing Tests
When a test fails because it executes more than 100 SQL queries there are two
solutions to this problem:
1. Reduce the number of SQL queries that are executed.
2. Whitelist the controller or API endpoint.
You should only resort to whitelisting when an existing controller or endpoint
is to blame as in this case reducing the number of SQL queries can take a lot of
effort. Newly added controllers and endpoints are not allowed to execute more
than 100 SQL queries and no exceptions will be made for this rule. _If_ a large
number of SQL queries is necessary to perform certain work it's best to have
this work performed by Sidekiq instead of doing this directly in a web request.
## Whitelisting
In the event that you _have_ to whitelist a controller you'll first need to
create an issue. This issue should (preferably in the title) mention the
controller or endpoint and include the appropriate labels (`database`,
`performance`, and at least a team specific label such as `Discussion`).
Once the issue has been created you can whitelist the code in question. For
Rails controllers it's best to create a `before_action` hook that runs as early
as possible. The called method in turn should call
`Gitlab::QueryLimiting.whitelist('issue URL here')`. For example:
```ruby
class MyController < ApplicationController
before_action :whitelist_query_limiting, only: [:show]
def index
# ...
end
def show
# ...
end
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/...')
end
end
```
By using a `before_action` you don't have to modify the controller method in
question, reducing the likelihood of merge conflicts.
For Grape API endpoints there unfortunately is not a reliable way of running a
hook before a specific endpoint. This means that you have to add the whitelist
call directly into the endpoint like so:
```ruby
get '/projects/:id/foo' do
Gitlab::QueryLimiting.whitelist('...')
# ...
end
```
......@@ -29,6 +29,8 @@ def find_branch!(branch_name)
use :pagination
end
get ':id/repository/branches' do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329')
repository = user_project.repository
branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name))
merged_branch_names = repository.merged_branch_names(branches.map(&:name))
......
......@@ -161,6 +161,8 @@ def find_issues(args = {})
use :issue_params
end
post ':id/issues' do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42320')
authorize! :create_issue, user_project
# Setting created_at time only allowed for admins and project owners
......@@ -201,6 +203,8 @@ def find_issues(args = {})
:labels, :created_at, :due_date, :confidential, :state_event
end
put ':id/issues/:issue_iid' do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322')
issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
authorize! :update_issue, issue
......@@ -234,6 +238,8 @@ def find_issues(args = {})
requires :to_project_id, type: Integer, desc: 'The ID of the new project'
end
post ':id/issues/:issue_iid/move' do
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323')
issue = user_project.issues.find_by(iid: params[:issue_iid])
not_found!('Issue') unless issue
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment