This instance will be upgraded to Heptapod 0.27.1 on 2021-12-02 at 15:00 UTC+1 (a few minutes of down time)

Commit d2aa74a6 authored by Francisco Javier López's avatar Francisco Javier López Committed by James Lopez
Browse files

Add improvements to the global search process

Removed the conditions added to
Project.with_feature_available_for_user, and moved to the
IssuableFinder. Now, we ensure that, in the projects retrieved
in the Finder, the user has enough access for the feature.
parent 3d3d8b9320cd
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
# updated_after: datetime # updated_after: datetime
# updated_before: datetime # updated_before: datetime
# attempt_group_search_optimizations: boolean # attempt_group_search_optimizations: boolean
# attempt_project_search_optimizations: boolean
# #
class IssuableFinder class IssuableFinder
prepend FinderWithCrossProjectAccess prepend FinderWithCrossProjectAccess
...@@ -184,7 +185,6 @@ def project ...@@ -184,7 +185,6 @@ def project
@project = project @project = project
end end
# rubocop: disable CodeReuse/ActiveRecord
def projects def projects
return @projects if defined?(@projects) return @projects if defined?(@projects)
...@@ -192,17 +192,25 @@ def projects ...@@ -192,17 +192,25 @@ def projects
projects = projects =
if current_user && params[:authorized_only].presence && !current_user_related? if current_user && params[:authorized_only].presence && !current_user_related?
current_user.authorized_projects current_user.authorized_projects(min_access_level)
elsif group elsif group
finder_options = { include_subgroups: params[:include_subgroups], only_owned: true } find_group_projects
GroupProjectsFinder.new(group: group, current_user: current_user, options: finder_options).execute # rubocop: disable CodeReuse/Finder
else else
ProjectsFinder.new(current_user: current_user).execute # rubocop: disable CodeReuse/Finder Project.public_or_visible_to_user(current_user, min_access_level)
end end
@projects = projects.with_feature_available_for_user(klass, current_user).reorder(nil) @projects = projects.with_feature_available_for_user(klass, current_user).reorder(nil) # rubocop: disable CodeReuse/ActiveRecord
end
def find_group_projects
return Project.none unless group
if params[:include_subgroups]
Project.where(namespace_id: group.self_and_descendants) # rubocop: disable CodeReuse/ActiveRecord
else
group.projects
end.public_or_visible_to_user(current_user, min_access_level)
end end
# rubocop: enable CodeReuse/ActiveRecord
def search def search
params[:search].presence params[:search].presence
...@@ -570,4 +578,8 @@ def current_user_related? ...@@ -570,4 +578,8 @@ def current_user_related?
scope = params[:scope] scope = params[:scope]
scope == 'created_by_me' || scope == 'authored' || scope == 'assigned_to_me' scope == 'created_by_me' || scope == 'authored' || scope == 'assigned_to_me'
end end
def min_access_level
ProjectFeature.required_minimum_access_level(klass)
end
end end
...@@ -48,9 +48,9 @@ def with_confidentiality_access_check ...@@ -48,9 +48,9 @@ def with_confidentiality_access_check
OR (issues.confidential = TRUE OR (issues.confidential = TRUE
AND (issues.author_id = :user_id AND (issues.author_id = :user_id
OR EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = :user_id AND issue_id = issues.id) OR EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = :user_id AND issue_id = issues.id)
OR issues.project_id IN(:project_ids)))', OR EXISTS (:authorizations)))',
user_id: current_user.id, user_id: current_user.id,
project_ids: current_user.authorized_projects(CONFIDENTIAL_ACCESS_LEVEL).select(:id)) authorizations: current_user.authorizations_for_projects(min_access_level: CONFIDENTIAL_ACCESS_LEVEL, related_project_column: "issues.project_id"))
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
......
...@@ -62,7 +62,7 @@ def filter_projects(collection) ...@@ -62,7 +62,7 @@ def filter_projects(collection)
collection = by_personal(collection) collection = by_personal(collection)
collection = by_starred(collection) collection = by_starred(collection)
collection = by_trending(collection) collection = by_trending(collection)
collection = by_visibilty_level(collection) collection = by_visibility_level(collection)
collection = by_tags(collection) collection = by_tags(collection)
collection = by_search(collection) collection = by_search(collection)
collection = by_archived(collection) collection = by_archived(collection)
...@@ -71,12 +71,11 @@ def filter_projects(collection) ...@@ -71,12 +71,11 @@ def filter_projects(collection)
collection collection
end end
# rubocop: disable CodeReuse/ActiveRecord
def collection_with_user def collection_with_user
if owned_projects? if owned_projects?
current_user.owned_projects current_user.owned_projects
elsif min_access_level? elsif min_access_level?
current_user.authorized_projects.where('project_authorizations.access_level >= ?', params[:min_access_level]) current_user.authorized_projects(params[:min_access_level])
else else
if private_only? if private_only?
current_user.authorized_projects current_user.authorized_projects
...@@ -85,7 +84,6 @@ def collection_with_user ...@@ -85,7 +84,6 @@ def collection_with_user
end end
end end
end end
# rubocop: enable CodeReuse/ActiveRecord
# Builds a collection for an anonymous user. # Builds a collection for an anonymous user.
def collection_without_user def collection_without_user
...@@ -131,7 +129,7 @@ def by_trending(items) ...@@ -131,7 +129,7 @@ def by_trending(items)
end end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def by_visibilty_level(items) def by_visibility_level(items)
params[:visibility_level].present? ? items.where(visibility_level: params[:visibility_level]) : items params[:visibility_level].present? ? items.where(visibility_level: params[:visibility_level]) : items
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
......
...@@ -464,10 +464,12 @@ def self.eager_load_namespace_and_owner ...@@ -464,10 +464,12 @@ def self.eager_load_namespace_and_owner
# Returns a collection of projects that is either public or visible to the # Returns a collection of projects that is either public or visible to the
# logged in user. # logged in user.
def self.public_or_visible_to_user(user = nil) def self.public_or_visible_to_user(user = nil, min_access_level = nil)
min_access_level = nil if user&.admin?
if user if user
where('EXISTS (?) OR projects.visibility_level IN (?)', where('EXISTS (?) OR projects.visibility_level IN (?)',
user.authorizations_for_projects, user.authorizations_for_projects(min_access_level: min_access_level),
Gitlab::VisibilityLevel.levels_for_user(user)) Gitlab::VisibilityLevel.levels_for_user(user))
else else
public_to_user public_to_user
...@@ -477,30 +479,32 @@ def self.public_or_visible_to_user(user = nil) ...@@ -477,30 +479,32 @@ def self.public_or_visible_to_user(user = nil)
# project features may be "disabled", "internal", "enabled" or "public". If "internal", # project features may be "disabled", "internal", "enabled" or "public". If "internal",
# they are only available to team members. This scope returns projects where # they are only available to team members. This scope returns projects where
# the feature is either public, enabled, or internal with permission for the user. # the feature is either public, enabled, or internal with permission for the user.
# Note: this scope doesn't enforce that the user has access to the projects, it just checks
# that the user has access to the feature. It's important to use this scope with others
# that checks project authorizations first.
# #
# This method uses an optimised version of `with_feature_access_level` for # This method uses an optimised version of `with_feature_access_level` for
# logged in users to more efficiently get private projects with the given # logged in users to more efficiently get private projects with the given
# feature. # feature.
def self.with_feature_available_for_user(feature, user) def self.with_feature_available_for_user(feature, user)
visible = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC] visible = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC]
min_access_level = ProjectFeature.required_minimum_access_level(feature)
if user&.admin? if user&.admin?
with_feature_enabled(feature) with_feature_enabled(feature)
elsif user elsif user
min_access_level = ProjectFeature.required_minimum_access_level(feature)
column = ProjectFeature.quoted_access_level_column(feature) column = ProjectFeature.quoted_access_level_column(feature)
with_project_feature with_project_feature
.where( .where("#{column} IS NULL OR #{column} IN (:public_visible) OR (#{column} = :private_visible AND EXISTS (:authorizations))",
"(projects.visibility_level > :private AND (#{column} IS NULL OR #{column} >= (:public_visible) OR (#{column} = :private_visible AND EXISTS(:authorizations))))"\ {
" OR (projects.visibility_level = :private AND (#{column} IS NULL OR #{column} >= :private_visible) AND EXISTS(:authorizations))", public_visible: visible,
{ private_visible: ProjectFeature::PRIVATE,
private: Gitlab::VisibilityLevel::PRIVATE, authorizations: user.authorizations_for_projects(min_access_level: min_access_level)
public_visible: ProjectFeature::ENABLED, })
private_visible: ProjectFeature::PRIVATE,
authorizations: user.authorizations_for_projects(min_access_level: min_access_level)
})
else else
# This has to be added to include features whose value is nil in the db
visible << nil
with_feature_access_level(feature, visible) with_feature_access_level(feature, visible)
end end
end end
......
...@@ -757,11 +757,15 @@ def authorized_project?(project, min_access_level = nil) ...@@ -757,11 +757,15 @@ def authorized_project?(project, min_access_level = nil)
# Typically used in conjunction with projects table to get projects # Typically used in conjunction with projects table to get projects
# a user has been given access to. # a user has been given access to.
# The param `related_project_column` is the column to compare to the
# project_authorizations. By default is projects.id
# #
# Example use: # Example use:
# `Project.where('EXISTS(?)', user.authorizations_for_projects)` # `Project.where('EXISTS(?)', user.authorizations_for_projects)`
def authorizations_for_projects(min_access_level: nil) def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id')
authorizations = project_authorizations.select(1).where('project_authorizations.project_id = projects.id') authorizations = project_authorizations
.select(1)
.where("project_authorizations.project_id = #{related_project_column}")
return authorizations unless min_access_level.present? return authorizations unless min_access_level.present?
......
---
title: Add improvements to global search of issues and merge requests
merge_request: 27817
author:
type: performance
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
module Gitlab module Gitlab
class GroupSearchResults < SearchResults class GroupSearchResults < SearchResults
attr_reader :group
def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20) def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20)
super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page) super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page)
...@@ -26,5 +28,9 @@ def users ...@@ -26,5 +28,9 @@ def users
.where(id: groups.select('members.user_id')) .where(id: groups.select('members.user_id'))
end end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
def issuable_params
super.merge(group_id: group.id)
end
end end
end end
...@@ -145,5 +145,9 @@ def repository_project_ref ...@@ -145,5 +145,9 @@ def repository_project_ref
def repository_wiki_ref def repository_wiki_ref
@repository_wiki_ref ||= repository_ref || project.wiki.default_branch @repository_wiki_ref ||= repository_ref || project.wiki.default_branch
end end
def issuable_params
super.merge(project_id: project.id)
end
end end
end end
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
module Gitlab module Gitlab
class SearchResults class SearchResults
COUNT_LIMIT = 1001
attr_reader :current_user, :query, :per_page attr_reader :current_user, :query, :per_page
# Limit search results by passed projects # Limit search results by passed projects
...@@ -25,29 +27,26 @@ def initialize(current_user, limit_projects, query, default_project_filter: fals ...@@ -25,29 +27,26 @@ def initialize(current_user, limit_projects, query, default_project_filter: fals
def objects(scope, page = nil, without_count = true) def objects(scope, page = nil, without_count = true)
collection = case scope collection = case scope
when 'projects' when 'projects'
projects.page(page).per(per_page) projects
when 'issues' when 'issues'
issues.page(page).per(per_page) issues
when 'merge_requests' when 'merge_requests'
merge_requests.page(page).per(per_page) merge_requests
when 'milestones' when 'milestones'
milestones.page(page).per(per_page) milestones
when 'users' when 'users'
users.page(page).per(per_page) users
else else
Kaminari.paginate_array([]).page(page).per(per_page) Kaminari.paginate_array([])
end end.page(page).per(per_page)
without_count ? collection.without_count : collection without_count ? collection.without_count : collection
end end
# rubocop: disable CodeReuse/ActiveRecord
def limited_projects_count def limited_projects_count
@limited_projects_count ||= projects.limit(count_limit).count @limited_projects_count ||= limited_count(projects)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def limited_issues_count def limited_issues_count
return @limited_issues_count if @limited_issues_count return @limited_issues_count if @limited_issues_count
...@@ -56,35 +55,28 @@ def limited_issues_count ...@@ -56,35 +55,28 @@ def limited_issues_count
# and confidential issues user has access to, is too complex. # and confidential issues user has access to, is too complex.
# It's faster to try to fetch all public issues first, then only # It's faster to try to fetch all public issues first, then only
# if necessary try to fetch all issues. # if necessary try to fetch all issues.
sum = issues(public_only: true).limit(count_limit).count sum = limited_count(issues(public_only: true))
@limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum @limited_issues_count = sum < count_limit ? limited_count(issues) : sum
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def limited_merge_requests_count def limited_merge_requests_count
@limited_merge_requests_count ||= merge_requests.limit(count_limit).count @limited_merge_requests_count ||= limited_count(merge_requests)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def limited_milestones_count def limited_milestones_count
@limited_milestones_count ||= milestones.limit(count_limit).count @limited_milestones_count ||= limited_count(milestones)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def limited_users_count def limited_users_count
@limited_users_count ||= users.limit(count_limit).count @limited_users_count ||= limited_count(users)
end end
# rubocop:enable CodeReuse/ActiveRecord
def single_commit_result? def single_commit_result?
false false
end end
def count_limit def count_limit
1001 COUNT_LIMIT
end end
def users def users
...@@ -99,23 +91,15 @@ def projects ...@@ -99,23 +91,15 @@ def projects
limit_projects.search(query) limit_projects.search(query)
end end
# rubocop: disable CodeReuse/ActiveRecord
def issues(finder_params = {}) def issues(finder_params = {})
issues = IssuesFinder.new(current_user, finder_params).execute issues = IssuesFinder.new(current_user, issuable_params.merge(finder_params)).execute
unless default_project_filter unless default_project_filter
issues = issues.where(project_id: project_ids_relation) issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord
end end
issues = issues
if query =~ /#(\d+)\z/
issues.where(iid: $1)
else
issues.full_search(query)
end
issues.reorder('issues.updated_at DESC')
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def milestones def milestones
...@@ -125,23 +109,15 @@ def milestones ...@@ -125,23 +109,15 @@ def milestones
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def merge_requests def merge_requests
merge_requests = MergeRequestsFinder.new(current_user).execute merge_requests = MergeRequestsFinder.new(current_user, issuable_params).execute
unless default_project_filter unless default_project_filter
merge_requests = merge_requests.in_projects(project_ids_relation) merge_requests = merge_requests.in_projects(project_ids_relation)
end end
merge_requests = merge_requests
if query =~ /[#!](\d+)\z/
merge_requests.where(iid: $1)
else
merge_requests.full_search(query)
end
merge_requests.reorder('merge_requests.updated_at DESC')
end end
# rubocop: enable CodeReuse/ActiveRecord
def default_scope def default_scope
'projects' 'projects'
...@@ -152,5 +128,23 @@ def project_ids_relation ...@@ -152,5 +128,23 @@ def project_ids_relation
limit_projects.select(:id).reorder(nil) limit_projects.select(:id).reorder(nil)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
def issuable_params
{}.tap do |params|
params[:sort] = 'updated_desc'
if query =~ /#(\d+)\z/
params[:iids] = $1
else
params[:search] = query
end
end
end
# rubocop: disable CodeReuse/ActiveRecord
def limited_count(relation)
relation.reorder(nil).limit(count_limit).size
end
# rubocop: enable CodeReuse/ActiveRecord
end end
end end
...@@ -260,6 +260,7 @@ ...@@ -260,6 +260,7 @@
trait(:merge_requests_enabled) { merge_requests_access_level ProjectFeature::ENABLED } trait(:merge_requests_enabled) { merge_requests_access_level ProjectFeature::ENABLED }
trait(:merge_requests_disabled) { merge_requests_access_level ProjectFeature::DISABLED } trait(:merge_requests_disabled) { merge_requests_access_level ProjectFeature::DISABLED }
trait(:merge_requests_private) { merge_requests_access_level ProjectFeature::PRIVATE } trait(:merge_requests_private) { merge_requests_access_level ProjectFeature::PRIVATE }
trait(:merge_requests_public) { merge_requests_access_level ProjectFeature::PUBLIC }
trait(:repository_enabled) { repository_access_level ProjectFeature::ENABLED } trait(:repository_enabled) { repository_access_level ProjectFeature::ENABLED }
trait(:repository_disabled) { repository_access_level ProjectFeature::DISABLED } trait(:repository_disabled) { repository_access_level ProjectFeature::DISABLED }
trait(:repository_private) { repository_access_level ProjectFeature::PRIVATE } trait(:repository_private) { repository_access_level ProjectFeature::PRIVATE }
......
...@@ -641,9 +641,7 @@ ...@@ -641,9 +641,7 @@
end end
it 'filters by confidentiality' do it 'filters by confidentiality' do
expect(Issue).to receive(:where).with(a_string_matching('confidential'), anything) expect(subject.to_sql).to match("issues.confidential")
subject
end end
end end
...@@ -660,9 +658,7 @@ ...@@ -660,9 +658,7 @@
end end
it 'filters by confidentiality' do it 'filters by confidentiality' do
expect(Issue).to receive(:where).with(a_string_matching('confidential'), anything) expect(subject.to_sql).to match("issues.confidential")
subject
end end
end end
......
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
end end
context 'filtering by group' do context 'filtering by group' do
it 'includes all merge requests when user has access exceluding merge requests from projects the user does not have access to' do it 'includes all merge requests when user has access excluding merge requests from projects the user does not have access to' do
private_project = allow_gitaly_n_plus_1 { create(:project, :private, group: group) } private_project = allow_gitaly_n_plus_1 { create(:project, :private, group: group) }
private_project.add_guest(user) private_project.add_guest(user)
create(:merge_request, :simple, author: user, source_project: private_project, target_project: private_project) create(:merge_request, :simple, author: user, source_project: private_project, target_project: private_project)
......
...@@ -3164,61 +3164,105 @@ def enable_lfs ...@@ -3164,61 +3164,105 @@ def enable_lfs
end end
describe '.with_feature_available_for_user' do describe '.with_feature_available_for_user' do
let!(:user) { create(:user) } let(:user) { create(:user) }
let!(:feature) { MergeRequest } let(:feature) { MergeRequest }
let!(:project) { create(:project, :public, :merge_requests_enabled) }
subject { described_class.with_feature_available_for_user(feature, user) } subject { described_class.with_feature_available_for_user(feature, user) }
context 'when user has access to project' do shared_examples 'feature disabled' do
subject { described_class.with_feature_available_for_user(feature, user) } let(:project) { create(:project, :public, :merge_requests_disabled) }
it 'does not return projects with the project feature disabled' do