Commit 70bf6dc7 authored by Robert Speicher's avatar Robert Speicher
Browse files

Merge branch 'trigram-index-searching' into 'master'

Refactor searching and use PostgreSQL trigram indexes for significantly
improved performance

Related issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/13743.

Also fixes #12410

See merge request !2987
......@@ -17,6 +17,7 @@ v 8.6.0 (unreleased)
- Return empty array instead of 404 when commit has no statuses in commit status API
- Decrease the font size and the padding of the `.anchor` icons used in the README (Roberto Dip)
- Rewrite logo to simplify SVG code (Sean Lang)
- Refactor and greatly improve search performance
- Add support for cross-project label references
- Update documentation to reflect Guest role not being enforced on internal projects
- Allow search for logged out users
......
......@@ -52,7 +52,10 @@ def group_projects(current_user, group)
def all_projects(current_user)
if current_user
[current_user.authorized_projects, public_and_internal_projects]
[
*current_user.project_relations,
public_and_internal_projects
]
else
[Project.public_only]
end
......
......@@ -23,7 +23,7 @@ class Runner < ActiveRecord::Base
LAST_CONTACT_TIME = 5.minutes.ago
AVAILABLE_SCOPES = ['specific', 'shared', 'active', 'paused', 'online']
has_many :builds, class_name: 'Ci::Build'
has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject'
has_many :projects, through: :runner_projects, class_name: '::Project', foreign_key: :gl_project_id
......@@ -46,9 +46,23 @@ class Runner < ActiveRecord::Base
acts_as_taggable
# Searches for runners matching the given query.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# This method performs a *partial* match on tokens, thus a query for "a"
# will match any runner where the token contains the letter "a". As a result
# you should *not* use this method for non-admin purposes as otherwise users
# might be able to query a list of all runners.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def self.search(query)
where('LOWER(ci_runners.token) LIKE :query OR LOWER(ci_runners.description) like :query',
query: "%#{query.try(:downcase)}%")
t = arel_table
pattern = "%#{query}%"
where(t[:token].matches(pattern).or(t[:description].matches(pattern)))
end
def set_default_values
......
......@@ -61,12 +61,29 @@ module Issuable
end
module ClassMethods
# Searches for records with a matching title.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def search(query)
where("LOWER(title) like :query", query: "%#{query.downcase}%")
where(arel_table[:title].matches("%#{query}%"))
end
# Searches for records with a matching title or description.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def full_search(query)
where("LOWER(title) like :query OR LOWER(description) like :query", query: "%#{query.downcase}%")
t = arel_table
pattern = "%#{query}%"
where(t[:title].matches(pattern).or(t[:description].matches(pattern)))
end
def sort(method)
......
......@@ -33,8 +33,18 @@ class Group < Namespace
after_destroy :post_destroy_hook
class << self
# Searches for groups matching the given query.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def search(query)
where("LOWER(namespaces.name) LIKE :query or LOWER(namespaces.path) LIKE :query", query: "%#{query.downcase}%")
table = Namespace.arel_table
pattern = "%#{query}%"
where(table[:name].matches(pattern).or(table[:path].matches(pattern)))
end
def sort(method)
......
......@@ -135,7 +135,6 @@ class MergeRequest < ActiveRecord::Base
scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }
scope :cared, ->(user) { where('assignee_id = :user OR author_id = :user', user: user.id) }
scope :by_milestone, ->(milestone) { where(milestone_id: milestone) }
scope :in_projects, ->(project_ids) { where("source_project_id in (:project_ids) OR target_project_id in (:project_ids)", project_ids: project_ids) }
scope :of_projects, ->(ids) { where(target_project_id: ids) }
scope :merged, -> { with_state(:merged) }
scope :closed_and_merged, -> { with_states(:closed, :merged) }
......@@ -161,6 +160,24 @@ def self.link_reference_pattern
super("merge_requests", /(?<merge_request>\d+)/)
end
# Returns all the merge requests from an ActiveRecord:Relation.
#
# This method uses a UNION as it usually operates on the result of
# ProjectsFinder#execute. PostgreSQL in particular doesn't always like queries
# using multiple sub-queries especially when combined with an OR statement.
# UNIONs on the other hand perform much better in these cases.
#
# relation - An ActiveRecord::Relation that returns a list of Projects.
#
# Returns an ActiveRecord::Relation.
def self.in_projects(relation)
source = where(source_project_id: relation).select(:id)
target = where(target_project_id: relation).select(:id)
union = Gitlab::SQL::Union.new([source, target])
where("merge_requests.id IN (#{union.to_sql})")
end
def to_reference(from_project = nil)
reference = "#{self.class.reference_prefix}#{iid}"
......
......@@ -58,9 +58,18 @@ class Milestone < ActiveRecord::Base
alias_attribute :name, :title
class << self
# Searches for milestones matching the given query.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def search(query)
query = "%#{query}%"
where("title like ? or description like ?", query, query)
t = arel_table
pattern = "%#{query}%"
where(t[:title].matches(pattern).or(t[:description].matches(pattern)))
end
end
......
......@@ -52,8 +52,18 @@ def find_by_path_or_name(path)
find_by("lower(path) = :path OR lower(name) = :path", path: path.downcase)
end
# Searches for namespaces matching the given query.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation
def search(query)
where("name LIKE :query OR path LIKE :query", query: "%#{query}%")
t = arel_table
pattern = "%#{query}%"
where(t[:name].matches(pattern).or(t[:path].matches(pattern)))
end
def clean_path(path)
......
......@@ -105,8 +105,18 @@ def build_discussion_id(type, id, line_code)
[:discussion, type.try(:underscore), id, line_code].join("-").to_sym
end
# Searches for notes matching the given query.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String.
#
# Returns an ActiveRecord::Relation.
def search(query)
where("LOWER(note) like :query", query: "%#{query.downcase}%")
table = arel_table
pattern = "%#{query}%"
where(table[:note].matches(pattern))
end
def grouped_awards
......
......@@ -266,13 +266,31 @@ def active
joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC')
end
# Searches for a list of projects based on the query given in `query`.
#
# On PostgreSQL this method uses "ILIKE" to perform a case-insensitive
# search. On MySQL a regular "LIKE" is used as it's already
# case-insensitive.
#
# query - The search query as a String.
def search(query)
joins(:namespace).
where('LOWER(projects.name) LIKE :query OR
LOWER(projects.path) LIKE :query OR
LOWER(namespaces.name) LIKE :query OR
LOWER(projects.description) LIKE :query',
query: "%#{query.try(:downcase)}%")
ptable = arel_table
ntable = Namespace.arel_table
pattern = "%#{query}%"
projects = select(:id).where(
ptable[:path].matches(pattern).
or(ptable[:name].matches(pattern)).
or(ptable[:description].matches(pattern))
)
namespaces = select(:id).
joins(:namespace).
where(ntable[:name].matches(pattern))
union = Gitlab::SQL::Union.new([projects, namespaces])
where("projects.id IN (#{union.to_sql})")
end
def search_by_visibility(level)
......@@ -280,7 +298,10 @@ def search_by_visibility(level)
end
def search_by_title(query)
non_archived.where('LOWER(projects.name) LIKE :query', query: "%#{query.downcase}%")
pattern = "%#{query}%"
table = Project.arel_table
non_archived.where(table[:name].matches(pattern))
end
def find_with_namespace(id)
......
......@@ -113,12 +113,32 @@ def visibility_level_field
end
class << self
# Searches for snippets with a matching title or file name.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String.
#
# Returns an ActiveRecord::Relation.
def search(query)
where('(title LIKE :query OR file_name LIKE :query)', query: "%#{query}%")
t = arel_table
pattern = "%#{query}%"
where(t[:title].matches(pattern).or(t[:file_name].matches(pattern)))
end
# Searches for snippets with matching content.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String.
#
# Returns an ActiveRecord::Relation.
def search_code(query)
where('(content LIKE :query)', query: "%#{query}%")
table = Snippet.arel_table
pattern = "%#{query}%"
where(table[:content].matches(pattern))
end
def accessible_to(user)
......
......@@ -286,8 +286,22 @@ def filter(filter_name)
end
end
# Searches users matching the given query.
#
# This method uses ILIKE on PostgreSQL and LIKE on MySQL.
#
# query - The search query as a String
#
# Returns an ActiveRecord::Relation.
def search(query)
where("lower(name) LIKE :query OR lower(email) LIKE :query OR lower(username) LIKE :query", query: "%#{query.downcase}%")
table = arel_table
pattern = "%#{query}%"
where(
table[:name].matches(pattern).
or(table[:email].matches(pattern)).
or(table[:username].matches(pattern))
)
end
def by_login(login)
......@@ -428,6 +442,11 @@ def authorized_projects
Project.where("projects.id IN (#{projects_union.to_sql})")
end
# Returns all the project relations
def project_relations
[personal_projects, groups_projects, projects]
end
def owned_projects
@owned_projects ||=
Project.where('namespace_id IN (?) OR namespace_id = ?',
......@@ -816,9 +835,7 @@ def ci_authorized_runners
private
def projects_union
Gitlab::SQL::Union.new([personal_projects.select(:id),
groups_projects.select(:id),
projects.select(:id)])
Gitlab::SQL::Union.new(project_relations.map { |r| r.select(:id) })
end
def ci_projects_union
......
......@@ -10,9 +10,8 @@ def execute
group = Group.find_by(id: params[:group_id]) if params[:group_id].present?
projects = ProjectsFinder.new.execute(current_user)
projects = projects.in_namespace(group.id) if group
project_ids = projects.pluck(:id)
Gitlab::SearchResults.new(project_ids, params[:search])
Gitlab::SearchResults.new(projects, params[:search])
end
end
end
......@@ -7,7 +7,7 @@ def initialize(project, user, params)
end
def execute
Gitlab::ProjectSearchResults.new(project.id,
Gitlab::ProjectSearchResults.new(project,
params[:search],
params[:repository_ref])
end
......
......@@ -7,8 +7,9 @@ def initialize(user, params)
end
def execute
snippet_ids = Snippet.accessible_to(current_user).pluck(:id)
Gitlab::SnippetSearchResults.new(snippet_ids, params[:search])
snippets = Snippet.accessible_to(current_user)
Gitlab::SnippetSearchResults.new(snippets, params[:search])
end
end
end
# This patches ActiveRecord so indexes created using the MySQL adapter ignore
# any PostgreSQL specific options (e.g. `using: :gin`).
#
# These patches do the following for MySQL:
#
# 1. Indexes created using the :opclasses option are ignored (as they serve no
# purpose on MySQL).
# 2. When creating an index with `using: :gin` the `using` option is discarded
# as :gin is not a valid value for MySQL.
# 3. The `:opclasses` option is stripped from add_index_options in case it's
# used anywhere other than in the add_index methods.
if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
module ActiveRecord
module ConnectionAdapters
class Mysql2Adapter < AbstractMysqlAdapter
alias_method :__gitlab_add_index, :add_index
alias_method :__gitlab_add_index_sql, :add_index_sql
alias_method :__gitlab_add_index_options, :add_index_options
def add_index(table_name, column_name, options = {})
unless options[:opclasses]
__gitlab_add_index(table_name, column_name, options)
end
end
def add_index_sql(table_name, column_name, options = {})
unless options[:opclasses]
__gitlab_add_index_sql(table_name, column_name, options)
end
end
def add_index_options(table_name, column_name, options = {})
if options[:using] and options[:using] == :gin
options = options.dup
options.delete(:using)
end
if options[:opclasses]
options = options.dup
options.delete(:opclasses)
end
__gitlab_add_index_options(table_name, column_name, options)
end
end
end
end
end
# rubocop:disable all
# These changes add support for PostgreSQL operator classes when creating
# indexes and dumping/loading schemas. Taken from Rails pull request
# https://github.com/rails/rails/pull/19090.
#
# License:
#
# Copyright (c) 2004-2016 David Heinemeier Hansson
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
require 'date'
require 'set'
require 'bigdecimal'
require 'bigdecimal/util'
# As the Struct definition is changed in this PR/patch we have to first remove
# the existing one.
ActiveRecord::ConnectionAdapters.send(:remove_const, :IndexDefinition)
module ActiveRecord
module ConnectionAdapters #:nodoc:
# Abstract representation of an index definition on a table. Instances of
# this type are typically created and returned by methods in database
# adapters. e.g. ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter#indexes
class IndexDefinition < Struct.new(:table, :name, :unique, :columns, :lengths, :orders, :where, :type, :using, :opclasses) #:nodoc:
end
end
end
module ActiveRecord
module ConnectionAdapters # :nodoc:
module SchemaStatements
def add_index_options(table_name, column_name, options = {}) #:nodoc:
column_names = Array(column_name)
index_name = index_name(table_name, column: column_names)
options.assert_valid_keys(:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type, :opclasses)
index_type = options[:unique] ? "UNIQUE" : ""
index_type = options[:type].to_s if options.key?(:type)
index_name = options[:name].to_s if options.key?(:name)
max_index_length = options.fetch(:internal, false) ? index_name_length : allowed_index_name_length
if options.key?(:algorithm)
algorithm = index_algorithms.fetch(options[:algorithm]) {
raise ArgumentError.new("Algorithm must be one of the following: #{index_algorithms.keys.map(&:inspect).join(', ')}")
}
end
using = "USING #{options[:using]}" if options[:using].present?
if supports_partial_index?
index_options = options[:where] ? " WHERE #{options[:where]}" : ""
end
if index_name.length > max_index_length
raise ArgumentError, "Index name '#{index_name}' on table '#{table_name}' is too long; the limit is #{max_index_length} characters"
end
if table_exists?(table_name) && index_name_exists?(table_name, index_name, false)
raise ArgumentError, "Index name '#{index_name}' on table '#{table_name}' already exists"
end
index_columns = quoted_columns_for_index(column_names, options).join(", ")
[index_name, index_type, index_columns, index_options, algorithm, using]
end
end
end
end
module ActiveRecord
module ConnectionAdapters
module PostgreSQL
module SchemaStatements
# Returns an array of indexes for the given table.
def indexes(table_name, name = nil)
result = query(<<-SQL, 'SCHEMA')
SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), t.oid
FROM pg_class t
INNER JOIN pg_index d ON t.oid = d.indrelid
INNER JOIN pg_class i ON d.indexrelid = i.oid
WHERE i.relkind = 'i'
AND d.indisprimary = 'f'
AND t.relname = '#{table_name}'
AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = ANY (current_schemas(false)) )
ORDER BY i.relname
SQL
result.map do |row|
index_name = row[0]
unique = row[1] == 't'
indkey = row[2].split(" ")
inddef = row[3]
oid = row[4]
columns = Hash[query(<<-SQL, "SCHEMA")]
SELECT a.attnum, a.attname
FROM pg_attribute a
WHERE a.attrelid = #{oid}
AND a.attnum IN (#{indkey.join(",")})
SQL
column_names = columns.values_at(*indkey).compact
unless column_names.empty?
# add info on sort order for columns (only desc order is explicitly specified, asc is the default)
desc_order_columns = inddef.scan(/(\w+) DESC/).flatten
orders = desc_order_columns.any? ? Hash[desc_order_columns.map {|order_column| [order_column, :desc]}] : {}
where = inddef.scan(/WHERE (.+)$/).flatten[0]
using = inddef.scan(/USING (.+?) /).flatten[0].to_sym
opclasses = Hash[inddef.scan(/\((.+)\)$/).flatten[0].split(',').map do |column_and_opclass|
column, opclass = column_and_opclass.split(' ').map(&:strip)
[column, opclass] if opclass
end.compact]
IndexDefinition.new(table_name, index_name, unique, column_names, [], orders, where, nil, using, opclasses)
end
end.compact
end
def add_index(table_name, column_name, options = {}) #:nodoc:
index_name, index_type, index_columns_and_opclasses, index_options, index_algorithm, index_using = add_index_options(table_name, column_name, options)
execute "CREATE #{index_type} INDEX #{index_algorithm} #{quote_column_name(index_name)} ON #{quote_table_name(table_name)} #{index_using} (#{index_columns_and_opclasses})#{index_options}"
end
protected
def quoted_columns_for_index(column_names, options = {})
column_opclasses = options[:opclasses] || {}
column_names.map {|name| "#{quote_column_name(name)} #{column_opclasses[name]}"}
end
end
end
end
end
module ActiveRecord
class SchemaDumper
private
def indexes(table, stream)
if (indexes = @connection.indexes(table)).any?
add_index_statements = indexes.map do |index|
statement_parts = [
"add_index #{remove_prefix_and_suffix(index.table).inspect}",
index.columns.inspect,
"name: #{index.name.inspect}",
]
statement_parts << 'unique: true' if index.unique
index_lengths = (index.lengths || []).compact
statement_parts << "length: #{Hash[index.columns.zip(index.lengths)].inspect}" if index_lengths.any?
index_orders = index.orders || {}
statement_parts << "order: #{index.orders.inspect}" if index_orders.any?
statement_parts << "where: #{index.where.inspect}" if index.where
statement_parts << "using: #{index.using.inspect}" if index.using