Read about our upcoming Code of Conduct on this issue

Commit e94cd6fd authored by Nick Thomas's avatar Nick Thomas
Browse files

Add markdown cache columns to the database, but don't use them yet

This commit adds a number of _html columns and, with the exception of Note,
starts updating them whenever the content of their partner fields changes.

Note has a collision with the note_html attr_accessor; that will be fixed later

A background worker for clearing these cache columns is also introduced - use
`rake cache:clear` to set it off. You can clear the database or Redis caches
separately by running `rake cache:clear:db` or `rake cache:clear:redis`,
respectively.
parent f2af12d39ac7
...@@ -51,17 +51,15 @@ def markdown(text, context = {}) ...@@ -51,17 +51,15 @@ def markdown(text, context = {})
context[:project] ||= @project context[:project] ||= @project
html = Banzai.render(text, context) html = Banzai.render(text, context)
banzai_postprocess(html, context)
end
context.merge!( def markdown_field(object, field)
current_user: (current_user if defined?(current_user)), object = object.for_display if object.respond_to?(:for_display)
return "" unless object.present?
# RelativeLinkFilter html = Banzai.render_field(object, field)
requested_path: @path, banzai_postprocess(html, object.banzai_render_context(field))
project_wiki: @project_wiki,
ref: @ref
)
Banzai.post_process(html, context)
end end
def asciidoc(text) def asciidoc(text)
...@@ -196,4 +194,18 @@ def markdown_toolbar_button(options = {}) ...@@ -196,4 +194,18 @@ def markdown_toolbar_button(options = {})
icon(options[:icon]) icon(options[:icon])
end end
end end
# Calls Banzai.post_process with some common context options
def banzai_postprocess(html, context)
context.merge!(
current_user: (current_user if defined?(current_user)),
# RelativeLinkFilter
requested_path: @path,
project_wiki: @project_wiki,
ref: @ref
)
Banzai.post_process(html, context)
end
end end
class AbuseReport < ActiveRecord::Base class AbuseReport < ActiveRecord::Base
include CacheMarkdownField
cache_markdown_field :message, pipeline: :single_line
belongs_to :reporter, class_name: 'User' belongs_to :reporter, class_name: 'User'
belongs_to :user belongs_to :user
...@@ -7,6 +11,9 @@ class AbuseReport < ActiveRecord::Base ...@@ -7,6 +11,9 @@ class AbuseReport < ActiveRecord::Base
validates :message, presence: true validates :message, presence: true
validates :user_id, uniqueness: { message: 'has already been reported' } validates :user_id, uniqueness: { message: 'has already been reported' }
# For CacheMarkdownField
alias_method :author, :reporter
def remove_user(deleted_by:) def remove_user(deleted_by:)
user.block user.block
DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true) DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true)
......
class Appearance < ActiveRecord::Base class Appearance < ActiveRecord::Base
include CacheMarkdownField
cache_markdown_field :description
validates :title, presence: true validates :title, presence: true
validates :description, presence: true validates :description, presence: true
validates :logo, file_size: { maximum: 1.megabyte } validates :logo, file_size: { maximum: 1.megabyte }
......
class ApplicationSetting < ActiveRecord::Base class ApplicationSetting < ActiveRecord::Base
include CacheMarkdownField
include TokenAuthenticatable include TokenAuthenticatable
add_authentication_token_field :runners_registration_token add_authentication_token_field :runners_registration_token
add_authentication_token_field :health_check_access_token add_authentication_token_field :health_check_access_token
...@@ -17,6 +19,11 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -17,6 +19,11 @@ class ApplicationSetting < ActiveRecord::Base
serialize :domain_whitelist, Array serialize :domain_whitelist, Array
serialize :domain_blacklist, Array serialize :domain_blacklist, Array
cache_markdown_field :sign_in_text
cache_markdown_field :help_page_text
cache_markdown_field :shared_runners_text, pipeline: :plain_markdown
cache_markdown_field :after_sign_up_text
attr_accessor :domain_whitelist_raw, :domain_blacklist_raw attr_accessor :domain_whitelist_raw, :domain_blacklist_raw
validates :session_expire_delay, validates :session_expire_delay,
......
class BroadcastMessage < ActiveRecord::Base class BroadcastMessage < ActiveRecord::Base
include CacheMarkdownField
include Sortable include Sortable
cache_markdown_field :message, pipeline: :broadcast_message
validates :message, presence: true validates :message, presence: true
validates :starts_at, presence: true validates :starts_at, presence: true
validates :ends_at, presence: true validates :ends_at, presence: true
......
# This module takes care of updating cache columns for Markdown-containing
# fields. Use like this in the body of your class:
#
# include CacheMarkdownField
# cache_markdown_field :foo
# cache_markdown_field :bar
# cache_markdown_field :baz, pipeline: :single_line
#
# Corresponding foo_html, bar_html and baz_html fields should exist.
module CacheMarkdownField
# Knows about the relationship between markdown and html field names, and
# stores the rendering contexts for the latter
class FieldData
extend Forwardable
def initialize
@data = {}
end
def_delegators :@data, :[], :[]=
def_delegator :@data, :keys, :markdown_fields
def html_field(markdown_field)
"#{markdown_field}_html"
end
def html_fields
markdown_fields.map {|field| html_field(field) }
end
end
# Dynamic registries don't really work in Rails as it's not guaranteed that
# every class will be loaded, so hardcode the list.
CACHING_CLASSES = %w[
AbuseReport
Appearance
ApplicationSetting
BroadcastMessage
Issue
Label
MergeRequest
Milestone
Namespace
Note
Project
Release
Snippet
]
def self.caching_classes
CACHING_CLASSES.map(&:constantize)
end
extend ActiveSupport::Concern
included do
cattr_reader :cached_markdown_fields do
FieldData.new
end
# Returns the default Banzai render context for the cached markdown field.
def banzai_render_context(field)
raise ArgumentError.new("Unknown field: #{field.inspect}") unless
cached_markdown_fields.markdown_fields.include?(field)
# Always include a project key, or Banzai complains
project = self.project if self.respond_to?(:project)
context = cached_markdown_fields[field].merge(project: project)
# Banzai is less strict about authors, so don't always have an author key
context[:author] = self.author if self.respond_to?(:author)
context
end
# Allow callers to look up the cache field name, rather than hardcoding it
def markdown_cache_field_for(field)
raise ArgumentError.new("Unknown field: #{field}") unless
cached_markdown_fields.markdown_fields.include?(field)
cached_markdown_fields.html_field(field)
end
# Always exclude _html fields from attributes (including serialization).
# They contain unredacted HTML, which would be a security issue
alias_method :attributes_before_markdown_cache, :attributes
def attributes
attrs = attributes_before_markdown_cache
cached_markdown_fields.html_fields.each do |field|
attrs.delete(field)
end
attrs
end
end
class_methods do
private
# Specify that a field is markdown. Its rendered output will be cached in
# a corresponding _html field. Any custom rendering options may be provided
# as a context.
def cache_markdown_field(markdown_field, context = {})
raise "Add #{self} to CacheMarkdownField::CACHING_CLASSES" unless
CacheMarkdownField::CACHING_CLASSES.include?(self.to_s)
cached_markdown_fields[markdown_field] = context
html_field = cached_markdown_fields.html_field(markdown_field)
cache_method = "#{markdown_field}_cache_refresh".to_sym
invalidation_method = "#{html_field}_invalidated?".to_sym
define_method(cache_method) do
html = Banzai::Renderer.cacheless_render_field(self, markdown_field)
__send__("#{html_field}=", html)
true
end
# The HTML becomes invalid if any dependent fields change. For now, assume
# author and project invalidate the cache in all circumstances.
define_method(invalidation_method) do
changed_fields = changed_attributes.keys
invalidations = changed_fields & [markdown_field.to_s, "author", "project"]
!invalidations.empty?
end
before_save cache_method, if: invalidation_method
end
end
end
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
# #
module Issuable module Issuable
extend ActiveSupport::Concern extend ActiveSupport::Concern
include CacheMarkdownField
include Participable include Participable
include Mentionable include Mentionable
include Subscribable include Subscribable
...@@ -13,6 +14,9 @@ module Issuable ...@@ -13,6 +14,9 @@ module Issuable
include Awardable include Awardable
included do included do
cache_markdown_field :title, pipeline: :single_line
cache_markdown_field :description
belongs_to :author, class_name: "User" belongs_to :author, class_name: "User"
belongs_to :assignee, class_name: "User" belongs_to :assignee, class_name: "User"
belongs_to :updated_by, class_name: "User" belongs_to :updated_by, class_name: "User"
......
...@@ -4,6 +4,10 @@ class GlobalLabel ...@@ -4,6 +4,10 @@ class GlobalLabel
delegate :color, :description, to: :@first_label delegate :color, :description, to: :@first_label
def for_display
@first_label
end
def self.build_collection(labels) def self.build_collection(labels)
labels = labels.group_by(&:title) labels = labels.group_by(&:title)
......
...@@ -4,6 +4,10 @@ class GlobalMilestone ...@@ -4,6 +4,10 @@ class GlobalMilestone
attr_accessor :title, :milestones attr_accessor :title, :milestones
alias_attribute :name, :title alias_attribute :name, :title
def for_display
@first_milestone
end
def self.build_collection(milestones) def self.build_collection(milestones)
milestones = milestones.group_by(&:title) milestones = milestones.group_by(&:title)
...@@ -17,6 +21,7 @@ def initialize(title, milestones) ...@@ -17,6 +21,7 @@ def initialize(title, milestones)
@title = title @title = title
@name = title @name = title
@milestones = milestones @milestones = milestones
@first_milestone = milestones.find {|m| m.description.present? } || milestones.first
end end
def safe_title def safe_title
......
class Label < ActiveRecord::Base class Label < ActiveRecord::Base
include CacheMarkdownField
include Referable include Referable
include Subscribable include Subscribable
...@@ -8,6 +9,8 @@ class Label < ActiveRecord::Base ...@@ -8,6 +9,8 @@ class Label < ActiveRecord::Base
None = LabelStruct.new('No Label', 'No Label') None = LabelStruct.new('No Label', 'No Label')
Any = LabelStruct.new('Any Label', '') Any = LabelStruct.new('Any Label', '')
cache_markdown_field :description, pipeline: :single_line
DEFAULT_COLOR = '#428BCA' DEFAULT_COLOR = '#428BCA'
default_value_for :color, DEFAULT_COLOR default_value_for :color, DEFAULT_COLOR
......
...@@ -6,12 +6,16 @@ class Milestone < ActiveRecord::Base ...@@ -6,12 +6,16 @@ class Milestone < ActiveRecord::Base
Any = MilestoneStruct.new('Any Milestone', '', -1) Any = MilestoneStruct.new('Any Milestone', '', -1)
Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2) Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2)
include CacheMarkdownField
include InternalId include InternalId
include Sortable include Sortable
include Referable include Referable
include StripAttribute include StripAttribute
include Milestoneish include Milestoneish
cache_markdown_field :title, pipeline: :single_line
cache_markdown_field :description
belongs_to :project belongs_to :project
has_many :issues has_many :issues
has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues
......
class Namespace < ActiveRecord::Base class Namespace < ActiveRecord::Base
acts_as_paranoid acts_as_paranoid
include CacheMarkdownField
include Sortable include Sortable
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
cache_markdown_field :description, pipeline: :description
has_many :projects, dependent: :destroy has_many :projects, dependent: :destroy
belongs_to :owner, class_name: "User" belongs_to :owner, class_name: "User"
......
...@@ -6,6 +6,7 @@ class Project < ActiveRecord::Base ...@@ -6,6 +6,7 @@ class Project < ActiveRecord::Base
include Gitlab::VisibilityLevel include Gitlab::VisibilityLevel
include Gitlab::CurrentSettings include Gitlab::CurrentSettings
include AccessRequestable include AccessRequestable
include CacheMarkdownField
include Referable include Referable
include Sortable include Sortable
include AfterCommitQueue include AfterCommitQueue
...@@ -17,6 +18,8 @@ class Project < ActiveRecord::Base ...@@ -17,6 +18,8 @@ class Project < ActiveRecord::Base
UNKNOWN_IMPORT_URL = 'http://unknown.git' UNKNOWN_IMPORT_URL = 'http://unknown.git'
cache_markdown_field :description, pipeline: :description
delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true
default_value_for :archived, false default_value_for :archived, false
......
class Release < ActiveRecord::Base class Release < ActiveRecord::Base
include CacheMarkdownField
cache_markdown_field :description
belongs_to :project belongs_to :project
validates :description, :project, :tag, presence: true validates :description, :project, :tag, presence: true
......
class Snippet < ActiveRecord::Base class Snippet < ActiveRecord::Base
include Gitlab::VisibilityLevel include Gitlab::VisibilityLevel
include Linguist::BlobHelper include Linguist::BlobHelper
include CacheMarkdownField
include Participable include Participable
include Referable include Referable
include Sortable include Sortable
include Awardable include Awardable
cache_markdown_field :title, pipeline: :single_line
cache_markdown_field :content
# If file_name changes, it invalidates content
alias_method :default_content_html_invalidator, :content_html_invalidated?
def content_html_invalidated?
default_content_html_invalidator || file_name_changed?
end
default_value_for :visibility_level, Snippet::PRIVATE default_value_for :visibility_level, Snippet::PRIVATE
belongs_to :author, class_name: 'User' belongs_to :author, class_name: 'User'
......
# This worker clears all cache fields in the database, working in batches.
class ClearDatabaseCacheWorker
include Sidekiq::Worker
BATCH_SIZE = 1000
def perform
CacheMarkdownField.caching_classes.each do |kls|
fields = kls.cached_markdown_fields.html_fields
clear_cache_fields = fields.each_with_object({}) do |field, memo|
memo[field] = nil
end
Rails.logger.debug("Clearing Markdown cache for #{kls}: #{fields.inspect}")
kls.unscoped.in_batches(of: BATCH_SIZE) do |relation|
relation.update_all(clear_cache_fields)
end
end
nil
end
end
# Port ActiveRecord::Relation#in_batches from ActiveRecord 5.
# https://github.com/rails/rails/blob/ac027338e4a165273607dccee49a3d38bc836794/activerecord/lib/active_record/relation/batches.rb#L184
# TODO: this can be removed once we're using AR5.
raise "Vendored ActiveRecord 5 code! Delete #{__FILE__}!" if ActiveRecord::VERSION::MAJOR >= 5
module ActiveRecord
module Batches
# Differences from upstream: enumerator support was removed, and custom
# order/limit clauses are ignored without a warning.
def in_batches(of: 1000, start: nil, finish: nil, load: false)
raise "Must provide a block" unless block_given?
relation = self.reorder(batch_order).limit(of)
relation = relation.where(arel_table[primary_key].gteq(start)) if start
relation = relation.where(arel_table[primary_key].lteq(finish)) if finish
batch_relation = relation
loop do
if load
records = batch_relation.records
ids = records.map(&:id)
yielded_relation = self.where(primary_key => ids)
yielded_relation.load_records(records)
else
ids = batch_relation.pluck(primary_key)
yielded_relation = self.where(primary_key => ids)
end
break if ids.empty?
primary_key_offset = ids.last
raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
yield yielded_relation
break if ids.length < of
batch_relation = relation.where(arel_table[primary_key].gt(primary_key_offset))
end
end
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddMarkdownCacheColumns < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
COLUMNS = {
abuse_reports: [:message],
appearances: [:description],
application_settings: [
:sign_in_text,
:help_page_text,
:shared_runners_text,
:after_sign_up_text
],
broadcast_messages: [:message],
issues: [:title, :description],
labels: [:description],
merge_requests: [:title, :description],
milestones: [:title, :description],
namespaces: [:description],
notes: [:note],
projects: [:description],
releases: [:description],
snippets: [:title, :content],
}
def change
COLUMNS.each do |table, columns|
columns.each do |column|
add_column table, "#{column}_html", :text
end
end
end
end
...@@ -23,6 +23,7 @@ ...@@ -23,6 +23,7 @@
t.text "message" t.text "message"
t.datetime "created_at" t.datetime "created_at"
t.datetime "updated_at" t.datetime "updated_at"
t.text "message_html"
end end
create_table "appearances", force: :cascade do |t| create_table "appearances", force: :cascade do |t|
...@@ -32,6 +33,7 @@ ...@@ -32,6 +33,7 @@
t.string "logo" t.string "logo"
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.text "description_html"
end end
create_table "application_settings", force: :cascade do |t| create_table "application_settings", force: :cascade do |t|
...@@ -92,6 +94,10 @@ ...@@ -92,6 +94,10 @@
t.text "domain_blacklist" t.text "domain_blacklist"