Commit e09fd9a8 authored by Andrew Newdigate's avatar Andrew Newdigate
Browse files

Add experimental support for Puma

This allows us (and others) to test drive Puma without it affecting all
users. Puma can be enabled by setting the environment variable
"EXPERIMENTAL_PUMA" to a non empty value.
parent dc137d562345
......@@ -40,6 +40,7 @@ eslint-report.html
/config/redis.queues.yml
/config/redis.shared_state.yml
/config/unicorn.rb
/config/puma.rb
/config/secrets.yml
/config/sidekiq.yml
/config/registry.key
......
......@@ -153,6 +153,11 @@ group :unicorn do
gem 'unicorn-worker-killer', '~> 0.4.4'
end
group :puma do
gem 'puma', '~> 3.12', require: false
gem 'puma_worker_killer', require: false
end
# State machine
gem 'state_machines-activerecord', '~> 0.5.1'
......
......@@ -598,6 +598,10 @@ GEM
pry-rails (0.3.6)
pry (>= 0.10.4)
public_suffix (3.0.3)
puma (3.12.0)
puma_worker_killer (0.1.0)
get_process_mem (~> 0.2)
puma (>= 2.7, < 4)
pyu-ruby-sasl (0.0.3.3)
rack (1.6.10)
rack-accept (0.4.5)
......@@ -1076,6 +1080,8 @@ DEPENDENCIES
prometheus-client-mmap (~> 0.9.4)
pry-byebug (~> 3.4.1)
pry-rails (~> 0.3.4)
puma (~> 3.12)
puma_worker_killer
rack-attack (~> 4.4.1)
rack-cors (~> 1.0.0)
rack-oauth2 (~> 1.2.1)
......
......@@ -602,6 +602,10 @@ GEM
pry-rails (0.3.6)
pry (>= 0.10.4)
public_suffix (3.0.3)
puma (3.12.0)
puma_worker_killer (0.1.0)
get_process_mem (~> 0.2)
puma (>= 2.7, < 4)
pyu-ruby-sasl (0.0.3.3)
rack (2.0.5)
rack-accept (0.4.5)
......@@ -1085,6 +1089,8 @@ DEPENDENCIES
prometheus-client-mmap (~> 0.9.4)
pry-byebug (~> 3.4.1)
pry-rails (~> 0.3.4)
puma (~> 3.12)
puma_worker_killer
rack-attack (~> 4.4.1)
rack-cors (~> 1.0.0)
rack-oauth2 (~> 1.2.1)
......
......@@ -3,6 +3,11 @@
cd $(dirname $0)/..
app_root=$(pwd)
# Switch to experimental PUMA configuration
if [ -n "${EXPERIMENTAL_PUMA}" ]; then
exec bin/web_puma "$@"
fi
unicorn_pidfile="$app_root/tmp/pids/unicorn.pid"
unicorn_config="$app_root/config/unicorn.rb"
unicorn_cmd="bundle exec unicorn_rails -c $unicorn_config -E $RAILS_ENV"
......
#!/bin/sh
set -e
cd $(dirname $0)/..
app_root=$(pwd)
puma_pidfile="$app_root/tmp/pids/puma.pid"
puma_config="$app_root/config/puma.rb"
spawn_puma()
{
exec bundle exec puma --config "${puma_config}" "$@"
}
get_puma_pid()
{
pid=$(cat "${puma_pidfile}")
if [ -z "$pid" ] ; then
echo "Could not find a PID in $puma_pidfile"
exit 1
fi
echo "${pid}"
}
start()
{
spawn_puma -d
}
start_foreground()
{
spawn_puma
}
stop()
{
get_puma_pid
kill -QUIT "$(get_puma_pid)"
}
reload()
{
kill -USR2 "$(get_puma_pid)"
}
case "$1" in
start)
start
;;
start_foreground)
start_foreground
;;
stop)
stop
;;
reload)
reload
;;
*)
echo "Usage: RAILS_ENV=your_env $0 {start|stop|reload}"
;;
esac
---
title: Experimental support for running Puma multithreaded web-server
merge_request: 22372
author:
type: performance
......@@ -26,9 +26,25 @@
end
if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
unless Sidekiq.server?
Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
Gitlab::Cluster::LifecycleEvents.on_worker_start do
defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change
unless Sidekiq.server?
Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
end
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
end
end
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
Gitlab::Cluster::LifecycleEvents.on_master_restart do
# The following is necessary to ensure stale Prometheus metrics don't
# accumulate over time. It needs to be done in this hook as opposed to
# inside an init script to ensure metrics files aren't deleted after new
# unicorn workers start after a SIGUSR2 is received.
prometheus_multiproc_dir = ENV['prometheus_multiproc_dir']
if prometheus_multiproc_dir
old_metrics = Dir[File.join(prometheus_multiproc_dir, '*.db')]
FileUtils.rm_rf(old_metrics)
end
end
......@@ -158,7 +158,9 @@ def instrument_classes(instrumentation)
GC::Profiler.enable
Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start
Gitlab::Cluster::LifecycleEvents.on_worker_start do
Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start
end
module TrackNewRedisConnections
def connect(*args)
......
# frozen_string_literal: true
# Don't handle sidekiq configuration as it
# has its own special active record configuration here
if defined?(ActiveRecord::Base) && !Sidekiq.server?
Gitlab::Cluster::LifecycleEvents.on_worker_start do
ActiveSupport.on_load(:active_record) do
ActiveRecord::Base.establish_connection
Rails.logger.debug("ActiveRecord connection established")
end
end
end
if defined?(ActiveRecord::Base)
Gitlab::Cluster::LifecycleEvents.on_before_fork do
# the following is highly recommended for Rails + "preload_app true"
# as there's no need for the master process to hold a connection
ActiveRecord::Base.connection.disconnect!
Rails.logger.debug("ActiveRecord connection disconnected")
end
end
# frozen_string_literal: true
if /darwin/ =~ RUBY_PLATFORM
Gitlab::Cluster::LifecycleEvents.on_before_fork do
require 'fiddle'
# Dynamically load Foundation.framework, ~implicitly~ initialising
# the Objective-C runtime before any forking happens in Unicorn
#
# From https://bugs.ruby-lang.org/issues/14009
Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation'
end
end
# frozen_string_literal: true
if ENV['ENABLE_RBTRACE']
Gitlab::Cluster::LifecycleEvents.on_worker_start do
# Unicorn clears out signals before it forks, so rbtrace won't work
# unless it is enabled after the fork.
require 'rbtrace'
end
end
......@@ -14,8 +14,6 @@
enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
Sidekiq.configure_server do |config|
require 'rbtrace' if ENV['ENABLE_RBTRACE']
config.redis = queues_config_hash
config.server_middleware do |chain|
......
# frozen_string_literal: true
# -----------------------------------------------------------------------
# This file is used by the GDK to generate a default config/puma.rb file
# Note that `/home/git` will be substituted for the actual GDK root
# directory when this file is generated
# -----------------------------------------------------------------------
# Load "path" as a rackup file.
#
# The default is "config.ru".
#
rackup 'config.ru'
pidfile '/home/git/gitlab/tmp/pids/puma.pid'
state_path '/home/git/gitlab/tmp/pids/puma.state'
stdout_redirect '/home/git/gitlab/log/puma.stdout.log',
'/home/git/gitlab/log/puma.stderr.log',
true
# Configure "min" to be the minimum number of threads to use to answer
# requests and "max" the maximum.
#
# The default is "0, 16".
#
threads 1, 4
# By default, workers accept all requests and queue them to pass to handlers.
# When false, workers accept the number of simultaneous requests configured.
#
# Queueing requests generally improves performance, but can cause deadlocks if
# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
#
# When set to false this may require a reverse proxy to handle slow clients and
# queue requests before they reach puma. This is due to disabling HTTP keepalive
queue_requests false
# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
# accepted protocols.
bind 'unix:///home/git/gitlab.socket'
workers 2
require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
on_restart do
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_master_restart
end
before_fork do
# Signal to the puma killer
Gitlab::Cluster::PumaWorkerKillerInitializer.start @config.options unless ENV['DISABLE_PUMA_WORKER_KILLER']
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
end
Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options
on_worker_boot do
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
end
# Preload the application before starting the workers; this conflicts with
# phased restart feature. (off by default)
preload_app!
tag 'gitlab-puma-worker'
# Verifies that all workers have checked in to the master process within
# the given timeout. If not the worker process will be restarted. Default
# value is 60 seconds.
#
worker_timeout 60
......@@ -81,22 +81,16 @@ preload_app true
# fast LAN.
check_client_connection false
require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
before_exec do |server|
# The following is necessary to ensure stale Prometheus metrics don't
# accumulate over time. It needs to be done in this hook as opposed to
# inside an init script to ensure metrics files aren't deleted after new
# unicorn workers start after a SIGUSR2 is received.
if ENV['prometheus_multiproc_dir']
old_metrics = Dir[File.join(ENV['prometheus_multiproc_dir'], '*.db')]
FileUtils.rm_rf(old_metrics)
end
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_master_restart
end
before_fork do |server, worker|
# the following is highly recommended for Rails + "preload_app true"
# as there's no need for the master process to hold a connection
defined?(ActiveRecord::Base) &&
ActiveRecord::Base.connection.disconnect!
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
......@@ -124,25 +118,10 @@ before_fork do |server, worker|
end
after_fork do |server, worker|
# Unicorn clears out signals before it forks, so rbtrace won't work
# unless it is enabled after the fork.
require 'rbtrace' if ENV['ENABLE_RBTRACE']
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
# the following is *required* for Rails + "preload_app true",
defined?(ActiveRecord::Base) &&
ActiveRecord::Base.establish_connection
# reset prometheus client, this will cause any opened metrics files to be closed
defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
Prometheus::Client.reinitialize_on_pid_change
# if preload_app is true, then you may also want to check and
# restart any other shared sockets/descriptors such as Memcached,
# and Redis. TokyoCabinet file handles are safe to reuse
# between any number of forked children (assuming your kernel
# correctly implements pread()/pwrite() system calls)
end
# frozen_string_literal: true
# -------------------------------------------------------------------------
# This file is used by the GDK to generate a default config/unicorn.rb file
# Note that `/home/git` will be substituted for the actual GDK root
# directory when this file is generated
# -------------------------------------------------------------------------
worker_processes 2
timeout 60
listen '/home/git/gitlab.socket'
preload_app true
check_client_connection false
require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
before_exec do |server|
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_master_restart
end
before_fork do |server, worker|
# the following is highly recommended for Rails + "preload_app true"
# as there's no need for the master process to hold a connection
defined?(ActiveRecord::Base) &&
ActiveRecord::Base.connection.disconnect!
if /darwin/ =~ RUBY_PLATFORM
require 'fiddle'
# Dynamically load Foundation.framework, ~implicitly~ initialising
# the Objective-C runtime before any forking happens in Unicorn
#
# From https://bugs.ruby-lang.org/issues/14009
Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation'
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
# twice as many worker_processes as you have configured.
#
# This allows a new master process to incrementally
# phase out the old master process with SIGTTOU to avoid a
# thundering herd (especially in the "preload_app false" case)
# when doing a transparent upgrade. The last worker spawned
# will then kill off the old master process with a SIGQUIT.
old_pid = "#{server.config[:pid]}.oldbin"
if old_pid != server.pid
begin
sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
Process.kill(sig, File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
end
end
#
# Throttle the master from forking too quickly by sleeping. Due
# to the implementation of standard Unix signal handlers, this
# helps (but does not completely) prevent identical, repeated signals
# from being lost when the receiving process is busy.
# sleep 1
end
after_fork do |server, worker|
# Unicorn clears out signals before it forks, so rbtrace won't work
# unless it is enabled after the fork.
require 'rbtrace' if ENV['ENABLE_RBTRACE']
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
# the following is *required* for Rails + "preload_app true",
defined?(ActiveRecord::Base) &&
ActiveRecord::Base.establish_connection
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
end
---
comments: false
---
# From 11.4 to 11.5
Make sure you view this update guide from the branch (version) of GitLab you would
like to install (e.g., `11-5-stable`. You can select the branch in the version
dropdown at the top left corner of GitLab (below the menu bar).
If the highest number stable branch is unclear please check the
[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation
guide links by version.
### 1. Stop server
```bash
sudo service gitlab stop
```
### 2. Backup
NOTE: If you installed GitLab from source, make sure `rsync` is installed.
```bash
cd /home/git/gitlab
sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
```
### 3. Update Ruby
NOTE: GitLab 11.0 and higher only support Ruby 2.4.x and dropped support for Ruby 2.3.x. Be
sure to upgrade your interpreter if necessary.
You can check which version you are running with `ruby -v`.
Download Ruby and compile it:
```bash
mkdir /tmp/ruby && cd /tmp/ruby
curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.4/ruby-2.4.5.tar.gz
echo '4d650f302f1ec00256450b112bb023644b6ab6dd ruby-2.4.5.tar.gz' | shasum -c - && tar xzf ruby-2.4.5.tar.gz
cd ruby-2.4.5
./configure --disable-install-rdoc
make
sudo make install
```
Install Bundler:
```bash
sudo gem install bundler --no-ri --no-rdoc
```
### 4. Update Node
GitLab utilizes [webpack](http://webpack.js.org) to compile frontend assets.
This requires a minimum version of node v6.0.0.
You can check which version you are running with `node -v`. If you are running
a version older than `v6.0.0` you will need to update to a newer version. You
can find instructions to install from community maintained packages or compile
from source at the nodejs.org website.
<https://nodejs.org/en/download/>
GitLab also requires the use of yarn `>= v1.2.0` to manage JavaScript
dependencies.
```bash
curl --silent --show-error https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
sudo apt-get update
sudo apt-get install yarn
```
More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install).
### 5. Update Go
NOTE: GitLab 11.4 and higher only supports Go 1.10.x and newer, and dropped support for Go
1.9.x. Be sure to upgrade your installation if necessary.
You can check which version you are running with `go version`.
Download and install Go:
```bash
# Remove former Go installation folder
sudo rm -rf /usr/local/go
curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz
echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \
sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz
sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/
rm go1.10.3.linux-amd64.tar.gz
```
### 6. Get latest code
```bash
cd /home/git/gitlab
sudo -u git -H git fetch --all --prune
sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically
sudo -u git -H git checkout -- locale
```
For GitLab Community Edition:
```bash
cd /home/git/gitlab
sudo -u git -H git checkout 11-5-stable
```
OR
For GitLab Enterprise Edition:
```bash
cd /home/git/gitlab
sudo -u git -H git checkout 11-5-stable-ee
```
### 7. Update gitlab-shell
```bash
cd /home/git/gitlab-shell
sudo -u git -H git fetch --all --tags --prune
sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_SHELL_VERSION)
sudo -u git -H bin/compile
```
### 8. Update gitlab-workhorse
Install and compile gitlab-workhorse. GitLab-Workhorse uses
[GNU Make](https://www.gnu.org/software/make/).
If you are not using Linux you may have to run `gmake` instead of
`make` below.
```bash
cd /home/git/gitlab-workhorse
sudo -u git -H git fetch --all --tags --prune
sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_WORKHORSE_VERSION)
sudo -u git -H make
```
### 9. Update Gitaly
#### New Gitaly configuration options required
In order to function Gitaly needs some additional configuration information. Below we assume you installed Gitaly in `/home/git/gitaly` and GitLab Shell in `/home/git/gitlab-shell`.
```shell
echo '
[gitaly-ruby]
dir = "/home/git/gitaly/ruby"
[gitlab-shell]
dir = "/home/git/gitlab-shell"
' | sudo -u git tee -a /home/git/gitaly/config.toml
```
<