# HG changeset patch # User Brian Brazil <brian.brazil@robustperception.io> # Date 1480327376 0 # Mon Nov 28 10:02:56 2016 +0000 # Node ID 7162dbee444ad2b61a32feb233fd7b982fa51283 # Parent 296fd67184bf8473449553d757615885b672321f Add optional postgres exporter to omnibus Fix typo in account helper username. diff --git a/config/projects/gitlab.rb b/config/projects/gitlab.rb --- a/config/projects/gitlab.rb +++ b/config/projects/gitlab.rb @@ -117,6 +117,7 @@ dependency "node-exporter" dependency "prometheus" dependency "redis-exporter" +dependency "postgres-exporter" # version manifest file dependency "version-manifest" diff --git a/config/software/postgres-exporter.rb b/config/software/postgres-exporter.rb new file mode 100644 --- /dev/null +++ b/config/software/postgres-exporter.rb @@ -0,0 +1,38 @@ +# +## Copyright:: Copyright (c) 2016 GitLab Inc. +## License:: Apache License, Version 2.0 +## +## Licensed under the Apache License, Version 2.0 (the 'License'); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an 'AS IS' BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +# + +name 'postgres-exporter' +default_version 'v0.1.1' + +license 'Apache-2.0' +license_file "https://raw.githubusercontent.com/wrouesnel/postgres_exporter/#{version}/LICENSE" + +source git: 'https://github.com/wrouesnel/postgres_exporter.git' + +relative_path 'src/github.com/wrouesnel/postgres_exporter' + +build do + env = { + 'GOPATH' => "#{Omnibus::Config.source_dir}/postgres-exporter", + 'GO15VENDOREXPERIMENT' => '1' # Build machines have go 1.5.x, use vendor directory + } + exporter_source_dir = "#{Omnibus::Config.source_dir}/postgres-exporter" + + command 'go build', env: env + copy 'postgres_exporter', "#{install_dir}/embedded/bin/" +end diff --git a/doc/package-information/defaults.md b/doc/package-information/defaults.md --- a/doc/package-information/defaults.md +++ b/doc/package-information/defaults.md @@ -20,6 +20,7 @@ | Node exporter | No | Port | X | 9100 | | Prometheus | No | Port | X | 9090 | | Redis exporter | No | Port | X | 9121 | +| Postgres exporter | No | Port | X | 9187 | | Redis Sentinel | No | Port | X | 26379 | | Incoming email | No | Port | X | 143 | | Elastic search | No | Port | X | 9200 | diff --git a/files/gitlab-config-template/gitlab.rb.template b/files/gitlab-config-template/gitlab.rb.template --- a/files/gitlab-config-template/gitlab.rb.template +++ b/files/gitlab-config-template/gitlab.rb.template @@ -1195,6 +1195,23 @@ # redis_exporter['listen_address'] = 'localhost:9121' ################################################################################ +# Postgres exporter +################################################################################ + +##! Docs: https://docs.gitlab.com/ce/administration/monitoring/performance/prometheus.html + +#default['gitlab']['postgres-exporter']['enable'] = false +#default['gitlab']['postgres-exporter']['log_directory'] = "/var/log/gitlab/postgres-exporter" +#default['gitlab']['postgres-exporter']['flags'] = { +#} +#default['gitlab']['postgres-exporter']['env'] = { +# 'DATA_SOURCE_NAME' => "user=#{node['gitlab']['postgresql']['username']} host=#{node['gitlab']['gitlab-rails']['db_host']} database=template1" +#} + +##! Advanced settings. Should be changed only if absolutely needed. +# postgres_exporter['listen_address'] = 'localhost:9187' + +################################################################################ ################################################################################ ## Configuration Settings for GitLab EE only ## ################################################################################ diff --git a/files/gitlab-cookbooks/gitlab/attributes/default.rb b/files/gitlab-cookbooks/gitlab/attributes/default.rb --- a/files/gitlab-cookbooks/gitlab/attributes/default.rb +++ b/files/gitlab-cookbooks/gitlab/attributes/default.rb @@ -1002,3 +1002,15 @@ 'redis.addr' => "unix://#{node['gitlab']['gitlab-rails']['redis_socket']}", } default['gitlab']['redis-exporter']['listen_address'] = 'localhost:9121' + +#### +# Postgres exporter +### +default['gitlab']['postgres-exporter']['enable'] = false +default['gitlab']['postgres-exporter']['log_directory'] = "/var/log/gitlab/postgres-exporter" +default['gitlab']['postgres-exporter']['flags'] = { +} +default['gitlab']['postgres-exporter']['env'] = { + 'DATA_SOURCE_NAME' => "user=#{node['gitlab']['postgresql']['username']} host=#{node['gitlab']['gitlab-rails']['db_host']} database=template1" +} +default['gitlab']['postgres-exporter']['listen_address'] = 'localhost:9187' diff --git a/files/gitlab-cookbooks/gitlab/libraries/account_helper.rb b/files/gitlab-cookbooks/gitlab/libraries/account_helper.rb --- a/files/gitlab-cookbooks/gitlab/libraries/account_helper.rb +++ b/files/gitlab-cookbooks/gitlab/libraries/account_helper.rb @@ -46,11 +46,11 @@ node['gitlab']['redis']['username'] end - def postgresgl_user + def postgresql_user node['gitlab']['postgresql']['username'] end - def postgresgl_group + def postgresql_group node['gitlab']['postgresql']['username'] end diff --git a/files/gitlab-cookbooks/gitlab/libraries/gitlab.rb b/files/gitlab-cookbooks/gitlab/libraries/gitlab.rb --- a/files/gitlab-cookbooks/gitlab/libraries/gitlab.rb +++ b/files/gitlab-cookbooks/gitlab/libraries/gitlab.rb @@ -83,6 +83,7 @@ node_exporter Mash.new prometheus Mash.new redis_exporter Mash.new + postgres_exporter Mash.new sentinel Mash.new node nil external_url nil @@ -202,6 +203,7 @@ "node_exporter", "prometheus", "redis_exporter", + "postgres_exporter", "sentinel" ].each do |key| rkey = key.gsub('_', '-') diff --git a/files/gitlab-cookbooks/gitlab/recipes/gitlab-prometheus.rb b/files/gitlab-cookbooks/gitlab/recipes/gitlab-prometheus.rb --- a/files/gitlab-cookbooks/gitlab/recipes/gitlab-prometheus.rb +++ b/files/gitlab-cookbooks/gitlab/recipes/gitlab-prometheus.rb @@ -19,7 +19,8 @@ [ "prometheus", "node-exporter", - "redis-exporter" + "redis-exporter", + "postgres-exporter" ].each do |service| if node["gitlab"][service]["enable"] include_recipe "gitlab::#{service}" diff --git a/files/gitlab-cookbooks/gitlab/recipes/postgres-exporter.rb b/files/gitlab-cookbooks/gitlab/recipes/postgres-exporter.rb new file mode 100644 --- /dev/null +++ b/files/gitlab-cookbooks/gitlab/recipes/postgres-exporter.rb @@ -0,0 +1,42 @@ +# +# Copyright:: Copyright (c) 2012 Opscode, Inc. +# Copyright:: Copyright (c) 2016 Gitlab Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +account_helper = AccountHelper.new(node) +postgresql_user = account_helper.postgresql_user +postgres_exporter_log_dir = node['gitlab']['postgres-exporter']['log_directory'] +postgres_exporter_static_etc_dir = "/opt/gitlab/etc/postgres-exporter" + +directory postgres_exporter_log_dir do + owner postgresql_user + mode '0700' + recursive true +end + +env_dir File.join(postgres_exporter_static_etc_dir, 'env') do + variables node['gitlab']['postgres-exporter']['env'] + restarts ["service[postgres-exporter]"] +end + +runtime_flags = PrometheusHelper.new(node).flags('postgres-exporter') +runit_service 'postgres-exporter' do + options({ + log_directory: postgres_exporter_log_dir, + flags: runtime_flags + }.merge(params)) + log_options node['gitlab']['logging'].to_hash.merge(node['gitlab']['registry'].to_hash) +end + diff --git a/files/gitlab-cookbooks/gitlab/recipes/postgres-exporter_disable.rb b/files/gitlab-cookbooks/gitlab/recipes/postgres-exporter_disable.rb new file mode 100644 --- /dev/null +++ b/files/gitlab-cookbooks/gitlab/recipes/postgres-exporter_disable.rb @@ -0,0 +1,21 @@ +# +# Copyright:: Copyright (c) 2012 Opscode, Inc. +# Copyright:: Copyright (c) 2016 GitLab Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +runit_service "postgres-exporter" do + action :disable +end diff --git a/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb b/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb --- a/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb +++ b/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb @@ -23,7 +23,7 @@ postgresql_data_dir_symlink = File.join(postgresql_dir, "data") postgresql_log_dir = node['gitlab']['postgresql']['log_directory'] postgresql_socket_dir = node['gitlab']['postgresql']['unix_socket_directory'] -postgresql_user = account_helper.postgresgl_user +postgresql_user = account_helper.postgresql_user pg_helper = PgHelper.new(node) diff --git a/files/gitlab-cookbooks/gitlab/templates/default/prometheus.yml.erb b/files/gitlab-cookbooks/gitlab/templates/default/prometheus.yml.erb --- a/files/gitlab-cookbooks/gitlab/templates/default/prometheus.yml.erb +++ b/files/gitlab-cookbooks/gitlab/templates/default/prometheus.yml.erb @@ -16,3 +16,8 @@ static_configs: - targets: ['<%= node['gitlab']['redis-exporter']['listen_address'] %>'] <% end -%> +<% if node['gitlab']['postgres-exporter']['enable'] %> + - job_name: postgres + static_configs: + - targets: ['<%= node['gitlab']['postgres-exporter']['listen_address'] %>'] +<% end -%> diff --git a/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-log-config.erb b/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-log-config.erb new file mode 100644 --- /dev/null +++ b/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-log-config.erb @@ -0,0 +1,6 @@ +<%= "s#@svlogd_size" if @svlogd_size %> +<%= "n#@svlogd_num" if @svlogd_num %> +<%= "t#@svlogd_timeout" if @svlogd_timeout %> +<%= "!#@svlogd_filter" if @svlogd_filter %> +<%= "u#@svlogd_udp" if @svlogd_udp %> +<%= "p#@svlogd_prefix" if @svlogd_prefix %> diff --git a/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-log-run.erb b/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-log-run.erb new file mode 100644 --- /dev/null +++ b/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-log-run.erb @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd -tt <%= @options[:log_directory] %> diff --git a/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-run.erb b/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-run.erb new file mode 100644 --- /dev/null +++ b/files/gitlab-cookbooks/gitlab/templates/default/sv-postgres-exporter-run.erb @@ -0,0 +1,6 @@ +#!/bin/sh +exec 2>&1 +<%= render("mount_point_check.erb") %> +umask 077 +exec chpst -e /opt/gitlab/etc/postgres-exporter/env -P -U <%= node['gitlab']['postgresql']['username'] %>:<%= node['gitlab']['user']['group'] %> -u <%= node['gitlab']['postgresql']['username'] %>:<%= node['gitlab']['user']['group'] %> /opt/gitlab/embedded/bin/postgres_exporter <%= @options[:flags] %> + diff --git a/spec/chef/recipes/postgres_exporter_spec.rb b/spec/chef/recipes/postgres_exporter_spec.rb new file mode 100644 --- /dev/null +++ b/spec/chef/recipes/postgres_exporter_spec.rb @@ -0,0 +1,58 @@ +require 'chef_helper' + +describe 'gitlab::postgres-exporter' do + let(:chef_run) { ChefSpec::SoloRunner.converge('gitlab::default') } + + before do + allow(Gitlab).to receive(:[]).and_call_original + end + + context 'when postgres-exporter is enabled' do + let(:config_template) { chef_run.template('/var/log/gitlab/postgres-exporter/config') } + + before do + stub_gitlab_rb( + postgres_exporter: { enable: true } + ) + end + + it_behaves_like 'enabled runit service', 'postgres-exporter', 'root', 'root' + + it 'populates the files with expected configuration' do + expect(config_template).to notify('ruby_block[reload postgres-exporter svlogd configuration]') + + expect(chef_run).to render_file('/opt/gitlab/sv/postgres-exporter/run') + .with_content { |content| + expect(content).to match(/exec chpst -P/) + expect(content).to match(/\/opt\/gitlab\/embedded\/bin\/postgres_exporter/) + } + + expect(chef_run).to render_file('/opt/gitlab/sv/postgres-exporter/log/run') + .with_content(/exec svlogd -tt \/var\/log\/gitlab\/postgres-exporter/) + end + + it 'creates default set of directories' do + expect(chef_run).to create_directory('/var/log/gitlab/postgres-exporter').with( + owner: 'gitlab-psql', + group: nil, + mode: '0700' + ) + end + end + + context 'when log dir is changed' do + before do + stub_gitlab_rb( + postgres_exporter: { + log_directory: 'foo', + enable: true + } + ) + end + + it 'populates the files with expected configuration' do + expect(chef_run).to render_file('/opt/gitlab/sv/postgres-exporter/log/run') + .with_content(/exec svlogd -tt foo/) + end + end +end