Commit d6de1bb2 authored by Stan Hu's avatar Stan Hu
Browse files

Merge branch 'json-logging-for-k8s' into 'master'

Json logging for k8s Integration

See merge request gitlab-org/gitlab-ce!23328
......@@ -11,6 +11,25 @@ def initialize(app)
protected
def log_error(error)
meta = {
exception: error.class.name,
error_code: error.respond_to?(:error_code) ? error.error_code : nil,
service: self.class.name,
app_id: app.id,
project_ids: app.cluster.project_ids,
group_ids: app.cluster.group_ids,
message: error.message
}
logger.error(meta)
Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def cluster
app.cluster
end
......
......@@ -15,8 +15,7 @@ def execute
check_timeout
end
rescue Kubeclient::HttpError => e
Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
log_error(e)
app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored?
end
......
......@@ -13,12 +13,10 @@ def execute
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
log_error(e)
app.make_errored!("Kubernetes error: #{e.error_code}")
rescue StandardError => e
Rails.logger.error "Can't start installation process: #{e.class.name} #{e.message}"
Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
log_error(e)
app.make_errored!("Can't start installation process.")
end
end
......
......@@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec
{"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"}
```
## `kubernetes.log`
Introduced in GitLab 11.6. This file lives in
`/var/log/gitlab/gitlab-rails/kubernetes.log` for Omnibus GitLab
packages or in `/home/git/gitlab/log/kubernetes.log` for
installations from source.
It logs information related to the Kubernetes Integration including errors
during installing cluster applications on your GitLab managed Kubernetes
clusters.
Each line contains a JSON line that can be ingested by Elasticsearch, Splunk,
etc. For example:
```json
{"severity":"ERROR","time":"2018-11-23T15:14:54.652Z","exception":"Kubeclient::HttpError","error_code":401,"service":"Clusters::Applications::CheckInstallationProgressService","app_id":14,"project_ids":[1],"group_ids":[],"message":"Unauthorized"}
{"severity":"ERROR","time":"2018-11-23T15:42:11.647Z","exception":"Kubeclient::HttpError","error_code":null,"service":"Clusters::Applications::InstallService","app_id":2,"project_ids":[19],"group_ids":[],"message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)"}
```
## `githost.log`
This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for
......
......@@ -75,7 +75,7 @@ To create a new file:
module Import
class Logger < ::Gitlab::JsonLogger
def self.file_name_noext
'importer_json'
'importer'
end
end
end
......@@ -105,7 +105,7 @@ To create a new file:
```ruby
# GOOD
logger.info("Unable to create project", project_id: project.id)
logger.info(message: "Unable to create project", project_id: project.id)
```
1. Be sure to create a common base structure of your log messages. For example,
......@@ -118,13 +118,13 @@ To create a new file:
```ruby
# BAD
logger.info("Import error", error: 1)
logger.info("Import error", error: "I/O failure")
logger.info(message: "Import error", error: 1)
logger.info(message: "Import error", error: "I/O failure")
```
```ruby
# GOOD
logger.info("Import error", error_code: 1, error: "I/O failure")
logger.info(message: "Import error", error_code: 1, error: "I/O failure")
```
## Additional steps with new log files
......
# frozen_string_literal: true
module Gitlab
module Kubernetes
class Logger < ::Gitlab::JsonLogger
def self.file_name_noext
'kubernetes'
end
end
end
end
......@@ -105,6 +105,12 @@
expect(application).to be_errored
expect(application.status_reason).to eq('Kubernetes error: 401')
end
it 'should log error' do
expect(service.send(:logger)).to receive(:error)
service.execute
end
end
end
end
......@@ -33,8 +33,9 @@
end
context 'when k8s cluster communication fails' do
let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
before do
error = Kubeclient::HttpError.new(500, 'system failure', nil)
expect(helm_client).to receive(:install).with(install_command).and_raise(error)
end
......@@ -44,18 +45,81 @@
expect(application).to be_errored
expect(application.status_reason).to match('Kubernetes error: 500')
end
it 'logs errors' do
expect(service.send(:logger)).to receive(:error).with(
{
exception: 'Kubeclient::HttpError',
message: 'system failure',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.project_ids,
group_ids: [],
error_code: 500
}
)
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
error,
extra: {
exception: 'Kubeclient::HttpError',
message: 'system failure',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.project_ids,
group_ids: [],
error_code: 500
}
)
service.execute
end
end
context 'when application cannot be persisted' do
context 'a non kubernetes error happens' do
let(:application) { create(:clusters_applications_helm, :scheduled) }
let(:error) { StandardError.new("something bad happened") }
before do
expect(application).to receive(:make_installing!).once.and_raise(error)
end
it 'make the application errored' do
expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid)
expect(helm_client).not_to receive(:install)
service.execute
expect(application).to be_errored
expect(application.status_reason).to eq("Can't start installation process.")
end
it 'logs errors' do
expect(service.send(:logger)).to receive(:error).with(
{
exception: 'StandardError',
error_code: nil,
message: 'something bad happened',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.projects.pluck(:id),
group_ids: []
}
)
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
error,
extra: {
exception: 'StandardError',
error_code: nil,
message: 'something bad happened',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.projects.pluck(:id),
group_ids: []
}
)
service.execute
end
end
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment