Commit bf952673 authored by Jacob Vosmaer's avatar Jacob Vosmaer
Browse files

Rebase repo check MR

parent 5d16a532747c
class Admin::ProjectsController < Admin::ApplicationController
before_action :project, only: [:show, :transfer]
before_action :project, only: [:show, :transfer, :repo_check]
before_action :group, only: [:show, :transfer]
def index
......@@ -8,6 +8,7 @@ def index
@projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present?
@projects = @projects.with_push if params[:with_push].present?
@projects = @projects.abandoned if params[:abandoned].present?
@projects = @projects.where(last_repo_check_failed: true) if params[:last_repo_check_failed].present?
@projects = @projects.non_archived unless params[:with_archived].present?
@projects = @projects.search(params[:name]) if params[:name].present?
@projects = @projects.sort(@sort = params[:sort])
......@@ -30,6 +31,27 @@ def transfer
redirect_to admin_namespace_project_path(@project.namespace, @project)
end
def repo_check
SingleRepoCheckWorker.perform_async(@project.id)
redirect_to(
admin_namespace_project_path(@project.namespace, @project),
notice: 'Repo check was triggered'
)
end
def clear_repo_check_states
Project.update_all(
last_repo_check_failed: false,
last_repo_check_at: nil
)
redirect_to(
admin_namespaces_projects_path,
notice: 'All project repo check states were cleared'
)
end
protected
def project
......
class RepoCheckMailer < BaseMailer
include ActionView::Helpers::TextHelper
def notify(failed_count)
if failed_count == 1
@message = "One project failed its last repository check"
else
@message = "#{failed_count} projects failed their last repository check"
end
mail(
to: User.admins.pluck(:email),
subject: @message
)
end
end
- page_title "Logs"
- loggers = [Gitlab::GitLogger, Gitlab::AppLogger,
Gitlab::ProductionLogger, Gitlab::SidekiqLogger]
Gitlab::ProductionLogger, Gitlab::SidekiqLogger,
Gitlab::RepoCheckLogger]
%ul.nav-links.log-tabs
- loggers.each do |klass|
%li{ class: (klass == Gitlab::GitLogger ? 'active' : '') }
......
......@@ -3,7 +3,7 @@
.row.prepend-top-default
%aside.col-md-3
.admin-filter
.panel.admin-filter
= form_tag admin_namespaces_projects_path, method: :get, class: '' do
.form-group
= label_tag :name, 'Name:'
......@@ -38,11 +38,22 @@
%span.descr
= visibility_level_icon(level)
= label
%hr
%fieldset
%strong Problems
.checkbox
= label_tag :last_repo_check_failed do
= check_box_tag :last_repo_check_failed, 1, params[:last_repo_check_failed]
%span Last repo check failed
= hidden_field_tag :sort, params[:sort]
= button_tag "Search", class: "btn submit btn-primary"
= link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel"
.panel.panel-default.repo-check-states
.panel-heading
Repo check states
.panel-body
= link_to 'Clear all', clear_repo_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repo check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
%section.col-md-9
.panel.panel-default
.panel-heading
......
......@@ -5,6 +5,14 @@
%i.fa.fa-pencil-square-o
Edit
%hr
- if @project.last_repo_check_failed?
.row
.col-md-12
.panel
.panel-heading.alert.alert-danger
Last repo check failed. See
= link_to 'repocheck.log', admin_logs_path
for error messages.
.row
.col-md-6
.panel.panel-default
......@@ -95,6 +103,30 @@
.col-sm-offset-2.col-sm-10
= f.submit 'Transfer', class: 'btn btn-primary'
.panel.panel-default.repo-check
.panel-heading
Repo check
.panel-body
= form_for @project, url: repo_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f|
.form-group
- if @project.last_repo_check_at.nil?
This repository has never been checked.
- else
This repository was last checked
= @project.last_repo_check_at.to_s(:medium) + '.'
The check
- if @project.last_repo_check_failed?
= succeed '.' do
%strong.cred failed
See
= link_to 'repocheck.log', admin_logs_path
for error messages.
- else
passed.
.form-group
= f.submit 'Trigger repo check', class: 'btn btn-primary'
.col-md-6
- if @group
.panel.panel-default
......
%p
#{@message}.
%p
= link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repo_check_failed: 1)
#{@message}.
\
View details: #{admin_namespaces_projects_url(last_repo_check_failed: 1)}
class AdminEmailWorker
include Sidekiq::Worker
sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
def perform
repo_check_failed_count = Project.where(last_repo_check_failed: true).count
return if repo_check_failed_count.zero?
RepoCheckMailer.notify(repo_check_failed_count).deliver_now
end
end
class RepoCheckWorker
include Sidekiq::Worker
RUN_TIME = 3600
sidekiq_options retry: false
def perform
start = Time.now
# This loop will break after a little more than one hour ('a little
# more' because `git fsck` may take a few minutes), or if it runs out of
# projects to check. By default sidekiq-cron will start a new
# RepoCheckWorker each hour so that as long as there are repositories to
# check, only one (or two) will be checked at a time.
project_ids.each do |project_id|
break if Time.now - start >= RUN_TIME
next if !try_obtain_lease(project_id)
SingleRepoCheckWorker.new.perform(project_id)
end
end
private
# In an ideal world we would use Project.where(...).find_each.
# Unfortunately, calling 'find_each' drops the 'where', so we must build
# an array of IDs instead.
def project_ids
limit = 10_000
never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit).
pluck(:id)
old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago).
reorder('last_repo_check_at ASC').limit(limit).pluck(:id)
never_checked_projects + old_check_projects
end
def try_obtain_lease(id)
lease = Gitlab::ExclusiveLease.new(
"project_repo_check:#{id}",
timeout: RUN_TIME
)
lease.try_obtain
end
end
class SingleRepoCheckWorker
include Sidekiq::Worker
sidekiq_options retry: false
def perform(project_id)
project = Project.find(project_id)
update(project, success: check(project))
end
private
def check(project)
[project.repository.path_to_repo, project.wiki.wiki.path].all? do |path|
git_fsck(path)
end
end
def git_fsck(path)
cmd = %W(nice git --git-dir=#{path} fsck)
output, status = Gitlab::Popen.popen(cmd)
return true if status.zero?
Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
false
end
def update(project, success:)
project.update_columns(
last_repo_check_failed: !success,
last_repo_check_at: Time.now,
)
end
end
......@@ -155,6 +155,13 @@ production: &base
# Flag stuck CI builds as failed
stuck_ci_builds_worker:
cron: "0 0 * * *"
# Periodically run 'git fsck' on all repositories. If started more than
# once per hour you will have concurrent 'git fsck' jobs.
repo_check_worker:
cron: "20 * * * *"
# Send admin emails once a day
admin_email_worker:
cron: "0 0 * * *"
#
......
......@@ -239,7 +239,12 @@ def host(url)
Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
Settings.cron_jobs['repo_check_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repo_check_worker']['cron'] ||= '20 * * * *'
Settings.cron_jobs['repo_check_worker']['job_class'] = 'RepoCheckWorker'
Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
#
# GitLab Shell
......
......@@ -264,6 +264,11 @@
member do
put :transfer
post :repo_check
end
collection do
put :clear_repo_check_states
end
resources :runner_projects
......
class ProjectAddRepoCheck < ActiveRecord::Migration
def change
add_column :projects, :last_repo_check_failed, :boolean, default: false
add_column :projects, :last_repo_check_at, :datetime
end
end
......@@ -732,6 +732,8 @@
t.boolean "public_builds", default: true, null: false
t.string "main_language"
t.integer "pushes_since_gc", default: 0
t.boolean "last_repo_check_failed", default: false
t.datetime "last_repo_check_at"
end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
# Repo checks
_**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_
---
Git has a built-in mechanism [git fsck][git-fsck] to verify the
integrity of all data commited to a repository. GitLab administrators can
trigger such a check for a project via the admin panel. The checks run
asynchronously so it may take a few minutes before the check result is
visible on the project admin page. If the checks failed you can see their
output on the admin log page under 'repocheck.log'.
## Periodical checks
GitLab periodically runs a repo check on all project repositories and
wiki repositories in order to detect data corruption problems. A
project will be checked no more than once per week. If any projects
fail their repo checks all GitLab administrators will receive an email
notification of the situation. This notification is sent out no more
than once a day.
## What to do if a check failed
If the repo check fails for some repository you should look up the error
in repocheck.log (in the admin panel or on disk; see
`/var/log/gitlab/gitlab-rails` for Omnibus installations or
`/home/git/gitlab/log` for installations from source). Once you have
resolved the issue use the admin panel to trigger a new repo check on
the project. This will clear the 'check failed' state.
If for some reason the periodical repo check caused a lot of false
alarms you can choose to clear ALL repo check states from the admin
project index page.
---
[ce-3232]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232 "Auto git fsck"
[git-fsck]: https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html "git fsck documentation"
\ No newline at end of file
module Gitlab
class RepoCheckLogger < Gitlab::Logger
def self.file_name_noext
'repocheck'
end
end
end
require 'spec_helper'
require 'rails_helper'
describe "Admin::Projects", feature: true do
describe "Admin Projects", feature: true do
before do
@project = create(:project)
login_as :admin
......@@ -31,4 +32,38 @@
expect(page).to have_content(@project.name)
end
end
feature 'repo checks' do
scenario 'trigger repo check' do
visit_admin_project_page
page.within('.repo-check') do
click_button 'Trigger repo check'
end
expect(page).to have_content('Repo check was triggered')
end
scenario 'see failed repo check' do
@project.update_column(:last_repo_check_failed, true)
visit_admin_project_page
expect(page).to have_content('Last repo check failed')
end
scenario 'clear repo checks', js: true do
@project.update_column(:last_repo_check_failed, true)
visit admin_namespaces_projects_path
page.within('.repo-check-states') do
click_link 'Clear all' # pop-up should be auto confirmed
end
expect(@project.reload.last_repo_check_failed).to eq(false)
end
end
def visit_admin_project_page
visit admin_namespace_project_path(@project.namespace, @project)
end
end
require 'rails_helper'
describe RepoCheckMailer do
include EmailSpec::Matchers
describe '.notify' do
it 'emails all admins' do
admins = 3.times.map { create(:admin) }
mail = described_class.notify(1)
expect(mail).to deliver_to admins.map(&:email)
end
it 'mentions the number of failed checks' do
mail = described_class.notify(3)
expect(mail).to have_subject '3 projects failed their last repository check'
end
end
end
require 'spec_helper'
describe RepoCheckWorker do
subject { RepoCheckWorker.new }
it 'prefers projects that have never been checked' do
projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.weeks.ago)
expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id))
end
it 'sorts projects by last_repo_check_at' do
projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 2.weeks.ago)
projects[1].update_column(:last_repo_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.weeks.ago)
expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id))
end
it 'excludes projects that were checked recently' do
projects = 3.times.map { create(:project) }
projects[0].update_column(:last_repo_check_at, 2.days.ago)
projects[1].update_column(:last_repo_check_at, 1.month.ago)
projects[2].update_column(:last_repo_check_at, 3.days.ago)
expect(subject.perform).to eq([projects[1].id])
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment