Commit ebdfda69 authored by Robert Speicher's avatar Robert Speicher
Browse files

Merge branch 'gitaly-refs' into 'master'

Implement Gitaly refs client

See merge request !9291
......@@ -59,7 +59,7 @@ def initialize(path_with_namespace, project)
def raw_repository
return nil unless path_with_namespace
@raw_repository ||= Gitlab::Git::Repository.new(path_to_repo)
@raw_repository ||= initialize_raw_repository
end
# Return absolute path to repository
......@@ -146,12 +146,7 @@ def find_branch(name, fresh_repo: true)
# may cause the branch to "disappear" erroneously or have the wrong SHA.
#
# See: https://github.com/libgit2/libgit2/issues/1534 and https://gitlab.com/gitlab-org/gitlab-ce/issues/15392
raw_repo =
if fresh_repo
Gitlab::Git::Repository.new(path_to_repo)
else
raw_repository
end
raw_repo = fresh_repo ? initialize_raw_repository : raw_repository
raw_repo.find_branch(name)
end
......@@ -505,9 +500,7 @@ def commit_count_for_ref(ref)
end
end
def branch_names
branches.map(&:name)
end
delegate :branch_names, to: :raw_repository
cache_method :branch_names, fallback: []
delegate :tag_names, to: :raw_repository
......@@ -1168,4 +1161,8 @@ def repository_event(event, tags = {})
def repository_storage_path
@project.repository_storage_path
end
def initialize_raw_repository
Gitlab::Git::Repository.new(project.repository_storage, path_with_namespace + '.git')
end
end
---
title: Incorporate Gitaly client for refs service
merge_request: 9291
author:
......@@ -8,11 +8,10 @@ def up
project_data.each do |project|
id = project['id']
namespace_path = project['namespace_path'] || ''
repos_path = Gitlab.config.gitlab_shell['repos_path'] || Gitlab.config.repositories.storages.default['path']
path = File.join(repos_path, namespace_path, project['project_path'] + '.git')
path = File.join(namespace_path, project['project_path'] + '.git')
begin
repo = Gitlab::Git::Repository.new(path)
repo = Gitlab::Git::Repository.new('default', path)
if repo.empty?
print '-'
else
......
......@@ -138,8 +138,11 @@ class Internal < Grape::API
return unless Gitlab::GitalyClient.enabled?
relative_path = Gitlab::RepoPath.strip_storage_path(params[:repo_path])
project = Project.find_by_full_path(relative_path.sub(/\.(git|wiki)\z/, ''))
begin
Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive
Gitlab::GitalyClient::Notifications.new(project.repository_storage, relative_path).post_receive
rescue GRPC::Unavailable => e
render_api_error(e, 500)
end
......
......@@ -4,6 +4,8 @@ module Git
TAG_REF_PREFIX = "refs/tags/".freeze
BRANCH_REF_PREFIX = "refs/heads/".freeze
CommandError = Class.new(StandardError)
class << self
def ref_name(ref)
ref.sub(/\Arefs\/(tags|heads)\//, '')
......
......@@ -25,9 +25,13 @@ class Repository
# 'path' must be the path to a _bare_ git repository, e.g.
# /path/to/my-repo.git
def initialize(path)
@path = path
@name = path.split("/").last
def initialize(repository_storage, relative_path)
@repository_storage = repository_storage
@relative_path = relative_path
storage_path = Gitlab.config.repositories.storages[@repository_storage]['path']
@path = File.join(storage_path, @relative_path)
@name = @relative_path.split("/").last
@attributes = Gitlab::Git::Attributes.new(path)
end
......@@ -37,7 +41,15 @@ def initialize(path)
# Default branch in the repository
def root_ref
@root_ref ||= discover_default_branch
@root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled|
if is_enabled
gitaly_ref_client.default_branch_name
else
discover_default_branch
end
end
rescue GRPC::BadStatus => e
raise CommandError.new(e)
end
# Alias to old method for compatibility
......@@ -54,7 +66,15 @@ def rugged
# Returns an Array of branch names
# sorted by name ASC
def branch_names
branches.map(&:name)
Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled|
if is_enabled
gitaly_ref_client.branch_names
else
branches.map(&:name)
end
end
rescue GRPC::BadStatus => e
raise CommandError.new(e)
end
# Returns an Array of Branches
......@@ -107,7 +127,15 @@ def branch_count
# Returns an Array of tag names
def tag_names
rugged.tags.map { |t| t.name }
Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled|
if is_enabled
gitaly_ref_client.tag_names
else
rugged.tags.map { |t| t.name }
end
end
rescue GRPC::BadStatus => e
raise CommandError.new(e)
end
# Returns an Array of Tags
......@@ -1202,6 +1230,10 @@ def diff_patches(from, to, options = {}, *paths)
diff.find_similar!(break_rewrites: break_rewrites)
diff.each_patch
end
def gitaly_ref_client
@gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(@repository_storage, @relative_path)
end
end
end
end
......@@ -3,18 +3,13 @@ module GitalyClient
class Notifications
attr_accessor :stub
def initialize(repo_path)
full_path = Gitlab::RepoPath.strip_storage_path(repo_path).
sub(/\.git\z/, '').sub(/\.wiki\z/, '')
@project = Project.find_by_full_path(full_path)
channel = GitalyClient.get_channel(@project.repository_storage)
@stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel)
def initialize(repository_storage, relative_path)
@channel, @repository = Util.process_path(repository_storage, relative_path)
@stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: @channel)
end
def post_receive
repository = Gitaly::Repository.new(path: @project.repository.path_to_repo)
request = Gitaly::PostReceiveRequest.new(repository: repository)
request = Gitaly::PostReceiveRequest.new(repository: @repository)
@stub.post_receive(request)
end
end
......
module Gitlab
module GitalyClient
class Ref
attr_accessor :stub
def initialize(repository_storage, relative_path)
@channel, @repository = Util.process_path(repository_storage, relative_path)
@stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: @channel)
end
def default_branch_name
request = Gitaly::FindDefaultBranchNameRequest.new(repository: @repository)
stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '')
end
def branch_names
request = Gitaly::FindAllBranchNamesRequest.new(repository: @repository)
consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/')
end
def tag_names
request = Gitaly::FindAllTagNamesRequest.new(repository: @repository)
consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/')
end
private
def consume_refs_response(response, prefix:)
response.flat_map do |r|
r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') }
end
end
end
end
end
module Gitlab
module GitalyClient
module Util
def self.process_path(repository_storage, relative_path)
channel = GitalyClient.get_channel(repository_storage)
storage_path = Gitlab.config.repositories.storages[repository_storage]['path']
repository = Gitaly::Repository.new(path: File.join(storage_path, relative_path))
[channel, repository]
end
end
end
end
......@@ -2,7 +2,7 @@
describe Gitlab::Git::Attributes, seed_helper: true do
let(:path) do
File.join(SEED_REPOSITORY_PATH, 'with-git-attributes.git')
File.join(SEED_STORAGE_PATH, 'with-git-attributes.git')
end
subject { described_class.new(path) }
......@@ -141,7 +141,7 @@
end
it 'does not yield when the attributes file has an unsupported encoding' do
path = File.join(SEED_REPOSITORY_PATH, 'with-invalid-git-attributes.git')
path = File.join(SEED_STORAGE_PATH, 'with-invalid-git-attributes.git')
attrs = described_class.new(path)
expect { |b| attrs.each_line(&b) }.not_to yield_control
......
......@@ -2,7 +2,7 @@
require "spec_helper"
describe Gitlab::Git::Blame, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
let(:blame) do
Gitlab::Git::Blame.new(repository, SeedRepo::Commit::ID, "CONTRIBUTING.md")
end
......
......@@ -3,7 +3,7 @@
require "spec_helper"
describe Gitlab::Git::Blob, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
describe 'initialize' do
let(:blob) { Gitlab::Git::Blob.new(name: 'test') }
......
require "spec_helper"
describe Gitlab::Git::Branch, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
subject { repository.branches }
......
require "spec_helper"
describe Gitlab::Git::Commit, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
let(:commit) { Gitlab::Git::Commit.find(repository, SeedRepo::Commit::ID) }
let(:rugged_commit) do
repository.rugged.lookup(SeedRepo::Commit::ID)
......@@ -9,7 +9,7 @@
describe "Commit info" do
before do
repo = Gitlab::Git::Repository.new(TEST_REPO_PATH).rugged
repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
@committer = {
email: 'mike@smith.com',
......@@ -59,7 +59,7 @@
after do
# Erase the new commit so other tests get the original repo
repo = Gitlab::Git::Repository.new(TEST_REPO_PATH).rugged
repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
repo.references.update("refs/heads/master", SeedRepo::LastCommit::ID)
end
end
......@@ -95,7 +95,7 @@
end
context 'with broken repo' do
let(:repository) { Gitlab::Git::Repository.new(TEST_BROKEN_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_BROKEN_REPO_PATH) }
it 'returns nil' do
expect(Gitlab::Git::Commit.find(repository, SeedRepo::Commit::ID)).to be_nil
......
require "spec_helper"
describe Gitlab::Git::Compare, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
let(:compare) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::Commit::ID, false) }
let(:compare_straight) { Gitlab::Git::Compare.new(repository, SeedRepo::BigCommit::ID, SeedRepo::Commit::ID, true) }
......
require "spec_helper"
describe Gitlab::Git::Diff, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
before do
@raw_diff_hash = {
......
......@@ -2,7 +2,7 @@
describe Gitlab::Git::EncodingHelper do
let(:ext_class) { Class.new { extend Gitlab::Git::EncodingHelper } }
let(:binary_string) { File.join(SEED_REPOSITORY_PATH, 'gitlab_logo.png') }
let(:binary_string) { File.join(SEED_STORAGE_PATH, 'gitlab_logo.png') }
describe '#encode!' do
[
......
require 'spec_helper'
describe Gitlab::Git::Index, seed_helper: true do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
let(:index) { described_class.new(repository) }
before do
......
......@@ -3,7 +3,7 @@
describe Gitlab::Git::Repository, seed_helper: true do
include Gitlab::Git::EncodingHelper
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
describe "Respond to" do
subject { repository }
......@@ -14,6 +14,32 @@
it { is_expected.to respond_to(:tags) }
end
describe '#root_ref' do
context 'with gitaly disabled' do
before { allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(false) }
it 'calls #discover_default_branch' do
expect(repository).to receive(:discover_default_branch)
repository.root_ref
end
end
context 'with gitaly enabled' do
before { stub_gitaly }
it 'gets the branch name from GitalyClient' do
expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:default_branch_name)
repository.root_ref
end
it 'wraps GRPC exceptions' do
expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:default_branch_name).
and_raise(GRPC::Unknown)
expect { repository.root_ref }.to raise_error(Gitlab::Git::CommandError)
end
end
end
describe "#discover_default_branch" do
let(:master) { 'master' }
let(:feature) { 'feature' }
......@@ -55,6 +81,21 @@
end
it { is_expected.to include("master") }
it { is_expected.not_to include("branch-from-space") }
context 'with gitaly enabled' do
before { stub_gitaly }
it 'gets the branch names from GitalyClient' do
expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:branch_names)
subject
end
it 'wraps GRPC exceptions' do
expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:branch_names).
and_raise(GRPC::Unknown)
expect { subject }.to raise_error(Gitlab::Git::CommandError)
end
end
end
describe '#tag_names' do
......@@ -71,6 +112,21 @@
end
it { is_expected.to include("v1.0.0") }
it { is_expected.not_to include("v5.0.0") }
context 'with gitaly enabled' do
before { stub_gitaly }
it 'gets the tag names from GitalyClient' do
expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:tag_names)
subject
end
it 'wraps GRPC exceptions' do
expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:tag_names).
and_raise(GRPC::Unknown)
expect { subject }.to raise_error(Gitlab::Git::CommandError)
end
end
end
shared_examples 'archive check' do |extenstion|
......@@ -221,7 +277,7 @@
end
context '#submodules' do
let(:repository) { Gitlab::Git::Repository.new(TEST_REPO_PATH) }
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
context 'where repo has submodules' do
let(:submodules) { repository.submodules('master') }
......@@ -290,9 +346,9 @@
end
describe "#reset" do
change_path = File.join(TEST_NORMAL_REPO_PATH, "CHANGELOG")
untracked_path = File.join(TEST_NORMAL_REPO_PATH, "UNTRACKED")
tracked_path = File.join(TEST_NORMAL_REPO_PATH, "files", "ruby", "popen.rb")
change_path = File.join(SEED_STORAGE_PATH, TEST_NORMAL_REPO_PATH, "CHANGELOG")
untracked_path = File.join(SEED_STORAGE_PATH, TEST_NORMAL_REPO_PATH, "UNTRACKED")
tracked_path = File.join(SEED_STORAGE_PATH, TEST_NORMAL_REPO_PATH, "files", "ruby", "popen.rb")
change_text = "New changelog text"
untracked_text = "This file is untracked"
......@@ -311,7 +367,7 @@
f.write(untracked_text)
end
@normal_repo = Gitlab::Git::Repository.new(TEST_NORMAL_REPO_PATH)
@normal_repo = Gitlab::Git::Repository.new('default', TEST_NORMAL_REPO_PATH)
@normal_repo.reset("HEAD", :hard)
end
......@@ -354,7 +410,7 @@
context "-b" do
before(:all) do
@normal_repo = Gitlab::Git::Repository.new(TEST_NORMAL_REPO_PATH)
@normal_repo = Gitlab::Git::Repository.new('default', TEST_NORMAL_REPO_PATH)
@normal_repo.checkout(new_branch, { b: true }, "origin/feature")
end
......@@ -382,7 +438,7 @@
context "without -b" do
context "and specifying a nonexistent branch" do
it "should not do anything" do
normal_repo = Gitlab::Git::Repository.new(TEST_NORMAL_REPO_PATH)
normal_repo = Gitlab::Git::Repository.new('default', TEST_NORMAL_REPO_PATH)
expect { normal_repo.checkout(new_branch) }.to raise_error(Rugged::ReferenceError)
expect(normal_repo.rugged.branches[new_branch]).to be_nil
......@@ -402,7 +458,7 @@
context "and with a valid branch" do
before(:all) do
@normal_repo = Gitlab::Git::Repository.new(TEST_NORMAL_REPO_PATH)
@normal_repo = Gitlab::Git::Repository.new('default', TEST_NORMAL_REPO_PATH)
@normal_repo.rugged.branches.create("feature", "origin/feature")
@normal_repo.checkout("feature")
end
......@@ -414,13 +470,13 @@
end
it "should update the working directory" do
File.open(File.join(TEST_NORMAL_REPO_PATH, ".gitignore"), "r") do |f|
File.open(File.join(SEED_STORAGE_PATH, TEST_NORMAL_REPO_PATH, ".gitignore"), "r") do |f|
expect(f.read.each_line.to_a).not_to include(".DS_Store\n")
end
end
after(:all) do
FileUtils.rm_rf(TEST_NORMAL_REPO_PATH)
FileUtils.rm_rf(SEED_STORAGE_PATH, TEST_NORMAL_REPO_PATH)
ensure_seeds
end
end
......@@ -429,7 +485,7 @@
describe "#delete_branch" do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
@repo.delete_branch("feature")
end
......@@ -449,7 +505,7 @@
describe "#create_branch" do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
end
it "should create a new branch" do
......@@ -496,7 +552,7 @@
describe "#remote_delete" do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
@repo.remote_delete("expendable")
end
......@@ -512,7 +568,7 @@
describe "#remote_add" do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
@repo.remote_add("new_remote", SeedHelper::GITLAB_GIT_TEST_REPO_URL)
end
......@@ -528,7 +584,7 @@
describe "#remote_update" do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
@repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
end
......@@ -551,7 +607,7 @@
before(:context) do
# Add new commits so that there's a renamed file in the commit history
repo = Gitlab::Git::Repository.new(TEST_REPO_PATH).rugged
repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
commit_with_old_name = new_commit_edit_old_file(repo)
rename_commit = new_commit_move_file(repo)
......@@ -560,7 +616,7 @@
after(:context) do
# Erase our commits so other tests get the original repo
repo = Gitlab::Git::Repository.new(TEST_REPO_PATH).rugged
repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged
repo.references.update("refs/heads/master", SeedRepo::LastCommit::ID)
end
......@@ -885,7 +941,7 @@ def commit_files(commit)
describe '#autocrlf' do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
@repo.rugged.config['core.autocrlf'] = true
end
......@@ -900,14 +956,14 @@ def commit_files(commit)
describe '#autocrlf=' do
before(:all) do
@repo = Gitlab::Git::Repository.new(TEST_MUTABLE_REPO_PATH)
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH)
@repo.rugged.config['core.autocrlf'] = false
end
it 'should set the autocrlf option to the provided option' do
@repo.autocrlf = :input
File.open(File.join(TEST_MUTABLE_REPO_PATH, '.git', 'config')) do |config_file|
File.open(File.join(SEED_STORAGE_PATH, TEST_MUTABLE_REPO_PATH, '.git', 'config')) do |config_file|
expect(config_file.read).to match('autocrlf = input')
end
end
......@@ -999,7 +1055,7 @@ def commit_files(commit)
end