Commit 99ddd1dc authored by Rémy Coutable's avatar Rémy Coutable
Browse files

Modify GithubImport to support Gitea



The reason is that Gitea plan to be GitHub-compatible so it makes sense
to just modify GitHubImport a bit for now, and hopefully we can change
it to GitHubishImport once Gitea is 100%-compatible.
Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent d91401c3af6c
......@@ -11,7 +11,7 @@ def personal_access_token
end
def status
@gitea_root_url = session[:host_url]
@gitea_host_url = session[:host_url]
super
end
......
......@@ -8,8 +8,8 @@ def github_project_link(path_with_namespace)
link_to path_with_namespace, github_project_url(path_with_namespace), target: '_blank'
end
def gitea_project_link(root_url, path_with_namespace)
link_to path_with_namespace, gitea_project_url(root_url, path_with_namespace), target: '_blank'
def gitea_project_link(path_with_namespace)
link_to path_with_namespace, gitea_project_url(path_with_namespace), target: '_blank'
end
private
......@@ -25,7 +25,7 @@ def github_root_url
@github_url = provider.fetch('url', 'https://github.com') if provider
end
def gitea_project_url(root_url, path_with_namespace)
"#{root_url}/#{path_with_namespace}"
def gitea_project_url(path_with_namespace)
"#{@gitea_host_url.sub(%r{/+\z}, '')}/#{path_with_namespace}"
end
end
- page_title "Gitea import"
- page_title "Gitea Import"
- header_title "Projects", root_path
%h3.page-title
= custom_icon('go_logo')
Import projects from Gitea
Import Projects from Gitea
%p.light
Select projects you want to import.
......@@ -26,7 +26,7 @@
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td
= gitea_project_link(@gitea_root_url, project.import_source)
= gitea_project_link(project.import_source)
%td
= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
%td.job-status
......@@ -43,7 +43,7 @@
- @repos.each do |repo|
%tr{id: "repo_#{repo.id}"}
%td
= gitea_project_link(@gitea_root_url, repo.full_name)
= gitea_project_link(repo.full_name)
%td.import-target
%fieldset.row
.input-group
......
- page_title "GitHub import"
- page_title "GitHub Import"
- header_title "Projects", root_path
%h3.page-title
%i.fa.fa-github
Import projects from GitHub
= icon 'github', text: 'Import Projects from GitHub'
%p.light
Select projects you want to import.
......
---
title: New Gitea importer
merge_request: 6945
merge_request: 8116
author:
......@@ -15,6 +15,10 @@ def create!
end
end
def url
raw_data.url || ''
end
private
def gitlab_user_id(github_id)
......
......@@ -8,7 +8,7 @@ class Client
def initialize(access_token, host: nil, api_version: 'v3')
@access_token = access_token
@host = host
@host = host.to_s.sub(%r{/+\z}, '')
@api_version = api_version
if access_token
......@@ -16,10 +16,6 @@ def initialize(access_token, host: nil, api_version: 'v3')
end
end
def api_endpoint
host.present? && api_version.present? ? "#{host}/api/#{api_version}" : github_options[:site]
end
def api
@api ||= ::Octokit::Client.new(
access_token: access_token,
......@@ -27,7 +23,7 @@ def api
# If there is no config, we're connecting to github.com and we
# should verify ssl.
connection_options: {
ssl: { verify: config ? config['verify_ssl'] : false }
ssl: { verify: config ? config['verify_ssl'] : true }
}
)
end
......@@ -70,6 +66,14 @@ def respond_to?(method)
private
def api_endpoint
if host.present? && api_version.present?
"#{host}/api/#{api_version}"
else
github_options[:site]
end
end
def config
Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" }
end
......
......@@ -3,7 +3,7 @@ module GithubImport
class Importer
include Gitlab::ShellAdapter
attr_reader :client, :errors, :project, :repo, :repo_url
attr_reader :errors, :project, :repo, :repo_url
def initialize(project)
@project = project
......@@ -11,12 +11,27 @@ def initialize(project)
@repo_url = project.import_url
@errors = []
@labels = {}
end
def client
return @client if defined?(@client)
unless credentials
raise Projects::ImportService::Error,
"Unable to find project import data credentials for project ID: #{@project.id}"
end
if credentials
@client = Client.new(credentials[:user])
else
raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
opts = {}
# Gitea plan to be GitHub compliant
if project.import_type == 'gitea'
uri = URI.parse(project.import_url)
host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '')
opts = {
host: host,
api_version: 'v1'
}
end
@client = Client.new(credentials[:user], opts)
end
def execute
......@@ -35,7 +50,13 @@ def execute
import_comments(:issues)
import_comments(:pull_requests)
import_wiki
import_releases
# Gitea doesn't have a Release API yet
# See https://github.com/go-gitea/gitea/issues/330
unless project.import_type == 'gitea'
import_releases
end
handle_errors
true
......@@ -44,7 +65,9 @@ def execute
private
def credentials
@credentials ||= project.import_data.credentials if project.import_data
return @credentials if defined?(@credentials)
@credentials = project.import_data ? project.import_data.credentials : nil
end
def handle_errors
......@@ -60,9 +83,10 @@ def import_labels
fetch_resources(:labels, repo, per_page: 100) do |labels|
labels.each do |raw|
begin
GithubImport::LabelFormatter.new(project, raw).create!
gh_label = LabelFormatter.new(project, raw)
gh_label.create!
rescue => e
errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message }
end
end
end
......@@ -74,9 +98,10 @@ def import_milestones
fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
milestones.each do |raw|
begin
GithubImport::MilestoneFormatter.new(project, raw).create!
gh_milestone = MilestoneFormatter.new(project, raw)
gh_milestone.create!
rescue => e
errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message }
end
end
end
......@@ -85,7 +110,7 @@ def import_milestones
def import_issues
fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
issues.each do |raw|
gh_issue = GithubImport::IssueFormatter.new(project, raw)
gh_issue = IssueFormatter.new(project, raw)
begin
issuable =
......@@ -97,7 +122,7 @@ def import_issues
apply_labels(issuable, raw)
rescue => e
errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message }
end
end
end
......@@ -106,18 +131,23 @@ def import_issues
def import_pull_requests
fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
pull_requests.each do |raw|
pull_request = GithubImport::PullRequestFormatter.new(project, raw)
next unless pull_request.valid?
gh_pull_request = PullRequestFormatter.new(project, raw)
next unless gh_pull_request.valid?
begin
restore_source_branch(pull_request) unless pull_request.source_branch_exists?
restore_target_branch(pull_request) unless pull_request.target_branch_exists?
restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists?
restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists?
merge_request = gh_pull_request.create!
pull_request.create!
# Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage
if project.import_type == 'gitea'
apply_labels(merge_request, raw)
end
rescue => e
errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message }
errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message }
ensure
clean_up_restored_branches(pull_request)
clean_up_restored_branches(gh_pull_request)
end
end
end
......@@ -179,7 +209,7 @@ def create_comments(comments)
ActiveRecord::Base.no_touching do
comments.each do |raw|
begin
comment = GithubImport::CommentFormatter.new(project, raw)
comment = CommentFormatter.new(project, raw)
# GH does not return info about comment's parent, so we guess it by checking its URL!
*_, parent, iid = URI(raw.html_url).path.split('/')
issuable_class = parent == 'issues' ? Issue : MergeRequest
......@@ -198,7 +228,7 @@ def discard_inserted_comments(comments, last_note)
last_note_attrs = nil
cut_off_index = comments.find_index do |raw|
comment = GithubImport::CommentFormatter.new(project, raw)
comment = CommentFormatter.new(project, raw)
comment_attrs = comment.attributes
last_note_attrs ||= last_note.slice(*comment_attrs.keys)
......@@ -214,7 +244,7 @@ def discard_inserted_comments(comments, last_note)
def import_wiki
unless project.wiki.repository_exists?
wiki = GithubImport::WikiFormatter.new(project)
wiki = WikiFormatter.new(project)
gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
end
rescue Gitlab::Shell::Error => e
......@@ -230,10 +260,10 @@ def import_releases
fetch_resources(:releases, repo, per_page: 100) do |releases|
releases.each do |raw|
begin
gh_release = GithubImport::ReleaseFormatter.new(project, raw)
gh_release = ReleaseFormatter.new(project, raw)
gh_release.create! if gh_release.valid?
rescue => e
errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message }
end
end
end
......
module Gitlab
module GithubImport
class IssuableFormatter < BaseFormatter
def project_association
raise NotImplementedError
end
def number
raw_data.number
end
def find_condition
{ iid: number }
end
private
def state
raw_data.state == 'closed' ? 'closed' : 'opened'
end
def assigned?
raw_data.assignee.present?
end
def assignee_id
if assigned?
gitlab_user_id(raw_data.assignee.id)
end
end
def author
raw_data.user.login
end
def author_id
gitlab_author_id || project.creator_id
end
def body
raw_data.body || ""
end
def description
if gitlab_author_id
body
else
formatter.author_line(author) + body
end
end
def milestone
if raw_data.milestone.present?
milestone = MilestoneFormatter.new(project, raw_data.milestone)
project.milestones.find_by(milestone.find_condition)
end
end
end
end
end
module Gitlab
module GithubImport
class IssueFormatter < BaseFormatter
class IssueFormatter < IssuableFormatter
def attributes
{
iid: number,
......@@ -24,59 +24,9 @@ def project_association
:issues
end
def find_condition
{ iid: number }
end
def number
raw_data.number
end
def pull_request?
raw_data.pull_request.present?
end
private
def assigned?
raw_data.assignee.present?
end
def assignee_id
if assigned?
gitlab_user_id(raw_data.assignee.id)
end
end
def author
raw_data.user.login
end
def author_id
gitlab_author_id || project.creator_id
end
def body
raw_data.body || ""
end
def description
if gitlab_author_id
body
else
formatter.author_line(author) + body
end
end
def milestone
if raw_data.milestone.present?
project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr))
end
end
def state
raw_data.state == 'closed' ? 'closed' : 'opened'
end
end
end
end
......@@ -3,7 +3,7 @@ module GithubImport
class MilestoneFormatter < BaseFormatter
def attributes
{
iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr),
iid: number,
project: project,
title: raw_data.title,
description: raw_data.description,
......@@ -19,11 +19,15 @@ def project_association
end
def find_condition
{ iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr) }
{ iid: number }
end
def self.iid_attr
:number
def number
if project.import_type == 'gitea'
raw_data.id
else
raw_data.number
end
end
private
......
module Gitlab
module GithubImport
class PullRequestFormatter < BaseFormatter
class PullRequestFormatter < IssuableFormatter
delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true
delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true
......@@ -28,14 +28,6 @@ def project_association
:merge_requests
end
def find_condition
{ iid: number }
end
def number
raw_data.number
end
def valid?
source_branch.valid? && target_branch.valid?
end
......@@ -60,57 +52,15 @@ def target_branch_name
end
end
def url
raw_data.url
end
private
def assigned?
raw_data.assignee.present?
end
def assignee_id
if assigned?
gitlab_user_id(raw_data.assignee.id)
end
end
def author
raw_data.user.login
end
def author_id
gitlab_author_id || project.creator_id
end
def body
raw_data.body || ""
end
def description
if gitlab_author_id
body
def state
if raw_data.state == 'closed' && raw_data.merged_at.present?
'merged'
else
formatter.author_line(author) + body
super
end
end
def milestone
if raw_data.milestone.present?
project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr))
end
end
def state
@state ||= if raw_data.state == 'closed' && raw_data.merged_at.present?
'merged'
elsif raw_data.state == 'closed'
'closed'
else
'opened'
end
end
end
end
end
......@@ -29,7 +29,7 @@ def assign_host_url
before do
assign_host_url
end
let(:extra_assign_expectations) { { gitea_root_url: host_url } }
let(:extra_assign_expectations) { { gitea_host_url: host_url } }
end
end
......
......@@ -45,20 +45,46 @@
end
end
context 'when provider does not specity an API endpoint' do
it 'uses GitHub root API endpoint' do
expect(client.api.api_endpoint).to eq 'https://api.github.com/'
describe '#api_endpoint' do
context 'when provider does not specity an API endpoint' do
it 'uses GitHub root API endpoint' do
expect(client.api.api_endpoint).to eq 'https://api.github.com/'
end
end
end
context 'when provider specify a custom API endpoint' do
before do
github_provider['args']['client_options']['site'] = 'https://github.company.com/'
context 'when provider specify a custom API endpoint' do
before do
github_provider['args']['client_options']['site'] = 'https://github.company.com/'
end
it 'uses the custom API endpoint' do
expect(OmniAuth::Strategies::GitHub).not_to receive(:default_options)
expect(client.api.api_endpoint).to eq 'https://github.company.com/'
end
end
context 'when given a host' do
subject(:client) { described_class.new(token, host: 'https://try.gitea.io/') }
it 'builds a endpoint with the given host and the default API version' do
expect(client.api.api_endpoint).to eq 'https://try.gitea.io/api/v3/'
end
end
it 'uses the custom API endpoint' do
expect(OmniAuth::Strategies::GitHub).not_to receive(:default_options)
expect(client.api.api_endpoint).to eq 'https://github.company.com/'
context 'when given an API version' do
subject(:client) { described_class.new(token, api_version: 'v3') }
it 'does not use the API version without a host' do
expect(client.api.api_endpoint).to eq 'https://api.github.com/'
end
end