Commit c380c142 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Bitbucket import: removed most of the code

There aren't Mercurial repositories to import from Bitbucket
any more: our divergence with upstream on this is now just a
drag.

Removal is partial in order not to jeopardize the installation
of 0.18.0rc1 on foss.h.n (second OAuth, worker queue registration
etc.). In all cases, behaviour on such configurations will be
tested and adjusted before release.

--HG--
branch : heptapod
parent e3b221981d54
...@@ -30,80 +30,33 @@ def realtime_changes ...@@ -30,80 +30,33 @@ def realtime_changes
super super
end end
def import_form
target_namespace = Namespace.find(params[:namespace_id].to_i)
begin
project = create_from_bb_rpath(params[:repo_id].to_s, target_namespace)
rescue StandardError => err
redirect_to(status_import_bitbucket_path, status: 303, alert: "Error: #{err}")
return
end
if project.nil?
redirect_to(status_import_bitbucket_path,
status: 303,
alert: _('This namespace has already been taken! Please choose another one.'),
)
elsif project.persisted?
redirect_to(status_import_bitbucket_path, status: 303, notice: _('Project import started'))
else
err = project_save_error(project)
Rails.logger.info("testgr Bitbucket controller project err=#{err}")
redirect_to(status_import_bitbucket_path, status: 303, alert: "Error: #{err}")
end
end
def create def create
repo_id = params[:repo_id].to_s
project = create_from_bb_rpath(repo_id.gsub('___', '/'))
if project.nil?
render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity
else
if project.persisted?
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
end
end
# name is the rpath in Bitbucket Cloud
def create_from_bb_rpath(name, target_namespace = nil)
bitbucket_client = Bitbucket::Client.new(credentials) bitbucket_client = Bitbucket::Client.new(credentials)
repo_id = params[:repo_id].to_s
name = repo_id.gsub('___', '/')
repo = bitbucket_client.repo(name) repo = bitbucket_client.repo(name)
project_name = params[:new_name].presence || repo.name project_name = params[:new_name].presence || repo.name
if target_namespace.nil?
repo_owner = repo.owner
repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
namespace_path = params[:new_namespace].presence || repo_owner
target_namespace = find_or_create_namespace(namespace_path, current_user)
end
# The token in a session can be expired, we need to get most recent one because repo_owner = repo.owner
# Bitbucket::Connection class refreshes it. repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
session[:bitbucket_token] = bitbucket_client.connection.token namespace_path = params[:new_namespace].presence || repo_owner
project = Project.where(namespace_id: target_namespace.id, target_namespace = find_or_create_namespace(namespace_path, current_user)
path: project_name).first
if !project.nil? if current_user.can?(:create_projects, target_namespace)
record_url = URI(project.import_url) # The token in a session can be expired, we need to get most recent one because
current_url = URI(repo.clone_url) # Bitbucket::Connection class refreshes it.
if record_url.host != current_url.host || record_url.path != current_url.path session[:bitbucket_token] = bitbucket_client.connection.token
raise StandardError,
"Found project '#{project_name}' in target namespace "\
"'#{target_namespace.full_path}', but its previous import URL "\
"#{record_url} doesn't match the expected #{current_url}"
end
project.create_or_update_import_data(credentials: credentials)
project.save
Rails.logger.info("Bitbucket controller: scheduling asynchronous "\
"reimport for project #{project.id}")
BitbucketReimportWorker.perform_async(project.id) project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
project
elsif current_user.can?(:create_projects, target_namespace)
Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute if project.persisted?
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
else
render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity
end end
end end
...@@ -140,8 +93,7 @@ def oauth_client ...@@ -140,8 +93,7 @@ def oauth_client
end end
def provider def provider
Gitlab::Auth::OAuth::Provider.config_for('bitbucket_import', Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
fallback='bitbucket')
end end
def client def client
......
...@@ -18,11 +18,6 @@ def issue_comments(repo, issue_id) ...@@ -18,11 +18,6 @@ def issue_comments(repo, issue_id)
get_collection(path, :comment) get_collection(path, :comment)
end end
def issue_attachments(repo, issue_id)
path = "/repositories/#{repo}/issues/#{issue_id}/attachments"
get_collection(path, :attachment)
end
def pull_requests(repo) def pull_requests(repo)
path = "/repositories/#{repo}/pullrequests?state=ALL" path = "/repositories/#{repo}/pullrequests?state=ALL"
get_collection(path, :pull_request) get_collection(path, :pull_request)
......
...@@ -26,18 +26,6 @@ def get(path, extra_query = {}) ...@@ -26,18 +26,6 @@ def get(path, extra_query = {})
response.parsed response.parsed
end end
# Return a response for an absolute URL
#
# This is useful for parts of the Bitbucket API (e.g, attachments) that
# provide a full URL.
#
# Returns response object
def raw_get(absolute_url, follow_redirect = true)
refresh! if expired?
opts = follow_redirect ? {} : {redirect_count: connection.client.options[:max_redirects]}
connection.get(absolute_url, opts)
end
delegate :expired?, to: :connection delegate :expired?, to: :connection
def refresh! def refresh!
...@@ -71,8 +59,7 @@ def root_url ...@@ -71,8 +59,7 @@ def root_url
end end
def provider def provider
Gitlab::Auth::OAuth::Provider.config_for('bitbucket_import', Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
fallback='bitbucket')
end end
def options def options
......
...@@ -31,12 +31,7 @@ def next_url ...@@ -31,12 +31,7 @@ def next_url
end end
def fetch_next_page def fetch_next_page
batching_extra = {pagelen: PAGE_LENGTH} parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
# TODO maybe better to have an explicit attribute
# (but it's also true that the default sorting just can't work with
# attachments)
batching_extra[:sort] = :created_on unless type == :attachment
parsed_response = connection.get(next_url, batching_extra)
Page.new(parsed_response, type) Page.new(parsed_response, type)
end end
end end
......
# frozen_string_literal: true
module Bitbucket
module Representation
class Attachment < Representation::Base
def name
raw.fetch('name')
end
def retrieval_url
# honestly, don't know why the URL should be a list. I don't
# give up easily on documentation, but Bitbucket API documentation
# is inscrutable, couldn't find a reference for attachments.
raw.fetch('links').fetch('self').fetch('href')[0]
end
end
end
end
...@@ -4,14 +4,7 @@ module Bitbucket ...@@ -4,14 +4,7 @@ module Bitbucket
module Representation module Representation
class Comment < Representation::Base class Comment < Representation::Base
def author def author
user&.fetch('nickname', nil) user['nickname']
end
def author_info
# gracinet: would be simpler to unmark user() as public
# but this way gives us more uniformity with issues and
# probably less conflicts down the road.
raw.fetch('user', nil)
end end
def note def note
......
...@@ -17,10 +17,6 @@ def author ...@@ -17,10 +17,6 @@ def author
raw.dig('reporter', 'nickname') raw.dig('reporter', 'nickname')
end end
def author_info
raw.fetch('reporter', nil)
end
def description def description
raw.fetch('content', {}).fetch('raw', nil) raw.fetch('content', {}).fetch('raw', nil)
end end
......
...@@ -4,11 +4,7 @@ module Bitbucket ...@@ -4,11 +4,7 @@ module Bitbucket
module Representation module Representation
class PullRequest < Representation::Base class PullRequest < Representation::Base
def author def author
raw.dig('author', 'nickname') raw.fetch('author', {}).fetch('nickname', nil)
end
def author_info
raw.fetch('author', nil)
end end
def description def description
...@@ -43,39 +39,19 @@ def title ...@@ -43,39 +39,19 @@ def title
end end
def source_branch_name def source_branch_name
source_branch.dig('branch', 'name') source_branch.fetch('branch', {}).fetch('name', nil)
end
def source_project_full_name
source_branch.dig('repository', 'full_name')
end
def source_project_uuid
source_branch.dig('repository', 'uuid')
end
def source_project_url
source_branch.dig('repository', 'links', 'html', 'href')
end end
def source_branch_sha def source_branch_sha
source_branch.dig('commit', 'hash') source_branch.fetch('commit', {}).fetch('hash', nil)
end end
def target_branch_name def target_branch_name
target_branch.dig('branch', 'name') target_branch.fetch('branch', {}).fetch('name', nil)
end end
def target_branch_sha def target_branch_sha
target_branch.dig('commit', 'hash') target_branch.fetch('commit', {}).fetch('hash', nil)
end
def target_project_full_name
target_branch.dig('repository', 'full_name')
end
def target_project_uuid
target_branch.dig('repository', 'uuid')
end end
private private
......
...@@ -47,12 +47,8 @@ def name ...@@ -47,12 +47,8 @@ def name
raw['name'] raw['name']
end end
def main_branch
raw['mainbranch']
end
def valid? def valid?
raw['scm'] == 'hg' raw['scm'] == 'git'
end end
def has_wiki? def has_wiki?
......
...@@ -10,8 +10,6 @@ class Importer ...@@ -10,8 +10,6 @@ class Importer
attr_reader :project, :client, :errors, :users attr_reader :project, :client, :errors, :users
AttachmentError = Class.new(StandardError)
def initialize(project) def initialize(project)
@project = project @project = project
@client = Bitbucket::Client.new(project.import_data.credentials) @client = Bitbucket::Client.new(project.import_data.credentials)
...@@ -25,24 +23,12 @@ def execute ...@@ -25,24 +23,12 @@ def execute
import_wiki import_wiki
import_issues import_issues
import_pull_requests import_pull_requests
import_repo_config
handle_errors handle_errors
metrics.track_finished_import metrics.track_finished_import
true true
end end
def reimport
Rails.logger.info("Bitbucket incremental reimport starting for "\
"project id=#{project.id}")
import_issues
import_pull_requests
success = errors.empty?
handle_errors
success
end
private private
def handle_errors def handle_errors
...@@ -88,25 +74,7 @@ def repo ...@@ -88,25 +74,7 @@ def repo
@repo ||= client.repo(project.import_source) @repo ||= client.repo(project.import_source)
end end
# import repository config items that aren't stored on the Project model
def import_repo_config
branch_info = repo.main_branch
return unless branch_info['type'] == 'named_branch'
gitlab_branch = 'branch/' + branch_info['name']
begin
project.change_head(gitlab_branch)
rescue StandardError => exc
errors << { type: :global,
errors: "Setting default branch to #{gitlab_branch}, got #{exc.class.name} #{exc.message}" }
end
end
def import_wiki def import_wiki
# gracinet: this isn't obvious, but BitbucketImport::ProjectCreator,
# the class responsible for repo creations will creates the wiki repo
# *only if* there's no wiki on the Bitbucket side.
# Therefore the condition below effectively avoids a useless and
# doomed import. This is unchanged in Heptapod at this point.
return if project.wiki.repository_exists? return if project.wiki.repository_exists?
wiki = WikiFormatter.new(project) wiki = WikiFormatter.new(project)
...@@ -116,86 +84,25 @@ def import_wiki ...@@ -116,86 +84,25 @@ def import_wiki
errors << { type: :wiki, errors: e.message } errors << { type: :wiki, errors: e.message }
end end
def author_markdown(author_info)
if author_info.nil?
return "Anonymous"
end
nick = author_info['nickname']
if nick.nil?
return "Anonymous"
end
if nick == "Former user"
# in this case author link on Bitbucket gives a useless 404
# couldn't find better identification of former users
return nick
end
display = author_info['display_name']
label =
if nick == display
nick
else
"#{nick} (#{display})"
end
link = author_info.dig('links', 'html', 'href')
if link.nil?
label
else
"[#{label}](#{link})"
end
end
def author_line(author_info)
"*Created originally on Bitbucket by #{author_markdown(author_info)}*\n\n"
end
def logprefix
"Bitbucket importer for #{project.namespace.path}/#{project.path} "\
"(project id=#{project.id})"
end
# rubocop: disable CodeReuse/ActiveRecord
def import_issues def import_issues
return unless repo.issues_enabled? return unless repo.issues_enabled?
Rails.logger.info("#{logprefix}: starting import of issues")
create_labels create_labels
client.issues(repo).each do |issue| client.issues(repo).each do |issue|
gitlab_issue = project.issues.where(iid: issue.iid).first
unless gitlab_issue.nil?
if has_attachments(gitlab_issue)
Rails.logger.info(
"#{logprefix}: skipping already imported "\
"issue #{issue.iid} having attachments on the Heptapod side")
else
Rails.logger.info(
"#{logprefix}: reimport attachments if any for "\
"issue #{issue.iid}")
reimport_issue_attachments(issue, gitlab_issue)
end
next
end
Rails.logger.info("#{logprefix}: importing issue #{issue.iid}")
import_issue(issue) import_issue(issue)
end end
end end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def import_issue(issue) def import_issue(issue)
log_issue = "#{logprefix} for issue #{issue&.iid}"
description = '' description = ''
description += author_line(issue.author_info) unless find_user_id(issue.author) description += @formatter.author_line(issue.author) unless find_user_id(issue.author)
description += issue.description description += issue.description
label_name = issue.kind label_name = issue.kind
milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil
description += import_link_issue_attachments(issue, log_issue)
gitlab_issue = project.issues.create!( gitlab_issue = project.issues.create!(
iid: issue.iid, iid: issue.iid,
title: issue.title, title: issue.title,
...@@ -213,9 +120,6 @@ def import_issue(issue) ...@@ -213,9 +120,6 @@ def import_issue(issue)
import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted?
rescue StandardError => e rescue StandardError => e
Rails.logger.error(
"#{log_issue}: failed with "\
"#{e.class.name}(#{e.message})\n" + e.backtrace.join("\n"))
errors << { type: :issue, iid: issue.iid, errors: e.message } errors << { type: :issue, iid: issue.iid, errors: e.message }
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
...@@ -230,7 +134,7 @@ def import_issue_comments(issue, gitlab_issue) ...@@ -230,7 +134,7 @@ def import_issue_comments(issue, gitlab_issue)
next unless comment.note.present? next unless comment.note.present?
note = '' note = ''
note += author_line(comment.author_info) unless find_user_id(comment.author) note += @formatter.author_line(comment.author) unless find_user_id(comment.author)
note += comment.note note += comment.note
begin begin
...@@ -247,150 +151,6 @@ def import_issue_comments(issue, gitlab_issue) ...@@ -247,150 +151,6 @@ def import_issue_comments(issue, gitlab_issue)
end end
end end
# Follow redirects unless they are not on Bitbucket
#
# Requests to the Bitbucket API must be done with the OAuth2 token,
# but, requests outside of Bitbucket (typically to their AWS S3 storage)
# must not, for two reasons:
#
# - would leak the token
# - AWS S3 doesn't permit coexistence of the `Authorization` header
# and of the auth in query string that last Bitbucket redirect provides
#
# Return [external?, response or URL], either
# - [false, response]:
# response would be a non streaming response from the
# OAuth2 library (itself relying on Faraday for HTTP)
# - [true, URL]
def follow_inner_bitbucket_redirects(url)
redir_count = 0
while redir_count < 10
response = client.connection.raw_get(url, false)
case response.status
when 301, 302, 303, 307
redir_count += 1
url = response.headers['location']
Rails.logger.info("testgr: got redirection #{response.status} to #{url}")
if url.start_with?("https://bitbucket.org")
return [false, response]
else
return [true, url]
end
else
return [false, response]
end
end
raise AttachmentError.new("Too much redirection")
end
def retrieve_attachment(url, file)