Commit c380c142 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Bitbucket import: removed most of the code

There aren't Mercurial repositories to import from Bitbucket
any more: our divergence with upstream on this is now just a
drag.

Removal is partial in order not to jeopardize the installation
of 0.18.0rc1 on foss.h.n (second OAuth, worker queue registration
etc.). In all cases, behaviour on such configurations will be
tested and adjusted before release.

--HG--
branch : heptapod
parent e3b221981d54
......@@ -30,80 +30,33 @@ def realtime_changes
super
end
def import_form
target_namespace = Namespace.find(params[:namespace_id].to_i)
begin
project = create_from_bb_rpath(params[:repo_id].to_s, target_namespace)
rescue StandardError => err
redirect_to(status_import_bitbucket_path, status: 303, alert: "Error: #{err}")
return
end
if project.nil?
redirect_to(status_import_bitbucket_path,
status: 303,
alert: _('This namespace has already been taken! Please choose another one.'),
)
elsif project.persisted?
redirect_to(status_import_bitbucket_path, status: 303, notice: _('Project import started'))
else
err = project_save_error(project)
Rails.logger.info("testgr Bitbucket controller project err=#{err}")
redirect_to(status_import_bitbucket_path, status: 303, alert: "Error: #{err}")
end
end
def create
repo_id = params[:repo_id].to_s
project = create_from_bb_rpath(repo_id.gsub('___', '/'))
if project.nil?
render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity
else
if project.persisted?
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
end
end
# name is the rpath in Bitbucket Cloud
def create_from_bb_rpath(name, target_namespace = nil)
bitbucket_client = Bitbucket::Client.new(credentials)
repo_id = params[:repo_id].to_s
name = repo_id.gsub('___', '/')
repo = bitbucket_client.repo(name)
project_name = params[:new_name].presence || repo.name
if target_namespace.nil?
repo_owner = repo.owner
repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
namespace_path = params[:new_namespace].presence || repo_owner
target_namespace = find_or_create_namespace(namespace_path, current_user)
end
# The token in a session can be expired, we need to get most recent one because
# Bitbucket::Connection class refreshes it.
session[:bitbucket_token] = bitbucket_client.connection.token
project = Project.where(namespace_id: target_namespace.id,
path: project_name).first
if !project.nil?
record_url = URI(project.import_url)
current_url = URI(repo.clone_url)
if record_url.host != current_url.host || record_url.path != current_url.path
raise StandardError,
"Found project '#{project_name}' in target namespace "\
"'#{target_namespace.full_path}', but its previous import URL "\
"#{record_url} doesn't match the expected #{current_url}"
end
project.create_or_update_import_data(credentials: credentials)
project.save
Rails.logger.info("Bitbucket controller: scheduling asynchronous "\
"reimport for project #{project.id}")
repo_owner = repo.owner
repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
namespace_path = params[:new_namespace].presence || repo_owner
target_namespace = find_or_create_namespace(namespace_path, current_user)
if current_user.can?(:create_projects, target_namespace)
# The token in a session can be expired, we need to get most recent one because
# Bitbucket::Connection class refreshes it.
session[:bitbucket_token] = bitbucket_client.connection.token
BitbucketReimportWorker.perform_async(project.id)
project
elsif current_user.can?(:create_projects, target_namespace)
project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
if project.persisted?
render json: ProjectSerializer.new.represent(project, serializer: :import)
else
render json: { errors: project_save_error(project) }, status: :unprocessable_entity
end
else
render json: { errors: _('This namespace has already been taken! Please choose another one.') }, status: :unprocessable_entity
end
end
......@@ -140,8 +93,7 @@ def oauth_client
end
def provider
Gitlab::Auth::OAuth::Provider.config_for('bitbucket_import',
fallback='bitbucket')
Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
end
def client
......
......@@ -18,11 +18,6 @@ def issue_comments(repo, issue_id)
get_collection(path, :comment)
end
def issue_attachments(repo, issue_id)
path = "/repositories/#{repo}/issues/#{issue_id}/attachments"
get_collection(path, :attachment)
end
def pull_requests(repo)
path = "/repositories/#{repo}/pullrequests?state=ALL"
get_collection(path, :pull_request)
......
......@@ -26,18 +26,6 @@ def get(path, extra_query = {})
response.parsed
end
# Return a response for an absolute URL
#
# This is useful for parts of the Bitbucket API (e.g, attachments) that
# provide a full URL.
#
# Returns response object
def raw_get(absolute_url, follow_redirect = true)
refresh! if expired?
opts = follow_redirect ? {} : {redirect_count: connection.client.options[:max_redirects]}
connection.get(absolute_url, opts)
end
delegate :expired?, to: :connection
def refresh!
......@@ -71,8 +59,7 @@ def root_url
end
def provider
Gitlab::Auth::OAuth::Provider.config_for('bitbucket_import',
fallback='bitbucket')
Gitlab::Auth::OAuth::Provider.config_for('bitbucket')
end
def options
......
......@@ -31,12 +31,7 @@ def next_url
end
def fetch_next_page
batching_extra = {pagelen: PAGE_LENGTH}
# TODO maybe better to have an explicit attribute
# (but it's also true that the default sorting just can't work with
# attachments)
batching_extra[:sort] = :created_on unless type == :attachment
parsed_response = connection.get(next_url, batching_extra)
parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
Page.new(parsed_response, type)
end
end
......
# frozen_string_literal: true
module Bitbucket
module Representation
class Attachment < Representation::Base
def name
raw.fetch('name')
end
def retrieval_url
# honestly, don't know why the URL should be a list. I don't
# give up easily on documentation, but Bitbucket API documentation
# is inscrutable, couldn't find a reference for attachments.
raw.fetch('links').fetch('self').fetch('href')[0]
end
end
end
end
......@@ -4,14 +4,7 @@ module Bitbucket
module Representation
class Comment < Representation::Base
def author
user&.fetch('nickname', nil)
end
def author_info
# gracinet: would be simpler to unmark user() as public
# but this way gives us more uniformity with issues and
# probably less conflicts down the road.
raw.fetch('user', nil)
user['nickname']
end
def note
......
......@@ -17,10 +17,6 @@ def author
raw.dig('reporter', 'nickname')
end
def author_info
raw.fetch('reporter', nil)
end
def description
raw.fetch('content', {}).fetch('raw', nil)
end
......
......@@ -4,11 +4,7 @@ module Bitbucket
module Representation
class PullRequest < Representation::Base
def author
raw.dig('author', 'nickname')
end
def author_info
raw.fetch('author', nil)
raw.fetch('author', {}).fetch('nickname', nil)
end
def description
......@@ -43,39 +39,19 @@ def title
end
def source_branch_name
source_branch.dig('branch', 'name')
end
def source_project_full_name
source_branch.dig('repository', 'full_name')
end
def source_project_uuid
source_branch.dig('repository', 'uuid')
end
def source_project_url
source_branch.dig('repository', 'links', 'html', 'href')
source_branch.fetch('branch', {}).fetch('name', nil)
end
def source_branch_sha
source_branch.dig('commit', 'hash')
source_branch.fetch('commit', {}).fetch('hash', nil)
end
def target_branch_name
target_branch.dig('branch', 'name')
target_branch.fetch('branch', {}).fetch('name', nil)
end
def target_branch_sha
target_branch.dig('commit', 'hash')
end
def target_project_full_name
target_branch.dig('repository', 'full_name')
end
def target_project_uuid
target_branch.dig('repository', 'uuid')
target_branch.fetch('commit', {}).fetch('hash', nil)
end
private
......
......@@ -47,12 +47,8 @@ def name
raw['name']
end
def main_branch
raw['mainbranch']
end
def valid?
raw['scm'] == 'hg'
raw['scm'] == 'git'
end
def has_wiki?
......
......@@ -10,8 +10,6 @@ class Importer
attr_reader :project, :client, :errors, :users
AttachmentError = Class.new(StandardError)
def initialize(project)
@project = project
@client = Bitbucket::Client.new(project.import_data.credentials)
......@@ -25,24 +23,12 @@ def execute
import_wiki
import_issues
import_pull_requests
import_repo_config
handle_errors
metrics.track_finished_import
true
end
def reimport
Rails.logger.info("Bitbucket incremental reimport starting for "\
"project id=#{project.id}")
import_issues
import_pull_requests
success = errors.empty?
handle_errors
success
end
private
def handle_errors
......@@ -88,25 +74,7 @@ def repo
@repo ||= client.repo(project.import_source)
end
# import repository config items that aren't stored on the Project model
def import_repo_config
branch_info = repo.main_branch
return unless branch_info['type'] == 'named_branch'
gitlab_branch = 'branch/' + branch_info['name']
begin
project.change_head(gitlab_branch)
rescue StandardError => exc
errors << { type: :global,
errors: "Setting default branch to #{gitlab_branch}, got #{exc.class.name} #{exc.message}" }
end
end
def import_wiki
# gracinet: this isn't obvious, but BitbucketImport::ProjectCreator,
# the class responsible for repo creations will creates the wiki repo
# *only if* there's no wiki on the Bitbucket side.
# Therefore the condition below effectively avoids a useless and
# doomed import. This is unchanged in Heptapod at this point.
return if project.wiki.repository_exists?
wiki = WikiFormatter.new(project)
......@@ -116,86 +84,25 @@ def import_wiki
errors << { type: :wiki, errors: e.message }
end
def author_markdown(author_info)
if author_info.nil?
return "Anonymous"
end
nick = author_info['nickname']
if nick.nil?
return "Anonymous"
end
if nick == "Former user"
# in this case author link on Bitbucket gives a useless 404
# couldn't find better identification of former users
return nick
end
display = author_info['display_name']
label =
if nick == display
nick
else
"#{nick} (#{display})"
end
link = author_info.dig('links', 'html', 'href')
if link.nil?
label
else
"[#{label}](#{link})"
end
end
def author_line(author_info)
"*Created originally on Bitbucket by #{author_markdown(author_info)}*\n\n"
end
def logprefix
"Bitbucket importer for #{project.namespace.path}/#{project.path} "\
"(project id=#{project.id})"
end
# rubocop: disable CodeReuse/ActiveRecord
def import_issues
return unless repo.issues_enabled?
Rails.logger.info("#{logprefix}: starting import of issues")
create_labels
client.issues(repo).each do |issue|
gitlab_issue = project.issues.where(iid: issue.iid).first
unless gitlab_issue.nil?
if has_attachments(gitlab_issue)
Rails.logger.info(
"#{logprefix}: skipping already imported "\
"issue #{issue.iid} having attachments on the Heptapod side")
else
Rails.logger.info(
"#{logprefix}: reimport attachments if any for "\
"issue #{issue.iid}")
reimport_issue_attachments(issue, gitlab_issue)
end
next
end
Rails.logger.info("#{logprefix}: importing issue #{issue.iid}")
import_issue(issue)
end
end
# rubocop: disable CodeReuse/ActiveRecord
def import_issue(issue)
log_issue = "#{logprefix} for issue #{issue&.iid}"
description = ''
description += author_line(issue.author_info) unless find_user_id(issue.author)
description += @formatter.author_line(issue.author) unless find_user_id(issue.author)
description += issue.description
label_name = issue.kind
milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil
description += import_link_issue_attachments(issue, log_issue)
gitlab_issue = project.issues.create!(
iid: issue.iid,
title: issue.title,
......@@ -213,9 +120,6 @@ def import_issue(issue)
import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted?
rescue StandardError => e
Rails.logger.error(
"#{log_issue}: failed with "\
"#{e.class.name}(#{e.message})\n" + e.backtrace.join("\n"))
errors << { type: :issue, iid: issue.iid, errors: e.message }
end
# rubocop: enable CodeReuse/ActiveRecord
......@@ -230,7 +134,7 @@ def import_issue_comments(issue, gitlab_issue)
next unless comment.note.present?
note = ''
note += author_line(comment.author_info) unless find_user_id(comment.author)
note += @formatter.author_line(comment.author) unless find_user_id(comment.author)
note += comment.note
begin
......@@ -247,150 +151,6 @@ def import_issue_comments(issue, gitlab_issue)
end
end
# Follow redirects unless they are not on Bitbucket
#
# Requests to the Bitbucket API must be done with the OAuth2 token,
# but, requests outside of Bitbucket (typically to their AWS S3 storage)
# must not, for two reasons:
#
# - would leak the token
# - AWS S3 doesn't permit coexistence of the `Authorization` header
# and of the auth in query string that last Bitbucket redirect provides
#
# Return [external?, response or URL], either
# - [false, response]:
# response would be a non streaming response from the
# OAuth2 library (itself relying on Faraday for HTTP)
# - [true, URL]
def follow_inner_bitbucket_redirects(url)
redir_count = 0
while redir_count < 10
response = client.connection.raw_get(url, false)
case response.status
when 301, 302, 303, 307
redir_count += 1
url = response.headers['location']
Rails.logger.info("testgr: got redirection #{response.status} to #{url}")
if url.start_with?("https://bitbucket.org")
return [false, response]
else
return [true, url]
end
else
return [false, response]
end
end
raise AttachmentError.new("Too much redirection")
end
def retrieve_attachment(url, file)
is_external, url_or_response = follow_inner_bitbucket_redirects(url)
unless is_external
file.write(url_or_response.body)
return
end
HTTParty.get(url_or_response, stream_body: true) do |fragment|
case fragment.code
when 301, 302, 303, 307
next
when 200
file.write(fragment)
else
raise AttachmentError.new("Non-success status code #{fragment.code} while streaming")
end
end
end
def import_issue_attachments(issue)
attachments = []
client.issue_attachments(repo, issue.iid).each do |att|
Rails.logger.info(
"#{logprefix}: issue #{issue.iid} "\
"processing attachment name=#{att.name} url=#{att.retrieval_url}")
uploader = FileUploader.new(@project, nil)
# uploader's filename can be set, but calling `store!` on a
# File object resets it to the basename of that file, that's
# why we're using a tmpdir and create the file directly with
# its target name
Dir.mktmpdir do |dir|
File.open(File.join(dir, att.name), 'w+b') do |file|
retrieve_attachment(att.retrieval_url, file)
file.seek(0)
uploader.store!(file)
end
end
attachments << uploader.to_h
rescue StandardError => e
Rails.logger.error("#{logprefix} failed to process attachment "\
"#{att.name} url=#{att.retrieval_url}"\
"#{e.class}: #{e}\n" + e.backtrace.join("\n"))
errors << { type: :issue_attachment,
iid: issue.iid,
errors: "#{e.class}: #{e}",
}
end
attachments
end
# Import attachments and return linking markdown to add to description
def import_link_issue_attachments(issue, log_issue)
attachments = import_issue_attachments(issue)
return "" if attachments.empty?
Rails.logger.info(
"#{log_issue}: uploaded attachments #{attachments} \
now adding Markdown links to description")
linking_md = "\n\n *Attachments:* "
attachments.each do |att|
markdown = att[:markdown]
if markdown.nil?
Rails.logger.warn(
"#{log_issue}: missing Markdown link for attachment #{att}")
next
end
# Turn off automatic renderings because chances are high
# that our rendering would not bear well with the issue
# layout in GitLab. Users can add the exclamation mark if
# they feel like it anyway.
markdown = markdown[1..] if markdown[0] == '!'
linking_md += markdown
end
linking_md
end
# Reimport attachments for an issue
def reimport_issue_attachments(bb_issue, gitlab_issue)
log_issue = "#{logprefix} reimport of attachments "\
"for issue #{bb_issue.iid}"
linking_md = import_link_issue_attachments(bb_issue, log_issue)
unless linking_md
Rails.logger.info("#{log_issue} no attachments found")
return
end
gitlab_issue.description += linking_md
gitlab_issue.save!
end
# Rough heuristic method to tell if a GitLab issue has attachments
#
# Since attachments are tied only to the project, there is no
# way to really tell, the best we can do is to look for Markdown links
# to attachments in the description.
#
# If someone has removed such links manually, it can trigger reimport
# of all attachments.
# Conversely, if only one attachment over tens did work in a
# previous run, nothing will be reimported.
#
# Only global attachments are considered.
def has_attachments(gitlab_issue)
match = /\[.*\]\(\/uploads/.match(gitlab_issue.description)
!match.nil?
end
def create_labels
LABELS.each do |label_params|
label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true)
......@@ -403,7 +163,6 @@ def create_labels
end