Commit 7b5bc32c authored by Douwe Maan's avatar Douwe Maan
Browse files

Allow projects to be imported from Google Code.

parent a1697424474c
......@@ -119,6 +119,22 @@
li {
line-height: 1.5;
}
a[href*="/uploads/"], a[href*="storage.googleapis.com/google-code-attachments/"] {
&:before {
margin-right: 4px;
font: normal normal normal 14px/1 FontAwesome;
font-size: inherit;
text-rendering: auto;
-webkit-font-smoothing: antialiased;
content: "\f0c6";
}
&:hover:before {
text-decoration: none;
}
}
}
@mixin str-truncated($max_width: 82%) {
......
......@@ -62,20 +62,8 @@ ul.notes {
word-wrap: break-word;
@include md-typography;
a[href*="/uploads/"] {
&:before {
margin-right: 4px;
font: normal normal normal 14px/1 FontAwesome;
font-size: inherit;
text-rendering: auto;
-webkit-font-smoothing: antialiased;
content: "\f0c6";
}
&:hover:before {
text-decoration: none;
}
hr {
margin: 10px 0;
}
}
}
......
class Import::GoogleCodeController < Import::BaseController
def new
end
def callback
dump_file = params[:dump_file]
unless dump_file.respond_to?(:read)
return redirect_to :back, alert: "You need to upload a Google Takeout JSON file."
end
begin
dump = JSON.parse(dump_file.read)
rescue
return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout JSON file."
end
unless Gitlab::GoogleCodeImport::Client.new(dump).valid?
return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout JSON file."
end
session[:google_code_dump] = dump
redirect_to status_import_google_code_path
end
def status
unless client.valid?
return redirect_to new_import_google_path
end
@repos = client.repos
@already_added_projects = current_user.created_projects.where(import_type: "google_code")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include? repo.name }
end
def jobs
jobs = current_user.created_projects.where(import_type: "google_code").to_json(only: [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id]
repo = client.repo(@repo_id)
@target_namespace = current_user.namespace
@project_name = repo.name
namespace = @target_namespace
@project = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::GoogleCodeImport::Client.new(session[:google_code_dump])
end
end
......@@ -27,6 +27,7 @@
# import_type :string(255)
# import_source :string(255)
# avatar :string(255)
# import_data :text
#
require 'carrierwave/orm/activerecord'
......@@ -50,6 +51,8 @@ class Project < ActiveRecord::Base
default_value_for :wall_enabled, false
default_value_for :snippets_enabled, gitlab_config_features.snippets
serialize :import_data, JSON
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
......@@ -185,6 +188,7 @@ def set_last_activity_at
state :failed
after_transition any => :started, do: :add_import_job
after_transition any => :finished, do: :clear_import_data
end
class << self
......@@ -262,6 +266,11 @@ def add_import_job
RepositoryImportWorker.perform_in(2.seconds, id)
end
def clear_import_data
self.import_data = nil
self.save
end
def import?
import_url.present?
end
......
%h3.page-title
%i.fa.fa-google
Import projects from Google Code
%hr
= form_tag callback_import_google_code_path, class: 'form-horizontal', multipart: true do
%ul
%li
Use Google Takeout etc
.form-group
= label_tag :dump_file, "Google Takeout JSON file", class: 'control-label'
.col-sm-10
%input{type: "file", name: "dump_file", id: "dump_file"}
.form-actions
= submit_tag 'Select projects to import', class: "btn btn-create"
%h3.page-title
%i.fa.fa-google
Import projects from Google Code
%p.light
Select projects you want to import.
%hr
%p
= button_tag 'Import all projects', class: "btn btn-success js-import-all"
%table.table.import-jobs
%thead
%tr
%th From Google Code
%th To GitLab
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td
= link_to project.import_source, "https://code.google.com/p/#{project.import_source}", target: "_blank"
%td
%strong= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
%td.job-status
- if project.import_status == 'finished'
%span
%i.fa.fa-check
done
- elsif project.import_status == 'started'
%i.fa.fa-spinner.fa-spin
started
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo.id}"}
%td
= link_to repo.name, "https://code.google.com/p/#{repo.name}", target: "_blank"
%td.import-target
= "#{current_user.username}/#{repo.name}"
%td.import-actions.job-status
= button_tag "Import", class: "btn js-add-to-import"
:coffeescript
new ImporterStatus("#{jobs_import_google_code_path}", "#{import_google_code_path}")
......@@ -62,6 +62,10 @@
%i.icon-gitorious.icon-gitorious-small
Gitorious.org
= link_to new_import_google_code_path, class: 'btn' do
%i.fa.fa-google
Google Code
= link_to "#", class: 'btn js-toggle-button' do
%i.fa.fa-git
%span Any repo by URL
......
......@@ -18,6 +18,8 @@ def perform(project_id)
Gitlab::GitlabImport::Importer.new(project).execute
elsif project.import_type == 'bitbucket'
Gitlab::BitbucketImport::Importer.new(project).execute
elsif project.import_type == 'google_code'
Gitlab::GoogleCodeImport::Importer.new(project).execute
else
true
end
......
......@@ -81,6 +81,12 @@
get :callback
get :jobs
end
resource :google_code, only: [:create, :new], controller: :google_code do
get :status
post :callback
get :jobs
end
end
#
......
class AddImportDataToProject < ActiveRecord::Migration
def change
add_column :projects, :import_data, :text
end
end
......@@ -342,6 +342,7 @@
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.text "import_data"
end
add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree
......
module Gitlab
module GoogleCodeImport
class Client
attr_reader :raw_data
def initialize(raw_data)
@raw_data = raw_data
end
def valid?
raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects")
end
def repos
@repos ||= raw_data["projects"].map { |raw_repo| GoogleCodeImport::Repository.new(raw_repo) }.select(&:git?)
end
def repo(id)
repos.find { |repo| repo.id == id }
end
end
end
end
module Gitlab
module GoogleCodeImport
class Importer
attr_reader :project, :repo
def initialize(project)
@project = project
@repo = GoogleCodeImport::Repository.new(project.import_data)
@closed_statuses = []
@known_labels = Set.new
end
def execute
return true unless repo.valid?
import_status_labels
import_labels
import_issues
true
end
private
def import_status_labels
repo.raw_data["issuesConfig"]["statuses"].each do |status|
closed = !status["meansOpen"]
@closed_statuses << status["status"] if closed
name = nice_status_name(status["status"])
create_label(name)
@known_labels << name
end
end
def import_labels
repo.raw_data["issuesConfig"]["labels"].each do |label|
name = nice_label_name(label["label"])
create_label(name)
@known_labels << name
end
end
def import_issues
return unless repo.raw_data["issues"]
last_id = 0
deleted_issues = []
issues = repo.raw_data["issues"]["items"]
issues.each_with_index do |raw_issue, i|
while raw_issue["id"] > last_id + 1
last_id += 1
issue = project.issues.create!(
title: "Deleted issue",
description: "*This issue has been deleted*",
author_id: project.creator_id,
state: "closed"
)
deleted_issues << issue
end
last_id = raw_issue["id"]
author = mask_email(raw_issue["author"]["name"])
author_link = raw_issue["author"]["htmlLink"]
date = DateTime.parse(raw_issue["published"]).to_formatted_s(:long)
body = []
body << "*By [#{author}](#{author_link}) on #{date}*"
body << "---"
comments = raw_issue["comments"]["items"]
issue_comment = comments.shift
content = format_content(issue_comment["content"])
if content.blank?
content = "*(No description has been entered for this issue)*"
end
body << content
attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"])
if attachments.any?
body << "---"
body += attachments
end
labels = []
raw_issue["labels"].each do |label|
name = nice_label_name(label)
labels << name
unless @known_labels.include?(name)
create_label(name)
@known_labels << name
end
end
labels << nice_status_name(raw_issue["status"])
issue = project.issues.create!(
title: raw_issue["title"],
description: body.join("\n\n"),
author_id: project.creator_id,
state: raw_issue["state"] == "closed" ? "closed" : "opened"
)
issue.add_labels_by_names(labels)
import_issue_comments(issue, comments)
end
deleted_issues.each(&:destroy!)
end
def import_issue_comments(issue, comments)
comments.each_with_index do |raw_comment, i|
next if raw_comment.has_key?("deletedBy")
author = mask_email(raw_comment["author"]["name"])
author_link = raw_comment["author"]["htmlLink"]
date = DateTime.parse(raw_comment["published"]).to_formatted_s(:long)
body = []
body << "*By [#{author}](#{author_link}) on #{date}*"
body << "---"
content = format_content(raw_comment["content"])
if content.blank?
content = "*(No comment has been entered for this change)*"
end
body << content
updates = format_updates(raw_comment["updates"])
if updates.any?
body << "---"
body += updates
end
attachments = format_attachments(issue.iid, raw_comment["id"], raw_comment["attachments"])
if attachments.any?
body << "---"
body += attachments
end
comment = issue.notes.create!(
project_id: project.id,
author_id: project.creator_id,
note: body.join("\n\n")
)
end
end
def nice_label_color(name)
case name
when /\AComponent:/
"#fff39e"
when /\AOpSys:/
"#e2e2e2"
when /\AMilestone:/
"#fee3ff"
when *@closed_statuses.map { |s| nice_status_name(s) }
"#cfcfcf"
when "Status: New"
"#428bca"
when "Status: Accepted"
"#5cb85c"
when "Status: NeedInfo"
"#f0ad4e"
when "Status: Started"
"#8e44ad"
when "Status: Wishlist"
"#a8d695"
#
when "Priority: Critical"
"#ffcfcf"
when "Priority: High"
"#deffcf"
when "Priority: Medium"
"#fff5cc"
when "Priority: Low"
"#cfe9ff"
#
when "Type: Defect"
"#d9534f"
when "Type: Enhancement"
"#44ad8e"
when "Type: Other"
"#7f8c8d"
when "Type: Review"
"#8e44ad"
when "Type: Task"
"#4b6dd0"
else
"#e2e2e2"
end
end
def nice_label_name(name)
name.sub("-", ": ")
end
def nice_status_name(name)
"Status: #{name}"
end
def mask_email(author)
parts = author.split("@", 2)
parts[0] = "#{parts[0][0...-3]}..."
parts.join("@")
end
def linkify_issues(s)
s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
end
def escape_for_markdown(s)
s = s.gsub("*", "\\*")
s = s.gsub("#", "\\#")
s = s.gsub("`", "\\`")
s = s.gsub(":", "\\:")
s = s.gsub("-", "\\-")
s = s.gsub("+", "\\+")
s = s.gsub("_", "\\_")
s = s.gsub("(", "\\(")
s = s.gsub(")", "\\)")
s = s.gsub("[", "\\[")
s = s.gsub("]", "\\]")
s = s.gsub("<", "\\<")
s = s.gsub(">", "\\>")
s = s.gsub("\r", "")
s = s.gsub("\n", " \n")
s
end
def create_label(name)
color = nice_label_color(name)
project.labels.create!(name: name, color: color)
end
def format_content(raw_content)
linkify_issues(escape_for_markdown(raw_content))
end
def format_updates(raw_updates)
updates = []
if raw_updates.has_key?("status")
updates << "*Status: #{raw_updates["status"]}*"
end
if raw_updates.has_key?("cc")
cc = raw_updates["cc"].map do |l|
deleted = l.start_with?("-")
l = l[1..-1] if deleted
l = mask_email(l)
l = "~~#{l}~~" if deleted
l
end
updates << "*Cc: #{cc.join(", ")}*"
end
if raw_updates.has_key?("labels")
labels = raw_updates["labels"].map do |l|
deleted = l.start_with?("-")
l = l[1..-1] if deleted
l = nice_label_name(l)
l = "~~#{l}~~" if deleted
l
end
updates << "*Labels: #{labels.join(", ")}*"
end
if raw_updates.has_key?("owner")
updates << "*Owner: #{raw_updates["owner"]}*"
end
if raw_updates.has_key?("mergedInto")
updates << "*Merged into: ##{raw_updates["mergedInto"]}*"
end
if raw_updates.has_key?("blockedOn")
blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
name, id = raw_blocked_on.split(":", 2)
if name == project.import_source
"##{id}"
else
"#{project.namespace.path}/#{name}##{id}"
end
end
updates << "*Blocked on: #{blocked_ons.join(", ")}*"
end
if raw_updates.has_key?("blocking")
blockings = raw_updates["blocking"].map do |raw_blocked_on|
name, id = raw_blocked_on.split(":", 2)
if name == project.import_source
"##{id}"
else
"#{project.namespace.path}/#{name}##{id}"
end
end
updates << "*Blocking: #{blockings.join(", ")}*"
end
updates
end
def format_attachments(issue_id, comment_id, raw_attachments)
return [] unless raw_attachments
raw_attachments.map do |attachment|