Commit cef3fd28 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files

Wrap all rate limiting logic inside GitHub API client

parent f9524605d9fc
module Gitlab
module GithubImport
class Client
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :client, :api
def initialize(access_token)
......@@ -11,7 +14,7 @@ def initialize(access_token)
)
if access_token
::Octokit.auto_paginate = true
::Octokit.auto_paginate = false
@api = ::Octokit::Client.new(
access_token: access_token,
......@@ -36,7 +39,7 @@ def get_token(code)
def method_missing(method, *args, &block)
if api.respond_to?(method)
api.send(method, *args, &block)
request { api.send(method, *args, &block) }
else
super(method, *args, &block)
end
......@@ -55,6 +58,34 @@ def config
def github_options
config["args"]["client_options"].deep_symbolize_keys
end
def rate_limit
api.rate_limit!
end
def rate_limit_exceed?
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
end
def rate_limit_sleep_time
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
end
def request
sleep rate_limit_sleep_time if rate_limit_exceed?
data = yield
last_response = api.last_response
while last_response.rels[:next]
sleep rate_limit_sleep_time if rate_limit_exceed?
last_response = last_response.rels[:next].get
data.concat(last_response.data) if last_response.data.is_a?(Array)
end
data
end
end
end
end
......@@ -3,9 +3,6 @@ module GithubImport
class Importer
include Gitlab::ShellAdapter
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :client, :project, :repo, :repo_url
def initialize(project)
......@@ -28,52 +25,12 @@ def execute
private
def turn_auto_pagination_off!
client.auto_paginate = false
end
def turn_auto_pagination_on!
client.auto_paginate = true
end
def rate_limit
client.rate_limit!
end
def rate_limit_exceed?
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
end
def rate_limit_sleep_time
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
end
def paginate
turn_auto_pagination_off!
sleep rate_limit_sleep_time if rate_limit_exceed?
data = yield
last_response = client.last_response
while last_response.rels[:next]
sleep rate_limit_sleep_time if rate_limit_exceed?
last_response = last_response.rels[:next].get
data.concat(last_response.data) if last_response.data.is_a?(Array)
end
turn_auto_pagination_on!
data
end
def credentials
@credentials ||= project.import_data.credentials if project.import_data
end
def import_labels
labels = paginate { client.labels(repo, per_page: 100) }
labels = client.labels(repo, per_page: 100)
labels.each { |raw| LabelFormatter.new(project, raw).create! }
true
......@@ -82,7 +39,7 @@ def import_labels
end
def import_milestones
milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
milestones = client.milestones(repo, state: :all, per_page: 100)
milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
true
......@@ -91,9 +48,9 @@ def import_milestones
end
def import_issues
data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
data.each do |raw|
issues.each do |raw|
gh_issue = IssueFormatter.new(project, raw)
if gh_issue.valid?
......@@ -112,7 +69,7 @@ def import_pull_requests
hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
disable_webhooks(hooks)
pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
......@@ -146,14 +103,12 @@ def clean_up_disabled_webhooks(hooks)
def update_webhooks(hooks, options)
hooks.each do |hook|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.edit_hook(repo, hook.id, hook.name, hook.config, options)
end
end
def restore_branches(branches)
branches.each do |name, sha|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.create_ref(repo, "refs/heads/#{name}", sha)
end
......@@ -162,15 +117,12 @@ def restore_branches(branches)
def clean_up_restored_branches(branches)
branches.each do |name, _|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.delete_ref(repo, "heads/#{name}")
project.repository.rm_branch(project.creator, name)
end
end
def apply_labels(issuable)
sleep rate_limit_sleep_time if rate_limit_exceed?
issue = client.issue(repo, issuable.iid)
if issue.labels.count > 0
......@@ -183,12 +135,12 @@ def apply_labels(issuable)
end
def import_comments(issuable)
comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
comments = client.issue_comments(repo, issuable.iid, per_page: 100)
create_comments(issuable, comments)
end
def import_comments_on_diff(merge_request)
comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100)
create_comments(merge_request, comments)
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment