This instance will be upgraded to Heptapod 0.31.0rc on 2022-05-19 at 11:00 UTC+2 (a few minutes of down time)

20160302152808_remove_wrong_import_url_from_projects.rb 4.92 KB
Newer Older
James Lopez's avatar
James Lopez committed
1
2
3
# Loops through old importer projects that kept a token/password in the import URL
# and encrypts the credentials into a separate field in project#import_data
# #down method not supported
James Lopez's avatar
James Lopez committed
4
class RemoveWrongImportUrlFromProjects < ActiveRecord::Migration
5

6
  class ProjectImportDataFake
7
8
    extend AttrEncrypted
    attr_accessor :credentials
9
    attr_encrypted :credentials, key: Gitlab::Application.secrets.db_key_base, marshal: true, encode: true, :mode => :per_attribute_iv_and_salt
10
11
  end

James Lopez's avatar
James Lopez committed
12
  def up
13
14
    say("Encrypting and migrating project import credentials...")

James Lopez's avatar
James Lopez committed
15
    # This should cover GitHub, GitLab, Bitbucket user:password, token@domain, and other similar URLs.
16
    in_transaction(message: "Projects including GitHub and GitLab projects with an unsecured URL.") { process_projects_with_wrong_url }
17

18
    in_transaction(message: "Migrating Bitbucket credentials...") { process_project(import_type: 'bitbucket', credentials_keys: ['bb_session']) }
19

20
    in_transaction(message: "Migrating FogBugz credentials...") { process_project(import_type: 'fogbugz', credentials_keys: ['fb_session']) }
21

22
23
24
  end

  def process_projects_with_wrong_url
25
    projects_with_wrong_import_url.each do |project|
26
      begin
27
        import_url = Gitlab::UrlSanitizer.new(project["import_url"])
28

29
30
31
32
33
        update_import_url(import_url, project)
        update_import_data(import_url, project)
      rescue URI::InvalidURIError
        nullify_import_url(project)
      end
34
35
36
    end
  end

37
  def process_project(import_type:, credentials_keys: [])
38
    unencrypted_import_data(import_type: import_type).each do |data|
James Lopez's avatar
James Lopez committed
39
      replace_data_credentials(data, credentials_keys)
40
41
42
    end
  end

James Lopez's avatar
James Lopez committed
43
  def replace_data_credentials(data, credentials_keys)
44
    data_hash = JSON.load(data['data']) if data['data']
James Lopez's avatar
James Lopez committed
45
46
47
48
    unless data_hash.blank?
      encrypted_data_hash = encrypt_data(data_hash, credentials_keys)
      unencrypted_data = data_hash.empty? ? ' NULL ' :  quote(data_hash.to_json)
      update_with_encrypted_data(encrypted_data_hash, data['id'], unencrypted_data)
James Lopez's avatar
James Lopez committed
49
50
51
    end
  end

James Lopez's avatar
James Lopez committed
52
  def encrypt_data(data_hash, credentials_keys)
53
    new_data_hash = {}
James Lopez's avatar
James Lopez committed
54
    credentials_keys.each do |key|
55
      new_data_hash[key.to_sym] = data_hash.delete(key) if data_hash[key]
56
    end
57
    new_data_hash.deep_symbolize_keys
58
59
  end

60
61
  def in_transaction(message:)
    say_with_time(message) do
62
63
64
65
66
67
      ActiveRecord::Base.transaction do
        yield
      end
    end
  end

68
  def update_import_data(import_url, project)
69
    fake_import_data = ProjectImportDataFake.new
70
    fake_import_data.credentials = import_url.credentials
71
72
73
    import_data_id = project['import_data_id']
    if import_data_id
      execute(update_import_data_sql(import_data_id, fake_import_data))
74
75
76
77
78
    else
      execute(insert_import_data_sql(project['id'], fake_import_data))
    end
  end

James Lopez's avatar
James Lopez committed
79
  def update_with_encrypted_data(data_hash, import_data_id, unencrypted_data = ' NULL ')
80
    fake_import_data = ProjectImportDataFake.new
81
    fake_import_data.credentials = data_hash
James Lopez's avatar
James Lopez committed
82
    execute(update_import_data_sql(import_data_id, fake_import_data, unencrypted_data))
83
84
85
86
87
88
  end

  def update_import_url(import_url, project)
    execute("UPDATE projects SET import_url = #{quote(import_url.sanitized_url)} WHERE id = #{project['id']}")
  end

89
90
91
92
  def nullify_import_url(project)
    execute("UPDATE projects SET import_url = NULL WHERE id = #{project['id']}")
  end

93
  def insert_import_data_sql(project_id, fake_import_data)
94
95
96
97
98
99
100
101
102
103
104
    %(
      INSERT INTO project_import_data
                  (encrypted_credentials,
                   project_id,
                   encrypted_credentials_iv,
                   encrypted_credentials_salt)
      VALUES      ( #{quote(fake_import_data.encrypted_credentials)},
                    '#{project_id}',
                    #{quote(fake_import_data.encrypted_credentials_iv)},
                    #{quote(fake_import_data.encrypted_credentials_salt)})
    ).squish
105
106
  end

107
  def update_import_data_sql(id, fake_import_data, data = 'NULL')
108
109
110
111
112
113
114
115
    %(
      UPDATE project_import_data
      SET    encrypted_credentials = #{quote(fake_import_data.encrypted_credentials)},
             encrypted_credentials_iv = #{quote(fake_import_data.encrypted_credentials_iv)},
             encrypted_credentials_salt = #{quote(fake_import_data.encrypted_credentials_salt)},
             data = #{data}
      WHERE  id = '#{id}'
    ).squish
116
117
  end

James Lopez's avatar
James Lopez committed
118
  #GitHub projects with token, and any user:password@ based URL
119
  def projects_with_wrong_import_url
120
    select_all("SELECT p.id, p.import_url, i.id as import_data_id FROM projects p LEFT JOIN project_import_data i on p.id = i.project_id WHERE p.import_url <> '' AND p.import_url LIKE '%//%@%'")
121
122
  end

123
124
  # All imports with data for import_type
  def unencrypted_import_data(import_type: )
125
    select_all("SELECT i.id, p.import_url, i.data FROM projects p INNER JOIN project_import_data i ON p.id = i.project_id WHERE p.import_url <> '' AND p.import_type = '#{import_type}' ")
126
127
  end

128
129
  def quote(value)
    ActiveRecord::Base.connection.quote(value)
James Lopez's avatar
James Lopez committed
130
131
  end
end