Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
heptapod
heptapod
Commits
72ef768e37c6
Commit
72ef768e
authored
Jun 25, 2018
by
James Lopez
Browse files
Update Import/Export to use object storage (based on aa feature flag)
parent
7df607327bda
Changes
31
Hide whitespace changes
Inline
Side-by-side
app/controllers/projects_controller.rb
View file @
72ef768e
...
...
@@ -2,6 +2,7 @@
include
IssuableCollections
include
ExtractsPath
include
PreviewMarkdown
include
SendFileUpload
before_action
:whitelist_query_limiting
,
only:
[
:create
]
before_action
:authenticate_user!
,
except:
[
:index
,
:show
,
:activity
,
:refs
]
...
...
@@ -188,9 +189,9 @@
end
def
download_export
export_project_
path
=
@project
.
export_project_path
if
export_project_path
if
export_project_
object_storage?
send_upload
(
@project
.
import_export_upload
.
export_file
)
els
if
export_project_path
send_file
export_project_path
,
disposition:
'attachment'
else
redirect_to
(
...
...
@@ -265,8 +266,6 @@
render
json:
options
.
to_json
end
private
# Render project landing depending of which features are available
# So if page is not availble in the list it renders the next page
#
...
...
@@ -424,4 +423,12 @@
def
whitelist_query_limiting
Gitlab
::
QueryLimiting
.
whitelist
(
'https://gitlab.com/gitlab-org/gitlab-ce/issues/42440'
)
end
def
export_project_path
@export_project_path
||=
@project
.
export_project_path
end
def
export_project_object_storage?
@project
.
export_project_object_exists?
end
end
app/models/import_export_upload.rb
0 → 100644
View file @
72ef768e
class
ImportExportUpload
<
ActiveRecord
::
Base
include
WithUploads
include
ObjectStorage
::
BackgroundMove
belongs_to
:project
mount_uploader
:import_file
,
ImportExportUploader
mount_uploader
:export_file
,
ImportExportUploader
def
retrieve_upload
(
_identifier
,
paths
)
Upload
.
find_by
(
model:
self
,
path:
paths
)
end
end
app/models/project.rb
View file @
72ef768e
...
...
@@ -171,6 +171,7 @@
has_one
:fork_network
,
through: :fork_network_member
has_one
:import_state
,
autosave:
true
,
class_name:
'ProjectImportState'
,
inverse_of: :project
has_one
:import_export_upload
,
dependent: :destroy
# rubocop:disable Cop/ActiveRecordDependent
# Merge Requests for target project should be removed with it
has_many
:merge_requests
,
foreign_key:
'target_project_id'
...
...
@@ -1712,7 +1713,7 @@
:started
elsif
after_export_in_progress?
:after_export_action
elsif
export_project_path
elsif
export_project_path
||
export_project_object_exists?
:finished
else
:none
...
...
@@ -1727,10 +1728,13 @@
import_export_shared
.
after_export_in_progress?
end
def
remove_exports
return
nil
unless
export_path
.
present?
FileUtils
.
rm_rf
(
export_path
)
def
remove_exports
(
path
=
export_path
)
if
path
.
present?
FileUtils
.
rm_rf
(
path
)
elsif
export_project_object_exists?
import_export_upload
.
remove_export_file!
import_export_upload
.
save
end
end
def
remove_exported_project_file
...
...
@@ -1734,5 +1738,6 @@
end
def
remove_exported_project_file
return
unless
export_project_path
.
present?
remove_exports
(
export_project_path
)
end
...
...
@@ -1738,5 +1743,6 @@
FileUtils
.
rm_f
(
export_project_path
)
def
export_project_object_exists?
Gitlab
::
ImportExport
.
object_storage?
&&
import_export_upload
&
.
export_file
&
.
file
end
def
full_path_slug
...
...
app/services/import_export_clean_up_service.rb
View file @
72ef768e
...
...
@@ -10,7 +10,9 @@
def
execute
Gitlab
::
Metrics
.
measure
(
:import_export_clean_up
)
do
next
unless
File
.
directory?
(
path
)
clean_up_export_object_files
break
unless
File
.
directory?
(
path
)
clean_up_export_files
end
...
...
@@ -21,4 +23,11 @@
def
clean_up_export_files
Gitlab
::
Popen
.
popen
(
%W(find
#{
path
}
-not -path
#{
path
}
-mmin +
#{
mmin
}
-delete)
)
end
def
clean_up_export_object_files
ImportExportUpload
.
where
(
'updated_at < ?'
,
mmin
.
minutes
.
ago
).
each
do
|
upload
|
upload
.
remove_export_file!
upload
.
save!
end
end
end
app/uploaders/import_export_uploader.rb
0 → 100644
View file @
72ef768e
class
ImportExportUploader
<
AttachmentUploader
EXTENSION_WHITELIST
=
%w[tar.gz]
.
freeze
def
extension_whitelist
EXTENSION_WHITELIST
end
def
move_to_store
true
end
def
move_to_cache
false
end
end
app/views/projects/_export.html.haml
View file @
72ef768e
...
...
@@ -31,7 +31,7 @@
%li
Any encrypted tokens
%p
Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
-
if
project
.
export_
project_path
-
if
project
.
export_
status
==
:finished
=
link_to
'Download export'
,
download_export_project_path
(
project
),
rel:
'nofollow'
,
download:
''
,
method: :get
,
class:
"btn btn-default"
=
link_to
'Generate new export'
,
generate_new_export_project_path
(
project
),
...
...
changelogs/unreleased/46246-gitlab-project-export-should-use-object-storage.yml
0 → 100644
View file @
72ef768e
---
title
:
Add Object Storage to project export
merge_request
:
20105
author
:
type
:
added
db/migrate/20180625113853_create_import_export_uploads.rb
0 → 100644
View file @
72ef768e
class
CreateImportExportUploads
<
ActiveRecord
::
Migration
DOWNTIME
=
false
def
change
create_table
:import_export_uploads
do
|
t
|
t
.
datetime_with_timezone
:updated_at
,
null:
false
t
.
references
:project
,
index:
true
,
foreign_key:
{
on_delete: :cascade
},
unique:
true
t
.
text
:import_file
t
.
text
:export_file
end
add_index
:import_export_uploads
,
:updated_at
end
end
db/schema.rb
View file @
72ef768e
...
...
@@ -949,6 +949,16 @@
add_index
"identities"
,
[
"user_id"
],
name:
"index_identities_on_user_id"
,
using: :btree
create_table
"import_export_uploads"
,
force: :cascade
do
|
t
|
t
.
datetime_with_timezone
"updated_at"
,
null:
false
t
.
integer
"project_id"
t
.
text
"import_file"
t
.
text
"export_file"
end
add_index
"import_export_uploads"
,
[
"project_id"
],
name:
"index_import_export_uploads_on_project_id"
,
using: :btree
add_index
"import_export_uploads"
,
[
"updated_at"
],
name:
"index_import_export_uploads_on_updated_at"
,
using: :btree
create_table
"internal_ids"
,
id: :bigserial
,
force: :cascade
do
|
t
|
t
.
integer
"project_id"
t
.
integer
"usage"
,
null:
false
...
...
doc/administration/raketasks/project_import_export.md
View file @
72ef768e
...
...
@@ -30,4 +30,10 @@
bundle
exec
rake gitlab:import_export:data
RAILS_ENV
=
production
```
In order to enable Object Storage on the Export, you can use the
[
feature flag
][
feature-flags
]
:
```
import_export_object_storage
```
[
ce-3050
]:
https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
...
...
@@ -33,2 +39,3 @@
[
ce-3050
]:
https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
[
feature-flags
]:
https://docs.gitlab.com/ee/api/features.html
[
tmp
]:
../../development/shared_files.md
lib/api/project_export.rb
View file @
72ef768e
...
...
@@ -23,9 +23,13 @@
get
':id/export/download'
do
path
=
user_project
.
export_project_path
render_api_error!
(
'404 Not found or has expired'
,
404
)
unless
path
present_disk_file!
(
path
,
File
.
basename
(
path
),
'application/gzip'
)
if
path
present_disk_file!
(
path
,
File
.
basename
(
path
),
'application/gzip'
)
elsif
user_project
.
export_project_object_exists?
present_carrierwave_file!
(
user_project
.
import_export_upload
.
export_file
)
else
render_api_error!
(
'404 Not found or has expired'
,
404
)
end
end
desc
'Start export'
do
...
...
lib/gitlab/import_export.rb
View file @
72ef768e
...
...
@@ -40,6 +40,10 @@
"
#{
basename
[
0
..
FILENAME_LIMIT
]
}
_export.tar.gz"
end
def
object_storage?
Feature
.
enabled?
(
:import_export_object_storage
)
end
def
version
VERSION
end
...
...
lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
View file @
72ef768e
...
...
@@ -24,5 +24,5 @@
end
def
execute
(
current_user
,
project
)
return
unless
project
&
.
export_
project
_path
@project
=
project
...
...
@@ -28,5 +28,6 @@
@project
=
project
return
unless
@project
.
export_status
==
:finished
@current_user
=
current_user
if
invalid?
...
...
@@ -51,5 +52,5 @@
end
def
self
.
lock_file_path
(
project
)
return
unless
project
&
.
export_path
return
unless
project
.
export_path
||
object_storage?
...
...
@@ -55,5 +56,8 @@
File
.
join
(
project
.
export_path
,
AFTER_EXPORT_LOCK_FILE_NAME
)
lock_path
=
project
.
import_export_shared
.
archive_path
FileUtils
.
mkdir_p
(
lock_path
)
File
.
join
(
lock_path
,
AFTER_EXPORT_LOCK_FILE_NAME
)
end
protected
...
...
@@ -77,6 +81,10 @@
def
log_validation_errors
errors
.
full_messages
.
each
{
|
msg
|
project
.
import_export_shared
.
add_error_message
(
msg
)
}
end
def
object_storage?
project
.
export_project_object_exists?
end
end
end
end
...
...
lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
View file @
72ef768e
...
...
@@ -38,7 +38,5 @@
private
def
send_file
export_file
=
File
.
open
(
project
.
export_project_path
)
Gitlab
::
HTTP
.
public_send
(
http_method
.
downcase
,
url
,
send_file_options
(
export_file
))
# rubocop:disable GitlabSecurity/PublicSend
Gitlab
::
HTTP
.
public_send
(
http_method
.
downcase
,
url
,
send_file_options
)
# rubocop:disable GitlabSecurity/PublicSend
ensure
...
...
@@ -44,4 +42,4 @@
ensure
export_file
.
close
if
export_file
export_file
.
close
if
export_file
&&
!
object_storage?
end
...
...
@@ -46,6 +44,14 @@
end
def
send_file_options
(
export_file
)
def
export_file
if
object_storage?
project
.
import_export_upload
.
export_file
.
file
.
open
else
File
.
open
(
project
.
export_project_path
)
end
end
def
send_file_options
{
body_stream:
export_file
,
headers:
headers
...
...
@@ -53,7 +59,15 @@
end
def
headers
{
'Content-Length'
=>
File
.
size
(
project
.
export_project_path
).
to_s
}
{
'Content-Length'
=>
export_size
.
to_s
}
end
def
export_size
if
object_storage?
project
.
import_export_upload
.
export_file
.
file
.
size
else
File
.
size
(
project
.
export_project_path
)
end
end
end
end
...
...
lib/gitlab/import_export/saver.rb
View file @
72ef768e
...
...
@@ -15,4 +15,5 @@
def
save
if
compress_and_save
remove_export_path
Rails
.
logger
.
info
(
"Saved project export
#{
archive_file
}
"
)
...
...
@@ -18,3 +19,4 @@
Rails
.
logger
.
info
(
"Saved project export
#{
archive_file
}
"
)
archive_file
save_on_object_storage
if
use_object_storage?
else
...
...
@@ -20,7 +22,7 @@
else
@shared
.
error
(
Gitlab
::
ImportExport
::
Error
.
new
(
"Unable to save
#{
archive_file
}
into
#{
@shared
.
export_path
}
"
))
@shared
.
error
(
Gitlab
::
ImportExport
::
Error
.
new
(
error_message
))
false
end
rescue
=>
e
@shared
.
error
(
e
)
false
...
...
@@ -22,8 +24,13 @@
false
end
rescue
=>
e
@shared
.
error
(
e
)
false
ensure
if
use_object_storage?
remove_archive
remove_export_path
end
end
private
...
...
@@ -36,6 +43,10 @@
FileUtils
.
rm_rf
(
@shared
.
export_path
)
end
def
remove_archive
FileUtils
.
rm_rf
(
@shared
.
archive_path
)
end
def
archive_file
@archive_file
||=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
project:
@project
))
end
...
...
@@ -39,6 +50,22 @@
def
archive_file
@archive_file
||=
File
.
join
(
@shared
.
archive_path
,
Gitlab
::
ImportExport
.
export_filename
(
project:
@project
))
end
def
save_on_object_storage
upload
=
ImportExportUpload
.
find_or_initialize_by
(
project:
@project
)
File
.
open
(
archive_file
)
{
|
file
|
upload
.
export_file
=
file
}
upload
.
save!
end
def
use_object_storage?
Gitlab
::
ImportExport
.
object_storage?
end
def
error_message
"Unable to save
#{
archive_file
}
into
#{
@shared
.
export_path
}
. Object storage enabled:
#{
use_object_storage?
}
"
end
end
end
end
spec/controllers/projects_controller_spec.rb
View file @
72ef768e
...
...
@@ -790,7 +790,12 @@
project
.
add_master
(
user
)
end
context
'when project export is enabled'
do
it
'returns 302'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
context
'object storage disabled'
do
before
do
stub_feature_flags
(
import_export_object_storage:
false
)
end
context
'when project export is enabled'
do
it
'returns 302'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
...
...
@@ -796,5 +801,18 @@
expect
(
response
).
to
have_gitlab_http_status
(
302
)
expect
(
response
).
to
have_gitlab_http_status
(
302
)
end
end
context
'when project export is disabled'
do
before
do
stub_application_setting
(
project_export_enabled?:
false
)
end
it
'returns 404'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
404
)
end
end
end
...
...
@@ -798,5 +816,5 @@
end
end
context
'
when project export is dis
abled'
do
context
'
object storage en
abled'
do
before
do
...
...
@@ -802,4 +820,4 @@
before
do
stub_
application_setting
(
project_export_enabled?:
fals
e
)
stub_
feature_flags
(
import_export_object_storage:
tru
e
)
end
...
...
@@ -804,5 +822,10 @@
end
it
'returns 404'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
context
'when project export is enabled'
do
it
'returns 302'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
302
)
end
end
...
...
@@ -808,5 +831,14 @@
expect
(
response
).
to
have_gitlab_http_status
(
404
)
context
'when project export is disabled'
do
before
do
stub_application_setting
(
project_export_enabled?:
false
)
end
it
'returns 404'
do
get
:download_export
,
namespace_id:
project
.
namespace
,
id:
project
expect
(
response
).
to
have_gitlab_http_status
(
404
)
end
end
end
end
...
...
spec/factories/import_export_uploads.rb
0 → 100644
View file @
72ef768e
FactoryBot
.
define
do
factory
:import_export_upload
do
project
{
create
(
:project
)
}
end
end
spec/factories/projects.rb
View file @
72ef768e
...
...
@@ -103,6 +103,22 @@
end
trait
:with_export
do
before
(
:create
)
do
|
_project
,
_evaluator
|
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
:import_export_object_storage
)
{
false
}
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
'import_export_object_storage'
)
{
false
}
end
after
(
:create
)
do
|
project
,
_evaluator
|
ProjectExportWorker
.
new
.
perform
(
project
.
creator
.
id
,
project
.
id
)
end
end
trait
:with_object_export
do
before
(
:create
)
do
|
_project
,
_evaluator
|
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
:import_export_object_storage
)
{
true
}
allow
(
Feature
).
to
receive
(
:enabled?
).
with
(
'import_export_object_storage'
)
{
true
}
end
after
(
:create
)
do
|
project
,
evaluator
|
ProjectExportWorker
.
new
.
perform
(
project
.
creator
.
id
,
project
.
id
)
end
...
...
spec/features/projects/import_export/export_file_spec.rb
View file @
72ef768e
...
...
@@ -25,6 +25,7 @@
before
do
allow_any_instance_of
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
stub_feature_flags
(
import_export_object_storage:
false
)
end
after
do
...
...
spec/features/projects/import_export/namespace_export_file_spec.rb
View file @
72ef768e
...
...
@@ -5,6 +5,7 @@
before
do
allow
(
Gitlab
::
ImportExport
).
to
receive
(
:storage_path
).
and_return
(
export_path
)
stub_feature_flags
(
import_export_object_storage:
false
)
end
after
do
...
...
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment