Read about our upcoming Code of Conduct on this issue

Commit 9ea56c4d authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@master

parent e915a1e83e88
......@@ -116,6 +116,10 @@ def objects
@objects ||= (params[:objects] || []).to_a
end
def objects_oids
objects.map { |o| o['oid'].to_s }
end
def has_authentication_ability?(capability)
(authentication_abilities || []).include?(capability)
end
......
......@@ -45,15 +45,9 @@ def upload_request?
params[:operation] == 'upload'
end
# rubocop: disable CodeReuse/ActiveRecord
def existing_oids
@existing_oids ||= begin
project.all_lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid)
end
end
# rubocop: enable CodeReuse/ActiveRecord
def download_objects!
existing_oids = project.all_lfs_objects_oids(oids: objects_oids)
objects.each do |object|
if existing_oids.include?(object[:oid])
object[:actions] = download_actions(object)
......@@ -68,13 +62,17 @@ def download_objects!
}
end
end
objects
end
def upload_objects!
existing_oids = project.lfs_objects_oids(oids: objects_oids)
objects.each do |object|
object[:actions] = upload_actions(object) unless existing_oids.include?(object[:oid])
end
objects
end
......
......@@ -1398,12 +1398,17 @@ def all_lfs_objects
.where(lfs_objects_projects: { project_id: [self, lfs_storage_project] })
end
# TODO: Call `#lfs_objects` instead once all LfsObjectsProject records are
# backfilled. At that point, projects can look at their own `lfs_objects`.
# TODO: Remove this method once all LfsObjectsProject records are backfilled
# for forks. At that point, projects can look at their own `lfs_objects` so
# `lfs_objects_oids` can be used instead.
#
# See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info.
def lfs_objects_oids
all_lfs_objects.pluck(:oid)
def all_lfs_objects_oids(oids: [])
oids(all_lfs_objects, oids: oids)
end
def lfs_objects_oids(oids: [])
oids(lfs_objects, oids: oids)
end
def personal?
......@@ -2515,6 +2520,12 @@ def ensure_pages_metadatum
reset
retry
end
def oids(objects, oids: [])
collection = oids.any? ? objects.where(oid: oids) : objects
collection.pluck(:oid)
end
end
Project.prepend_if_ee('EE::Project')
......@@ -60,6 +60,7 @@ class User < ApplicationRecord
MINIMUM_INACTIVE_DAYS = 180
enum bot_type: ::UserBotTypeEnums.bots
enum user_type: ::UserTypeEnums.types
# Override Devise::Models::Trackable#update_tracked_fields!
# to limit database writes to at most once every hour
......@@ -336,7 +337,7 @@ def inactive_message
scope :with_dashboard, -> (dashboard) { where(dashboard: dashboard) }
scope :with_public_profile, -> { where(private_profile: false) }
scope :bots, -> { where.not(bot_type: nil) }
scope :humans, -> { where(bot_type: nil) }
scope :humans, -> { where(user_type: nil, bot_type: nil) }
scope :with_expiring_and_not_notified_personal_access_tokens, ->(at) do
where('EXISTS (?)',
......
......@@ -2,7 +2,9 @@
module UserBotTypeEnums
def self.bots
# When adding a new key, please ensure you are not conflicting with EE-only keys in app/models/user_bot_type_enums.rb
# When adding a new key, please ensure you are not conflicting
# with EE-only keys in app/models/user_type_enums.rb
# or app/models/user_bot_type_enums.rb
{
alert_bot: 2
}
......
# frozen_string_literal: true
module UserTypeEnums
def self.types
# When adding a new key, please ensure you are not conflicting
# with EE-only keys in app/models/user_type_enums.rb
# or app/models/user_bot_type_enums.rb
bots
end
def self.bots
{
AlertBot: 2
}
end
end
UserTypeEnums.prepend_if_ee('EE::UserTypeEnums')
......@@ -52,7 +52,7 @@ def start_fork(project)
def link_lfs_objects(source_project, target_project)
Projects::LfsPointers::LfsLinkService
.new(target_project)
.execute(source_project.lfs_objects_oids)
.execute(source_project.all_lfs_objects_oids)
rescue Projects::LfsPointers::LfsLinkService::TooManyOidsError
raise_fork_failure(
source_project,
......
---
title: Mark existing LFS object for upload for forks
merge_request: 26344
author:
type: fixed
# frozen_string_literal: true
class AddUserType < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
with_lock_retries do
add_column :users, :user_type, :integer, limit: 2
end
end
def down
with_lock_retries do
remove_column :users, :user_type
end
end
end
# frozen_string_literal: true
class AddUserTypeIndex < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :users, :user_type
end
def down
remove_concurrent_index :users, :user_type
end
end
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_03_03_074328) do
ActiveRecord::Schema.define(version: 2020_03_04_090155) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm"
......@@ -4325,6 +4325,7 @@
t.string "last_name", limit: 255
t.string "static_object_token", limit: 255
t.integer "role", limit: 2
t.integer "user_type", limit: 2
t.index "lower((name)::text)", name: "index_on_users_name_lower"
t.index ["accepted_term_id"], name: "index_users_on_accepted_term_id"
t.index ["admin"], name: "index_users_on_admin"
......@@ -4347,6 +4348,7 @@
t.index ["state"], name: "index_users_on_state_and_internal_ee", where: "((ghost IS NOT TRUE) AND (bot_type IS NULL))"
t.index ["static_object_token"], name: "index_users_on_static_object_token", unique: true
t.index ["unconfirmed_email"], name: "index_users_on_unconfirmed_email", where: "(unconfirmed_email IS NOT NULL)"
t.index ["user_type"], name: "index_users_on_user_type"
t.index ["username"], name: "index_users_on_username"
t.index ["username"], name: "index_users_on_username_trigram", opclass: :gin_trgm_ops, using: :gin
end
......
......@@ -160,7 +160,7 @@ Parameters:
| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true |
| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API |
| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes |
| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate (needed if API is using a self-signed TLS certificate |
| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. |
| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. |
| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** |
......@@ -227,7 +227,7 @@ Parameters:
| `domain` | string | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster |
| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API |
| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes |
| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate (needed if API is using a self-signed TLS certificate |
| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. |
| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** |
NOTE: **Note:**
......
......@@ -142,3 +142,212 @@ ActiveRecord hooks:
The framework behind all this is located in
[`ee/lib/gitlab/geo/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/lib/gitlab/geo).
## Existing Replicator Strategies
Before writing a new kind of Replicator Strategy, check below to see if your
resource can already be handled by one of the existing strategies. Consult with
the Geo team if you are unsure.
### Blob Replicator Strategy
Models that use
[CarrierWave's](https://github.com/carrierwaveuploader/carrierwave) `Uploader::Base`
can be easily supported by Geo with the `Geo::BlobReplicatorStrategy` module.
First, each file should have its own primary ID and model. Geo strongly
recommends treating *every single file* as a first-class citizen, because in
our experience this greatly simplifies tracking replication and verification
state.
For example, to add support for files referenced by a `Widget` model with a
`widgets` table, you would perform the following steps:
1. Add verification state fields to the `widgets` table so the Geo primary can
track verification state:
```ruby
# frozen_string_literal: true
class AddVerificationStateToWidgets < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
add_column :widgets, :verification_retry_at, :datetime_with_timezone
add_column :widgets, :last_verification_ran_at, :datetime_with_timezone
add_column :widgets, :verification_checksum, :string
add_column :widgets, :verification_failure, :string
add_column :widgets, :verification_retry_count, :integer
end
end
```
1. Add a partial index on `verification_failure` to ensure re-verification can
be performed efficiently:
```ruby
# frozen_string_literal: true
class AddVerificationFailureIndexToWidgets < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :widgets, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial"
end
def down
remove_concurrent_index :widgets, :verification_failure
end
end
```
1. Include `Gitlab::Geo::ReplicableModel` in the `Widget` class, and specify
the Replicator class `with_replicator Geo::WidgetReplicator`.
At this point the `Widget` class should look like this:
```ruby
# frozen_string_literal: true
class Widget < ApplicationRecord
include ::Gitlab::Geo::ReplicableModel
with_replicator Geo::WidgetReplicator
mount_uploader :file, WidgetUploader
...
end
```
1. Create `ee/app/replicators/geo/widget_replicator.rb`. Implement the
`#carrierwave_uploader` method which should return a `CarrierWave::Uploader`.
And implement the private `#model` method to return the `Widget` class.
```ruby
# frozen_string_literal: true
module Geo
class WidgetReplicator < Gitlab::Geo::Replicator
include ::Geo::BlobReplicatorStrategy
def carrierwave_uploader
model_record.file
end
private
def model
::Widget
end
end
end
```
1. Create `ee/spec/replicators/geo/widget_replicator_spec.rb` and perform
the setup necessary to define the `model_record` variable for the shared
examples.
```ruby
# frozen_string_literal: true
require 'spec_helper'
describe Geo::WidgetReplicator do
let(:model_record) { build(:widget) }
it_behaves_like 'a blob replicator'
end
```
1. Create the `widget_registry` table so Geo secondaries can track the sync and
verification state of each Widget's file:
```ruby
# frozen_string_literal: true
class CreateWidgetRegistry < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :widget_registry, id: :serial, force: :cascade do |t|
t.integer :widget_id, null: false
t.integer :state, default: 0, null: false
t.integer :retry_count, default: 0
t.string :last_sync_failure, limit: 255
t.datetime_with_timezone :retry_at
t.datetime_with_timezone :last_synced_at
t.datetime_with_timezone :created_at, null: false
t.index :widget_id, name: :index_widget_registry_on_repository_id, using: :btree
t.index :retry_at, name: :index_widget_registry_on_retry_at, using: :btree
t.index :state, name: :index_widget_registry_on_state, using: :btree
end
end
end
```
1. Create `ee/app/models/geo/widget_registry.rb`:
```ruby
# frozen_string_literal: true
class Geo::WidgetRegistry < Geo::BaseRegistry
include Geo::StateMachineRegistry
belongs_to :widget, class_name: 'Widget'
end
```
1. Create `ee/spec/factories/geo/widget_registry.rb`:
```ruby
# frozen_string_literal: true
FactoryBot.define do
factory :widget_registry, class: 'Geo::WidgetRegistry' do
widget
state { Geo::WidgetRegistry.state_value(:pending) }
trait :synced do
state { Geo::WidgetRegistry.state_value(:synced) }
last_synced_at { 5.days.ago }
end
trait :failed do
state { Geo::WidgetRegistry.state_value(:failed) }
last_synced_at { 1.day.ago }
retry_count { 2 }
last_sync_failure { 'Random error' }
end
trait :started do
state { Geo::WidgetRegistry.state_value(:started) }
last_synced_at { 1.day.ago }
retry_count { 0 }
end
end
end
```
1. Create `ee/spec/models/geo/widget_registry.rb`:
```ruby
# frozen_string_literal: true
require 'spec_helper'
describe Geo::WidgetRegistry, :geo, type: :model do
let_it_be(:registry) { create(:widget_registry) }
specify 'factory is valid' do
expect(registry).to be_valid
end
end
```
Widget files should now be replicated and verified by Geo!
......@@ -293,11 +293,11 @@ On the EC2 dashboard, look for Load Balancer in the left navigation bar:
1. Click the **Create Load Balancer** button.
1. Choose the **Classic Load Balancer**.
1. Give it a name (`gitlab-loadbalancer`) and for the **Create LB Inside** option, select `gitlab-vpc` from the dropdown menu.
1. Give it a name (we'll use `gitlab-loadbalancer`) and for the **Create LB Inside** option, select `gitlab-vpc` from the dropdown menu.
1. In the **Listeners** section, set HTTP port 80, HTTPS port 443, and TCP port 22 for both load balancer and instance protocols and ports.
1. In the **Select Subnets** section, select both public subnets from the list.
1. Click **Assign Security Groups** and select **Create a new security group**, give it a name
(`gitlab-loadbalancer-sec-group`) and description, and allow both HTTP and HTTPS traffic
(we'll use `gitlab-loadbalancer-sec-group`) and description, and allow both HTTP and HTTPS traffic
from anywhere (`0.0.0.0/0, ::/0`).
1. Click **Configure Security Settings** and select an SSL/TLS certificate from ACM or upload a certificate to IAM.
1. Click **Configure Health Check** and set up a health check for your EC2 instances.
......
......@@ -5692,6 +5692,53 @@ def enable_lfs
end
end
describe '#all_lfs_objects_oids' do
let(:project) { create(:project) }
let(:lfs_object) { create(:lfs_object) }
let(:another_lfs_object) { create(:lfs_object) }
subject { project.all_lfs_objects_oids }
context 'when project has associated LFS objects' do
before do
create(:lfs_objects_project, lfs_object: lfs_object, project: project)
create(:lfs_objects_project, lfs_object: another_lfs_object, project: project)
end
it 'returns OIDs of LFS objects' do
expect(subject).to match_array([lfs_object.oid, another_lfs_object.oid])
end
context 'and there are specified oids' do
subject { project.all_lfs_objects_oids(oids: [lfs_object.oid]) }
it 'returns OIDs of LFS objects that match specified oids' do
expect(subject).to eq([lfs_object.oid])
end
end
end
context 'when fork has associated LFS objects to itself and source' do
let(:source) { create(:project) }
let(:project) { fork_project(source) }
before do
create(:lfs_objects_project, lfs_object: lfs_object, project: source)
create(:lfs_objects_project, lfs_object: another_lfs_object, project: project)
end
it 'returns OIDs of LFS objects' do
expect(subject).to match_array([lfs_object.oid, another_lfs_object.oid])
end
end
context 'when project has no associated LFS objects' do
it 'returns empty array' do
expect(subject).to be_empty
end
end
end
describe '#lfs_objects_oids' do
let(:project) { create(:project) }
let(:lfs_object) { create(:lfs_object) }
......@@ -5708,6 +5755,14 @@ def enable_lfs
it 'returns OIDs of LFS objects' do
expect(subject).to match_array([lfs_object.oid, another_lfs_object.oid])
end
context 'and there are specified oids' do
subject { project.lfs_objects_oids(oids: [lfs_object.oid]) }
it 'returns OIDs of LFS objects that match specified oids' do
expect(subject).to eq([lfs_object.oid])
end
end
end
context 'when project has no associated LFS objects' do
......
......@@ -690,22 +690,34 @@ def authorization_in_action(action)
end
context 'when pushing an LFS object that already exists' do
shared_examples_for 'batch upload with existing LFS object' do
it_behaves_like 'LFS http 200 response'
it 'responds with links the object to the project' do
expect(json_response['objects']).to be_kind_of(Array)
expect(json_response['objects'].first).to include(sample_object)
expect(lfs_object.projects.pluck(:id)).not_to include(project.id)
expect(lfs_object.projects.pluck(:id)).to include(other_project.id)
expect(json_response['objects'].first['actions']['upload']['href']).to eq(objects_url(project, sample_oid, sample_size))
expect(json_response['objects'].first['actions']['upload']['header']).to include('Content-Type' => 'application/octet-stream')
end
it_behaves_like 'process authorization header', renew_authorization: true
end
let(:update_lfs_permissions) do
other_project.lfs_objects << lfs_object
end
it_behaves_like 'LFS http 200 response'
it 'responds with links the object to the project' do
expect(json_response['objects']).to be_kind_of(Array)
expect(json_response['objects'].first).to include(sample_object)
expect(lfs_object.projects.pluck(:id)).not_to include(project.id)
expect(lfs_object.projects.pluck(:id)).to include(other_project.id)
expect(json_response['objects'].first['actions']['upload']['href']).to eq(objects_url(project, sample_oid, sample_size))
expect(json_response['objects'].first['actions']['upload']['header']).to include('Content-Type' => 'application/octet-stream')
context 'in another project' do
it_behaves_like 'batch upload with existing LFS object'
end
it_behaves_like 'process authorization header', renew_authorization: true
context 'in source of fork project' do
let(:project) { fork_project(other_project) }
it_behaves_like 'batch upload with existing LFS object'
end
end
context 'when pushing a LFS object that does not exist' do
......
# frozen_string_literal: true
# Include these shared examples in specs of Replicators that include
# BlobReplicatorStrategy.
#
# A let variable called model_record should be defined in the spec. It should be
# a valid, unpersisted instance of the model class.
#
RSpec.shared_examples 'a blob replicator' do
include EE::GeoHelpers
let_it_be(:primary) { create(:geo_node, :primary) }
let_it_be(:secondary) { create(:geo_node) }
subject(:replicator) { model_record.replicator }
before do
stub_current_geo_node(primary)
end
describe '#handle_after_create_commit' do
it 'creates a Geo::Event' do
expect do
replicator.handle_after_create_commit
end.to change { ::Geo::Event.count }.by(1)
expect(::Geo::Event.last.attributes).to include(
"replicable_name" => replicator.replicable_name, "event_name" => "created", "payload" => { "model_record_id" => replicator.model_record.id })
end
end
describe '#consume_created_event' do
it 'invokes Geo::BlobDownloadService' do
service = double(:service)