Commit 155c9728 authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@master

parent e2c13d9bd911
......@@ -256,7 +256,8 @@ export default class Clusters {
eventHub.$on('uninstallApplication', data => this.uninstallApplication(data));
eventHub.$on('setCrossplaneProviderStack', data => this.setCrossplaneProviderStack(data));
eventHub.$on('setIngressModSecurityEnabled', data => this.setIngressModSecurityEnabled(data));
eventHub.$on('resetIngressModSecurityEnabled', id => this.resetIngressModSecurityEnabled(id));
eventHub.$on('setIngressModSecurityMode', data => this.setIngressModSecurityMode(data));
eventHub.$on('resetIngressModSecurityChanges', id => this.resetIngressModSecurityChanges(id));
// Add event listener to all the banner close buttons
this.addBannerCloseHandler(this.unreachableContainer, 'unreachable');
this.addBannerCloseHandler(this.authenticationFailureContainer, 'authentication_failure');
......@@ -271,7 +272,8 @@ export default class Clusters {
eventHub.$off('setCrossplaneProviderStack');
eventHub.$off('uninstallApplication');
eventHub.$off('setIngressModSecurityEnabled');
eventHub.$off('resetIngressModSecurityEnabled');
eventHub.$off('setIngressModSecurityMode');
eventHub.$off('resetIngressModSecurityChanges');
}
initPolling(method, successCallback, errorCallback) {
......@@ -525,8 +527,14 @@ export default class Clusters {
this.store.updateAppProperty(id, 'modsecurity_enabled', modSecurityEnabled);
}
resetIngressModSecurityEnabled(id) {
setIngressModSecurityMode({ id, modSecurityMode }) {
this.store.updateAppProperty(id, 'isEditingModSecurityMode', true);
this.store.updateAppProperty(id, 'modsecurity_mode', modSecurityMode);
}
resetIngressModSecurityChanges(id) {
this.store.updateAppProperty(id, 'isEditingModSecurityEnabled', false);
this.store.updateAppProperty(id, 'isEditingModSecurityMode', false);
}
destroy() {
......
......@@ -313,6 +313,7 @@ Crossplane runs inside your Kubernetes cluster and supports secure connectivity
:install-failed="applications.ingress.installFailed"
:install-application-request-params="{
modsecurity_enabled: applications.ingress.modsecurity_enabled,
modsecurity_mode: applications.ingress.modsecurity_mode,
}"
:uninstallable="applications.ingress.uninstallable"
:uninstall-successful="applications.ingress.uninstallSuccessful"
......
<script>
import _ from 'lodash';
import { __ } from '../../locale';
import { APPLICATION_STATUS, INGRESS } from '~/clusters/constants';
import { GlAlert, GlSprintf, GlLink, GlToggle, GlButton } from '@gitlab/ui';
import { escape as esc } from 'lodash';
import { s__, __ } from '../../locale';
import { APPLICATION_STATUS, INGRESS, LOGGING_MODE, BLOCKING_MODE } from '~/clusters/constants';
import {
GlAlert,
GlSprintf,
GlLink,
GlToggle,
GlButton,
GlDropdown,
GlDropdownItem,
GlIcon,
} from '@gitlab/ui';
import eventHub from '~/clusters/event_hub';
import modSecurityLogo from 'images/cluster_app_logos/modsecurity.png';
......@@ -17,6 +26,9 @@ export default {
GlLink,
GlToggle,
GlButton,
GlDropdown,
GlDropdownItem,
GlIcon,
},
props: {
ingress: {
......@@ -28,10 +40,23 @@ export default {
required: false,
default: '',
},
modes: {
type: Object,
required: false,
default: () => ({
[LOGGING_MODE]: {
name: s__('ClusterIntegration|Logging mode'),
},
[BLOCKING_MODE]: {
name: s__('ClusterIntegration|Blocking mode'),
},
}),
},
},
data: () => ({
modSecurityLogo,
hasValueChanged: false,
initialValue: null,
initialMode: null,
}),
computed: {
modSecurityEnabled: {
......@@ -39,19 +64,30 @@ export default {
return this.ingress.modsecurity_enabled;
},
set(isEnabled) {
if (this.initialValue === null) {
this.initialValue = this.ingress.modsecurity_enabled;
}
eventHub.$emit('setIngressModSecurityEnabled', {
id: INGRESS,
modSecurityEnabled: isEnabled,
});
if (this.hasValueChanged) {
this.resetStatus();
} else {
this.hasValueChanged = true;
}
},
},
hasValueChanged() {
return this.modSecurityEnabledChanged || this.modSecurityModeChanged;
},
modSecurityEnabledChanged() {
return this.initialValue !== null && this.initialValue !== this.ingress.modsecurity_enabled;
},
modSecurityModeChanged() {
return (
this.ingress.modsecurity_enabled &&
this.initialMode !== null &&
this.initialMode !== this.ingress.modsecurity_mode
);
},
ingressModSecurityDescription() {
return _.escape(this.ingressModSecurityHelpPath);
return esc(this.ingressModSecurityHelpPath);
},
saving() {
return [UPDATING].includes(this.ingress.status);
......@@ -73,18 +109,40 @@ export default {
this.saving || (this.hasValueChanged && [INSTALLED, UPDATED].includes(this.ingress.status))
);
},
modSecurityModeName() {
return this.modes[this.ingress.modsecurity_mode].name;
},
},
methods: {
updateApplication() {
eventHub.$emit('updateApplication', {
id: INGRESS,
params: { modsecurity_enabled: this.ingress.modsecurity_enabled },
params: {
modsecurity_enabled: this.ingress.modsecurity_enabled,
modsecurity_mode: this.ingress.modsecurity_mode,
},
});
this.resetStatus();
},
resetStatus() {
eventHub.$emit('resetIngressModSecurityEnabled', INGRESS);
this.hasValueChanged = false;
if (this.initialMode !== null) {
this.ingress.modsecurity_mode = this.initialMode;
}
if (this.initialValue !== null) {
this.ingress.modsecurity_enabled = this.initialValue;
}
this.initialValue = null;
this.initialMode = null;
eventHub.$emit('resetIngressModSecurityChanges', INGRESS);
},
selectMode(modeKey) {
if (this.initialMode === null) {
this.initialMode = this.ingress.modsecurity_mode;
}
eventHub.$emit('setIngressModSecurityMode', {
id: INGRESS,
modSecurityMode: modeKey,
});
},
},
};
......@@ -144,7 +202,35 @@ export default {
label-position="right"
/>
</div>
<div v-if="showButtons">
<div
v-if="ingress.modsecurity_enabled"
class="gl-responsive-table-row-layout mt-3"
role="row"
>
<div class="table-section section-wrap" role="gridcell">
<strong>
{{ s__('ClusterIntegration|Global default') }}
<gl-icon name="earth" class="align-text-bottom" />
</strong>
<div class="form-group">
<p class="form-text text-muted">
<strong>
{{
s__(
'ClusterIntegration|Set the global mode for the WAF in this cluster. This can be overridden at the environmental level.',
)
}}
</strong>
</p>
</div>
<gl-dropdown :text="modSecurityModeName" :disabled="saveButtonDisabled">
<gl-dropdown-item v-for="(mode, key) in modes" :key="key" @click="selectMode(key)">
{{ mode.name }}
</gl-dropdown-item>
</gl-dropdown>
</div>
</div>
<div v-if="showButtons" class="mt-3">
<gl-button
class="btn-success inline mr-1"
:loading="saving"
......
......@@ -66,3 +66,6 @@ export const APPLICATIONS = [
];
export const INGRESS_DOMAIN_SUFFIX = '.nip.io';
export const LOGGING_MODE = 'logging';
export const BLOCKING_MODE = 'blocking';
......@@ -53,9 +53,11 @@ export default class ClusterStore {
...applicationInitialState,
title: s__('ClusterIntegration|Ingress'),
modsecurity_enabled: false,
modsecurity_mode: null,
externalIp: null,
externalHostname: null,
isEditingModSecurityEnabled: false,
isEditingModSecurityMode: false,
updateFailed: false,
},
cert_manager: {
......@@ -214,6 +216,9 @@ export default class ClusterStore {
if (!this.state.applications.ingress.isEditingModSecurityEnabled) {
this.state.applications.ingress.modsecurity_enabled = serverAppEntry.modsecurity_enabled;
}
if (!this.state.applications.ingress.isEditingModSecurityMode) {
this.state.applications.ingress.modsecurity_mode = serverAppEntry.modsecurity_mode;
}
} else if (appId === CERT_MANAGER) {
this.state.applications.cert_manager.email =
this.state.applications.cert_manager.email || serverAppEntry.email;
......
......@@ -47,7 +47,7 @@ def cluster
end
def cluster_application_params
params.permit(:application, :hostname, :email, :stack, :modsecurity_enabled)
params.permit(:application, :hostname, :email, :stack, :modsecurity_enabled, :modsecurity_mode)
end
def cluster_application_destroy_params
......
......@@ -65,6 +65,14 @@ class SnippetType < BaseObject
calls_gitaly: true,
null: false
field :ssh_url_to_repo, type: GraphQL::STRING_TYPE,
description: 'SSH URL to the snippet repository',
null: true
field :http_url_to_repo, type: GraphQL::STRING_TYPE,
description: 'HTTP URL to the snippet repository',
null: true
markdown_field :description_html, null: true, method: :description
end
end
......@@ -6,6 +6,9 @@ class Ingress < ApplicationRecord
VERSION = '1.29.7'
INGRESS_CONTAINER_NAME = 'nginx-ingress-controller'
MODSECURITY_LOG_CONTAINER_NAME = 'modsecurity-log'
MODSECURITY_MODE_LOGGING = "DetectionOnly"
MODSECURITY_MODE_BLOCKING = "On"
MODSECURITY_OWASP_RULES_FILE = "/etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf"
self.table_name = 'clusters_applications_ingress'
......@@ -18,11 +21,14 @@ class Ingress < ApplicationRecord
default_value_for :ingress_type, :nginx
default_value_for :modsecurity_enabled, true
default_value_for :version, VERSION
default_value_for :modsecurity_mode, :logging
enum ingress_type: {
nginx: 1
}
enum modsecurity_mode: { logging: 0, blocking: 1 }
FETCH_IP_ADDRESS_DELAY = 30.seconds
MODSEC_SIDECAR_INITIAL_DELAY_SECONDS = 10
......@@ -82,7 +88,8 @@ def specification
"controller" => {
"config" => {
"enable-modsecurity" => "true",
"enable-owasp-modsecurity-crs" => "true",
"enable-owasp-modsecurity-crs" => "false",
"modsecurity-snippet" => modsecurity_snippet_content,
"modsecurity.conf" => modsecurity_config_content
},
"extraContainers" => [
......@@ -157,6 +164,11 @@ def content_values
def application_jupyter_nil_or_installable?
cluster.application_jupyter.nil? || cluster.application_jupyter&.installable?
end
def modsecurity_snippet_content
sec_rule_engine = logging? ? MODSECURITY_MODE_LOGGING : MODSECURITY_MODE_BLOCKING
"SecRuleEngine #{sec_rule_engine}\nInclude #{MODSECURITY_OWASP_RULES_FILE}"
end
end
end
end
......@@ -312,6 +312,10 @@ def hexdigest
Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
end
def versioned_enabled_for?(user)
repository_exists? && ::Feature.enabled?(:version_snippets, user)
end
class << self
# Searches for snippets with a matching title, description or file name.
#
......
......@@ -11,6 +11,14 @@ def raw_url
Gitlab::UrlBuilder.build(snippet, raw: true)
end
def ssh_url_to_repo
snippet.ssh_url_to_repo if snippet.versioned_enabled_for?(current_user)
end
def http_url_to_repo
snippet.http_url_to_repo if snippet.versioned_enabled_for?(current_user)
end
def can_read_snippet?
can_access_resource?("read")
end
......
......@@ -15,4 +15,5 @@ class ClusterApplicationEntity < Grape::Entity
expose :can_uninstall?, as: :can_uninstall
expose :available_domains, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:available_domains) }
expose :pages_domain, using: Serverless::DomainEntity, if: -> (e, _) { e.respond_to?(:pages_domain) }
expose :modsecurity_mode, if: -> (e, _) { e.respond_to?(:modsecurity_mode) }
end
......@@ -31,6 +31,10 @@ def execute(request)
application.modsecurity_enabled = params[:modsecurity_enabled] || false
end
if application.has_attribute?(:modsecurity_mode)
application.modsecurity_mode = params[:modsecurity_mode] || 0
end
if application.respond_to?(:oauth_application)
application.oauth_application = create_oauth_application(application, request)
end
......
---
title: Resolve Improve format support message in issue design
merge_request: 27409
author:
type: fixed
---
title: Add option for switching between blocking and logging for WAF
merge_request: 27133
author:
type: added
# frozen_string_literal: true
# See https://docs.gitlab.com/ee/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddModsecurityModeToIngressApplication < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default(:clusters_applications_ingress, :modsecurity_mode, :smallint, default: 0, allow_null: false)
end
def down
remove_column :clusters_applications_ingress, :modsecurity_mode
end
end
......@@ -1199,6 +1199,7 @@
t.string "external_ip"
t.string "external_hostname"
t.boolean "modsecurity_enabled"
t.integer "modsecurity_mode", limit: 2, default: 0, null: false
t.index ["cluster_id"], name: "index_clusters_applications_ingress_on_cluster_id", unique: true
end
......
......@@ -1016,6 +1016,15 @@ When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']`
When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/issues/2521) is resolved.
### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node
If this error occurs even though file permissions are correct, it's likely that
the Gitaly node is experiencing
[clock drift](https://en.wikipedia.org/wiki/Clock_drift).
Please ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time
server to keep them synchronized if possible.
### Praefect
Praefect is an experimental daemon that allows for replication of the Git data.
......
......@@ -573,6 +573,127 @@ Particular attention should be shown to:
Congratulations! You have configured a highly available Praefect cluster.
### Failover
There are two ways to do a failover from one internal Gitaly node to another as the primary. Manually, or automatically.
As an example, in this `config.toml` we have one virtual storage named "default" with two internal Gitaly nodes behind it.
One is deemed the "primary". This means that read and write traffic will go to `internal_storage_0`, and writes
will get replicated to `internal_storage_1`:
```toml
socket_path = "/path/to/Praefect.socket"
# failover_enabled will enable automatic failover
failover_enabled = false
[logging]
format = "json"
level = "info"
[[virtual_storage]]
name = "default"
[[virtual_storage.node]]
name = "internal_storage_0"
address = "tcp://localhost:9999"
primary = true
token = "supersecret"
[[virtual_storage.node]]
name = "internal_storage_1"
address = "tcp://localhost:9998"
token = "supersecret"
```
#### Manual failover
In order to failover from using one internal Gitaly node to using another, a manual failover step can be used. Unless `failover_enabled` is set to `true`
in the `config.toml`, the only way to fail over from one primary to using another node as the primary is to do a manual failover.
1. Move `primary = true` from the current `[[virtual_storage.node]]` to another node in `/etc/gitlab/gitlab.rb`:
```ruby
praefect['virtual_storages'] = {
'praefect' => {
'gitaly-1' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
# no longer the primary
},
'gitaly-2' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
# this is the new primary
'primary' => true
},
'gitaly-3' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
}
}
}
```
1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
On a restart, Praefect will send write traffic to `internal_storage_1`. `internal_storage_0` is the new secondary now,
and replication jobs will be created to replicate repository data to `internal_storage_0` **from** `internal_storage_1`
#### Automatic failover
When automatic failover is enabled, Praefect will do automatic detection of the health of internal Gitaly nodes. If the
primary has a certain amount of healthchecks fail, it will decide to promote one of the secondaries to be primary, and
demote the primary to be a secondary.
1. To enable automatic failover, edit `/etc/gitlab/gitlab.rb`:
```ruby
# failover_enabled turns on automatic failover
praefect['failover_enabled'] = true
praefect['virtual_storages'] = {
'praefect' => {
'gitaly-1' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN'
},
'gitaly-3' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN'
}
}
}
```
1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
Below is the picture when Praefect starts up with the config.toml above:
```mermaid
graph TD
A[Praefect] -->|Mutator RPC| B(internal_storage_0)
B --> |Replication|C[internal_storage_1]
```
Let's say suddenly `internal_storage_0` goes down. Praefect will detect this and
automatically switch over to `internal_storage_1`, and `internal_storage_0` will serve as a secondary:
```mermaid
graph TD
A[Praefect] -->|Mutator RPC| B(internal_storage_1)
B --> |Replication|C[internal_storage_0]
```
NOTE: **Note:**: Currently this feature is supported for setups that only have 1 Praefect instance. Praefect instances running,
for example behind a load balancer, `failover_enabled` should be disabled. The reason is The reason is because there
is no coordination that currently happens across different Praefect instances, so there could be a situation where
two Praefect instances think two different Gitaly nodes are the primary.
## Migrating existing repositories to Praefect
If your GitLab instance already has repositories, these won't be migrated
......
......@@ -110,7 +110,7 @@ them.
| [PgBouncer](../../development/architecture.md#pgbouncer) | Database Pool Manager | [PgBouncer HA configuration](pgbouncer.md) **(PREMIUM ONLY)** |
| [Redis](../../development/architecture.md#redis)[^3] with Redis Sentinel | Key/Value store for shared data with HA watcher service | [Redis HA configuration](redis.md) |
| [Gitaly](../../development/architecture.md#gitaly)[^2] [^5] [^7] | Recommended high-level storage for Git repository data | [Gitaly HA configuration](gitaly.md) |
| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | |
| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | [Sidekiq configuration](sidekiq.md) |
| [GitLab application nodes](../../development/architecture.md#unicorn)[^1] | (Unicorn / Puma, Workhorse) - Web-requests (UI, API, Git over HTTP) | [GitLab app HA/scaling configuration](gitlab.md) |
| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling/HA](monitoring_node.md) |
......
---
type: reference
---
# Configuring Sidekiq
This section discusses how to configure an external Sidekiq instance.
Sidekiq requires connection to the Redis, PostgreSQL and Gitaly instance.
To configure the Sidekiq node:
1. SSH into the Sidekiq server.
1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package
you want using steps 1 and 2 from the GitLab downloads page.
**Do not complete any other steps on the download page.**
1. Open `/etc/gitlab/gitlab.rb` with your editor.
1. Generate the Sidekiq configuration:
```ruby
sidekiq['listen_address'] = "10.10.1.48"
## Optional: Enable extra Sidekiq processes
sidekiq_cluster['enable'] = true
sidekiq_cluster['enable'] = true
"elastic_indexer"
]
```