Read about our upcoming Code of Conduct on this issue

Commit db5130ff authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@master

parent ab6af775c9bd
......@@ -476,12 +476,12 @@ const Api = {
return axios.get(url);
},
lsifData(projectPath, commitId, path) {
lsifData(projectPath, commitId, paths) {
const url = Api.buildUrl(this.lsifPath)
.replace(':id', encodeURIComponent(projectPath))
.replace(':commit_id', commitId);
return axios.get(url, { params: { path } });
return axios.get(url, { params: { paths } });
},
environments(id) {
......
......@@ -13,9 +13,10 @@ export default {
commit(types.REQUEST_DATA);
api
.lsifData(state.projectPath, state.commitId, state.blobPath)
.lsifData(state.projectPath, state.commitId, [state.blobPath])
.then(({ data }) => {
const normalizedData = data.reduce((acc, d) => {
const dataForPath = data[state.blobPath];
const normalizedData = dataForPath.reduce((acc, d) => {
if (d.hover) {
acc[`${d.start_line}:${d.start_char}`] = d;
addInteractionClass(d);
......
......@@ -38,7 +38,7 @@ def date_range(params)
end
def to_utc_time(field)
date = field.is_a?(Date) ? field : Date.parse(field)
date = field.is_a?(Date) || field.is_a?(Time) ? field : Date.parse(field)
date.to_time.utc
end
end
......
......@@ -10,6 +10,11 @@ class SnippetsResolver < BaseResolver
def resolve(**args)
return Snippet.none if project.nil?
unless project.feature_available?(:snippets, current_user)
raise Gitlab::Graphql::Errors::ResourceNotAvailable,
'Snippets are not enabled for this Project'
end
super
end
......
......@@ -2039,6 +2039,16 @@ def set_repository_writable!
end
end
def change_repository_storage(new_repository_storage_key)
return if repository_read_only?
return if repository_storage == new_repository_storage_key
raise ArgumentError unless ::Gitlab.config.repositories.storages.key?(new_repository_storage_key)
run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) }
self.repository_read_only = true
end
def pushes_since_gc
Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
end
......
......@@ -469,6 +469,8 @@ class ProjectPolicy < BasePolicy
prevent :create_pipeline
end
rule { admin }.enable :change_repository_storage
private
def team_member?
......
......@@ -53,7 +53,7 @@ def issue_summary_markdown
#### Summary
#{metadata_list}
#{alert_details}
#{alert_details}#{metric_embed_for_alert}
MARKDOWN
end
......@@ -118,6 +118,10 @@ def backtick(value)
def host_links
Array(hosts.value).join(' ')
end
def metric_embed_for_alert; end
end
end
end
Projects::Prometheus::AlertPresenter.prepend_if_ee('EE::Projects::Prometheus::AlertPresenter')
......@@ -3,18 +3,15 @@
class SnippetBlobPresenter < BlobPresenter
def rich_data
return if blob.binary?
return unless blob.rich_viewer
if markup?
blob.rendered_markup
else
highlight(plain: false)
end
render_rich_partial
end
def plain_data
return if blob.binary?
highlight(plain: !markup?)
highlight(plain: false)
end
def raw_path
......@@ -27,10 +24,6 @@ def raw_path
private
def markup?
blob.rich_viewer&.partial_name == 'markup'
end
def snippet
blob.container
end
......@@ -38,4 +31,18 @@ def snippet
def language
nil
end
def render_rich_partial
renderer.render("projects/blob/viewers/_#{blob.rich_viewer.partial_name}",
locals: { viewer: blob.rich_viewer, blob: blob, blob_raw_path: raw_path },
layout: false)
end
def renderer
proxy = Warden::Proxy.new({}, Warden::Manager.new({})).tap do |proxy_instance|
proxy_instance.set_user(current_user, scope: :user)
end
ApplicationController.renderer.new('warden' => proxy)
end
end
......@@ -61,10 +61,15 @@ def order_by_date(tags)
end
def filter_by_name(tags)
regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
# Technical Debt: https://gitlab.com/gitlab-org/gitlab/issues/207267
# name_regex to be removed when container_expiration_policies is updated
# to have both regex columns
regex_delete = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z")
regex_retain = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z")
tags.select do |tag|
regex.scan(tag.name).any?
# regex_retain will override any overlapping matches by regex_delete
regex_delete.match?(tag.name) && !regex_retain.match?(tag.name)
end
end
......
......@@ -2,20 +2,22 @@
module Projects
class LsifDataService
attr_reader :file, :project, :path, :commit_id,
:docs, :doc_ranges, :ranges, :def_refs, :hover_refs
attr_reader :file, :project, :commit_id, :docs,
:doc_ranges, :ranges, :def_refs, :hover_refs
CACHE_EXPIRE_IN = 1.hour
def initialize(file, project, params)
def initialize(file, project, commit_id)
@file = file
@project = project
@path = params[:path]
@commit_id = params[:commit_id]
end
@commit_id = commit_id
def execute
fetch_data!
end
def execute(path)
doc_id = find_doc_id(docs, path)
dir_absolute_path = docs[doc_id]&.delete_suffix(path)
doc_ranges[doc_id]&.map do |range_id|
location, ref_id = ranges[range_id].values_at('loc', 'ref_id')
......@@ -26,7 +28,7 @@ def execute
end_line: line_data.last,
start_char: column_data.first,
end_char: column_data.last,
definition_url: definition_url_for(def_refs[ref_id]),
definition_url: definition_url_for(def_refs[ref_id], dir_absolute_path),
hover: highlighted_hover(hover_refs[ref_id])
}
end
......@@ -58,8 +60,8 @@ def fetch_data!
@hover_refs = data['hover_refs']
end
def doc_id
@doc_id ||= docs.reduce(nil) do |doc_id, (id, doc_path)|
def find_doc_id(docs, path)
docs.reduce(nil) do |doc_id, (id, doc_path)|
next doc_id unless doc_path =~ /#{path}$/
if doc_id.nil? || docs[doc_id].size > doc_path.size
......@@ -70,11 +72,7 @@ def doc_id
end
end
def dir_absolute_path
@dir_absolute_path ||= docs[doc_id]&.delete_suffix(path)
end
def definition_url_for(ref_id)
def definition_url_for(ref_id, dir_absolute_path)
return unless range = ranges[ref_id]
def_doc_id, location = range.values_at('doc_id', 'loc')
......
# frozen_string_literal: true
module Projects
class UpdateRepositoryStorageService < BaseService
include Gitlab::ShellAdapter
RepositoryAlreadyMoved = Class.new(StandardError)
def initialize(project)
@project = project
end
def execute(new_repository_storage_key)
# Raising an exception is a little heavy handed but this behavior (doing
# nothing if the repo is already on the right storage) prevents data
# loss, so it is valuable for us to be able to observe it via the
# exception.
raise RepositoryAlreadyMoved if project.repository_storage == new_repository_storage_key
if mirror_repositories(new_repository_storage_key)
mark_old_paths_for_archive
project.update(repository_storage: new_repository_storage_key, repository_read_only: false)
project.leave_pool_repository
project.track_project_repository
enqueue_housekeeping
else
project.update(repository_read_only: false)
end
end
private
def mirror_repositories(new_repository_storage_key)
result = mirror_repository(new_repository_storage_key)
if project.wiki.repository_exists?
result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI)
end
result
end
def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT)
return false unless wait_for_pushes(type)
repository = type.repository_for(project)
full_path = repository.full_path
raw_repository = repository.raw
# Initialize a git repository on the target path
gitlab_shell.create_repository(new_storage_key, raw_repository.relative_path, full_path)
new_repository = Gitlab::Git::Repository.new(new_storage_key,
raw_repository.relative_path,
raw_repository.gl_repository,
full_path)
new_repository.fetch_repository_as_mirror(raw_repository)
end
def mark_old_paths_for_archive
old_repository_storage = project.repository_storage
new_project_path = moved_path(project.disk_path)
# Notice that the block passed to `run_after_commit` will run with `project`
# as its context
project.run_after_commit do
GitlabShellWorker.perform_async(:mv_repository,
old_repository_storage,
disk_path,
new_project_path)
if wiki.repository_exists?
GitlabShellWorker.perform_async(:mv_repository,
old_repository_storage,
wiki.disk_path,
"#{new_project_path}.wiki")
end
end
end
def moved_path(path)
"#{path}+#{project.id}+moved+#{Time.now.to_i}"
end
# The underlying FetchInternalRemote call uses a `git fetch` to move data
# to the new repository, which leaves it in a less-well-packed state,
# lacking bitmaps and commit graphs. Housekeeping will boost performance
# significantly.
def enqueue_housekeeping
return unless Gitlab::CurrentSettings.housekeeping_enabled?
return unless Feature.enabled?(:repack_after_shard_migration, project)
Projects::HousekeepingService.new(project, :gc).execute
rescue Projects::HousekeepingService::LeaseTaken
# No action required
end
def wait_for_pushes(type)
reference_counter = project.reference_counter(type: type)
# Try for 30 seconds, polling every 10
3.times do
return true if reference_counter.value == 0
sleep 10
end
false
end
end
end
Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService')
......@@ -13,6 +13,10 @@ def execute
ensure_wiki_exists if enabling_wiki?
if changing_storage_size?
project.change_repository_storage(params.delete(:repository_storage))
end
yield if block_given?
validate_classification_label(project, :external_authorization_classification_label)
......@@ -140,6 +144,13 @@ def update_pages_config
def changing_pages_https_only?
project.previous_changes.include?(:pages_https_only)
end
def changing_storage_size?
new_repository_storage = params[:repository_storage]
new_repository_storage && project.repository.exists? &&
can?(current_user, :change_repository_storage, project)
end
end
end
......
......@@ -1151,6 +1151,13 @@
:resource_boundary: :unknown
:weight: 1
:idempotent:
- :name: project_update_repository_storage
:feature_category: :source_code_management
:has_external_dependencies:
:urgency: :default
:resource_boundary: :unknown
:weight: 1
:idempotent:
- :name: propagate_service_template
:feature_category: :source_code_management
:has_external_dependencies:
......
# frozen_string_literal: true
class ProjectUpdateRepositoryStorageWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
feature_category :source_code_management
def perform(project_id, new_repository_storage_key)
project = Project.find(project_id)
::Projects::UpdateRepositoryStorageService.new(project).execute(new_repository_storage_key)
rescue ::Projects::UpdateRepositoryStorageService::RepositoryAlreadyMoved
Rails.logger.info "#{self.class}: repository already moved: #{project}" # rubocop:disable Gitlab/RailsLogger
end
end
---
title: Add name_regex_keep param to container registry bulk delete API endpoint
merge_request: 25484
author:
type: added
---
title: Project Snippets GraphQL resolver checks feature status
merge_request: 26158
author:
type: performance
---
title: Fix snippet blob viewers for rich and plain data
merge_request: 25945
author:
type: fixed
---
title: "Backport API support to move between repository storages/shards"
merge_request: 18721
author: Ben Bodenmiller
type: added
---
title: Automatically include embedded metrics for GitLab alert incidents
merge_request: 25277
author:
type: added
......@@ -231,7 +231,9 @@ DELETE /projects/:id/registry/repositories/:repository_id/tags
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. |
| `repository_id` | integer | yes | The ID of registry repository. |
| `name_regex` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`.|
| `name_regex` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. **Note:** `name_regex` is deprecated in favor of `name_regex_delete`.|
| `name_regex_delete` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`.|
| `name_regex_keep` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to keep. This value will override any matches from `name_regex_delete`. Note: setting to `.*` will result in a no-op. |
| `keep_n` | integer | no | The amount of latest tags of given name to keep. |
| `older_than` | string | no | Tags to delete that are older than the given time, written in human readable form `1h`, `1d`, `1month`. |
......@@ -239,7 +241,7 @@ This API call performs the following operations:
1. It orders all tags by creation date. The creation date is the time of the
manifest creation, not the time of tag push.
1. It removes only the tags matching the given `name_regex`.
1. It removes only the tags matching the given `name_regex_delete` (or deprecated `name_regex`), keeping any that match `name_regex_keep`.
1. It never removes the tag named `latest`.
1. It keeps N latest matching tags (if `keep_n` is specified).
1. It only removes tags that are older than X amount of time (if `older_than` is specified).
......@@ -261,17 +263,23 @@ Examples:
and remove ones that are older than 2 days:
```shell
curl --request DELETE --data 'name_regex=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
curl --request DELETE --data 'name_regex_delete=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
1. Remove all tags, but keep always the latest 5:
```shell
curl --request DELETE --data 'name_regex=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
curl --request DELETE --data 'name_regex_delete=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
1. Remove all tags, but keep always tags beginning with `stable`:
```shell
curl --request DELETE --data 'name_regex_delete=.*' --data 'name_regex_keep=stable.*' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
1. Remove all tags that are older than 1 month:
```shell
curl --request DELETE --data 'name_regex=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
curl --request DELETE --data 'name_regex_delete=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment