Commit 74b14865 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Merged heptapod branch into release branch

--HG--
branch : heptapod-0-21
### Main reason driving the release
### Information about the release
#### Relevant series
<!--
Either `default`, `stable` or `oldstable`.
-->
#### Main reason driving the release
<!---
......@@ -34,11 +44,14 @@ reached".
+ [ ] merge stabler branches into the branch to release and run a pipeline
to check them (fast)
+ [ ] set the version in `setup.py`, commit
+ [ ] set the version in `hgitaly/VERSION`, commit
+ [ ] run `$python setup.py sdist and `push to PyPI with
[twine](https://twine.readthedocs.io/en/latest/)
+ [ ] tag accordingly to the pushed version (`hg tag 1.2.3`)
+ [ ] set a new development version in `setup.py`, commit
+ [ ] set a new development version in `hgitaly/VERSION`,
such as `1.2.4dev0` (no dot for proper interpretation by the
Rails app);
commit
+ [ ] push and publish these three changesets
- Heptapod Shell: again, if needed which is more than likely of
......@@ -63,10 +76,10 @@ reached".
+ [ ] merge stabler development branches into the development branch
(pipelines not really needed).
+ [ ] start a topic from the head of the release branch (e.g, `heptapod-0-20`)
+ [ ] merge the `heptapod` branch into the release branch if needed. Commit.
+ [ ] merge the development branch into the release branch if needed. Commit.
+ [ ] fill in [HEPTAPOD_CHANGELOG](HEPTAPOD_CHANGELOG.md)
Be sure to include the changes related to the stable branches. If needed
copy them from the stable *release* branches, as these won't get merged.
Be sure to include the changes related to the stabler branches. If needed
copy them from the stabler *release* branches, as these won't get merged.
+ [ ] set HEPTAPOD_VERSION
+ [ ] update HEPTAPOD_SHELL_VERSION if needed
+ [ ] update the requirements files in the [python](python) subdirectory
......
......@@ -117,7 +117,7 @@ with this data:
- `project`: full web path to the project
Note: disabling the fully native mode this way does not work if it is enabled for the
whole instance. Enable the `hgitaly_fallback` feature flag instead.
whole instance. Enable the `hgitaly1_fallback` feature flag instead.
##### Under the [Rails console]:
......
......@@ -450,6 +450,35 @@ def translate_params_for_compatibility(params)
user_project.repository.get_hgrc.transform_keys { |k| k.tr('-', '_') }
end
desc 'Censor file in a Mercurial revision'
params do
requires :hash, regexp: /\h{12,}/, type: String, desc: 'Censor file from specified hash of length >= 12'
requires :file, type: String, desc: 'File to be censored'
optional :tombstone, type: String, desc: 'Replacement tombstone data. Must be UTF8 and length + 15 (in bytes) must be <= of that of the original data.'
optional :file_is_base64, type: Boolean, desc: 'If `file` is base64 encoded. Otherwise is interpreted as UTF8. This is useful if the filename is not valid UTF8.'
end
post ':id/hg_censor_file' do
authorize_admin_project
unless user_project.mercurial?
render_api_error!("Not a Mercurial project!", 400)
end
args = declared_params(include_missing: false)
file = if args[:file_is_base64]
Base64.decode64(args[:file])
else
args[:file]
end
user_project.repository.hg_censor(current_user, args[:hash], file, tombstone: args[:tombstone])
no_content!
rescue Gitlab::Mercurial::HgGitRepository::HgError
render_api_error!("'hg censor' command failed. Please check the file name at the given changeset, and make sure that the tombstone size (default empty) is smaller or equal to the file size - 15 (in bytes). More details may be available in server logs", 400)
end
desc 'Archive a project' do
success Entities::Project
end
......
......@@ -21,8 +21,7 @@ def ls_files(revision)
revision: encode_binary(revision)
)
# TODO HGitaly2!
response = GitalyClient.call(@git_storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout)
response = GitalyClient.call(@storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout)
response.flat_map do |msg|
msg.paths.map { |d| EncodingHelper.encode!(d.dup) }
end
......
......@@ -1216,8 +1216,8 @@ def multi_action(
force: false
)
logprefix = "hg_multi_action for #{@relative_path} "\
"start_branch_name=#{start_branch_name} "\
"branch_name=#{branch_name}"
"start_branch_name=#{start_branch_name.inspect} "\
"branch_name=#{branch_name.inspect}"
LOGGER.info("#{logprefix} starting")
hg_env = hg_env_for_write(user)
......@@ -1312,7 +1312,7 @@ def hg_single_working_dir_action(working_dir, hg_env, action, logprefix)
abspath = File.join(working_dir, file_path)
LOGGER.info(
"#{logprefix} Performing '#{action[:action]}' for #{file_path}")
"#{logprefix} Performing '#{action[:action]}' for #{file_path.inspect}")
case action[:action].to_sym
when :create
if File.exist?(abspath)
......@@ -1503,6 +1503,24 @@ def get_hgrc
res
end
def hg_censor(user, hash, file, tombstone: nil)
args = [
"censor",
file,
"--config",
"extensions.censor=",
"--rev",
hash
]
unless tombstone.nil?
args << "--tombstone"
args << tombstone
end
hg_call(args, user: user, for_write: true)
end
# Check HGRC inheritance from group path and fix it if needed.
#
# This is especially useful in cases of repository relocation:
......
[upload.heptapod.net]:7022 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5HYl4+LHEY9n4+3JhXPLHLwj+55H3lik/cp91AlXpQmmIzVRcFlQEqzMjxFI1kfNOd9Y/k7lZjyF86AVmZ6+e726J4i8rCwyMtATkdM3acRNdtONFVcC9a5CvT71H6U0OFCOS8hVCv/pBs/FoYwC+foxmMltZp2kpmhhqRQyuCTLoFQUdYYaFysXRtoo3Yy5nO8Road1M1fIib0e+PB2ALzJsgKjTiINk7sWXF2AfC7Phw8kOkYIL285eBftoh6LQ/oKshYXs1AULe/gaFgbhfQ9H2M5xiR+ddqkVNAlV1nJLYtRvCdGavYzX3F/hKXQjRS9TDmiinspK9CnaGje9
[upload.heptapod.net]:7022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFhzij3QcQHkTZ/d5pia53FPBKAw6uo1MugXtppURwqYd1Wlk/yEoFP4VfamWpZvLW5anB2SDh0PrkPv8NYDBnM=
[download.heptapod.net]:7022 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGQYhX/90l97jZgG33PXfobTUoetWwuBdAMDS0rSaU3v
......@@ -5,7 +5,7 @@ export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
export BUNDLE_INSTALL_FLAGS=${BUNDLE_INSTALL_FLAGS:-"--without=production development --jobs=$(nproc) --path=vendor --retry=3 --quiet"}
if [ "$HEPTAPOD_USE_GITLAB_PREBUILDS" != "false" ]; then
GITLAB_PREBUILDS_URL=https://download.heptapod.net/gitlab-prebuilds/ci-$(cat VERSION).tar.xz
GITLAB_PREBUILDS_URL=https://upload.heptapod.net/gitlab-prebuilds/ci-$(cat VERSION).tar.xz
echo "Downloading $GITLAB_PREBUILDS_URL"
wget --progress=dot:giga -O /tmp/gitlab-prebuilds.tar.xz $GITLAB_PREBUILDS_URL
run_timed_command "tar -x --xz -f /tmp/gitlab-prebuilds.tar.xz"
......
......@@ -211,6 +211,21 @@ def gitsha(sha)
end
end
describe 'commit_service#list_files' do
subject! { non_empty_repository }
after do
subject.remove
end
it 'basically works' do
# we don't need comprehensive tests, just a proof that it works
# (roundtrip with HGitaly happened, everything was compatible)
# Full comparison tests with Gitaly are provided in HGitaly.
expect(subject.ls_files('branch/default')).to eq(["foo"])
end
end
describe 'commit_service#find_commits_by_message' do
subject! { non_empty_repository }
......
# frozen_string_literal: true
require "spec_helper"
# rubocop:disable RSpec/FilePath
RSpec.describe "Diff access HgitalyRepository", :seed_helper do
include Gitlab::EncodingHelper
include RepoHelpers
include Gitlab::Popen
def hgitaly_repo_init
repo = Gitlab::Mercurial::HgitalyRepository.new(
'default', 'hgitaly-repo.git', '', 'group/project')
repo.unit_tests_skip_hooks = true
repo
end
let(:hgitaly_repo) { hgitaly_repo_init }
before(:all) do
repo = hgitaly_repo_init
status = repo.create_repository(namespace_fs_path: "group", in_subgroup: false)
expect(status).to be(true)
user = build(:user)
repo.multi_action(user,
branch_name: 'branch/default',
message: "Commit 0",
author_email: "jdoe1@heptapod.test",
author_name: "John Doe1",
actions: [{ action: :create,
file_path: 'pet',
content: "first_pet\nsecond_pet\n" }])
repo.expire_has_local_branches_cache # we have no post-receive hooks
repo.multi_action(user,
branch_name: 'branch/default',
start_branch_name: 'branch/default',
message: "Commit 1",
author_email: "jdoe1@heptapod.test",
author_name: "John Doe1",
actions: [{ action: :create,
file_path: "sub/car",
content: "first_car\nsecond_car\n" }])
end
# Same repository as :repository, seen as HgGitRepository, so that
#
# 1. we can use HgGitRepository as a reference for assertions on method
# results
# 2. we have easy fixture preparation options
let(:hg_git_repo) do
repo = Gitlab::Mercurial::HgGitRepository.new(
hgitaly_repo.storage,
hgitaly_repo.relative_path,
hgitaly_repo.gl_repository,
hgitaly_repo.gl_project_path)
repo.unit_tests_skip_hooks = true
repo
end
after(:all) do
hgitaly_repo_init.remove
end
shared_examples 'DiffService common integration tests' do
let(:hg_sha1) { hgitaly_repo.hgsha_from_rev('default') }
let(:git_sha1) { hg_git_repo.commit.sha }
let(:hg_sha0) { hgitaly_repo.hgsha_from_rev('0') }
let(:git_sha0) do
hg_git_repo.hg_git_invalidate_maps!
hg_git_repo.sha_from_hgsha(hg_sha0)
end
it 'repo.diff_stats tests DiffStats' do
hg_resp = hgitaly_repo.diff_stats(hg_sha0, hg_sha1).to_a
git_resp = hg_git_repo.diff_stats(git_sha0, git_sha1).to_a
expect(hg_resp).to eq(git_resp)
stats = hg_resp[0]
expect(stats.path).to eq("sub/car")
expect(stats.additions).to eq(2)
expect(stats.deletions).to eq(0)
end
it 'repo.diff tests CommitDiff' do
hg_resp = hgitaly_repo.diff(hg_sha0, hg_sha1).to_a[0]
git_resp = hg_git_repo.diff(git_sha0, git_sha1).to_a[0]
expect(hg_resp.to_json).to eq(git_resp.to_json)
end
it 'commit.deltas tests CommitDelta' do
hg_commit1 = Gitlab::Git::Commit.find(hgitaly_repo, 'branch/default')
git_commit1 = Gitlab::Git::Commit.find(hg_git_repo, 'branch/default')
hg_resp = hg_commit1.deltas[0]
git_resp = git_commit1.deltas[0]
expect(hg_resp.to_json).to eq(git_resp.to_json)
end
# TODO: Decide if we want to test and how (probably using Request Specs)
# RawDiff and RawPatch methods, as Rails don't parse their responses
# anywhere and directly sent to the frontend.
#
# We would think about FindChangedPaths too, but for now Rails is not
# calling any RPC for this method.
# it 'Workhorse.send_git_patch tests RawPatch' do
# end
# it 'Workhorse.send_git_diff tests RawDiff' do
# end
end
describe 'In HGitaly1 mode' do
before do
hgitaly_repo.is_hgitaly1 = true
end
it_behaves_like 'DiffService common integration tests'
end
describe 'In fully native mode' do
before do
hgitaly_repo.is_hgitaly1 = false
end
it_behaves_like 'DiffService common integration tests'
end
end
......@@ -693,4 +693,50 @@ def hg_commit(cid, topic: nil)
expect(response).not_to have_key 'nonexistent'
end
end
describe '#log' do
subject! { non_empty_repository }
def hg_commit(cid)
subject.write_file(subject.hg_full_path + "/file#{cid}", "line\n")
subject.hg_call(["add", "file#{cid}"])
subject.hg_call(['ci', '-m', "Commit #{cid}", '--config',
'experimental.single-head-per-branch=False'],
user: user, for_write: true)
subject.hg_git_invalidate_maps!
end
# repo structure:
#
# o 1 (branch/default)
# |
# 0
#
let(:prepare_repo) do
subject.hg_call(%w[up 0])
hg_commit(1)
end
before do
prepare_repo
end
after do
subject.remove
end
it 'return commits for the given ref name' do
hg_sha0 = subject.hgsha_from_rev('0')
hg_sha1 = subject.hgsha_from_rev('1')
hg_git_repository.hg_git_invalidate_maps!
git_sha0 = hg_git_repository.sha_from_hgsha(hg_sha0)
git_sha1 = hg_git_repository.sha_from_hgsha(hg_sha1)
resp = subject.log({ ref: 'branch/default' }).map(&:id)
expect(resp).to eq([hg_sha1, hg_sha0])
# consistency with HgGitRepository
resp = hg_git_repository.log({ ref: 'branch/default' }).map(&:id)
expect(resp).to eq([git_sha1, git_sha0])
end
end
end
......@@ -3737,6 +3737,91 @@ def failure_message(diff)
end
end
describe 'POST /projects/:id/hg_censor_file' do
context 'when authenticated as admin' do
let(:project) { create(:hg_git_project, :hg_repository) }
after do
project.repository.remove
end
it 'fails to censor if not admin' do
post api("/projects/#{project.id}/hg_censor_file", user), params: { hash: "abcabcabcabc", file: "filename" }
expect(response).to have_gitlab_http_status(:not_found)
end
it 'fails to censor if not a hash' do
post api("/projects/#{project.id}/hg_censor_file", admin), params: { hash: 10, file: "filename" }
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['error']).to eq('hash is invalid')
end
it 'converts Mercurial errors in the expected errors' do
project.repository.multi_action(project.creator,
branch_name: 'branch/default',
message: "Commit 1",
author_email: "jdoe1@heptapod.test",
author_name: "John Doe1",
actions: [{ action: :update,
file_path: 'README',
content: "Cannot censor heads" }])
commit_hash = project.repository.hgsha_from_rev("0")
# tombstone is too big
post api("/projects/#{project.id}/hg_censor_file", admin),
params: { hash: commit_hash, tombstone: "abc" * 100, file: "README" }
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response['message']).to include("'hg censor' command failed")
end
it 'censors a file successfully' do
project.repository.multi_action(project.creator,
branch_name: 'branch/default',
message: "Commit 1",
author_email: "jdoe1@heptapod.test",
author_name: "John Doe1",
actions: [{ action: :update,
file_path: 'README',
content: "Cannot censor heads" }])
commit_hash = project.repository.hgsha_from_rev("0")
post api("/projects/#{project.id}/hg_censor_file", admin), params: { hash: commit_hash, file: "README" }
expect(response).to have_gitlab_http_status(:no_content)
end
it 'supports non-utf8 files' do
# 0xe9 isn't allowed in UTF-8. It is ISO-8859-1, but we don't indicate it on the
# string, otherwise Ruby will transparently convert to UTF-8
non_utf8 = [0xe9].pack("c*")
project.repository.multi_action(project.creator,
branch_name: 'branch/default',
message: "Commit 1",
author_email: "jdoe1@heptapod.test",
author_name: "John Doe1",
actions: [{ action: :create,
file_path: non_utf8,
content: "Non utf8 but still bigger than 15 bytes" }])
project.repository.multi_action(project.creator,
branch_name: 'branch/default',
message: "Commit 2",
author_email: "jdoe1@heptapod.test",
author_name: "John Doe1",
actions: [{ action: :update,
file_path: non_utf8,
content: "Cannot censor heads" }])
commit_hash = project.repository.hgsha_from_rev("1")
post api("/projects/#{project.id}/hg_censor_file", admin), params: { hash: commit_hash, file: Base64.encode64(non_utf8), file_is_base64: true }
expect(response).to have_gitlab_http_status(:no_content)
end
end
end
it_behaves_like 'custom attributes endpoints', 'projects' do
let(:attributable) { project }
let(:other_attributable) { project2 }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment