Read about our upcoming Code of Conduct on this issue

Commit 284b1c18 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Merged heptapod-stable into release branch

--HG--
branch : heptapod-0-21
......@@ -112,13 +112,15 @@ export const createCommitPayload = ({
actions: getCommitFiles(rootState.stagedFiles).map((f) => {
const isBlob = isBlobUrl(f.rawPath);
const content = isBlob ? btoa(f.content) : f.content;
const action = commitActionForFile(f);
return {
action: commitActionForFile(f),
action: action,
file_path: f.path,
previous_path: f.prevPath || undefined,
content: f.prevPath && !f.changed ? null : content || undefined,
encoding: isBlob ? 'base64' : 'text',
hg_preserve_line_endings: (action == 'update' && !isBlob) || undefined,
last_commit_id: newBranch || f.deleted || f.prevPath ? undefined : f.lastCommitSha,
};
}),
......
......@@ -22,6 +22,14 @@ def create_commit(service, success_path:, failure_path:, failure_view: nil, succ
branch_name: @branch_name
)
unless params[:file].present?
# except for file upload (can be binary and otherwise expected to be
# exactly what the user wants), we want to preserve line endings
# for Mercurial instead of replacing them by the CR/LF produced by
# textareas (see heptapod#445)
commit_params[:hg_preserve_line_endings] = true
end
result = service.new(@project_to_commit_into, current_user, commit_params).execute
if result[:status] == :success
......
......@@ -808,9 +808,18 @@ def create_file(user, path, content, **options)
def update_file(user, path, content, **options)
previous_path = options.delete(:previous_path)
action = previous_path && previous_path != path ? :move : :update
action = {
action: previous_path && previous_path != path ? :move : :update,
file_path: path,
previous_path: previous_path,
content: content
}
# in all cases we want to remove it from global options
hg_keep_eol = options.delete(:hg_preserve_line_endings)
# and push it down in the action options only if relevant
action[:hg_preserve_line_endings] = hg_keep_eol if mercurial?
options[:actions] = [{ action: action, file_path: path, previous_path: previous_path, content: content }]
options[:actions] = [action]
multi_action(user, **options)
end
......@@ -856,6 +865,11 @@ def multi_action(user, **options)
end
end
unless mercurial?
# remove Mercurial-specific action parameters
options[:actions].each { |action| action.delete(:hg_preserve_line_endings) }
end
with_cache_hooks { raw.multi_action(user, **options) }
end
......
......@@ -7,6 +7,7 @@ def create_commit!
message: @commit_message,
branch_name: @branch_name,
previous_path: @previous_path,
hg_preserve_line_endings: params[:hg_preserve_line_endings],
author_email: @author_email,
author_name: @author_name,
start_project: @start_project,
......
......@@ -13,9 +13,9 @@ def initialize(user, params)
end
def vcs_type(params)
case params[:import_type]
case params[:import_type]&.to_sym
# Same ordering as in _import_project_pane.html.haml
when :github, :bitbucket_server, :gitlab, :fogbugz, :gitea, :phabricator
when :github, :bitbucket, :bitbucket_server, :gitlab, :fogbugz, :gitea, :phabricator
'git'
else
return @project.group.default_vcs_type unless @project.group.nil?
......
......@@ -93,6 +93,7 @@ def authorize_push_to_branch!(branch)
end
given action: ->(action) { action == 'update' } do
requires :content, type: String, desc: 'File content'
optional :hg_preserve_line_endings, type: Boolean, desc: '(Ignored if not a Mercurial project). If true, try and keep line endings as they are in previous content', default: false
end
optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64]
given action: ->(action) { %w[update move delete].include? action } do
......
......@@ -25,6 +25,8 @@ class HgGitRepository < Gitlab::Git::Repository
# so let's subclass it.
ActionError = Class.new(Gitlab::Git::Index::IndexError)
UNIX_EOL = Gitlab::Utils::LineEndings::UNIX
DEFAULT_HGRC = [
"# This is the specific configuration for this repository",
"# You may modify it freely at your will.",
......@@ -344,7 +346,7 @@ def hgsha_from_rev(hgrev, path = nil)
hg_call(['log', '--limit', '1', '-r', hgrev, '-T', '{node}'],
path: path).first.strip
rescue HgError
raise HgRevisionError, "Could not find changeset #{hgrev} in #{@hgpath}"
raise HgRevisionError, "Could not find changeset #{hgrev} in #{path || @hgpath}"
end
# Find the named branch of a Mercurial changeset
......@@ -1234,7 +1236,6 @@ def multi_action(
"start_branch_name=#{start_branch_name.inspect} "\
"branch_name=#{branch_name.inspect}"
LOGGER.info("#{logprefix} starting")
hg_env = hg_env_for_write(user)
hg_env.delete('HEPTAPOD_SKIP_ALL_GITLAB_HOOKS') unless unit_tests_skip_hooks
......@@ -1348,7 +1349,17 @@ def hg_single_working_dir_action(working_dir, hg_env, action, logprefix)
raise ActionError, "A file with this name doesn't exist"
end
write_file(abspath, action[:content])
content = action[:content]
if action[:hg_preserve_line_endings]
content = Gitlab::Utils::LineEndings.replace_as_in(
content,
File.read(abspath),
solve_inconsistency_as: UNIX_EOL,
default: UNIX_EOL)
end
write_file(abspath, content)
when :create_dir
if File.file?(abspath)
raise ActionError, "A file with this name already exists "
......
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
# frozen_string_literal: true
module Gitlab
module Utils
module LineEndings
InconsistentLineEndings = Class.new(StandardError)
UNIX = "\n"
WINDOWS = "\r\n"
HTML_TEXTAREA = WINDOWS
MAC_CLASSIC = "\r" # MacOS before OSX
extend self
# Analyze content for consistent line endings
#
# returns
# - the line ending (`UNIX`, `WINDOWS`, `MAC_CLASSIC`) if it is consistently
# used in `content`,
# - :mixed if several distinct line endings are in use
# - or nil if there is no line ending
#
# If last line doesn't have an End-of-Line code, it doesn't count as an inconsistency
def analyze(content)
res = nil
lines = content.lines
return if lines.empty?
# ordering matters, because we'll use early breaks in early loops
# keeping as litterals instead of constants to make what happens clear
eols = ["\r\n", "\n", "\r"]
content.lines.each do |line|
eols.each do |eol|
# special cases for MAC_CLASSIC due to Ruby line spliting not recognizing it.
# this eol is checked last. With the `break` for other eols, include? is enough.
if eol == MAC_CLASSIC && line.include?(eol)
res ||= eol
return :mixed if res != eol
elsif line.ends_with?(eol)
# chomp considers \r to be a line ending, but it only removes the last
# line ending, so could be usable. Still, that makes a whole copy
# that we'd better avoid
if line[..-eol.length - 1].include?(MAC_CLASSIC)
return :mixed
end
if res.nil?
res = eol
elsif res != eol
return :mixed
end
break
end
end
end
res
end
# Return a version of `content` with line endings as in as in `reference_content`
#
# Limitations: does not understand MAC_CLASSIC line endings in `content` (not needed
# for current applications as of this writing)
#
# Param `default`: can be used to specify a default line ending in case `reference_content`
# has no line endings.
#
# Param `solve_inconsistency_as`: if `reference_content` has inconsistent line endings,
# this is an end of line to choose instead. If not given, `InconsistentLineEndings`
# is raised in that situation.
def replace_as_in(content, reference_content, default: UNIX, solve_inconsistency_as: nil)
ref_eol = analyze(reference_content)
if ref_eol == :mixed
raise InconsistentLineEndings if solve_inconsistency_as.nil?
ref_eol = solve_inconsistency_as
end
# could not find how to join iterators without collecting first in an Array
# special case: if last line endswith '\r' the chomp will remove it because
# line splitting and chomp aren't consistent. Is it really worth
content.each_line.map { |l| l.chomp }.to_a.join(ref_eol || default)
end
end
end
end
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
# frozen_string_literal: true
require 'fast_spec_helper'
RSpec.describe Gitlab::Utils::LineEndings do
describe '.analyze' do
context 'when input is Unix-style' do
it 'returns the Unix end-of-line' do
expect(described_class.analyze("abc\ndef\n")).to eq("\n")
expect(described_class.analyze("abc\ndef")).to eq("\n")
end
end
context 'when input is Windows style' do
it 'returns the Windows end-of-line' do
expect(described_class.analyze("abc\r\ndef\r\n")).to eq("\r\n")
expect(described_class.analyze("abc\r\ndef")).to eq("\r\n")
end
end
context 'when input is MacOS old style' do
it 'returns the Mac classic end-of-line' do
expect(described_class.analyze("abc\rdef\r")).to eq("\r")
expect(described_class.analyze("abc\rdef")).to eq("\r")
end
end
context 'when input does not have any line end' do
it 'returns nil' do
expect(described_class.analyze("just a word")).to be_nil
end
end
context 'when input is mix of Unix and Windows' do
it 'returns :mixed' do
expect(described_class.analyze("unix\n win\r\n")).to eq(:mixed)
expect(described_class.analyze("unix\n win\r\n one more word")).to eq(:mixed)
expect(described_class.analyze("win\r\n unix\n")).to eq(:mixed)
expect(described_class.analyze("win\r\n unix\n one more word")).to eq(:mixed)
end
end
context 'when input is mix of Unix and old Mac' do
it 'returns :mixed' do
expect(described_class.analyze("unix\n mac\r")).to eq(:mixed)
expect(described_class.analyze("unix\n mac\r one more word")).to eq(:mixed)
expect(described_class.analyze("mac\r unix\n")).to eq(:mixed)
expect(described_class.analyze("mac\r unix\n one more word")).to eq(:mixed)
end
end
context 'when input is mix of Windows and old Mac' do
it 'returns :mixed' do
expect(described_class.analyze("win\r\n mac\r")).to eq(:mixed)
expect(described_class.analyze("win\r\n mac\r one more word")).to eq(:mixed)
expect(described_class.analyze("mac\r win\r\n")).to eq(:mixed)
expect(described_class.analyze("mac\r win\r\n one more word")).to eq(:mixed)
end
end
context 'when input is mix of everything' do
it 'returns :mixed' do
expect(described_class.analyze("win\r\n mac\r unix\n addendum")).to eq(:mixed)
end
end
end
describe '.replace_as_in' do
context 'when reference content is consistent' do
it 'performs the replacement' do
expect(described_class.replace_as_in("a\r\n b", "reference\n content\n"))
.to eq("a\n b")
end
end
context 'when reference content is inconsistent' do
it 'raises by default' do
expect { described_class.replace_as_in("a\nb", "incon\n sis\r\n tent\r") }
.to raise_error(Gitlab::Utils::LineEndings::InconsistentLineEndings)
end
it 'uses solve_inconsistency_as if given' do
expect(described_class.replace_as_in("a\r\n bc\r\n d", "incon\n sis\r\n tent\r",
solve_inconsistency_as: "\n"))
.to eq("a\n bc\n d")
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment