Read about our upcoming Code of Conduct on this issue

Commit 23b11deb authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

heptapod#542: making 0.25 the new stable

Pipeline #27729 passed with stage
in 3 minutes and 29 seconds
......@@ -30,6 +30,8 @@
- gitlab-org-docker
.test:
variables:
GITALY_CONNECTION_INFO: '{"address":"tcp://gitaly:8075", "storage":"default"}'
before_script:
# Set up the environment to run integration tests (still written in Ruby)
- apt-get update -qq && apt-get install -y ruby ruby-dev
......@@ -42,6 +44,11 @@
- cp config.yml.example config.yml
- go version
- which go
services:
- name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
# Disable the hooks so we don't have to stub the GitLab API
command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && exec /usr/bin/env GITALY_TESTING_NO_GIT_HOOKS=1 /scripts/process-wrapper"]
alias: gitaly
script:
- make verify test
......@@ -77,7 +84,7 @@
rules: *workflow_rules
code_navigation:
image: sourcegraph/lsif-go:v1
image: sourcegraph/lsif-go:v1.3.1
allow_failure: true
script:
- lsif-go
......
......@@ -32,3 +32,5 @@
4def2dd0132fde7ae4fd57ea21d402860dcd3612 heptapod-13.17.0
ed9dd657a0ab833fd60fe809385f2b73947d0819 heptapod-13.18.0
3cfed32ac2038975b533352ff0f4839aa2a0a319 heptapod-13.18.1
aefbb5e3eac5411fc526a88d3df1762ecbdc2e76 heptapod-13.19.0
b84ce63181d5c0d07585be867dbe957eb6d35348 heptapod-13.19.1
v13.19.1
- Modify regex to prevent partial matches
v13.19.0
- Don't finish the opentracing span early !466
- gitlab-sshd: Respect the ssl_cert_dir config !467
- Stop changing directory to the filesystem root !470
- Fix opentracing setup for gitlab-sshd !473
v13.18.1
- Modify regex to prevent partial matches
......
......@@ -9,6 +9,16 @@
The corresponding tag is heptapod-x.y.z
## 13.19.1
- No actual changes
## 13.19.0
- Upstream version bump to v13.19.0 (GitLab 14.0 and 14.1 series)
(done as a merge on top of 13.18.1, hence includes the
fixes in upstream 13.19.1.)
## 13.18.1
- Upstream version bump to v13.18.1 (GitLab 13.12.9)
......
......@@ -3,7 +3,8 @@
GO_SOURCES := $(shell find . -name '*.go')
VERSION_STRING := $(shell ./compute_version || echo unknown)
BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S)
GOBUILD_FLAGS := -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)"
BUILD_TAGS := tracer_static tracer_static_jaeger continuous_profiler_stackdriver
GOBUILD_FLAGS := -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)" -tags "$(BUILD_TAGS)"
validate: verify test
......
......@@ -64,6 +64,20 @@
bundle install
make validate
### Gitaly
Some tests need a Gitaly server. The
[`docker-compose.yml`](./docker-compose.yml) file will run Gitaly on
port 8075. To tell the tests where Gitaly is, set
`GITALY_CONNECTION_INFO`:
export GITALY_CONNECTION_INFO='{"address": "tcp://localhost:8075", "storage": "default"}'
make test
If no `GITALY_CONNECTION_INFO` is set, the test suite will still run, but any
tests requiring Gitaly will be skipped. They will always run in the CI
environment.
## Git LFS remark
Starting with GitLab 8.12, GitLab supports Git LFS authentication through SSH.
......
......@@ -9,7 +9,7 @@
"testing"
"github.com/stretchr/testify/require"
pb "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
pb "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
......
......@@ -39,7 +39,7 @@
os.Exit(1)
}
ctx, finished := command.ContextWithCorrelationID()
ctx, finished := command.Setup(executable.Name, config)
defer finished()
if err = cmd.Execute(ctx); err != nil {
......
......@@ -42,7 +42,7 @@
os.Exit(1)
}
ctx, finished := command.ContextWithCorrelationID()
ctx, finished := command.Setup(executable.Name, config)
defer finished()
if err = cmd.Execute(ctx); err != nil {
......
......@@ -42,7 +42,7 @@
os.Exit(1)
}
ctx, finished := command.ContextWithCorrelationID()
ctx, finished := command.Setup(executable.Name, config)
defer finished()
if err = cmd.Execute(ctx); err != nil {
......
......@@ -57,7 +57,7 @@
os.Exit(1)
}
ctx, finished := command.ContextWithCorrelationID()
ctx, finished := command.Setup(executable.Name, config)
defer finished()
if err = cmd.Execute(ctx); err != nil {
......
......@@ -4,6 +4,7 @@
"bufio"
"context"
"crypto/ed25519"
"encoding/json"
"encoding/pem"
"fmt"
"io"
......@@ -16,8 +17,9 @@
"path/filepath"
"regexp"
"runtime"
"strings"
"testing"
"github.com/mikesmitty/edkey"
"github.com/pires/go-proxyproto"
"github.com/stretchr/testify/require"
......@@ -19,9 +21,12 @@
"testing"
"github.com/mikesmitty/edkey"
"github.com/pires/go-proxyproto"
"github.com/stretchr/testify/require"
gitalyClient "gitlab.com/gitlab-org/gitaly/v14/client"
pb "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitlab-shell/internal/testhelper"
"golang.org/x/crypto/ssh"
)
var (
......@@ -24,7 +29,8 @@
"golang.org/x/crypto/ssh"
)
var (
sshdPath = ""
sshdPath = ""
gitalyConnInfo *gitalyConnectionInfo
)
......@@ -29,5 +35,16 @@
)
const (
testRepo = "test-gitlab-shell/gitlab-test.git"
testRepoNamespace = "test-gitlab-shell"
testRepoImportUrl = "https://gitlab.com/gitlab-org/gitlab-test.git"
)
type gitalyConnectionInfo struct {
Address string `json:"address"`
Storage string `json:"storage"`
}
func init() {
rootDir := rootDir()
sshdPath = filepath.Join(rootDir, "bin", "gitlab-sshd")
......@@ -35,6 +52,11 @@
if _, err := os.Stat(sshdPath); os.IsNotExist(err) {
panic(fmt.Errorf("cannot find executable %s. Please run 'make compile'", sshdPath))
}
gci, exists := os.LookupEnv("GITALY_CONNECTION_INFO")
if exists {
json.Unmarshal([]byte(gci), &gitalyConnInfo)
}
}
func rootDir() string {
......@@ -46,7 +68,30 @@
return filepath.Join(filepath.Dir(currentFile), "..", "..")
}
func ensureGitalyRepository(t *testing.T) {
if os.Getenv("GITALY_CONNECTION_INFO") == "" {
t.Skip("GITALY_CONNECTION_INFO is not set")
}
conn, err := gitalyClient.Dial(gitalyConnInfo.Address, gitalyClient.DefaultDialOpts)
require.NoError(t, err)
namespace := pb.NewNamespaceServiceClient(conn)
repository := pb.NewRepositoryServiceClient(conn)
// Remove the repository if it already exists, for consistency
rmNsReq := &pb.RemoveNamespaceRequest{StorageName: gitalyConnInfo.Storage, Name: testRepoNamespace}
_, err = namespace.RemoveNamespace(context.Background(), rmNsReq)
require.NoError(t, err)
gl_repository := &pb.Repository{StorageName: gitalyConnInfo.Storage, RelativePath: testRepo}
createReq := &pb.CreateRepositoryFromURLRequest{Repository: gl_repository, Url: testRepoImportUrl}
_, err = repository.CreateRepositoryFromURL(context.Background(), createReq)
require.NoError(t, err)
}
func successAPI(t *testing.T) http.Handler {
t.Helper()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
......@@ -49,7 +94,11 @@
func successAPI(t *testing.T) http.Handler {
t.Helper()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
testDirCleanup, err := testhelper.PrepareTestRootDir()
require.NoError(t, err)
defer testDirCleanup()
t.Logf("gitlab-api-mock: received request: %s %s", r.Method, r.RequestURI)
w.Header().Set("Content-Type", "application/json")
......@@ -58,4 +107,24 @@
fmt.Fprintf(w, `{"id":1, "key":"%s"}`, r.FormValue("key"))
case "/api/v4/internal/discover":
fmt.Fprint(w, `{"id": 1000, "name": "Test User", "username": "test-user"}`)
case "/api/v4/internal/personal_access_token":
fmt.Fprint(w, `{"success": true, "token": "testtoken", "scopes": ["api"], "expires_at": ""}`)
case "/api/v4/internal/two_factor_recovery_codes":
fmt.Fprint(w, `{"success": true, "recovery_codes": ["code1", "code2"]}`)
case "/api/v4/internal/two_factor_otp_check":
fmt.Fprint(w, `{"success": true}`)
case "/api/v4/internal/allowed":
body, err := ioutil.ReadFile(filepath.Join(testhelper.TestRoot, "responses/allowed_without_console_messages.json"))
require.NoError(t, err)
response := strings.Replace(string(body), "GITALY_REPOSITORY", testRepo, 1)
if gitalyConnInfo != nil {
response = strings.Replace(response, "GITALY_ADDRESS", gitalyConnInfo.Address, 1)
}
fmt.Fprint(w, response)
require.NoError(t, err)
case "/api/v4/internal/lfs_authenticate":
fmt.Fprint(w, `{"username": "test-user", "lfs_token": "testlfstoken", "repo_path": "foo", "expires_in": 7200}`)
default:
......@@ -61,5 +130,5 @@
default:
t.Log("Unexpected request to successAPI!")
t.Logf("Unexpected request to successAPI: %s", r.URL.EscapedPath())
t.FailNow()
}
})
......@@ -217,3 +286,164 @@
require.NoError(t, err)
require.Equal(t, "Welcome to GitLab, @test-user!\n", string(output))
}
func TestPersonalAccessTokenSuccess(t *testing.T) {
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
output, err := session.Output("personal_access_token test api")
require.NoError(t, err)
require.Equal(t, "Token: testtoken\nScopes: api\nExpires: never\n", string(output))
}
func TestTwoFactorAuthRecoveryCodesSuccess(t *testing.T) {
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
stdin, err := session.StdinPipe()
require.NoError(t, err)
stdout, err := session.StdoutPipe()
require.NoError(t, err)
reader := bufio.NewReader(stdout)
err = session.Start("2fa_recovery_codes")
require.NoError(t, err)
line, err := reader.ReadString('\n')
require.NoError(t, err)
require.Equal(t, "Are you sure you want to generate new two-factor recovery codes?\n", line)
line, err = reader.ReadString('\n')
require.NoError(t, err)
require.Equal(t, "Any existing recovery codes you saved will be invalidated. (yes/no)\n", line)
_, err = fmt.Fprintln(stdin, "yes")
require.NoError(t, err)
output, err := ioutil.ReadAll(stdout)
require.NoError(t, err)
require.Equal(t, `
Your two-factor authentication recovery codes are:
code1
code2
During sign in, use one of the codes above when prompted for
your two-factor code. Then, visit your Profile Settings and add
a new device so you do not lose access to your account again.
`, string(output))
}
func TwoFactorAuthVerifySuccess(t *testing.T) {
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
stdin, err := session.StdinPipe()
require.NoError(t, err)
stdout, err := session.StdoutPipe()
require.NoError(t, err)
reader := bufio.NewReader(stdout)
err = session.Start("2fa_verify")
require.NoError(t, err)
line, err := reader.ReadString('\n')
require.NoError(t, err)
require.Equal(t, "OTP: ", line)
_, err = fmt.Fprintln(stdin, "otp123")
require.NoError(t, err)
output, err := ioutil.ReadAll(stdout)
require.NoError(t, err)
require.Equal(t, "OTP validation successful. Git operations are now allowed.\n", string(output))
}
func TestGitLfsAuthenticateSuccess(t *testing.T) {
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
output, err := session.Output("git-lfs-authenticate test-user/repo.git download")
require.NoError(t, err)
require.Equal(t, `{"header":{"Authorization":"Basic dGVzdC11c2VyOnRlc3RsZnN0b2tlbg=="},"href":"/info/lfs","expires_in":7200}
`, string(output))
}
func TestGitReceivePackSuccess(t *testing.T) {
ensureGitalyRepository(t)
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
output, err := session.Output(fmt.Sprintf("git-receive-pack %s", testRepo))
require.NoError(t, err)
outputLines := strings.Split(string(output), "\n")
for i := 0; i < (len(outputLines) - 1); i++ {
require.Regexp(t, "^[0-9a-f]{44} refs/(heads|tags)/[^ ]+", outputLines[i])
}
require.Equal(t, "0000", outputLines[len(outputLines)-1])
}
func TestGitUploadPackSuccess(t *testing.T) {
ensureGitalyRepository(t)
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
output, err := session.Output(fmt.Sprintf("git-upload-pack %s", testRepo))
require.NoError(t, err)
outputLines := strings.Split(string(output), "\n")
require.Regexp(t, "^[0-9a-f]{44} HEAD.+", outputLines[0])
for i := 1; i < (len(outputLines) - 1); i++ {
require.Regexp(t, "^[0-9a-f]{44} refs/(heads|tags)/[^ ]+", outputLines[i])
}
require.Equal(t, "0000", outputLines[len(outputLines)-1])
}
func TestGitUploadArchiveSuccess(t *testing.T) {
ensureGitalyRepository(t)
client := runSSHD(t, successAPI(t))
session, err := client.NewSession()
require.NoError(t, err)
defer session.Close()
output, err := session.Output(fmt.Sprintf("git-upload-archive %s", testRepo))
require.NoError(t, err)
outputLines := strings.Split(string(output), "\n")
require.Equal(t, "0008ACK", outputLines[0])
require.Regexp(t, "^0000", outputLines[1])
}
......@@ -6,6 +6,7 @@
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-shell/internal/command"
"gitlab.com/gitlab-org/gitlab-shell/internal/config"
"gitlab.com/gitlab-org/gitlab-shell/internal/logger"
"gitlab.com/gitlab-org/gitlab-shell/internal/sshd"
......@@ -54,5 +55,8 @@
}
log.Fatalf("configuration error: %v", err)
}
cfg.ApplyGlobalState()
logger.ConfigureStandalone(cfg)
......@@ -57,5 +61,8 @@
logger.ConfigureStandalone(cfg)
ctx, finished := command.Setup("gitlab-sshd", cfg)
defer finished()
// Startup monitoring endpoint.
if cfg.Server.WebListen != "" {
go func() {
......@@ -68,7 +75,7 @@
}()
}
if err := sshd.Run(cfg); err != nil {
if err := sshd.Run(ctx, cfg); err != nil {
log.Fatalf("Failed to start GitLab built-in sshd: %v", err)
}
}
version: '3.1'
services:
gitaly:
environment:
- GITALY_TESTING_NO_GIT_HOOKS=1
image: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
ports:
- '8075:8075'
......@@ -3,7 +3,9 @@
go 1.13
require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/mattn/go-shellwords v1.0.11
github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
github.com/otiai10/copy v1.4.2
github.com/pires/go-proxyproto v0.5.0
......@@ -6,10 +8,10 @@
github.com/mattn/go-shellwords v1.0.11
github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
github.com/otiai10/copy v1.4.2
github.com/pires/go-proxyproto v0.5.0
github.com/prometheus/client_golang v1.9.0
github.com/sirupsen/logrus v1.7.0
github.com/stretchr/testify v1.6.1
gitlab.com/gitlab-org/gitaly v1.68.0
gitlab.com/gitlab-org/labkit v1.3.0
github.com/prometheus/client_golang v1.10.0
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0
gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1
gitlab.com/gitlab-org/labkit v1.4.1
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
......@@ -15,5 +17,5 @@
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
google.golang.org/grpc v1.29.1
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
google.golang.org/grpc v1.37.0
gopkg.in/yaml.v2 v2.4.0
)
......@@ -18,7 +20,2 @@
gopkg.in/yaml.v2 v2.4.0
)
// go get tries to enforce semantic version compatibility via module paths.
// We can't upgrade to Gitaly v13.x.x from v1.x.x without using a manual override.
// See https://gitlab.com/gitlab-org/gitaly/-/issues/3177 for more details.
replace gitlab.com/gitlab-org/gitaly => gitlab.com/gitlab-org/gitaly v0.0.0-20201001041716-3f5e218def93
This diff is collapsed.
......@@ -16,6 +16,8 @@
image: golang:1.14
.test:
variables:
GITALY_CONNECTION_INFO: '{"address":"tcp://gitaly:8075", "storage":"default"}'
before_script:
# Set up the environment to run integration tests (still written in Ruby)
- apt-get update -qq && apt-get install -y ruby ruby-dev
......@@ -28,7 +30,11 @@
- cp config.yml.example config.yml
- go version
- which go
- make build verify_golang test_golang
services:
- name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
# Disable the hooks so we don't have to stub the GitLab API
command: ["bash", "-c", "mkdir -p /home/git/repositories && rm -rf /srv/gitlab-shell/hooks/* && exec /usr/bin/env GITALY_TESTING_NO_GIT_HOOKS=1 /scripts/process-wrapper"]
alias: gitaly
script:
- make verify test
......
......@@ -2,7 +2,6 @@
import (
"context"
"os"
"gitlab.com/gitlab-org/gitlab-shell/internal/command/authorizedkeys"
"gitlab.com/gitlab-org/gitlab-shell/internal/command/authorizedprincipals"
......@@ -23,7 +22,6 @@
"gitlab.com/gitlab-org/gitlab-shell/internal/executable"
"gitlab.com/gitlab-org/gitlab-shell/internal/sshenv"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/log"
"gitlab.com/gitlab-org/labkit/tracing"
)
......@@ -38,13 +36,9 @@
}
if cmd := buildCommand(e, args, config, readWriter); cmd != nil {
if config.SslCertDir != "" {
os.Setenv("SSL_CERT_DIR", config.SslCertDir)
}
return cmd, nil
}
return nil, disallowedcommand.Error
}
......@@ -45,12 +39,28 @@
return cmd, nil
}
return nil, disallowedcommand.Error
}
// ContextWithCorrelationID() will always return a background Context
// with a correlation ID. It will first attempt to extract the ID from
// an environment variable. If is not available, a random one will be
// generated.
func ContextWithCorrelationID() (context.Context, func()) {
// Setup() initializes tracing from the configuration file and generates a
// background context from which all other contexts in the process should derive
// from, as it has a service name and initial correlation ID set.
func Setup(serviceName string, config *config.Config) (context.Context, func()) {
closer := tracing.Initialize(
tracing.WithServiceName(serviceName),