Commit 60de6f7c authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Merged heptapod branch into heptapod-stable to make 0.23 the new stable

Pipeline #25471 passed with stage
in 8 minutes and 4 seconds
......@@ -5,7 +5,7 @@
- template: Security/Secret-Detection.gitlab-ci.yml
variables:
DOCKER_VERSION: "19.03.0"
DOCKER_VERSION: "20.10.3"
workflow:
rules: &workflow_rules
......@@ -17,7 +17,7 @@
- if: '$CI_COMMIT_TAG'
default:
image: golang:1.13
image: golang:1.14
tags:
- gitlab-org
......@@ -25,10 +25,6 @@
image: docker:${DOCKER_VERSION}
services:
- docker:${DOCKER_VERSION}-dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
tags:
# See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions
- gitlab-org-docker
......@@ -49,10 +45,6 @@
script:
- make verify test
go:1.13:
extends: .test
image: golang:1.13
go:1.14:
extends: .test
image: golang:1.14
......@@ -67,5 +59,12 @@
- make coverage
coverage: '/\d+.\d+%/'
go:1.16:
extends: .test
image: golang:1.16
after_script:
- make coverage
coverage: '/\d+.\d+%/'
race:
extends: .test
......@@ -70,6 +69,6 @@
race:
extends: .test
image: golang:1.15
image: golang:1.16
script:
- make test_golang_race
......
......@@ -30,3 +30,4 @@
f202fe1e8d044b1ced7dd31be9dc65d4d36a11b1 heptapod-13.14.1
3b2b3737c7a656cf2383fff51d57f51573e9b981 heptapod-13.16.0
4def2dd0132fde7ae4fd57ea21d402860dcd3612 heptapod-13.17.0
ed9dd657a0ab833fd60fe809385f2b73947d0819 heptapod-13.18.0
v13.18.0
- Fix thread-safety issues in gitlab-shell !463
- gitlab-sshd: Support the PROXY protocol !461
- sshd: Recover from per-session and per-connection panics !464
v13.17.0
- Fix gitlab-shell panic when log file not writable !453
......
......@@ -9,5 +9,9 @@
The corresponding tag is heptapod-x.y.z
## 13.18.0
- Upstream version bump to v13.18.0 (GitLab 13.12 series)
## 13.17.0
......@@ -12,6 +16,6 @@
## 13.17.0
- Upsteam version bump to v13.17.0 (Gitlab 13.9 *and* 13.10 series)
- Upstream version bump to v13.17.0 (GitLab 13.9 to 13.11 series)
## 13.16.0
......
......@@ -43,4 +43,4 @@
bin/check
clean:
rm -f bin/check bin/gitlab-shell bin/gitlab-shell-authorized-keys-check bin/gitlab-shell-authorized-principals-check
rm -f bin/check bin/gitlab-shell bin/gitlab-shell-authorized-keys-check bin/gitlab-shell-authorized-principals-check bin/gitlab-sshd
......@@ -35,6 +35,9 @@
Download and install the current version of Go from https://golang.org/dl/
We follow the [Golang Release Policy](https://golang.org/doc/devel/release.html#policy)
of supporting the current stable version and the previous two major versions.
## Setup
make setup
......
......@@ -26,7 +26,7 @@
desc string
relativeURLRoot string
caFile string
server func(*testing.T, []testserver.TestRequestHandler) (string, func())
server func(*testing.T, []testserver.TestRequestHandler) string
}{
{
desc: "Socket client",
......@@ -49,7 +49,7 @@
{
desc: "Https client",
caFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt"),
server: func(t *testing.T, handlers []testserver.TestRequestHandler) (string, func()) {
server: func(t *testing.T, handlers []testserver.TestRequestHandler) string {
return testserver.StartHttpsServer(t, handlers, "")
},
},
......@@ -57,8 +57,7 @@
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
url, cleanup := tc.server(t, buildRequests(t, tc.relativeURLRoot))
defer cleanup()
url := tc.server(t, buildRequests(t, tc.relativeURLRoot))
secret := "sssh, it's a secret"
......
......@@ -48,8 +48,7 @@
},
}
client, cleanup := setup(t, username, password, requests)
defer cleanup()
client := setup(t, username, password, requests)
response, err := client.Get(context.Background(), "/get_endpoint")
require.NoError(t, err)
......@@ -86,8 +85,7 @@
},
}
client, cleanup := setup(t, "", "", requests)
defer cleanup()
client := setup(t, "", "", requests)
_, err := client.Get(context.Background(), "/empty_basic_auth")
require.NoError(t, err)
......@@ -110,8 +108,7 @@
},
}
client, cleanup := setup(t, "", "", requests)
defer cleanup()
client := setup(t, "", "", requests)
_, err := client.Get(context.Background(), "/default_user_agent")
require.NoError(t, err)
......@@ -122,11 +119,11 @@
}
func setup(t *testing.T, username, password string, requests []testserver.TestRequestHandler) (*GitlabNetClient, func()) {
url, cleanup := testserver.StartHttpServer(t, requests)
func setup(t *testing.T, username, password string, requests []testserver.TestRequestHandler) *GitlabNetClient {
url := testserver.StartHttpServer(t, requests)
httpClient := NewHTTPClient(url, "", "", "", false, 1)
client, err := NewGitlabNetClient(username, password, "", httpClient)
require.NoError(t, err)
......@@ -127,8 +124,8 @@
httpClient := NewHTTPClient(url, "", "", "", false, 1)
client, err := NewGitlabNetClient(username, password, "", httpClient)
require.NoError(t, err)
return client, cleanup
return client
}
......@@ -51,8 +51,7 @@
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
client, cleanup := setupWithRequests(t, tc.caFile, tc.caPath, tc.clientCAPath, tc.clientCertPath, tc.clientKeyPath, tc.selfSigned)
defer cleanup()
client := setupWithRequests(t, tc.caFile, tc.caPath, tc.clientCAPath, tc.clientCertPath, tc.clientKeyPath, tc.selfSigned)
response, err := client.Get(context.Background(), "/hello")
require.NoError(t, err)
......@@ -88,8 +87,7 @@
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
client, cleanup := setupWithRequests(t, tc.caFile, tc.caPath, "", "", "", false)
defer cleanup()
client := setupWithRequests(t, tc.caFile, tc.caPath, "", "", "", false)
_, err := client.Get(context.Background(), "/hello")
require.Error(t, err)
......@@ -99,7 +97,7 @@
}
}
func setupWithRequests(t *testing.T, caFile, caPath, clientCAPath, clientCertPath, clientKeyPath string, selfSigned bool) (*GitlabNetClient, func()) {
func setupWithRequests(t *testing.T, caFile, caPath, clientCAPath, clientCertPath, clientKeyPath string, selfSigned bool) *GitlabNetClient {
testDirCleanup, err := testhelper.PrepareTestRootDir()
require.NoError(t, err)
defer testDirCleanup()
......@@ -115,7 +113,7 @@
},
}
url, cleanup := testserver.StartHttpsServer(t, requests, clientCAPath)
url := testserver.StartHttpsServer(t, requests, clientCAPath)
var opts []HTTPClientOpt
if clientCertPath != "" && clientKeyPath != "" {
......@@ -128,5 +126,5 @@
client, err := NewGitlabNetClient("", "", "", httpClient)
require.NoError(t, err)
return client, cleanup
return client
}
......@@ -58,6 +58,8 @@
return nil
}
func StartGitalyServer(t *testing.T) (string, *TestGitalyServer, func()) {
func StartGitalyServer(t *testing.T) (string, *TestGitalyServer) {
t.Helper()
tempDir, _ := ioutil.TempDir("", "gitlab-shell-test-api")
gitalySocketPath := path.Join(tempDir, "gitaly.sock")
......@@ -62,5 +64,6 @@
tempDir, _ := ioutil.TempDir("", "gitlab-shell-test-api")
gitalySocketPath := path.Join(tempDir, "gitaly.sock")
t.Cleanup(func() { os.RemoveAll(tempDir) })
err := os.MkdirAll(filepath.Dir(gitalySocketPath), 0700)
require.NoError(t, err)
......@@ -74,5 +77,6 @@
pb.RegisterSSHServiceServer(server, &testServer)
go server.Serve(listener)
t.Cleanup(func() { server.Stop() })
gitalySocketUrl := "unix:" + gitalySocketPath
......@@ -77,7 +81,3 @@
gitalySocketUrl := "unix:" + gitalySocketPath
cleanup := func() {
server.Stop()
os.RemoveAll(tempDir)
}
......@@ -83,3 +83,3 @@
return gitalySocketUrl, &testServer, cleanup
return gitalySocketUrl, &testServer
}
......@@ -27,6 +27,8 @@
Handler func(w http.ResponseWriter, r *http.Request)
}
func StartSocketHttpServer(t *testing.T, handlers []TestRequestHandler) (string, func()) {
func StartSocketHttpServer(t *testing.T, handlers []TestRequestHandler) string {
t.Helper()
err := os.MkdirAll(filepath.Dir(testSocket), 0700)
require.NoError(t, err)
......@@ -31,5 +33,6 @@
err := os.MkdirAll(filepath.Dir(testSocket), 0700)
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(tempDir) })
socketListener, err := net.Listen("unix", testSocket)
require.NoError(t, err)
......@@ -44,6 +47,6 @@
url := "http+unix://" + testSocket
return url, cleanupSocket
return url
}
......@@ -48,4 +51,6 @@
}
func StartHttpServer(t *testing.T, handlers []TestRequestHandler) (string, func()) {
func StartHttpServer(t *testing.T, handlers []TestRequestHandler) string {
t.Helper()
server := httptest.NewServer(buildHandler(handlers))
......@@ -51,2 +56,3 @@
server := httptest.NewServer(buildHandler(handlers))
t.Cleanup(func() { server.Close() })
......@@ -52,4 +58,4 @@
return server.URL, server.Close
return server.URL
}
......@@ -54,6 +60,8 @@
}
func StartHttpsServer(t *testing.T, handlers []TestRequestHandler, clientCAPath string) (string, func()) {
func StartHttpsServer(t *testing.T, handlers []TestRequestHandler, clientCAPath string) string {
t.Helper()
crt := path.Join(testhelper.TestRoot, "certs/valid/server.crt")
key := path.Join(testhelper.TestRoot, "certs/valid/server.key")
......@@ -63,7 +71,7 @@
server.TLS = &tls.Config{
Certificates: []tls.Certificate{cer},
MinVersion: tls.VersionTLS12,
MinVersion: tls.VersionTLS12,
}
server.TLS.BuildNameToCertificate()
......@@ -80,6 +88,5 @@
server.StartTLS()
return server.URL, server.Close
}
t.Cleanup(func() { server.Close() })
......@@ -85,6 +92,5 @@
func cleanupSocket() {
os.RemoveAll(tempDir)
return server.URL
}
func buildHandler(handlers []TestRequestHandler) http.Handler {
......
......@@ -9,6 +9,7 @@
"gitlab.com/gitlab-org/gitlab-shell/internal/config"
"gitlab.com/gitlab-org/gitlab-shell/internal/executable"
"gitlab.com/gitlab-org/gitlab-shell/internal/logger"
"gitlab.com/gitlab-org/gitlab-shell/internal/sshenv"
)
func main() {
......@@ -32,7 +33,7 @@
logger.Configure(config)
cmd, err := command.New(executable, os.Args[1:], config, readWriter)
cmd, err := command.New(executable, os.Args[1:], sshenv.Env{}, config, readWriter)
if err != nil {
fmt.Fprintf(readWriter.ErrOut, "%v\n", err)
os.Exit(1)
......
......@@ -10,6 +10,7 @@
"gitlab.com/gitlab-org/gitlab-shell/internal/console"
"gitlab.com/gitlab-org/gitlab-shell/internal/executable"
"gitlab.com/gitlab-org/gitlab-shell/internal/logger"
"gitlab.com/gitlab-org/gitlab-shell/internal/sshenv"
)
func main() {
......@@ -33,7 +34,7 @@
logger.Configure(config)
cmd, err := command.New(executable, os.Args[1:], config, readWriter)
cmd, err := command.New(executable, os.Args[1:], sshenv.Env{}, config, readWriter)
if err != nil {
// For now this could happen if `SSH_CONNECTION` is not set on
// the environment
......
......@@ -10,6 +10,7 @@
"gitlab.com/gitlab-org/gitlab-shell/internal/console"
"gitlab.com/gitlab-org/gitlab-shell/internal/executable"
"gitlab.com/gitlab-org/gitlab-shell/internal/logger"
"gitlab.com/gitlab-org/gitlab-shell/internal/sshenv"
)
func main() {
......@@ -33,7 +34,7 @@
logger.Configure(config)
cmd, err := command.New(executable, os.Args[1:], config, readWriter)
cmd, err := command.New(executable, os.Args[1:], sshenv.Env{}, config, readWriter)
if err != nil {
// For now this could happen if `SSH_CONNECTION` is not set on
// the environment
......
......@@ -10,6 +10,7 @@
"gitlab.com/gitlab-org/gitlab-shell/internal/console"
"gitlab.com/gitlab-org/gitlab-shell/internal/executable"
"gitlab.com/gitlab-org/gitlab-shell/internal/logger"
"gitlab.com/gitlab-org/gitlab-shell/internal/sshenv"
)
var (
......@@ -47,7 +48,8 @@
logger.Configure(config)
cmd, err := command.New(executable, os.Args[1:], config, readWriter)
env := sshenv.NewFromEnv()
cmd, err := command.New(executable, os.Args[1:], env, config, readWriter)
if err != nil {
// For now this could happen if `SSH_CONNECTION` is not set on
// the environment
......
package main_test
import (
"bufio"
"context"
"crypto/ed25519"
"encoding/pem"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"testing"
"github.com/mikesmitty/edkey"
"github.com/pires/go-proxyproto"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
var (
sshdPath = ""
)
func init() {
rootDir := rootDir()
sshdPath = filepath.Join(rootDir, "bin", "gitlab-sshd")
if _, err := os.Stat(sshdPath); os.IsNotExist(err) {
panic(fmt.Errorf("cannot find executable %s. Please run 'make compile'", sshdPath))
}
}
func rootDir() string {
_, currentFile, _, ok := runtime.Caller(0)
if !ok {
panic(fmt.Errorf("rootDir: calling runtime.Caller failed"))
}
return filepath.Join(filepath.Dir(currentFile), "..", "..")
}
func successAPI(t *testing.T) http.Handler {
t.Helper()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Logf("gitlab-api-mock: received request: %s %s", r.Method, r.RequestURI)
w.Header().Set("Content-Type", "application/json")
switch r.URL.EscapedPath() {
case "/api/v4/internal/authorized_keys":
fmt.Fprintf(w, `{"id":1, "key":"%s"}`, r.FormValue("key"))
case "/api/v4/internal/discover":
fmt.Fprint(w, `{"id": 1000, "name": "Test User", "username": "test-user"}`)
default:
t.Log("Unexpected request to successAPI!")
t.FailNow()
}
})
}
func genServerConfig(gitlabUrl, hostKeyPath string) []byte {
return []byte(`---
user: "git"
log_file: ""
log_format: json
secret: "0123456789abcdef"
gitlab_url: "` + gitlabUrl + `"
sshd:
listen: "127.0.0.1:0"
proxy_protocol: true
web_listen: ""
host_key_files:
- "` + hostKeyPath + `"`)
}
func buildClient(t *testing.T, addr string, hostKey ed25519.PublicKey) *ssh.Client {
t.Helper()
pubKey, err := ssh.NewPublicKey(hostKey)
require.NoError(t, err)
_, clientPrivKey, err := ed25519.GenerateKey(nil)
require.NoError(t, err)
clientSigner, err := ssh.NewSignerFromKey(clientPrivKey)
require.NoError(t, err)
// Use the proxy protocol to spoof our client address
target, err := net.ResolveTCPAddr("tcp", addr)
require.NoError(t, err)
conn, err := net.DialTCP("tcp", nil, target)
require.NoError(t, err)
t.Cleanup(func() { conn.Close() })
// Create a proxyprotocol header or use HeaderProxyFromAddrs() if you
// have two conn's
header := &proxyproto.Header{
Version: 2,
Command: proxyproto.PROXY,
TransportProtocol: proxyproto.TCPv4,
SourceAddr: &net.TCPAddr{
IP: net.ParseIP("10.1.1.1"),
Port: 1000,
},
DestinationAddr: target,
}
// After the connection was created write the proxy headers first
_, err = header.WriteTo(conn)
require.NoError(t, err)
sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, &ssh.ClientConfig{
User: "git",
Auth: []ssh.AuthMethod{ssh.PublicKeys(clientSigner)},
HostKeyCallback: ssh.FixedHostKey(pubKey),
})
require.NoError(t, err)
client := ssh.NewClient(sshConn, chans, reqs)
t.Cleanup(func() { client.Close() })
return client
}
func configureSSHD(t *testing.T, apiServer string) (string, ed25519.PublicKey) {
t.Helper()
dir, err := ioutil.TempDir("", "gitlab-sshd-acceptance-test-")
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(dir) })
configFile := filepath.Join(dir, "config.yml")
hostKeyFile := filepath.Join(dir, "hostkey")
pub, priv, err := ed25519.GenerateKey(nil)
require.NoError(t, err)
configFileData := genServerConfig(apiServer, hostKeyFile)
require.NoError(t, ioutil.WriteFile(configFile, configFileData, 0644))
block := &pem.Block{Type: "OPENSSH PRIVATE KEY", Bytes: edkey.MarshalED25519PrivateKey(priv)}
hostKeyData := pem.EncodeToMemory(block)
require.NoError(t, ioutil.WriteFile(hostKeyFile, hostKeyData, 0400))
return dir, pub
}
func startSSHD(t *testing.T, dir string) string {
t.Helper()
// We need to scan the first few lines of stderr to get the listen address.
// Once we've learned it, we'll start a goroutine to copy everything to
// the real stderr
pr, pw := io.Pipe()
t.Cleanup(func() { pr.Close() })
t.Cleanup(func() { pw.Close() })
scanner := bufio.NewScanner(pr)
extractor := regexp.MustCompile(`msg="Listening on ([0-9a-f\[\]\.:]+)"`)
ctx, cancel := context.WithCancel(context.Background())
cmd := exec.CommandContext(ctx, sshdPath, "-config-dir", dir)
cmd.Stdout = os.Stdout
cmd.Stderr = pw
require.NoError(t, cmd.Start())
t.Logf("gitlab-sshd: Start(): success")
t.Cleanup(func() { t.Logf("gitlab-sshd: Wait(): %v", cmd.Wait()) })
t.Cleanup(cancel)
var listenAddr string
for scanner.Scan() {
if matches := extractor.FindSubmatch(scanner.Bytes()); len(matches) == 2 {
listenAddr = string(matches[1])
break
}
}
require.NotEmpty(t, listenAddr, "Couldn't extract listen address from gitlab-sshd")
go io.Copy(os.Stderr, pr)
return listenAddr
}
// Starts an instance of gitlab-sshd with the given arguments, returning an SSH
// client already connected to it
func runSSHD(t *testing.T, apiHandler http.Handler) *ssh.Client {
t.Helper()