Inspect changes in Gitaly db9003ccaed4618cb6c9e1d8ce99f7794c868a65
Upstream merge commit 080470b2efa2 changes Gitaly server version from 3f55c303d2683913581119e79c03bded0588e153 to db9003ccaed4618cb6c9e1d8ce99f7794c868a65, with changes in protocol or one of the source subtrees currently under watch.
To further inspect, go to an up to date Gitaly source clone and run:
git diff 3f55c303d2683913581119e79c03bded0588e153..db9003ccaed4618cb6c9e1d8ce99f7794c868a65 -- proto/*.proto internal/backup
Here is a preview, truncated at 500 lines:
diff --git a/internal/backup/backup.go b/internal/backup/backup.go
index 9f32ee22e..e58fc8718 100644
--- a/internal/backup/backup.go
+++ b/internal/backup/backup.go
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"strings"
+ "time"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
@@ -38,6 +39,9 @@ type Sink interface {
// GetReader returns a reader that servers the data stored by relativePath.
// If relativePath doesn't exists the ErrDoesntExist will be returned.
GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error)
+ // SignedURL returns a URL that can be used to GET the blob for the duration
+ // specified in expiry.
+ SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error)
}
// Backup represents all the information needed to restore a backup for a repository
diff --git a/internal/backup/filesystem_sink.go b/internal/backup/filesystem_sink.go
index 6d68502db..2b4c04944 100644
--- a/internal/backup/filesystem_sink.go
+++ b/internal/backup/filesystem_sink.go
@@ -7,8 +7,10 @@ import (
"io"
"os"
"path/filepath"
+ "time"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
)
// FilesystemSink is a sink for creating and restoring backups from the local filesystem.
@@ -59,3 +61,8 @@ func (fs *FilesystemSink) GetReader(ctx context.Context, relativePath string) (i
func (fs *FilesystemSink) Close() error {
return nil
}
+
+// SignedURL is not supported by FilesystemSink.
+func (fs *FilesystemSink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) {
+ return "", structerr.NewUnimplemented("SignedURL not implemented for FilesystemSink")
+}
diff --git a/internal/backup/sink.go b/internal/backup/sink.go
index 330b1c5b4..9de909024 100644
--- a/internal/backup/sink.go
+++ b/internal/backup/sink.go
@@ -6,17 +6,22 @@ import (
"io"
"net/url"
"strings"
+ "time"
"gocloud.dev/blob"
"gocloud.dev/blob/azureblob"
"gocloud.dev/blob/gcsblob"
+ "gocloud.dev/blob/memblob"
"gocloud.dev/blob/s3blob"
"gocloud.dev/gcerrors"
)
-// ResolveSink returns a sink implementation based on the provided path.
-func ResolveSink(ctx context.Context, path string) (Sink, error) {
- parsed, err := url.Parse(path)
+// ResolveSink returns a sink implementation based on the provided uri.
+// The storage engine is chosen based on the provided uri.
+// It is the caller's responsibility to provide all required environment
+// variables in order to get properly initialized storage engine driver.
+func ResolveSink(ctx context.Context, uri string) (Sink, error) {
+ parsed, err := url.Parse(uri)
if err != nil {
return nil, err
}
@@ -29,11 +34,10 @@ func ResolveSink(ctx context.Context, path string) (Sink, error) {
}
switch scheme {
- case s3blob.Scheme, azureblob.Scheme, gcsblob.Scheme:
- sink, err := NewStorageServiceSink(ctx, path)
- return sink, err
+ case s3blob.Scheme, azureblob.Scheme, gcsblob.Scheme, memblob.Scheme:
+ return newStorageServiceSink(ctx, uri)
default:
- return NewFilesystemSink(path), nil
+ return NewFilesystemSink(uri), nil
}
}
@@ -42,11 +46,8 @@ type StorageServiceSink struct {
bucket *blob.Bucket
}
-// NewStorageServiceSink returns initialized instance of StorageServiceSink instance.
-// The storage engine is chosen based on the provided url value and a set of pre-registered
-// blank imports in that file. It is the caller's responsibility to provide all required environment
-// variables in order to get properly initialized storage engine driver.
-func NewStorageServiceSink(ctx context.Context, url string) (*StorageServiceSink, error) {
+// newStorageServiceSink returns initialized instance of StorageServiceSink instance.
+func newStorageServiceSink(ctx context.Context, url string) (*StorageServiceSink, error) {
bucket, err := blob.OpenBucket(ctx, url)
if err != nil {
return nil, fmt.Errorf("storage service sink: open bucket: %w", err)
@@ -98,3 +99,21 @@ func (s *StorageServiceSink) GetReader(ctx context.Context, relativePath string)
}
return reader, nil
}
+
+// SignedURL returns a URL that can be used to GET the blob for the duration
+// specified in expiry.
+func (s *StorageServiceSink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) {
+ opt := &blob.SignedURLOptions{
+ Expiry: expiry,
+ }
+
+ signed, err := s.bucket.SignedURL(ctx, relativePath, opt)
+ if err != nil {
+ if gcerrors.Code(err) == gcerrors.NotFound {
+ err = ErrDoesntExist
+ }
+ return "", fmt.Errorf("storage service sink: signed URL for %q: %w", relativePath, err)
+ }
+
+ return signed, err
+}
diff --git a/internal/backup/sink_test.go b/internal/backup/sink_test.go
index 0c40ee6ff..9ff6f3f8a 100644
--- a/internal/backup/sink_test.go
+++ b/internal/backup/sink_test.go
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
@@ -115,7 +116,7 @@ func TestStorageServiceSink(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
- sss, err := NewStorageServiceSink(ctx, "mem://test_bucket")
+ sss, err := ResolveSink(ctx, "mem://test_bucket")
require.NoError(t, err)
defer func() { require.NoError(t, sss.Close()) }()
@@ -147,3 +148,50 @@ func TestStorageServiceSink(t *testing.T) {
require.Nil(t, reader)
})
}
+
+func TestStorageServiceSink_SignedURL_notImplemented(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+ tmpDir := testhelper.TempDir(t)
+
+ for _, tc := range []struct {
+ desc string
+ bucketURL string
+ }{
+ {
+ desc: "memory bucket",
+ bucketURL: "mem://test_bucket",
+ },
+ {
+ desc: "fs bucket",
+ bucketURL: tmpDir,
+ },
+ } {
+ tc := tc
+
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ sss, err := ResolveSink(ctx, tc.bucketURL)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sss.Close()) })
+
+ const relativePath = "path/to/data"
+
+ data := []byte("test")
+
+ w, err := sss.GetWriter(ctx, relativePath)
+ require.NoError(t, err)
+
+ _, err = io.Copy(w, bytes.NewReader(data))
+ require.NoError(t, err)
+
+ require.NoError(t, w.Close())
+
+ _, err = sss.SignedURL(ctx, relativePath, 10*time.Minute)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "not implemented")
+ })
+ }
+}
diff --git a/proto/log.proto b/proto/log.proto
index 9b82789a3..1df859a7e 100644
--- a/proto/log.proto
+++ b/proto/log.proto
@@ -11,15 +11,21 @@ option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb";
// Schema for :
// - `partition/<partition_id>/log/entry/<log_index>`.
message LogEntry {
- // ReferenceUpdate models a single reference update.
- message ReferenceUpdate {
- // reference_name is the fully qualified name of the reference
- // to update.
- bytes reference_name = 1;
- // new_oid is the new oid to point the reference to. Deletions
- // are denoted as the SHA1 or SHA256 zero OID depending on the
- // hash type used in the repository.
- bytes new_oid = 2;
+ // ReferenceTransaction models a single reference transaction.
+ message ReferenceTransaction {
+ // Change models a single reference change.
+ message Change {
+ // reference_name is the fully qualified name of the reference
+ // to update.
+ bytes reference_name = 1;
+ // new_oid is the new oid to point the reference to. Deletions
+ // are denoted as the SHA1 or SHA256 zero OID depending on the
+ // hash type used in the repository.
+ bytes new_oid = 2;
+ }
+
+ // changes are the reference changes performed in this reference transaction.
+ repeated Change changes = 1;
}
// DefaultBranchUpdate models a default branch update.
@@ -57,10 +63,11 @@ message LogEntry {
// relative_path is the relative path of the repository the changes in the
// log entry are targeting.
string relative_path = 1;
- // reference_updates contains the reference updates this log
+ // reference_transactions contains the reference transactions this
// entry records. The logged reference updates have already passed
// through verification and are applied without any further checks.
- repeated ReferenceUpdate reference_updates = 2;
+ // The reference transactions are applied in order.
+ repeated ReferenceTransaction reference_transactions = 2;
// default_branch_update contains the information pertaining to updating
// the default branch of the repo.
DefaultBranchUpdate default_branch_update = 3;
diff --git a/proto/remote.proto b/proto/remote.proto
index 00d989fbb..110d7f058 100644
--- a/proto/remote.proto
+++ b/proto/remote.proto
@@ -22,7 +22,9 @@ service RemoteService {
};
}
- // FindRemoteRepository ...
+ // FindRemoteRepository detects whether a repository exists at the given URI. This is done by
+ // asking git-ls-remote(1) to enumerate the remote's HEAD and then checking that we recognize it
+ // as a valid reference.
rpc FindRemoteRepository(FindRemoteRepositoryRequest) returns (FindRemoteRepositoryResponse) {
option (op_type) = {
op: ACCESSOR
@@ -42,9 +44,9 @@ service RemoteService {
}
}
-// UpdateRemoteMirrorRequest ...
+// UpdateRemoteMirrorRequest is a request for the UpdateRemoteMirror RPC.
message UpdateRemoteMirrorRequest {
- // Remote ...
+ // Remote defines a remote repository that diverged references should be pushed to.
message Remote {
// url is the URL of the remote repository.
string url = 1;
@@ -91,16 +93,17 @@ message UpdateRemoteMirrorRequest {
reserved "ref_name";
}
-// UpdateRemoteMirrorResponse ...
+// UpdateRemoteMirrorResponse is a response for the UpdateRemoteMirror RPC.
message UpdateRemoteMirrorResponse {
// divergent_refs contains a list of references that had diverged in the
// mirror from the source repository.
repeated bytes divergent_refs = 1;
}
-// FindRemoteRepositoryRequest ...
+// FindRemoteRepositoryRequest is a request for the FindRemoteRepository RPC.
message FindRemoteRepositoryRequest {
- // remote ...
+ // remote is the remote repository being checked. Because this RPC is not executed within a
+ // repository, the remote parameter must specify a valid Git URL for the targeted repository.
string remote = 1;
// storage_name is used to redirect request to proper storage where it can be handled.
// As of now it doesn't matter what storage will be used, but it still must be a valid.
@@ -108,11 +111,10 @@ message FindRemoteRepositoryRequest {
string storage_name = 2 [(storage)=true];
}
-// FindRemoteRepositoryResponse ...
-// This migth throw a GRPC Unavailable code, to signal the request failure
-// is transient.
+// FindRemoteRepositoryResponse is a response for the FindRemoteRepository RPC. This might throw a
+// GRPC Unavailable code to signal the request failure is transient.
message FindRemoteRepositoryResponse {
- // exists ...
+ // exists specifies if the remote repository exists.
bool exists = 1;
}
A merge request may have already been submitted, check that on the merge commit, as usually changes in Gitaly are not breaking, compliance becoming mandatory in the following development cycles.