Inspect changes in Gitaly 3cd2a30b770d3312a1634d8eec49e34a592ef5d7
Upstream merge commit b8f3b85ddd08 changes Gitaly server version from 6835085898eb8d3881ee98c476a7bfc2981f0067 to 3cd2a30b770d3312a1634d8eec49e34a592ef5d7, with changes in protocol or one of the source subtrees currently under watch.
To further inspect, go to an up to date Gitaly source clone and run:
git diff 6835085898eb8d3881ee98c476a7bfc2981f0067..3cd2a30b770d3312a1634d8eec49e34a592ef5d7 -- proto/*.proto internal/backup
Here is a preview, truncated at 500 lines:
diff --git a/internal/backup/backup.go b/internal/backup/backup.go
index e58fc8718..4b05c9aba 100644
--- a/internal/backup/backup.go
+++ b/internal/backup/backup.go
@@ -7,9 +7,12 @@ import (
"errors"
"fmt"
"io"
+ "math"
"strings"
"time"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
@@ -28,6 +31,19 @@ var (
ErrSkipped = errors.New("repository skipped")
// ErrDoesntExist means that the data was not found.
ErrDoesntExist = errors.New("doesn't exist")
+
+ backupLatency = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "gitaly_backup_latency_seconds",
+ Help: "Latency of a repository backup by phase",
+ },
+ []string{"phase"})
+ backupBundleSize = promauto.NewHistogram(
+ prometheus.HistogramOpts{
+ Name: "gitaly_backup_bundle_bytes",
+ Help: "Size of a Git bundle uploaded in a backup",
+ Buckets: prometheus.ExponentialBucketsRange(1, 10*math.Pow(1024, 3), 20), // up to 10GB
+ })
)
// Sink is an abstraction over the real storage used for storing/restoring backups.
@@ -369,6 +385,9 @@ func setContextServerInfo(ctx context.Context, server *storage.ServerInfo, stora
}
func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step) (returnErr error) {
+ timer := prometheus.NewTimer(backupLatency.WithLabelValues("bundle"))
+ defer timer.ObserveDuration()
+
var patterns io.Reader
if len(step.PreviousRefPath) > 0 {
// If there is a previous ref path, then we are creating an increment
@@ -389,6 +408,7 @@ func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step
return mgr.sink.GetWriter(ctx, step.BundlePath)
})
defer func() {
+ backupBundleSize.Observe(float64(w.BytesWritten()))
if err := w.Close(); err != nil && returnErr == nil {
returnErr = fmt.Errorf("write bundle: %w", err)
}
@@ -481,6 +501,9 @@ func (mgr *Manager) restoreBundle(ctx context.Context, repo Repository, path str
}
func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path string) (returnErr error) {
+ timer := prometheus.NewTimer(backupLatency.WithLabelValues("custom_hooks"))
+ defer timer.ObserveDuration()
+
w := NewLazyWriter(func() (io.WriteCloser, error) {
return mgr.sink.GetWriter(ctx, path)
})
@@ -514,6 +537,9 @@ func (mgr *Manager) restoreCustomHooks(ctx context.Context, repo Repository, pat
// writeRefs writes the previously fetched list of refs in the same output
// format as `git-show-ref(1)`
func (mgr *Manager) writeRefs(ctx context.Context, path string, refs []git.Reference) (returnErr error) {
+ timer := prometheus.NewTimer(backupLatency.WithLabelValues("refs"))
+ defer timer.ObserveDuration()
+
w, err := mgr.sink.GetWriter(ctx, path)
if err != nil {
return fmt.Errorf("write refs: %w", err)
diff --git a/internal/backup/lazy.go b/internal/backup/lazy.go
index 7c38d5c6b..a66b90621 100644
--- a/internal/backup/lazy.go
+++ b/internal/backup/lazy.go
@@ -8,8 +8,9 @@ import (
// Write. This means it will only create a file if there will be data written
// to it.
type LazyWriter struct {
- create func() (io.WriteCloser, error)
- w io.WriteCloser
+ create func() (io.WriteCloser, error)
+ w io.WriteCloser
+ bytesWritten int
}
// NewLazyWriter initializes a new LazyWriter. create is called on the first
@@ -20,6 +21,12 @@ func NewLazyWriter(create func() (io.WriteCloser, error)) *LazyWriter {
}
}
+// BytesWritten returns the total number of bytes written to the underlying
+// WriteCloser. The count is never explicitly reset to 0.
+func (w *LazyWriter) BytesWritten() int {
+ return w.bytesWritten
+}
+
func (w *LazyWriter) Write(p []byte) (int, error) {
if w.w == nil {
var err error
@@ -29,7 +36,9 @@ func (w *LazyWriter) Write(p []byte) (int, error) {
}
}
- return w.w.Write(p)
+ n, err := w.w.Write(p)
+ w.bytesWritten += n
+ return n, err
}
// Close calls Close on the WriteCloser returned by Create, passing on any
diff --git a/internal/backup/lazy_test.go b/internal/backup/lazy_test.go
index 5f4ff514f..15cce1075 100644
--- a/internal/backup/lazy_test.go
+++ b/internal/backup/lazy_test.go
@@ -36,6 +36,7 @@ func TestLazyWriter(t *testing.T) {
_, err = io.Copy(w, iotest.OneByteReader(bytes.NewReader(expectedData)))
require.NoError(t, err)
require.FileExists(t, tempFilePath)
+ require.Equal(t, len(expectedData), w.BytesWritten())
data, err := os.ReadFile(tempFilePath)
require.NoError(t, err)
@@ -50,5 +51,6 @@ func TestLazyWriter(t *testing.T) {
n, err := w.Write(make([]byte, 100))
require.Equal(t, 0, n)
require.Equal(t, assert.AnError, err)
+ require.Equal(t, 0, w.BytesWritten())
})
}
diff --git a/proto/operations.proto b/proto/operations.proto
index 4b6264bb7..27a792572 100644
--- a/proto/operations.proto
+++ b/proto/operations.proto
@@ -831,7 +831,7 @@ message UserCommitFilesRequestHeader {
// point to the new commit.
bool force = 9;
// start_sha specifies the SHA of the commit to use as the parent of new commit. Takes priority
- // over start_branch_name and branc_name. Optional.
+ // over start_branch_name and branch_name. Optional.
string start_sha = 10;
// timestamp is the optional timestamp to use for the commits as author and
// committer date. If it's not set, the current time will be used.
A merge request may have already been submitted, check that on the merge commit, as usually changes in Gitaly are not breaking, compliance becoming mandatory in the following development cycles.