Skip to content
Snippets Groups Projects
Commit 40638610 authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

clonebundles: adds a auto-generate.serve-inline option

This new option disable the "external" serving of auto generated bundle in favor
of using the new "inline bundle" feature that allow serving such bundle through
the mercurial protocol.

This is a less efficient, but very easy to setup way to start using clone bundle.
parent 12995258
No related branches found
No related tags found
2 merge requests!618Draft: 6.5rc0,!614bridge automatic clone bundle generation and inline clone bundle serving
......@@ -254,6 +254,22 @@
auto-generate.on-change=yes
auto-generate.formats= zstd-v2, gzip-v2
Automatic Inline serving
........................
The simplest way to serve the generated bundle is through the Mercurial
protocol. However it is not the most efficient as request will still be served
by that main server. It is useful in case where authentication is complexe or
when an efficient mirror system is already in use anyway. See the `inline
clonebundles` section above for details about inline clonebundles
To automatically serve generated bundle through inline clonebundle, simply set
the following option::
auto-generate.serve-inline=yes
Enabling this option disable the managed upload and serving explained below.
Bundles Upload and Serving:
...........................
......@@ -257,6 +273,9 @@
Bundles Upload and Serving:
...........................
This is the most efficient way to serve automatically generated clone bundles,
but requires some setup.
The generated bundles need to be made available to users through a "public" URL.
This should be donne through `clone-bundles.upload-command` configuration. The
value of this command should be a shell command. It will have access to the
......@@ -344,6 +363,7 @@
configitem(b'clone-bundles', b'auto-generate.on-change', default=False)
configitem(b'clone-bundles', b'auto-generate.formats', default=list)
configitem(b'clone-bundles', b'auto-generate.serve-inline', default=False)
configitem(b'clone-bundles', b'trigger.below-bundled-ratio', default=0.95)
configitem(b'clone-bundles', b'trigger.revs', default=1000)
......@@ -753,6 +773,5 @@
The upload is done using the `clone-bundles.upload-command`
"""
cmd = repo.ui.config(b'clone-bundles', b'upload-command')
url = repo.ui.config(b'clone-bundles', b'url-template')
inline = repo.ui.config(b'clone-bundles', b'auto-generate.serve-inline')
basename = repo.vfs.basename(bundle.filepath)
......@@ -758,21 +777,31 @@
basename = repo.vfs.basename(bundle.filepath)
filepath = procutil.shellquote(bundle.filepath)
variables = {
b'HGCB_BUNDLE_PATH': filepath,
b'HGCB_BUNDLE_BASENAME': basename,
}
env = procutil.shellenviron(environ=variables)
ret = repo.ui.system(cmd, environ=env)
if ret:
raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
url = (
url.decode('utf8')
.format(basename=basename.decode('utf8'))
.encode('utf8')
)
return bundle.uploaded(url, basename)
if inline:
dest_dir = repo.vfs.join(bundlecaches.BUNDLE_CACHE_DIR)
repo.vfs.makedirs(dest_dir)
dest = repo.vfs.join(dest_dir, basename)
util.copyfiles(bundle.filepath, dest, hardlink=True)
url = bundlecaches.CLONEBUNDLESCHEME + basename
return bundle.uploaded(url, basename)
else:
cmd = repo.ui.config(b'clone-bundles', b'upload-command')
url = repo.ui.config(b'clone-bundles', b'url-template')
filepath = procutil.shellquote(bundle.filepath)
variables = {
b'HGCB_BUNDLE_PATH': filepath,
b'HGCB_BUNDLE_BASENAME': basename,
}
env = procutil.shellenviron(environ=variables)
ret = repo.ui.system(cmd, environ=env)
if ret:
raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
url = (
url.decode('utf8')
.format(basename=basename.decode('utf8'))
.encode('utf8')
)
return bundle.uploaded(url, basename)
def delete_bundle(repo, bundle):
"""delete a bundle from storage"""
assert bundle.ready
......@@ -774,12 +803,18 @@
def delete_bundle(repo, bundle):
"""delete a bundle from storage"""
assert bundle.ready
msg = b'clone-bundles: deleting bundle %s\n'
inline = bundle.file_url.startswith(bundlecaches.CLONEBUNDLESCHEME)
if inline:
msg = b'clone-bundles: deleting inline bundle %s\n'
else:
msg = b'clone-bundles: deleting bundle %s\n'
msg %= bundle.basename
if repo.ui.configbool(b'devel', b'debug.clonebundles'):
repo.ui.write(msg)
else:
repo.ui.debug(msg)
......@@ -780,18 +815,25 @@
msg %= bundle.basename
if repo.ui.configbool(b'devel', b'debug.clonebundles'):
repo.ui.write(msg)
else:
repo.ui.debug(msg)
cmd = repo.ui.config(b'clone-bundles', b'delete-command')
variables = {
b'HGCB_BUNDLE_URL': bundle.file_url,
b'HGCB_BASENAME': bundle.basename,
}
env = procutil.shellenviron(environ=variables)
ret = repo.ui.system(cmd, environ=env)
if ret:
raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
if inline:
inline_path = repo.vfs.join(
bundlecaches.BUNDLE_CACHE_DIR,
bundle.basename,
)
util.tryunlink(inline_path)
else:
cmd = repo.ui.config(b'clone-bundles', b'delete-command')
variables = {
b'HGCB_BUNDLE_URL': bundle.file_url,
b'HGCB_BASENAME': bundle.basename,
}
env = procutil.shellenviron(environ=variables)
ret = repo.ui.system(cmd, environ=env)
if ret:
raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
def auto_bundle_needed_actions(repo, bundles, op_id):
......
......@@ -382,3 +382,95 @@
$ cat ../server/.hg/clonebundles.manifest
https://example.com/final-upload/full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 REQUIRESNI=true (glob)
https://example.com/final-upload/full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 REQUIRESNI=true (glob)
Test serving them through inline-clone bundle
=============================================
$ cat >> ../server/.hg/hgrc << EOF
> [clone-bundles]
> auto-generate.serve-inline=yes
> EOF
$ hg -R ../server/ admin::clone-bundles-clear
clone-bundles: deleting bundle full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
clone-bundles: deleting bundle full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
initial generation
------------------
$ hg -R ../server/ admin::clone-bundles-refresh
clone-bundles: starting bundle generation: v1
11 changesets found
clone-bundles: starting bundle generation: v2
11 changesets found
$ cat ../server/.hg/clonebundles.manifest
peer-bundle-cache://full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
peer-bundle-cache://full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
$ ls -1 ../server/.hg/bundle-cache
full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
$ ls -1 ../final-upload
Regeneration eventually cleanup the old ones
--------------------------------------------
create more content
$ touch voit
$ hg -q commit -A -m 'add voit'
$ touch ar
$ hg -q commit -A -m 'add ar'
$ hg push
pushing to $TESTTMP/server
searching for changes
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 2 files
check first regeneration
$ hg -R ../server/ admin::clone-bundles-refresh
clone-bundles: starting bundle generation: v1
13 changesets found
clone-bundles: starting bundle generation: v2
13 changesets found
$ cat ../server/.hg/clonebundles.manifest
peer-bundle-cache://full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
peer-bundle-cache://full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
$ ls -1 ../server/.hg/bundle-cache
full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
$ ls -1 ../final-upload
check first regeneration (should cleanup the one before that last)
$ touch "investi"
$ hg -q commit -A -m 'add investi'
$ touch "lesgisla"
$ hg -q commit -A -m 'add lesgisla'
$ hg push
pushing to $TESTTMP/server
searching for changes
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 2 files
$ hg -R ../server/ admin::clone-bundles-refresh
clone-bundles: deleting inline bundle full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
clone-bundles: deleting inline bundle full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
clone-bundles: starting bundle generation: v1
15 changesets found
clone-bundles: starting bundle generation: v2
15 changesets found
$ cat ../server/.hg/clonebundles.manifest
peer-bundle-cache://full-v1-15_revs-17615b3984c2_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
peer-bundle-cache://full-v2-15_revs-17615b3984c2_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
$ ls -1 ../server/.hg/bundle-cache
full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
full-v1-15_revs-17615b3984c2_tip-*_acbr.hg (glob)
full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
full-v2-15_revs-17615b3984c2_tip-*_acbr.hg (glob)
$ ls -1 ../final-upload
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment