Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
P
Poulpe
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
octobus
Poulpe
Commits
26a2a246
Commit
26a2a246
authored
4 months ago
by
Raphaël Gomès
Browse files
Options
Downloads
Patches
Plain Diff
suites: add a setup script for the SWH git loader in a new `swh` suite
parent
64751f1b
No related branches found
Branches containing commit
No related tags found
1 merge request
!44
Suite agnostic tooling, improvements to simple-command, SWH suite
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
suites/swh/bin-env-setup/default.poulpe-bin-env-setup
+1
-0
1 addition, 0 deletions
suites/swh/bin-env-setup/default.poulpe-bin-env-setup
suites/swh/bin-env-setup/git-loader.poulpe-setup.py
+277
-0
277 additions, 0 deletions
suites/swh/bin-env-setup/git-loader.poulpe-setup.py
with
278 additions
and
0 deletions
suites/swh/bin-env-setup/default.poulpe-bin-env-setup
0 → 120000
+
1
−
0
View file @
26a2a246
git-loader.poulpe-setup.py
\ No newline at end of file
This diff is collapsed.
Click to expand it.
suites/swh/bin-env-setup/git-loader.poulpe-setup.py
0 → 100755
+
277
−
0
View file @
26a2a246
#!/bin/env python
# WARNING: Do not run this script directly, use `poulpe bin-env-util setup-one`
#
# The script works using a hardcoded temporary directory
# in `/tmp/poulpe-clone-cache/swh/git-loader`
import
hashlib
import
json
import
os
from
pathlib
import
Path
import
subprocess
import
sys
from
typing
import
Dict
,
List
import
click
import
poulpe
CLONE_PATH
=
"
/tmp/poulpe-clone-cache/swh/swh-loader-git
"
def
run_command
(
command
:
List
[
str
]):
environ
=
os
.
environ
.
copy
()
environ
[
"
GIT_CONFIG_GLOBAL
"
]
=
"
/dev/null
"
environ
[
"
GIT_CONFIG_SYSTEM
"
]
=
"
/dev/null
"
completed
=
subprocess
.
run
(
command
,
capture_output
=
True
,
env
=
environ
)
if
completed
.
returncode
!=
0
:
click
.
secho
(
completed
.
stderr
,
fg
=
"
red
"
,
file
=
sys
.
stderr
)
completed
.
check_returncode
()
return
completed
.
stdout
def
compute_unique_id
(
git_sha1
,
python_version
,
postgres_version
):
hasher
=
hashlib
.
sha256
()
# Version number to bust caches
hasher
.
update
(
b
"
0
"
)
hasher
.
update
(
git_sha1
.
encode
())
hasher
.
update
(
b
"
\x00
"
)
hasher
.
update
(
python_version
.
encode
())
hasher
.
update
(
b
"
\x00
"
)
if
postgres_version
is
not
None
:
hasher
.
update
(
postgres_version
.
encode
())
return
hasher
.
digest
().
hex
()
def
describe_parameters
(
ctx
:
click
.
Context
,
param
,
value
):
if
not
value
:
return
# TODO infer this from click context
print
(
json
.
dumps
(
{
"
parameters
"
:
{
"
loader-version
"
:
{
"
doc
"
:
"
an identifier of the swh-loader-git version we want to install
"
,
"
type
"
:
"
<git-revision>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
target-repo
"
:
{
"
default
"
:
"
https://gitlab.softwareheritage.org/swh/devel/swh-loader-git
"
,
"
doc
"
:
"
a path to the repo of swh-loader-git
"
,
"
type
"
:
"
<url>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
python-version
"
:
{
"
default
"
:
"
python3
"
,
"
doc
"
:
"
a path to the Python to use for the bin env
"
,
"
type
"
:
"
<file-path>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
}
}
)
)
ctx
.
exit
(
0
)
def
echo
(
message
:
str
):
"""
Wrapper around echo that is disabled if we
'
re computing the unique id
"""
click
.
echo
(
message
)
def
noop
(
message
:
str
):
"""
disabled echo when computing the unique id
"""
# Explicitly don't set `no_args_is_help=True` because we want the script
# to return an error code if nothing is passed
@click.command
()
@click.option
(
"
--describe-parameters
"
,
default
=
False
,
is_flag
=
True
,
is_eager
=
True
,
callback
=
describe_parameters
,
)
@click.option
(
"
--get-unique-id
"
,
default
=
False
,
is_flag
=
True
,
help
=
"
return the unique identifier derived from the all the parameters to this bin-env
"
,
)
@click.option
(
"
--loader-version
"
,
required
=
True
,
help
=
"
an identifier of the swh-loader-git version we want to install
"
,
)
@click.option
(
"
--target-repo
"
,
help
=
"
a path to the repo of swh-loader-git
"
,
default
=
"
https://gitlab.softwareheritage.org/swh/devel/swh-loader-git
"
,
)
@click.option
(
"
--python-version
"
,
default
=
"
python3
"
,
help
=
"
a path to the Python to use for the bin env
"
,
)
def
git_loader_setup
(
describe_parameters
:
bool
,
get_unique_id
:
bool
,
loader_version
:
str
,
target_repo
:
str
,
python_version
:
str
,
):
if
get_unique_id
:
# Don't print anything if we're only getting the ID
global
echo
echo
=
noop
echo
(
f
"
POULPE: using python version
{
python_version
}
"
)
echo
(
f
"
POULPE: using swh-git-loader version
{
loader_version
}
"
)
echo
(
f
"
POULPE: using loader url
{
target_repo
}
"
)
# Does it look like a clone? Don't bother to be more resistant for now
if
Path
(
CLONE_PATH
,
"
.git
"
).
is_dir
():
# Do we already have the revision?
try
:
run_command
([
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
rev-parse
"
,
loader_version
])
except
subprocess
.
CalledProcessError
:
# If not, fetch
echo
(
f
"
POULPE: failed to retrieve version, fetching
"
)
run_command
([
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
fetch
"
,
target_repo
])
echo
(
f
"
POULPE: fetch done
"
)
else
:
echo
(
f
"
POULPE: expected version already in cache
"
)
else
:
echo
(
f
"
POULPE: cloning swh-git-loader from
{
target_repo
}
"
)
run_command
([
"
git
"
,
"
clone
"
,
target_repo
,
CLONE_PATH
])
echo
(
f
"
POULPE: clone done
"
)
git_rank
=
int
(
(
run_command
(
[
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
rev-list
"
,
"
--count
"
,
loader_version
,
]
)
.
strip
()
.
decode
()
)
)
git_sha1
=
(
run_command
([
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
rev-parse
"
,
loader_version
])
.
strip
()
.
decode
()
)
unique_id
=
compute_unique_id
(
git_sha1
,
python_version
,
postgres_version
=
None
)
if
not
get_unique_id
:
# This means we've already got the unique ID, so make sure we match
name
=
Path
().
resolve
().
name
if
name
!=
unique_id
:
raise
poulpe
.
errors
.
BinEnvSetupFailure
(
f
"
Bin-env
'
{
name
}
'
doesn
'
t match expected ID
'
{
unique_id
}
'
,
"
f
"
environment changed during setup
"
)
else
:
click
.
echo
(
f
"
UNIQUE_ID:
{
unique_id
}
"
)
exit
(
0
)
echo
(
"
POULPE: creating a new virtual environment
"
)
run_command
([
python_version
,
"
-m
"
,
"
venv
"
,
"
.
"
])
echo
(
"
POULPE: installing swh-git-loader in the new venv
"
)
run_command
(
[
"
bin/pip
"
,
"
install
"
,
# install with no working copy for speed and making concurrent
# installs possible
f
"
git+file://
{
CLONE_PATH
}
@
{
git_sha1
}
"
,
"
--require-virtualenv
"
,
]
)
echo
(
"
POULPE: install OK
"
)
output
=
run_command
(
[
"
bin/pip
"
,
"
list
"
,
"
--pre
"
,
"
--include-editable
"
,
"
--require-virtualenv
"
,
"
--format
"
,
"
json
"
,
]
)
virtualenv_versions
:
Dict
[
str
,
str
]
=
{
p
[
"
name
"
]:
p
[
"
version
"
]
for
p
in
json
.
loads
(
output
)
}
bin_env_vars
=
{
"
python
"
:
{
"
version
"
:
run_command
([
"
bin/python
"
,
"
--version
"
])
.
decode
()
.
strip
(),
"
implementation
"
:
run_command
(
[
"
bin/python
"
,
"
-c
"
,
"
import sys; print(sys.implementation.name)
"
,
]
)
.
decode
()
.
strip
(),
"
dependencies
"
:
{},
},
"
git
"
:
{
# kept as a useful shortcut, but redundant with `python.dependencies`
"
version
"
:
virtualenv_versions
[
"
swh.loader.git
"
],
"
rank
"
:
git_rank
,
"
sha1
"
:
git_sha1
,
},
}
for
name
,
version
in
virtualenv_versions
.
items
():
bin_env_vars
[
"
python
"
][
"
dependencies
"
][
name
]
=
version
# XXX should we use version or sha1 here and in the sort key?
bin_env_vars
[
"
results-comparison
"
]
=
{
"
default-compare-key
"
:
"
bin-env-vars.git.version
"
,
"
sort-keys
"
:
{
"
bin-env-vars.git.version
"
:
[
"
bin-env-vars.git.rank
"
,
"
bin-env-vars.git.sha1
"
,
]
},
"
header-keys
"
:
[
"
data-env-vars.name
"
,
"
benchmark.name
"
,
"
bin-env-vars.python.version
"
,
"
bin-env-vars.python.implementation
"
,
],
}
poulpe
.
write_data
(
"
bin-env.poulpe
"
,
{
"
bin-env-vars
"
:
bin_env_vars
})
with
open
(
"
bin-env.shell
"
,
"
w
"
)
as
f
:
f
.
write
(
"""
#!/bin/bash
set -Eeuo pipefail
this_script=$(realpath
"
$0
"
)
POULPE_BIN_ENV_ROOT=
"
$(dirname
"
$this_script
"
)
"
export POULPE_BIN_ENV_ROOT
.
"
${POULPE_BIN_ENV_ROOT}
"
/bin/activate
"
$@
"
"""
)
os
.
chmod
(
"
bin-env.shell
"
,
0o775
)
if
__name__
==
"
__main__
"
:
git_loader_setup
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment