Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
P
Poulpe
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
octobus
Poulpe
Commits
86d5e2d2
Commit
86d5e2d2
authored
2 months ago
by
Pierre-Yves David
Browse files
Options
Downloads
Patches
Plain Diff
setup changes
parent
bef99438
Branches
topic/default/bench-1-import
No related tags found
No related merge requests found
Pipeline
#98038
failed
2 months ago
Stage: test
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
suites/swh/bin-env-setup/git-loader.poulpe-setup.py
+155
-68
155 additions, 68 deletions
suites/swh/bin-env-setup/git-loader.poulpe-setup.py
with
155 additions
and
68 deletions
suites/swh/bin-env-setup/git-loader.poulpe-setup.py
+
155
−
68
View file @
86d5e2d2
...
...
@@ -5,6 +5,7 @@
# in `/tmp/poulpe-clone-cache/swh/git-loader`
import
hashlib
import
shutil
import
json
import
os
from
pathlib
import
Path
...
...
@@ -14,7 +15,7 @@
import
click
import
poulpe
CLONE_PATH
=
"
/tmp/poulpe-clone-cache/swh/
swh-loader-git
"
CLONE_PATH
_TMPL
=
"
/tmp/poulpe-clone-cache/swh/
{name}
"
def
run_command
(
command
:
List
[
str
]):
...
...
@@ -24,8 +25,9 @@
completed
=
subprocess
.
run
(
command
,
capture_output
=
True
,
env
=
environ
)
if
completed
.
returncode
!=
0
:
click
.
secho
(
completed
.
stdout
,
fg
=
"
yellow
"
,
file
=
sys
.
stderr
)
click
.
secho
(
completed
.
stderr
,
fg
=
"
red
"
,
file
=
sys
.
stderr
)
completed
.
check_returncode
()
return
completed
.
stdout
...
...
@@ -27,9 +29,9 @@
click
.
secho
(
completed
.
stderr
,
fg
=
"
red
"
,
file
=
sys
.
stderr
)
completed
.
check_returncode
()
return
completed
.
stdout
def
compute_unique_id
(
git_sha1
,
python_version
,
postgres_version
):
def
compute_unique_id
(
all_
git_sha1
,
python_version
,
postgres_version
):
hasher
=
hashlib
.
sha256
()
# Version number to bust caches
hasher
.
update
(
b
"
0
"
)
...
...
@@ -33,7 +35,11 @@
hasher
=
hashlib
.
sha256
()
# Version number to bust caches
hasher
.
update
(
b
"
0
"
)
hasher
.
update
(
git_sha1
.
encode
())
for
n
,
h
in
sorted
(
all_git_sha1
):
hasher
.
update
(
b
"
\x00
"
)
hasher
.
update
(
n
.
encode
())
hasher
.
update
(
b
"
\x00
"
)
hasher
.
update
(
h
.
encode
())
hasher
.
update
(
b
"
\x00
"
)
hasher
.
update
(
python_version
.
encode
())
hasher
.
update
(
b
"
\x00
"
)
...
...
@@ -50,8 +56,8 @@
json
.
dumps
(
{
"
parameters
"
:
{
"
loader-version
"
:
{
"
swh-
loader-
git-
version
"
:
{
"
doc
"
:
"
an identifier of the swh-loader-git version we want to install
"
,
"
type
"
:
"
<git-revision>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
...
...
@@ -54,10 +60,10 @@
"
doc
"
:
"
an identifier of the swh-loader-git version we want to install
"
,
"
type
"
:
"
<git-revision>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
target-repo
"
:
{
"
swh-loader-git-
target-repo
"
:
{
"
default
"
:
"
https://gitlab.softwareheritage.org/swh/devel/swh-loader-git
"
,
"
doc
"
:
"
a path to the repo of swh-loader-git
"
,
"
type
"
:
"
<url>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
...
...
@@ -59,8 +65,30 @@
"
default
"
:
"
https://gitlab.softwareheritage.org/swh/devel/swh-loader-git
"
,
"
doc
"
:
"
a path to the repo of swh-loader-git
"
,
"
type
"
:
"
<url>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
swh-storage-version
"
:
{
"
doc
"
:
"
an identifier of the swh-storage version we want to install
"
,
"
type
"
:
"
<git-revision>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
swh-storage-target-repo
"
:
{
"
default
"
:
"
https://gitlab.softwareheritage.org/swh/devel/swh-storage
"
,
"
doc
"
:
"
a path to the repo of swh-storage
"
,
"
type
"
:
"
<url>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
swh-core-version
"
:
{
"
doc
"
:
"
an identifier of the swh-core version we want to install
"
,
"
type
"
:
"
<git-revision>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
swh-core-target-repo
"
:
{
"
default
"
:
"
https://gitlab.softwareheritage.org/swh/devel/swh-core
"
,
"
doc
"
:
"
a path to the repo of swh-core
"
,
"
type
"
:
"
<url>
"
,
"
arg-mode
"
:
"
CLI
"
,
},
"
python-version
"
:
{
"
default
"
:
"
python3
"
,
"
doc
"
:
"
a path to the Python to use for the bin env
"
,
...
...
@@ -83,6 +111,66 @@
"""
disabled echo when computing the unique id
"""
def
install_one
(
name
:
str
,
repo_url
:
str
,
version
:
str
,
get_unique_id
:
bool
=
False
)
->
tuple
[
int
,
str
]:
clone_path
=
Path
(
CLONE_PATH_TMPL
.
format
(
name
=
name
))
if
(
clone_path
).
is_dir
():
shutil
.
rmtree
(
clone_path
)
# Does it look like a clone? Don't bother to be more resistant for now
if
(
clone_path
/
"
.git
"
).
is_dir
():
# Do we already have the revision?
try
:
run_command
([
"
git
"
,
"
-C
"
,
clone_path
,
"
rev-parse
"
,
version
])
except
subprocess
.
CalledProcessError
:
# If not, fetch
echo
(
f
"
POULPE: failed to retrieve version, fetching
"
)
run_command
([
"
git
"
,
"
-C
"
,
clone_path
,
"
pull
"
,
repo_url
])
echo
(
f
"
POULPE: fetch done
"
)
else
:
echo
(
f
"
POULPE: expected version already in cache
"
)
else
:
echo
(
f
"
POULPE: cloning swh-git-loader from
{
repo_url
}
"
)
run_command
([
"
git
"
,
"
clone
"
,
"
--mirror
"
,
repo_url
,
clone_path
])
echo
(
f
"
POULPE: clone done
"
)
git_rank
=
int
(
(
run_command
(
[
"
git
"
,
"
-C
"
,
str
(
clone_path
),
"
rev-list
"
,
"
--count
"
,
version
,
]
)
.
strip
()
.
decode
()
)
)
git_sha1
=
(
run_command
([
"
git
"
,
"
-C
"
,
clone_path
,
"
rev-parse
"
,
version
])
.
strip
()
.
decode
()
)
if
not
get_unique_id
:
echo
(
f
"
POULPE: installing
{
name
}
in the new venv
"
)
run_command
(
[
"
bin/pip
"
,
"
install
"
,
# install with no working copy for speed and making concurrent
# installs possible
f
"
git+file://
{
clone_path
}
@
{
git_sha1
}
"
,
"
--require-virtualenv
"
,
]
)
return
git_rank
,
git_sha1
# Explicitly don't set `no_args_is_help=True` because we want the script
# to return an error code if nothing is passed
@click.command
()
...
...
@@ -100,8 +188,8 @@
help
=
"
return the unique identifier derived from the all the parameters to this bin-env
"
,
)
@click.option
(
"
--loader-version
"
,
"
--
swh-
loader-
git-
version
"
,
required
=
True
,
help
=
"
an identifier of the swh-loader-git version we want to install
"
,
)
@click.option
(
...
...
@@ -104,9 +192,9 @@
required
=
True
,
help
=
"
an identifier of the swh-loader-git version we want to install
"
,
)
@click.option
(
"
--target-repo
"
,
"
--
swh-loader-git-
target-repo
"
,
help
=
"
a path to the repo of swh-loader-git
"
,
default
=
"
https://gitlab.softwareheritage.org/swh/devel/swh-loader-git
"
,
)
@click.option
(
...
...
@@ -109,7 +197,27 @@
help
=
"
a path to the repo of swh-loader-git
"
,
default
=
"
https://gitlab.softwareheritage.org/swh/devel/swh-loader-git
"
,
)
@click.option
(
"
--swh-storage-version
"
,
required
=
True
,
help
=
"
an identifier of the swh-storage version we want to install
"
,
)
@click.option
(
"
--swh-storage-target-repo
"
,
help
=
"
a path to the repo of swh-storage
"
,
default
=
"
https://gitlab.softwareheritage.org/swh/devel/swh-storage
"
,
)
@click.option
(
"
--swh-core-version
"
,
required
=
True
,
help
=
"
an identifier of the swh-core version we want to install
"
,
)
@click.option
(
"
--swh-core-target-repo
"
,
help
=
"
a path to the repo of swh-core
"
,
default
=
"
https://gitlab.softwareheritage.org/swh/devel/swh-core
"
,
)
@click.option
(
"
--python-version
"
,
default
=
"
python3
"
,
help
=
"
a path to the Python to use for the bin env
"
,
...
...
@@ -117,8 +225,12 @@
def
git_loader_setup
(
describe_parameters
:
bool
,
get_unique_id
:
bool
,
loader_version
:
str
,
target_repo
:
str
,
swh_loader_git_version
:
str
,
swh_loader_git_target_repo
:
str
,
swh_storage_version
:
str
,
swh_storage_target_repo
:
str
,
swh_core_version
:
str
,
swh_core_target_repo
:
str
,
python_version
:
str
,
):
if
get_unique_id
:
...
...
@@ -127,6 +239,17 @@
echo
=
noop
echo
(
f
"
POULPE: using python version
{
python_version
}
"
)
echo
(
f
"
POULPE: using swh-git-loader version
{
loader_version
}
"
)
echo
(
f
"
POULPE: using loader url
{
target_repo
}
"
)
echo
(
f
"
POULPE: using swh-loader-git version
{
swh_loader_git_version
}
"
)
echo
(
f
"
POULPE: using swh-loader-git url
{
swh_loader_git_target_repo
}
"
)
echo
(
f
"
POULPE: using swh-storage version
{
swh_storage_version
}
"
)
echo
(
f
"
POULPE: using swh-storage url
{
swh_storage_target_repo
}
"
)
echo
(
f
"
POULPE: using swh-core version
{
swh_core_version
}
"
)
echo
(
f
"
POULPE: using swh-core url
{
swh_core_target_repo
}
"
)
to_install
=
[
(
"
swh-loader-git
"
,
swh_loader_git_target_repo
,
swh_loader_git_version
),
(
"
swh-storage
"
,
swh_storage_target_repo
,
swh_storage_version
),
(
"
swh-core
"
,
swh_core_target_repo
,
swh_core_version
),
]
...
...
@@ -132,18 +255,6 @@
# Does it look like a clone? Don't bother to be more resistant for now
if
Path
(
CLONE_PATH
,
"
.git
"
).
is_dir
():
# Do we already have the revision?
try
:
run_command
([
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
rev-parse
"
,
loader_version
])
except
subprocess
.
CalledProcessError
:
# If not, fetch
echo
(
f
"
POULPE: failed to retrieve version, fetching
"
)
run_command
([
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
fetch
"
,
target_repo
])
echo
(
f
"
POULPE: fetch done
"
)
else
:
echo
(
f
"
POULPE: expected version already in cache
"
)
else
:
echo
(
f
"
POULPE: cloning swh-git-loader from
{
target_repo
}
"
)
run_command
([
"
git
"
,
"
clone
"
,
target_repo
,
CLONE_PATH
])
echo
(
f
"
POULPE: clone done
"
)
echo
(
"
POULPE: creating a new virtual environment
"
)
run_command
([
python_version
,
"
-m
"
,
"
venv
"
,
"
.
"
])
...
...
@@ -149,18 +260,12 @@
git_rank
=
int
(
(
run_command
(
[
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
rev-list
"
,
"
--count
"
,
loader_version
,
]
)
.
strip
()
.
decode
()
)
all_git_sha1
=
[]
all_git_rank
=
[]
for
t
in
to_install
:
rank
,
gs1
=
install_one
(
*
t
,
get_unique_id
=
get_unique_id
)
all_git_sha1
.
append
((
t
[
0
],
gs1
))
all_git_rank
.
append
((
t
[
0
],
rank
))
unique_id
=
compute_unique_id
(
all_git_sha1
,
python_version
,
postgres_version
=
None
)
...
...
@@ -165,14 +270,7 @@
)
git_sha1
=
(
run_command
([
"
git
"
,
"
-C
"
,
CLONE_PATH
,
"
rev-parse
"
,
loader_version
])
.
strip
()
.
decode
()
)
unique_id
=
compute_unique_id
(
git_sha1
,
python_version
,
postgres_version
=
None
)
# XXX We now check the name after starting install, so we destroyed the
# existing bin env in the process
if
not
get_unique_id
:
# This means we've already got the unique ID, so make sure we match
name
=
Path
().
resolve
().
name
...
...
@@ -185,20 +283,6 @@
click
.
echo
(
f
"
UNIQUE_ID:
{
unique_id
}
"
)
exit
(
0
)
echo
(
"
POULPE: creating a new virtual environment
"
)
run_command
([
python_version
,
"
-m
"
,
"
venv
"
,
"
.
"
])
echo
(
"
POULPE: installing swh-git-loader in the new venv
"
)
run_command
(
[
"
bin/pip
"
,
"
install
"
,
# install with no working copy for speed and making concurrent
# installs possible
f
"
git+file://
{
CLONE_PATH
}
@
{
git_sha1
}
"
,
"
--require-virtualenv
"
,
]
)
echo
(
"
POULPE: install OK
"
)
output
=
run_command
(
...
...
@@ -234,9 +318,12 @@
},
"
git
"
:
{
# kept as a useful shortcut, but redundant with `python.dependencies`
"
version
"
:
virtualenv_versions
[
"
swh.loader.git
"
],
"
rank
"
:
git_rank
,
"
sha1
"
:
git_sha1
,
"
version
"
:
'
|
'
.
join
(
virtualenv_versions
[
n
]
for
n
in
[
"
swh.core
"
,
"
swh.storage
"
,
"
swh.loader.git
"
]
),
# XXX wrongy wrong
"
rank
"
:
sum
(
r
for
n
,
r
in
all_git_rank
),
},
}
for
name
,
version
in
virtualenv_versions
.
items
():
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment