# HG changeset patch # User Artem Tikhomirov <tikhomirov.artem@gmail.com> # Date 1344270633 -7200 # Mon Aug 06 18:30:33 2012 +0200 # Node ID b9ede5f917015c649a399bec0e22795575d4cec5 # Parent 03d8e33bf7761526048576c5374cb89735268d49 Subrepos: generate .hgsubstate and .hgsub based on gitlinks and .gitmodules, preserve gitlinks on hg commit export. Tests included. Dependency from PyPI's ordereddict to use OrderedDict diff --git a/hggit/git_handler.py b/hggit/git_handler.py --- a/hggit/git_handler.py +++ b/hggit/git_handler.py @@ -1,11 +1,14 @@ import os, math, urllib, re +import stat, posixpath, StringIO +import ordereddict from dulwich.errors import HangupException, GitProtocolError from dulwich.index import commit_tree -from dulwich.objects import Blob, Commit, Tag, Tree, parse_timezone +from dulwich.objects import Blob, Commit, Tag, Tree, parse_timezone, S_IFGITLINK from dulwich.pack import create_delta, apply_delta from dulwich.repo import Repo from dulwich import client +from dulwich import config as dul_config try: from mercurial import bookmarks @@ -499,7 +502,24 @@ return message def iterblobs(self, ctx): + if '.hgsubstate' in ctx: + hgsub = ordereddict.OrderedDict() + if '.hgsub' in ctx: + hgsub = util.parse_hgsub(ctx['.hgsub'].data().splitlines()) + hgsubstate = util.parse_hgsubstate(ctx['.hgsubstate'].data().splitlines()) + for path, sha in hgsubstate.iteritems(): + try: + if path in hgsub and not hgsub[path].startswith('[git]'): + # some other kind of a repository (e.g. [hg]) + # that keeps its state in .hgsubstate, shall ignore + continue + yield path, sha, S_IFGITLINK + except ValueError: + pass + for f in ctx: + if f == '.hgsubstate' or f == '.hgsub': + continue fctx = ctx[f] blobid = self.map_git_get(hex(fctx.filenode())) @@ -619,6 +639,35 @@ # get a list of the changed, added, removed files files = self.get_files_changed(commit) + # Handle gitlinks: collect + gitlinks = self.collect_gitlinks(commit.tree) + git_commit_tree = self.git[commit.tree] + + # Analyze hgsubstate and build an updated version + # using SHAs from gitlinks + hgsubstate = None + if gitlinks: + hgsubstate = util.parse_hgsubstate(self.git_file_readlines(git_commit_tree, '.hgsubstate')) + for path, sha in gitlinks: + hgsubstate[path] = sha + # in case .hgsubstate wasn't among changed files + # force its inclusion + files['.hgsubstate'] = (False, 0100644, None) + + # Analyze .hgsub and merge with .gitmodules + hgsub = None + gitmodules = self.parse_gitmodules(git_commit_tree) + if gitmodules or gitlinks: + hgsub = util.parse_hgsub(self.git_file_readlines(git_commit_tree, '.hgsub')) + for (sm_path, sm_url, sm_name) in gitmodules: + hgsub[sm_path] = '[git]' + sm_url + files['.hgsub'] = (False, 0100644, None) + elif commit.parents and '.gitmodules' in self.git[self.git[commit.parents[0]].tree]: + # no .gitmodules in this commit, however present in the parent + # mark its hg counterpart as deleted (assuming .hgsub is there + # due to the same import_git_commit process + files['.hgsub'] = (True, 0100644, None) + date = (commit.author_time, -commit.author_timezone) text = strip_message @@ -679,9 +728,17 @@ if delete: raise IOError - data = self.git[sha].data - copied_path = hg_renames.get(f) - e = self.convert_git_int_mode(mode) + if not sha: # indicates there's no git counterpart + e = '' + copied_path = None + if '.hgsubstate' == f: + data = util.serialize_hgsubstate(hgsubstate) + elif '.hgsub' == f: + data = util.serialize_hgsub(hgsub) + else: + data = self.git[sha].data + copied_path = hg_renames.get(f) + e = self.convert_git_int_mode(mode) else: # it's a converged file fc = context.filectx(self.repo, f, changeid=memctx.p1().rev()) @@ -1136,6 +1193,57 @@ return files + def collect_gitlinks(self, tree_id): + """Walk the tree and collect all gitlink entries + :param tree_id: sha of the commit tree + :return: list of tuples (commit sha, full entry path) + """ + queue = [(tree_id, '')] + gitlinks = [] + while queue: + e, path = queue.pop(0) + o = self.git.object_store[e] + for (name, mode, sha) in o.iteritems(): + if mode == S_IFGITLINK: + gitlinks.append((posixpath.join(path, name), sha)) + elif stat.S_ISDIR(mode): + queue.append((sha, posixpath.join(path, name))) + return gitlinks + + def parse_gitmodules(self, tree_obj): + """Parse .gitmodules from a git tree specified by tree_obj + + :return: list of tuples (submodule path, url, name), + where name is quoted part of the section's name, or + empty list if nothing found + """ + rv = [] + try: + unused_mode,gitmodules_sha = tree_obj['.gitmodules'] + except KeyError: + return rv + gitmodules_content = self.git[gitmodules_sha].data + fo = StringIO.StringIO(gitmodules_content) + tt = dul_config.ConfigFile.from_file(fo) + for section in tt.keys(): + section_kind, section_name = section + if section_kind == 'submodule': + sm_path = tt.get(section, 'path') + sm_url = tt.get(section, 'url') + rv.append((sm_path, sm_url, section_name)) + return rv + + def git_file_readlines(self, tree_obj, fname): + """Read content of a named entry from the git commit tree + + :return: list of lines + """ + if fname in tree_obj: + unused_mode, sha = tree_obj[fname] + content = self.git[sha].data + return content.splitlines() + return [] + def remote_name(self, remote): names = [name for name, path in self.paths if path == remote] if names: diff --git a/hggit/util.py b/hggit/util.py --- a/hggit/util.py +++ b/hggit/util.py @@ -1,5 +1,35 @@ """Compatability functions for old Mercurial versions.""" +import ordereddict def progress(ui, *args, **kwargs): """Shim for progress on hg < 1.4. Remove when 1.3 is dropped.""" getattr(ui, 'progress', lambda *x, **kw: None)(*args, **kwargs) + +def parse_hgsub(lines): + """Fills OrderedDict with hgsub file content passed as list of lines""" + rv = ordereddict.OrderedDict() + for l in lines: + ls = l.strip(); + if not ls or ls[0] == '#': continue + name, value = l.split('=', 1) + rv[name.strip()] = value.strip() + return rv + +def serialize_hgsub(data): + """Produces a string from OrderedDict hgsub content""" + return ''.join(['%s = %s\n' % (n,v) for n,v in data.iteritems()]) + +def parse_hgsubstate(lines): + """Fills OrderedDict with hgsubtate file content passed as list of lines""" + rv = ordereddict.OrderedDict() + for l in lines: + ls = l.strip(); + if not ls or ls[0] == '#': continue + value, name = l.split(' ', 1) + rv[name.strip()] = value.strip() + return rv + +def serialize_hgsubstate(data): + """Produces a string from OrderedDict hgsubstate content""" + return ''.join(['%s %s\n' % (data[n], n) for n in sorted(data)]) + diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='hg-git', - version='0.3.3', + version='0.3.2', author='Scott Chacon', maintainer='Augie Fackler', maintainer_email='durin42@gmail.com', @@ -19,5 +19,5 @@ keywords='hg git mercurial', license='GPLv2', packages=['hggit'], - install_requires=['dulwich>=0.8.0'], + install_requires=['dulwich>=0.8.0', 'ordereddict>=1.1'], ) diff --git a/tests/test-subrepos b/tests/test-subrepos new file mode 100644 --- /dev/null +++ b/tests/test-subrepos @@ -0,0 +1,136 @@ +#!/bin/sh + +# Fails for some reason, need to investigate +# "$TESTDIR/hghave" git || exit 80 + +# bail if the user does not have dulwich +python -c 'import dulwich, dulwich.repo' || exit 80 + +# bail early if the user is already running git-daemon +echo hi | nc localhost 9418 2>/dev/null && exit 80 + +echo "[extensions]" >> $HGRCPATH +echo "hggit=$(echo $(dirname $(dirname $0)))/hggit" >> $HGRCPATH +echo 'hgext.bookmarks =' >> $HGRCPATH + + +GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME +GIT_AUTHOR_EMAIL='test@example.org'; export GIT_AUTHOR_EMAIL +GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE +GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME +GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL +GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE + +count=10 +gitcommit() +{ + GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000" + GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE" + git commit "$@" >/dev/null 2>/dev/null || echo "git commit error" + count=`expr $count + 1` +} +hgcommit() +{ + HGDATE="2007-01-01 00:00:$count +0000" + hg commit -d "$HGDATE" "$@" >/dev/null 2>/dev/null || echo "hg commit error" + count=`expr $count + 1` +} + + +mkdir gitsubrepo +cd gitsubrepo +git init | python -c "import sys; print sys.stdin.read().replace('$(dirname $(pwd))/', '')" +echo beta > beta +git add beta +gitcommit -m 'add beta' +cd .. + +mkdir gitrepo1 +cd gitrepo1 +git init | python -c "import sys; print sys.stdin.read().replace('$(dirname $(pwd))/', '')" +echo alpha > alpha +git add alpha +gitcommit -m 'add alpha' +git submodule add ../gitsubrepo subrepo1 +gitcommit -m 'add subrepo1' +git submodule add ../gitsubrepo xyz/subrepo2 +gitcommit -m 'add subrepo2' +# we are going to push to this repo from our hg clone, +# allow commits despite working copy presense +git config receive.denyCurrentBranch ignore +cd .. +echo + +git daemon --base-path="$(pwd)"\ + --listen=localhost\ + --export-all\ + --pid-file="$DAEMON_PIDS" \ + --detach --reuseaddr \ + --enable=receive-pack + +echo % Ensure gitlinks are transformed to .hgsubstate on hg pull from git +hg clone git://localhost/gitrepo1 hgrepo +cd hgrepo +hg bookmarks -f -r default master +# 1. Ensure gitlinks are transformed to .hgsubstate on hg <- git pull +# Synopsis: clone, check .hgsub and .hgsubstate are in place +echo % .hgsub shall list two [git] subrepos +cat .hgsub +echo % .hgsubstate shall list two idenitcal revisions +cat .hgsubstate +echo % hg status shall NOT report .hgsub and .hgsubstate as untracked - either ignored or unmodified +hg status --unknown .hgsub .hgsubstate +hg status --modified .hgsub .hgsubstate +cd .. +echo + +# 2. Check gitmodules are preserved during hg -> git push +# Synopsis: modify git sumodule, update hg subrepo to this new revision, +# hg commit, push to git, see gitlinks are correct (preserved for subrepo1, +# and updated for subrepo2 +echo % Check gitmodules are preserved during hg push to git +cd gitsubrepo +echo gamma > gamma +git add gamma +gitcommit -m 'add gamma' +cd .. +cd hgrepo +cd xyz/subrepo2 +git pull +cd ../.. +echo xxx >> alpha +hg commit -m 'Update subrepo2 from hg' +hg push +cd .. +cd gitrepo1 +echo % there shall be two gitlink entries, with values matching that in .hgsubstate +git ls-tree -r HEAD^{tree} | grep 'commit' +# bring working copy to HEAD state (it's not bare repo) +git reset --hard +cd .. +echo + +# 3. Check .hgsub and .hgsubstate from git repository are merged, not overwritten +# Synopsis: add hg repo to become a subrepo for existing repo. +# Commit .hgsub and .hgsubstate in gitrepo, that point to newly added hg repo +# pull changes to exising hgrepo, see preserved values +echo Check .hgsub and .hgsubstate from git repository are merged, not overwritten +hg init hgsub +cd hgsub +echo delta > delta +hg add delta +hgcommit -m "add delta" +echo "`hg tip --template '{node}'` hgsub" > ../gitrepo1/.hgsubstate +echo "hgsub = $(pwd)" > ../gitrepo1/.hgsub +cd ../gitrepo1 +git add .hgsubstate .hgsub +gitcommit -m "Test3. Prepare .hgsub and .hgsubstate sources" +cd ../hgrepo +hg pull +hg checkout -C | sed s_$(dirname $(pwd))_TEMPLOCATION_ +cd .. +echo % pull shall bring .hgsub entry which was added to the git repo +cat hgrepo/.hgsub | sed s_$(pwd)_TEMPLOCATION_ +echo % .hgsubstate shall list revision of the subrepo added through git repo +cat hgrepo/.hgsubstate | sed s_$(pwd)_TEMPLOCATION_ + diff --git a/tests/test-subrepos.out b/tests/test-subrepos.out new file mode 100644 --- /dev/null +++ b/tests/test-subrepos.out @@ -0,0 +1,54 @@ +Initialized empty Git repository in gitsubrepo/.git/ + +Initialized empty Git repository in gitrepo1/.git/ + +Cloning into 'subrepo1'... +done. +Cloning into 'xyz/subrepo2'... +done. + +% Ensure gitlinks are transformed to .hgsubstate on hg pull from git +importing git objects into hg +updating to branch default +cloning subrepo subrepo1 from git://localhost/gitsubrepo +cloning subrepo xyz/subrepo2 from git://localhost/gitsubrepo +4 files updated, 0 files merged, 0 files removed, 0 files unresolved +% .hgsub shall list two [git] subrepos +subrepo1 = [git]../gitsubrepo +xyz/subrepo2 = [git]../gitsubrepo +% .hgsubstate shall list two idenitcal revisions +56f0304c5250308f14cfbafdc27bd12d40154d17 subrepo1 +56f0304c5250308f14cfbafdc27bd12d40154d17 xyz/subrepo2 +% hg status shall NOT report .hgsub and .hgsubstate as untracked - either ignored or unmodified + +% Check gitmodules are preserved during hg push to git +From git://localhost/gitsubrepo + 56f0304..aabf7cd master -> origin/master +Updating 56f0304..aabf7cd +Fast-forward + gamma | 1 + + 1 file changed, 1 insertion(+) + create mode 100644 gamma +pushing to git://localhost/gitrepo1 +exporting hg objects to git +creating and sending data + default::refs/heads/master => GIT:4663c492 +% there shall be two gitlink entries, with values matching that in .hgsubstate +160000 commit 56f0304c5250308f14cfbafdc27bd12d40154d17 subrepo1 +160000 commit aabf7cd015089aff0b84596e69aa37b24a3d090a xyz/subrepo2 +HEAD is now at 4663c49 Update subrepo2 from hg + +Check .hgsub and .hgsubstate from git repository are merged, not overwritten +pulling from git://localhost/gitrepo1 +importing git objects into hg +(run 'hg update' to get a working copy) +cloning subrepo hgsub from TEMPLOCATION/hgsub +2 files updated, 0 files merged, 0 files removed, 0 files unresolved +% pull shall bring .hgsub entry which was added to the git repo +hgsub = TEMPLOCATION/hgsub +subrepo1 = [git]../gitsubrepo +xyz/subrepo2 = [git]../gitsubrepo +% .hgsubstate shall list revision of the subrepo added through git repo +481ec30d580f333ae3a77f94c973ce37b69d5bda hgsub +56f0304c5250308f14cfbafdc27bd12d40154d17 subrepo1 +aabf7cd015089aff0b84596e69aa37b24a3d090a xyz/subrepo2