# HG changeset patch
# User Artem Tikhomirov <tikhomirov.artem@gmail.com>
# Date 1344270633 -7200
#      Mon Aug 06 18:30:33 2012 +0200
# Node ID b9ede5f917015c649a399bec0e22795575d4cec5
# Parent  03d8e33bf7761526048576c5374cb89735268d49
Subrepos: generate .hgsubstate and .hgsub based on gitlinks and .gitmodules, preserve gitlinks on hg commit export. Tests included. Dependency from PyPI's ordereddict to use OrderedDict

diff --git a/hggit/git_handler.py b/hggit/git_handler.py
--- a/hggit/git_handler.py
+++ b/hggit/git_handler.py
@@ -1,11 +1,14 @@
 import os, math, urllib, re
+import stat, posixpath, StringIO
+import ordereddict
 
 from dulwich.errors import HangupException, GitProtocolError
 from dulwich.index import commit_tree
-from dulwich.objects import Blob, Commit, Tag, Tree, parse_timezone
+from dulwich.objects import Blob, Commit, Tag, Tree, parse_timezone, S_IFGITLINK
 from dulwich.pack import create_delta, apply_delta
 from dulwich.repo import Repo
 from dulwich import client
+from dulwich import config as dul_config
 
 try:
     from mercurial import bookmarks
@@ -499,7 +502,24 @@
         return message
 
     def iterblobs(self, ctx):
+        if '.hgsubstate' in ctx:
+            hgsub = ordereddict.OrderedDict()
+            if '.hgsub' in ctx:
+                hgsub = util.parse_hgsub(ctx['.hgsub'].data().splitlines())
+            hgsubstate = util.parse_hgsubstate(ctx['.hgsubstate'].data().splitlines())
+            for path, sha in hgsubstate.iteritems():
+                try:
+                    if path in hgsub and not hgsub[path].startswith('[git]'):
+                        # some other kind of a repository (e.g. [hg])
+                        # that keeps its state in .hgsubstate, shall ignore
+                        continue
+                    yield path, sha, S_IFGITLINK
+                except ValueError:
+                    pass
+
         for f in ctx:
+            if f == '.hgsubstate' or f == '.hgsub':
+                continue
             fctx = ctx[f]
             blobid = self.map_git_get(hex(fctx.filenode()))
 
@@ -619,6 +639,35 @@
         # get a list of the changed, added, removed files
         files = self.get_files_changed(commit)
 
+        # Handle gitlinks: collect
+        gitlinks = self.collect_gitlinks(commit.tree)
+        git_commit_tree = self.git[commit.tree]
+
+        # Analyze hgsubstate and build an updated version
+        # using SHAs from gitlinks
+        hgsubstate = None
+        if gitlinks:
+            hgsubstate = util.parse_hgsubstate(self.git_file_readlines(git_commit_tree, '.hgsubstate'))
+            for path, sha in gitlinks:
+                hgsubstate[path] = sha
+            # in case .hgsubstate wasn't among changed files
+            # force its inclusion
+            files['.hgsubstate'] = (False, 0100644, None)
+
+        # Analyze .hgsub and merge with .gitmodules
+        hgsub = None
+        gitmodules = self.parse_gitmodules(git_commit_tree)
+	if gitmodules or gitlinks:
+            hgsub = util.parse_hgsub(self.git_file_readlines(git_commit_tree, '.hgsub'))
+            for (sm_path, sm_url, sm_name) in gitmodules:
+                hgsub[sm_path] = '[git]' + sm_url
+            files['.hgsub'] = (False, 0100644, None)
+        elif commit.parents and '.gitmodules' in self.git[self.git[commit.parents[0]].tree]:
+            # no .gitmodules in this commit, however present in the parent
+            # mark its hg counterpart as deleted (assuming .hgsub is there
+            # due to the same import_git_commit process
+            files['.hgsub'] = (True, 0100644, None)
+
         date = (commit.author_time, -commit.author_timezone)
         text = strip_message
 
@@ -679,9 +728,17 @@
                 if delete:
                     raise IOError
 
-                data = self.git[sha].data
-                copied_path = hg_renames.get(f)
-                e = self.convert_git_int_mode(mode)
+                if not sha: # indicates there's no git counterpart
+                    e = ''
+                    copied_path = None
+                    if '.hgsubstate' == f:
+                        data = util.serialize_hgsubstate(hgsubstate)
+                    elif '.hgsub' == f:
+                        data = util.serialize_hgsub(hgsub)
+                else:
+                    data = self.git[sha].data
+                    copied_path = hg_renames.get(f)
+                    e = self.convert_git_int_mode(mode)
             else:
                 # it's a converged file
                 fc = context.filectx(self.repo, f, changeid=memctx.p1().rev())
@@ -1136,6 +1193,57 @@
 
         return files
 
+    def collect_gitlinks(self, tree_id):
+        """Walk the tree and collect all gitlink entries
+          :param tree_id: sha of the commit tree
+          :return: list of tuples (commit sha, full entry path)
+        """
+        queue = [(tree_id, '')]
+        gitlinks = []
+        while queue:
+            e, path = queue.pop(0)
+            o = self.git.object_store[e]
+            for (name, mode, sha) in o.iteritems():
+                if mode == S_IFGITLINK:
+                    gitlinks.append((posixpath.join(path, name), sha))
+                elif stat.S_ISDIR(mode):
+                    queue.append((sha, posixpath.join(path, name)))
+        return gitlinks
+
+    def parse_gitmodules(self, tree_obj):
+        """Parse .gitmodules from a git tree specified by tree_obj
+
+           :return: list of tuples (submodule path, url, name),
+           where name is quoted part of the section's name, or
+           empty list if nothing found
+        """
+        rv = []
+        try:
+            unused_mode,gitmodules_sha = tree_obj['.gitmodules']
+        except KeyError:
+            return rv
+        gitmodules_content = self.git[gitmodules_sha].data
+        fo = StringIO.StringIO(gitmodules_content)
+        tt = dul_config.ConfigFile.from_file(fo)
+        for section in tt.keys():
+            section_kind, section_name = section
+            if section_kind == 'submodule':
+                sm_path = tt.get(section, 'path')
+                sm_url  = tt.get(section, 'url')
+                rv.append((sm_path, sm_url, section_name))
+        return rv
+
+    def git_file_readlines(self, tree_obj, fname):
+        """Read content of a named entry from the git commit tree
+
+           :return: list of lines
+        """
+        if fname in tree_obj:
+            unused_mode, sha = tree_obj[fname]
+            content = self.git[sha].data
+            return content.splitlines()
+        return []
+
     def remote_name(self, remote):
         names = [name for name, path in self.paths if path == remote]
         if names:
diff --git a/hggit/util.py b/hggit/util.py
--- a/hggit/util.py
+++ b/hggit/util.py
@@ -1,5 +1,35 @@
 """Compatability functions for old Mercurial versions."""
+import ordereddict
 
 def progress(ui, *args, **kwargs):
     """Shim for progress on hg < 1.4. Remove when 1.3 is dropped."""
     getattr(ui, 'progress', lambda *x, **kw: None)(*args, **kwargs)
+
+def parse_hgsub(lines):
+    """Fills OrderedDict with hgsub file content passed as list of lines"""
+    rv = ordereddict.OrderedDict()
+    for l in lines:
+        ls = l.strip();
+        if not ls or ls[0] == '#': continue
+        name, value = l.split('=', 1)
+        rv[name.strip()] = value.strip()
+    return rv
+
+def serialize_hgsub(data):
+    """Produces a string from OrderedDict hgsub content"""
+    return ''.join(['%s = %s\n' % (n,v) for n,v in data.iteritems()])
+
+def parse_hgsubstate(lines):
+    """Fills OrderedDict with hgsubtate file content passed as list of lines"""
+    rv = ordereddict.OrderedDict()
+    for l in lines:
+        ls = l.strip();
+        if not ls or ls[0] == '#': continue
+        value, name = l.split(' ', 1)
+        rv[name.strip()] = value.strip()
+    return rv
+
+def serialize_hgsubstate(data):
+    """Produces a string from OrderedDict hgsubstate content"""
+    return ''.join(['%s %s\n' % (data[n], n) for n in sorted(data)])
+
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name='hg-git',
-    version='0.3.3',
+    version='0.3.2',
     author='Scott Chacon',
     maintainer='Augie Fackler',
     maintainer_email='durin42@gmail.com',
@@ -19,5 +19,5 @@
     keywords='hg git mercurial',
     license='GPLv2',
     packages=['hggit'],
-    install_requires=['dulwich>=0.8.0'],
+    install_requires=['dulwich>=0.8.0', 'ordereddict>=1.1'],
 )
diff --git a/tests/test-subrepos b/tests/test-subrepos
new file mode 100644
--- /dev/null
+++ b/tests/test-subrepos
@@ -0,0 +1,136 @@
+#!/bin/sh
+
+# Fails for some reason, need to investigate
+# "$TESTDIR/hghave" git || exit 80
+
+# bail if the user does not have dulwich
+python -c 'import dulwich, dulwich.repo' || exit 80
+
+# bail early if the user is already running git-daemon
+echo hi | nc localhost 9418 2>/dev/null && exit 80
+
+echo "[extensions]" >> $HGRCPATH
+echo "hggit=$(echo $(dirname $(dirname $0)))/hggit" >> $HGRCPATH
+echo 'hgext.bookmarks =' >> $HGRCPATH
+
+
+GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME
+GIT_AUTHOR_EMAIL='test@example.org'; export GIT_AUTHOR_EMAIL
+GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE
+GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME
+GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL
+GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE
+
+count=10
+gitcommit()
+{
+    GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000"
+    GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"
+    git commit "$@" >/dev/null 2>/dev/null || echo "git commit error"
+    count=`expr $count + 1`
+}
+hgcommit()
+{
+    HGDATE="2007-01-01 00:00:$count +0000"
+    hg commit -d "$HGDATE" "$@" >/dev/null 2>/dev/null || echo "hg commit error"
+    count=`expr $count + 1`
+}
+
+
+mkdir gitsubrepo
+cd gitsubrepo
+git init | python -c "import sys; print sys.stdin.read().replace('$(dirname $(pwd))/', '')"
+echo beta > beta
+git add beta
+gitcommit -m 'add beta'
+cd ..
+
+mkdir gitrepo1
+cd gitrepo1
+git init | python -c "import sys; print sys.stdin.read().replace('$(dirname $(pwd))/', '')"
+echo alpha > alpha
+git add alpha
+gitcommit -m 'add alpha'
+git submodule add ../gitsubrepo subrepo1
+gitcommit -m 'add subrepo1'
+git submodule add ../gitsubrepo xyz/subrepo2
+gitcommit -m 'add subrepo2'
+# we are going to push to this repo from our hg clone, 
+# allow commits despite working copy presense
+git config receive.denyCurrentBranch ignore
+cd ..
+echo
+
+git daemon --base-path="$(pwd)"\
+ --listen=localhost\
+ --export-all\
+ --pid-file="$DAEMON_PIDS" \
+ --detach --reuseaddr \
+ --enable=receive-pack
+
+echo % Ensure gitlinks are transformed to .hgsubstate on hg pull from git
+hg clone git://localhost/gitrepo1 hgrepo
+cd hgrepo
+hg bookmarks -f -r default master
+# 1. Ensure gitlinks are transformed to .hgsubstate on hg <- git pull
+#    Synopsis: clone, check .hgsub and .hgsubstate are in place
+echo % .hgsub shall list two [git] subrepos
+cat .hgsub
+echo % .hgsubstate shall list two idenitcal revisions
+cat .hgsubstate
+echo % hg status shall NOT report .hgsub and .hgsubstate as untracked - either ignored or unmodified
+hg status --unknown .hgsub .hgsubstate
+hg status --modified .hgsub .hgsubstate
+cd ..
+echo
+
+# 2. Check gitmodules are preserved during hg -> git push
+#    Synopsis: modify git sumodule, update hg subrepo to this new revision,
+#    hg commit, push to git, see gitlinks are correct (preserved for subrepo1,
+#    and updated for subrepo2
+echo % Check gitmodules are preserved during hg push to git
+cd gitsubrepo
+echo gamma > gamma
+git add gamma
+gitcommit -m 'add gamma'
+cd ..
+cd hgrepo
+cd xyz/subrepo2
+git pull
+cd ../..
+echo xxx >> alpha
+hg commit -m 'Update subrepo2 from hg'
+hg push
+cd ..
+cd gitrepo1
+echo % there shall be two gitlink entries, with values matching that in .hgsubstate
+git ls-tree -r HEAD^{tree} | grep 'commit'
+# bring working copy to HEAD state (it's not bare repo)
+git reset --hard
+cd ..
+echo
+
+# 3. Check .hgsub and .hgsubstate from git repository are merged, not overwritten
+#    Synopsis: add hg repo to become a subrepo for existing repo.
+#    Commit .hgsub and .hgsubstate in gitrepo, that point to newly added hg repo
+#    pull changes to exising hgrepo, see preserved values
+echo Check .hgsub and .hgsubstate from git repository are merged, not overwritten
+hg init hgsub
+cd hgsub
+echo delta > delta
+hg add delta
+hgcommit -m "add delta"
+echo "`hg tip --template '{node}'` hgsub" > ../gitrepo1/.hgsubstate
+echo "hgsub = $(pwd)" > ../gitrepo1/.hgsub
+cd ../gitrepo1
+git add .hgsubstate .hgsub
+gitcommit -m "Test3. Prepare .hgsub and .hgsubstate sources"
+cd ../hgrepo
+hg pull
+hg checkout -C  | sed s_$(dirname $(pwd))_TEMPLOCATION_
+cd ..
+echo % pull shall bring .hgsub entry which was added to the git repo
+cat hgrepo/.hgsub | sed s_$(pwd)_TEMPLOCATION_
+echo % .hgsubstate shall list revision of the subrepo added through git repo
+cat hgrepo/.hgsubstate | sed s_$(pwd)_TEMPLOCATION_
+
diff --git a/tests/test-subrepos.out b/tests/test-subrepos.out
new file mode 100644
--- /dev/null
+++ b/tests/test-subrepos.out
@@ -0,0 +1,54 @@
+Initialized empty Git repository in gitsubrepo/.git/
+
+Initialized empty Git repository in gitrepo1/.git/
+
+Cloning into 'subrepo1'...
+done.
+Cloning into 'xyz/subrepo2'...
+done.
+
+% Ensure gitlinks are transformed to .hgsubstate on hg pull from git
+importing git objects into hg
+updating to branch default
+cloning subrepo subrepo1 from git://localhost/gitsubrepo
+cloning subrepo xyz/subrepo2 from git://localhost/gitsubrepo
+4 files updated, 0 files merged, 0 files removed, 0 files unresolved
+% .hgsub shall list two [git] subrepos
+subrepo1 = [git]../gitsubrepo
+xyz/subrepo2 = [git]../gitsubrepo
+% .hgsubstate shall list two idenitcal revisions
+56f0304c5250308f14cfbafdc27bd12d40154d17 subrepo1
+56f0304c5250308f14cfbafdc27bd12d40154d17 xyz/subrepo2
+% hg status shall NOT report .hgsub and .hgsubstate as untracked - either ignored or unmodified
+
+% Check gitmodules are preserved during hg push to git
+From git://localhost/gitsubrepo
+   56f0304..aabf7cd  master     -> origin/master
+Updating 56f0304..aabf7cd
+Fast-forward
+ gamma |    1 +
+ 1 file changed, 1 insertion(+)
+ create mode 100644 gamma
+pushing to git://localhost/gitrepo1
+exporting hg objects to git
+creating and sending data
+    default::refs/heads/master => GIT:4663c492
+% there shall be two gitlink entries, with values matching that in .hgsubstate
+160000 commit 56f0304c5250308f14cfbafdc27bd12d40154d17	subrepo1
+160000 commit aabf7cd015089aff0b84596e69aa37b24a3d090a	xyz/subrepo2
+HEAD is now at 4663c49 Update subrepo2 from hg
+
+Check .hgsub and .hgsubstate from git repository are merged, not overwritten
+pulling from git://localhost/gitrepo1
+importing git objects into hg
+(run 'hg update' to get a working copy)
+cloning subrepo hgsub from TEMPLOCATION/hgsub
+2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+% pull shall bring .hgsub entry which was added to the git repo
+hgsub = TEMPLOCATION/hgsub
+subrepo1 = [git]../gitsubrepo
+xyz/subrepo2 = [git]../gitsubrepo
+% .hgsubstate shall list revision of the subrepo added through git repo
+481ec30d580f333ae3a77f94c973ce37b69d5bda hgsub
+56f0304c5250308f14cfbafdc27bd12d40154d17 subrepo1
+aabf7cd015089aff0b84596e69aa37b24a3d090a xyz/subrepo2