Remote fetching

Added classes to fetch packages/trees etc. from remote repositories of
all kinds mentioned in overlays.xml: git, mercurial, svn, rsync

Added utility functions for git repos and to interact with filesystem
repository hierarchy

7 files changed, 304 insertions, 13 deletions

diff --git a/pomu/repo/remote/git.py b/pomu/repo/remote/git.py
new file mode 100644
index 0000000..217a0c6
--- /dev/null
+++ b/pomu/repo/remote/git.py
@@ -0,0 +1,68 @@
+"""A class for remote git repos"""
+from os import chdir, mkdtemp, path
+from subprocess import call
+
+from git import Repo
+
+from pomu.repo.remote import RemoteRepo, normalize_key
+from pomu.util.git import parse_object
+from pomu.util.result import Result
+
+class RemoteGitRepo(RemoteRepo):
+    """A class responsible for git remotes"""
+    def __init__(self, url):
+        self.uri = uri
+        self.dir = mkdtemp()
+        chdir(self.dir)
+        if call('git', 'clone', '--depth=1', '--bare', url) > 0: # we've a problem
+            raise RuntimeError()
+        self.repo = Repo(self.dir)
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *_):
+        self.cleanup()
+
+    def get_object(self, oid):
+        head, tail = oid[0:2], oid[2:]
+        opath = path.join(self.dir, 'objects', head, tail)
+        return a.read()
+
+    def _fetch_tree(self, obj, tpath):
+        res = []
+        ents = parse_object(self.get_object(tid), tpath).unwrap()
+        for is_dir, sha, opath in ents:
+            res.append((opath + '/' if is_dir else '', sha))
+            if is_dir:
+                res.extend(self._fetch_tree(sha, tpath + opath))
+        return res
+
+    def fetch_tree(self):
+        """Returns repos hierarchy"""
+        if hasattr(self, '_tree'):
+            return self._tree
+        tid = self.repo.tree().hexsha
+        res = self._fetch_tree(tid, '')
+        self._tree = res
+        return [x for x, y in res]
+
+    def fetch_subtree(self, key):
+        """Lists a subtree"""
+        k = normalize_key(key, True)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        l = len(key)
+        return Result.Ok(
+                [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)])
+
+    def fetch_file(self, key):
+        """Fetches a file from the repo"""
+        k = normalize_key(key)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        return Result.Ok(parse_object(self.get_object(dic[k])))
diff --git a/pomu/repo/remote/hg.py b/pomu/repo/remote/hg.py
new file mode 100644
index 0000000..678f843
--- /dev/null
+++ b/pomu/repo/remote/hg.py
@@ -0,0 +1,58 @@
+"""A class for remote hg repos"""
+from os import chdir, mkdtemp, path
+from shutil import rmtree
+from subprocess import call, run, check_output
+
+from pomu.repo.remote import RemoteRepo, normalize_key
+from pomu.util.result import Result
+
+class RemoteHgRepo(RemoteRepo):
+    """A class responsible for hg remotes"""
+    def __init__(self, url):
+        self.uri = uri
+        self.dir = mkdtemp()
+        chdir(self.dir)
+        if call('hg', 'clone', '-U', url, '.') > 0: # we've a problem
+            raise RuntimeError()
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *_):
+        self.cleanup()
+
+    def fetch_tree(self):
+        """Returns repos hierarchy"""
+        if hasattr(self, '_tree'):
+            return self._tree
+        p = run('hg', 'files', '-rdefault')
+        if p.returncode:
+            return []
+        self._tree = ['/' + x for x in p.stdout.split('\n')]
+        return self._tree
+
+    def fetch_subtree(self, key):
+        """Lists a subtree"""
+        k = normalize_key(key, True)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        l = len(key)
+        return Result.Ok(
+                [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)])
+
+    def fetch_file(self, key):
+        """Fetches a file from the repo"""
+        k = normalize_key(key)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        p = run('hg', 'cat', '-rdefault', k)
+        if p.returncode:
+            return Result.Err()
+        return Result.Ok(p.stdout)
+
+    def cleanup(self):
+        rmtree(self.dir)
diff --git a/pomu/repo/remote/remote.py b/pomu/repo/remote/remote.py
index e177f68..d057e6f 100644
--- a/pomu/repo/remote/remote.py
+++ b/pomu/repo/remote/remote.py
@@ -4,16 +4,28 @@
 class RemoteRepo():
     """A class responsible for remotes"""
     def __init__(self, url):
-        self.uri = uri
         raise NotImplementedError()
 
     def fetch_package(self, name, category=None, version=None):
         """Fetches a package, determined by the parametres"""
-        raise NotImplementedError()
+        cat, n, ver = get_full_cpv(self.fetch_tree(), name, category, version).unwrap()
+        ebuild = '{}/{}/{}-{}.ebuild'.format(cat, n, n, ver)
+        subdir = '/{}/{}'.format(category, name)
+        filemap = {}
+        filemap[ebuild] = self.fetch_file(ebuild).unwrap()
+        subtree = self.fetch_subtree('/{}/{}/'.format(category, name))
+        for fpath in subtree:
+            if '/' in fpath:
+                parent, _, child = fpath.rpartition('/')
+                if parent != 'files': continue
+            if fpath.endswith('.ebuild') or fpath.endswith('/'): continue
+            p = path.join(subdir, fpath)
+            filemap[p] = self.fetch_file(p).unwrap()
+        return Package(name, '/', None, category, version, filemap=filemap)
 
     def list_cpvs(self):
         """Gets a list of all pebuilds in the repo"""
-        raise NotImplementedError()
+        return filelist_to_cpvs(self.fetch_tree())
 
     def fetch_tree(self):
         """Returns repos hierarchy"""
@@ -26,3 +38,8 @@ class RemoteRepo():
     def fetch_file(self, key):
         """Fetches a file from the repo"""
         raise NotImplementedError()
+
+def normalize_key(key, trail=False):
+    k = '/' + key.lstrip('/')
+    if trail:
+        k = k.rstrip('/') + '/'
diff --git a/pomu/repo/remote/rsync.py b/pomu/repo/remote/rsync.py
new file mode 100644
index 0000000..8f00e5e
--- /dev/null
+++ b/pomu/repo/remote/rsync.py
@@ -0,0 +1,59 @@
+"""A class for remote rsync repos"""
+from os import mkdtemp, rmdir, mkfifo, unlink
+import re
+from subprocess import run
+
+from pomu.repo.remote import RemoteRepo, normalize_key
+from pomu.util.result import Result
+
+class RemoteRsyncRepo(RemoteRepo):
+    """A class responsible for rsync remotes"""
+    def __init__(self, url):
+        self.uri = uri
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *_):
+        pass
+
+    def fetch_tree(self):
+        """Returns repos hierarchy"""
+        if hasattr(self, '_tree'):
+            return self._tree
+        d = mkdtemp()
+        p = run('rsync', '-rn', '--out-format="%n"' self.uri, d)
+        rmdir(d)
+        if p.returncode:
+            return Result.Err()
+        self._tree = ['/' + x for x in p.stdout.split('\n')]
+        return self._tree
+
+    def fetch_subtree(self, key):
+        """Lists a subtree"""
+        k = normalize_key(key, True)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return []
+        l = len(key)
+        return Result.Ok(
+                [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)])
+
+    def fetch_file(self, key):
+        """Fetches a file from the repo"""
+        k = normalize_key(key)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        d = mkdtemp()
+        fip = path.join(d, 'fifo')
+        mkfifo(fip)
+        p = run('rsync', self.uri.rstrip('/') + key, fip)
+        fout = fip.read()
+        unlink(fip)
+        rmdir(d)
+        if p.returncode:
+            return Result.Err()
+        return Result.Ok(fout)
diff --git a/pomu/repo/remote/svn.py b/pomu/repo/remote/svn.py
new file mode 100644
index 0000000..4b1e1d3
--- /dev/null
+++ b/pomu/repo/remote/svn.py
@@ -0,0 +1,49 @@
+"""A class for remote svn repos"""
+from subprocess import run
+
+from pomu.repo.remote import RemoteRepo, normalize_key
+from pomu.util.result import Result
+
+class RemoteSvnRepo(RemoteRepo):
+    """A class responsible for svn remotes"""
+    def __init__(self, url):
+        self.uri = uri
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *_):
+        pass
+
+    def fetch_tree(self):
+        """Returns repos hierarchy"""
+        if hasattr(self, '_tree'):
+            return self._tree
+        p = run('svn', 'ls', '-R', self.uri)
+        if p.returncode:
+            return []
+        self._tree = p.stdout.split('\n')
+        return self._tree
+
+    def fetch_subtree(self, key):
+        """Lists a subtree"""
+        k = normalize_key(key, True)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        l = len(key)
+        return Result.Ok(
+                [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)])
+
+    def fetch_file(self, key):
+        """Fetches a file from the repo"""
+        k = normalize_key(key)
+        self.fetch_tree()
+        dic = dict(self._tree)
+        if k not in dic:
+            return Result.Err()
+        p = run('svn', 'cat', k)
+        if p.returncode:
+            return Result.Err()
+        return Result.Ok(p.stdout)
diff --git a/pomu/util/git.py b/pomu/util/git.py
index 4c3e2ed..611b27c 100644
--- a/pomu/util/git.py
+++ b/pomu/util/git.py
@@ -1,23 +1,25 @@
 """Miscellaneous utility functions for git structures"""
 
 from base64 import b16encode
+import zlib
 
 from pomu.util.result import Result
 
-def parse_tree(blob):
+def parse_tree(blob, path=''):
     """Parses a git tree"""
     res = []
-    leng, _, tree = blob.partition('\0')
+    leng, _, tree = blob.partition(b'\0')
     if not tree:
         return Result.Err('Invalid tree')
-    while len(data) > 0:
-        data = data[7:] # strip access code
-        name, _, data = data.partition('\0')
-        sha = b16encode(data[0:20]).decode('utf-8')
-        data = data[20:]
+    while len(tree) > 0:
+        mode, _, tree = tree.partition(b' ')
+        name, _, tree = tree.partition(b'\0')
+        sha = b16encode(tree[0:20]).decode('utf-8').lower()
+        tree = tree[20:]
         if not name or not sha:
             return Result.Err()
-        res.append((sha, name))
+        is_dir = mode[0:1] != b'1'
+        res.append((is_dir, sha, path.encode('utf-8') + b'/' + name))
     return Result.Ok(res)
 
 def parse_blob(blob):
@@ -39,8 +41,8 @@ def commit_head(blob):
 def parse_object(obj):
     """Parses a git object"""
     data = zlib.decompress(obj)
-    if data[0:5] == 'blob ':
+    if data[0:5] == b'blob ':
         return parse_blob(data[5:])
-    elif data[0:5] == 'tree ':
+    elif data[0:5] == b'tree ':
         return parse_tree(data[5:])
     return Result.Err('Unsupported object type')
diff --git a/pomu/util/remote.py b/pomu/util/remote.py
new file mode 100644
index 0000000..20691e2
--- /dev/null
+++ b/pomu/util/remote.py
@@ -0,0 +1,38 @@
+"""
+Utilities for remotes
+"""
+
+from portage.versions import best
+
+from pomu.util.pkg import ver_str, cpv_split
+from pomu.util.portage import misc_dirs
+
+def filelist_to_cpvs(tree):
+    """Converts a list of files to list of cpvs"""
+    res = []
+    for opath in tree:
+        comps = opath.split('/')
+        if (opath.endswith('/') or
+            any(opath.startswith('/' + x + '/') for x in misc_dirs) or
+            len(comps) != 3 or
+            not comps[2].endswith('.ebuild')):
+            continue
+        category, name, ebuild = comps[0], comps[1], comps[2][:-7]
+        c, n, v, s, r = cpv_split(ebuild)
+        if category or n != name:
+            continue
+        ver = ver_str(v, s, r)
+        res.append((category, name, ver))
+    return res
+
+def get_full_cpv(cpvs, name, category=None, version=None):
+    cpvl = cpvs.filter(lambda x: x[1] == name and (not category or x[0] == category))
+    if not cpvl: return Result.Err()
+    if version:
+        cpvl = cpvl.filter(lambda x: x[2] == version)[:1]
+    b = best('{}/{}-{}'.format(c, n, v) for c, n, v in cpvl)
+    if b:
+        cat, name, v, s, r = cpv_split(b)
+        ver = ver_str(v, s, r)
+        return Result.Ok((cat, name, ver))
+    return Result.Err()