Blame tools/util.py

Owen W. Taylor 646539e
import gzip
Owen W. Taylor 646539e
import hashlib
Owen W. Taylor 646539e
import pickle
Owen W. Taylor 646539e
import rpm
Owen W. Taylor 646539e
import os
Owen W. Taylor 646539e
import sys
Owen W. Taylor 646539e
import xml.etree.ElementTree as ET
Owen W. Taylor 646539e
import xml.sax
Owen W. Taylor 646539e
Owen W. Taylor 646539e
XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")
Owen W. Taylor 646539e
Owen W. Taylor 646539e
# This needs to be in sync with fedmod
Owen W. Taylor 646539e
REPOS = [
Owen W. Taylor adbd27a
    "f29-packages"
Owen W. Taylor 646539e
]
Owen W. Taylor 646539e
Owen W. Taylor 646539e
_log_name = None
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def set_log_name(name):
Owen W. Taylor 646539e
    global _log_name
Owen W. Taylor 646539e
    _log_name = name
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def warn(msg):
Owen W. Taylor 646539e
    print("{}: \033[31m{}\033[39m".format(_log_name, msg), file=sys.stderr)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def error(msg):
Owen W. Taylor 646539e
    print("{}: \033[31m{}\033[39m".format(_log_name, msg), file=sys.stderr)
Owen W. Taylor 646539e
    sys.exit(1)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def start(msg):
Owen W. Taylor 646539e
    print("{}: \033[90m{} ... \033[39m".format(_log_name, msg), file=sys.stderr, end="")
Owen W. Taylor 646539e
    sys.stderr.flush()
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def done():
Owen W. Taylor 646539e
    print("\033[90mdone\033[39m", file=sys.stderr)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def package_cmp(p1, p2):
Owen W. Taylor 646539e
    n1, e1, v1, r1, a1 = p1
Owen W. Taylor 646539e
    n2, e2, v2, r2, a2 = p2
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    if a1 == 'i686' and a2 != 'i686':
Owen W. Taylor 646539e
        return 1
Owen W. Taylor 646539e
    if a1 != 'i686' and a2 == 'i686':
Owen W. Taylor 646539e
        return -1
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    if n1.startswith('compat-') and not n2.startswith('compat-'):
Owen W. Taylor 646539e
        return 1
Owen W. Taylor 646539e
    elif n2.startswith('compat-') and not n1.startswith('compat-'):
Owen W. Taylor 646539e
        return -1
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    if n1 < n2:
Owen W. Taylor 646539e
        return -1
Owen W. Taylor 646539e
    elif n1 > n2:
Owen W. Taylor 646539e
        return 1
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    if e1 is None:
Owen W. Taylor 646539e
        e1 = '0'
Owen W. Taylor 646539e
    if e2 is None:
Owen W. Taylor 646539e
        e2 = '0'
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    return - rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
Owen W. Taylor 646539e
Owen W. Taylor 646539e
class FilesMapHandler(xml.sax.handler.ContentHandler):
Owen W. Taylor 646539e
    def __init__(self, cb):
Owen W. Taylor 646539e
        self.cb = cb
Owen W. Taylor 646539e
        self.package_info = None
Owen W. Taylor 646539e
        self.name = None
Owen W. Taylor 646539e
        self.arch = None
Owen W. Taylor 646539e
        self.epoch = None
Owen W. Taylor 646539e
        self.version = None
Owen W. Taylor 646539e
        self.release = None
Owen W. Taylor 646539e
        self.file = None
Owen W. Taylor 646539e
        self.package_info = None
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    def startElement(self, name, attrs):
Owen W. Taylor 646539e
        if name == 'package':
Owen W. Taylor 646539e
            self.name = attrs['name']
Owen W. Taylor 646539e
            self.arch = attrs['arch']
Owen W. Taylor 646539e
        elif name == 'version':
Owen W. Taylor 646539e
            if self.name is not None:
Owen W. Taylor 646539e
                self.epoch = attrs['epoch']
Owen W. Taylor 646539e
                self.version = attrs['ver']
Owen W. Taylor 646539e
                self.release = attrs['rel']
Owen W. Taylor 646539e
        elif name == 'file':
Owen W. Taylor 646539e
            self.file = ''
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    def endElement(self, name):
Owen W. Taylor 646539e
        if name == 'package':
Owen W. Taylor 646539e
            self.package_info = None
Owen W. Taylor 646539e
        elif name == 'file':
Owen W. Taylor 646539e
            if self.package_info is None:
Owen W. Taylor 646539e
                self.package_info = (self.name, self.epoch, self.version, self.release, self.arch)
Owen W. Taylor 646539e
            self.cb(self.package_info, self.file)
Owen W. Taylor 646539e
            self.file = None
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    def characters(self, content):
Owen W. Taylor 646539e
        if self.file is not None:
Owen W. Taylor 646539e
            self.file += content
Owen W. Taylor 646539e
Owen W. Taylor 646539e
def foreach_file(repo_info, cb):
Owen W. Taylor 646539e
    for repo in REPOS:
Owen W. Taylor 646539e
        start("Scanning files for {}".format(repo))
Owen W. Taylor 646539e
        repo_dir, repomd_contents = repo_info[repo]
Owen W. Taylor 646539e
        root = ET.fromstring(repomd_contents)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
        ns = {'repo': 'http://linux.duke.edu/metadata/repo'}
Owen W. Taylor 646539e
        filelists_location = root.find("./repo:data[@type='filelists']/repo:location", ns).attrib['href']
Owen W. Taylor 646539e
        filelists_path = os.path.join(repo_dir, filelists_location)
Owen W. Taylor 646539e
        if os.path.commonprefix([filelists_path, repo_dir]) != repo_dir:
Owen W. Taylor 646539e
            done()
Owen W. Taylor 646539e
            error("{}: filelists directory is outside of repository".format(repo_dir))
Owen W. Taylor 646539e
Owen W. Taylor 646539e
        handler = FilesMapHandler(cb)
Owen W. Taylor 646539e
        with gzip.open(filelists_path, 'rb') as f:
Owen W. Taylor 646539e
            xml.sax.parse(f, handler)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
        done()
Owen W. Taylor 646539e
Owen W. Taylor c9816cd
class PackageMapHandler(xml.sax.handler.ContentHandler):
Owen W. Taylor c9816cd
    def __init__(self, cb):
Owen W. Taylor c9816cd
        self.cb = cb
Owen W. Taylor c9816cd
        self.name = None
Owen W. Taylor c9816cd
        self.sourcerpm = None
Owen W. Taylor c9816cd
        self.chars = None
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
    def startElement(self, name, attrs):
Owen W. Taylor c9816cd
        if name == 'package':
Owen W. Taylor c9816cd
            pass
Owen W. Taylor c9816cd
        elif name in ('name', 'rpm:sourcerpm'):
Owen W. Taylor c9816cd
            self.chars = ''
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
    def endElement(self, name):
Owen W. Taylor c9816cd
        if name == 'package':
Owen W. Taylor c9816cd
            self.cb(self.name, self.sourcerpm)
Owen W. Taylor c9816cd
            self.name = None
Owen W. Taylor c9816cd
            self.sourcerpm = None
Owen W. Taylor c9816cd
        elif name == 'name':
Owen W. Taylor c9816cd
            self.name = self.chars
Owen W. Taylor c9816cd
            self.chars = None
Owen W. Taylor c9816cd
        elif name == 'rpm:sourcerpm':
Owen W. Taylor c9816cd
            self.sourcerpm = self.chars
Owen W. Taylor c9816cd
            self.chars = None
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
    def characters(self, content):
Owen W. Taylor c9816cd
        if self.chars is not None:
Owen W. Taylor c9816cd
            self.chars += content
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
def foreach_package(repo_info, cb):
Owen W. Taylor c9816cd
    for repo in REPOS:
Owen W. Taylor c9816cd
        start("Scanning files for {}".format(repo))
Owen W. Taylor c9816cd
        repo_dir, repomd_contents = repo_info[repo]
Owen W. Taylor c9816cd
        root = ET.fromstring(repomd_contents)
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
        ns = {'repo': 'http://linux.duke.edu/metadata/repo'}
Owen W. Taylor c9816cd
        filelists_location = root.find("./repo:data[@type='primary']/repo:location", ns).attrib['href']
Owen W. Taylor c9816cd
        filelists_path = os.path.join(repo_dir, filelists_location)
Owen W. Taylor c9816cd
        if os.path.commonprefix([filelists_path, repo_dir]) != repo_dir:
Owen W. Taylor c9816cd
            done()
Owen W. Taylor c9816cd
            error("{}: filelists directory is outside of repository".format(repo_dir))
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
        handler = PackageMapHandler(cb)
Owen W. Taylor c9816cd
        with gzip.open(filelists_path, 'rb') as f:
Owen W. Taylor c9816cd
            xml.sax.parse(f, handler)
Owen W. Taylor c9816cd
Owen W. Taylor c9816cd
        done()
Owen W. Taylor c9816cd
Owen W. Taylor 646539e
def get_repo_cacheable(name, generate):
Owen W. Taylor 646539e
    hash_text = ''
Owen W. Taylor 646539e
    repos_dir = os.path.join(XDG_CACHE_HOME, "fedmod/repos")
Owen W. Taylor 646539e
    repo_info = {}
Owen W. Taylor 646539e
    for repo in REPOS:
Owen W. Taylor 646539e
        repo_dir = os.path.join(repos_dir, repo, 'x86_64')
Owen W. Taylor 646539e
        repomd_path = os.path.join(repo_dir, 'repodata/repomd.xml')
Owen W. Taylor 646539e
        try:
Owen W. Taylor 646539e
            with open(repomd_path, 'rb') as f:
Owen W. Taylor 646539e
                repomd_contents = f.read()
Owen W. Taylor 646539e
        except (OSError, IOError):
Owen W. Taylor adbd27a
            print("Cannot read {}, try 'fedmod --dataset=f29 fetch-metadata'".format(repomd_path), file=sys.stderr)
Owen W. Taylor 646539e
            sys.exit(1)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
        repo_info[repo] = (repo_dir, repomd_contents)
Owen W. Taylor 646539e
        hash_text += '{}|{}\n'.format(repo, hashlib.sha256(repomd_contents).hexdigest())
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    repo_hash = hashlib.sha256(hash_text.encode("UTF-8")).hexdigest()
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    cache_path = os.path.join('out', name + ".gz")
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    try:
Owen W. Taylor 646539e
        with gzip.open(cache_path, 'rb') as f:
Owen W. Taylor 646539e
            old_repo_hash = f.read(64).decode('utf-8')
Owen W. Taylor 646539e
            if old_repo_hash == repo_hash:
Owen W. Taylor 646539e
                start("Reading " + name)
Owen W. Taylor 646539e
                data = pickle.load(f)
Owen W. Taylor 646539e
                done()
Owen W. Taylor 646539e
Owen W. Taylor 646539e
                return data
Owen W. Taylor 646539e
    except FileNotFoundError:
Owen W. Taylor 646539e
        pass
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    data = generate(repo_info)
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    start("Writing " + name)
Owen W. Taylor 646539e
    with gzip.open(cache_path, 'wb') as f:
Owen W. Taylor 646539e
        f.write(repo_hash.encode('utf-8'))
Owen W. Taylor 646539e
        pickle.dump(data, f)
Owen W. Taylor 646539e
    done()
Owen W. Taylor 646539e
Owen W. Taylor 646539e
    return data
Owen W. Taylor 646539e