|
Owen W. Taylor |
646539e |
import gzip
|
|
Owen W. Taylor |
646539e |
import hashlib
|
|
Owen W. Taylor |
646539e |
import pickle
|
|
Owen W. Taylor |
646539e |
import rpm
|
|
Owen W. Taylor |
646539e |
import os
|
|
Owen W. Taylor |
646539e |
import sys
|
|
Owen W. Taylor |
646539e |
import xml.etree.ElementTree as ET
|
|
Owen W. Taylor |
646539e |
import xml.sax
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
# This needs to be in sync with fedmod
|
|
Owen W. Taylor |
646539e |
REPOS = [
|
|
Owen W. Taylor |
adbd27a |
"f29-packages"
|
|
Owen W. Taylor |
646539e |
]
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
_log_name = None
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def set_log_name(name):
|
|
Owen W. Taylor |
646539e |
global _log_name
|
|
Owen W. Taylor |
646539e |
_log_name = name
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def warn(msg):
|
|
Owen W. Taylor |
646539e |
print("{}: \033[31m{}\033[39m".format(_log_name, msg), file=sys.stderr)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def error(msg):
|
|
Owen W. Taylor |
646539e |
print("{}: \033[31m{}\033[39m".format(_log_name, msg), file=sys.stderr)
|
|
Owen W. Taylor |
646539e |
sys.exit(1)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def start(msg):
|
|
Owen W. Taylor |
646539e |
print("{}: \033[90m{} ... \033[39m".format(_log_name, msg), file=sys.stderr, end="")
|
|
Owen W. Taylor |
646539e |
sys.stderr.flush()
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def done():
|
|
Owen W. Taylor |
646539e |
print("\033[90mdone\033[39m", file=sys.stderr)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def package_cmp(p1, p2):
|
|
Owen W. Taylor |
646539e |
n1, e1, v1, r1, a1 = p1
|
|
Owen W. Taylor |
646539e |
n2, e2, v2, r2, a2 = p2
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
if a1 == 'i686' and a2 != 'i686':
|
|
Owen W. Taylor |
646539e |
return 1
|
|
Owen W. Taylor |
646539e |
if a1 != 'i686' and a2 == 'i686':
|
|
Owen W. Taylor |
646539e |
return -1
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
if n1.startswith('compat-') and not n2.startswith('compat-'):
|
|
Owen W. Taylor |
646539e |
return 1
|
|
Owen W. Taylor |
646539e |
elif n2.startswith('compat-') and not n1.startswith('compat-'):
|
|
Owen W. Taylor |
646539e |
return -1
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
if n1 < n2:
|
|
Owen W. Taylor |
646539e |
return -1
|
|
Owen W. Taylor |
646539e |
elif n1 > n2:
|
|
Owen W. Taylor |
646539e |
return 1
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
if e1 is None:
|
|
Owen W. Taylor |
646539e |
e1 = '0'
|
|
Owen W. Taylor |
646539e |
if e2 is None:
|
|
Owen W. Taylor |
646539e |
e2 = '0'
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
return - rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
class FilesMapHandler(xml.sax.handler.ContentHandler):
|
|
Owen W. Taylor |
646539e |
def __init__(self, cb):
|
|
Owen W. Taylor |
646539e |
self.cb = cb
|
|
Owen W. Taylor |
646539e |
self.package_info = None
|
|
Owen W. Taylor |
646539e |
self.name = None
|
|
Owen W. Taylor |
646539e |
self.arch = None
|
|
Owen W. Taylor |
646539e |
self.epoch = None
|
|
Owen W. Taylor |
646539e |
self.version = None
|
|
Owen W. Taylor |
646539e |
self.release = None
|
|
Owen W. Taylor |
646539e |
self.file = None
|
|
Owen W. Taylor |
646539e |
self.package_info = None
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def startElement(self, name, attrs):
|
|
Owen W. Taylor |
646539e |
if name == 'package':
|
|
Owen W. Taylor |
646539e |
self.name = attrs['name']
|
|
Owen W. Taylor |
646539e |
self.arch = attrs['arch']
|
|
Owen W. Taylor |
646539e |
elif name == 'version':
|
|
Owen W. Taylor |
646539e |
if self.name is not None:
|
|
Owen W. Taylor |
646539e |
self.epoch = attrs['epoch']
|
|
Owen W. Taylor |
646539e |
self.version = attrs['ver']
|
|
Owen W. Taylor |
646539e |
self.release = attrs['rel']
|
|
Owen W. Taylor |
646539e |
elif name == 'file':
|
|
Owen W. Taylor |
646539e |
self.file = ''
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def endElement(self, name):
|
|
Owen W. Taylor |
646539e |
if name == 'package':
|
|
Owen W. Taylor |
646539e |
self.package_info = None
|
|
Owen W. Taylor |
646539e |
elif name == 'file':
|
|
Owen W. Taylor |
646539e |
if self.package_info is None:
|
|
Owen W. Taylor |
646539e |
self.package_info = (self.name, self.epoch, self.version, self.release, self.arch)
|
|
Owen W. Taylor |
646539e |
self.cb(self.package_info, self.file)
|
|
Owen W. Taylor |
646539e |
self.file = None
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def characters(self, content):
|
|
Owen W. Taylor |
646539e |
if self.file is not None:
|
|
Owen W. Taylor |
646539e |
self.file += content
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
def foreach_file(repo_info, cb):
|
|
Owen W. Taylor |
646539e |
for repo in REPOS:
|
|
Owen W. Taylor |
646539e |
start("Scanning files for {}".format(repo))
|
|
Owen W. Taylor |
646539e |
repo_dir, repomd_contents = repo_info[repo]
|
|
Owen W. Taylor |
646539e |
root = ET.fromstring(repomd_contents)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
ns = {'repo': 'http://linux.duke.edu/metadata/repo'}
|
|
Owen W. Taylor |
646539e |
filelists_location = root.find("./repo:data[@type='filelists']/repo:location", ns).attrib['href']
|
|
Owen W. Taylor |
646539e |
filelists_path = os.path.join(repo_dir, filelists_location)
|
|
Owen W. Taylor |
646539e |
if os.path.commonprefix([filelists_path, repo_dir]) != repo_dir:
|
|
Owen W. Taylor |
646539e |
done()
|
|
Owen W. Taylor |
646539e |
error("{}: filelists directory is outside of repository".format(repo_dir))
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
handler = FilesMapHandler(cb)
|
|
Owen W. Taylor |
646539e |
with gzip.open(filelists_path, 'rb') as f:
|
|
Owen W. Taylor |
646539e |
xml.sax.parse(f, handler)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
done()
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
c9816cd |
class PackageMapHandler(xml.sax.handler.ContentHandler):
|
|
Owen W. Taylor |
c9816cd |
def __init__(self, cb):
|
|
Owen W. Taylor |
c9816cd |
self.cb = cb
|
|
Owen W. Taylor |
c9816cd |
self.name = None
|
|
Owen W. Taylor |
c9816cd |
self.sourcerpm = None
|
|
Owen W. Taylor |
c9816cd |
self.chars = None
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
def startElement(self, name, attrs):
|
|
Owen W. Taylor |
c9816cd |
if name == 'package':
|
|
Owen W. Taylor |
c9816cd |
pass
|
|
Owen W. Taylor |
c9816cd |
elif name in ('name', 'rpm:sourcerpm'):
|
|
Owen W. Taylor |
c9816cd |
self.chars = ''
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
def endElement(self, name):
|
|
Owen W. Taylor |
c9816cd |
if name == 'package':
|
|
Owen W. Taylor |
c9816cd |
self.cb(self.name, self.sourcerpm)
|
|
Owen W. Taylor |
c9816cd |
self.name = None
|
|
Owen W. Taylor |
c9816cd |
self.sourcerpm = None
|
|
Owen W. Taylor |
c9816cd |
elif name == 'name':
|
|
Owen W. Taylor |
c9816cd |
self.name = self.chars
|
|
Owen W. Taylor |
c9816cd |
self.chars = None
|
|
Owen W. Taylor |
c9816cd |
elif name == 'rpm:sourcerpm':
|
|
Owen W. Taylor |
c9816cd |
self.sourcerpm = self.chars
|
|
Owen W. Taylor |
c9816cd |
self.chars = None
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
def characters(self, content):
|
|
Owen W. Taylor |
c9816cd |
if self.chars is not None:
|
|
Owen W. Taylor |
c9816cd |
self.chars += content
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
def foreach_package(repo_info, cb):
|
|
Owen W. Taylor |
c9816cd |
for repo in REPOS:
|
|
Owen W. Taylor |
c9816cd |
start("Scanning files for {}".format(repo))
|
|
Owen W. Taylor |
c9816cd |
repo_dir, repomd_contents = repo_info[repo]
|
|
Owen W. Taylor |
c9816cd |
root = ET.fromstring(repomd_contents)
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
ns = {'repo': 'http://linux.duke.edu/metadata/repo'}
|
|
Owen W. Taylor |
c9816cd |
filelists_location = root.find("./repo:data[@type='primary']/repo:location", ns).attrib['href']
|
|
Owen W. Taylor |
c9816cd |
filelists_path = os.path.join(repo_dir, filelists_location)
|
|
Owen W. Taylor |
c9816cd |
if os.path.commonprefix([filelists_path, repo_dir]) != repo_dir:
|
|
Owen W. Taylor |
c9816cd |
done()
|
|
Owen W. Taylor |
c9816cd |
error("{}: filelists directory is outside of repository".format(repo_dir))
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
handler = PackageMapHandler(cb)
|
|
Owen W. Taylor |
c9816cd |
with gzip.open(filelists_path, 'rb') as f:
|
|
Owen W. Taylor |
c9816cd |
xml.sax.parse(f, handler)
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
c9816cd |
done()
|
|
Owen W. Taylor |
c9816cd |
|
|
Owen W. Taylor |
646539e |
def get_repo_cacheable(name, generate):
|
|
Owen W. Taylor |
646539e |
hash_text = ''
|
|
Owen W. Taylor |
646539e |
repos_dir = os.path.join(XDG_CACHE_HOME, "fedmod/repos")
|
|
Owen W. Taylor |
646539e |
repo_info = {}
|
|
Owen W. Taylor |
646539e |
for repo in REPOS:
|
|
Owen W. Taylor |
646539e |
repo_dir = os.path.join(repos_dir, repo, 'x86_64')
|
|
Owen W. Taylor |
646539e |
repomd_path = os.path.join(repo_dir, 'repodata/repomd.xml')
|
|
Owen W. Taylor |
646539e |
try:
|
|
Owen W. Taylor |
646539e |
with open(repomd_path, 'rb') as f:
|
|
Owen W. Taylor |
646539e |
repomd_contents = f.read()
|
|
Owen W. Taylor |
646539e |
except (OSError, IOError):
|
|
Owen W. Taylor |
adbd27a |
print("Cannot read {}, try 'fedmod --dataset=f29 fetch-metadata'".format(repomd_path), file=sys.stderr)
|
|
Owen W. Taylor |
646539e |
sys.exit(1)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
repo_info[repo] = (repo_dir, repomd_contents)
|
|
Owen W. Taylor |
646539e |
hash_text += '{}|{}\n'.format(repo, hashlib.sha256(repomd_contents).hexdigest())
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
repo_hash = hashlib.sha256(hash_text.encode("UTF-8")).hexdigest()
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
cache_path = os.path.join('out', name + ".gz")
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
try:
|
|
Owen W. Taylor |
646539e |
with gzip.open(cache_path, 'rb') as f:
|
|
Owen W. Taylor |
646539e |
old_repo_hash = f.read(64).decode('utf-8')
|
|
Owen W. Taylor |
646539e |
if old_repo_hash == repo_hash:
|
|
Owen W. Taylor |
646539e |
start("Reading " + name)
|
|
Owen W. Taylor |
646539e |
data = pickle.load(f)
|
|
Owen W. Taylor |
646539e |
done()
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
return data
|
|
Owen W. Taylor |
646539e |
except FileNotFoundError:
|
|
Owen W. Taylor |
646539e |
pass
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
data = generate(repo_info)
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
start("Writing " + name)
|
|
Owen W. Taylor |
646539e |
with gzip.open(cache_path, 'wb') as f:
|
|
Owen W. Taylor |
646539e |
f.write(repo_hash.encode('utf-8'))
|
|
Owen W. Taylor |
646539e |
pickle.dump(data, f)
|
|
Owen W. Taylor |
646539e |
done()
|
|
Owen W. Taylor |
646539e |
|
|
Owen W. Taylor |
646539e |
return data
|
|
Owen W. Taylor |
646539e |
|