diff --git a/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch b/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch new file mode 100644 index 0000000..9714a79 --- /dev/null +++ b/0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch @@ -0,0 +1,871 @@ +From 3f60e62ea86a3180b45290102d3519347cf8788a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Wed, 11 Apr 2018 09:18:59 +0200 +Subject: [PATCH 1/3] Revert "Move ostree phase and pipelines for running + phases" + +This reverts commit 660c04368ba1abed310f121d01f0fa029eea5f11. +--- + bin/pungi-koji | 52 +++++--- + doc/_static/phases.svg | 339 ++++++++++++++++++++++------------------------- + pungi/phases/__init__.py | 19 ++- + pungi/phases/weaver.py | 72 ---------- + tests/test_phase_base.py | 129 ++++-------------- + 5 files changed, 230 insertions(+), 381 deletions(-) + delete mode 100644 pungi/phases/weaver.py + +diff --git a/bin/pungi-koji b/bin/pungi-koji +index 2a62a125..e5d31084 100755 +--- a/bin/pungi-koji ++++ b/bin/pungi-koji +@@ -358,27 +358,41 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): + # Store the password + compose.conf["signing_key_password"] = signing_key_password + ++ # INIT phase + init_phase.start() + init_phase.stop() + ++ # PKGSET phase + pkgset_phase.start() + pkgset_phase.stop() + +- # WEAVER phase - launches other phases which can safely run in parallel +- essentials_schema = ( +- buildinstall_phase, +- (gather_phase, extrafiles_phase, createrepo_phase), +- (ostree_phase, ostree_installer_phase), +- ) +- essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema) +- essentials_phase.start() +- essentials_phase.stop() ++ # BUILDINSTALL phase - start, we can run gathering, extra files and ++ # createrepo while buildinstall is in progress. ++ buildinstall_phase.start() ++ ++ # If any of the following three phases fail, we must ensure that ++ # buildinstall is stopped. Otherwise the whole process will hang. ++ try: ++ gather_phase.start() ++ gather_phase.stop() ++ ++ extrafiles_phase.start() ++ extrafiles_phase.stop() ++ ++ createrepo_phase.start() ++ createrepo_phase.stop() ++ ++ finally: ++ buildinstall_phase.stop() + + if not buildinstall_phase.skip(): + buildinstall_phase.copy_files() + +- productimg_phase.start() +- productimg_phase.stop() ++ ostree_phase.start() ++ ostree_phase.stop() ++ ++ pungi.phases.run_all([productimg_phase, ++ ostree_installer_phase]) + + # write treeinfo before ISOs are created + for variant in compose.get_variants(): +@@ -394,16 +408,11 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): + pungi.metadata.write_media_repo(compose, arch, variant, timestamp) + + # Start all phases for image artifacts +- compose_images_schema = ( +- createiso_phase, +- liveimages_phase, +- image_build_phase, +- livemedia_phase, +- osbs_phase, +- ) +- compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) +- compose_images_phase.start() +- compose_images_phase.stop() ++ pungi.phases.run_all([createiso_phase, ++ liveimages_phase, ++ image_build_phase, ++ livemedia_phase, ++ osbs_phase]) + + image_checksum_phase.start() + image_checksum_phase.stop() +@@ -412,6 +421,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): + compose.im.dump(compose.paths.compose.metadata("images.json")) + osbs_phase.dump_metadata() + ++ # TEST phase + test_phase.start() + test_phase.stop() + +diff --git a/doc/_static/phases.svg b/doc/_static/phases.svg +index 66184b15..072e5dd2 100644 +--- a/doc/_static/phases.svg ++++ b/doc/_static/phases.svg +@@ -14,7 +14,7 @@ + viewBox="0 0 839.33334 220.33335" + id="svg2" + version="1.1" +- inkscape:version="0.91 r13725" ++ inkscape:version="0.92+devel unknown" + sodipodi:docname="phases.svg" + inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" + inkscape:export-xdpi="90" +@@ -26,16 +26,16 @@ + borderopacity="1.0" + inkscape:pageopacity="1" + inkscape:pageshadow="2" +- inkscape:zoom="1.6532468" +- inkscape:cx="337.4932" +- inkscape:cy="70.825454" ++ inkscape:zoom="0.8266234" ++ inkscape:cx="226.63066" ++ inkscape:cy="201.60264" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="false" + inkscape:window-width="1920" +- inkscape:window-height="1020" +- inkscape:window-x="1920" +- inkscape:window-y="31" ++ inkscape:window-height="1016" ++ inkscape:window-x="0" ++ inkscape:window-y="27" + inkscape:window-maximized="1" + units="px" + inkscape:document-rotation="0" +@@ -67,7 +67,7 @@ + image/svg+xml + +- ++ + + + +@@ -100,7 +100,87 @@ + style="font-size:13.14787769px;line-height:1.25">Pkgset + + ++ ++ Buildinstall ++ ++ Gather ++ ++ ExtraFiles ++ ++ Createrepo ++ ++ + ImageChecksum + + + Test + + ++ transform="translate(2.318656,-80.47309)" ++ id="g3406"> + + Init + + ++ d="M 100.90864,859.8891 H 712.42335" ++ style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.23489845px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" /> + ++ transform="translate(86.469501,49.471116)" ++ id="g3408"> + + Productimg ++ id="tspan3406" ++ style="font-size:13.14787769px;line-height:1.25">OSTree + + ++ id="g251" ++ transform="translate(0,42.1407)"> + ++ id="g241"> + ++ id="rect3350" ++ style="fill:#a40000;fill-rule:evenodd;stroke:none;stroke-width:1.10477591px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + Buildinstall +- +- +- +- Gather +- +- +- +- +- +- ExtraFiles +- +- +- +- Createrepo ++ y="847.65234" ++ x="403.15945" ++ id="tspan3382" ++ sodipodi:role="line">Productimg + + ++ transform="translate(-89.482556,-154.87768)" ++ id="g288"> + ++ id="rect3428" ++ style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> + OSTree +- +- +- +- +- OSTreeInstaller +- ++ x="492.642" ++ sodipodi:role="line" ++ id="tspan283">OSTreeInstaller + + + ++ transform="translate(0,42.1407)"> + +@@ -367,8 +343,7 @@ + y="971.54041" + x="422.99252" + style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" +- xml:space="preserve" +- sodipodi:linespacing="0%"> + +Date: Wed, 11 Apr 2018 09:19:53 +0200 +Subject: [PATCH 2/3] Revert "Other repo for OstreeInstaller" + +This reverts commit 5c081cb545715c2a912ff50fa57554e89d905868. +--- + pungi/checks.py | 3 ++- + pungi/phases/ostree_installer.py | 8 ++----- + pungi/util.py | 21 +++++------------- + tests/test_ostree_installer_phase.py | 43 +++++++++++++++--------------------- + 4 files changed, 28 insertions(+), 47 deletions(-) + +diff --git a/pungi/checks.py b/pungi/checks.py +index 6233d5d7..189ca948 100644 +--- a/pungi/checks.py ++++ b/pungi/checks.py +@@ -938,7 +938,7 @@ def make_schema(): + "tag_ref": {"type": "boolean"}, + "ostree_ref": {"type": "string"}, + }, +- "required": ["treefile", "config_url", "ostree_repo"], ++ "required": ["treefile", "config_url", "repo", "ostree_repo"], + "additionalProperties": False, + }), + ] +@@ -959,6 +959,7 @@ def make_schema(): + "template_repo": {"type": "string"}, + "template_branch": {"type": "string"}, + }, ++ "required": ["repo"], + "additionalProperties": False, + }), + +diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py +index b7843195..9bf5e255 100644 +--- a/pungi/phases/ostree_installer.py ++++ b/pungi/phases/ostree_installer.py +@@ -9,7 +9,7 @@ from kobo import shortcuts + + from .base import ConfigGuardedPhase, PhaseLoggerMixin + from .. import util +-from ..util import get_volid, get_repo_urls, version_generator, translate_path ++from ..util import get_volid, get_repo_urls, version_generator + from ..wrappers import kojiwrapper, iso, lorax, scm + + +@@ -64,11 +64,7 @@ class OstreeInstallerThread(WorkerThread): + self.pool.log_info('[BEGIN] %s' % msg) + self.logdir = compose.paths.log.topdir('%s/%s/ostree_installer-%s' % (arch, variant, self.num)) + +- repo_baseurl = compose.paths.work.arch_repo('$basearch', create_dir=False) +- repos = get_repo_urls(None, # compose==None. Special value says that method should ignore deprecated variant-type repo +- shortcuts.force_list(config['repo']) + shortcuts.force_list(translate_path(compose, repo_baseurl)), +- arch=arch, +- logger=self.pool) ++ repos = get_repo_urls(compose, shortcuts.force_list(config['repo']), arch=arch) + repos = [url.replace('$arch', arch) for url in repos] + output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer') + util.makedirs(os.path.dirname(output_dir)) +diff --git a/pungi/util.py b/pungi/util.py +index 261f3f25..8ea945da 100644 +--- a/pungi/util.py ++++ b/pungi/util.py +@@ -678,8 +678,6 @@ def get_repo_url(compose, repo, arch='$basearch'): + Convert repo to repo URL. + + @param compose - required for access to variants +- special value compose==None determines that method is called during +- OSTreeInstaller phase where variant-type source repository is deprecated + @param repo - string or a dict which at least contains 'baseurl' key + @param arch - string to be used as arch in repo url + """ +@@ -690,17 +688,14 @@ def get_repo_url(compose, repo, arch='$basearch'): + raise RuntimeError('Baseurl is required in repo dict %s' % str(repo)) + if '://' not in repo: + # this is a variant name +- if compose is not None: +- v = compose.all_variants.get(repo) +- if not v: +- raise RuntimeError('There is no variant %s to get repo from.' % repo) +- else: +- return None ++ v = compose.all_variants.get(repo) ++ if not v: ++ raise RuntimeError('There is no variant %s to get repo from.' % repo) + repo = translate_path(compose, compose.paths.compose.repository(arch, v, create_dir=False)) + return repo + + +-def get_repo_urls(compose, repos, arch='$basearch', logger=None): ++def get_repo_urls(compose, repos, arch='$basearch'): + """ + Convert repos to a list of repo URLs. + +@@ -711,11 +706,7 @@ def get_repo_urls(compose, repos, arch='$basearch', logger=None): + urls = [] + for repo in repos: + repo = get_repo_url(compose, repo, arch=arch) +- if repo is None: +- if logger: +- logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'OSTreeInstaller' phase: %s" % (repo)) +- else: +- urls.append(repo) ++ urls.append(repo) + return urls + + +@@ -773,7 +764,7 @@ def get_repo_dicts(repos, logger=None): + repo_dict = get_repo_dict(repo) + if repo_dict == {}: + if logger: +- logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'OSTree' phase: %s" % (repo)) ++ logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'ostree' phase: %s" % (repo)) + else: + repo_dicts.append(repo_dict) + return repo_dicts +diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py +index 1c7af1ad..ba56d8c7 100644 +--- a/tests/test_ostree_installer_phase.py ++++ b/tests/test_ostree_installer_phase.py +@@ -14,7 +14,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + + from tests import helpers + from pungi.phases import ostree_installer as ostree +-from six.moves import shlex_quote + + + LOG_PATH = 'logs/x86_64/Everything/ostree_installer-1' +@@ -59,9 +58,6 @@ class OstreeThreadTest(helpers.PungiTestCase): + 'koji_profile': 'koji', + 'runroot_tag': 'rrt', + 'image_volid_formats': ['{release_short}-{variant}-{arch}'], +- 'translate_paths': [ +- (self.topdir + '/work', 'http://example.com/work') +- ], + }) + + def assertImageAdded(self, compose, ImageCls, iso): +@@ -88,7 +84,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + ] + + for s in force_list(sources): +- lorax_cmd.append(shlex_quote('--source=%s' % s)) ++ lorax_cmd.append('--source=%s' % s) + + lorax_cmd.append('--variant=Everything') + lorax_cmd.append('--nomacboot') +@@ -141,7 +137,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + self.compose.supported = False + pool = mock.Mock() + cfg = { +- 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo ++ 'repo': 'Everything', + 'release': '20160321.n.0', + } + koji = KojiWrapper.return_value +@@ -159,7 +155,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + self.assertRunrootCall(koji, +- 'http://example.com/work/$basearch/repo', ++ 'file://%s/compose/Everything/x86_64/os' % self.topdir, + cfg['release'], + extra=['--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)]) + self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) +@@ -194,11 +190,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + +- self.assertRunrootCall(koji, +- ('http://example.com/repo/x86_64/', +- 'http://example.com/work/$basearch/repo'), +- cfg['release'], +- isfinal=True, ++ self.assertRunrootCall(koji, 'http://example.com/repo/x86_64/', cfg['release'], isfinal=True, + extra=['--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)]) + self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) + self.assertImageAdded(self.compose, ImageCls, iso) +@@ -217,7 +209,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + cfg = { + 'release': '20160321.n.0', + 'repo': [ +- 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo ++ 'Everything', + 'https://example.com/extra-repo1.repo', + 'https://example.com/extra-repo2.repo', + ], +@@ -234,9 +226,9 @@ class OstreeThreadTest(helpers.PungiTestCase): + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + sources = [ ++ 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'https://example.com/extra-repo1.repo', +- 'https://example.com/extra-repo2.repo', +- 'http://example.com/work/$basearch/repo', ++ 'https://example.com/extra-repo2.repo' + ] + + self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True, +@@ -255,8 +247,8 @@ class OstreeThreadTest(helpers.PungiTestCase): + cfg = { + 'release': '20160321.n.0', + 'repo': [ +- 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo +- 'Server', # this variant-type repo is deprecated, in result will be replaced with default repo ++ 'Everything', ++ 'Server', + 'https://example.com/extra-repo1.repo', + 'https://example.com/extra-repo2.repo', + ], +@@ -273,9 +265,10 @@ class OstreeThreadTest(helpers.PungiTestCase): + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + sources = [ ++ 'file://%s/compose/Everything/x86_64/os' % self.topdir, ++ 'file://%s/compose/Server/x86_64/os' % self.topdir, + 'https://example.com/extra-repo1.repo', +- 'https://example.com/extra-repo2.repo', +- 'http://example.com/work/$basearch/repo', ++ 'https://example.com/extra-repo2.repo' + ] + + self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True, +@@ -326,7 +319,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + get_dir_from_scm): + pool = mock.Mock() + cfg = { +- 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo ++ 'repo': 'Everything', + 'release': '20160321.n.0', + 'add_template': ['some_file.txt'], + 'add_arch_template': ['other_file.txt'], +@@ -353,7 +346,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + 'branch': 'f24', 'dir': '.'}, + templ_dir, logger=pool._logger)]) + self.assertRunrootCall(koji, +- 'http://example.com/work/$basearch/repo', ++ 'file://%s/compose/Everything/x86_64/os' % self.topdir, + cfg['release'], + isfinal=True, + extra=['--add-template=%s/some_file.txt' % templ_dir, +@@ -374,7 +367,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + get_file_size, get_mtime, ImageCls, run): + pool = mock.Mock() + cfg = { +- 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo ++ 'repo': 'Everything', + 'release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', + "installpkgs": ["fedora-productimg-atomic"], + "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], +@@ -406,7 +399,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + + self.assertRunrootCall( + koji, +- 'http://example.com/work/$basearch/repo', ++ 'file://%s/compose/Everything/x86_64/os' % self.topdir, + '20151203.t.0', + isfinal=True, + extra=['--installpkgs=fedora-productimg-atomic', +@@ -435,7 +428,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + get_file_size, get_mtime, ImageCls, run): + pool = mock.Mock() + cfg = { +- 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo ++ 'repo': 'Everything', + 'release': None, + "installpkgs": ["fedora-productimg-atomic"], + "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], +@@ -467,7 +460,7 @@ class OstreeThreadTest(helpers.PungiTestCase): + + self.assertRunrootCall( + koji, +- 'http://example.com/work/$basearch/repo', ++ 'file://%s/compose/Everything/x86_64/os' % self.topdir, + '20151203.t.0', + isfinal=True, + extra=['--installpkgs=fedora-productimg-atomic', +-- +2.13.6 + diff --git a/0003-Revert-Ostree-can-use-pkgset-repos.patch b/0003-Revert-Ostree-can-use-pkgset-repos.patch new file mode 100644 index 0000000..3e8e08e --- /dev/null +++ b/0003-Revert-Ostree-can-use-pkgset-repos.patch @@ -0,0 +1,272 @@ +From 8a7bf97434cbbf2397d3209498eacc326fc130f2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Wed, 11 Apr 2018 09:20:51 +0200 +Subject: [PATCH 3/3] Revert "Ostree can use pkgset repos" + +This reverts commit c7cc200246300c6a3946b2e3a9f5f7693896a7d6. +--- + pungi/phases/ostree.py | 3 +-- + pungi/util.py | 35 +++++++++++++++++------------- + tests/test_config_validate_script.py | 4 +++- + tests/test_ostree_phase.py | 41 ++++++++---------------------------- + tests/test_util.py | 26 +++++++++++++++-------- + 5 files changed, 50 insertions(+), 59 deletions(-) + +diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py +index 787bf07a..0550b18c 100644 +--- a/pungi/phases/ostree.py ++++ b/pungi/phases/ostree.py +@@ -58,8 +58,7 @@ class OSTreeThread(WorkerThread): + repodir = os.path.join(workdir, 'config_repo') + self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) + +- repo_baseurl = compose.paths.work.arch_repo('$basearch', create_dir=False) +- repos = get_repo_dicts(shortcuts.force_list(config['repo']) + shortcuts.force_list(translate_path(compose, repo_baseurl)), logger=self.pool) ++ repos = get_repo_dicts(compose, shortcuts.force_list(config['repo'])) + + # copy the original config and update before save to a json file + new_config = copy.copy(config) +diff --git a/pungi/util.py b/pungi/util.py +index 8ea945da..b095598c 100644 +--- a/pungi/util.py ++++ b/pungi/util.py +@@ -718,16 +718,19 @@ def _translate_url_to_repo_id(url): + return ''.join([s if s in list(_REPOID_CHARS) else '_' for s in url]) + + +-def get_repo_dict(repo): ++def get_repo_dict(compose, repo, arch='$basearch'): + """ + Convert repo to a dict of repo options. + +- If repo is a string that represents url, set it as 'baseurl' in result dict, +- also generate a repo id/name as 'name' key in result dict. +- If repo is a dict, and if 'name' key is missing in the dict, generate one for it. +- Repo (str or dict) that has not url format is no longer processed. ++ If repo is a string, translate it to repo url if necessary (when it's ++ not a url), and set it as 'baseurl' in result dict, also generate ++ a repo id/name as 'name' key in result dict. ++ If repo is a dict, translate value of 'baseurl' key to url if necessary, ++ if 'name' key is missing in the dict, generate one for it. + ++ @param compose - required for access to variants + @param repo - A string or dict, if it is a dict, key 'baseurl' is required ++ @param arch - string to be used as arch in repo url + """ + repo_dict = {} + if isinstance(repo, dict): +@@ -737,8 +740,10 @@ def get_repo_dict(repo): + if name is None: + name = _translate_url_to_repo_id(url) + else: +- # url is variant uid - this possibility is now discontinued +- return {} ++ # url is variant uid ++ if name is None: ++ name = '%s-%s' % (compose.compose_id, url) ++ url = get_repo_url(compose, url, arch=arch) + repo['name'] = name + repo['baseurl'] = url + return repo +@@ -749,24 +754,24 @@ def get_repo_dict(repo): + repo_dict['name'] = _translate_url_to_repo_id(repo) + repo_dict['baseurl'] = repo + else: +- return {} ++ repo_dict['name'] = '%s-%s' % (compose.compose_id, repo) ++ repo_dict['baseurl'] = get_repo_url(compose, repo) ++ + return repo_dict + + +-def get_repo_dicts(repos, logger=None): ++def get_repo_dicts(compose, repos, arch='$basearch'): + """ + Convert repos to a list of repo dicts. + ++ @param compose - required for access to variants + @param repo - A list of string or dict, if item is a dict, key 'baseurl' is required ++ @param arch - string to be used as arch in repo url + """ + repo_dicts = [] + for repo in repos: +- repo_dict = get_repo_dict(repo) +- if repo_dict == {}: +- if logger: +- logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'ostree' phase: %s" % (repo)) +- else: +- repo_dicts.append(repo_dict) ++ repo_dict = get_repo_dict(compose, repo, arch=arch) ++ repo_dicts.append(repo_dict) + return repo_dicts + + +diff --git a/tests/test_config_validate_script.py b/tests/test_config_validate_script.py +index 031a15a4..c73fe126 100644 +--- a/tests/test_config_validate_script.py ++++ b/tests/test_config_validate_script.py +@@ -1,6 +1,7 @@ + # -*- coding: utf-8 -*- + + ++import mock + import os + import subprocess + import sys +@@ -19,7 +20,8 @@ from tests import helpers + + class ConfigValidateScriptTest(helpers.PungiTestCase): + +- def test_validate_dummy_config(self): ++ @mock.patch('kobo.shortcuts.run') ++ def test_validate_dummy_config(self, run): + DUMMY_CONFIG = os.path.join(HERE, 'data/dummy-pungi.conf') + interp = 'python2' if six.PY2 else 'python3' + p = subprocess.Popen([interp, PUNGI_CONFIG_VALIDATE, DUMMY_CONFIG], +diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py +index d57dc682..3a9ebf32 100644 +--- a/tests/test_ostree_phase.py ++++ b/tests/test_ostree_phase.py +@@ -118,7 +118,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): + 'koji_profile': 'koji', + 'runroot_tag': 'rrt', + 'translate_paths': [ +- (self.topdir, 'http://example.com') ++ (self.topdir + '/compose', 'http://example.com') + ] + }) + self.pool = mock.Mock() +@@ -150,28 +150,6 @@ class OSTreeThreadTest(helpers.PungiTestCase): + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') +- def test_extra_config_content(self, KojiWrapper, get_dir_from_scm): +- get_dir_from_scm.side_effect = self._dummy_config_repo +- self.compose.conf['runroot_weights'] = {'ostree': 123} +- +- koji = KojiWrapper.return_value +- koji.run_runroot_cmd.side_effect = self._mock_runroot(0) +- +- t = ostree.OSTreeThread(self.pool) +- +- extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json') +- self.assertFalse(os.path.isfile(extra_config_file)) +- +- t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1) +- +- self.assertTrue(os.path.isfile(extra_config_file)) +- with open(extra_config_file, 'r') as f: +- extraconf_content = json.load(f) +- proper_extraconf_content = json.loads('{"repo": [{"name": "http:__example.com_work__basearch_repo", "baseurl": "http://example.com/work/$basearch/repo"}]}') +- self.assertEqual(proper_extraconf_content, extraconf_content) +- +- @mock.patch('pungi.wrappers.scm.get_dir_from_scm') +- @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + self.compose.conf['runroot_weights'] = {'ostree': 123} +@@ -291,7 +269,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): + arch='x86_64', + ref='fedora-atomic/25/x86_64', + commitid=None, +- repo_path='http://example.com/place/for/atomic', ++ repo_path=self.repo, + local_repo_path=self.repo)]) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') +@@ -424,14 +402,14 @@ class OSTreeThreadTest(helpers.PungiTestCase): + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + cfg = { +- 'repo': [ # Variant type repos will not be included into extra_config. This part of the config is deprecated +- 'Everything', # do not include ++ 'repo': [ ++ 'Everything', + { + 'name': 'repo_a', + 'baseurl': 'http://url/to/repo/a', + 'exclude': 'systemd-container' + }, +- { # do not include ++ { + 'name': 'Server', + 'baseurl': 'Server', + 'exclude': 'systemd-container' +@@ -450,13 +428,12 @@ class OSTreeThreadTest(helpers.PungiTestCase): + + extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json') + self.assertTrue(os.path.isfile(extra_config_file)) +- with open(extra_config_file, 'r') as extra_config_fd: +- extra_config = json.load(extra_config_fd) ++ extra_config = json.load(open(extra_config_file, 'r')) + self.assertTrue(extra_config.get('keep_original_sources', False)) +- self.assertEqual(len(extra_config.get('repo', [])), 2) # should equal to number of valid repositories in cfg['repo'] + default repository +- self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/work/$basearch/repo') ++ self.assertEqual(len(extra_config.get('repo', [])), len(cfg['repo'])) ++ self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/Server/$basearch/os') + self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://url/to/repo/a') +- ++ self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/Everything/$basearch/os') + + if __name__ == '__main__': + unittest.main() +diff --git a/tests/test_util.py b/tests/test_util.py +index 2b4c2d2c..b4c85a57 100644 +--- a/tests/test_util.py ++++ b/tests/test_util.py +@@ -588,35 +588,43 @@ class GetRepoFuncsTestCase(unittest.TestCase): + self.assertEqual(util.get_repo_urls(self.compose, repos), expect) + + def test_get_repo_dict_from_normal_url(self): +- repo_dict = util.get_repo_dict('http://example.com/repo') ++ repo_dict = util.get_repo_dict(self.compose, 'http://example.com/repo') + expect = {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'} + self.assertEqual(repo_dict, expect) + + def test_get_repo_dict_from_variant_uid(self): +- repo_dict = util.get_repo_dict('Server') # this repo format is deprecated +- expect = {} ++ repo_dict = util.get_repo_dict(self.compose, 'Server') ++ expect = { ++ 'name': "%s-%s" % (self.compose.compose_id, 'Server'), ++ 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os', ++ } + self.assertEqual(repo_dict, expect) + + def test_get_repo_dict_from_repo_dict(self): +- repo = {'baseurl': 'Server'} # this repo format is deprecated +- expect = {} +- repo_dict = util.get_repo_dict(repo) ++ repo = {'baseurl': 'Server'} ++ expect = { ++ 'name': '%s-%s' % (self.compose.compose_id, 'Server'), ++ 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os' ++ } ++ repo_dict = util.get_repo_dict(self.compose, repo) + self.assertEqual(repo_dict, expect) + + def test_get_repo_dicts(self): + repos = [ + 'http://example.com/repo', +- 'Server', # this repo format is deprecated (and will not be included into final repo_dict) +- {'baseurl': 'Client'}, # this repo format is deprecated ++ 'Server', ++ {'baseurl': 'Client'}, + {'baseurl': 'ftp://example.com/linux/repo'}, + {'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'}, + ] + expect = [ + {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'}, ++ {'name': '%s-%s' % (self.compose.compose_id, 'Server'), 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os'}, ++ {'name': '%s-%s' % (self.compose.compose_id, 'Client'), 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Client/$basearch/os'}, + {'name': 'ftp:__example.com_linux_repo', 'baseurl': 'ftp://example.com/linux/repo'}, + {'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'}, + ] +- repos = util.get_repo_dicts(repos) ++ repos = util.get_repo_dicts(self.compose, repos) + self.assertEqual(repos, expect) + + +-- +2.13.6 + diff --git a/pungi.spec b/pungi.spec index 6ce9413..108d4fc 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,6 +1,6 @@ Name: pungi Version: 4.1.23 -Release: 1%{?dist} +Release: 2%{?dist} Summary: Distribution compose tool Group: Development/Tools @@ -8,6 +8,9 @@ License: GPLv2 URL: https://pagure.io/pungi Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 Patch0: 0001-tests-Use-dummy-modulesdir-for-DNF.patch +Patch1: 0001-Revert-Move-ostree-phase-and-pipelines-for-running-p.patch +Patch2: 0002-Revert-Other-repo-for-OstreeInstaller.patch +Patch3: 0003-Revert-Ostree-can-use-pkgset-repos.patch BuildRequires: python3-nose BuildRequires: python3-mock BuildRequires: python2-devel @@ -172,6 +175,9 @@ nosetests-3 --exe %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Wed Apr 11 2018 Lubomír Sedlář - 4.1.23-2 +- Revert reordering of ostree phases + * Wed Apr 4 2018 Lubomír Sedlář - 4.1.23-1 - Update documentation section 'contributing' (onosek) - Write module metadata (onosek)