diff --git a/0001-ostree-installer-Clone-repo-with-templates.patch b/0001-ostree-installer-Clone-repo-with-templates.patch new file mode 100644 index 0000000..4604be9 --- /dev/null +++ b/0001-ostree-installer-Clone-repo-with-templates.patch @@ -0,0 +1,260 @@ +From 9b0edaafe902a0885d2e9d35986784053269d8d1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Mon, 11 Apr 2016 10:25:10 +0200 +Subject: [PATCH 1/2] [ostree-installer] Clone repo with templates +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +We can't assume the templates will just be available. This patch adds a +configuration option to point to the git repo with them. It will be +cloned at compose box and relative paths to templates will be made +absolute respective to this clone. + +Signed-off-by: Lubomír Sedlář +--- + doc/configuration.rst | 13 +++- + pungi/phases/ostree_installer.py | 30 ++++++++- + tests/test_ostree_installer_phase.py | 117 +++++++++++++++++++++++++++++++++++ + 3 files changed, 155 insertions(+), 5 deletions(-) + +diff --git a/doc/configuration.rst b/doc/configuration.rst +index ff26b44..94704ae 100644 +--- a/doc/configuration.rst ++++ b/doc/configuration.rst +@@ -1005,6 +1005,13 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. + * ``add_arch_template`` -- (*[str]*) + * ``add_template_var`` -- (*[str]*) + * ``add_arch_template_var`` -- (*[str]*) ++ * ``template_repo`` -- (*str*) Git repository with extra templates. ++ * ``template_branch`` -- (*str*) Branch to use from ``template_repo``. ++ ++ The templates can either be absolute paths, in which case they will be used ++ as configured; or they can be relative paths, in which case ++ ``template_repo`` needs to point to a Git repository from which to take the ++ templates. + + + Example config +@@ -1018,17 +1025,19 @@ Example config + "release": None, + "filename": "%(release_short)s-%(variant)s-%(arch)s-%(version)s-%(compose_date)s.iso", + "installpkgs": ["fedora-productimg-atomic"], +- "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], ++ "add_template": ["atomic-installer/lorax-configure-repo.tmpl"], + "add_template_var": [ + "ostree_osname=fedora-atomic", + "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", + ], +- "add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"], ++ "add_arch_template": ["atomic-installer/lorax-embed-repo.tmpl"], + "add_arch_template_var": [ + "ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/", + "ostree_osname=fedora-atomic", + "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", + ] ++ 'template_repo': 'https://git.fedorahosted.org/git/spin-kickstarts.git', ++ 'template_branch': 'f24', + } + }) + ] +diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py +index c73bbea..7399874 100644 +--- a/pungi/phases/ostree_installer.py ++++ b/pungi/phases/ostree_installer.py +@@ -10,7 +10,7 @@ from kobo import shortcuts + from .base import ConfigGuardedPhase + from .. import util + from ..paths import translate_path +-from ..wrappers import kojiwrapper, iso, lorax ++from ..wrappers import kojiwrapper, iso, lorax, scm + + + class OstreeInstallerPhase(ConfigGuardedPhase): +@@ -56,6 +56,9 @@ class OstreeInstallerThread(WorkerThread): + output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer') + util.makedirs(os.path.dirname(output_dir)) + ++ self.template_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'lorax_templates') ++ self._clone_templates(config.get('template_repo'), config.get('template_branch')) ++ + self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir) + + disc_type = compose.conf.get('disc_types', {}).get('dvd', 'dvd') +@@ -65,6 +68,13 @@ class OstreeInstallerThread(WorkerThread): + self._add_to_manifest(compose, variant, arch, filename) + self.pool.log_info('[DONE ] %s' % msg) + ++ def _clone_templates(self, url, branch='master'): ++ if not url: ++ self.template_dir = None ++ return ++ scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, ++ self.template_dir, logger=self.pool._logger) ++ + def _get_release(self, compose, config): + if 'release' in config and config['release'] is None: + return compose.image_release +@@ -105,6 +115,20 @@ class OstreeInstallerThread(WorkerThread): + pass + compose.im.add(variant.uid, arch, img) + ++ def _get_templates(self, config, key): ++ """Retrieve all templates from configuration and make sure the paths ++ are absolute. Raises RuntimeError if template repo is needed but not ++ configured. ++ """ ++ templates = [] ++ for template in config.get(key, []): ++ if template[0] != '/': ++ if not self.template_dir: ++ raise RuntimeError('Relative path to template without setting template_repo.') ++ template = os.path.join(self.template_dir, template) ++ templates.append(template) ++ return templates ++ + def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir): + lorax_wrapper = lorax.LoraxWrapper() + cmd = lorax_wrapper.get_lorax_cmd( +@@ -116,8 +140,8 @@ class OstreeInstallerThread(WorkerThread): + variant=variant.uid, + nomacboot=True, + buildinstallpackages=config.get('installpkgs'), +- add_template=config.get('add_template'), +- add_arch_template=config.get('add_arch_template'), ++ add_template=self._get_templates(config, 'add_template'), ++ add_arch_template=self._get_templates(config, 'add_arch_template'), + add_template_var=config.get('add_template_var'), + add_arch_template_var=config.get('add_arch_template_var') + ) +diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py +index 6a8205c..7007a3f 100644 +--- a/tests/test_ostree_installer_phase.py ++++ b/tests/test_ostree_installer_phase.py +@@ -172,6 +172,123 @@ class OstreeThreadTest(helpers.PungiTestCase): + @mock.patch('pungi.wrappers.iso.IsoWrapper') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') ++ def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link, ++ IsoWrapper, get_file_size, ++ get_mtime, ImageCls, run): ++ compose = helpers.DummyCompose(self.topdir, { ++ 'release_name': 'Fedora', ++ 'release_version': 'Rawhide', ++ 'koji_profile': 'koji', ++ 'runroot_tag': 'rrt', ++ }) ++ pool = mock.Mock() ++ cfg = { ++ 'source_repo_from': 'Everything', ++ 'release': '20160321.n.0', ++ 'filename': 'Fedora-Atomic.iso', ++ 'add_template': ['some-file.txt'], ++ } ++ koji = KojiWrapper.return_value ++ koji.run_runroot_cmd.return_value = { ++ 'task_id': 1234, ++ 'retcode': 0, ++ 'output': 'Foo bar\n', ++ } ++ get_file_size.return_value = 1024 ++ get_mtime.return_value = 13579 ++ ++ t = ostree.OstreeInstallerThread(pool) ++ ++ with self.assertRaises(RuntimeError) as ctx: ++ t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) ++ ++ self.assertIn('template_repo', str(ctx.exception)) ++ ++ @mock.patch('pungi.wrappers.scm.get_dir_from_scm') ++ @mock.patch('kobo.shortcuts.run') ++ @mock.patch('productmd.images.Image') ++ @mock.patch('pungi.util.get_mtime') ++ @mock.patch('pungi.util.get_file_size') ++ @mock.patch('pungi.wrappers.iso.IsoWrapper') ++ @mock.patch('os.link') ++ @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') ++ def test_run_clone_templates(self, KojiWrapper, link, IsoWrapper, ++ get_file_size, get_mtime, ImageCls, run, ++ get_dir_from_scm): ++ compose = helpers.DummyCompose(self.topdir, { ++ 'release_name': 'Fedora', ++ 'release_version': 'Rawhide', ++ 'koji_profile': 'koji', ++ 'runroot_tag': 'rrt', ++ }) ++ pool = mock.Mock() ++ cfg = { ++ 'source_repo_from': 'Everything', ++ 'release': '20160321.n.0', ++ 'filename': 'Fedora-Atomic.iso', ++ 'add_template': ['some_file.txt'], ++ 'add_arch_template': ['other_file.txt'], ++ 'template_repo': 'git://example.com/templates.git', ++ 'template_branch': 'f24', ++ } ++ koji = KojiWrapper.return_value ++ koji.run_runroot_cmd.return_value = { ++ 'task_id': 1234, ++ 'retcode': 0, ++ 'output': 'Foo bar\n', ++ } ++ get_file_size.return_value = 1024 ++ get_mtime.return_value = 13579 ++ final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name' ++ templ_dir = self.topdir + '/work/x86_64/Everything/lorax_templates' ++ ++ t = ostree.OstreeInstallerThread(pool) ++ ++ t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) ++ ++ self.assertEqual(get_dir_from_scm.call_args_list, ++ [mock.call({'scm': 'git', 'repo': 'git://example.com/templates.git', ++ 'branch': 'f24', 'dir': '.'}, ++ templ_dir, logger=pool._logger)]) ++ self.assertEqual(koji.get_runroot_cmd.call_args_list, ++ [mock.call('rrt', 'x86_64', ++ ['lorax', ++ '--product=Fedora', ++ '--version=Rawhide', ++ '--release=20160321.n.0', ++ '--source=file://{}/compose/Everything/x86_64/os'.format(self.topdir), ++ '--variant=Everything', ++ '--nomacboot', ++ '--add-template={}/some_file.txt'.format(templ_dir), ++ '--add-arch-template={}/other_file.txt'.format(templ_dir), ++ self.topdir + '/work/x86_64/Everything/ostree_installer'], ++ channel=None, mounts=[self.topdir], ++ packages=['pungi', 'lorax'], ++ task_id=True, use_shell=True)]) ++ self.assertEqual(koji.run_runroot_cmd.call_args_list, ++ [mock.call(koji.get_runroot_cmd.return_value, ++ log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')]) ++ self.assertEqual(link.call_args_list, ++ [mock.call(self.topdir + '/work/x86_64/Everything/ostree_installer/images/boot.iso', ++ final_iso_path)]) ++ self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)]) ++ self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)]) ++ self.assertImageAdded(compose, ImageCls, IsoWrapper) ++ self.assertEqual(compose.get_image_name.call_args_list, ++ [mock.call('x86_64', compose.variants['Everything'], ++ disc_type='dvd', format='Fedora-Atomic.iso')]) ++ self.assertTrue(os.path.isdir(self.topdir + '/work/x86_64/Everything/')) ++ self.assertFalse(os.path.isdir(self.topdir + '/work/x86_64/Everything/ostree_installer')) ++ self.assertEqual(run.call_args_list, ++ [mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/iso/'.format(self.topdir))]) ++ ++ @mock.patch('kobo.shortcuts.run') ++ @mock.patch('productmd.images.Image') ++ @mock.patch('pungi.util.get_mtime') ++ @mock.patch('pungi.util.get_file_size') ++ @mock.patch('pungi.wrappers.iso.IsoWrapper') ++ @mock.patch('os.link') ++ @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_implicit_release(self, KojiWrapper, link, IsoWrapper, + get_file_size, get_mtime, ImageCls, run): + compose = helpers.DummyCompose(self.topdir, { +-- +2.7.3 + diff --git a/0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch b/0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch new file mode 100644 index 0000000..0d5f1e7 --- /dev/null +++ b/0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch @@ -0,0 +1,56 @@ +From bdc309f20677d719b7e5b972ff4a44fc10634f1f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= +Date: Mon, 11 Apr 2016 15:34:26 +0200 +Subject: [PATCH 2/2] [pkgset] Print more detailed logs when rpm is not found +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fixes: #265 +Signed-off-by: Lubomír Sedlář +--- + pungi/phases/pkgset/pkgsets.py | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py +index 834fe31..adc2d07 100644 +--- a/pungi/phases/pkgset/pkgsets.py ++++ b/pungi/phases/pkgset/pkgsets.py +@@ -225,12 +225,14 @@ class KojiPackageSet(PackageSetBase): + rpm_path = None + found = False + pathinfo = self.koji_wrapper.koji_module.pathinfo ++ paths = [] + for sigkey in self.sigkey_ordering: + if sigkey is None: + # we're looking for *signed* copies here + continue + sigkey = sigkey.lower() + rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey)) ++ paths.append(rpm_path) + if os.path.isfile(rpm_path): + found = True + break +@@ -239,14 +241,17 @@ class KojiPackageSet(PackageSetBase): + if None in self.sigkey_ordering: + # use an unsigned copy (if allowed) + rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info)) ++ paths.append(rpm_path) + if os.path.isfile(rpm_path): + found = True + else: + # or raise an exception +- raise RuntimeError("RPM %s not found for sigs: %s" % (rpm_info, self.sigkey_ordering)) ++ raise RuntimeError("RPM %s not found for sigs: %s. Paths checked: %s" ++ % (rpm_info, self.sigkey_ordering, paths)) + + if not found: +- raise RuntimeError("Package not found: %s" % rpm_info) ++ raise RuntimeError("Package not found: %s. Paths checked: %s" ++ % (rpm_info, paths)) + return rpm_path + + def populate(self, tag, event=None, inherit=True): +-- +2.7.3 + diff --git a/pungi.spec b/pungi.spec index 9c0fdcf..77f2fdd 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,12 +1,14 @@ Name: pungi Version: 4.0.14 -Release: 1%{?dist} +Release: 2%{?dist} Summary: Distribution compose tool Group: Development/Tools License: GPLv2 URL: https://pagure.io/pungi Source0: https://fedorahosted.org/pungi/attachment/wiki/%{version}/%{name}-%{version}.tar.bz2 +Patch0: 0001-ostree-installer-Clone-repo-with-templates.patch +Patch1: 0002-pkgset-Print-more-detailed-logs-when-rpm-is-not-foun.patch BuildRequires: python-nose, python-nose-cov, python-mock @@ -49,6 +51,8 @@ A tool to create anaconda based installation trees/isos of a set of rpms. %prep %setup -q +%patch0 -p1 +%patch1 -p1 %build %{__python} setup.py build @@ -83,6 +87,10 @@ cd tests && ./test_compose.sh /var/cache/pungi %changelog +* Mon Apr 11 2016 Dennis Gilmore - 4.0.14-2 +- add patch to print more infor for missing rpms +- add patch to clone repo with extra lorax templates for ostree_installer + * Fri Apr 08 2016 Dennis Gilmore - 4.0.14-1 - [ostree-installer] Copy all lorax outputs (lsedlar) - [ostree] Log to stdout as well (lsedlar)