From c4078c160caad8c019c4e890757fbde34aa83f60 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Dec 08 2017 13:30:51 +0000 Subject: If pungi_runroot_enabled is true, use koji runroot also to remove expired composes. --- diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index c7c77b9..539b23d 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -143,6 +143,33 @@ class RemoveExpiredComposesThread(BackendThread): else: shutil.rmtree(toplevel_dir) + def _get_remove_compose_dir_cmds(self, toplevel_dir): + """ + Returns the commands needed to remove the compose with toplevel_dir + compose dir as list. + """ + + # Be nice and don't fail when directory does not exist. + if not os.path.exists(toplevel_dir): + log.warn("Cannot remove directory %s, it does not exist", + toplevel_dir) + return [] + + cmds = [] + + # If toplevel_dir is a symlink, remove the symlink and + # its target. If toplevel_dir is normal directory, just + # remove it using rmtree. + if os.path.realpath(toplevel_dir) != toplevel_dir: + targetpath = os.path.realpath(toplevel_dir) + cmds += ["rm", "-f", toplevel_dir] + if os.path.exists(targetpath): + cmds += ["&&", "rm", "-rf", targetpath] + else: + cmds += ["rm", "-rf", toplevel_dir] + + return cmds + def _get_compose_id_from_path(self, path): """ Returns the ID of compose from directory path in conf.target_dir. @@ -163,6 +190,15 @@ class RemoveExpiredComposesThread(BackendThread): """ log.info("Checking for expired composes") + # We store all the compose directories to remove in this list and + # removes them all in the end of this method. + # The reason to do it this way is that if "conf.pungi_runroot_enabled" + # is True, we spawn Koji runroot task which calls "rm" commands for + # every compose directory we want to remove. We need a list + # of directories to be able to generate that set of rm commands later + # if neede and if not, just remove the directories by this thread. + compose_dirs_to_remove = [] + composes = Compose.composes_to_expire() for compose in composes: log.info("%r: Removing compose", compose) @@ -170,7 +206,7 @@ class RemoveExpiredComposesThread(BackendThread): compose.time_removed = datetime.utcnow() db.session.commit() if not compose.reused_id: - self._remove_compose_dir(compose.toplevel_dir) + compose_dirs_to_remove.append(str(compose.toplevel_dir)) # In case of ODCS error, there might be left-over directories # belonging to already expired composes. Try to find them in the @@ -197,16 +233,34 @@ class RemoveExpiredComposesThread(BackendThread): if not composes: log.info("Removing data of compose %d - it is not in " "database: %s", compose_id, path) - self._remove_compose_dir(path) + compose_dirs_to_remove.append(path) continue compose = composes[0] if compose.state == COMPOSE_STATES["removed"]: log.info("%r: Removing data of compose - it has already " "expired some time ago: %s", compose_id, path) - self._remove_compose_dir(path) + compose_dirs_to_remove.append(path) continue + koji_cmds = [] + for compose_dir in compose_dirs_to_remove: + if conf.pungi_runroot_enabled: + cmds = self._get_remove_compose_dir_cmds(compose_dir) + if cmds: + koji_cmds += cmds + koji_cmds += ["&&"] + else: + self._remove_compose_dir(compose_dir) + + # In case we have some commands to run in Koji, spawn the Koji runroot + # task and wait for result. + if koji_cmds: + # Remove last "&&" + koji_cmds = koji_cmds[:-1] + koji_session = odcs.server.utils.make_koji_session() + odcs.server.utils.run_koji_runroot(koji_session, koji_cmds) + def create_koji_session(): """ diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py index aab57e4..9dbd1d3 100644 --- a/server/odcs/server/pungi.py +++ b/server/odcs/server/pungi.py @@ -25,8 +25,6 @@ import os import shutil import tempfile import jinja2 -import koji -import munch import time import random import string @@ -35,7 +33,8 @@ import odcs.server.utils from odcs.server import conf, log from odcs.server import comps from odcs.common.types import PungiSourceType, COMPOSE_RESULTS -from odcs.server.utils import makedirs, download_file +from odcs.server.utils import ( + makedirs, download_file, make_koji_session, run_koji_runroot) class PungiConfig(object): @@ -185,47 +184,6 @@ class Pungi(object): output_path = os.path.join(topdir, "pungi.conf") download_file(self.pungi_cfg, output_path) - def make_koji_session(self): - """ - Creates new KojiSession according to odcs.server.conf, logins to - Koji using this session and returns it. - :rtype: koji.KojiSession - :return: KojiSession - """ - koji_config = munch.Munch(koji.read_config( - profile_name=conf.koji_profile, - user_config=conf.koji_config, - )) - - address = koji_config.server - authtype = koji_config.authtype - log.info("Connecting to koji %r with %r." % (address, authtype)) - koji_session = koji.ClientSession(address, opts=koji_config) - if authtype == "kerberos": - ccache = getattr(conf, "krb_ccache", None) - keytab = getattr(conf, "krb_keytab", None) - principal = getattr(conf, "krb_principal", None) - log.debug(" ccache: %r, keytab: %r, principal: %r" % ( - ccache, keytab, principal)) - if keytab and principal: - koji_session.krb_login( - principal=principal, - keytab=keytab, - ccache=ccache, - ) - else: - koji_session.krb_login(ccache=ccache) - elif authtype == "ssl": - koji_session.ssl_login( - os.path.expanduser(koji_config.cert), - None, - os.path.expanduser(koji_config.serverca), - ) - else: - raise ValueError("Unrecognized koji authtype %r" % authtype) - - return koji_session - def get_pungi_cmd(self, conf_topdir, targetdir): """ Returns list with pungi command line arguments needed to generate @@ -305,7 +263,7 @@ class Pungi(object): makedirs(conf_topdir) self._write_cfgs(conf_topdir) - koji_session = self.make_koji_session() + koji_session = make_koji_session() serverdir = self.upload_files_to_koji(koji_session, conf_topdir) # TODO: Copy keytab from secret repo and generate koji profile. @@ -313,30 +271,7 @@ class Pungi(object): cmd += ["cp", "/mnt/koji/work/%s/*" % serverdir, ".", "&&"] cmd += self.get_pungi_cmd("./", conf.pungi_runroot_target_dir) - kwargs = { - 'channel': conf.pungi_parent_runroot_channel, - 'packages': conf.pungi_parent_runroot_packages, - 'mounts': conf.pungi_parent_runroot_mounts, - 'weight': conf.pungi_parent_runroot_weight - } - - task_id = koji_session.runroot( - conf.pungi_parent_runroot_tag, conf.pungi_parent_runroot_arch, - " ".join(cmd), **kwargs) - - while True: - # wait for the task to finish - if koji_session.taskFinished(task_id): - break - log.info("Waiting for Koji runroot task %r to finish...", task_id) - time.sleep(60) - - info = koji_session.getTaskInfo(task_id) - if info is None: - raise RuntimeError("Cannot get status of Koji task %r" % task_id) - state = koji.TASK_STATES[info['state']] - if state in ('FAILED', 'CANCELED'): - raise RuntimeError("Koji runroot task %r failed." % task_id) + run_koji_runroot(koji_session, cmd) def run(self): """ diff --git a/server/odcs/server/utils.py b/server/odcs/server/utils.py index 63c3574..20e2e50 100644 --- a/server/odcs/server/utils.py +++ b/server/odcs/server/utils.py @@ -26,11 +26,86 @@ import os import time import subprocess import requests +import munch +import koji from distutils.spawn import find_executable from odcs.server import conf, log +def make_koji_session(): + """ + Creates new KojiSession according to odcs.server.conf, logins to + Koji using this session and returns it. + :rtype: koji.KojiSession + :return: KojiSession + """ + koji_config = munch.Munch(koji.read_config( + profile_name=conf.koji_profile, + user_config=conf.koji_config, + )) + + address = koji_config.server + authtype = koji_config.authtype + log.info("Connecting to koji %r with %r." % (address, authtype)) + koji_session = koji.ClientSession(address, opts=koji_config) + if authtype == "kerberos": + ccache = getattr(conf, "krb_ccache", None) + keytab = getattr(conf, "krb_keytab", None) + principal = getattr(conf, "krb_principal", None) + log.debug(" ccache: %r, keytab: %r, principal: %r" % ( + ccache, keytab, principal)) + if keytab and principal: + koji_session.krb_login( + principal=principal, + keytab=keytab, + ccache=ccache, + ) + else: + koji_session.krb_login(ccache=ccache) + elif authtype == "ssl": + koji_session.ssl_login( + os.path.expanduser(koji_config.cert), + None, + os.path.expanduser(koji_config.serverca), + ) + else: + raise ValueError("Unrecognized koji authtype %r" % authtype) + + return koji_session + + +def run_koji_runroot(koji_session, cmd): + """ + Runs the cmd defined as list of commands in Koji runroot and waits for the + task to finish. Raises RuntimeError in case the task execution failed. + """ + kwargs = { + 'channel': conf.pungi_parent_runroot_channel, + 'packages': conf.pungi_parent_runroot_packages, + 'mounts': conf.pungi_parent_runroot_mounts, + 'weight': conf.pungi_parent_runroot_weight + } + + task_id = koji_session.runroot( + conf.pungi_parent_runroot_tag, conf.pungi_parent_runroot_arch, + " ".join(cmd), **kwargs) + + while True: + # wait for the task to finish + if koji_session.taskFinished(task_id): + break + log.info("Waiting for Koji runroot task %r to finish...", task_id) + time.sleep(60) + + info = koji_session.getTaskInfo(task_id) + if info is None: + raise RuntimeError("Cannot get status of Koji task %r" % task_id) + state = koji.TASK_STATES[info['state']] + if state in ('FAILED', 'CANCELED'): + raise RuntimeError("Koji runroot task %r failed." % task_id) + + def download_file(url, output_path): """ Downloads file from URL `url` to `output_path`. diff --git a/server/tests/test_pungi.py b/server/tests/test_pungi.py index 4d2591d..8709780 100644 --- a/server/tests/test_pungi.py +++ b/server/tests/test_pungi.py @@ -201,7 +201,7 @@ class TestPungiRunroot(unittest.TestCase): self.config_patcher.patch('pungi_runroot_target_dir_url', 'http://kojipkgs.fedoraproject.org/compose/odcs') self.config_patcher.start() - self.patch_make_koji_session = patch("odcs.server.pungi.Pungi.make_koji_session") + self.patch_make_koji_session = patch("odcs.server.pungi.make_koji_session") self.make_koji_session = self.patch_make_koji_session.start() self.koji_session = MagicMock() self.koji_session.runroot.return_value = 123 diff --git a/server/tests/test_remove_expired_composes_thread.py b/server/tests/test_remove_expired_composes_thread.py index fb387f7..e4551dc 100644 --- a/server/tests/test_remove_expired_composes_thread.py +++ b/server/tests/test_remove_expired_composes_thread.py @@ -21,17 +21,18 @@ # Written by Jan Kaluza from odcs.server import db, conf +import odcs.server.backend from odcs.server.models import Compose from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS from odcs.server.backend import RemoveExpiredComposesThread from odcs.server.pungi import PungiSourceType from datetime import datetime, timedelta -from .utils import ModelsBaseTest +from .utils import ModelsBaseTest, ConfigPatcher import os import mock -from mock import patch +from mock import patch, MagicMock class TestRemoveExpiredComposesThread(ModelsBaseTest): @@ -211,3 +212,149 @@ class TestRemoveExpiredComposesThread(ModelsBaseTest): self.thread._remove_compose_dir(toplevel_dir) unlink.assert_not_called() rmtree.assert_called_once() + + +class TestRemoveExpiredComposesThreadPungiRunroot(ModelsBaseTest): + maxDiff = None + + def setUp(self): + super(TestRemoveExpiredComposesThreadPungiRunroot, self).setUp() + + compose = Compose.create( + db.session, "unknown", PungiSourceType.MODULE, "testmodule-master", + COMPOSE_RESULTS["repository"], 60) + db.session.add(compose) + db.session.commit() + + self.config_patcher = ConfigPatcher(odcs.server.backend.conf) + self.config_patcher.patch("pungi_runroot_enabled", True) + self.config_patcher.start() + + self.thread = RemoveExpiredComposesThread() + + self.patch_make_koji_session = patch("odcs.server.utils.make_koji_session") + self.make_koji_session = self.patch_make_koji_session.start() + self.koji_session = MagicMock() + self.make_koji_session.return_value = self.koji_session + + self.patch_run_koji_runroot = patch("odcs.server.utils.run_koji_runroot") + self.run_koji_runroot = self.patch_run_koji_runroot.start() + + def tearDown(self): + super(TestRemoveExpiredComposesThreadPungiRunroot, self).tearDown() + self.config_patcher.stop() + self.patch_make_koji_session.stop() + + def _mock_glob(self, glob, dirs): + glob_ret_values = [[], []] + for d in dirs: + path = os.path.join(conf.target_dir, d) + if d.startswith("latest-"): + glob_ret_values[0].append(path) + else: + glob_ret_values[1].append(path) + glob.side_effect = glob_ret_values + + @patch("os.path.isdir") + @patch("glob.glob") + def test_remove_left_composes(self, glob, isdir): + isdir.return_value = True + self._mock_glob(glob, ["latest-odcs-96-1", "odcs-96-1-20171005.n.0"]) + self.thread.do_work() + + self.run_koji_runroot.assert_not_called() + + @patch("os.path.isdir") + @patch("glob.glob") + def test_remove_left_composes_not_dir( + self, glob, isdir): + isdir.return_value = False + self._mock_glob(glob, ["latest-odcs-96-1"]) + self.thread.do_work() + self.run_koji_runroot.assert_not_called() + + @patch("os.path.isdir") + @patch("glob.glob") + def test_remove_left_composes_wrong_dir( + self, glob, isdir): + isdir.return_value = True + self._mock_glob(glob, ["latest-odcs-", "odcs-", "odcs-abc"]) + self.thread.do_work() + self.run_koji_runroot.assert_not_called() + + @patch("os.path.isdir") + @patch("glob.glob") + def test_remove_left_composes_valid_compose( + self, glob, isdir): + isdir.return_value = True + self._mock_glob(glob, ["latest-odcs-1-1", "odcs-1-1-2017.n.0"]) + c = db.session.query(Compose).filter(Compose.id == 1).one() + c.state = COMPOSE_STATES["done"] + db.session.add(c) + db.session.commit() + self.thread.do_work() + self.run_koji_runroot.assert_not_called() + + @patch("os.path.isdir") + @patch("glob.glob") + @patch("os.path.realpath") + @patch("os.path.exists") + def test_remove_left_composes_expired_compose( + self, exists, realpath, glob, isdir): + exists.return_value = True + realpath.return_value = "/odcs-real" + isdir.return_value = True + self._mock_glob(glob, ["latest-odcs-1-1", "odcs-1-1-2017.n.0"]) + c = db.session.query(Compose).filter(Compose.id == 1).one() + c.state = COMPOSE_STATES["removed"] + db.session.add(c) + db.session.commit() + self.thread.do_work() + self.run_koji_runroot.assert_called_once_with( + self.koji_session, + ['rm', '-f', os.path.join(conf.target_dir, 'latest-odcs-1-1'), '&&', + 'rm', '-rf', '/odcs-real', '&&', + 'rm', '-f', os.path.join(conf.target_dir, 'odcs-1-1-2017.n.0'), '&&', + 'rm', '-rf', '/odcs-real']) + + @patch("os.path.realpath") + @patch("os.path.exists") + def test_remove_compose_dir_symlink( + self, exists, realpath): + exists.return_value = True + toplevel_dir = "/odcs" + realpath.return_value = "/odcs-real" + + cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir) + self.assertEqual( + cmds, ['rm', '-f', '/odcs', '&&', 'rm', '-rf', '/odcs-real']) + + @patch("shutil.rmtree") + @patch("os.unlink") + @patch("os.path.realpath") + @patch("os.path.exists") + def test_remove_compose_dir_broken_symlink( + self, exists, realpath, unlink, rmtree): + def mocked_exists(p): + return p != "/odcs-real" + exists.side_effect = mocked_exists + toplevel_dir = "/odcs" + realpath.return_value = "/odcs-real" + + cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir) + self.assertEqual( + cmds, ['rm', '-f', '/odcs']) + + @patch("shutil.rmtree") + @patch("os.unlink") + @patch("os.path.realpath") + @patch("os.path.exists") + def test_remove_compose_dir_real_dir( + self, exists, realpath, unlink, rmtree): + exists.return_value = True + toplevel_dir = "/odcs" + realpath.return_value = "/odcs" + + cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir) + self.assertEqual( + cmds, ['rm', '-rf', '/odcs'])