From c4078c160caad8c019c4e890757fbde34aa83f60 Mon Sep 17 00:00:00 2001

From: Jan Kaluza <jkaluza@redhat.com>

Date: Dec 08 2017 13:30:51 +0000

Subject: If pungi_runroot_enabled is true, use koji runroot also to remove expired composes.

---

diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py

index c7c77b9..539b23d 100644

--- a/server/odcs/server/backend.py

+++ b/server/odcs/server/backend.py

@@ -143,6 +143,33 @@ class RemoveExpiredComposesThread(BackendThread):

         else:

             shutil.rmtree(toplevel_dir)

+    def _get_remove_compose_dir_cmds(self, toplevel_dir):

+        """

+        Returns the commands needed to remove the compose with toplevel_dir

+        compose dir as list.

+        """

+

+        # Be nice and don't fail when directory does not exist.

+        if not os.path.exists(toplevel_dir):

+            log.warn("Cannot remove directory %s, it does not exist",

+                     toplevel_dir)

+            return []

+

+        cmds = []

+

+        # If toplevel_dir is a symlink, remove the symlink and

+        # its target. If toplevel_dir is normal directory, just

+        # remove it using rmtree.

+        if os.path.realpath(toplevel_dir) != toplevel_dir:

+            targetpath = os.path.realpath(toplevel_dir)

+            cmds += ["rm", "-f", toplevel_dir]

+            if os.path.exists(targetpath):

+                cmds += ["&&", "rm", "-rf", targetpath]

+        else:

+            cmds += ["rm", "-rf", toplevel_dir]

+

+        return cmds

+

     def _get_compose_id_from_path(self, path):

         """

         Returns the ID of compose from directory path in conf.target_dir.

@@ -163,6 +190,15 @@ class RemoveExpiredComposesThread(BackendThread):

         """

         log.info("Checking for expired composes")

+        # We store all the compose directories to remove in this list and

+        # removes them all in the end of this method.

+        # The reason to do it this way is that if "conf.pungi_runroot_enabled"

+        # is True, we spawn Koji runroot task which calls "rm" commands for

+        # every compose directory we want to remove. We need a list

+        # of directories to be able to generate that set of rm commands later

+        # if neede and if not, just remove the directories by this thread.

+        compose_dirs_to_remove = []

+

         composes = Compose.composes_to_expire()

         for compose in composes:

             log.info("%r: Removing compose", compose)

@@ -170,7 +206,7 @@ class RemoveExpiredComposesThread(BackendThread):

             compose.time_removed = datetime.utcnow()

             db.session.commit()

             if not compose.reused_id:

-                self._remove_compose_dir(compose.toplevel_dir)

+                compose_dirs_to_remove.append(str(compose.toplevel_dir))

         # In case of ODCS error, there might be left-over directories

         # belonging to already expired composes. Try to find them in the

@@ -197,16 +233,34 @@ class RemoveExpiredComposesThread(BackendThread):

             if not composes:

                 log.info("Removing data of compose %d - it is not in "

                          "database: %s", compose_id, path)

-                self._remove_compose_dir(path)

+                compose_dirs_to_remove.append(path)

                 continue

             compose = composes[0]

             if compose.state == COMPOSE_STATES["removed"]:

                 log.info("%r: Removing data of compose - it has already "

                          "expired some time ago: %s", compose_id, path)

-                self._remove_compose_dir(path)

+                compose_dirs_to_remove.append(path)

                 continue

+        koji_cmds = []

+        for compose_dir in compose_dirs_to_remove:

+            if conf.pungi_runroot_enabled:

+                cmds = self._get_remove_compose_dir_cmds(compose_dir)

+                if cmds:

+                    koji_cmds += cmds

+                    koji_cmds += ["&&"]

+            else:

+                self._remove_compose_dir(compose_dir)

+

+        # In case we have some commands to run in Koji, spawn the Koji runroot

+        # task and wait for result.

+        if koji_cmds:

+            # Remove last "&&"

+            koji_cmds = koji_cmds[:-1]

+            koji_session = odcs.server.utils.make_koji_session()

+            odcs.server.utils.run_koji_runroot(koji_session, koji_cmds)

+

 def create_koji_session():

     """

diff --git a/server/odcs/server/pungi.py b/server/odcs/server/pungi.py

index aab57e4..9dbd1d3 100644

--- a/server/odcs/server/pungi.py

+++ b/server/odcs/server/pungi.py

@@ -25,8 +25,6 @@ import os

 import shutil

 import tempfile

 import jinja2

-import koji

-import munch

 import time

 import random

 import string

@@ -35,7 +33,8 @@ import odcs.server.utils

 from odcs.server import conf, log

 from odcs.server import comps

 from odcs.common.types import PungiSourceType, COMPOSE_RESULTS

-from odcs.server.utils import makedirs, download_file

+from odcs.server.utils import (

+    makedirs, download_file, make_koji_session, run_koji_runroot)

 class PungiConfig(object):

@@ -185,47 +184,6 @@ class Pungi(object):

             output_path = os.path.join(topdir, "pungi.conf")

             download_file(self.pungi_cfg, output_path)

-    def make_koji_session(self):

-        """

-        Creates new KojiSession according to odcs.server.conf, logins to

-        Koji using this session and returns it.

-        :rtype: koji.KojiSession

-        :return: KojiSession

-        """

-        koji_config = munch.Munch(koji.read_config(

-            profile_name=conf.koji_profile,

-            user_config=conf.koji_config,

-        ))

-

-        address = koji_config.server

-        authtype = koji_config.authtype

-        log.info("Connecting to koji %r with %r." % (address, authtype))

-        koji_session = koji.ClientSession(address, opts=koji_config)

-        if authtype == "kerberos":

-            ccache = getattr(conf, "krb_ccache", None)

-            keytab = getattr(conf, "krb_keytab", None)

-            principal = getattr(conf, "krb_principal", None)

-            log.debug("  ccache: %r, keytab: %r, principal: %r" % (

-                ccache, keytab, principal))

-            if keytab and principal:

-                koji_session.krb_login(

-                    principal=principal,

-                    keytab=keytab,

-                    ccache=ccache,

-                )

-            else:

-                koji_session.krb_login(ccache=ccache)

-        elif authtype == "ssl":

-            koji_session.ssl_login(

-                os.path.expanduser(koji_config.cert),

-                None,

-                os.path.expanduser(koji_config.serverca),

-            )

-        else:

-            raise ValueError("Unrecognized koji authtype %r" % authtype)

-

-        return koji_session

-

     def get_pungi_cmd(self, conf_topdir, targetdir):

         """

         Returns list with pungi command line arguments needed to generate

@@ -305,7 +263,7 @@ class Pungi(object):

         makedirs(conf_topdir)

         self._write_cfgs(conf_topdir)

-        koji_session = self.make_koji_session()

+        koji_session = make_koji_session()

         serverdir = self.upload_files_to_koji(koji_session, conf_topdir)

         # TODO: Copy keytab from secret repo and generate koji profile.

@@ -313,30 +271,7 @@ class Pungi(object):

         cmd += ["cp", "/mnt/koji/work/%s/*" % serverdir, ".", "&&"]

         cmd += self.get_pungi_cmd("./", conf.pungi_runroot_target_dir)

-        kwargs = {

-            'channel': conf.pungi_parent_runroot_channel,

-            'packages': conf.pungi_parent_runroot_packages,

-            'mounts': conf.pungi_parent_runroot_mounts,

-            'weight': conf.pungi_parent_runroot_weight

-        }

-

-        task_id = koji_session.runroot(

-            conf.pungi_parent_runroot_tag, conf.pungi_parent_runroot_arch,

-            " ".join(cmd), **kwargs)

-

-        while True:

-            # wait for the task to finish

-            if koji_session.taskFinished(task_id):

-                break

-            log.info("Waiting for Koji runroot task %r to finish...", task_id)

-            time.sleep(60)

-

-        info = koji_session.getTaskInfo(task_id)

-        if info is None:

-            raise RuntimeError("Cannot get status of Koji task %r" % task_id)

-        state = koji.TASK_STATES[info['state']]

-        if state in ('FAILED', 'CANCELED'):

-            raise RuntimeError("Koji runroot task %r failed." % task_id)

+        run_koji_runroot(koji_session, cmd)

     def run(self):

         """

diff --git a/server/odcs/server/utils.py b/server/odcs/server/utils.py

index 63c3574..20e2e50 100644

--- a/server/odcs/server/utils.py

+++ b/server/odcs/server/utils.py

@@ -26,11 +26,86 @@ import os

 import time

 import subprocess

 import requests

+import munch

+import koji

 from distutils.spawn import find_executable

 from odcs.server import conf, log

+def make_koji_session():

+    """

+    Creates new KojiSession according to odcs.server.conf, logins to

+    Koji using this session and returns it.

+    :rtype: koji.KojiSession

+    :return: KojiSession

+    """

+    koji_config = munch.Munch(koji.read_config(

+        profile_name=conf.koji_profile,

+        user_config=conf.koji_config,

+    ))

+

+    address = koji_config.server

+    authtype = koji_config.authtype

+    log.info("Connecting to koji %r with %r." % (address, authtype))

+    koji_session = koji.ClientSession(address, opts=koji_config)

+    if authtype == "kerberos":

+        ccache = getattr(conf, "krb_ccache", None)

+        keytab = getattr(conf, "krb_keytab", None)

+        principal = getattr(conf, "krb_principal", None)

+        log.debug("  ccache: %r, keytab: %r, principal: %r" % (

+            ccache, keytab, principal))

+        if keytab and principal:

+            koji_session.krb_login(

+                principal=principal,

+                keytab=keytab,

+                ccache=ccache,

+            )

+        else:

+            koji_session.krb_login(ccache=ccache)

+    elif authtype == "ssl":

+        koji_session.ssl_login(

+            os.path.expanduser(koji_config.cert),

+            None,

+            os.path.expanduser(koji_config.serverca),

+        )

+    else:

+        raise ValueError("Unrecognized koji authtype %r" % authtype)

+

+    return koji_session

+

+

+def run_koji_runroot(koji_session, cmd):

+    """

+    Runs the cmd defined as list of commands in Koji runroot and waits for the

+    task to finish. Raises RuntimeError in case the task execution failed.

+    """

+    kwargs = {

+        'channel': conf.pungi_parent_runroot_channel,

+        'packages': conf.pungi_parent_runroot_packages,

+        'mounts': conf.pungi_parent_runroot_mounts,

+        'weight': conf.pungi_parent_runroot_weight

+    }

+

+    task_id = koji_session.runroot(

+        conf.pungi_parent_runroot_tag, conf.pungi_parent_runroot_arch,

+        " ".join(cmd), **kwargs)

+

+    while True:

+        # wait for the task to finish

+        if koji_session.taskFinished(task_id):

+            break

+        log.info("Waiting for Koji runroot task %r to finish...", task_id)

+        time.sleep(60)

+

+    info = koji_session.getTaskInfo(task_id)

+    if info is None:

+        raise RuntimeError("Cannot get status of Koji task %r" % task_id)

+    state = koji.TASK_STATES[info['state']]

+    if state in ('FAILED', 'CANCELED'):

+        raise RuntimeError("Koji runroot task %r failed." % task_id)

+

+

 def download_file(url, output_path):

     """

     Downloads file from URL `url` to `output_path`.

diff --git a/server/tests/test_pungi.py b/server/tests/test_pungi.py

index 4d2591d..8709780 100644

--- a/server/tests/test_pungi.py

+++ b/server/tests/test_pungi.py

@@ -201,7 +201,7 @@ class TestPungiRunroot(unittest.TestCase):

         self.config_patcher.patch('pungi_runroot_target_dir_url', 'http://kojipkgs.fedoraproject.org/compose/odcs')

         self.config_patcher.start()

-        self.patch_make_koji_session = patch("odcs.server.pungi.Pungi.make_koji_session")

+        self.patch_make_koji_session = patch("odcs.server.pungi.make_koji_session")

         self.make_koji_session = self.patch_make_koji_session.start()

         self.koji_session = MagicMock()

         self.koji_session.runroot.return_value = 123

diff --git a/server/tests/test_remove_expired_composes_thread.py b/server/tests/test_remove_expired_composes_thread.py

index fb387f7..e4551dc 100644

--- a/server/tests/test_remove_expired_composes_thread.py

+++ b/server/tests/test_remove_expired_composes_thread.py

@@ -21,17 +21,18 @@

 # Written by Jan Kaluza <jkaluza@redhat.com>

 from odcs.server import db, conf

+import odcs.server.backend

 from odcs.server.models import Compose

 from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS

 from odcs.server.backend import RemoveExpiredComposesThread

 from odcs.server.pungi import PungiSourceType

 from datetime import datetime, timedelta

-from .utils import ModelsBaseTest

+from .utils import ModelsBaseTest, ConfigPatcher

 import os

 import mock

-from mock import patch

+from mock import patch, MagicMock

 class TestRemoveExpiredComposesThread(ModelsBaseTest):

@@ -211,3 +212,149 @@ class TestRemoveExpiredComposesThread(ModelsBaseTest):

         self.thread._remove_compose_dir(toplevel_dir)

         unlink.assert_not_called()

         rmtree.assert_called_once()

+

+

+class TestRemoveExpiredComposesThreadPungiRunroot(ModelsBaseTest):

+    maxDiff = None

+

+    def setUp(self):

+        super(TestRemoveExpiredComposesThreadPungiRunroot, self).setUp()

+

+        compose = Compose.create(

+            db.session, "unknown", PungiSourceType.MODULE, "testmodule-master",

+            COMPOSE_RESULTS["repository"], 60)

+        db.session.add(compose)

+        db.session.commit()

+

+        self.config_patcher = ConfigPatcher(odcs.server.backend.conf)

+        self.config_patcher.patch("pungi_runroot_enabled", True)

+        self.config_patcher.start()

+

+        self.thread = RemoveExpiredComposesThread()

+

+        self.patch_make_koji_session = patch("odcs.server.utils.make_koji_session")

+        self.make_koji_session = self.patch_make_koji_session.start()

+        self.koji_session = MagicMock()

+        self.make_koji_session.return_value = self.koji_session

+

+        self.patch_run_koji_runroot = patch("odcs.server.utils.run_koji_runroot")

+        self.run_koji_runroot = self.patch_run_koji_runroot.start()

+

+    def tearDown(self):

+        super(TestRemoveExpiredComposesThreadPungiRunroot, self).tearDown()

+        self.config_patcher.stop()

+        self.patch_make_koji_session.stop()

+

+    def _mock_glob(self, glob, dirs):

+        glob_ret_values = [[], []]

+        for d in dirs:

+            path = os.path.join(conf.target_dir, d)

+            if d.startswith("latest-"):

+                glob_ret_values[0].append(path)

+            else:

+                glob_ret_values[1].append(path)

+        glob.side_effect = glob_ret_values

+

+    @patch("os.path.isdir")

+    @patch("glob.glob")

+    def test_remove_left_composes(self, glob, isdir):

+        isdir.return_value = True

+        self._mock_glob(glob, ["latest-odcs-96-1", "odcs-96-1-20171005.n.0"])

+        self.thread.do_work()

+

+        self.run_koji_runroot.assert_not_called()

+

+    @patch("os.path.isdir")

+    @patch("glob.glob")

+    def test_remove_left_composes_not_dir(

+            self, glob, isdir):

+        isdir.return_value = False

+        self._mock_glob(glob, ["latest-odcs-96-1"])

+        self.thread.do_work()

+        self.run_koji_runroot.assert_not_called()

+

+    @patch("os.path.isdir")

+    @patch("glob.glob")

+    def test_remove_left_composes_wrong_dir(

+            self, glob, isdir):

+        isdir.return_value = True

+        self._mock_glob(glob, ["latest-odcs-", "odcs-", "odcs-abc"])

+        self.thread.do_work()

+        self.run_koji_runroot.assert_not_called()

+

+    @patch("os.path.isdir")

+    @patch("glob.glob")

+    def test_remove_left_composes_valid_compose(

+            self, glob, isdir):

+        isdir.return_value = True

+        self._mock_glob(glob, ["latest-odcs-1-1", "odcs-1-1-2017.n.0"])

+        c = db.session.query(Compose).filter(Compose.id == 1).one()

+        c.state = COMPOSE_STATES["done"]

+        db.session.add(c)

+        db.session.commit()

+        self.thread.do_work()

+        self.run_koji_runroot.assert_not_called()

+

+    @patch("os.path.isdir")

+    @patch("glob.glob")

+    @patch("os.path.realpath")

+    @patch("os.path.exists")

+    def test_remove_left_composes_expired_compose(

+            self, exists, realpath, glob, isdir):

+        exists.return_value = True

+        realpath.return_value = "/odcs-real"

+        isdir.return_value = True

+        self._mock_glob(glob, ["latest-odcs-1-1", "odcs-1-1-2017.n.0"])

+        c = db.session.query(Compose).filter(Compose.id == 1).one()

+        c.state = COMPOSE_STATES["removed"]

+        db.session.add(c)

+        db.session.commit()

+        self.thread.do_work()

+        self.run_koji_runroot.assert_called_once_with(

+            self.koji_session,

+            ['rm', '-f', os.path.join(conf.target_dir, 'latest-odcs-1-1'), '&&',

+             'rm', '-rf', '/odcs-real', '&&',

+             'rm', '-f', os.path.join(conf.target_dir, 'odcs-1-1-2017.n.0'), '&&',

+             'rm', '-rf', '/odcs-real'])

+

+    @patch("os.path.realpath")

+    @patch("os.path.exists")

+    def test_remove_compose_dir_symlink(

+            self, exists, realpath):

+        exists.return_value = True

+        toplevel_dir = "/odcs"

+        realpath.return_value = "/odcs-real"

+

+        cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir)

+        self.assertEqual(

+            cmds, ['rm', '-f', '/odcs', '&&', 'rm', '-rf', '/odcs-real'])

+

+    @patch("shutil.rmtree")

+    @patch("os.unlink")

+    @patch("os.path.realpath")

+    @patch("os.path.exists")

+    def test_remove_compose_dir_broken_symlink(

+            self, exists, realpath, unlink, rmtree):

+        def mocked_exists(p):

+            return p != "/odcs-real"

+        exists.side_effect = mocked_exists

+        toplevel_dir = "/odcs"

+        realpath.return_value = "/odcs-real"

+

+        cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir)

+        self.assertEqual(

+            cmds, ['rm', '-f', '/odcs'])

+

+    @patch("shutil.rmtree")

+    @patch("os.unlink")

+    @patch("os.path.realpath")

+    @patch("os.path.exists")

+    def test_remove_compose_dir_real_dir(

+            self, exists, realpath, unlink, rmtree):

+        exists.return_value = True

+        toplevel_dir = "/odcs"

+        realpath.return_value = "/odcs"

+

+        cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir)

+        self.assertEqual(

+            cmds, ['rm', '-rf', '/odcs'])