Blame mbs-cg-upload.patch

4f51c34
diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py
87848f5
index 078cc1f..4c9ee67 100644
4f51c34
--- a/module_build_service/builder/KojiContentGenerator.py
4f51c34
+++ b/module_build_service/builder/KojiContentGenerator.py
87848f5
@@ -32,6 +32,7 @@ import platform
87848f5
 import shutil
87848f5
 import subprocess
87848f5
 import tempfile
87848f5
+import time
87848f5
 
87848f5
 import koji
87848f5
 
a241acc
@@ -273,7 +274,7 @@ class KojiContentGenerator(object):
a241acc
                 u'filesize': len(self.mmd),
a241acc
                 u'checksum_type': u'md5',
a241acc
                 u'checksum': unicode(hashlib.md5(self.mmd).hexdigest()),
a241acc
-                u'filename': u'modulemd.yaml',
a241acc
+                u'filename': u'modulemd.txt',
a241acc
                 u'components': components
a241acc
             }
a241acc
         )
a241acc
@@ -317,7 +318,7 @@ class KojiContentGenerator(object):
a241acc
         Returns path to the temporary directory
a241acc
         """
a241acc
         prepdir = tempfile.mkdtemp(prefix="koji-cg-import")
a241acc
-        mmd_path = os.path.join(prepdir, "modulemd.yaml")
a241acc
+        mmd_path = os.path.join(prepdir, "modulemd.txt")
a241acc
         with open(mmd_path, "w") as mmd_f:
a241acc
             mmd_f.write(self.mmd)
a241acc
 
4f51c34
@@ -328,6 +328,32 @@ class KojiContentGenerator(object):
4f51c34
         return prepdir
4f51c34
 
4f51c34
 
4f51c34
+    def _upload_outputs(self, session, metadata, file_dir):
4f51c34
+        """
4f51c34
+        Uploads output files to Koji hub.
4f51c34
+        """
4f51c34
+        to_upload = []
4f51c34
+        for info in metadata['output']:
4f51c34
+            if info.get('metadata_only', False):
4f51c34
+                continue
4f51c34
+            localpath = os.path.join(file_dir, info['filename'])
4f51c34
+            if not os.path.exists(localpath):
4f51c34
+                err = "Cannot upload %s to Koji. No such file." % localpath
4f51c34
+                log.error(err)
4f51c34
+                raise RuntimeError(err)
4f51c34
+
4f51c34
+            to_upload.append([localpath, info])
4f51c34
+
4f51c34
+        # Create unique server directory.
4f51c34
+        serverdir = 'mbs/%r.%d' % (time.time(), self.module.id)
4f51c34
+
4f51c34
+        for localpath, info in to_upload:
4f51c34
+            log.info("Uploading %s to Koji" % localpath)
4f51c34
+            session.uploadWrapper(localpath, serverdir, callback=None)
4f51c34
+            log.info("Upload of %s to Koji done" % localpath)
4f51c34
+
4f51c34
+        return serverdir
4f51c34
+
4f51c34
     def koji_import(self):
4f51c34
         """This method imports given module into the configured koji instance as
4f51c34
         a content generator based build
4f51c34
@@ -338,7 +364,8 @@ class KojiContentGenerator(object):
4f51c34
         file_dir = self._prepare_file_directory()
4f51c34
         metadata = self._get_content_generator_metadata(file_dir)
4f51c34
         try:
4f51c34
-            build_info = session.CGImport(metadata, file_dir)
4f51c34
+            serverdir = self._upload_outputs(session, metadata, file_dir)
4f51c34
+            build_info = session.CGImport(metadata, serverdir)
4f51c34
             log.debug("Content generator import done: %s",
4f51c34
                       json.dumps(build_info, sort_keys=True, indent=4))
4f51c34
         except Exception, e:
87848f5
87848f5
diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py
87848f5
index 7a6f7d8..56c8668 100644
87848f5
--- a/module_build_service/scheduler/handlers/repos.py
87848f5
+++ b/module_build_service/scheduler/handlers/repos.py
87848f5
@@ -89,13 +89,6 @@ def done(config, session, msg):
87848f5
         tag_name=tag, components=[c.package for c in module_build.component_builds])
87848f5
     builder.buildroot_connect(groups)
87848f5
 
87848f5
-    # Ok, for the subset of builds that did complete successfully, check to
87848f5
-    # see if they are in the buildroot.
87848f5
-    artifacts = [component_build.nvr for component_build in good]
87848f5
-    if not builder.buildroot_ready(artifacts):
87848f5
-        log.info("Not all of %r are in the buildroot.  Waiting." % artifacts)
87848f5
-        return
87848f5
-
87848f5
     # If we have reached here then we know the following things:
87848f5
     #
87848f5
     # - All components in this batch have finished (failed or succeeded)
87848f5
@@ -116,6 +109,13 @@ def done(config, session, msg):
87848f5
 
87848f5
     further_work = []
87848f5
     if has_unbuilt_components and not has_failed_components:
87848f5
+        # Ok, for the subset of builds that did complete successfully, check to
87848f5
+        # see if they are in the buildroot before starting new batch.
87848f5
+        artifacts = [component_build.nvr for component_build in good]
87848f5
+        if not builder.buildroot_ready(artifacts):
87848f5
+            log.info("Not all of %r are in the buildroot.  Waiting." % artifacts)
87848f5
+            return
87848f5
+
87848f5
         # Try to start next batch build, because there are still unbuilt
87848f5
         # components in a module.
87848f5
         further_work += start_next_batch_build(
87848f5
diff --git a/module_build_service/scheduler/handlers/tags.py b/module_build_service/scheduler/handlers/tags.py
87848f5
index 493a5da..bdf7d78 100644
87848f5
--- a/module_build_service/scheduler/handlers/tags.py
87848f5
+++ b/module_build_service/scheduler/handlers/tags.py
87848f5
@@ -27,7 +27,7 @@ import module_build_service.builder
87848f5
 import module_build_service.pdc
87848f5
 import logging
87848f5
 import koji
87848f5
-from module_build_service import models, log
87848f5
+from module_build_service import models, log, messaging
87848f5
 from module_build_service.utils import start_next_batch_build
87848f5
 
87848f5
 logging.basicConfig(level=logging.DEBUG)
87848f5
@@ -78,13 +78,30 @@ def tagged(config, session, msg):
87848f5
         if not c.tagged and c.state == koji.BUILD_STATES['COMPLETE']
87848f5
     ]
87848f5
 
87848f5
+    further_work = []
87848f5
+
87848f5
     # If all components are tagged, start newRepo task.
87848f5
     if not untagged_components:
87848f5
-        log.info("All components tagged, regenerating repo for tag %s", tag)
87848f5
         builder = module_build_service.builder.GenericBuilder.create_from_module(
87848f5
             session, module_build, config)
87848f5
-        task_id = builder.koji_session.newRepo(tag)
87848f5
-        module_build.new_repo_task_id = task_id
87848f5
+
87848f5
+        unbuilt_components = [
87848f5
+            c for c in module_build.component_builds
87848f5
+            if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
87848f5
+        ]
87848f5
+        if unbuilt_components:
87848f5
+            log.info("All components in batch tagged, regenerating repo for tag %s", tag)
87848f5
+            task_id = builder.koji_session.newRepo(tag)
87848f5
+            module_build.new_repo_task_id = task_id
87848f5
+        else:
87848f5
+            # In case this is the last batch, we do not need to regenerate the
87848f5
+            # buildroot, because we will not build anything else in it. It
87848f5
+            # would be useless to wait for a repository we will not use anyway.
87848f5
+            log.info("All components in module tagged and built, skipping the "
87848f5
+                     "last repo regeneration")
87848f5
+            further_work += [messaging.KojiRepoChange(
87848f5
+                'components::_finalize: fake msg',
87848f5
+                builder.module_build_tag['name'])]
87848f5
         session.commit()
87848f5
 
87848f5
-    return []
87848f5
+    return further_work