|
|
4f51c34 |
diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py
|
|
|
87848f5 |
index 078cc1f..4c9ee67 100644
|
|
|
4f51c34 |
--- a/module_build_service/builder/KojiContentGenerator.py
|
|
|
4f51c34 |
+++ b/module_build_service/builder/KojiContentGenerator.py
|
|
|
87848f5 |
@@ -32,6 +32,7 @@ import platform
|
|
|
87848f5 |
import shutil
|
|
|
87848f5 |
import subprocess
|
|
|
87848f5 |
import tempfile
|
|
|
87848f5 |
+import time
|
|
|
87848f5 |
|
|
|
87848f5 |
import koji
|
|
|
87848f5 |
|
|
|
a241acc |
@@ -273,7 +274,7 @@ class KojiContentGenerator(object):
|
|
|
a241acc |
u'filesize': len(self.mmd),
|
|
|
a241acc |
u'checksum_type': u'md5',
|
|
|
a241acc |
u'checksum': unicode(hashlib.md5(self.mmd).hexdigest()),
|
|
|
a241acc |
- u'filename': u'modulemd.yaml',
|
|
|
a241acc |
+ u'filename': u'modulemd.txt',
|
|
|
a241acc |
u'components': components
|
|
|
a241acc |
}
|
|
|
a241acc |
)
|
|
|
a241acc |
@@ -317,7 +318,7 @@ class KojiContentGenerator(object):
|
|
|
a241acc |
Returns path to the temporary directory
|
|
|
a241acc |
"""
|
|
|
a241acc |
prepdir = tempfile.mkdtemp(prefix="koji-cg-import")
|
|
|
a241acc |
- mmd_path = os.path.join(prepdir, "modulemd.yaml")
|
|
|
a241acc |
+ mmd_path = os.path.join(prepdir, "modulemd.txt")
|
|
|
a241acc |
with open(mmd_path, "w") as mmd_f:
|
|
|
a241acc |
mmd_f.write(self.mmd)
|
|
|
a241acc |
|
|
|
4f51c34 |
@@ -328,6 +328,32 @@ class KojiContentGenerator(object):
|
|
|
4f51c34 |
return prepdir
|
|
|
4f51c34 |
|
|
|
4f51c34 |
|
|
|
4f51c34 |
+ def _upload_outputs(self, session, metadata, file_dir):
|
|
|
4f51c34 |
+ """
|
|
|
4f51c34 |
+ Uploads output files to Koji hub.
|
|
|
4f51c34 |
+ """
|
|
|
4f51c34 |
+ to_upload = []
|
|
|
4f51c34 |
+ for info in metadata['output']:
|
|
|
4f51c34 |
+ if info.get('metadata_only', False):
|
|
|
4f51c34 |
+ continue
|
|
|
4f51c34 |
+ localpath = os.path.join(file_dir, info['filename'])
|
|
|
4f51c34 |
+ if not os.path.exists(localpath):
|
|
|
4f51c34 |
+ err = "Cannot upload %s to Koji. No such file." % localpath
|
|
|
4f51c34 |
+ log.error(err)
|
|
|
4f51c34 |
+ raise RuntimeError(err)
|
|
|
4f51c34 |
+
|
|
|
4f51c34 |
+ to_upload.append([localpath, info])
|
|
|
4f51c34 |
+
|
|
|
4f51c34 |
+ # Create unique server directory.
|
|
|
4f51c34 |
+ serverdir = 'mbs/%r.%d' % (time.time(), self.module.id)
|
|
|
4f51c34 |
+
|
|
|
4f51c34 |
+ for localpath, info in to_upload:
|
|
|
4f51c34 |
+ log.info("Uploading %s to Koji" % localpath)
|
|
|
4f51c34 |
+ session.uploadWrapper(localpath, serverdir, callback=None)
|
|
|
4f51c34 |
+ log.info("Upload of %s to Koji done" % localpath)
|
|
|
4f51c34 |
+
|
|
|
4f51c34 |
+ return serverdir
|
|
|
4f51c34 |
+
|
|
|
4f51c34 |
def koji_import(self):
|
|
|
4f51c34 |
"""This method imports given module into the configured koji instance as
|
|
|
4f51c34 |
a content generator based build
|
|
|
4f51c34 |
@@ -338,7 +364,8 @@ class KojiContentGenerator(object):
|
|
|
4f51c34 |
file_dir = self._prepare_file_directory()
|
|
|
4f51c34 |
metadata = self._get_content_generator_metadata(file_dir)
|
|
|
4f51c34 |
try:
|
|
|
4f51c34 |
- build_info = session.CGImport(metadata, file_dir)
|
|
|
4f51c34 |
+ serverdir = self._upload_outputs(session, metadata, file_dir)
|
|
|
4f51c34 |
+ build_info = session.CGImport(metadata, serverdir)
|
|
|
4f51c34 |
log.debug("Content generator import done: %s",
|
|
|
4f51c34 |
json.dumps(build_info, sort_keys=True, indent=4))
|
|
|
4f51c34 |
except Exception, e:
|
|
|
87848f5 |
|
|
|
87848f5 |
diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py
|
|
|
87848f5 |
index 7a6f7d8..56c8668 100644
|
|
|
87848f5 |
--- a/module_build_service/scheduler/handlers/repos.py
|
|
|
87848f5 |
+++ b/module_build_service/scheduler/handlers/repos.py
|
|
|
87848f5 |
@@ -89,13 +89,6 @@ def done(config, session, msg):
|
|
|
87848f5 |
tag_name=tag, components=[c.package for c in module_build.component_builds])
|
|
|
87848f5 |
builder.buildroot_connect(groups)
|
|
|
87848f5 |
|
|
|
87848f5 |
- # Ok, for the subset of builds that did complete successfully, check to
|
|
|
87848f5 |
- # see if they are in the buildroot.
|
|
|
87848f5 |
- artifacts = [component_build.nvr for component_build in good]
|
|
|
87848f5 |
- if not builder.buildroot_ready(artifacts):
|
|
|
87848f5 |
- log.info("Not all of %r are in the buildroot. Waiting." % artifacts)
|
|
|
87848f5 |
- return
|
|
|
87848f5 |
-
|
|
|
87848f5 |
# If we have reached here then we know the following things:
|
|
|
87848f5 |
#
|
|
|
87848f5 |
# - All components in this batch have finished (failed or succeeded)
|
|
|
87848f5 |
@@ -116,6 +109,13 @@ def done(config, session, msg):
|
|
|
87848f5 |
|
|
|
87848f5 |
further_work = []
|
|
|
87848f5 |
if has_unbuilt_components and not has_failed_components:
|
|
|
87848f5 |
+ # Ok, for the subset of builds that did complete successfully, check to
|
|
|
87848f5 |
+ # see if they are in the buildroot before starting new batch.
|
|
|
87848f5 |
+ artifacts = [component_build.nvr for component_build in good]
|
|
|
87848f5 |
+ if not builder.buildroot_ready(artifacts):
|
|
|
87848f5 |
+ log.info("Not all of %r are in the buildroot. Waiting." % artifacts)
|
|
|
87848f5 |
+ return
|
|
|
87848f5 |
+
|
|
|
87848f5 |
# Try to start next batch build, because there are still unbuilt
|
|
|
87848f5 |
# components in a module.
|
|
|
87848f5 |
further_work += start_next_batch_build(
|
|
|
87848f5 |
diff --git a/module_build_service/scheduler/handlers/tags.py b/module_build_service/scheduler/handlers/tags.py
|
|
|
87848f5 |
index 493a5da..bdf7d78 100644
|
|
|
87848f5 |
--- a/module_build_service/scheduler/handlers/tags.py
|
|
|
87848f5 |
+++ b/module_build_service/scheduler/handlers/tags.py
|
|
|
87848f5 |
@@ -27,7 +27,7 @@ import module_build_service.builder
|
|
|
87848f5 |
import module_build_service.pdc
|
|
|
87848f5 |
import logging
|
|
|
87848f5 |
import koji
|
|
|
87848f5 |
-from module_build_service import models, log
|
|
|
87848f5 |
+from module_build_service import models, log, messaging
|
|
|
87848f5 |
from module_build_service.utils import start_next_batch_build
|
|
|
87848f5 |
|
|
|
87848f5 |
logging.basicConfig(level=logging.DEBUG)
|
|
|
87848f5 |
@@ -78,13 +78,30 @@ def tagged(config, session, msg):
|
|
|
87848f5 |
if not c.tagged and c.state == koji.BUILD_STATES['COMPLETE']
|
|
|
87848f5 |
]
|
|
|
87848f5 |
|
|
|
87848f5 |
+ further_work = []
|
|
|
87848f5 |
+
|
|
|
87848f5 |
# If all components are tagged, start newRepo task.
|
|
|
87848f5 |
if not untagged_components:
|
|
|
87848f5 |
- log.info("All components tagged, regenerating repo for tag %s", tag)
|
|
|
87848f5 |
builder = module_build_service.builder.GenericBuilder.create_from_module(
|
|
|
87848f5 |
session, module_build, config)
|
|
|
87848f5 |
- task_id = builder.koji_session.newRepo(tag)
|
|
|
87848f5 |
- module_build.new_repo_task_id = task_id
|
|
|
87848f5 |
+
|
|
|
87848f5 |
+ unbuilt_components = [
|
|
|
87848f5 |
+ c for c in module_build.component_builds
|
|
|
87848f5 |
+ if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
|
|
|
87848f5 |
+ ]
|
|
|
87848f5 |
+ if unbuilt_components:
|
|
|
87848f5 |
+ log.info("All components in batch tagged, regenerating repo for tag %s", tag)
|
|
|
87848f5 |
+ task_id = builder.koji_session.newRepo(tag)
|
|
|
87848f5 |
+ module_build.new_repo_task_id = task_id
|
|
|
87848f5 |
+ else:
|
|
|
87848f5 |
+ # In case this is the last batch, we do not need to regenerate the
|
|
|
87848f5 |
+ # buildroot, because we will not build anything else in it. It
|
|
|
87848f5 |
+ # would be useless to wait for a repository we will not use anyway.
|
|
|
87848f5 |
+ log.info("All components in module tagged and built, skipping the "
|
|
|
87848f5 |
+ "last repo regeneration")
|
|
|
87848f5 |
+ further_work += [messaging.KojiRepoChange(
|
|
|
87848f5 |
+ 'components::_finalize: fake msg',
|
|
|
87848f5 |
+ builder.module_build_tag['name'])]
|
|
|
87848f5 |
session.commit()
|
|
|
87848f5 |
|
|
|
87848f5 |
- return []
|
|
|
87848f5 |
+ return further_work
|