From 5bb52a7846ba966eafe0e056c1b3d8fd570859ff Mon Sep 17 00:00:00 2001 From: Jakub Kadlčík Date: Mar 28 2016 21:40:31 +0000 Subject: Update copr-backend to 1.82-1 Resolves: #1179713 - workaround for 1179806 --- diff --git a/.gitignore b/.gitignore index 43b7e0d..2bc1754 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /copr-backend-1.32.tar.gz /copr-backend-1.37.tar.gz +/copr-backend-1.82.tar.gz diff --git a/copr-backend.spec b/copr-backend.spec index 671898c..7ead382 100644 --- a/copr-backend.spec +++ b/copr-backend.spec @@ -1,10 +1,10 @@ -%if 0%{?rhel} < 7 && 0%{?rhel} > 0 +%if 0%{?rhel} <= 7 && 0%{?rhel} > 0 %global _pkgdocdir %{_docdir}/%{name}-%{version} %endif Name: copr-backend -Version: 1.37 -Release: 3%{?dist} +Version: 1.82 +Release: 1%{?dist} Summary: Backend for Copr Group: Applications/Productivity @@ -20,17 +20,42 @@ BuildArch: noarch BuildRequires: asciidoc BuildRequires: libxslt BuildRequires: util-linux +BuildRequires: yum-utils BuildRequires: python-setuptools BuildRequires: python-requests BuildRequires: python2-devel +BuildRequires: python-copr >= 1.60 BuildRequires: systemd -%if 0%{?rhel} < 7 && 0%{?rhel} > 0 -BuildRequires: python-argparse -%endif +BuildRequires: redis + +BuildRequires: pytest +BuildRequires: python-pytest-cov +BuildRequires: python-mock +BuildRequires: python-six +BuildRequires: python-munch +BuildRequires: python-lockfile +# missing python3 +BuildRequires: python-daemon +BuildRequires: python-requests +BuildRequires: python-setproctitle +# missing python3 +BuildRequires: python-retask +BuildRequires: python-copr >= 1.60 +BuildRequires: ansible >= 1.2 +BuildRequires: python-IPy +BuildRequires: python-paramiko +BuildRequires: python-psutil +BuildRequires: python-futures +BuildRequires: python-dateutil +BuildRequires: pytz +# BuildRequires: python-plumbum +# BuildRequires: wget -- ??? + #for doc package -BuildRequires: epydoc -BuildRequires: graphviz +BuildRequires: sphinx +BuildRequires: python-sphinx +Requires: obs-signd Requires: ansible >= 1.2 Requires: lighttpd Requires: euca2ools @@ -39,17 +64,32 @@ Requires: openssh-clients Requires: mock Requires: yum-utils Requires: createrepo_c >= 0.2.1-3 -Requires: python-bunch +Requires: python-munch Requires: python-daemon Requires: python-lockfile Requires: python-requests Requires: python-setproctitle Requires: python-retask +Requires: python-copr +Requires: python-six +Requires: python-IPy +Requires: python-psutil +Requires: python-futures +Requires: python-dateutil +Requires: pytz Requires: redis Requires: logrotate Requires: fedmsg Requires: gawk Requires: crontabs +Requires: prunerepo +Requires: python-paramiko +Requires: python-lockfile +# Requires: python-ipdb +Requires: logstash +Requires: libappstream-glib-builder >= 0.4.0 +# Requires: python-plumbum + Requires(post): systemd Requires(preun): systemd Requires(postun): systemd @@ -75,10 +115,14 @@ only. %build + +%if 0%{?fedora} # build documentation -pushd documentation -make %{?_smp_mflags} python +pushd docs + make %{?_smp_mflags} html + rm build/html/.buildinfo popd +%endif # ?fedora %install @@ -86,7 +130,6 @@ install -d %{buildroot}%{_sharedstatedir}/copr install -d %{buildroot}%{_sharedstatedir}/copr/jobs install -d %{buildroot}%{_sharedstatedir}/copr/public_html/results install -d %{buildroot}%{_var}/log/copr -install -d %{buildroot}%{_var}/log/copr/workers/ install -d %{buildroot}%{_pkgdocdir}/lighttpd/ install -d %{buildroot}%{_datadir}/copr/backend install -d %{buildroot}%{_sysconfdir}/copr @@ -97,29 +140,56 @@ install -d %{buildroot}/%{_var}/run/copr-backend/ install -d %{buildroot}/%{_tmpfilesdir} install -d %{buildroot}/%{_sbindir} install -d %{buildroot}%{_sysconfdir}/cron.daily +install -d %{buildroot}%{_sysconfdir}/sudoers.d +install -d %{buildroot}%{_bindir}/ +cp -a copr-backend-service %{buildroot}/%{_sbindir}/ cp -a backend/* %{buildroot}%{_datadir}/copr/backend -cp -a copr-be.py %{buildroot}%{_datadir}/copr/ -cp -a copr-be.conf.example %{buildroot}%{_sysconfdir}/copr/copr-be.conf -install -p -m 755 copr-prune-repo %{buildroot}%{_sbindir}/copr-prune-repo -install -p -m 755 crontab/copr-backend %{buildroot}%{_sysconfdir}/cron.daily/copr-backend +cp -a run/* %{buildroot}%{_bindir}/ +cp -a conf/copr-be.conf.example %{buildroot}%{_sysconfdir}/copr/copr-be.conf -cp -a dist/lighttpd/* %{buildroot}%{_pkgdocdir}/lighttpd/ -cp -a logrotate/* %{buildroot}%{_sysconfdir}/logrotate.d/ -cp -a tmpfiles.d/* %{buildroot}/%{_tmpfilesdir} +install -p -m 755 conf/crontab/copr-backend %{buildroot}%{_sysconfdir}/cron.daily/copr-backend + +cp -a conf/lighttpd/* %{buildroot}%{_pkgdocdir}/lighttpd/ +cp -a conf/logrotate/* %{buildroot}%{_sysconfdir}/logrotate.d/ +cp -a conf/tmpfiles.d/* %{buildroot}/%{_tmpfilesdir} # for ghost files touch %{buildroot}%{_var}/log/copr/copr.log -for i in `seq 7`; do - touch %{buildroot}%{_var}/log/copr/workers/worker-$i.log -done +touch %{buildroot}%{_var}/log/copr/prune_old.log + touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid -install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/ +#install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/ +cp -a run/systemd/*.service %{buildroot}/%{_unitdir}/ +install -m 0644 conf/copr.sudoers.d %{buildroot}%{_sysconfdir}/sudoers.d/copr + + +install -d %{buildroot}%{_sysconfdir}/logstash.d + +install -d %{buildroot}%{_datadir}/logstash/patterns/ +cp -a conf/logstash/lighttpd.pattern %{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern + #doc -cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/ -cp -a playbooks %{buildroot}%{_pkgdocdir}/ +# cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/ +cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/ + +install -d %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d +cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysconfdir}/logstash.d/copr_backend.conf + +%if 0%{?fedora} + cp -a docs/build/html %{buildroot}%{_pkgdocdir}/ +%endif + +%check + +redis-server --port 7777 &> /dev/null & + +#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \ +# -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/ + +kill %1 %pre getent group copr >/dev/null || groupadd -r copr @@ -129,6 +199,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %post %systemd_post copr-backend.service +%systemd_post logstash.service %preun %systemd_preun copr-backend.service @@ -137,18 +208,17 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %systemd_postun_with_restart copr-backend.service %files -%doc LICENSE -%dir %{_datadir}/copr +%license LICENSE + +%{_datadir}/copr/* %dir %{_sharedstatedir}/copr %dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/jobs/ %dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/ %dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/results %dir %attr(0755, copr, copr) %{_var}/log/copr -%dir %attr(0755, copr, copr) %{_var}/log/copr/workers %dir %attr(0755, copr, copr) %{_var}/run/copr-backend -%ghost %{_var}/log/copr/copr.log -%ghost %{_var}/log/copr/workers/worker-*.log +%ghost %{_var}/log/copr/*.log %ghost %{_var}/run/copr-backend/copr-be.pid %config(noreplace) %{_sysconfdir}/logrotate.d/copr-backend @@ -157,26 +227,336 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %doc %{_pkgdocdir}/playbooks %dir %{_sysconfdir}/copr %config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf -%{_unitdir}/copr-backend.service +%{_unitdir}/*.service %{_tmpfilesdir}/copr-backend.conf -%{_sbindir}/copr-prune-repo +%{_bindir}/* +%{_sbindir}/* + %config(noreplace) %{_sysconfdir}/cron.daily/copr-backend +%{_datadir}/logstash/patterns/lighttpd.pattern -%{_datadir}/copr/backend -%{_datadir}/copr/copr-be.py* + +%config(noreplace) %attr(0600, root, root) %{_sysconfdir}/sudoers.d/copr %files doc -%doc LICENSE -%doc %{_pkgdocdir}/python-doc +%license LICENSE +%doc +# %{_pkgdocdir}/python-doc +%{_pkgdocdir}/ %exclude %{_pkgdocdir}/lighttpd %exclude %{_pkgdocdir}/playbooks %changelog -* Wed Feb 03 2016 Fedora Release Engineering - 1.37-3 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild - -* Wed Jun 17 2015 Fedora Release Engineering - 1.37-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild +* Thu Mar 24 2016 Jakub Kadlčík 1.82-1 +- use timeout variable from config + +* Mon Mar 14 2016 Jakub Kadlčík 1.81-1 +- support project forking +- support building from PyPI +- support for redis_host, redis_port, redis_db config options +- dockerized-backend project moved under backend/docker +- run createrepo in rawhide_to_release +- specify rawhide name when calling rawhide_to_release + +* Fri Jan 29 2016 Miroslav Suchý 1.80-1 +- do not fail when when you recieve job with architecture which does not have + queue +- fix 1260780 - Build fails after successful package generation - + just add a log error message pointing to an rsync log +- jobgrabcontrol.py/retask misuse fix +- "localhost-targeted" spawn and terminate playbooks added for testing +- [frontend]implement rawhide to release feature First create new + chroots: python manage.py create_chroot fedora-24-i386 fedora-24-x86_64 +- abstraction above [BE <-> JG <-> Builders] channels +- don't traceback backend if frontend is not yet up&running +- do not preserve user and group when rsyncing + +* Wed Dec 23 2015 Miroslav Suchý 1.79-1 +- fix packaging issues in epel-7+ + +* Mon Nov 16 2015 Miroslav Suchý 1.78-1 +- handle_generate_gpg_key skips key creation when signing is disabled +- Added test_handle_generate_gpg_key +- fixed failing tests +- show when createrepo is waiting for lock +- do not block builds when processing too much actions + +* Fri Nov 06 2015 Miroslav Suchý 1.77-1 +- we need to have recent python-copr +- create copr-backend-service script to handle all copr services +- [backend] fix not starting job_grab + +* Tue Oct 13 2015 Miroslav Suchý 1.76-1 +- createrepo action run infinitely when applied to + deleted project + +* Mon Sep 21 2015 Valentin Gologuzov 1.75-1 +- [backend] run copr-backend-log service before other components + +* Mon Sep 21 2015 Valentin Gologuzov 1.74-1 +- [backend] add executible bit to run/copr_run_job_grab.py + +* Mon Sep 21 2015 Valentin Gologuzov 1.73-1 +- added context manager `local_file_logger` +- eliminated global multiprocessing.Lock +- split backend daemon: extracted RedisLogHandler, JobGrab, VMM +- replace python-bunch with python-munch +- added comps.xml support + +* Tue Aug 04 2015 Valentin Gologuzov 1.72-1 +- support new results naming in the build deletion action +- fix BuildJob.results_dir; eliminated MockRemote.pkg_dest_path +- using package name and versiong given in the build task; cleanup; +- handle error's caused by failure to obtain srpm from dist-git +- repairing test for newest changes +- rsync update + several fixes +- building from dist git +- fix vm spawn check: spawner count child processes per build group; + +* Wed Jul 01 2015 Valentin Gologuzov 1.71-1 +- add small script to print queues +- AppData supproted +- copy mockchain and rsync logs to resdir (RhBug:1221519) +- note which modules still stops us from migrating to python3 + +* Mon Jun 15 2015 Miroslav Suchý 1.70-1 +- alter vm_name= regexp +- polishing Bug 1195867 - Move or delete logs when rebuilding failed + build. +- backup only info and log files +- have just one backup directory per results directory +- clean results from previous build +- alter IP= regexp +- disabled appdata until fixed +- unable appdata in createrepo +- more safe VmMaster.check_one_vm_for_dead_builder function +- adding support for AppData +- new requirement form AppData support +- createrepo_unsafe now returns only STDOUT and raise exception on + errors + +* Mon Jun 01 2015 Valentin Gologuzov 1.69-1 +- removed creation of symlinks for log.gz +- catch exception during Worker.can_start_job +- config cleanup + +* Thu May 28 2015 Valentin Gologuzov 1.68-1 +- [backend] add config option for VM health check timeout +- [backend] moved config parameters from Threshold class into the backend + config file + +* Thu May 21 2015 Valentin Gologuzov 1.67-1 +- [backend] Handle unexpected exception VmMaster::check_one_vm_for_dead_builder + +* Thu May 21 2015 Valentin Gologuzov 1.66-1 +- [backend] fix race condition in check for dead worker + +* Wed May 20 2015 Valentin Gologuzov 1.65-1 +- [backend] Rescheduling unfinished builds before stop +- fix indentation +- [backend] request frontend to reschedule old unfinished builds at startup +- [backend] update sytemd unit: removed obsolete + EnviromentFile=/home/copr/cloud/ec2rc.variable directive + +* Tue May 19 2015 Valentin Gologuzov 1.64-1 +- [backend] check for aux process state and restart if needed + +* Fri May 15 2015 Miroslav Suchý 1.63-1 +- BR python-sphinx + +* Fri May 15 2015 Miroslav Suchý 1.62-1 +- [backend] small cleanup, need more tests + +* Fri May 15 2015 Miroslav Suchý 1.61-1 +- [backend] notify job_grab to remove job from added when start_job failed +- [backend] [vmm] terminate VM with state IN_USE only when builder process is + missed +- [backend] bugfix VMM.get_all_vm_in_group : VM could be removed before load + occures +- [backend] job_grab: postpone build is we already serving more builds + than`max_vm_per_user` option +- [backend] fix build logging +- [backend] defer sending job to worker if job owner acquired too much VMs +- [backend] replaced Thresholds.dirty_vm_terminating_timeout with config option + vm_dirty_terminating_timeout +- [backend] Thread's doesn't have a pid +- [backend] vm manage: user threading instead of multiprocessing +- [backend] VMM aware cleunup_vm_nova +- [backend] moving to nginx to serve results. lighttpd couldnt server pre- + compressed properly +- [backend] script to clean up erred and forgotten VM's using python-novaclient +- [backend] new documentation +- [backend] repaired unittests +- [backend] updated builder playbooks +- [backend] updated example spawn playbook +- [backend] don't provide logstash config directly, add only example to + documentation +- [backend] do logging from multiply processes through redis pubsub; some fixes + to VM-management +- [image_builder] initial release, due to OS bug, we cannot create snapshot + after provision through API, need to do it manually in the WebUI. +- [backend] tests cleanup +- [backend] ensure that prune script running under the copr user; simpler + `copr_find_obsolete_builds` +- [backend] safer copr_prune_results script, unittests +- [backend][frontend] Send for delete action only `src_pkg_name` instead of + original URL. +- [backend] returned script to call createrepo from cli +- [copr] don't allow acquire VMs that was last checked before server restart. +- [backend] Added limit to acquire_vm based on VMs used by the same username at + the current moment. +- New python dependencies +- [backend] run tmp redis-server for tests +- [backend] Dedicated and more complex management for builder machines. + [frontend] Now builds failed due to VM errors reschedulted faster. + +* Fri Mar 20 2015 Valentin Gologuzov 1.60-1 +- [backend][spec] start/stop redis server during package build tests + +* Fri Mar 20 2015 Valentin Gologuzov 1.59-1 +- [backend][hotfix] 1203753 : don't process delete action if src_pkg is + mallformed + +* Mon Mar 02 2015 Valentin Gologuzov 1.58-1 +- [rhbz:#1185959] - RFE: Present statistics about project + popularity. A few more counters for downloads from backend's result + directory. +- [backend] [rhbz:#1191037] RFE: Include package name and version in fedmsg + notification +- [rhbz:#1091640] RFE: Release specific additional repos +- [rhbz:#1119300] [RFE] allow easy add copr repos in using + repository lis +- [backend][frontend] removing code related to multiply source rpms in build. + Build.pkgs now expected to have exactly one src.rpm. +- [copr] backend: script fixes, dropped create_repo cli script +- more file descriptors on builder +- [rhbz:#1171796] copr sometimes doesn't delete build from repository +- [rhbz:#1073333] Record consecutive builds fails to redis. Added + script to produce warnings for nagios check from failures recorded to redis. +- correctly print job representation + +* Fri Jan 23 2015 Valentin Gologuzov 1.57-1 +- call correct Worker method on backend termination +- put gpg pubkey to the project results root dir (one level up from + the chroot dir) +- don't kill Worker from errors during job build +- [rhbz:#1169782] RFE - Show package "version-release" instead of + just "version" +- [rhbz:#1117446] add a build id tagfile into the package directory +- Updated unittests to reflect latest changes. +- builder: use only one log file for rsync per build +- dispatcher: run terminate_instance safely +- cleanup example config +- cleanup mockremote.builder +- Builder.download don't use Popen+PIPE.communicate with rsync, + output redirected to the files. +- disable networking only when required; python style exception + handling in mockremote*; removed run/copr_mockremote +- test build with disabled networking +- simplified mockremote.builder.Builder.check_for_ans_error; new + method mockremote.builder.Builder.run_ansible_with_check +- daemons.dispatched.Worker: don't fail when wrong group_id was + provided +- add vm_ip to worker process title (rhbz: 1182637) + +* Wed Jan 14 2015 Valentin Gologuzov 1.56-1 +- [backend] [.spec] fix %files section + +* Wed Jan 14 2015 Valentin Gologuzov 1.55-1 +- [backend] [bugfix] set pythonpath in systemd unit to run /usr/bin/copr_be.py +- [backend] [RHBZ:#1182106] JobGrabber dies when action raises an exception. +- [backend] Moved scripts into /usr/bin/ Renamed copr{-,_}be.py. + +* Wed Jan 07 2015 Miroslav Suchý 1.54-1 +- 1179713 - workaround for 1179806 +- run script unbufferred otherwise log is written after full block +- express that it is n-th projects +- fix permissions on prune script + +* Mon Jan 05 2015 Valentin Gologuzov 1.53-1 +- [backend, frontend] [RHBZ:#1176364] Wrong value for the build timeout. + +* Mon Dec 15 2014 Valentin Gologuzov 1.52-1 +- fixed config option `results_baseurl` usage, in mockremote + +* Fri Dec 12 2014 Valentin Gologuzov 1.51-1 +- updated BuildRequires; cleanup imports +- package sign: generate gpg usermail with special symbol +- bugfix: when dispatcher has vm_ip it shouldn't start new VM; +- run tests during rpm build +- minor docstring fix + +* Wed Dec 10 2014 Valentin Gologuzov 1.50-1 +- [backend] added option to control ansible ssh transport, changed by default + to `paramiko` [frontend] bugfix api create new +- [backend] removed spawn_vars options, to be able to spawn VMs in advance +- [backend] unittest for backend.daemons.log +- [backend] massive refactoring and unittest coverage +- [backend] backend.sign: discover `keygen_host` from backend config file + +* Tue Nov 25 2014 Valentin Gologuzov 1.49-1 +- [backend] small bug in dispatcher + +* Tue Nov 25 2014 Valentin Gologuzov 1.48-1 +- bugfixes, disabled debug prints, fixed PEP8 violations + +* Thu Nov 20 2014 Valentin Gologuzov 1.47-1 +- refactored mockremote, added explicit BuildJob class +- allow to spawn builder in advance +- copr-prune-repo respects auto_createrepo option +- bugfix: repeated config reads produced constantly growing lists + +* Fri Oct 24 2014 Valentin Gologuzov 1.46-1 +- [backend] added handling of new action type: "createrepo" +- [backend] added dependency on `python-copr` +- [backend] added to mockchroot -a /devel/repodata subfolder +- [backend] new config option to define the public frontend api endpoint +- [backend] conditional execution of createrepo_c +- [backend] unittest for Action and minor refactoring +- [backend] rotate backend.log as well + +* Thu Sep 18 2014 Miroslav Suchý 1.45-1 +- [backend][keygen] minor fixes/typos + +* Thu Sep 18 2014 Valentin Gologuzov 1.44-1 +- [backend] type fix + +* Thu Sep 18 2014 Valentin Gologuzov 1.43-1 +- [backend] config parsing: convert fields to proper data type. +- [backend] added option to disable package signing. +- [keygen] new component for copr: gpg key generation for package sign +- [backend] broadcast both submitter and owner to fedmsg +- [backend] example backend config: changes url protocol to HTTPS. + +* Mon Aug 25 2014 Adam Samalik 1.42-1 +- [backend] [RHBZ:1128606 ] For rhel-5 builds pass "--checksum md5" to + `createrepo_c` command. +- [backend] fix of builder test +- [backend] test builder instance after spawning +- [backend] never give up while spawning an OpenStack VM +- [backend] worker's log filename correction +- [backend] task id in worker process' name +- [backend] async build playbooks + +* Thu Aug 14 2014 Miroslav Suchý 1.41-1 +- [backend] fix of fix +- [backend] couple of fixes + +* Wed Aug 13 2014 Miroslav Suchý 1.40-1 +- [backend] queue cleaning +- [backend] experimental build groups for more architectures +- [backend] fix of a strange beaviour of retask +- [backend] fedmsg shows submitter instead of project owner +- [backend] new task queue for workers using retask +- epel-7 comps workaround is need no more, since CENTOS7 have been released + +* Tue Jul 22 2014 Miroslav Suchý 1.39-1 +- FrontendCallback prettified +- Starting state implemented, cancelling fixed +- [backend] faster skipping + +* Tue Jul 15 2014 Miroslav Suchý 1.38-1 +- [backend] built pkgs fix * Tue Jul 15 2014 Miroslav Suchý 1.37-1 - [backend] shell command uses pipes.quote diff --git a/sources b/sources index 7468a84..912557c 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -c8e1068738d83537bae5ca40b0447822 copr-backend-1.37.tar.gz +93851268640182b81dc550e7624cd146 copr-backend-1.82.tar.gz