From 147986062cfb3815268c91065527dabc2b6b6adc Mon Sep 17 00:00:00 2001 From: Fabian Affolter Date: Sep 10 2019 07:27:45 +0000 Subject: [PATCH 1/23] Incomplete --- diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e69de29..0000000 --- a/.gitignore +++ /dev/null diff --git a/dead.package b/dead.package new file mode 100644 index 0000000..09b8255 --- /dev/null +++ b/dead.package @@ -0,0 +1 @@ +Incomplete diff --git a/sources b/sources deleted file mode 100644 index e69de29..0000000 --- a/sources +++ /dev/null From 4d52cb0f03df597af61a6ee6601d8a57cdf06a09 Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: Aug 13 2020 08:30:01 +0000 Subject: [PATCH 2/23] Revert "Incomplete" This reverts commit 147986062cfb3815268c91065527dabc2b6b6adc. Unretirement request: https://pagure.io/releng/issue/9676 Signed-off-by: Tomas Hrcka --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/.gitignore diff --git a/dead.package b/dead.package deleted file mode 100644 index 09b8255..0000000 --- a/dead.package +++ /dev/null @@ -1 +0,0 @@ -Incomplete diff --git a/sources b/sources new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sources From f71cf1d0864f0cb1eea07f17f6d7584eebfed7c2 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 3/23] Zuul 3.19.0 packaging --- diff --git a/.gitignore b/.gitignore index e69de29..b7ad809 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,4 @@ +/zuul-3.18.0.tar.gz +results_zuul +*.rpm +/zuul-3.19.0.tar.gz diff --git a/0001-Remove-another-shebang-and-remove-useless-exec-bits.patch b/0001-Remove-another-shebang-and-remove-useless-exec-bits.patch new file mode 100644 index 0000000..991660c --- /dev/null +++ b/0001-Remove-another-shebang-and-remove-useless-exec-bits.patch @@ -0,0 +1,58 @@ +From 6feb399854ef55a3e43c5a46c56fae487dcd8d28 Mon Sep 17 00:00:00 2001 +From: Fabien Boucher +Date: Tue, 19 May 2020 14:40:51 +0200 +Subject: [PATCH] Remove another shebang and remove useless exec bits + +Change-Id: Id49bc9a4b50948b1f07e4fa917c93acac0c2a17e +--- + zuul/cmd/__init__.py | 0 + zuul/cmd/client.py | 0 + zuul/cmd/executor.py | 0 + zuul/cmd/merger.py | 5 ----- + zuul/cmd/scheduler.py | 0 + zuul/cmd/web.py | 0 + 6 files changed, 5 deletions(-) + mode change 100755 => 100644 zuul/cmd/__init__.py + mode change 100755 => 100644 zuul/cmd/client.py + mode change 100755 => 100644 zuul/cmd/executor.py + mode change 100755 => 100644 zuul/cmd/merger.py + mode change 100755 => 100644 zuul/cmd/scheduler.py + mode change 100755 => 100644 zuul/cmd/web.py + +diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py +old mode 100755 +new mode 100644 +diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py +old mode 100755 +new mode 100644 +diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py +old mode 100755 +new mode 100644 +diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py +old mode 100755 +new mode 100644 +index 1d4b6fef..9b827427 +--- a/zuul/cmd/merger.py ++++ b/zuul/cmd/merger.py +@@ -1,4 +1,3 @@ +-#!/usr/bin/env python + # Copyright 2012 Hewlett-Packard Development Company, L.P. + # Copyright 2013-2014 OpenStack Foundation + # +@@ -69,7 +68,3 @@ class Merger(zuul.cmd.ZuulDaemonApp): + + def main(): + Merger().main() +- +- +-if __name__ == "__main__": +- main() +diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py +old mode 100755 +new mode 100644 +diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py +old mode 100755 +new mode 100644 +-- +2.25.1 + diff --git a/0001-requirements-add-explicit-reference-to-dateutil.patch b/0001-requirements-add-explicit-reference-to-dateutil.patch new file mode 100644 index 0000000..95f9c39 --- /dev/null +++ b/0001-requirements-add-explicit-reference-to-dateutil.patch @@ -0,0 +1,28 @@ +From ea0b50c90d22a2f24b288e1a91e2b85f17059cbc Mon Sep 17 00:00:00 2001 +From: Tristan Cacqueray +Date: Mon, 25 May 2020 17:39:59 +0000 +Subject: [PATCH] requirements: add explicit reference to dateutil + +Zuul is using a dateutil provided transitively. This change +makes the requirements explicit. + +Change-Id: I672d409a1bb241e5d199c77acdd51a8260d99ee2 +--- + requirements.txt | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/requirements.txt b/requirements.txt +index 732aa35a..78eb980f 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -4,6 +4,7 @@ importlib-resources==1.0.2 + # Early virtualenv 20 had bad file location assumptions + virtualenv!=20.0.0,!=20.0.1,>20 + ++python-dateutil + github3.py>=1.1.0 + PyYAML>=3.1.0 + paramiko>=2.0.1 +-- +2.25.4 + diff --git a/README.fedora b/README.fedora new file mode 100644 index 0000000..66d83f6 --- /dev/null +++ b/README.fedora @@ -0,0 +1,70 @@ +TODO(fbo) This is still a draft here. Will be improved once we are clear with runtime deps. + +zuul-manage-ansible +------------------- + +sudo -u zuul bash +cd +zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin +ls -al /var/lib/zuul/ansible-bin + + +zuul-scheduler +-------------- + +Add in /etc/zuul/main.yaml: +- tenant: + name: demo-tenant + source: + local_git: + config-projects: + - zuul-config + +Install zookeeper with nodepool tests.yml +Comment MQTT section from zuul.conf +Comment Gerrit section +Comment the MySQL section + +systemctl start zuul-scheduler + +zuul-executor +------------- + +systemctl start zuul-executor + + +prepare basic config +-------------------- + +In zuul.conf add: +[connection local_git] +driver=git +baseurl=http://localhost:8000/home/fedora +poll_delay=300 + + +pip3 install --user dulwich +dulwich web-daemon -l 0.0.0.0 / +git init /home/fedora/zuul-config +vim /home/fedora/zuul-config/.zuul.yaml +- pipeline: + name: periodic + post-review: true + description: Jobs in this queue are triggered every minute. + manager: independent + precedence: low + trigger: + timer: + - time: '* * * * *' + +- project: + periodic: + jobs: + - noop +git config user.name "John Doe" +git config user.email "john@localhost" +git add .zuul.yaml +git commit -m"Init demo config" + +Dans les logs /var/log/zuul/zuul.log +zuul.ExecutorClient: [e: 3f6a0ad4b46f4a9aa23744e3e6b9eea5] Execute job noop diff --git a/logging.conf b/logging.conf new file mode 100644 index 0000000..edfd3f6 --- /dev/null +++ b/logging.conf @@ -0,0 +1,46 @@ +[loggers] +keys=root,zuul,gerrit + +[handlers] +keys=console,debug,normal + +[formatters] +keys=simple + +[logger_root] +level=WARNING +handlers=console + +[logger_zuul] +level=DEBUG +handlers=debug,normal +qualname=zuul +propagate=0 + +[logger_gerrit] +level=DEBUG +handlers=debug,normal +qualname=gerrit +propagate=0 + +[handler_console] +level=WARNING +class=StreamHandler +formatter=simple +args=(sys.stdout,) + +[handler_debug] +level=DEBUG +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/zuul/debug.log', 'midnight', 1, 30,) + +[handler_normal] +level=INFO +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/zuul/zuul.log', 'midnight', 1, 30,) + +[formatter_simple] +format=%(asctime)s %(levelname)s %(name)s: %(message)s +datefmt= diff --git a/sources b/sources index e69de29..e739312 100644 --- a/sources +++ b/sources @@ -0,0 +1 @@ +SHA512 (zuul-3.19.0.tar.gz) = b75430879b6165e87e0f9d05de3cda67cc1e0b84e65ac1407ce6f853e27f66811ac438cbff2d6c45243025e5aba26820fa478929744f1eed9487f3fc957bcda1 diff --git a/zuul-executor.service b/zuul-executor.service new file mode 100644 index 0000000..bf00a50 --- /dev/null +++ b/zuul-executor.service @@ -0,0 +1,15 @@ +[Unit] +Description=Zuul Executor Service +After=network.target + +[Service] +Type=simple +User=zuul +Group=zuul +SyslogIdentifier=zuul-executor +EnvironmentFile=-/etc/sysconfig/zuul +ExecStart=/usr/bin/zuul-executor -f +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/zuul-fingergw.service b/zuul-fingergw.service new file mode 100644 index 0000000..71a832f --- /dev/null +++ b/zuul-fingergw.service @@ -0,0 +1,14 @@ +[Unit] +Description=Zuul Finger Gateway Service +After=network.target + +[Service] +Type=simple +User=root +Group=root +SyslogIdentifier=zuul-fingergw +EnvironmentFile=-/etc/sysconfig/zuul +ExecStart=/usr/bin/zuul-fingergw -f + +[Install] +WantedBy=multi-user.target diff --git a/zuul-merger.service b/zuul-merger.service new file mode 100644 index 0000000..3f035da --- /dev/null +++ b/zuul-merger.service @@ -0,0 +1,15 @@ +[Unit] +Description=Zuul Merger Service +After=network.target + +[Service] +Type=simple +User=zuul +Group=zuul +LimitNOFILE=8192 +SyslogIdentifier=zuul-merger +EnvironmentFile=-/etc/sysconfig/zuul +ExecStart=/usr/bin/zuul-merger -f + +[Install] +WantedBy=multi-user.target diff --git a/zuul-scheduler.service b/zuul-scheduler.service new file mode 100644 index 0000000..5083062 --- /dev/null +++ b/zuul-scheduler.service @@ -0,0 +1,15 @@ +[Unit] +Description=Zuul Scheduler Service +After=network.target + +[Service] +Type=simple +User=zuul +Group=zuul +SyslogIdentifier=zuul-scheduler +EnvironmentFile=-/etc/sysconfig/zuul +ExecStart=/usr/bin/zuul-scheduler -f +ExecReload=/usr/bin/zuul-scheduler smart-reconfigure + +[Install] +WantedBy=multi-user.target diff --git a/zuul-web.service b/zuul-web.service new file mode 100644 index 0000000..acddd1a --- /dev/null +++ b/zuul-web.service @@ -0,0 +1,14 @@ +[Unit] +Description=Zuul Web Service +After=network.target + +[Service] +Type=simple +User=zuul +Group=zuul +SyslogIdentifier=zuul-web +EnvironmentFile=-/etc/sysconfig/zuul +ExecStart=/usr/bin/zuul-web -f + +[Install] +WantedBy=multi-user.target diff --git a/zuul.conf b/zuul.conf new file mode 100644 index 0000000..6a4f09c --- /dev/null +++ b/zuul.conf @@ -0,0 +1,80 @@ +[gearman] +server=127.0.0.1 +;port=4730 +;ssl_ca=/path/to/ca.pem +;ssl_cert=/path/to/client.pem +;ssl_key=/path/to/client.key + +[statsd] +server=127.0.0.1 + +[zookeeper] +hosts=127.0.0.1:2181 + +[gearman_server] +start=true +;ssl_ca=/path/to/ca.pem +;ssl_cert=/path/to/server.pem +;ssl_key=/path/to/server.key +;port=4730 + +[scheduler] +tenant_config=/etc/zuul/main.yaml +log_config=/etc/zuul/logging.conf +pidfile=/var/run/zuul/zuul.pid +state_dir=/var/lib/zuul + +[merger] +git_dir=/var/lib/zuul/git +;git_user_email=zuul@example.com +;git_user_name=zuul + +[executor] +default_username=zuul +log_config=/etc/zuul/logging.conf +trusted_ro_paths=/opt/zuul-scripts:/var/cache +trusted_rw_paths=/opt/zuul-logs + +[web] +listen_address=127.0.0.1 +port=9000 +static_cache_expiry=0 +status_url=https://zuul.example.com/status + +[webclient] +url=https://zuul.example.com +verify_ssl=true + +[auth zuul_operator] +driver=HS256 +allow_authz_override=true +realm=zuul.example.com +default=true +client_id=zuul.example.com +issuer_id=zuul_operator +secret=NoDanaOnlyZuul +max_validity_time=36000 +skew=0 + +;[connection gerrit] +;driver=gerrit +;server=review.example.com +;baseurl=https://review.example.com/r +;user=jenkins +;sshkey=/home/jenkins/.ssh/id_rsa +;keepalive=60 + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com + +;[connection mqtt] +;driver=mqtt +;server=localhost +;user=zuul +;password=zuul +;keepalive=60 + diff --git a/zuul.rpmlintrc b/zuul.rpmlintrc new file mode 100644 index 0000000..80109a9 --- /dev/null +++ b/zuul.rpmlintrc @@ -0,0 +1,52 @@ +# Documentation in a separate sub-package +addFilter("zuul.noarch: W: no-documentation") +addFilter("zuul-executor.noarch: W: no-documentation") +addFilter("zuul-fingergw.noarch: W: no-documentation") +addFilter("zuul-merger.noarch: W: no-documentation") +addFilter("zuul-migrate.noarch: W: no-documentation") +addFilter("zuul-scheduler.noarch: W: no-documentation") +addFilter("zuul-web.noarch: W: no-documentation") +addFilter("zuul-webui.noarch: W: no-documentation") +# Expected - Zuul services should run as the zuul user identity +addFilter("zuul.noarch: W: non-standard-uid /etc/zuul/logging.conf zuul") +addFilter("zuul.noarch: W: non-standard-gid /etc/zuul/logging.conf zuul") +addFilter("zuul.noarch: W: non-standard-uid /etc/zuul/zuul.conf zuul") +addFilter("zuul.noarch: W: non-standard-gid /etc/zuul/zuul.conf zuul") +addFilter("zuul.noarch: W: non-standard-uid /var/lib/zuul zuul") +addFilter("zuul.noarch: W: non-standard-gid /var/lib/zuul zuul") +addFilter("zuul.noarch: W: non-standard-uid /var/lib/zuul/.ssh zuul") +addFilter("zuul.noarch: W: non-standard-gid /var/lib/zuul/.ssh zuul") +addFilter("zuul.noarch: W: non-standard-uid /var/lib/zuul/ansible zuul") +addFilter("zuul.noarch: W: non-standard-gid /var/lib/zuul/ansible zuul") +addFilter("zuul.noarch: W: non-standard-uid /var/lib/zuul/keys zuul") +addFilter("zuul.noarch: W: non-standard-gid /var/lib/zuul/keys zuul") +addFilter("zuul.noarch: W: non-standard-uid /var/log/zuul zuul") +addFilter("zuul.noarch: W: non-standard-gid /var/log/zuul zuul") +addFilter("zuul-executor.noarch: W: non-standard-uid /var/lib/zuul/ansible zuul") +addFilter("zuul-executor.noarch: W: non-standard-gid /var/lib/zuul/ansible zuul") +addFilter("zuul-executor.noarch: W: non-standard-uid /var/lib/zuul/executor zuul") +addFilter("zuul-executor.noarch: W: non-standard-gid /var/lib/zuul/executor zuul") +# Expected - zuul.conf can store credentials +addFilter("zuul.noarch: E: non-readable /etc/zuul/zuul.conf 640") +# Expected for now - https://review.opendev.org/728955 - does not âss the upstream CI +addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.8/site-packages/zuul/ansible/base/library/command.py 644 /usr/bin/python3 ") +addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.8/site-packages/zuul/ansible/base/library/zuul_console.py 644 /usr/bin/python3 ") +# Expected - Zuul runtime expects to find that directory +addFilter("zuul.noarch: W: hidden-file-or-dir /var/lib/zuul/.ssh") +addFilter("zuul.noarch: W: hidden-file-or-dir /var/lib/zuul/.ssh") +# Expected - Provided Python logging configuration configures the rotation +addFilter("zuul.noarch: W: log-files-without-logrotate \['/var/log/zuul'\]") +# Expected - upstream does not provide man pages +addFilter("zuul.noarch: W: no-manual-page-for-binary zuul") +addFilter("zuul.noarch: W: no-manual-page-for-binary zuul-bwrap") +addFilter("zuul-executor.noarch: W: no-manual-page-for-binary zuul-executor") +addFilter("zuul-executor.noarch: W: no-manual-page-for-binary zuul-manage-ansible") +addFilter("zuul-fingergw.noarch: W: no-manual-page-for-binary zuul-fingergw") +addFilter("zuul-merger.noarch: W: no-manual-page-for-binary zuul-merger") +addFilter("zuul-migrate.noarch: W: no-manual-page-for-binary zuul-migrate") +addFilter("zuul-scheduler.noarch: W: no-manual-page-for-binary zuul-scheduler") +addFilter("zuul-web.noarch: W: no-manual-page-for-binary zuul-web") +# Expected - executor service require python3-devel to install ansible virtualenvs +addFilter("zuul-executor.noarch: E: devel-dependency python3-devel") +# Expected - gearman is a protocol name +addFilter("zuul-scheduler.noarch: W: spelling-error %description -l en_US gearman -> gear man, gear-man, germane") diff --git a/zuul.spec b/zuul.spec new file mode 100644 index 0000000..5c7831e --- /dev/null +++ b/zuul.spec @@ -0,0 +1,535 @@ +Name: zuul +Version: 3.19.0 +Release: 1%{?dist} +Summary: Trunk Gating System + +License: ASL 2.0 +URL: https://zuul-ci.org +Source0: %pypi_source + +Source2: zuul-scheduler.service +Source3: zuul-merger.service +Source4: zuul-executor.service +Source5: zuul-web.service +Source6: zuul-fingergw.service +Source7: README.fedora +Source8: zuul.conf +Source9: logging.conf + +Patch01: 0001-Remove-another-shebang-and-remove-useless-exec-bits.patch +Patch02: 0001-requirements-add-explicit-reference-to-dateutil.patch + +BuildArch: noarch + + +#Error: +# Problem: conflicting requests + +# https://src.fedoraproject.org/rpms/python-virtualenv/pull-request/20 +# - nothing provides ((python3.8dist(virtualenv) < 20 or python3.8dist(virtualenv) > 20) with (python3.8dist(virtualenv) < 20.0.1 or python3.8dist(virtualenv) > 20.0.1) with python3.8dist(virtualenv) > 20) needed by zuul-3.19.0-1.fc33.noarch + +# https://github.com/cherrypy/cheroot/issues/263 # the zuul cap reason +# The proposed fix https://github.com/cherrypy/cheroot/pull/277 +# https://bugzilla.redhat.com/show_bug.cgi?id=1834207 +# - nothing provides python3.8dist(cheroot) < 8.1 needed by zuul-3.19.0-1.fc33.noarch + +# - nothing provides python3.8dist(cherrypy) = 18.3 needed by zuul-3.19.0-1.fc33.noarch + + +BuildRequires: python3-devel +BuildRequires: python3-pbr +BuildRequires: python3-setuptools +BuildRequires: python3-zuul-sphinx +BuildRequires: python3-snowballstemmer +BuildRequires: python3-fixtures +BuildRequires: python3-sphinx-autodoc-typehints +BuildRequires: python3-sphinxcontrib-blockdiag +BuildRequires: python3-sphinxcontrib-programoutput +BuildRequires: python3-sphinxcontrib-openapi +BuildRequires: python3-reno +BuildRequires: python3-CacheControl +BuildRequires: python3-jwt +BuildRequires: python3-github3py +BuildRequires: python3-pyyaml +BuildRequires: python3-paramiko +BuildRequires: python3-GitPython +BuildRequires: python3-daemon +BuildRequires: python3-extras +BuildRequires: python3-statsd +BuildRequires: python3-voluptuous +BuildRequires: python3-gear +BuildRequires: python3-APScheduler +BuildRequires: python3-prettytable +BuildRequires: python3-babel +BuildRequires: python3-kazoo +BuildRequires: python3-sqlalchemy +BuildRequires: python3-alembic +BuildRequires: python3-cryptography +BuildRequires: python3-cherrypy +BuildRequires: python3-ws4py +BuildRequires: python3-routes +BuildRequires: python3-netaddr +BuildRequires: python3-paho-mqtt +BuildRequires: python3-psutil +BuildRequires: python3-fb-re2 +BuildRequires: python3-cachetools +BuildRequires: python3-jsonpath-rw +BuildRequires: python3-dateutil +BuildRequires: python3-iso8601 +BuildRequires: python3-PyMySQL +BuildRequires: python3-psycopg2 +BuildRequires: python3-pathspec +# BuildRequires: python3-graphene +BuildRequires: systemd +BuildRequires: ansible + + +%description +Zuul is a program that drives continuous integration, delivery, +and deployment systems with a focus on project gating and +interrelated projects. + + +%package webui +Summary: The Zuul web interface + +%description webui +This package provides the Zuul web interface source code. +Please refers to README.fedora for build and deployment instruction. + +%package scheduler +Summary: The Zuul scheduler service +Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} + +%description scheduler +The main Zuul process. Handles receiving events, executing jobs, +collecting results and posting reports. Coordinates the work of +the other components. It also provides a gearman daemon which +the other components use for coordination. + + +%package merger +Summary: The Zuul merger service +Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} + +%description merger +Scale-out component that performs git merge operations. +Zuul performs a large number of git operations in the course of +its work. Adding merger processes can help speed Zuul’s processing. +This component is optional (zero or more of these can be run). + + +%package web +Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Summary: The Zuul web service + +%description web +A web server that receives “web-hook” events from external providers, +supplies a web dashboard, and provides web-socket access to live +streaming of logs. + + +%package executor +Summary: The Zuul executor service +Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: git-core +Requires: bubblewrap +# zuul-manage-ansible requires them to create ansible virtualenvs +## Requires: python3-virtualenv +Requires: gcc +Requires: python3-devel + +%description executor +Scale-out component for executing jobs. At least one of these is +required. Depending on system configuration, you can expect a single +executor to handle up to about 100 simultaneous jobs. Can handle +the functions of a merger if dedicated mergers are not provided. +One or more of these must be run. + + +%package fingergw +Summary: Executor finger gateway service +Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} + +%description fingergw +A gateway which provides finger protocol access to live streaming of logs. + +%package migrate +Summary: Migrate zuul v2 and Jenkins Job Builder to Zuul v3 +Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: bubblewrap + +%description migrate +Migrate zuul v2 and Jenkins Job Builder to Zuul v3 + +%package doc +Summary: Zuul documentation + +%description doc +The Zuul HTML documentation + + +%prep +%autosetup -n zuul-%{version} -p1 +sed -i '/^importlib-resources.*/d' requirements.txt +# remove extra uneeded shebangs in zuul_return +# archive on pypi do not preserve links +# could be removed at next release +find zuul/ansible/ -type f -name "zuul_return.py" \ + -exec sed -i '/\/usr\/bin\/python/d' {} \; + +# Inject package version +cat << EOF > zuul/version.py +is_release = True +release_string = "%{version}-%{release}" +class version_info: + def release_string(): + return release_string +EOF +# Fix non compliant shebangs +/usr/bin/pathfix.py -i %{python3} -p -n -k zuul/ansible +install -m 0644 %{SOURCE7} README.fedora +# Fix wrong-file-end-of-line-encoding +sed -i 's/\r$//' LICENSE + +%build +%py3_build + +# Create fake zuul clients suitable for sphinx programoutput +cp zuul/cmd/client.py build/zuul +sed -i '1i \#!/usr/bin/env python3' build/zuul +chmod +x build/zuul +cp zuul/cmd/manage_ansible.py build/zuul-manage-ansible +sed -i '1i \#!/usr/bin/env python3' build/zuul-manage-ansible +chmod +x build/zuul-manage-ansible +# Generate documentation (without release note because source doesn't have git log) +sed -e 's/^ *releasenotes$//' -i doc/source/reference/index.rst +rm doc/source/reference/releasenotes.rst +PYTHONPATH=../../build/lib PATH=$PATH:$(pwd)/build PBR_VERSION=%{version} SPHINX_DEBUG=1 sphinx-build-3 \ + -b html doc/source build/html +# Remove empty stub files +find build -type f -name "*.pyi" -size 0 -delete +# rm doc build leftovers +rm -Rf build/html/.buildinfo build/html/.doctrees + +%install +install -p -d -m 0755 %{buildroot}/%{_datadir}/zuul-ui +mv web/* %{buildroot}/%{_datadir}/zuul-ui/ +%py3_install +rm -Rf %{buildroot}%{python3_sitelib}/zuul/web/static +install -p -D -m 0644 %{SOURCE2} %{buildroot}%{_unitdir}/zuul-scheduler.service +install -p -D -m 0644 %{SOURCE3} %{buildroot}%{_unitdir}/zuul-merger.service +install -p -D -m 0644 %{SOURCE4} %{buildroot}%{_unitdir}/zuul-executor.service +install -p -D -m 0644 %{SOURCE5} %{buildroot}%{_unitdir}/zuul-web.service +install -p -D -m 0644 %{SOURCE6} %{buildroot}%{_unitdir}/zuul-fingergw.service +install -p -D -m 0640 %{SOURCE8} %{buildroot}%{_sysconfdir}/zuul/zuul.conf +install -p -D -m 0640 %{SOURCE9} %{buildroot}%{_sysconfdir}/zuul/logging.conf +install -p -d -m 0700 %{buildroot}%{_sharedstatedir}/zuul +install -p -d -m 0700 %{buildroot}%{_localstatedir}/log/zuul + +# Prepare lib directory +install -p -d -m 0700 %{buildroot}%{_sharedstatedir}/zuul/.ssh +install -p -d -m 0755 %{buildroot}%{_sharedstatedir}/zuul/ansible +install -p -d -m 0755 %{buildroot}%{_sharedstatedir}/zuul/executor +install -p -d -m 0755 %{buildroot}%{_sharedstatedir}/zuul/git +install -p -d -m 0700 %{buildroot}%{_sharedstatedir}/zuul/keys + + +%pre +getent group zuul >/dev/null || groupadd -r zuul +if ! getent passwd zuul >/dev/null; then + useradd -r -g zuul -G zuul -d %{_sharedstatedir}/zuul -s /sbin/nologin -c "Zuul Daemon" zuul +fi +exit 0 + + +%post scheduler +%systemd_post zuul-scheduler.service +%post merger +%systemd_post zuul-merger.service +%post web +%systemd_post zuul-web.service +%post executor +%systemd_post zuul-executor.service +%post fingergw +%systemd_post zuul-fingergw.service + + +%preun scheduler +%systemd_preun zuul-scheduler.service +%preun merger +%systemd_preun zuul-merger.service +%preun web +%systemd_preun zuul-web.service +%preun executor +%systemd_preun zuul-executor.service +%preun fingergw +%systemd_preun zuul-fingergw.service + + +%files +%license LICENSE +%config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/zuul.conf +%config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/logging.conf +%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul +%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/.ssh +%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/ansible +%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/keys +%dir %attr(0755,zuul,zuul) %{_localstatedir}/log/zuul +%{python3_sitelib}/zuul +%{python3_sitelib}/zuul-*.egg-info/ +%{_bindir}/zuul +%{_bindir}/zuul-bwrap + +%files webui +%license LICENSE +%{_datadir}/zuul-ui + +%files scheduler +%license LICENSE +%{_bindir}/zuul-scheduler +%{_unitdir}/zuul-scheduler.service + +%files merger +%license LICENSE +%{_bindir}/zuul-merger +%{_unitdir}/zuul-merger.service + +%files web +%license LICENSE +%{_bindir}/zuul-web +%{_unitdir}/zuul-web.service + +%files executor +%license LICENSE +%{_bindir}/zuul-executor +%{_bindir}/zuul-manage-ansible +%{_unitdir}/zuul-executor.service +%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/ansible +%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/executor + +%files fingergw +%license LICENSE +%{_bindir}/zuul-fingergw +%{_unitdir}/zuul-fingergw.service + +%files migrate +%license LICENSE +%{_bindir}/zuul-migrate + +%files doc +%license LICENSE +%doc build/html README.fedora + + +%changelog +* Wed Mar 11 2020 Fabien Boucher - 3.19.0-1 +- Bump to 3.19.0 +- Fedora rawhide compat + +* Tue Mar 3 2020 Tristan Cacqueray - 3.18.0-1 +- Bump to 3.18.0 +- Use -f argument for services +- Use smart-reconfigure command for scheduler reload + +* Wed Feb 26 2020 Tristan Cacqueray - 3.13.0-2 +- Add security fix + +* Tue Dec 10 2019 Fabien Boucher - 3.13.0-1 +- Bump to 3.13.0 + +* Tue Oct 22 2019 Fabien Boucher - 3.11.1-1 +- Bump to 3.11.1 + +* Mon Sep 23 2019 Tristan Cacqueray - 3.10.2-2 +- Remove SCL leftovers + +* Tue Sep 17 2019 Tristan Cacqueray - 3.10.2-1 +- Add synchronize rsh security fix + +* Thu Aug 15 2019 Tristan Cacqueray - 3.10.1-1 +- Bump to 3.10.1 + +* Mon May 20 2019 Tristan Cacqueray - 3.8.1-3 +- Add merger optimization + +* Tue May 14 2019 Tristan Cacqueray - 3.8.1-2 +- Remove a couple of un-needed patches + +* Wed May 8 2019 Tristan Cacqueray - 3.8.1-1 +- Bump to 3.8.1 +- Remove React service worker + +* Wed Apr 17 2019 Tristan Cacqueray - 3.8.0-1 +- Bump to 3.8.0 + +* Mon Mar 25 2019 Tristan Cacqueray - 3.7.1-1 +- Bump to 3.7.1 + +* Mon Mar 18 2019 Tristan Cacqueray - 3.7.0-1 +- Bump to 3.7.0 + +* Mon Feb 11 2019 Tristan Cacqueray - 3.6.0-1 +- Bump to 3.6.0 + +* Fri Feb 8 2019 Tristan Cacqueray - 3.5.0-2 +- Cherry-pick github fix +- Fix config endpoint + +* Wed Jan 23 2019 Javier Peña - 3.5.0-1 +- Bump to 3.5.0 + +* Wed Jan 2 2019 Tristan Cacqueray - 3.4.0-1 +- Bump to 3.4.0 + +* Thu Nov 29 2018 Tristan Cacqueray - 3.3.1-1 +- Update the openshift resources patches +- Use package version in the zuul.version module + +* Tue Nov 6 2018 Tristan Cacqueray - 3.3.0-1 +- Bump to 3.3.0 + +* Mon Sep 24 2018 Tristan Cacqueray - 3.2.0-6 +- Add react web interface + +* Wed Sep 19 2018 Tristan Cacqueray - 3.2.0-5 +- Bump to latest master + +* Thu Aug 9 2018 Tristan Cacqueray - 3.2.0-4 +- Add timer trigger fix + +* Mon Aug 6 2018 Tristan Cacqueray - 3.2.0-3 +- Add missing canonical name in scheduler status + +* Sat Aug 4 2018 Tristan Cacqueray - 3.2.0-2 +- Add node age to webpage + +* Mon Jul 30 2018 Tristan Cacqueray - 3.2.0-1 +- Bump version + +* Wed Jul 18 2018 Tristan Cacqueray - 3.1.0-5 +- Bump to last master for ui fix + +* Mon Jul 2 2018 Tristan Cacqueray - 3.1.0-4 +- Bump to 3.1.1 tech preview + +* Fri Jun 22 2018 Tristan Cacqueray - 3.1.0-3 +- Add resource connection type patch +- Add patch to get the zuul-scheduler -t validation option +- Add /etc/localtime patch + +* Thu Jun 21 2018 Tristan Cacqueray - 3.1.0-2 +- Fix tenant status page reload issue +- Add glyphicon status balls + +* Fri Jun 15 2018 Tristan Cacqueray - 3.1.0-1 +- Bump version to 3.1.0 +- Add angular6 patch + +* Mon May 28 2018 Tristan Cacqueray - 3.0.2-2 +- Bump ansible patch to 2.5 version + +* Fri Apr 13 2018 Tristan Cacqueray - 3.0.2-1 +- Update version to 3.0.2 release +- Update the MQTT driver patch + +* Fri Apr 13 2018 Tristan Cacqueray - 3.0.1-1 +- Update version to 3.0.1 release +- Add patch to fix tag reporter +- Update the MQTT driver patch + +* Thu Mar 29 2018 Tristan Cacqueray - 3.0.0-26 +- Update version to 3.0.0 release + +* Tue Mar 20 2018 Fabien Boucher - 3.0.0-25 +- Bump version for security fixes +- Add patches for supporting zuul to start with a broken config + +* Fri Mar 16 2018 Tristan Cacqueray - 3.0.0-24 +- Bump version for security fixes +- Add new Nodepool dashboards + +* Thu Mar 15 2018 Tristan Cacqueray - 3.0.0-23 +- Bump version + +* Tue Mar 13 2018 Tristan Cacqueray - 3.0.0-22 +- Bump version + +* Thu Feb 22 2018 Tristan Cacqueray - 3.0.0-21 +- Add missing ansible-2.4 fix + +* Wed Feb 21 2018 Tristan Cacqueray - 3.0.0-20 +- Bump version +- Add branch-column patch +- Fix dynamic config loader + +* Mon Feb 19 2018 Tristan Cacqueray - 3.0.0-19 +- Bump version and switch to ansible 2.4 requirement + +* Wed Feb 07 2018 Tristan Cacqueray - 3.0.0-18 +- Bump version +- Add pipelines.json endpoint + +* Wed Jan 31 2018 Tristan Cacqueray - 3.0.0-17 +- Bump version +- Add external webui built with npm + +* Mon Jan 22 2018 Tristan Cacqueray - 3.0.0-16 +- Bump version +- Add config-loader optimization patch +- Add job page description patch +- Add dynamic config load +- Remove with_restart in systemd unit +- Add doc sub package + +* Wed Dec 27 2017 Tristan Cacqueray - 3.0.0-15 +- Bump version +- Add jmespath to the executor requirements +- Add fingergw sub-package +- Add zookeeper retry logic patch + +* Tue Dec 05 2017 Tristan Cacqueray - 3.0.0-14 +- Bump version + +* Wed Nov 29 2017 Tristan Cacqueray - 3.0.0-13 +- Bump version +- Add MQTT driver patch +- Add log_stream options patch + +* Fri Nov 17 2017 Tristan Cacqueray - 3.0.0-12 +- Force ssh known_host to be in scl user home + +* Mon Nov 6 2017 Tristan Cacqueray - 3.0.0-11 +- Bump version and slightly update the patches + +* Wed Nov 1 2017 Tristan Cacqueray - 3.0.0-10 +- Bump version and remove merged patches + +* Wed Sep 27 2017 Tristan Cacqueray - 3.0.0-9 +- Add newrev patch + +* Thu Sep 14 2017 Tristan Cacqueray - 3.0.0-8 +- Add zuul-web interfaces + +* Tue Sep 05 2017 Tristan Cacqueray - 3.0.0-7 +- Add scheduler StartPost command to wait for gearman server. + +* Fri Aug 25 2017 Tristan Cacqueray - 3.0.0-6 +- Fix zuul-web static file missing from python module +- Fix bwrap usage (LD_LIBRARY_PATH is removed by setuid) +- Bump version + +* Tue Jul 25 2017 Tristan Cacqueray - 3.0.0-5 +- Bump version and fix executor reload + +* Wed Jul 12 2017 Tristan Cacqueray - 3.0.0-4 +- Bump version and add zuul-web package + +* Thu Jul 6 2017 Tristan Cacqueray - 3.0.0-3 +- Remove demonization from service file + +* Thu Jun 29 2017 Tristan Cacqueray - 3.0.0-2 +- Fix service reload + +* Mon Jun 19 2017 Tristan Cacqueray - 3.0.0-1 +- Initial packaging From 18f64deb45ff8132fce70e90eec355b6669eac9d Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 4/23] bypass cheroot req and move foward --- diff --git a/README.fedora b/README.fedora index 66d83f6..4c4dacf 100644 --- a/README.fedora +++ b/README.fedora @@ -1,12 +1,39 @@ TODO(fbo) This is still a draft here. Will be improved once we are clear with runtime deps. +https://zuul-ci.org/docs/zuul/howtos/installation.html + zuul-manage-ansible ------------------- -sudo -u zuul bash -cd -zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin -ls -al /var/lib/zuul/ansible-bin +Prepare Ansible Virtualenv for every supported Ansible versions for the executor service. + +sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" + +zuul-web +-------- + +systemctl start zuul-web +curl http://localhost:9000/api/tenants + +zuul-webui +---------- + +dnf install -y nodejs +curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo +sudo dnf install yarn +cd /usr/share/zuul-ui/ +yarn install +REACT_APP_ZUUL_API='http://zuul-web.example.com' yarn build + +httpd +----- + +dnf install httpd +setsebool -P httpd_can_network_connect on +Add in /etc/httpd/conf.d/zuul.conf +RewriteEngine on +RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] +RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] zuul-scheduler diff --git a/logging.conf b/logging.conf index edfd3f6..d47fa1f 100644 --- a/logging.conf +++ b/logging.conf @@ -24,7 +24,7 @@ qualname=gerrit propagate=0 [handler_console] -level=WARNING +level=DEBUG class=StreamHandler formatter=simple args=(sys.stdout,) @@ -36,7 +36,7 @@ formatter=simple args=('/var/log/zuul/debug.log', 'midnight', 1, 30,) [handler_normal] -level=INFO +level=DEBUG class=logging.handlers.TimedRotatingFileHandler formatter=simple args=('/var/log/zuul/zuul.log', 'midnight', 1, 30,) diff --git a/main.yaml b/main.yaml new file mode 100644 index 0000000..a62fde9 --- /dev/null +++ b/main.yaml @@ -0,0 +1,3 @@ +- tenant: + name: default + source: {} diff --git a/zuul.conf b/zuul.conf index 6a4f09c..13a2c38 100644 --- a/zuul.conf +++ b/zuul.conf @@ -36,10 +36,12 @@ trusted_ro_paths=/opt/zuul-scripts:/var/cache trusted_rw_paths=/opt/zuul-logs [web] +log_config=/etc/zuul/logging.conf listen_address=127.0.0.1 port=9000 static_cache_expiry=0 status_url=https://zuul.example.com/status +static_path=/usr/share/zuul-ui/build [webclient] url=https://zuul.example.com diff --git a/zuul.spec b/zuul.spec index 5c7831e..770bf81 100644 --- a/zuul.spec +++ b/zuul.spec @@ -1,3 +1,5 @@ +%{?python_disable_dependency_generator} + Name: zuul Version: 3.19.0 Release: 1%{?dist} @@ -15,6 +17,7 @@ Source6: zuul-fingergw.service Source7: README.fedora Source8: zuul.conf Source9: logging.conf +Source10: main.yaml Patch01: 0001-Remove-another-shebang-and-remove-useless-exec-bits.patch Patch02: 0001-requirements-add-explicit-reference-to-dateutil.patch @@ -24,16 +27,12 @@ BuildArch: noarch #Error: # Problem: conflicting requests - -# https://src.fedoraproject.org/rpms/python-virtualenv/pull-request/20 -# - nothing provides ((python3.8dist(virtualenv) < 20 or python3.8dist(virtualenv) > 20) with (python3.8dist(virtualenv) < 20.0.1 or python3.8dist(virtualenv) > 20.0.1) with python3.8dist(virtualenv) > 20) needed by zuul-3.19.0-1.fc33.noarch - # https://github.com/cherrypy/cheroot/issues/263 # the zuul cap reason # The proposed fix https://github.com/cherrypy/cheroot/pull/277 # https://bugzilla.redhat.com/show_bug.cgi?id=1834207 # - nothing provides python3.8dist(cheroot) < 8.1 needed by zuul-3.19.0-1.fc33.noarch - # - nothing provides python3.8dist(cherrypy) = 18.3 needed by zuul-3.19.0-1.fc33.noarch +# TODO: https://bugzilla.redhat.com/show_bug.cgi?id=1849654 BuildRequires: python3-devel @@ -42,7 +41,8 @@ BuildRequires: python3-setuptools BuildRequires: python3-zuul-sphinx BuildRequires: python3-snowballstemmer BuildRequires: python3-fixtures -BuildRequires: python3-sphinx-autodoc-typehints +# TODO: https://bugzilla.redhat.com/show_bug.cgi?id=1849654 +# BuildRequires: python3-sphinx-autodoc-typehints BuildRequires: python3-sphinxcontrib-blockdiag BuildRequires: python3-sphinxcontrib-programoutput BuildRequires: python3-sphinxcontrib-openapi @@ -79,10 +79,61 @@ BuildRequires: python3-iso8601 BuildRequires: python3-PyMySQL BuildRequires: python3-psycopg2 BuildRequires: python3-pathspec -# BuildRequires: python3-graphene +BuildRequires: python3-graphene BuildRequires: systemd BuildRequires: ansible +#rpm -qpR results_zuul/3.19.0/1.fc33/zuul-3.19.0-1.fc33.noarch.rpm +Requires: ((python3.9dist(gear) < 0.15 or python3.9dist(gear) > 0.15) with python3.9dist(gear) < 1 with python3.9dist(gear) >= 0.13) +Requires: ((python3.9dist(urllib3) < 1.25.4 or python3.9dist(urllib3) > 1.25.4) with (python3.9dist(urllib3) < 1.25.5 or python3.9dist(urllib3) > 1.25.5)) +Requires: ((python3.9dist(virtualenv) < 20 or python3.9dist(virtualenv) > 20) with (python3.9dist(virtualenv) < 20.0.1 or python3.9dist(virtualenv) > 20.0.1) with python3.9dist(virtualenv) > 20) +Requires: (python3.9dist(prettytable) < 0.8 with python3.9dist(prettytable) >= 0.6) +Requires: /bin/sh +Requires: /usr/bin/python3 +Requires: config(zuul) = 3.19.0-1.fc33 +Requires: python(abi) = 3.9 +Requires: python3-cheroot +Requires: python3.9dist(alembic) +Requires: python3.9dist(apscheduler) >= 3 +Requires: python3.9dist(babel) >= 1 +Requires: python3.9dist(cachecontrol) +Requires: python3.9dist(cachetools) + +#Requires: python3.9dist(cheroot) < 8.1 +#Requires: python3.9dist(cherrypy) = 18.3 +Requires: python3.9dist(cheroot) +Requires: python3.9dist(cherrypy) + +Requires: python3.9dist(cryptography) >= 1.6 +Requires: python3.9dist(extras) +Requires: python3.9dist(fb-re2) >= 1.0.6 +Requires: python3.9dist(github3-py) >= 1.1 +Requires: python3.9dist(gitpython) >= 2.1.8 +Requires: python3.9dist(iso8601) +Requires: python3.9dist(jsonpath-rw) +Requires: python3.9dist(kazoo) +Requires: python3.9dist(netaddr) +Requires: python3.9dist(paho-mqtt) +Requires: python3.9dist(paramiko) >= 2.0.1 +Requires: python3.9dist(pathspec) +Requires: python3.9dist(pbr) >= 1.1 +Requires: python3.9dist(psutil) +Requires: python3.9dist(pyjwt) +Requires: python3.9dist(python-daemon) >= 2.0.4 +Requires: python3.9dist(python-dateutil) +Requires: python3.9dist(pyyaml) >= 3.1 +Requires: python3.9dist(routes) +Requires: python3.9dist(setuptools) +Requires: python3.9dist(sqlalchemy) +Requires: python3.9dist(statsd) >= 3 +Requires: python3.9dist(voluptuous) >= 0.10.2 +Requires: python3.9dist(ws4py) +Requires: rpmlib(CompressedFileNames) <= 3.0.4-1 +Requires: rpmlib(FileDigests) <= 4.6.0-1 +Requires: rpmlib(PartialHardlinkSets) <= 4.0.4-1 +Requires: rpmlib(PayloadFilesHavePrefix) <= 4.0-1 +Requires: rpmlib(PayloadIsZstd) <= 5.4.18-1 +Requires: rpmlib(RichDependencies) <= 4.12.0-1 %description Zuul is a program that drives continuous integration, delivery, @@ -138,6 +189,7 @@ Requires: bubblewrap ## Requires: python3-virtualenv Requires: gcc Requires: python3-devel +Requires: libffi-devel %description executor Scale-out component for executing jobs. At least one of these is @@ -205,6 +257,8 @@ chmod +x build/zuul-manage-ansible # Generate documentation (without release note because source doesn't have git log) sed -e 's/^ *releasenotes$//' -i doc/source/reference/index.rst rm doc/source/reference/releasenotes.rst +# TODO: https://bugzilla.redhat.com/show_bug.cgi?id=1849654 +sed '/sphinx_autodoc_typehints/d' -i doc/source/conf.py PYTHONPATH=../../build/lib PATH=$PATH:$(pwd)/build PBR_VERSION=%{version} SPHINX_DEBUG=1 sphinx-build-3 \ -b html doc/source build/html # Remove empty stub files @@ -224,6 +278,7 @@ install -p -D -m 0644 %{SOURCE5} %{buildroot}%{_unitdir}/zuul-web.service install -p -D -m 0644 %{SOURCE6} %{buildroot}%{_unitdir}/zuul-fingergw.service install -p -D -m 0640 %{SOURCE8} %{buildroot}%{_sysconfdir}/zuul/zuul.conf install -p -D -m 0640 %{SOURCE9} %{buildroot}%{_sysconfdir}/zuul/logging.conf +install -p -D -m 0640 %{SOURCE10} %{buildroot}%{_sysconfdir}/zuul/main.yaml install -p -d -m 0700 %{buildroot}%{_sharedstatedir}/zuul install -p -d -m 0700 %{buildroot}%{_localstatedir}/log/zuul @@ -270,12 +325,15 @@ exit 0 %files %license LICENSE %config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/zuul.conf +%config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/main.yaml %config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/logging.conf %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/.ssh %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/ansible %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/keys -%dir %attr(0755,zuul,zuul) %{_localstatedir}/log/zuul +# Zuul runtime check +# Project key directory /var/lib/zuul/keys must be mode 0700; current mode is 755 +%dir %attr(0700,zuul,zuul) %{_localstatedir}/log/zuul %{python3_sitelib}/zuul %{python3_sitelib}/zuul-*.egg-info/ %{_bindir}/zuul From c6b7f7e9a2bd9227a6ee1687bd373fbc07f31868 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 5/23] Improve README.fedora --- diff --git a/0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch b/0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch new file mode 100644 index 0000000..da2689e --- /dev/null +++ b/0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch @@ -0,0 +1,54 @@ +From b27dc5df5dc1617fc2f1d438611b87b3e63383c4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Mon, 1 Jun 2020 12:49:07 +0200 +Subject: [PATCH] Replace deprecated Thread.isAlive() with Thread.is_alive() + +The isAlive() method of threading.Thread has been removed in Python 3.9. +The is_alive() method is available on Python 2.6+. + +See https://bugs.python.org/issue37804 + +Change-Id: I951b1ae331c3101722fe34babf81d6f82d838380 +--- + zuul/ansible/base/library/command.py | 4 ++-- + zuul/lib/log_streamer.py | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/zuul/ansible/base/library/command.py b/zuul/ansible/base/library/command.py +index a3b969d9..0c461b3a 100755 +--- a/zuul/ansible/base/library/command.py ++++ b/zuul/ansible/base/library/command.py +@@ -474,7 +474,7 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex + if t: + t.join(10) + with Console(zuul_log_id) as console: +- if t.isAlive(): ++ if t.is_alive(): + console.addLine("[Zuul] standard output/error still open " + "after child exited") + # ZUUL: stdout and stderr are in the console log file +@@ -495,7 +495,7 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex + finally: + if t: + with Console(zuul_log_id) as console: +- if t.isAlive(): ++ if t.is_alive(): + console.addLine("[Zuul] standard output/error still open " + "after child exited") + if fail_json_kwargs: +diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py +index 16b72227..9ed124c5 100644 +--- a/zuul/lib/log_streamer.py ++++ b/zuul/lib/log_streamer.py +@@ -181,7 +181,7 @@ class LogStreamer(object): + raise + + def stop(self): +- if self.thd.isAlive(): ++ if self.thd.is_alive(): + self.server.shutdown() + self.server.server_close() + self.thd.join() +-- +2.25.4 + diff --git a/README.fedora b/README.fedora index 4c4dacf..3fb0da4 100644 --- a/README.fedora +++ b/README.fedora @@ -1,79 +1,125 @@ -TODO(fbo) This is still a draft here. Will be improved once we are clear with runtime deps. +# Getting started with Zuul on Fedora -https://zuul-ci.org/docs/zuul/howtos/installation.html +## Upstream documentation -zuul-manage-ansible -------------------- +The upstream documentation is available at that page: https://zuul-ci.org/docs/zuul/index.html -Prepare Ansible Virtualenv for every supported Ansible versions for the executor service. +## Installation -sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" +This section describes the installation process of the minimal set of Zuul components to +get a working Zuul deployment. Please refer to the upstream documentation for advanced +setup. -zuul-web --------- +### Zookeeper -systemctl start zuul-web -curl http://localhost:9000/api/tenants +TODO -zuul-webui ----------- +### Install Zuul components -dnf install -y nodejs -curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo -sudo dnf install yarn -cd /usr/share/zuul-ui/ -yarn install -REACT_APP_ZUUL_API='http://zuul-web.example.com' yarn build +To install the packages run: -httpd ------ + $ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui -dnf install httpd -setsebool -P httpd_can_network_connect on -Add in /etc/httpd/conf.d/zuul.conf -RewriteEngine on -RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] -RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] +### Install and setup postgresql + $ sudo dnf install -y posgresql python3-psycopg2 + $ su - postgres + $ psql + ALTER USER postgres WITH PASSWORD 'mypassword'; + $ createdb --owner=postgres zuul + $ exit -zuul-scheduler --------------- +Update the local access setting: -Add in /etc/zuul/main.yaml: -- tenant: - name: demo-tenant - source: - local_git: - config-projects: - - zuul-config + $ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' + $ sudo systemctl restart posgreql -Install zookeeper with nodepool tests.yml -Comment MQTT section from zuul.conf -Comment Gerrit section -Comment the MySQL section +Validate server connection by running: -systemctl start zuul-scheduler + $ psql -h 127.0.0.1 -U postgres -W zuul -zuul-executor -------------- +### Update the zuul configuration to define the sql connection -systemctl start zuul-executor +In /etc/zuul/zuul.conf add the following: + [connection sqlreporter] + driver=sql + dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul -prepare basic config --------------------- +### Setup Ansible virtual environment for the Zuul executor -In zuul.conf add: -[connection local_git] -driver=git -baseurl=http://localhost:8000/home/fedora -poll_delay=300 +The Zuul executor is the component in charge of running Zuul Job. A Zuul job is +a set of Ansible playbook. The Zuul executor support multiple version of Ansible. +Zuul provides a tool to manage the Ansible virtual environments. To initialize them +run: + $ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" + +### Build the Zuul web UI + +The Zuul web UI is a React application. The Zuul packaging provide the source code of +the application. The following process describe how to compile the source to get a +production build: + + $ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo + $ sudo dnf install yarn + $ cd /usr/share/zuul-ui/ + $ yarn install + $ REACT_APP_ZUUL_API='' yarn build + +Replace by the URL where the zuul-web API is listening. + +### Start the scheduler and executor + +To do so run: + + $ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" + $ sudo systemctl start zuul-scheduler + $ sudo systemctl start zuul-executor + +The services logs are available into /var/log/zuul/zuul.log. + +### Setup the web API + +Install Apache: + + $ sudo dnf install httpd + $ setsebool -P httpd_can_network_connect on + +Then setup the reverve proxy by adding to /etc/httpd/conf.d/zuul.conf the +following content: + + RewriteEngine on + RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] + RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] + +Reload Apache: + + $ sudo systemctl relead httpd + +Finally start the zuul-web process + + $ sudo systemctl start zuul-web + +Validate the direct access to the API by running: + + $ curl http://localhost:9000/api/tenants + +Validate the access to the API via the reverse proxy: + + $ curl http:///api/tenants + +## Run a first Zuul job + +### Prepare a Git repository to host the CI configuration + +We need to prepare a local Git repository to host the pipelines and jobs configuration. +Follow the process below: + + $ git init /home/fedora/zuul-config + +In /home/fedora/zuul-config/.zuul.yaml add the following content: -pip3 install --user dulwich -dulwich web-daemon -l 0.0.0.0 / -git init /home/fedora/zuul-config -vim /home/fedora/zuul-config/.zuul.yaml - pipeline: name: periodic post-review: true @@ -83,15 +129,72 @@ vim /home/fedora/zuul-config/.zuul.yaml trigger: timer: - time: '* * * * *' + success: + sqlreporter: + failure: + sqlreporter: + +- job: + name: my-noop + description: Minimal working job + parent: null + run: my-noop.yaml - project: periodic: jobs: - - noop -git config user.name "John Doe" -git config user.email "john@localhost" -git add .zuul.yaml -git commit -m"Init demo config" - -Dans les logs /var/log/zuul/zuul.log -zuul.ExecutorClient: [e: 3f6a0ad4b46f4a9aa23744e3e6b9eea5] Execute job noop + - my-noop + +Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: + +--- +- hosts: localhost + tasks: + - name: List working directory + command: ls -al {{ ansible_user_dir }} + - name: Sleep 30 seconds + wait_for: + timeout: 30 + +Then commit the configuration by running: + + $ cd /home/fedora/zuul-config/ + $ git config user.name "John Doe" + $ git config user.email "john@localhost" + $ git add -A . + $ git commit -m"Init demo config" + + +### Run a simple Git deamon to serve the config repository + + $ dulwich web-daemon -l 0.0.0.0 / + +### Add the new connection in the Zuul configuration + +In /etc/zuul/zuul.conf add the following: + + [connection local_git] + driver=git + baseurl=http://localhost:8000/home/fedora + poll_delay=300 + +### Setup the Zuul tenant configuration file + +In /etc/zuul/main.yaml add the following: + + - tenant: + name: default + source: + local_git: + config-projects: + - zuul-config + +### Restart the Zuul services + + $ sudo systemctl restart zuul-* + +## Access the web ui + + Access the build page, where you should see the periodic job my-noop runs + + http:///t/default/builds diff --git a/zuul.conf b/zuul.conf index 13a2c38..b7d7d06 100644 --- a/zuul.conf +++ b/zuul.conf @@ -32,8 +32,8 @@ git_dir=/var/lib/zuul/git [executor] default_username=zuul log_config=/etc/zuul/logging.conf -trusted_ro_paths=/opt/zuul-scripts:/var/cache -trusted_rw_paths=/opt/zuul-logs +;trusted_ro_paths=/opt/zuul-scripts:/var/cache +;trusted_rw_paths=/opt/zuul-logs [web] log_config=/etc/zuul/logging.conf diff --git a/zuul.spec b/zuul.spec index 770bf81..500b619 100644 --- a/zuul.spec +++ b/zuul.spec @@ -21,6 +21,7 @@ Source10: main.yaml Patch01: 0001-Remove-another-shebang-and-remove-useless-exec-bits.patch Patch02: 0001-requirements-add-explicit-reference-to-dateutil.patch +Patch03: 0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch BuildArch: noarch @@ -172,6 +173,7 @@ This component is optional (zero or more of these can be run). %package web Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: nodejs Summary: The Zuul web service %description web @@ -243,6 +245,11 @@ EOF install -m 0644 %{SOURCE7} README.fedora # Fix wrong-file-end-of-line-encoding sed -i 's/\r$//' LICENSE +# Fix 0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch not fully apply +# due to pypi archive removing the symlinks +cp zuul/ansible/base/library/command.py zuul/ansible/2.7/library/ +cp zuul/ansible/base/library/command.py zuul/ansible/2.8/library/ +cp zuul/ansible/base/library/command.py zuul/ansible/2.9/library/ %build %py3_build @@ -330,7 +337,7 @@ exit 0 %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/.ssh %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/ansible -%dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/keys +%dir %attr(0700,zuul,zuul) %{_sharedstatedir}/zuul/keys # Zuul runtime check # Project key directory /var/lib/zuul/keys must be mode 0700; current mode is 755 %dir %attr(0700,zuul,zuul) %{_localstatedir}/log/zuul From c5454940e74ce3bfcb8974f26e5a0bf4b7985fdf Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 6/23] Readme update --- diff --git a/README.fedora b/README.fedora deleted file mode 100644 index 3fb0da4..0000000 --- a/README.fedora +++ /dev/null @@ -1,200 +0,0 @@ -# Getting started with Zuul on Fedora - -## Upstream documentation - -The upstream documentation is available at that page: https://zuul-ci.org/docs/zuul/index.html - -## Installation - -This section describes the installation process of the minimal set of Zuul components to -get a working Zuul deployment. Please refer to the upstream documentation for advanced -setup. - -### Zookeeper - -TODO - -### Install Zuul components - -To install the packages run: - - $ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui - -### Install and setup postgresql - - $ sudo dnf install -y posgresql python3-psycopg2 - $ su - postgres - $ psql - ALTER USER postgres WITH PASSWORD 'mypassword'; - $ createdb --owner=postgres zuul - $ exit - -Update the local access setting: - - $ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' - $ sudo systemctl restart posgreql - -Validate server connection by running: - - $ psql -h 127.0.0.1 -U postgres -W zuul - -### Update the zuul configuration to define the sql connection - -In /etc/zuul/zuul.conf add the following: - - [connection sqlreporter] - driver=sql - dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul - -### Setup Ansible virtual environment for the Zuul executor - -The Zuul executor is the component in charge of running Zuul Job. A Zuul job is -a set of Ansible playbook. The Zuul executor support multiple version of Ansible. -Zuul provides a tool to manage the Ansible virtual environments. To initialize them -run: - - $ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" - -### Build the Zuul web UI - -The Zuul web UI is a React application. The Zuul packaging provide the source code of -the application. The following process describe how to compile the source to get a -production build: - - $ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo - $ sudo dnf install yarn - $ cd /usr/share/zuul-ui/ - $ yarn install - $ REACT_APP_ZUUL_API='' yarn build - -Replace by the URL where the zuul-web API is listening. - -### Start the scheduler and executor - -To do so run: - - $ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" - $ sudo systemctl start zuul-scheduler - $ sudo systemctl start zuul-executor - -The services logs are available into /var/log/zuul/zuul.log. - -### Setup the web API - -Install Apache: - - $ sudo dnf install httpd - $ setsebool -P httpd_can_network_connect on - -Then setup the reverve proxy by adding to /etc/httpd/conf.d/zuul.conf the -following content: - - RewriteEngine on - RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] - RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] - -Reload Apache: - - $ sudo systemctl relead httpd - -Finally start the zuul-web process - - $ sudo systemctl start zuul-web - -Validate the direct access to the API by running: - - $ curl http://localhost:9000/api/tenants - -Validate the access to the API via the reverse proxy: - - $ curl http:///api/tenants - -## Run a first Zuul job - -### Prepare a Git repository to host the CI configuration - -We need to prepare a local Git repository to host the pipelines and jobs configuration. -Follow the process below: - - $ git init /home/fedora/zuul-config - -In /home/fedora/zuul-config/.zuul.yaml add the following content: - -- pipeline: - name: periodic - post-review: true - description: Jobs in this queue are triggered every minute. - manager: independent - precedence: low - trigger: - timer: - - time: '* * * * *' - success: - sqlreporter: - failure: - sqlreporter: - -- job: - name: my-noop - description: Minimal working job - parent: null - run: my-noop.yaml - -- project: - periodic: - jobs: - - my-noop - -Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: - ---- -- hosts: localhost - tasks: - - name: List working directory - command: ls -al {{ ansible_user_dir }} - - name: Sleep 30 seconds - wait_for: - timeout: 30 - -Then commit the configuration by running: - - $ cd /home/fedora/zuul-config/ - $ git config user.name "John Doe" - $ git config user.email "john@localhost" - $ git add -A . - $ git commit -m"Init demo config" - - -### Run a simple Git deamon to serve the config repository - - $ dulwich web-daemon -l 0.0.0.0 / - -### Add the new connection in the Zuul configuration - -In /etc/zuul/zuul.conf add the following: - - [connection local_git] - driver=git - baseurl=http://localhost:8000/home/fedora - poll_delay=300 - -### Setup the Zuul tenant configuration file - -In /etc/zuul/main.yaml add the following: - - - tenant: - name: default - source: - local_git: - config-projects: - - zuul-config - -### Restart the Zuul services - - $ sudo systemctl restart zuul-* - -## Access the web ui - - Access the build page, where you should see the periodic job my-noop runs - - http:///t/default/builds diff --git a/README.fedora.md b/README.fedora.md new file mode 100644 index 0000000..ace395e --- /dev/null +++ b/README.fedora.md @@ -0,0 +1,229 @@ +# Getting started with Zuul on Fedora + +## Upstream documentation + +The upstream documentation is available at: https://zuul-ci.org/docs/zuul/index.html + +## Installation + +This section describes the installation process of the minimal set of Zuul components to +get a working Zuul deployment. Please refer to the upstream documentation for advanced +setup. + +Zuul requires a Web, a Zookeeper and SQL server running. This documentation covers all the step +to bootstrap Zuul. + +### Install and setup Zookeeper + +This process describes the minimal steps to get a Zookeeper service running. You +should refer to the Zookeeper documentation to get a production setup if needed. + + $ mkdir /tmp/zookeeper && cd /tmp/zookeeper + $ curl -OL https://downloads.apache.org/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz + $ tar -xvzf apache-zookeeper-3.6.1-bin.tar.gz + $ cp apache-zookeeper-3.6.1-bin/conf/zoo_sample.cfg apache-zookeeper-3.6.1-bin/conf/zoo.cfg + $ sudo apache-zookeeper-3.6.1-bin/bin/zkServer.sh start + +### Install and setup postgresql + +This process describes the minimal steps to get a postgres service running. You +should refer to the postgrres documentation to get a production setup if needed. + + $ sudo dnf install -y posgresql python3-psycopg2 + $ su - postgres + $ psql + ALTER USER postgres WITH PASSWORD 'mypassword'; + $ createdb --owner=postgres zuul + $ exit + +Update the local access setting: + + $ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' + $ sudo systemctl restart posgreql + +Validate server connection by running: + + $ psql -h 127.0.0.1 -U postgres -W zuul + +### Install Zuul components + +To install the packages run: + + $ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui + +### Update the zuul configuration to define the sql connection + +In /etc/zuul/zuul.conf add the following: + + [connection sqlreporter] + driver=sql + dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul + +### Setup Ansible virtual environment for the Zuul executor + +The Zuul executor is the component in charge of running Zuul Jobs. A Zuul job is +a set of Ansible playbook. The Zuul executor supports multiple version of Ansible. +Zuul provides a tool to manage the Ansible virtual environments. To initialize them +run: + + $ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" + +### Build the Zuul web UI + +The Zuul web UI is a React application. The Zuul packaging provides the source code of +the application. The following process describe how to compile the source to get a +production build: + + $ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo + $ sudo dnf install yarn + $ cd /usr/share/zuul-ui/ + $ yarn install + $ REACT_APP_ZUUL_API='' yarn build + +Replace by the URL where the zuul-web API is listening. + +### Start the scheduler and executor + +To do so run: + + $ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" + $ sudo systemctl start zuul-scheduler + $ sudo systemctl start zuul-executor + +The services logs are available into /var/log/zuul/zuul.log. + +### Setup the web API + +Install Apache: + + $ sudo dnf install httpd + $ setsebool -P httpd_can_network_connect on + +Then setup the reverve proxy by adding to /etc/httpd/conf.d/zuul.conf the +following content: + + RewriteEngine on + RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] + RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] + +Reload Apache: + + $ sudo systemctl relead httpd + +Finally start the zuul-web process + + $ sudo systemctl start zuul-web + +Validate the direct access to the API by running: + + $ curl http://localhost:9000/api/tenants + +Validate the access to the API via the reverse proxy: + + $ curl http:///api/tenants + +## Run a first Zuul job + +In this section, you'll find the steps to get a first periodic job running. + +Please note that, this is a really basic configuration to ensure the services are working +as expected. To go further, please refer to the Zuul documentation to learn how to connect +Zuul to Code Review systems such as Github, Gerrit, Pagure. This demo uses the local system +to run jobs but this has serious limitation and should not be used in production. +Thus Nodepool must be used as the nodes and containers provider for Zuul. +Nodepool supports various cloud providers. Nodepool is packaged into Fedora and here is +its documentaion: https://zuul-ci.org/docs/nodepool/ + +### Prepare a Git repository to host the CI configuration + +We need to prepare a local Git repository to host the pipelines and jobs configuration. + +Follow the process below: + + $ git init /home/fedora/zuul-config + +In /home/fedora/zuul-config/.zuul.yaml add the following content: + +- pipeline: + name: periodic + post-review: true + description: Jobs in this queue are triggered every minute. + manager: independent + precedence: low + trigger: + timer: + - time: '* * * * *' + success: + sqlreporter: + failure: + sqlreporter: + +- job: + name: my-noop + description: Minimal working job + parent: null + run: my-noop.yaml + +- project: + periodic: + jobs: + - my-noop + +Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: + +--- +- hosts: localhost + tasks: + - name: List working directory + command: ls -al {{ ansible_user_dir }} + - name: Sleep 30 seconds + wait_for: + timeout: 30 + +Then commit the configuration by running: + + $ cd /home/fedora/zuul-config/ + $ git config user.name "John Doe" + $ git config user.email "john@localhost" + $ git add -A . + $ git commit -m"Init demo config" + + +### Run a Git deamon to serve the config repository + + $ dulwich web-daemon -l 0.0.0.0 / + +### Add the new connection in the Zuul configuration + +In /etc/zuul/zuul.conf add the following: + + [connection local_git] + driver=git + baseurl=http://localhost:8000/home/fedora + poll_delay=300 + +### Setup the Zuul tenant configuration file + +In /etc/zuul/main.yaml add the following: + + - tenant: + name: default + source: + local_git: + config-projects: + - zuul-config + +### Restart the Zuul services + + $ sudo systemctl restart zuul-* + +### Verify the job executed periodically + +Run the following command to access the builds API endpoint and ensure the +my-noop job run as expected every minutes w/o errors. + + $ curl http:///api/tenant/default/builds | python -m json.tool + +Access the build page, where you should see the periodic job my-noop runs + + http:///t/default/builds From d44c8599804cc56642adcd3f284176b7d2af59be Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 7/23] Fix README path --- diff --git a/README.fedora.md b/README.fedora.md deleted file mode 100644 index ace395e..0000000 --- a/README.fedora.md +++ /dev/null @@ -1,229 +0,0 @@ -# Getting started with Zuul on Fedora - -## Upstream documentation - -The upstream documentation is available at: https://zuul-ci.org/docs/zuul/index.html - -## Installation - -This section describes the installation process of the minimal set of Zuul components to -get a working Zuul deployment. Please refer to the upstream documentation for advanced -setup. - -Zuul requires a Web, a Zookeeper and SQL server running. This documentation covers all the step -to bootstrap Zuul. - -### Install and setup Zookeeper - -This process describes the minimal steps to get a Zookeeper service running. You -should refer to the Zookeeper documentation to get a production setup if needed. - - $ mkdir /tmp/zookeeper && cd /tmp/zookeeper - $ curl -OL https://downloads.apache.org/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz - $ tar -xvzf apache-zookeeper-3.6.1-bin.tar.gz - $ cp apache-zookeeper-3.6.1-bin/conf/zoo_sample.cfg apache-zookeeper-3.6.1-bin/conf/zoo.cfg - $ sudo apache-zookeeper-3.6.1-bin/bin/zkServer.sh start - -### Install and setup postgresql - -This process describes the minimal steps to get a postgres service running. You -should refer to the postgrres documentation to get a production setup if needed. - - $ sudo dnf install -y posgresql python3-psycopg2 - $ su - postgres - $ psql - ALTER USER postgres WITH PASSWORD 'mypassword'; - $ createdb --owner=postgres zuul - $ exit - -Update the local access setting: - - $ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' - $ sudo systemctl restart posgreql - -Validate server connection by running: - - $ psql -h 127.0.0.1 -U postgres -W zuul - -### Install Zuul components - -To install the packages run: - - $ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui - -### Update the zuul configuration to define the sql connection - -In /etc/zuul/zuul.conf add the following: - - [connection sqlreporter] - driver=sql - dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul - -### Setup Ansible virtual environment for the Zuul executor - -The Zuul executor is the component in charge of running Zuul Jobs. A Zuul job is -a set of Ansible playbook. The Zuul executor supports multiple version of Ansible. -Zuul provides a tool to manage the Ansible virtual environments. To initialize them -run: - - $ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" - -### Build the Zuul web UI - -The Zuul web UI is a React application. The Zuul packaging provides the source code of -the application. The following process describe how to compile the source to get a -production build: - - $ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo - $ sudo dnf install yarn - $ cd /usr/share/zuul-ui/ - $ yarn install - $ REACT_APP_ZUUL_API='' yarn build - -Replace by the URL where the zuul-web API is listening. - -### Start the scheduler and executor - -To do so run: - - $ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" - $ sudo systemctl start zuul-scheduler - $ sudo systemctl start zuul-executor - -The services logs are available into /var/log/zuul/zuul.log. - -### Setup the web API - -Install Apache: - - $ sudo dnf install httpd - $ setsebool -P httpd_can_network_connect on - -Then setup the reverve proxy by adding to /etc/httpd/conf.d/zuul.conf the -following content: - - RewriteEngine on - RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] - RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] - -Reload Apache: - - $ sudo systemctl relead httpd - -Finally start the zuul-web process - - $ sudo systemctl start zuul-web - -Validate the direct access to the API by running: - - $ curl http://localhost:9000/api/tenants - -Validate the access to the API via the reverse proxy: - - $ curl http:///api/tenants - -## Run a first Zuul job - -In this section, you'll find the steps to get a first periodic job running. - -Please note that, this is a really basic configuration to ensure the services are working -as expected. To go further, please refer to the Zuul documentation to learn how to connect -Zuul to Code Review systems such as Github, Gerrit, Pagure. This demo uses the local system -to run jobs but this has serious limitation and should not be used in production. -Thus Nodepool must be used as the nodes and containers provider for Zuul. -Nodepool supports various cloud providers. Nodepool is packaged into Fedora and here is -its documentaion: https://zuul-ci.org/docs/nodepool/ - -### Prepare a Git repository to host the CI configuration - -We need to prepare a local Git repository to host the pipelines and jobs configuration. - -Follow the process below: - - $ git init /home/fedora/zuul-config - -In /home/fedora/zuul-config/.zuul.yaml add the following content: - -- pipeline: - name: periodic - post-review: true - description: Jobs in this queue are triggered every minute. - manager: independent - precedence: low - trigger: - timer: - - time: '* * * * *' - success: - sqlreporter: - failure: - sqlreporter: - -- job: - name: my-noop - description: Minimal working job - parent: null - run: my-noop.yaml - -- project: - periodic: - jobs: - - my-noop - -Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: - ---- -- hosts: localhost - tasks: - - name: List working directory - command: ls -al {{ ansible_user_dir }} - - name: Sleep 30 seconds - wait_for: - timeout: 30 - -Then commit the configuration by running: - - $ cd /home/fedora/zuul-config/ - $ git config user.name "John Doe" - $ git config user.email "john@localhost" - $ git add -A . - $ git commit -m"Init demo config" - - -### Run a Git deamon to serve the config repository - - $ dulwich web-daemon -l 0.0.0.0 / - -### Add the new connection in the Zuul configuration - -In /etc/zuul/zuul.conf add the following: - - [connection local_git] - driver=git - baseurl=http://localhost:8000/home/fedora - poll_delay=300 - -### Setup the Zuul tenant configuration file - -In /etc/zuul/main.yaml add the following: - - - tenant: - name: default - source: - local_git: - config-projects: - - zuul-config - -### Restart the Zuul services - - $ sudo systemctl restart zuul-* - -### Verify the job executed periodically - -Run the following command to access the builds API endpoint and ensure the -my-noop job run as expected every minutes w/o errors. - - $ curl http:///api/tenant/default/builds | python -m json.tool - -Access the build page, where you should see the periodic job my-noop runs - - http:///t/default/builds diff --git a/README.md b/README.md new file mode 100644 index 0000000..ace395e --- /dev/null +++ b/README.md @@ -0,0 +1,229 @@ +# Getting started with Zuul on Fedora + +## Upstream documentation + +The upstream documentation is available at: https://zuul-ci.org/docs/zuul/index.html + +## Installation + +This section describes the installation process of the minimal set of Zuul components to +get a working Zuul deployment. Please refer to the upstream documentation for advanced +setup. + +Zuul requires a Web, a Zookeeper and SQL server running. This documentation covers all the step +to bootstrap Zuul. + +### Install and setup Zookeeper + +This process describes the minimal steps to get a Zookeeper service running. You +should refer to the Zookeeper documentation to get a production setup if needed. + + $ mkdir /tmp/zookeeper && cd /tmp/zookeeper + $ curl -OL https://downloads.apache.org/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz + $ tar -xvzf apache-zookeeper-3.6.1-bin.tar.gz + $ cp apache-zookeeper-3.6.1-bin/conf/zoo_sample.cfg apache-zookeeper-3.6.1-bin/conf/zoo.cfg + $ sudo apache-zookeeper-3.6.1-bin/bin/zkServer.sh start + +### Install and setup postgresql + +This process describes the minimal steps to get a postgres service running. You +should refer to the postgrres documentation to get a production setup if needed. + + $ sudo dnf install -y posgresql python3-psycopg2 + $ su - postgres + $ psql + ALTER USER postgres WITH PASSWORD 'mypassword'; + $ createdb --owner=postgres zuul + $ exit + +Update the local access setting: + + $ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' + $ sudo systemctl restart posgreql + +Validate server connection by running: + + $ psql -h 127.0.0.1 -U postgres -W zuul + +### Install Zuul components + +To install the packages run: + + $ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui + +### Update the zuul configuration to define the sql connection + +In /etc/zuul/zuul.conf add the following: + + [connection sqlreporter] + driver=sql + dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul + +### Setup Ansible virtual environment for the Zuul executor + +The Zuul executor is the component in charge of running Zuul Jobs. A Zuul job is +a set of Ansible playbook. The Zuul executor supports multiple version of Ansible. +Zuul provides a tool to manage the Ansible virtual environments. To initialize them +run: + + $ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" + +### Build the Zuul web UI + +The Zuul web UI is a React application. The Zuul packaging provides the source code of +the application. The following process describe how to compile the source to get a +production build: + + $ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo + $ sudo dnf install yarn + $ cd /usr/share/zuul-ui/ + $ yarn install + $ REACT_APP_ZUUL_API='' yarn build + +Replace by the URL where the zuul-web API is listening. + +### Start the scheduler and executor + +To do so run: + + $ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" + $ sudo systemctl start zuul-scheduler + $ sudo systemctl start zuul-executor + +The services logs are available into /var/log/zuul/zuul.log. + +### Setup the web API + +Install Apache: + + $ sudo dnf install httpd + $ setsebool -P httpd_can_network_connect on + +Then setup the reverve proxy by adding to /etc/httpd/conf.d/zuul.conf the +following content: + + RewriteEngine on + RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] + RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] + +Reload Apache: + + $ sudo systemctl relead httpd + +Finally start the zuul-web process + + $ sudo systemctl start zuul-web + +Validate the direct access to the API by running: + + $ curl http://localhost:9000/api/tenants + +Validate the access to the API via the reverse proxy: + + $ curl http:///api/tenants + +## Run a first Zuul job + +In this section, you'll find the steps to get a first periodic job running. + +Please note that, this is a really basic configuration to ensure the services are working +as expected. To go further, please refer to the Zuul documentation to learn how to connect +Zuul to Code Review systems such as Github, Gerrit, Pagure. This demo uses the local system +to run jobs but this has serious limitation and should not be used in production. +Thus Nodepool must be used as the nodes and containers provider for Zuul. +Nodepool supports various cloud providers. Nodepool is packaged into Fedora and here is +its documentaion: https://zuul-ci.org/docs/nodepool/ + +### Prepare a Git repository to host the CI configuration + +We need to prepare a local Git repository to host the pipelines and jobs configuration. + +Follow the process below: + + $ git init /home/fedora/zuul-config + +In /home/fedora/zuul-config/.zuul.yaml add the following content: + +- pipeline: + name: periodic + post-review: true + description: Jobs in this queue are triggered every minute. + manager: independent + precedence: low + trigger: + timer: + - time: '* * * * *' + success: + sqlreporter: + failure: + sqlreporter: + +- job: + name: my-noop + description: Minimal working job + parent: null + run: my-noop.yaml + +- project: + periodic: + jobs: + - my-noop + +Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: + +--- +- hosts: localhost + tasks: + - name: List working directory + command: ls -al {{ ansible_user_dir }} + - name: Sleep 30 seconds + wait_for: + timeout: 30 + +Then commit the configuration by running: + + $ cd /home/fedora/zuul-config/ + $ git config user.name "John Doe" + $ git config user.email "john@localhost" + $ git add -A . + $ git commit -m"Init demo config" + + +### Run a Git deamon to serve the config repository + + $ dulwich web-daemon -l 0.0.0.0 / + +### Add the new connection in the Zuul configuration + +In /etc/zuul/zuul.conf add the following: + + [connection local_git] + driver=git + baseurl=http://localhost:8000/home/fedora + poll_delay=300 + +### Setup the Zuul tenant configuration file + +In /etc/zuul/main.yaml add the following: + + - tenant: + name: default + source: + local_git: + config-projects: + - zuul-config + +### Restart the Zuul services + + $ sudo systemctl restart zuul-* + +### Verify the job executed periodically + +Run the following command to access the builds API endpoint and ensure the +my-noop job run as expected every minutes w/o errors. + + $ curl http:///api/tenant/default/builds | python -m json.tool + +Access the build page, where you should see the periodic job my-noop runs + + http:///t/default/builds diff --git a/zuul.spec b/zuul.spec index 500b619..ea4bb25 100644 --- a/zuul.spec +++ b/zuul.spec @@ -14,7 +14,7 @@ Source3: zuul-merger.service Source4: zuul-executor.service Source5: zuul-web.service Source6: zuul-fingergw.service -Source7: README.fedora +Source7: README.md Source8: zuul.conf Source9: logging.conf Source10: main.yaml From 4e329e7ae7972af9d1edf19d2aa174e84efcbcf3 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 8/23] Fix markdown --- diff --git a/README.md b/README.md index ace395e..1deef3f 100644 --- a/README.md +++ b/README.md @@ -18,46 +18,58 @@ to bootstrap Zuul. This process describes the minimal steps to get a Zookeeper service running. You should refer to the Zookeeper documentation to get a production setup if needed. - $ mkdir /tmp/zookeeper && cd /tmp/zookeeper - $ curl -OL https://downloads.apache.org/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz - $ tar -xvzf apache-zookeeper-3.6.1-bin.tar.gz - $ cp apache-zookeeper-3.6.1-bin/conf/zoo_sample.cfg apache-zookeeper-3.6.1-bin/conf/zoo.cfg - $ sudo apache-zookeeper-3.6.1-bin/bin/zkServer.sh start +``` +$ mkdir /tmp/zookeeper && cd /tmp/zookeeper +$ curl -OL https://downloads.apache.org/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz +$ tar -xvzf apache-zookeeper-3.6.1-bin.tar.gz +$ cp apache-zookeeper-3.6.1-bin/conf/zoo_sample.cfg apache-zookeeper-3.6.1-bin/conf/zoo.cfg +$ sudo apache-zookeeper-3.6.1-bin/bin/zkServer.sh start +``` ### Install and setup postgresql This process describes the minimal steps to get a postgres service running. You should refer to the postgrres documentation to get a production setup if needed. - $ sudo dnf install -y posgresql python3-psycopg2 - $ su - postgres - $ psql - ALTER USER postgres WITH PASSWORD 'mypassword'; - $ createdb --owner=postgres zuul - $ exit +``` +$ sudo dnf install -y posgresql python3-psycopg2 +$ su - postgres +$ psql + ALTER USER postgres WITH PASSWORD 'mypassword'; +$ createdb --owner=postgres zuul +$ exit +``` Update the local access setting: - $ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' - $ sudo systemctl restart posgreql +``` +$ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' +$ sudo systemctl restart posgreql +``` Validate server connection by running: - $ psql -h 127.0.0.1 -U postgres -W zuul +``` +$ psql -h 127.0.0.1 -U postgres -W zuul +``` ### Install Zuul components To install the packages run: - $ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui +``` +$ sudo dnf install zuul-scheduler zuul-executor zuul-web zuul-webui +``` ### Update the zuul configuration to define the sql connection In /etc/zuul/zuul.conf add the following: - [connection sqlreporter] - driver=sql - dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul +``` +[connection sqlreporter] +driver=sql +dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul +``` ### Setup Ansible virtual environment for the Zuul executor @@ -66,7 +78,9 @@ a set of Ansible playbook. The Zuul executor supports multiple version of Ansibl Zuul provides a tool to manage the Ansible virtual environments. To initialize them run: - $ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" +``` +$ sudo -u zuul bash -c "cd && zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin" +``` ### Build the Zuul web UI @@ -74,11 +88,13 @@ The Zuul web UI is a React application. The Zuul packaging provides the source c the application. The following process describe how to compile the source to get a production build: - $ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo - $ sudo dnf install yarn - $ cd /usr/share/zuul-ui/ - $ yarn install - $ REACT_APP_ZUUL_API='' yarn build +``` +$ curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo +$ sudo dnf install yarn +$ cd /usr/share/zuul-ui/ +$ yarn install +$ REACT_APP_ZUUL_API='' yarn build +``` Replace by the URL where the zuul-web API is listening. @@ -86,9 +102,11 @@ Replace by the URL where the zuul-web API is listening. To do so run: - $ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" - $ sudo systemctl start zuul-scheduler - $ sudo systemctl start zuul-executor +``` +$ sudo -u zuul bash -c "ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa" +$ sudo systemctl start zuul-scheduler +$ sudo systemctl start zuul-executor +``` The services logs are available into /var/log/zuul/zuul.log. @@ -96,31 +114,43 @@ The services logs are available into /var/log/zuul/zuul.log. Install Apache: - $ sudo dnf install httpd - $ setsebool -P httpd_can_network_connect on +``` +$ sudo dnf install httpd +$ setsebool -P httpd_can_network_connect on +``` Then setup the reverve proxy by adding to /etc/httpd/conf.d/zuul.conf the following content: - RewriteEngine on - RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] - RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] +``` +RewriteEngine on +RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] +RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] +``` Reload Apache: - $ sudo systemctl relead httpd +``` +$ sudo systemctl relead httpd +``` Finally start the zuul-web process - $ sudo systemctl start zuul-web +``` +$ sudo systemctl start zuul-web +``` Validate the direct access to the API by running: - $ curl http://localhost:9000/api/tenants +``` +$ curl http://localhost:9000/api/tenants +``` Validate the access to the API via the reverse proxy: - $ curl http:///api/tenants +``` +$ curl http:///api/tenants +``` ## Run a first Zuul job @@ -140,10 +170,13 @@ We need to prepare a local Git repository to host the pipelines and jobs configu Follow the process below: - $ git init /home/fedora/zuul-config +``` +$ git init /home/fedora/zuul-config +``` In /home/fedora/zuul-config/.zuul.yaml add the following content: +``` - pipeline: name: periodic post-review: true @@ -168,9 +201,11 @@ In /home/fedora/zuul-config/.zuul.yaml add the following content: periodic: jobs: - my-noop +``` Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: +``` --- - hosts: localhost tasks: @@ -179,51 +214,65 @@ Create the /home/fedora/zuul-config/my-noop.yaml and add the following content: - name: Sleep 30 seconds wait_for: timeout: 30 +``` Then commit the configuration by running: - $ cd /home/fedora/zuul-config/ - $ git config user.name "John Doe" - $ git config user.email "john@localhost" - $ git add -A . - $ git commit -m"Init demo config" - +``` +$ cd /home/fedora/zuul-config/ +$ git config user.name "John Doe" +$ git config user.email "john@localhost" +$ git add -A . +$ git commit -m"Init demo config" +``` ### Run a Git deamon to serve the config repository - $ dulwich web-daemon -l 0.0.0.0 / +``` +$ dulwich web-daemon -l 0.0.0.0 / +``` ### Add the new connection in the Zuul configuration In /etc/zuul/zuul.conf add the following: - [connection local_git] - driver=git - baseurl=http://localhost:8000/home/fedora - poll_delay=300 +``` +[connection local_git] +driver=git +baseurl=http://localhost:8000/home/fedora +poll_delay=300 +``` ### Setup the Zuul tenant configuration file In /etc/zuul/main.yaml add the following: - - tenant: - name: default - source: - local_git: - config-projects: - - zuul-config +``` +- tenant: + name: default + source: + local_git: + config-projects: + - zuul-config +``` ### Restart the Zuul services - $ sudo systemctl restart zuul-* +``` +$ sudo systemctl restart zuul-* +``` ### Verify the job executed periodically Run the following command to access the builds API endpoint and ensure the my-noop job run as expected every minutes w/o errors. - $ curl http:///api/tenant/default/builds | python -m json.tool +``` +$ curl http:///api/tenant/default/builds | python -m json.tool +``` Access the build page, where you should see the periodic job my-noop runs - http:///t/default/builds +``` +http:///t/default/builds +``` From 686543e4aa052f2d87e5dd9895689dc59cbf7552 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 9/23] Add functional tests --- diff --git a/README.md b/README.md index 1deef3f..7ddc297 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ $ exit Update the local access setting: ``` -$ sudo sed -i /var/lib/pgsql/data/pg_hba.conf 's/127.0.0.1/32 ident/127.0.0.1/32 md5/' +$ sudo sed -i 's|127.0.0.1/32 ident|127.0.0.1/32 md5|' /var/lib/pgsql/data/pg_hba.conf $ sudo systemctl restart posgreql ``` diff --git a/tests/install_zuul.yml b/tests/install_zuul.yml new file mode 100644 index 0000000..3a3e1c1 --- /dev/null +++ b/tests/install_zuul.yml @@ -0,0 +1,10 @@ +- name: Install Zuul packages + package: + name: + - zuul-scheduler + - zuul-executor + - zuul-web + - zuul-webui + - zuul-doc + state: present + become: true diff --git a/tests/setup_postgres.yml b/tests/setup_postgres.yml new file mode 100644 index 0000000..4b6357e --- /dev/null +++ b/tests/setup_postgres.yml @@ -0,0 +1,40 @@ +- name: Install postgres + package: + name: + - postgresql + - python3-psycopg2 + state: present + become: true + +- name: Setup a postgres admin password + shell: psql -c "ALTER USER postgres WITH PASSWORD 'mypassword';" + become_user: postgres + become: true + environment: + PGPASSWORD: mypassword + +- name: Create the database for zuul + shell: createdb --owner=postgres zuul + register: cmd + failed_when: cmd.rc != 0 and not "'already exists' in cmd.stderr" + ignore_errors: true + become_user: postgres + become: true + environment: + PGPASSWORD: mypassword + +- name: Enable auth access + shell: sed -i 's|127.0.0.1/32 ident|127.0.0.1/32 md5|' /var/lib/pgsql/data/pg_hba.conf + become: true + +- name: Restart postgres + service: + name: postgresql + state: restarted + become: true + +- name: Ensure postgres access + shell: psql -h 127.0.0.1 -U postgres zuul -c "\dt" + environment: + PGPASSWORD: mypassword + diff --git a/tests/setup_zookeeper.yml b/tests/setup_zookeeper.yml new file mode 100644 index 0000000..b9d2c03 --- /dev/null +++ b/tests/setup_zookeeper.yml @@ -0,0 +1,29 @@ +- name: Install JDK + package: + name: java-latest-openjdk + state: present + become: true + +- name: Create /tmp/zookeeper + file: + path: /tmp/zookeeper + state: directory + +- name: Get and extract Zookeeper + unarchive: + src: https://downloads.apache.org/zookeeper/zookeeper-{{ zookeeper_version }}/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz + dest: /tmp/zookeeper + remote_src: yes + +- name: Setup Zookeeper config file + copy: + src: /tmp/zookeeper/apache-zookeeper-{{ zookeeper_version }}-bin/conf/zoo_sample.cfg + dest: /tmp/zookeeper/apache-zookeeper-{{ zookeeper_version }}-bin/conf/zoo.cfg + remote_src: true + +- name: Ensure Zookeeper not running + command: pkill -f zookeeper + ignore_errors: true + +- name: Start Zookeeper + shell: /tmp/zookeeper/apache-zookeeper-{{ zookeeper_version }}-bin/bin/zkServer.sh start diff --git a/tests/setup_zuul_core.yml b/tests/setup_zuul_core.yml new file mode 100644 index 0000000..961d236 --- /dev/null +++ b/tests/setup_zuul_core.yml @@ -0,0 +1,49 @@ +- name: Ensure scheduler and executor service are down + service: + name: "{{ item }}" + state: stopped + loop: + - zuul-executor + - zuul-scheduler + become: true + +- name: Define the sql connection + blockinfile: + path: /etc/zuul/zuul.conf + block: | + [connection sqlreporter] + driver=sql + dburi=postgresql://postgres:mypassword@127.0.0.1:5432/zuul + become: true + +- name: Setup Ansible virtual environment for the Zuul executor + shell: zuul-manage-ansible -u -r /var/lib/zuul/ansible-bin + become_user: zuul + become: true + +- name: Create ssh keypair for Zuul + shell: ssh-keygen -t rsa -N '' -f /var/lib/zuul/.ssh/id_rsa + args: + creates: /var/lib/zuul/.ssh/id_rsa + become_user: zuul + become: true + +- name: Start scheduler and executor services + service: + name: "{{ item }}" + state: started + loop: + - zuul-executor + - zuul-scheduler + become: true + +- name: Give some time to services to fully start + pause: + seconds: 15 + +- name: Check zuul core services are up + shell: systemctl is-active {{ item }} + loop: + - zuul-executor + - zuul-scheduler + become: true diff --git a/tests/setup_zuul_web.yml b/tests/setup_zuul_web.yml new file mode 100644 index 0000000..eb43ac5 --- /dev/null +++ b/tests/setup_zuul_web.yml @@ -0,0 +1,79 @@ +- name: Ensure zuul-web service is down + service: + name: zuul-web + state: stopped + become: true + +- name: Install httpd + package: + name: + - httpd + state: present + become: true + +- name: Ensure httpd can connect to zuul-web + command: setsebool -P httpd_can_network_connect on + become: true + +- name: Add the reverse proxy setting in httpd + blockinfile: + path: /etc/httpd/conf.d/zuul.conf + block: | + RewriteEngine on + RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] + RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] + become: true + +- name: Reload httpd + service: + name: httpd + state: reloaded + become: true + +- name: Add the yarn yum repository + shell: "curl --silent --location https://dl.yarnpkg.com/rpm/yarn.repo | sudo tee /etc/yum.repos.d/yarn.repo" + become: true + +- name: Install yarn + package: + name: yarn + state: present + become: true + +- name: Start a production build of the Zuul web UI + shell: "{{ item }}" + args: + chdir: /usr/share/zuul-ui/ + environment: + REACT_APP_ZUUL_API: 'http://domain.com' + loop: + - yarn install + - yarn build + become: true + +- name: Ensure zuul-web service is up + service: + name: zuul-web + state: started + become: true + +- name: Give some time to services to fully start + pause: + seconds: 15 + +- name: Check zuul web services are up + shell: systemctl is-active {{ item }} + loop: + - zuul-web + - httpd + become: true + +- name: Check we can access the Zuul API + shell: curl http://localhost:9000/api/tenants + register: cmd + failed_when: not "'default' in cmd.stdout" + +- name: Check we can access the Zuul API via httpd + shell: curl http://localhost/api/tenants + register: cmd + failed_when: not "'default' in cmd.stdout" diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..020c5f2 --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,19 @@ +- hosts: localhost + vars: + zookeeper_version: "3.6.1" + tasks: + - import_tasks: setup_zookeeper.yml + tags: + - zookeeper + - import_tasks: setup_postgres.yml + tags: + - postgres + - import_tasks: install_zuul.yml + tags: + - install_zuul + - import_tasks: setup_zuul_core.yml + tags: + - setup_zuul_core + - import_tasks: setup_zuul_web.yml + tags: + - setup_zuul_web From 50eaa3fbabd3c4c8c73fed56d5608829bf035cad Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 10/23] functional test second step --- diff --git a/README.md b/README.md index 7ddc297..7ab2483 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ $ git commit -m"Init demo config" ### Run a Git deamon to serve the config repository ``` +$ sudo dnf install -y python3-dulwich $ dulwich web-daemon -l 0.0.0.0 / ``` diff --git a/tests/configure_zuul_jobs.yml b/tests/configure_zuul_jobs.yml new file mode 100644 index 0000000..93b2ae8 --- /dev/null +++ b/tests/configure_zuul_jobs.yml @@ -0,0 +1,84 @@ +- name: Install dulwich + package: + name: python3-dulwich + state: present + become: true + +- name: Create a local git repository + shell: git init {{ ansible_user_dir }}/zuul-config + +- name: Create .zuul.yaml + copy: + src: zuul.yaml + dest: "{{ ansible_user_dir }}/zuul-config/.zuul.yaml" + +- name: Create my-noop.yaml + copy: + src: my-noop.yaml + dest: "{{ ansible_user_dir }}/zuul-config/my-noop.yaml" + +- name: Commit zuul-config changes + shell: "{{ item }}" + register: cmd + failed_when: + - cmd.rc != 0 + - not "'nothing to commit' in cmd.stdout" + args: + chdir: "{{ ansible_user_dir }}/zuul-config/" + loop: + - git config user.name "John Doe" + - git config user.email "john@localhost" + - git add -A . + - git commit -m"Init demo config" + +- name: Ensure dulwich down + shell: pkill -f dulwich + +- name: Ensure dulwich running + shell: "dulwich web-daemon -l 0.0.0.0 / &" + +- name: Add the git connection into the zuul config + blockinfile: + path: /etc/zuul/zuul.conf + marker: "# {mark} local_git" + block: | + [connection local_git] + driver=git + baseurl=http://localhost:8000/home/fedora + poll_delay=300 + become: true + +- name: Setup the Zuul default tenant to load zuul-config + copy: + src: main.yaml + dest: /etc/zuul/main.yaml + become: true + +- name: Restart Zuul components + service: + name: "{{ item }}" + state: restarted + become: true + loop: + - zuul-scheduler + - zuul-executor + - zuul-web + +- name: Give some time to services to fully start + pause: + seconds: 15 + +- name: Check zuul services are up + shell: systemctl is-active {{ item }} + loop: + - zuul-scheduler + - zuul-executor + - zuul-web + become: true + +- name: Give some time to Zuul to trigger and execute the job + pause: + minutes: 1 + +- name: Ensure the my-noop job was executed + shell: python -c "import requests; r = requests.get('http://localhost/api/tenant/default/builds').json(); assert r[0]['job_name'] == 'my-noop'; assert r[0]['result'] == 'SUCCESS'" diff --git a/tests/main.yaml b/tests/main.yaml new file mode 100644 index 0000000..d6aca6e --- /dev/null +++ b/tests/main.yaml @@ -0,0 +1,6 @@ +- tenant: + name: default + source: + local_git: + config-projects: + - zuul-config diff --git a/tests/my-noop.yaml b/tests/my-noop.yaml new file mode 100644 index 0000000..46deb7f --- /dev/null +++ b/tests/my-noop.yaml @@ -0,0 +1,8 @@ +--- +- hosts: localhost + tasks: + - name: List working directory + command: ls -al {{ ansible_user_dir }} + - name: Sleep 30 seconds + wait_for: + timeout: 30 diff --git a/tests/orig_main.yaml b/tests/orig_main.yaml new file mode 100644 index 0000000..a62fde9 --- /dev/null +++ b/tests/orig_main.yaml @@ -0,0 +1,3 @@ +- tenant: + name: default + source: {} diff --git a/tests/setup_postgres.yml b/tests/setup_postgres.yml index 4b6357e..49b5898 100644 --- a/tests/setup_postgres.yml +++ b/tests/setup_postgres.yml @@ -17,7 +17,7 @@ shell: createdb --owner=postgres zuul register: cmd failed_when: cmd.rc != 0 and not "'already exists' in cmd.stderr" - ignore_errors: true + # ignore_errors: true become_user: postgres become: true environment: diff --git a/tests/setup_zuul_core.yml b/tests/setup_zuul_core.yml index 961d236..283da66 100644 --- a/tests/setup_zuul_core.yml +++ b/tests/setup_zuul_core.yml @@ -7,9 +7,16 @@ - zuul-scheduler become: true +- name: Setup the Zuul default tenant to be empty + copy: + src: orig_main.yaml + dest: /etc/zuul/main.yaml + become: true + - name: Define the sql connection blockinfile: path: /etc/zuul/zuul.conf + marker: "# {mark} sqlreporter" block: | [connection sqlreporter] driver=sql diff --git a/tests/tests.yml b/tests/tests.yml index 020c5f2..faf7fa9 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -17,3 +17,7 @@ - import_tasks: setup_zuul_web.yml tags: - setup_zuul_web + # Now configure Zuul to run a job + - import_tasks: configure_zuul_jobs.yml + tags: + - configure_zuul_jobs diff --git a/tests/zuul.yaml b/tests/zuul.yaml new file mode 100644 index 0000000..ab7c5fe --- /dev/null +++ b/tests/zuul.yaml @@ -0,0 +1,24 @@ +- pipeline: + name: periodic + post-review: true + description: Jobs in this queue are triggered every minute. + manager: independent + precedence: low + trigger: + timer: + - time: '* * * * *' + success: + sqlreporter: + failure: + sqlreporter: + +- job: + name: my-noop + description: Minimal working job + parent: null + run: my-noop.yaml + +- project: + periodic: + jobs: + - my-noop From 7357b3d954959f02a009f2e5ca0a6e5739a7162d Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 11/23] Tests + spec fixes --- diff --git a/tests/setup_postgres.yml b/tests/setup_postgres.yml index 49b5898..7074def 100644 --- a/tests/setup_postgres.yml +++ b/tests/setup_postgres.yml @@ -2,10 +2,26 @@ package: name: - postgresql + - postgresql-server - python3-psycopg2 state: present become: true +- name: Start postgres + service: + name: postgresql + state: started + become: true + +- name: Init postgreql DB + shell: /usr/bin/postgresql-setup --initdb + ignore_errors: true + become: true + +- name: Wait for the service to be up + pause: + seconds: 15 + - name: Setup a postgres admin password shell: psql -c "ALTER USER postgres WITH PASSWORD 'mypassword';" become_user: postgres diff --git a/tests/setup_zookeeper.yml b/tests/setup_zookeeper.yml index b9d2c03..5144648 100644 --- a/tests/setup_zookeeper.yml +++ b/tests/setup_zookeeper.yml @@ -8,22 +8,27 @@ file: path: /tmp/zookeeper state: directory + become: true - name: Get and extract Zookeeper unarchive: src: https://downloads.apache.org/zookeeper/zookeeper-{{ zookeeper_version }}/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz dest: /tmp/zookeeper remote_src: yes + become: true - name: Setup Zookeeper config file copy: src: /tmp/zookeeper/apache-zookeeper-{{ zookeeper_version }}-bin/conf/zoo_sample.cfg dest: /tmp/zookeeper/apache-zookeeper-{{ zookeeper_version }}-bin/conf/zoo.cfg remote_src: true + become: true - name: Ensure Zookeeper not running command: pkill -f zookeeper ignore_errors: true + become: true - name: Start Zookeeper shell: /tmp/zookeeper/apache-zookeeper-{{ zookeeper_version }}-bin/bin/zkServer.sh start + become: true diff --git a/zuul.rpmlintrc b/zuul.rpmlintrc index 80109a9..d2ac5f0 100644 --- a/zuul.rpmlintrc +++ b/zuul.rpmlintrc @@ -12,6 +12,8 @@ addFilter("zuul.noarch: W: non-standard-uid /etc/zuul/logging.conf zuul") addFilter("zuul.noarch: W: non-standard-gid /etc/zuul/logging.conf zuul") addFilter("zuul.noarch: W: non-standard-uid /etc/zuul/zuul.conf zuul") addFilter("zuul.noarch: W: non-standard-gid /etc/zuul/zuul.conf zuul") +addFilter("zuul.noarch: W: non-standard-uid /etc/zuul/main.yaml zuul") +addFilter("zuul.noarch: W: non-standard-gid /etc/zuul/main.yaml zuul") addFilter("zuul.noarch: W: non-standard-uid /var/lib/zuul zuul") addFilter("zuul.noarch: W: non-standard-gid /var/lib/zuul zuul") addFilter("zuul.noarch: W: non-standard-uid /var/lib/zuul/.ssh zuul") @@ -28,9 +30,9 @@ addFilter("zuul-executor.noarch: W: non-standard-uid /var/lib/zuul/executor zuul addFilter("zuul-executor.noarch: W: non-standard-gid /var/lib/zuul/executor zuul") # Expected - zuul.conf can store credentials addFilter("zuul.noarch: E: non-readable /etc/zuul/zuul.conf 640") -# Expected for now - https://review.opendev.org/728955 - does not âss the upstream CI -addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.8/site-packages/zuul/ansible/base/library/command.py 644 /usr/bin/python3 ") -addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.8/site-packages/zuul/ansible/base/library/zuul_console.py 644 /usr/bin/python3 ") +# Expected for now - https://review.opendev.org/728955 - does not pass the upstream CI +addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.9/site-packages/zuul/ansible/base/library/command.py 644 /usr/bin/python3 ") +addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.9/site-packages/zuul/ansible/base/library/zuul_console.py 644 /usr/bin/python3 ") # Expected - Zuul runtime expects to find that directory addFilter("zuul.noarch: W: hidden-file-or-dir /var/lib/zuul/.ssh") addFilter("zuul.noarch: W: hidden-file-or-dir /var/lib/zuul/.ssh") @@ -50,3 +52,8 @@ addFilter("zuul-web.noarch: W: no-manual-page-for-binary zuul-web") addFilter("zuul-executor.noarch: E: devel-dependency python3-devel") # Expected - gearman is a protocol name addFilter("zuul-scheduler.noarch: W: spelling-error %description -l en_US gearman -> gear man, gear-man, germane") +# Expected - executor devel-dependency +addFilter("zuul-executor.noarch: E: devel-dependency libffi-devel") +addFilter("zuul-executor.noarch: E: explicit-lib-dependency libffi-devel") +# No idea for this one +addFilter("zuul-doc.noarch: W: wrong-file-end-of-line-encoding /usr/share/doc/zuul-doc/html/objects.inv") diff --git a/zuul.spec b/zuul.spec index ea4bb25..091e4a0 100644 --- a/zuul.spec +++ b/zuul.spec @@ -42,8 +42,7 @@ BuildRequires: python3-setuptools BuildRequires: python3-zuul-sphinx BuildRequires: python3-snowballstemmer BuildRequires: python3-fixtures -# TODO: https://bugzilla.redhat.com/show_bug.cgi?id=1849654 -# BuildRequires: python3-sphinx-autodoc-typehints +BuildRequires: python3-sphinx-autodoc-typehints BuildRequires: python3-sphinxcontrib-blockdiag BuildRequires: python3-sphinxcontrib-programoutput BuildRequires: python3-sphinxcontrib-openapi @@ -243,8 +242,6 @@ EOF # Fix non compliant shebangs /usr/bin/pathfix.py -i %{python3} -p -n -k zuul/ansible install -m 0644 %{SOURCE7} README.fedora -# Fix wrong-file-end-of-line-encoding -sed -i 's/\r$//' LICENSE # Fix 0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch not fully apply # due to pypi archive removing the symlinks cp zuul/ansible/base/library/command.py zuul/ansible/2.7/library/ @@ -265,13 +262,19 @@ chmod +x build/zuul-manage-ansible sed -e 's/^ *releasenotes$//' -i doc/source/reference/index.rst rm doc/source/reference/releasenotes.rst # TODO: https://bugzilla.redhat.com/show_bug.cgi?id=1849654 -sed '/sphinx_autodoc_typehints/d' -i doc/source/conf.py +# sed '/sphinx_autodoc_typehints/d' -i doc/source/conf.py PYTHONPATH=../../build/lib PATH=$PATH:$(pwd)/build PBR_VERSION=%{version} SPHINX_DEBUG=1 sphinx-build-3 \ -b html doc/source build/html # Remove empty stub files find build -type f -name "*.pyi" -size 0 -delete # rm doc build leftovers rm -Rf build/html/.buildinfo build/html/.doctrees +# Fix file-not-utf8 +iconv -f iso8859-1 -t utf-8 build/html/objects.inv > build/html/objects.inv.conv && \ + mv -f build/html/objects.inv.conv build/html/objects.inv +# Fix wrong-file-end-of-line-encoding +sed -i 's/\r$//' LICENSE +sed -i 's/\r$//' build/html/objects.inv %install install -p -d -m 0755 %{buildroot}/%{_datadir}/zuul-ui @@ -284,10 +287,10 @@ install -p -D -m 0644 %{SOURCE4} %{buildroot}%{_unitdir}/zuul-executor.service install -p -D -m 0644 %{SOURCE5} %{buildroot}%{_unitdir}/zuul-web.service install -p -D -m 0644 %{SOURCE6} %{buildroot}%{_unitdir}/zuul-fingergw.service install -p -D -m 0640 %{SOURCE8} %{buildroot}%{_sysconfdir}/zuul/zuul.conf -install -p -D -m 0640 %{SOURCE9} %{buildroot}%{_sysconfdir}/zuul/logging.conf -install -p -D -m 0640 %{SOURCE10} %{buildroot}%{_sysconfdir}/zuul/main.yaml +install -p -D -m 0644 %{SOURCE9} %{buildroot}%{_sysconfdir}/zuul/logging.conf +install -p -D -m 0644 %{SOURCE10} %{buildroot}%{_sysconfdir}/zuul/main.yaml install -p -d -m 0700 %{buildroot}%{_sharedstatedir}/zuul -install -p -d -m 0700 %{buildroot}%{_localstatedir}/log/zuul +install -p -d -m 0755 %{buildroot}%{_localstatedir}/log/zuul # Prepare lib directory install -p -d -m 0700 %{buildroot}%{_sharedstatedir}/zuul/.ssh @@ -332,15 +335,15 @@ exit 0 %files %license LICENSE %config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/zuul.conf -%config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/main.yaml +%config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/main.yaml %config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/logging.conf %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/.ssh %dir %attr(0755,zuul,zuul) %{_sharedstatedir}/zuul/ansible -%dir %attr(0700,zuul,zuul) %{_sharedstatedir}/zuul/keys -# Zuul runtime check +# Zuul runtime check complains # Project key directory /var/lib/zuul/keys must be mode 0700; current mode is 755 -%dir %attr(0700,zuul,zuul) %{_localstatedir}/log/zuul +%dir %attr(0700,zuul,zuul) %{_sharedstatedir}/zuul/keys +%dir %attr(0755,zuul,zuul) %{_localstatedir}/log/zuul %{python3_sitelib}/zuul %{python3_sitelib}/zuul-*.egg-info/ %{_bindir}/zuul From 9ce704c2369ce9b2343b4ccb036bed5f5fe8d34e Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 12/23] Fix ordering in for postgresql setup --- diff --git a/tests/setup_postgres.yml b/tests/setup_postgres.yml index 7074def..5934618 100644 --- a/tests/setup_postgres.yml +++ b/tests/setup_postgres.yml @@ -7,17 +7,17 @@ state: present become: true +- name: Init postgreql DB + shell: /usr/bin/postgresql-setup --initdb + ignore_errors: true + become: true + - name: Start postgres service: name: postgresql state: started become: true -- name: Init postgreql DB - shell: /usr/bin/postgresql-setup --initdb - ignore_errors: true - become: true - - name: Wait for the service to be up pause: seconds: 15 From 5c34fc6d6e4b4ecda8aa3448bd0324f28486f6c4 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:25 +0000 Subject: [PATCH 13/23] Intregrate cherrot pinning patch --- diff --git a/0001-Update-cherrypy-and-cheroot-requirement-pinning.patch b/0001-Update-cherrypy-and-cheroot-requirement-pinning.patch new file mode 100644 index 0000000..0b58089 --- /dev/null +++ b/0001-Update-cherrypy-and-cheroot-requirement-pinning.patch @@ -0,0 +1,36 @@ +From e8935676a8b6db3b29d419d440f9e82b5090a5b2 Mon Sep 17 00:00:00 2001 +From: Fabien Boucher +Date: Mon, 13 Jul 2020 15:32:53 +0000 +Subject: [PATCH] Update cherrypy and cheroot requirement pinning + +Cheroot 8.3.1 have been released with the https://github.com/cherrypy/cheroot/issues/263 +fix. + +Change-Id: I83aad30a6145ed2c7689d6c25c624c453c07ed2b +--- + requirements.txt | 7 ++----- + 1 file changed, 2 insertions(+), 5 deletions(-) + +diff --git a/requirements.txt b/requirements.txt +index 78eb980f..829c7606 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -29,13 +29,10 @@ iso8601 + psutil + fb-re2>=1.0.6 + paho-mqtt +-cherrypy==18.3.0; # last version to support cheroot below ++cherrypy + ws4py + routes + pathspec + jsonpath-rw + urllib3!=1.25.4,!=1.25.5 # https://github.com/urllib3/urllib3/pull/1684 +-# TODO(tobiash): cheroot 8.1.0 introduced a regression when handling concurrent +-# requests. Remove the following line when this issue is resolved: +-# https://github.com/cherrypy/cheroot/issues/263 +-cheroot<8.1.0 ++cheroot!=8.1.*,!=8.2.*,!=8.3.0 # https://github.com/cherrypy/cheroot/issues/263 +-- +2.27.0 + diff --git a/zuul.spec b/zuul.spec index 091e4a0..3ae27d0 100644 --- a/zuul.spec +++ b/zuul.spec @@ -1,5 +1,3 @@ -%{?python_disable_dependency_generator} - Name: zuul Version: 3.19.0 Release: 1%{?dist} @@ -22,20 +20,10 @@ Source10: main.yaml Patch01: 0001-Remove-another-shebang-and-remove-useless-exec-bits.patch Patch02: 0001-requirements-add-explicit-reference-to-dateutil.patch Patch03: 0001-Replace-deprecated-Thread.isAlive-with-Thread.is_ali.patch +Patch04: 0001-Update-cherrypy-and-cheroot-requirement-pinning.patch BuildArch: noarch - -#Error: -# Problem: conflicting requests -# https://github.com/cherrypy/cheroot/issues/263 # the zuul cap reason -# The proposed fix https://github.com/cherrypy/cheroot/pull/277 -# https://bugzilla.redhat.com/show_bug.cgi?id=1834207 -# - nothing provides python3.8dist(cheroot) < 8.1 needed by zuul-3.19.0-1.fc33.noarch -# - nothing provides python3.8dist(cherrypy) = 18.3 needed by zuul-3.19.0-1.fc33.noarch -# TODO: https://bugzilla.redhat.com/show_bug.cgi?id=1849654 - - BuildRequires: python3-devel BuildRequires: python3-pbr BuildRequires: python3-setuptools @@ -83,58 +71,6 @@ BuildRequires: python3-graphene BuildRequires: systemd BuildRequires: ansible -#rpm -qpR results_zuul/3.19.0/1.fc33/zuul-3.19.0-1.fc33.noarch.rpm -Requires: ((python3.9dist(gear) < 0.15 or python3.9dist(gear) > 0.15) with python3.9dist(gear) < 1 with python3.9dist(gear) >= 0.13) -Requires: ((python3.9dist(urllib3) < 1.25.4 or python3.9dist(urllib3) > 1.25.4) with (python3.9dist(urllib3) < 1.25.5 or python3.9dist(urllib3) > 1.25.5)) -Requires: ((python3.9dist(virtualenv) < 20 or python3.9dist(virtualenv) > 20) with (python3.9dist(virtualenv) < 20.0.1 or python3.9dist(virtualenv) > 20.0.1) with python3.9dist(virtualenv) > 20) -Requires: (python3.9dist(prettytable) < 0.8 with python3.9dist(prettytable) >= 0.6) -Requires: /bin/sh -Requires: /usr/bin/python3 -Requires: config(zuul) = 3.19.0-1.fc33 -Requires: python(abi) = 3.9 -Requires: python3-cheroot -Requires: python3.9dist(alembic) -Requires: python3.9dist(apscheduler) >= 3 -Requires: python3.9dist(babel) >= 1 -Requires: python3.9dist(cachecontrol) -Requires: python3.9dist(cachetools) - -#Requires: python3.9dist(cheroot) < 8.1 -#Requires: python3.9dist(cherrypy) = 18.3 -Requires: python3.9dist(cheroot) -Requires: python3.9dist(cherrypy) - -Requires: python3.9dist(cryptography) >= 1.6 -Requires: python3.9dist(extras) -Requires: python3.9dist(fb-re2) >= 1.0.6 -Requires: python3.9dist(github3-py) >= 1.1 -Requires: python3.9dist(gitpython) >= 2.1.8 -Requires: python3.9dist(iso8601) -Requires: python3.9dist(jsonpath-rw) -Requires: python3.9dist(kazoo) -Requires: python3.9dist(netaddr) -Requires: python3.9dist(paho-mqtt) -Requires: python3.9dist(paramiko) >= 2.0.1 -Requires: python3.9dist(pathspec) -Requires: python3.9dist(pbr) >= 1.1 -Requires: python3.9dist(psutil) -Requires: python3.9dist(pyjwt) -Requires: python3.9dist(python-daemon) >= 2.0.4 -Requires: python3.9dist(python-dateutil) -Requires: python3.9dist(pyyaml) >= 3.1 -Requires: python3.9dist(routes) -Requires: python3.9dist(setuptools) -Requires: python3.9dist(sqlalchemy) -Requires: python3.9dist(statsd) >= 3 -Requires: python3.9dist(voluptuous) >= 0.10.2 -Requires: python3.9dist(ws4py) -Requires: rpmlib(CompressedFileNames) <= 3.0.4-1 -Requires: rpmlib(FileDigests) <= 4.6.0-1 -Requires: rpmlib(PartialHardlinkSets) <= 4.0.4-1 -Requires: rpmlib(PayloadFilesHavePrefix) <= 4.0-1 -Requires: rpmlib(PayloadIsZstd) <= 5.4.18-1 -Requires: rpmlib(RichDependencies) <= 4.12.0-1 - %description Zuul is a program that drives continuous integration, delivery, and deployment systems with a focus on project gating and From 440e36684e40e8a6a6b3fbdfa52db1c666419394 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 14/23] Fix missing /etc/zuul and refector logging.conf --- diff --git a/logging.conf b/logging.conf index d47fa1f..ee2954c 100644 --- a/logging.conf +++ b/logging.conf @@ -1,42 +1,30 @@ [loggers] -keys=root,zuul,gerrit +keys=root,zuul [handlers] -keys=console,debug,normal +keys=console,normal [formatters] keys=simple [logger_root] -level=WARNING +level=INFO handlers=console [logger_zuul] -level=DEBUG -handlers=debug,normal +level=INFO +handlers=normal qualname=zuul propagate=0 -[logger_gerrit] -level=DEBUG -handlers=debug,normal -qualname=gerrit -propagate=0 - [handler_console] -level=DEBUG +level=INFO class=StreamHandler formatter=simple args=(sys.stdout,) -[handler_debug] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('/var/log/zuul/debug.log', 'midnight', 1, 30,) - [handler_normal] -level=DEBUG +level=INFO class=logging.handlers.TimedRotatingFileHandler formatter=simple args=('/var/log/zuul/zuul.log', 'midnight', 1, 30,) diff --git a/zuul.spec b/zuul.spec index 3ae27d0..719f460 100644 --- a/zuul.spec +++ b/zuul.spec @@ -270,6 +270,7 @@ exit 0 %files %license LICENSE +%dir %attr(0755,zuul,zuul) %{_sysconfdir}/zuul %config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/zuul.conf %config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/main.yaml %config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/logging.conf From 4a81f22a4a0e1da72c4e67d9e240db8f90e7518b Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 15/23] Bump to 3.19.1 --- diff --git a/.gitignore b/.gitignore index b7ad809..ac174da 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ results_zuul *.rpm /zuul-3.19.0.tar.gz +/zuul-3.19.1.tar.gz diff --git a/sources b/sources index e739312..feff18a 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (zuul-3.19.0.tar.gz) = b75430879b6165e87e0f9d05de3cda67cc1e0b84e65ac1407ce6f853e27f66811ac438cbff2d6c45243025e5aba26820fa478929744f1eed9487f3fc957bcda1 +SHA512 (zuul-3.19.1.tar.gz) = 0460e7e8407cd9cc005d1c05be718ca350a526cb7db44bfb1e8fe702a28c127ad828d6f6a7b1786205722e778def571fae80663db3d371df8a06e6b37ca50d89 diff --git a/zuul.rpmlintrc b/zuul.rpmlintrc index d2ac5f0..3fe02f6 100644 --- a/zuul.rpmlintrc +++ b/zuul.rpmlintrc @@ -30,9 +30,9 @@ addFilter("zuul-executor.noarch: W: non-standard-uid /var/lib/zuul/executor zuul addFilter("zuul-executor.noarch: W: non-standard-gid /var/lib/zuul/executor zuul") # Expected - zuul.conf can store credentials addFilter("zuul.noarch: E: non-readable /etc/zuul/zuul.conf 640") -# Expected for now - https://review.opendev.org/728955 - does not pass the upstream CI -addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.9/site-packages/zuul/ansible/base/library/command.py 644 /usr/bin/python3 ") -addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.9/site-packages/zuul/ansible/base/library/zuul_console.py 644 /usr/bin/python3 ") +# Expected - See upstream comments https://review.opendev.org/728955 +addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.9/site-packages/zuul/ansible/base/library/command.py 644 /usr/bin/python3") +addFilter("zuul.noarch: E: non-executable-script /usr/lib/python3.9/site-packages/zuul/ansible/base/library/zuul_console.py 644 /usr/bin/python3") # Expected - Zuul runtime expects to find that directory addFilter("zuul.noarch: W: hidden-file-or-dir /var/lib/zuul/.ssh") addFilter("zuul.noarch: W: hidden-file-or-dir /var/lib/zuul/.ssh") diff --git a/zuul.spec b/zuul.spec index 719f460..1e8c56b 100644 --- a/zuul.spec +++ b/zuul.spec @@ -1,5 +1,5 @@ Name: zuul -Version: 3.19.0 +Version: 3.19.1 Release: 1%{?dist} Summary: Trunk Gating System @@ -328,8 +328,8 @@ exit 0 %changelog -* Wed Mar 11 2020 Fabien Boucher - 3.19.0-1 -- Bump to 3.19.0 +* Mon Jul 27 2020 Fabien Boucher - 3.19.1-1 +- Bump to 3.19.1 - Fedora rawhide compat * Tue Mar 3 2020 Tristan Cacqueray - 3.18.0-1 From 231e7e5bd12f11b7cf86ae2149095fa69db3cefe Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 16/23] setup_zuul_web: set create flag in blockinfile --- diff --git a/tests/setup_zuul_web.yml b/tests/setup_zuul_web.yml index eb43ac5..5e22c76 100644 --- a/tests/setup_zuul_web.yml +++ b/tests/setup_zuul_web.yml @@ -17,6 +17,7 @@ - name: Add the reverse proxy setting in httpd blockinfile: + create: true path: /etc/httpd/conf.d/zuul.conf block: | RewriteEngine on From c32129c9a58ca703f72def788f6678179482977a Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 17/23] configure_zuul_jobs: add ignore_errors on pkill task --- diff --git a/tests/configure_zuul_jobs.yml b/tests/configure_zuul_jobs.yml index 93b2ae8..0131dff 100644 --- a/tests/configure_zuul_jobs.yml +++ b/tests/configure_zuul_jobs.yml @@ -33,6 +33,7 @@ - name: Ensure dulwich down shell: pkill -f dulwich + ignore_errors: true - name: Ensure dulwich running shell: "dulwich web-daemon -l 0.0.0.0 / &" From e8ec04ab07d56a0f5f15fdf8de58ebb8f8fe7f19 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 18/23] tests: fix dulwich web-daemon call and repo path --- diff --git a/tests/configure_zuul_jobs.yml b/tests/configure_zuul_jobs.yml index 0131dff..d8b1196 100644 --- a/tests/configure_zuul_jobs.yml +++ b/tests/configure_zuul_jobs.yml @@ -32,11 +32,12 @@ - git commit -m"Init demo config" - name: Ensure dulwich down - shell: pkill -f dulwich + shell: pkill -f web-daemon ignore_errors: true +# https://bugzilla.redhat.com/show_bug.cgi?id=1866463 - name: Ensure dulwich running - shell: "dulwich web-daemon -l 0.0.0.0 / &" + shell: "python3 /usr/lib64/python3.9/site-packages/dulwich/cli.py web-daemon -l 0.0.0.0 / &" - name: Add the git connection into the zuul config blockinfile: @@ -45,7 +46,7 @@ block: | [connection local_git] driver=git - baseurl=http://localhost:8000/home/fedora + baseurl=http://localhost:8000/{{ ansible_user_dir }} poll_delay=300 become: true From 0c67bb6181b8fd7e0fc7299ef10a4f6934b10c40 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 19/23] Fix license and missing provides --- diff --git a/zuul.spec b/zuul.spec index 1e8c56b..4318ae5 100644 --- a/zuul.spec +++ b/zuul.spec @@ -3,7 +3,8 @@ Version: 3.19.1 Release: 1%{?dist} Summary: Trunk Gating System -License: ASL 2.0 +# The entire source code is ASL 2.0 except files under %%{python3_sitelib}/zuul/ansible which is GPLv3+ +License: ASL 2.0 and GPLv3+ URL: https://zuul-ci.org Source0: %pypi_source @@ -71,6 +72,10 @@ BuildRequires: python3-graphene BuildRequires: systemd BuildRequires: ansible +Provides: bundled(ansible) = 2.7 +Provides: bundled(ansible) = 2.8 +Provides: bundled(ansible) = 2.9 + %description Zuul is a program that drives continuous integration, delivery, and deployment systems with a focus on project gating and From 0dd67c4715f2792d2235a44dcce3351c038e2cc3 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 20/23] Remove multiple provide: zuul.noarch: E: useless-provides bundled(ansible) --- diff --git a/zuul.spec b/zuul.spec index 4318ae5..7dd7cd5 100644 --- a/zuul.spec +++ b/zuul.spec @@ -72,9 +72,7 @@ BuildRequires: python3-graphene BuildRequires: systemd BuildRequires: ansible -Provides: bundled(ansible) = 2.7 -Provides: bundled(ansible) = 2.8 -Provides: bundled(ansible) = 2.9 +Provides: bundled(ansible) %description Zuul is a program that drives continuous integration, delivery, From 24225066d8a9af6bf05313616a02c989192ee823 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 21/23] Fix missing systemd require for subpackages --- diff --git a/zuul.rpmlintrc b/zuul.rpmlintrc index 3fe02f6..e33bc31 100644 --- a/zuul.rpmlintrc +++ b/zuul.rpmlintrc @@ -57,3 +57,5 @@ addFilter("zuul-executor.noarch: E: devel-dependency libffi-devel") addFilter("zuul-executor.noarch: E: explicit-lib-dependency libffi-devel") # No idea for this one addFilter("zuul-doc.noarch: W: wrong-file-end-of-line-encoding /usr/share/doc/zuul-doc/html/objects.inv") +# Expected - https://github.com/rpm-software-management/rpmlint/issues/427 +addFilter("zuul.noarch: E: useless-provides bundled(ansible)") diff --git a/zuul.spec b/zuul.spec index 7dd7cd5..2a1d532 100644 --- a/zuul.spec +++ b/zuul.spec @@ -72,7 +72,9 @@ BuildRequires: python3-graphene BuildRequires: systemd BuildRequires: ansible -Provides: bundled(ansible) +Provides: bundled(ansible) = 2.9 +Provides: bundled(ansible) = 2.8 +Provides: bundled(ansible) = 2.7 %description Zuul is a program that drives continuous integration, delivery, @@ -90,6 +92,7 @@ Please refers to README.fedora for build and deployment instruction. %package scheduler Summary: The Zuul scheduler service Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: systemd %description scheduler The main Zuul process. Handles receiving events, executing jobs, @@ -101,6 +104,7 @@ the other components use for coordination. %package merger Summary: The Zuul merger service Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: systemd %description merger Scale-out component that performs git merge operations. @@ -123,6 +127,7 @@ streaming of logs. %package executor Summary: The Zuul executor service Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: systemd Requires: git-core Requires: bubblewrap # zuul-manage-ansible requires them to create ansible virtualenvs @@ -142,6 +147,7 @@ One or more of these must be run. %package fingergw Summary: Executor finger gateway service Requires: %{name} = %{?epoch:%{epoch}:}%{version}-%{release} +Requires: systemd %description fingergw A gateway which provides finger protocol access to live streaming of logs. @@ -273,6 +279,7 @@ exit 0 %files %license LICENSE +%doc README.rst %dir %attr(0755,zuul,zuul) %{_sysconfdir}/zuul %config(noreplace) %attr(0640,zuul,zuul) %{_sysconfdir}/zuul/zuul.conf %config(noreplace) %attr(0644,zuul,zuul) %{_sysconfdir}/zuul/main.yaml From 814d0946a0366e1ae45df8bf183a154f1a003842 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 08:52:26 +0000 Subject: [PATCH 22/23] Give more time for the periodic job to execute --- diff --git a/tests/configure_zuul_jobs.yml b/tests/configure_zuul_jobs.yml index d8b1196..6204222 100644 --- a/tests/configure_zuul_jobs.yml +++ b/tests/configure_zuul_jobs.yml @@ -80,7 +80,7 @@ - name: Give some time to Zuul to trigger and execute the job pause: - minutes: 1 + minutes: 2 - name: Ensure the my-noop job was executed shell: python -c "import requests; r = requests.get('http://localhost/api/tenant/default/builds').json(); assert r[0]['job_name'] == 'my-noop'; assert r[0]['result'] == 'SUCCESS'" From dfa8ac8495b05a76ce529aa9cabe3853452ff8b4 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Aug 13 2020 09:53:17 +0000 Subject: [PATCH 23/23] Fix rmplint rule --- diff --git a/zuul.rpmlintrc b/zuul.rpmlintrc index e33bc31..d5dc470 100644 --- a/zuul.rpmlintrc +++ b/zuul.rpmlintrc @@ -58,4 +58,4 @@ addFilter("zuul-executor.noarch: E: explicit-lib-dependency libffi-devel") # No idea for this one addFilter("zuul-doc.noarch: W: wrong-file-end-of-line-encoding /usr/share/doc/zuul-doc/html/objects.inv") # Expected - https://github.com/rpm-software-management/rpmlint/issues/427 -addFilter("zuul.noarch: E: useless-provides bundled(ansible)") +addFilter("zuul.noarch: E: useless-provides bundled\(ansible\)")