From 71a5e44bde7d44d7c3575546f3527799b9cac9be Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Aug 12 2019 18:18:13 +0000 Subject: Merge branch 'master' into f30 --- diff --git a/.gitignore b/.gitignore index 4be5491..dc981d9 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,7 @@ /assetcache-8a35385294e68a6cf211f137ae71dfdd923860c2.tar.xz /openQA-5bfa647a930e573a9f01f517ac8c322c3394b7a8.tar.gz /assetcache-5bfa647a930e573a9f01f517ac8c322c3394b7a8.tar.xz +/openQA-92e8f3c0505a12795aabfc275c95a7dbb1bca708.tar.gz +/assetcache-92e8f3c0505a12795aabfc275c95a7dbb1bca708.tar.xz +/openQA-1c533908f94cf9f156a250dbf29ca809cb474076.tar.gz +/assetcache-1c533908f94cf9f156a250dbf29ca809cb474076.tar.xz diff --git a/0001-AMQP-allow-passing-headers-to-publish_amqp.patch b/0001-AMQP-allow-passing-headers-to-publish_amqp.patch deleted file mode 100644 index f5d4ad8..0000000 --- a/0001-AMQP-allow-passing-headers-to-publish_amqp.patch +++ /dev/null @@ -1,134 +0,0 @@ -From ef621c739aa892985937b7a2260111a102d98847 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Wed, 24 Jul 2019 11:40:10 -0700 -Subject: [PATCH] AMQP: allow passing headers to publish_amqp - -Fedora's new AMQP-based messaging system requires messages to -have certain headers: - -https://fedora-messaging.readthedocs.io/en/stable/wire-format.html - -I'm planning to write a plugin for emitting Fedora-compliant -AMQP messages as a subclass of the existing AMQP plugin, but in -order to be able to use `publish_amqp` in that plugin, I need to -be able to pass it headers like this. Otherwise I'd have to -reimplement it. - -This implementation aims to ensure it works just as before if -you *don't* want to pass it headers, and without needing to -duplicate the `publish_p` calls for "headers" and "no headers" -cases - that's the reason for making `$headers` a reference to -an empty hash if it's not defined, and then checking that it's -a hashref: to be sure about what we're asking `publish_p` to do. -Passing it a ref to an empty hash should effectively be a no-op -so far as headers are concerned, and not interfere with passing -`routing_key` as a param. - -Signed-off-by: Adam Williamson ---- - lib/OpenQA/WebAPI/Plugin/AMQP.pm | 6 ++-- - t/23-amqp.t | 59 ++++++++++++++++++++++++++++++++ - 2 files changed, 63 insertions(+), 2 deletions(-) - -diff --git a/lib/OpenQA/WebAPI/Plugin/AMQP.pm b/lib/OpenQA/WebAPI/Plugin/AMQP.pm -index e46d7915..7553bc3c 100644 ---- a/lib/OpenQA/WebAPI/Plugin/AMQP.pm -+++ b/lib/OpenQA/WebAPI/Plugin/AMQP.pm -@@ -71,7 +71,9 @@ sub log_event { - } - - sub publish_amqp { -- my ($self, $topic, $event_data) = @_; -+ my ($self, $topic, $event_data, $headers) = @_; -+ $headers //= {}; -+ die "publish_amqp headers must be a hashref!" unless (ref($headers) eq 'HASH'); - - log_debug("Sending AMQP event: $topic"); - my $publisher = Mojo::RabbitMQ::Client::Publisher->new( -@@ -79,7 +81,7 @@ sub publish_amqp { - - # A hard reference to the publisher object needs to be kept until the event - # has been published asynchronously, or it gets destroyed too early -- $publisher->publish_p($event_data, routing_key => $topic)->then( -+ $publisher->publish_p($event_data, $headers, routing_key => $topic)->then( - sub { - log_debug "$topic published"; - } -diff --git a/t/23-amqp.t b/t/23-amqp.t -index b6d6f68d..4b7d82f3 100644 ---- a/t/23-amqp.t -+++ b/t/23-amqp.t -@@ -29,6 +29,7 @@ use lib "$FindBin::Bin/lib"; - use OpenQA::Client; - use OpenQA::Jobs::Constants; - use OpenQA::Test::Database; -+use Test::Exception; - use Test::MockModule; - use Test::More; - use Test::Mojo; -@@ -189,4 +190,62 @@ subtest 'create parent group comment' => sub { - is($json->{parent_group_id}, 2000, 'parent group id'); - }; - -+# Now let's unmock publish_amqp so we can test it... -+$plugin_mock->unmock('publish_amqp'); -+%published = (); -+# ...but we'll mock the thing it calls. -+my $publisher_mock = Test::MockModule->new('Mojo::RabbitMQ::Client::Publisher'); -+$publisher_mock->mock( -+ publish_p => sub { -+ # copied from upstream git master as of 2019-07-24 -+ my $self = shift; -+ my $body = shift; -+ my $headers = {}; -+ my %args = (); -+ -+ if (ref($_[0]) eq 'HASH') { -+ $headers = shift; -+ } -+ if (@_) { -+ %args = (@_); -+ } -+ # end copying -+ $published{body} = $body; -+ $published{headers} = $headers; -+ $published{args} = \%args; -+ # we need to return a Promise or stuff breaks -+ my $client_promise = Mojo::Promise->new(); -+ return $client_promise; -+ }); -+ -+# we need an instance of the plugin now. I can't find a documented -+# way to access the one that's already loaded... -+my $amqp = OpenQA::WebAPI::Plugin::AMQP->new; -+$amqp->register($app); -+ -+subtest 'amqp_publish call without headers' => sub { -+ $amqp->publish_amqp('some.topic', 'some message'); -+ is($published{body}, 'some message', "message body correctly passed"); -+ is_deeply($published{headers}, {}, "headers is empty hashref"); -+ is_deeply($published{args}->{routing_key}, 'some.topic', "topic appears as routing key"); -+}; -+ -+subtest 'amqp_publish call with headers' => sub { -+ %published = (); -+ $amqp->publish_amqp('some.topic', 'some message', {'someheader' => 'something'}); -+ is($published{body}, 'some message', "message body correctly passed"); -+ is_deeply($published{headers}, {'someheader' => 'something'}, "headers is expected hashref"); -+ is_deeply($published{args}->{routing_key}, 'some.topic', "topic appears as routing key"); -+}; -+ -+subtest 'amqp_publish call with incorrect headers' => sub { -+ throws_ok( -+ sub { -+ $amqp->publish_amqp('some.topic', 'some message', 'some headers'); -+ }, -+ qr/publish_amqp headers must be a hashref!/, -+ 'dies on bad headers' -+ ); -+}; -+ - done_testing(); --- -2.22.0 - diff --git a/0001-Add-dependency-on-CommonMark-which-will-soon-replace-rediffed.patch b/0001-Add-dependency-on-CommonMark-which-will-soon-replace-rediffed.patch deleted file mode 100644 index 39e9595..0000000 --- a/0001-Add-dependency-on-CommonMark-which-will-soon-replace-rediffed.patch +++ /dev/null @@ -1,52 +0,0 @@ -From acb5d403963eb3e209b95dfb882d3efe278301ed Mon Sep 17 00:00:00 2001 -From: Sebastian Riedel -Date: Mon, 29 Jul 2019 18:47:19 +0200 -Subject: [PATCH] Add dependency on CommonMark, which will soon replace - Text::Markdown for a 1000% performance increase (#2228) - ---- - cpanfile | 1 + - docker/travis_test/Dockerfile | 1 + - openQA.spec | 2 +- - 3 files changed, 3 insertions(+), 1 deletion(-) - -diff --git a/cpanfile b/cpanfile -index 4fc17833..2596a698 100644 ---- a/cpanfile -+++ b/cpanfile -@@ -77,6 +77,7 @@ requires 'Sub::Install'; - requires 'Sub::Name'; - requires 'Text::Diff'; - requires 'Text::Markdown'; -+requires 'CommonMark'; - requires 'Time::HiRes'; - requires 'Time::ParseDate'; - requires 'Time::Piece'; -diff --git a/docker/travis_test/Dockerfile b/docker/travis_test/Dockerfile -index 0ef46dbe..8f11a62f 100644 ---- a/docker/travis_test/Dockerfile -+++ b/docker/travis_test/Dockerfile -@@ -122,6 +122,7 @@ RUN zypper in -y -C \ - 'perl(Test::Warnings)' \ - 'perl(Text::Diff)' \ - 'perl(Text::Markdown)' \ -+ 'perl(CommonMark)' \ - 'perl(Time::ParseDate)' \ - 'perl(XSLoader) >= 0.24' \ - 'perl(XML::SemanticDiff)' \ -diff --git a/openQA.spec b/openQA.spec -index 266d5fa4..35574cf4 100644 ---- a/openQA.spec -+++ b/openQA.spec -@@ -45,7 +45,7 @@ - %bcond_with tests - %endif - # runtime requirements that also the testsuite needs --%define t_requires perl(DBD::Pg) perl(DBIx::Class) perl(Config::IniFiles) perl(SQL::Translator) perl(Date::Format) perl(File::Copy::Recursive) perl(DateTime::Format::Pg) perl(Net::OpenID::Consumer) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Plugin::AssetPack) perl(aliased) perl(Config::Tiny) perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(IO::Socket::SSL) perl(Data::Dump) perl(DBIx::Class::OptimisticLocking) perl(Module::Pluggable) perl(Text::Diff) perl(Text::Markdown) perl(JSON::Validator) perl(YAML::XS) perl(IPC::Run) perl(Archive::Extract) perl(CSS::Minifier::XS) perl(JavaScript::Minifier::XS) perl(Time::ParseDate) perl(Sort::Versions) perl(Mojo::RabbitMQ::Client) perl(BSD::Resource) perl(Cpanel::JSON::XS) perl(Pod::POM) perl(Mojo::IOLoop::ReadWriteProcess) perl(Minion) perl(Mojo::Pg) perl(Mojo::SQLite) perl(Minion::Backend::SQLite) -+%define t_requires perl(DBD::Pg) perl(DBIx::Class) perl(Config::IniFiles) perl(SQL::Translator) perl(Date::Format) perl(File::Copy::Recursive) perl(DateTime::Format::Pg) perl(Net::OpenID::Consumer) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Plugin::AssetPack) perl(aliased) perl(Config::Tiny) perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(IO::Socket::SSL) perl(Data::Dump) perl(DBIx::Class::OptimisticLocking) perl(Module::Pluggable) perl(Text::Diff) perl(Text::Markdown) perl(CommonMark) perl(JSON::Validator) perl(YAML::XS) perl(IPC::Run) perl(Archive::Extract) perl(CSS::Minifier::XS) perl(JavaScript::Minifier::XS) perl(Time::ParseDate) perl(Sort::Versions) perl(Mojo::RabbitMQ::Client) perl(BSD::Resource) perl(Cpanel::JSON::XS) perl(Pod::POM) perl(Mojo::IOLoop::ReadWriteProcess) perl(Minion) perl(Mojo::Pg) perl(Mojo::SQLite) perl(Minion::Backend::SQLite) - Name: openQA - Version: 4.6 - Release: 0 --- -2.22.0 - diff --git a/0001-Improve-rendering-summary-in-test-result-overview.patch b/0001-Improve-rendering-summary-in-test-result-overview.patch deleted file mode 100644 index 078e68d..0000000 --- a/0001-Improve-rendering-summary-in-test-result-overview.patch +++ /dev/null @@ -1,190 +0,0 @@ -From c172e8883d8f32fced5e02f9b6faaacc913df27b Mon Sep 17 00:00:00 2001 -From: Marius Kittler -Date: Mon, 22 Jul 2019 13:35:41 +0200 -Subject: [PATCH] Improve rendering summary in test result overview - ---- - lib/OpenQA/WebAPI/Controller/Test.pm | 73 ++++++++++++++++++---------- - t/10-tests_overview.t | 19 ++++++++ - t/ui/10-tests_overview.t | 2 +- - templates/test/overview.html.ep | 4 +- - 4 files changed, 69 insertions(+), 29 deletions(-) - -diff --git a/lib/OpenQA/WebAPI/Controller/Test.pm b/lib/OpenQA/WebAPI/Controller/Test.pm -index 33cb5a4b..9de2ec9b 100644 ---- a/lib/OpenQA/WebAPI/Controller/Test.pm -+++ b/lib/OpenQA/WebAPI/Controller/Test.pm -@@ -1,4 +1,4 @@ --# Copyright (C) 2015-2016 SUSE LLC -+# Copyright (C) 2015-2019 SUSE LLC - # - # This program is free software; you can redistribute it and/or modify - # it under the terms of the GNU General Public License as published by -@@ -22,9 +22,10 @@ use OpenQA::Jobs::Constants; - use OpenQA::Schema::Result::Jobs; - use OpenQA::Schema::Result::JobDependencies; - use OpenQA::WebAPI::Controller::Developer; -+use OpenQA::Utils qw(determine_web_ui_web_socket_url get_ws_status_only_url); -+use Mojo::ByteStream; - use File::Basename; - use POSIX 'strftime'; --use OpenQA::Utils qw(determine_web_ui_web_socket_url get_ws_status_only_url); - - sub referer_check { - my ($self) = @_; -@@ -569,6 +570,31 @@ sub prepare_job_results { - return (\%archs, \%results, $aggregated); - } - -+# appends the specified $distri and $version to $array_to_add_parts_to as string or if $raw as Mojo::ByteStream -+sub _add_distri_and_version_to_summary { -+ my ($array_to_add_parts_to, $distri, $version, $raw) = @_; -+ -+ for my $part ($distri, $version) { -+ # handle case when multiple distri/version parameters have been specified -+ $part = $part->{-in} if (ref $part eq 'HASH'); -+ next unless $part; -+ -+ # separate distri and version with a whitespace -+ push(@$array_to_add_parts_to, ' ') if (@$array_to_add_parts_to); -+ -+ if (ref $part eq 'ARRAY') { -+ # separate multiple distris/versions using a slash -+ if (@$part) { -+ push(@$array_to_add_parts_to, map { ($raw ? Mojo::ByteStream->new($_) : $_, '/') } @$part); -+ pop(@$array_to_add_parts_to); -+ } -+ } -+ elsif (ref $part ne 'HASH') { -+ push(@$array_to_add_parts_to, $raw ? Mojo::ByteStream->new($part) : $part); -+ } -+ } -+} -+ - # A generic query page showing test results in a configurable matrix - sub overview { - my ($self) = @_; -@@ -585,43 +611,38 @@ sub overview { - ($stash{archs}, $stash{results}, $stash{aggregated}) = $self->prepare_job_results(\@latest_jobs); - - # determine distri/version from job results if not explicitely specified via search args -- my @distris = keys %{$stash{results}}; -+ my @distris = keys %{$stash{results}}; -+ my $formatted_distri; -+ my $formatted_version; - my $only_distri = scalar @distris == 1; - if (!defined $stash{distri} && $only_distri) { -- my $distri = $stash{distri} = $distris[0]; -+ $formatted_distri = $distris[0]; - if (!defined $stash{version}) { -- my @versions = keys %{$stash{results}->{$distri}}; -- $stash{version} = $versions[0] if (scalar @versions == 1); -+ my @versions = keys %{$stash{results}->{$formatted_distri}}; -+ $formatted_version = $versions[0] if (scalar @versions == 1); - } - } - -- # determine summary name for "Overall Summary of ..." -- my $summary_name; -+ # compose summary for "Overall Summary of ..." -+ my @summary_parts; - if (@$groups) { -- $summary_name = join(', ', -- map { $self->link_to($_->name => $self->url_for('group_overview', groupid => $_->id)) } @$groups); -+ # use groups if present -+ push(@summary_parts, -+ map { ($self->link_to($_->name => $self->url_for('group_overview', groupid => $_->id)), ', ') } @$groups); -+ pop(@summary_parts); - } - else { -- my @variables = ($stash{distri}, $stash{version}); -- my @formatted_parts; -- for my $part (@variables) { -- $part = $part->{-in} if (ref $part eq 'HASH'); -- next unless $part; -- -- if (ref $part eq 'ARRAY') { -- push(@formatted_parts, join('/', @$part)); -- } -- elsif (ref $part ne 'HASH') { -- push(@formatted_parts, $part); -- } -- } -- $summary_name = join(' ', @formatted_parts) if (@formatted_parts); -+ # add pre-formatted distri and version as Mojo::ByteStream -+ _add_distri_and_version_to_summary(\@summary_parts, $formatted_distri, $formatted_version, 1); -+ -+ # add distri and version from query parameters as regular strings -+ _add_distri_and_version_to_summary(\@summary_parts, $stash{distri}, $stash{version}, 0); - } - - $self->stash( - %stash, -- summary_name => $summary_name, -- only_distri => $only_distri, -+ summary_parts => \@summary_parts, -+ only_distri => $only_distri, - ); - $self->respond_to( - json => {json => \%stash}, -diff --git a/t/10-tests_overview.t b/t/10-tests_overview.t -index f6d9d294..e8f60a7c 100644 ---- a/t/10-tests_overview.t -+++ b/t/10-tests_overview.t -@@ -65,6 +65,25 @@ like(get_summary, qr/Overall Summary of opensuse 13\.1 build 0091/i, 'specifying - $form = {distri => 'opensuse', version => '13.1', build => '0091', groupid => 1001}; - $t->get_ok('/tests/overview' => form => $form)->status_is(200); - like(get_summary, qr/Overall Summary of opensuse build 0091/i, 'specifying groupid parameter'); -+subtest 'escaping works' => sub { -+ $form = { -+ distri => '', -+ version => ['', ''], -+ build => '' -+ }; -+ $t->get_ok('/tests/overview' => form => $form)->status_is(200); -+ my $body = $t->tx->res->body; -+ unlike($body, qr//, 'no unescaped image tag for distri'); -+ unlike($body, qr/.*/, 'no unescaped image tags for version'); -+ unlike($body, qr//, 'no unescaped image tag for build'); -+ like($body, qr/<img src="distri">/, 'image tag for distri escaped'); -+ like( -+ $body, -+ qr/<img src="version1">.*<img src="version2">/, -+ 'image tags for version escaped' -+ ); -+ like($body, qr/<img src="build">/, 'image tag for build escaped'); -+}; - - # - # Overview of build 0048 -diff --git a/t/ui/10-tests_overview.t b/t/ui/10-tests_overview.t -index 1e83f93d..ea78e440 100644 ---- a/t/ui/10-tests_overview.t -+++ b/t/ui/10-tests_overview.t -@@ -208,7 +208,7 @@ subtest 'filtering by distri' => sub { - $driver->get('/tests/overview?distri=foo&distri=opensuse&distri=bar&version=13.1&build=0091'); - check_build_0091_defaults; - is( -- OpenQA::Test::Case::trim_whitespace($driver->find_element('#summary .card-header b')->get_text()), -+ OpenQA::Test::Case::trim_whitespace($driver->find_element('#summary .card-header strong')->get_text()), - 'foo/opensuse/bar 13.1', - 'filter also visible in summary' - ); -diff --git a/templates/test/overview.html.ep b/templates/test/overview.html.ep -index f618337e..2d5cb72f 100644 ---- a/templates/test/overview.html.ep -+++ b/templates/test/overview.html.ep -@@ -11,8 +11,8 @@ -
-
- Overall Summary of -- % if ($summary_name) { -- <%= b $summary_name %> -+ % if (@$summary_parts) { -+ <% for my $part (@$summary_parts) { %><%= $part %><% } %> - % } - % else { - multiple distri/version --- -2.22.0 - diff --git a/0001-Setting-to-not-cancel-parallel-parents-with-still-pe.patch b/0001-Setting-to-not-cancel-parallel-parents-with-still-pe.patch deleted file mode 100644 index 53b3ba9..0000000 --- a/0001-Setting-to-not-cancel-parallel-parents-with-still-pe.patch +++ /dev/null @@ -1,226 +0,0 @@ -From 0ce7bf2cdb236c453af5f2d6c1b021c3a47a3a77 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Tue, 19 Feb 2019 12:08:32 -0800 -Subject: [PATCH] Setting to not cancel parallel parents with still-pending - children - -As discussed extensively in -https://progress.opensuse.org/issues/46295 , openQA job logic -makes an assumption that, any time a parallel child fails or is -cancelled, its parent and any other pending children of that -parent ought to be cancelled. This is the behaviour SUSE's tests -expect, but it is not the behaviour Fedora's tests expect. In -Fedora we have several cases of clusters where a parallel parent -acts as a server to multiple unrelated child tests; if one of -the children fails, that does not mean the parent and all other -children must be cancelled. - -This patch adds a job setting to set whether parallel parents -with other pending children (and hence those children) will be -cancelled when one child fails or is cancelled. The default is -the current behaviour. For the parent and the other pending -children *not* to be canceled, the parent must have the setting -`PARALLEL_CANCEL_WHOLE_CLUSTER` set to 0 (or anything false-y; -empty string also works). - -Signed-off-by: Adam Williamson ---- - docs/WritingTests.asciidoc | 5 +++ - lib/OpenQA/Schema/Result/Jobs.pm | 47 ++++++++++++++++++------ - t/05-scheduler-cancel.t | 63 ++++++++++++++++++++++++++++++++ - 3 files changed, 104 insertions(+), 11 deletions(-) - -diff --git a/docs/WritingTests.asciidoc b/docs/WritingTests.asciidoc -index 67851937..152c10ff 100644 ---- a/docs/WritingTests.asciidoc -+++ b/docs/WritingTests.asciidoc -@@ -298,6 +298,11 @@ Then, in job templates, add test suite(s) and all of its dependent test suite(s) - have been explicitly defined in a variable for each dependent test suite. - Checkout the example sections to get a better understanding: - -+===== Handling of related jobs on failure / cancellation / restart -+openQA tries to handle things sensibly when jobs with relations like this either fail, or are manually cancelled or restarted. When a chained or parallel parent fails or is cancelled, all children will be cancelled; if the parent is restarted, all children are also restarted. When a parallel child is restarted, the parent and any other children will also be restarted. When a chained child is restarted, the parent is not restarted; this will usually be fine, but be aware that if an asset uploaded by the chained parent has been cleaned up, the child may fail immediately. To deal with this case, just restart the parent. -+ -+By default, when a parallel *child* fails or is cancelled, the parent and all other children are also cancelled. This behaviour is intended for closely-related clusters of jobs, e.g. high availability tests, where it's sensible to assume the entire test is invalid if any of its components fails. A special variable can be used to change this behaviour. Setting a parallel parent job's PARALLEL_CANCEL_WHOLE_CLUSTER to any truth-y value (e.g. 1 or 'true') changes this so that, if one of its children fails or is cancelled but the parent has other pending or active children, the parent and the other children will not be cancelled. This behaviour makes more sense if the parent is providing services to the various children but the children themselves are not closely related and a failure of one does not imply that the tests run by the other children and the parent are invalid. -+ - ====== Example: correct dependency and machine placed - ---- - There is a test suite A placed on machine 64bit-8G, -diff --git a/lib/OpenQA/Schema/Result/Jobs.pm b/lib/OpenQA/Schema/Result/Jobs.pm -index f5ec2a1d..bc4f5c39 100644 ---- a/lib/OpenQA/Schema/Result/Jobs.pm -+++ b/lib/OpenQA/Schema/Result/Jobs.pm -@@ -729,9 +729,17 @@ sub create_clones { - # internal (recursive) function for duplicate - returns hash of all jobs in the - # cluster of the current job (in no order but with relations) - sub cluster_jobs { -- my ($self, $jobs) = @_; -+ my $self = shift; -+ my %args = ( -+ jobs => {}, -+ # set to 1 when called on a cluster job being cancelled or failing; -+ # affects whether we include parallel parents with -+ # PARALLEL_CANCEL_WHOLE_CLUSTER set if they have other pending children -+ cancelmode => 0, -+ @_ -+ ); - -- $jobs ||= {}; -+ my $jobs = $args{jobs}; - return $jobs if defined $jobs->{$self->id}; - $jobs->{$self->id} = { - parallel_parents => [], -@@ -741,7 +749,7 @@ sub cluster_jobs { - - ## if we have a parallel parent, go up recursively - my $parents = $self->parents; -- while (my $pd = $parents->next) { -+ PARENT: while (my $pd = $parents->next) { - my $p = $pd->parent; - - if ($pd->dependency eq OpenQA::Schema::Result::JobDependencies->CHAINED) { -@@ -751,7 +759,23 @@ sub cluster_jobs { - } - else { - push(@{$jobs->{$self->id}->{parallel_parents}}, $p->id); -- $p->cluster_jobs($jobs); -+ my $cancelwhole = 1; -+ # check if the setting to disable cancelwhole is set: the var -+ # must exist and be set to something false-y -+ my $cwset = $p->settings_hash->{PARALLEL_CANCEL_WHOLE_CLUSTER}; -+ $cancelwhole = 0 if (defined $cwset && !$cwset); -+ if ($args{cancelmode} && !$cancelwhole) { -+ # skip calling cluster_jobs (so cancelling it and its other -+ # related jobs) if job has pending children we are not -+ # cancelling -+ my $otherchildren = $p->children; -+ CHILD: while (my $childr = $otherchildren->next) { -+ my $child = $childr->child; -+ next CHILD unless grep { $child->state eq $_ } PENDING_STATES; -+ next PARENT unless $jobs->{$child->id}; -+ } -+ } -+ $p->cluster_jobs(jobs => $jobs); - } - } - -@@ -772,7 +796,7 @@ sub cluster_children { - next if $c->clone_id; - - # do not fear the recursion -- $c->cluster_jobs($jobs); -+ $c->cluster_jobs(jobs => $jobs); - if ($cd->dependency eq OpenQA::Schema::Result::JobDependencies->PARALLEL) { - push(@{$jobs->{$self->id}->{parallel_children}}, $c->id); - } -@@ -1714,8 +1738,9 @@ sub store_column { - return $self->SUPER::store_column(%args); - } - --# parent job failed, handle running children - send stop command --sub _job_stop_child { -+# used to stop jobs with some kind of dependency relationship to another -+# job that failed or was cancelled, see cluster_jobs(), cancel() and done() -+sub _job_stop_cluster { - my ($self, $job) = @_; - - # skip ourselves -@@ -1811,9 +1836,9 @@ sub done { - $self->update(\%new_val); - - if (defined $new_val{result} && !grep { $result eq $_ } OK_RESULTS) { -- my $jobs = $self->cluster_jobs; -+ my $jobs = $self->cluster_jobs(cancelmode => 1); - for my $job (sort keys %$jobs) { -- $self->_job_stop_child($job); -+ $self->_job_stop_cluster($job); - } - } - -@@ -1841,9 +1866,9 @@ sub cancel { - if ($self->worker) { - $self->worker->send_command(command => 'cancel', job_id => $self->id); - } -- my $jobs = $self->cluster_jobs; -+ my $jobs = $self->cluster_jobs(cancelmode => 1); - for my $job (sort keys %$jobs) { -- $count += $self->_job_stop_child($job); -+ $count += $self->_job_stop_cluster($job); - } - - return $count; -diff --git a/t/05-scheduler-cancel.t b/t/05-scheduler-cancel.t -index b95a322d..917fc28f 100644 ---- a/t/05-scheduler-cancel.t -+++ b/t/05-scheduler-cancel.t -@@ -304,4 +304,67 @@ subtest 'chained parent fails -> parallel parents of children are cancelled (ski - is($jobD->result, OpenQA::Jobs::Constants::SKIPPED, 'D result is skipped'); - }; - -+subtest 'parallel child with one parent fails -> parent is cancelled' => sub { -+ my %settingsA = %settings; -+ my %settingsB = %settings; -+ $settingsA{TEST} = 'A'; -+ $settingsB{TEST} = 'B'; -+ my $jobA = _job_create(\%settingsA); -+ $settingsB{_PARALLEL_JOBS} = [$jobA->id]; -+ my $jobB = _job_create(\%settingsB); -+ -+ # set B as failed and reload A from database -+ $jobB->done(result => OpenQA::Jobs::Constants::FAILED); -+ $jobA->discard_changes; -+ -+ is($jobA->state, OpenQA::Jobs::Constants::CANCELLED, 'A state is cancelled'); -+}; -+ -+subtest 'failure behaviour for multiple parallel children' => sub { -+ my %settingsA = %settings; -+ my %settingsB = %settings; -+ my %settingsC = %settings; -+ $settingsA{TEST} = 'A'; -+ $settingsB{TEST} = 'B'; -+ $settingsC{TEST} = 'C'; -+ my $jobA = _job_create(\%settingsA); -+ $settingsB{_PARALLEL_JOBS} = [$jobA->id]; -+ $settingsC{_PARALLEL_JOBS} = [$jobA->id]; -+ my $jobB = _job_create(\%settingsB); -+ my $jobC = _job_create(\%settingsC); -+ -+ # set B as failed and reload A and C from database -+ $jobB->done(result => OpenQA::Jobs::Constants::FAILED); -+ $jobA->discard_changes; -+ $jobC->discard_changes; -+ -+ # A and C should be cancelled -+ is($jobA->state, OpenQA::Jobs::Constants::CANCELLED, 'A state is cancelled'); -+ is($jobC->state, OpenQA::Jobs::Constants::CANCELLED, 'C state is cancelled'); -+ -+ # now test in 'do not cancel parent and other children' mode -+ $settingsA{PARALLEL_CANCEL_WHOLE_CLUSTER} = '0'; -+ $jobA = _job_create(\%settingsA); -+ $settingsB{_PARALLEL_JOBS} = [$jobA->id]; -+ $settingsC{_PARALLEL_JOBS} = [$jobA->id]; -+ $jobB = _job_create(\%settingsB); -+ $jobC = _job_create(\%settingsC); -+ -+ # set B as failed and reload A and C from database -+ $jobB->done(result => OpenQA::Jobs::Constants::FAILED); -+ $jobA->discard_changes; -+ $jobC->discard_changes; -+ -+ # this time A and C should still be scheduled (*not* cancelled) -+ is($jobA->state, OpenQA::Jobs::Constants::SCHEDULED, 'new A state is scheduled'); -+ is($jobC->state, OpenQA::Jobs::Constants::SCHEDULED, 'new C state is scheduled'); -+ -+ # now set C as failed and reload A -+ $jobC->done(result => OpenQA::Jobs::Constants::FAILED); -+ $jobA->discard_changes; -+ -+ # now A *should* be cancelled -+ is($jobA->state, OpenQA::Jobs::Constants::CANCELLED, 'new A state is cancelled'); -+}; -+ - done_testing(); --- -2.21.0 - diff --git a/0001-Skip-a-sometimes-failing-test-POO-28385.patch b/0001-Skip-a-sometimes-failing-test-POO-28385.patch deleted file mode 100644 index 4411b6b..0000000 --- a/0001-Skip-a-sometimes-failing-test-POO-28385.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 7f06e0ad63c631d8332bc793910043dc6951edf1 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Fri, 24 Nov 2017 15:01:37 -0800 -Subject: [PATCH] Skip a sometimes-failing test (POO #28385) - ---- - t/24-worker.t | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/t/24-worker.t b/t/24-worker.t -index cb480052..52138119 100644 ---- a/t/24-worker.t -+++ b/t/24-worker.t -@@ -283,7 +283,7 @@ subtest 'worker status timer calculation' => sub { - $instance = 205; - $pop = $instance; - compare_timers(40, 190, ++$pop) -- && compare_timers(30, 200, $pop) -+ && $pop == 270 ? 1 == 1 : compare_timers(30, 200, $pop) - && compare_timers(70, 254, $pop) - for $instance .. 300; - --- -2.15.0 - diff --git a/0001-load_templates-fix-clean-and-reading-perl-templates.patch b/0001-load_templates-fix-clean-and-reading-perl-templates.patch new file mode 100644 index 0000000..4c109a5 --- /dev/null +++ b/0001-load_templates-fix-clean-and-reading-perl-templates.patch @@ -0,0 +1,49 @@ +From 9c0a701321391bd390d970a9c15af7883eed2987 Mon Sep 17 00:00:00 2001 +From: Adam Williamson +Date: Tue, 6 Aug 2019 16:57:38 -0700 +Subject: [PATCH] load_templates: fix --clean and reading perl templates + +This fixes two bugs in load_templates since #2204 landed. First, +`--clean` couldn't work once JobGroups was added to `@tables` +because that API endpoint returns an array (not a hash) and the +script would choke on that (it only knows how to handle hashes). +The correct fix is just to skip JobGroups when doing `--clean` +because we usually cannot delete them anyway, as non-empty job +groups cannot be deleted. There's no point trying to handle the +array. + +Second, loading perl format template files was broken by the +change to make `$datafile` the slurped contents of the file +rather than simply the filename. We have to change `do` to `eval` +to handle that change. + +Signed-off-by: Adam Williamson +--- + script/load_templates | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/script/load_templates b/script/load_templates +index ac5bf071..8f7813b1 100755 +--- a/script/load_templates ++++ b/script/load_templates +@@ -105,7 +105,7 @@ try { + dd $info; + } + catch { +- $info = do $datafile; ++ $info = eval $datafile; + if (my $error = $@) { + die "Error in data file: $error\n"; + } +@@ -227,6 +227,8 @@ sub post_entry { + + if ($options{'clean'}) { + for my $table (@tables) { ++ # we can't clean job groups as they're not deletable unless empty ++ next if ($table eq "JobGroups"); + my $res = $client->get($url->path($options{'apibase'} . '/' . decamelize($table)))->res; + if ($res->code && $res->code == 200) { + my $result = $res->json; +-- +2.22.0 + diff --git a/0001-t-api-03-auth.t-don-t-actually-wipe-asset-files-from.patch b/0001-t-api-03-auth.t-don-t-actually-wipe-asset-files-from.patch deleted file mode 100644 index 516dd1b..0000000 --- a/0001-t-api-03-auth.t-don-t-actually-wipe-asset-files-from.patch +++ /dev/null @@ -1,46 +0,0 @@ -From ee7985495d92a9bafa84cea60104469bd35b73fe Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Wed, 17 Jul 2019 20:44:28 -0700 -Subject: [PATCH] t/api/03-auth.t: don't actually wipe asset files from disk - -In testing whether we're allowed to or forbidden from deleting -assets in various cases, we actually wind up deleting an asset. -This is a problem if t/ui/07-file.t runs after this test, -because one of its subtests relies on that asset being present. -This doesn't show up in Travis runs because the UI and API tests -are run in separate jobs, but it *does* show up in package -builds - you can see the test failing in OBS openQA package -build logs, for instance. - -Signed-off-by: Adam Williamson ---- - t/api/03-auth.t | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/t/api/03-auth.t b/t/api/03-auth.t -index 8cbe1e23..5a486672 100644 ---- a/t/api/03-auth.t -+++ b/t/api/03-auth.t -@@ -24,6 +24,7 @@ BEGIN { - - use FindBin; - use lib "$FindBin::Bin/../lib"; -+use Test::MockModule; - use Test::More; - use Test::Mojo; - use Test::Warnings ':all'; -@@ -42,6 +43,11 @@ my $app = $t->app; - $t->ua(OpenQA::Client->new()->ioloop(Mojo::IOLoop->singleton)); - $t->app($app); - -+# we don't want to *actually* delete any assets when we're testing -+# whether we're allowed to or not, so let's mock that out -+my $mock_asset = Test::MockModule->new('OpenQA::Schema::Result::Assets'); -+$mock_asset->mock(remove_from_disk => sub { return 1; }); -+ - subtest 'authentication routes for plugins' => sub { - my $ensure_admin = $t->app->routes->find('api_ensure_admin'); - ok $ensure_admin, 'api_ensure_admin route found'; --- -2.22.0 - diff --git a/2192.patch b/2192.patch deleted file mode 100644 index 3eab93a..0000000 --- a/2192.patch +++ /dev/null @@ -1,223 +0,0 @@ -From 99949a7b1a5170a26a83caa9b6bd4084111950b4 Mon Sep 17 00:00:00 2001 -From: Marius Kittler -Date: Mon, 15 Jul 2019 17:14:45 +0200 -Subject: [PATCH] Stop job directly with reason 'api-failure' on API failures - -* Add test for this as well -* Add test for worker behaviour when stopped due to an API failure - * It is not supposed to upload results anymore to the web UI - because that would likely fail, too. Note that the API client - has already tried to query the web UI multiple times at this - point. Maybe the web UI even already considers the job dead. - * It is supposed to re-register which might help in situations - like https://progress.opensuse.org/issues/47060#note-19. ---- - lib/OpenQA/Worker.pm | 2 +- - lib/OpenQA/Worker/WebUIConnection.pm | 2 +- - t/24-worker-job.t | 98 +++++++++++++++++++++++----- - t/24-worker-webui-connection.t | 12 ++-- - 4 files changed, 92 insertions(+), 22 deletions(-) - -diff --git a/lib/OpenQA/Worker.pm b/lib/OpenQA/Worker.pm -index 341d37189..f9301eada 100644 ---- a/lib/OpenQA/Worker.pm -+++ b/lib/OpenQA/Worker.pm -@@ -390,7 +390,7 @@ sub stop { - sub stop_current_job { - my ($self, $reason) = @_; - -- if (my $current_job = $self->current_job) { $current_job->stop; } -+ if (my $current_job = $self->current_job) { $current_job->stop($reason); } - } - - sub kill { -diff --git a/lib/OpenQA/Worker/WebUIConnection.pm b/lib/OpenQA/Worker/WebUIConnection.pm -index 5df1c619d..23fb04861 100644 ---- a/lib/OpenQA/Worker/WebUIConnection.pm -+++ b/lib/OpenQA/Worker/WebUIConnection.pm -@@ -359,7 +359,7 @@ sub send { - my $worker = $self->worker; - my $current_webui_host = $worker->current_webui_host; - if ($current_webui_host && $current_webui_host eq $self->webui_host) { -- $worker->stop_current_job; -+ $worker->stop_current_job('api-failure'); - } - $callback->(); - return; -diff --git a/t/24-worker-job.t b/t/24-worker-job.t -index 8ed1df6ae..d4f81365d 100644 ---- a/t/24-worker-job.t -+++ b/t/24-worker-job.t -@@ -57,12 +57,14 @@ $app->log->level('debug'); - has websocket_connection => sub { OpenQA::Test::FakeWebSocketTransaction->new }; - has ua => sub { Mojo::UserAgent->new }; - has url => sub { Mojo::URL->new }; -+ has register_called => 0; - sub send { - my ($self, $method, $path, %args) = @_; - push(@{shift->sent_messages}, {path => $path, json => $args{json}}); - $args{callback}->({}) if $args{callback}; - } - sub send_status { push(@{shift->sent_messages}, @_); } -+ sub register { shift->register_called(1); } - } - { - package Test::FakeProcess; -@@ -83,7 +85,11 @@ sub wait_until_upload_concluded { - - # mock isotovideo 'engine' here - we don't actually want to start it in that test - my $engine_mock = Test::MockModule->new('OpenQA::Worker::Engines::isotovideo'); --$engine_mock->mock(engine_workit => sub { {error => 'some error'} }); -+$engine_mock->mock( -+ engine_workit => sub { -+ note('pretending isotovideo startup error'); -+ return {error => 'some error'}; -+ }); - - # log which files and assets would have been uploaded - my $job_mock = Test::MockModule->new('OpenQA::Worker::Job'); -@@ -103,18 +109,18 @@ $client->ua->connect_timeout(0.1); - - # handle/log events - my @happended_events; --$job->on( -- status_changed => sub { -- my ($event_client, $event_data) = @_; -- -- is($event_client, $job, 'job passed correctly'); -- is(ref $event_data, 'HASH', 'event data is a HASH'); -- my $status = $event_data->{status}; -- my %event = (status => $status,); -- $event{error_message} = $event_data->{error_message} if $event_data->{error_message}; -- push(@happended_events, \%event); -- Mojo::IOLoop->stop if $status eq 'stopped'; -- }); -+my $handle_job_event = sub { -+ my ($event_client, $event_data) = @_; -+ -+ is($event_client, $job, 'job passed correctly'); -+ is(ref $event_data, 'HASH', 'event data is a HASH'); -+ my $status = $event_data->{status}; -+ my %event = (status => $status,); -+ $event{error_message} = $event_data->{error_message} if $event_data->{error_message}; -+ push(@happended_events, \%event); -+ Mojo::IOLoop->stop if $status eq 'stopped'; -+}; -+$job->on(status_changed => $handle_job_event); - - # start the actual testing of going though the job live-cycle (in particular a setup failure and a - # manually stopped job) -@@ -141,15 +147,19 @@ is($job->setup_error, 'some error', 'setup error recorded'); - ok(!-e $pool_directory->child('autoinst-log.txt'), 'autoinst-log.txt file has been deleted'); - ok(-e $pool_directory->child('worker-log.txt'), 'worker log is there'); - --# pretent that the failure didn't happen to be able to continue testing -+# pretend that the failure didn't happen to be able to continue testing - # note: The actual worker always creates a new OpenQA::Worker::Job object. - $job->{_status} = 'accepted'; - - # change the job ID before starting again to better distinguish the resulting messages - $job->{_id} = 2; - --# try to start job again pretenting isotovideo could actually be spawned --$engine_mock->mock(engine_workit => sub { {child => $isotovideo} }); -+# try to start job again pretending isotovideo could actually be spawned -+$engine_mock->mock( -+ engine_workit => sub { -+ note('pretending to run isotovideo'); -+ return {child => $isotovideo}; -+ }); - combined_like( - sub { - $job->start(); -@@ -371,4 +381,60 @@ is_deeply( - is_deeply(\@uploaded_assets, [], 'no assets uploaded because this test so far has none') - or diag explain \@uploaded_assets; - -+# reset fake isotovideo and 'test results' -+$isotovideo->is_running(1); -+@happended_events = (); -+@uploaded_files = (); -+@uploaded_assets = (); -+$client->sent_messages([]); -+$client->websocket_connection->sent_messages([]); -+ -+# exercise another job live-cycle simulating an API failure -+subtest 'handling API failures' => sub { -+ # create and accept new job -+ $job = OpenQA::Worker::Job->new($worker, $client, {id => 3, URL => 'url'}); -+ $job->on(status_changed => $handle_job_event); -+ $job->accept; -+ -+ combined_like( -+ sub { -+ $job->start(); -+ wait_until_upload_concluded($job); -+ }, -+ qr/isotovideo has been started/, -+ 'isotovideo startup logged' -+ ); -+ -+ is($client->register_called, 0, 'no re-registration attempted so far'); -+ -+ combined_like( -+ sub { -+ $job->stop('api-failure'); -+ wait_until_upload_concluded($job); -+ }, -+ qr/Result: api-failure/, -+ 'job stopped after an API failure' -+ ); -+ -+ is($client->register_called, 1, 'worker tries to register itself again after an API failure'); -+ -+ # verify that all status events for the job live-cycle have been emitted -+ is_deeply( -+ \@happended_events, -+ [map { {status => $_} } (qw(accepting accepted setup running stopping stopped))], -+ 'status changes emitted' -+ ) or diag explain \@happended_events; -+ -+ # verify messages sent via REST API and websocket connection during the job live-cycle -+ # note: This can also be seen as an example for other worker implementations. -+ is($client->sent_messages->[-1]->{path}, 'jobs/3/set_done', 'set_done is still attempted to be called') -+ or diag explain $client->sent_messages; -+ -+ # verify that the upload has been skipped -+ is_deeply(\@uploaded_files, [], 'file upload skipped after API failure') -+ or diag explain \@uploaded_files; -+ is_deeply(\@uploaded_assets, [], 'asset upload skipped after API failure') -+ or diag explain \@uploaded_assets; -+}; -+ - done_testing(); -diff --git a/t/24-worker-webui-connection.t b/t/24-worker-webui-connection.t -index bd78a3e70..59696f206 100644 ---- a/t/24-worker-webui-connection.t -+++ b/t/24-worker-webui-connection.t -@@ -91,8 +91,11 @@ like( - has stop_current_job_called => 0; - has current_error => undef; - has current_job => undef; -- sub stop_current_job { shift->stop_current_job_called(1); } -- sub status { {fake_status => 1} } -+ sub stop_current_job { -+ my ($self, $reason) = @_; -+ $self->stop_current_job_called($reason); -+ } -+ sub status { {fake_status => 1} } - sub accept_job { - my ($self, $client, $job_info) = @_; - $self->current_job(OpenQA::Worker::Job->new($self, $client, $job_info)); -@@ -183,8 +186,9 @@ subtest 'attempt to register and send a command' => sub { - 'error logged', - ); - -- is($callback_invoked, 1, 'callback has been invoked'); -- is($client->worker->stop_current_job_called, 1, 'attempted to stop current job'); -+ is($callback_invoked, 1, 'callback has been invoked'); -+ is($client->worker->stop_current_job_called, -+ 'api-failure', 'attempted to stop current job with reason "api-failure"'); - - is_deeply( - \@happended_events, diff --git a/2232-rediffed.patch b/2232-rediffed.patch deleted file mode 100644 index ed864f9..0000000 --- a/2232-rediffed.patch +++ /dev/null @@ -1,906 +0,0 @@ -From e63bdcdb56539bed3e4789721a764f719be2d663 Mon Sep 17 00:00:00 2001 -From: Sebastian Riedel -Date: Tue, 23 Jul 2019 13:48:16 +0200 -Subject: [PATCH 1/3] Abstract out markdown handling into a new class - ---- - lib/OpenQA/Markdown.pm | 52 +++++++++++++++++++++ - lib/OpenQA/Schema/Result/Comments.pm | 32 +------------ - lib/OpenQA/Schema/Result/JobGroupParents.pm | 9 ++-- - lib/OpenQA/Schema/Result/JobGroups.pm | 11 ++--- - t/16-markdown.t | 42 +++++++++++++++++ - 5 files changed, 104 insertions(+), 42 deletions(-) - create mode 100644 lib/OpenQA/Markdown.pm - create mode 100644 t/16-markdown.t - -diff --git a/lib/OpenQA/Markdown.pm b/lib/OpenQA/Markdown.pm -new file mode 100644 -index 000000000..b63806a2c ---- /dev/null -+++ b/lib/OpenQA/Markdown.pm -@@ -0,0 +1,52 @@ -+# Copyright (C) 2019 SUSE Linux Products GmbH -+# -+# This program is free software; you can redistribute it and/or modify -+# it under the terms of the GNU General Public License as published by -+# the Free Software Foundation; either version 2 of the License, or -+# (at your option) any later version. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License along -+# with this program; if not, see . -+package OpenQA::Markdown; -+use Mojo::Base -strict; -+ -+require Text::Markdown; -+our @ISA = qw(Text::Markdown); -+ -+use Exporter 'import'; -+use Regexp::Common 'URI'; -+use OpenQA::Utils 'bugref_to_href'; -+ -+our @EXPORT_OK = qw(markdown_to_html); -+ -+sub markdown_to_html { -+ my $text = shift; -+ my $m = __PACKAGE__->new; -+ my $html = $m->markdown($text); -+ return $html; -+} -+ -+# TODO: Kill it with fire -+sub _DoAutoLinks { -+ my ($self, $text) = @_; -+ -+ # auto-replace bugrefs with 'a href...' -+ $text = bugref_to_href($text); -+ -+ # auto-replace every http(s) reference which is not already either html -+ # 'a href...' or markdown link '[link](url)' or enclosed by Text::Markdown -+ # URL markers '<>' -+ $text =~ s@(?])($RE{URI})@<$1>@gi; -+ -+ # For tests make sure that references into test modules and needling steps also work -+ $text =~ s{\b(t#([\w/]+))}{$1}gi; -+ -+ return $self->SUPER::_DoAutoLinks($text); -+} -+ -+1; -diff --git a/lib/OpenQA/Schema/Result/Comments.pm b/lib/OpenQA/Schema/Result/Comments.pm -index ca09bf5e2..48288a6ca 100644 ---- a/lib/OpenQA/Schema/Result/Comments.pm -+++ b/lib/OpenQA/Schema/Result/Comments.pm -@@ -21,6 +21,7 @@ use warnings; - use base 'DBIx::Class::Core'; - - use OpenQA::Utils qw(find_bugref find_bugrefs); -+use OpenQA::Markdown qw(markdown_to_html); - - __PACKAGE__->load_components(qw(Core)); - __PACKAGE__->load_components(qw(InflateColumn::DateTime Timestamps)); -@@ -144,12 +145,7 @@ sub tag { - return $+{build}, $+{type}, $+{description}, $+{version}; - } - --sub rendered_markdown { -- my ($self) = @_; -- -- my $m = CommentsMarkdownParser->new; -- return Mojo::ByteStream->new($m->markdown($self->text)); --} -+sub rendered_markdown { Mojo::ByteStream->new(markdown_to_html(shift->text)) } - - sub hash { - my ($self) = @_; -@@ -174,28 +170,4 @@ sub extended_hash { - }; - } - --package CommentsMarkdownParser; --require Text::Markdown; --our @ISA = qw(Text::Markdown); --use Regexp::Common 'URI'; --use OpenQA::Utils 'bugref_to_href'; -- --sub _DoAutoLinks { -- my ($self, $text) = @_; -- -- # auto-replace bugrefs with 'a href...' -- $text = bugref_to_href($text); -- -- # auto-replace every http(s) reference which is not already either html -- # 'a href...' or markdown link '[link](url)' or enclosed by Text::Markdown -- # URL markers '<>' -- $text =~ s@(?])($RE{URI})@<$1>@gi; -- -- # For tests make sure that references into test modules and needling steps also work -- $text =~ s{\b(t#([\w/]+))}{$1}gi; -- -- $text =~ s{(http://\S*\.gif$)}{}gi; -- $self->SUPER::_DoAutoLinks($text); --} -- - 1; -diff --git a/lib/OpenQA/Schema/Result/JobGroupParents.pm b/lib/OpenQA/Schema/Result/JobGroupParents.pm -index 4ee040678..18656e3b2 100644 ---- a/lib/OpenQA/Schema/Result/JobGroupParents.pm -+++ b/lib/OpenQA/Schema/Result/JobGroupParents.pm -@@ -21,6 +21,7 @@ use warnings; - - use base 'DBIx::Class::Core'; - -+use OpenQA::Markdown 'markdown_to_html'; - use OpenQA::Schema::JobGroupDefaults; - use OpenQA::Utils 'parse_tags_from_comments'; - use Class::Method::Modifiers; -@@ -157,11 +158,9 @@ sub jobs { - } - - sub rendered_description { -- my ($self) = @_; -- -- return unless $self->description; -- my $m = CommentsMarkdownParser->new; -- return Mojo::ByteStream->new($m->markdown($self->description)); -+ my $self = shift; -+ return undef unless my $desc = $self->description; -+ return Mojo::ByteStream->new(markdown_to_html($desc)); - } - - sub tags { -diff --git a/lib/OpenQA/Schema/Result/JobGroups.pm b/lib/OpenQA/Schema/Result/JobGroups.pm -index 1e5073386..4ba138ea7 100644 ---- a/lib/OpenQA/Schema/Result/JobGroups.pm -+++ b/lib/OpenQA/Schema/Result/JobGroups.pm -@@ -20,6 +20,7 @@ use warnings; - - use base 'DBIx::Class::Core'; - -+use OpenQA::Markdown 'markdown_to_html'; - use OpenQA::Schema::JobGroupDefaults; - use Class::Method::Modifiers; - use OpenQA::Utils qw(log_debug parse_tags_from_comments); -@@ -153,13 +154,9 @@ around 'carry_over_bugrefs' => sub { - }; - - sub rendered_description { -- my ($self) = @_; -- -- if ($self->description) { -- my $m = CommentsMarkdownParser->new; -- return Mojo::ByteStream->new($m->markdown($self->description)); -- } -- return; -+ my $self = shift; -+ return undef unless my $desc = $self->description; -+ return Mojo::ByteStream->new(markdown_to_html($desc)); - } - - sub full_name { -diff --git a/t/16-markdown.t b/t/16-markdown.t -new file mode 100644 -index 000000000..99b1b8db8 ---- /dev/null -+++ b/t/16-markdown.t -@@ -0,0 +1,42 @@ -+#!/usr/bin/env perl -w -+ -+# Copyright (C) 2016 SUSE LLC -+# -+# This program is free software; you can redistribute it and/or modify -+# it under the terms of the GNU General Public License as published by -+# the Free Software Foundation; either version 2 of the License, or -+# (at your option) any later version. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License along -+# with this program; if not, write to the Free Software Foundation, Inc., -+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -+ -+use Mojo::Base -strict; -+ -+use Test::More; -+use OpenQA::Markdown 'markdown_to_html'; -+ -+subtest 'standard markdown' => sub { -+ is markdown_to_html('Test'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html("Test\n123\n\n456 789 tset\n"), qq{

Test\n123

\n\n

456 789 tset

\n}, 'HTML rendered'; -+ is markdown_to_html('*Test*'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('[Test](http://test.com)'), qq{

Test

\n}, 'HTML rendered'; -+}; -+ -+subtest 'bugrefs' => sub { -+ is markdown_to_html('boo#123'), -+ qq{

boo#123

\n}, 'bugref expanded'; -+}; -+ -+subtest 'openQA additions' => sub { -+ is markdown_to_html('https://example.com'), -+ qq{

https://example.com

\n}, 'URL turned into a link'; -+ is markdown_to_html('t#123'), qq{

t#123

\n}, 'testref expanded'; -+}; -+ -+done_testing; - -From cb3b0fce08be64503362271fa31d12621c8ff806 Mon Sep 17 00:00:00 2001 -From: Sebastian Riedel -Date: Thu, 25 Jul 2019 17:49:57 +0200 -Subject: [PATCH 2/3] Scrub dangerous HTML from comments - ---- - cpanfile | 1 + - docker/travis_test/Dockerfile | 1 + - lib/OpenQA/Markdown.pm | 53 +++++++++++++++++++++---------- - openQA.spec | 2 +- - t/16-markdown.t | 60 +++++++++++++++++++++++++++++++---- - 5 files changed, 92 insertions(+), 25 deletions(-) - -diff --git a/cpanfile b/cpanfile -index 2596a6980..8159d6554 100644 ---- a/cpanfile -+++ b/cpanfile -@@ -77,6 +77,7 @@ requires 'Sub::Install'; - requires 'Sub::Name'; - requires 'Text::Diff'; - requires 'Text::Markdown'; -+requires 'HTML::Restrict'; - requires 'CommonMark'; - requires 'Time::HiRes'; - requires 'Time::ParseDate'; -diff --git a/docker/travis_test/Dockerfile b/docker/travis_test/Dockerfile -index 8f11a62f6..1e14c0679 100644 ---- a/docker/travis_test/Dockerfile -+++ b/docker/travis_test/Dockerfile -@@ -122,6 +122,7 @@ RUN zypper in -y -C \ - 'perl(Test::Warnings)' \ - 'perl(Text::Diff)' \ - 'perl(Text::Markdown)' \ -+ 'perl(HTML::Restrict)' \ - 'perl(CommonMark)' \ - 'perl(Time::ParseDate)' \ - 'perl(XSLoader) >= 0.24' \ -diff --git a/lib/OpenQA/Markdown.pm b/lib/OpenQA/Markdown.pm -index b63806a2c..ddbcf0913 100644 ---- a/lib/OpenQA/Markdown.pm -+++ b/lib/OpenQA/Markdown.pm -@@ -15,38 +15,57 @@ - package OpenQA::Markdown; - use Mojo::Base -strict; - --require Text::Markdown; --our @ISA = qw(Text::Markdown); -- - use Exporter 'import'; - use Regexp::Common 'URI'; - use OpenQA::Utils 'bugref_to_href'; -+use Text::Markdown; -+use HTML::Restrict; - - our @EXPORT_OK = qw(markdown_to_html); - -+# Limit tags to a safe subset -+my $RULES = { -+ a => [qw(href)], -+ blockquote => [], -+ code => [], -+ em => [], -+ img => [qw(src alt)], -+ h1 => [], -+ h2 => [], -+ h3 => [], -+ h4 => [], -+ h5 => [], -+ h6 => [], -+ hr => [], -+ li => [], -+ ol => [], -+ p => [], -+ strong => [], -+ ul => []}; -+ -+# Only allow "href=/...", "href=http://..." and "href=https://..." -+my $SCHEMES = [undef, 'http', 'https']; -+ -+my $RESTRICT = HTML::Restrict->new(rules => $RULES, uri_schemes => $SCHEMES); -+my $MARKDOWN = Text::Markdown->new; -+ - sub markdown_to_html { - my $text = shift; -- my $m = __PACKAGE__->new; -- my $html = $m->markdown($text); -- return $html; --} -- --# TODO: Kill it with fire --sub _DoAutoLinks { -- my ($self, $text) = @_; - -- # auto-replace bugrefs with 'a href...' -+ # Replace bugrefs with links - $text = bugref_to_href($text); - -- # auto-replace every http(s) reference which is not already either html -- # 'a href...' or markdown link '[link](url)' or enclosed by Text::Markdown -- # URL markers '<>' -+ # Turn all remaining URLs into links - $text =~ s@(?])($RE{URI})@<$1>@gi; - -- # For tests make sure that references into test modules and needling steps also work -+ # Turn references to test modules and needling steps into links - $text =~ s{\b(t#([\w/]+))}{$1}gi; - -- return $self->SUPER::_DoAutoLinks($text); -+ # Markdown -> HTML -+ my $html = $MARKDOWN->markdown($text); -+ -+ # Unsafe -> safe -+ return $RESTRICT->process($html); - } - - 1; -diff --git a/openQA.spec b/openQA.spec -index 4a92f2766..1f61418e2 100644 ---- a/openQA.spec -+++ b/openQA.spec -@@ -45,7 +45,7 @@ - %bcond_with tests - %endif - # runtime requirements that also the testsuite needs --%define t_requires perl(DBD::Pg) perl(DBIx::Class) perl(Config::IniFiles) perl(SQL::Translator) perl(Date::Format) perl(File::Copy::Recursive) perl(DateTime::Format::Pg) perl(Net::OpenID::Consumer) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Plugin::AssetPack) perl(aliased) perl(Config::Tiny) perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(IO::Socket::SSL) perl(Data::Dump) perl(DBIx::Class::OptimisticLocking) perl(Module::Pluggable) perl(Text::Diff) perl(Text::Markdown) perl(CommonMark) perl(JSON::Validator) perl(YAML::XS) perl(IPC::Run) perl(Archive::Extract) perl(CSS::Minifier::XS) perl(JavaScript::Minifier::XS) perl(Time::ParseDate) perl(Sort::Versions) perl(Mojo::RabbitMQ::Client) perl(BSD::Resource) perl(Cpanel::JSON::XS) perl(Pod::POM) perl(Mojo::IOLoop::ReadWriteProcess) perl(Minion) perl(Mojo::Pg) perl(Mojo::SQLite) perl(Minion::Backend::SQLite) -+%define t_requires perl(DBD::Pg) perl(DBIx::Class) perl(Config::IniFiles) perl(SQL::Translator) perl(Date::Format) perl(File::Copy::Recursive) perl(DateTime::Format::Pg) perl(Net::OpenID::Consumer) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Plugin::AssetPack) perl(aliased) perl(Config::Tiny) perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(IO::Socket::SSL) perl(Data::Dump) perl(DBIx::Class::OptimisticLocking) perl(Module::Pluggable) perl(Text::Diff) perl(Text::Markdown) perl(HTML::Restrict) perl(CommonMark) perl(JSON::Validator) perl(YAML::XS) perl(IPC::Run) perl(Archive::Extract) perl(CSS::Minifier::XS) perl(JavaScript::Minifier::XS) perl(Time::ParseDate) perl(Sort::Versions) perl(Mojo::RabbitMQ::Client) perl(BSD::Resource) perl(Cpanel::JSON::XS) perl(Pod::POM) perl(Mojo::IOLoop::ReadWriteProcess) perl(Minion) perl(Mojo::Pg) perl(Mojo::SQLite) perl(Minion::Backend::SQLite) - Name: openQA - Version: 4.6 - Release: 0 -diff --git a/t/16-markdown.t b/t/16-markdown.t -index 99b1b8db8..d6839cd7a 100644 ---- a/t/16-markdown.t -+++ b/t/16-markdown.t -@@ -22,21 +22,67 @@ use Test::More; - use OpenQA::Markdown 'markdown_to_html'; - - subtest 'standard markdown' => sub { -- is markdown_to_html('Test'), "

Test

\n", 'HTML rendered'; -- is markdown_to_html("Test\n123\n\n456 789 tset\n"), qq{

Test\n123

\n\n

456 789 tset

\n}, 'HTML rendered'; -- is markdown_to_html('*Test*'), "

Test

\n", 'HTML rendered'; -- is markdown_to_html('[Test](http://test.com)'), qq{

Test

\n}, 'HTML rendered'; -+ is markdown_to_html('Test'), '

Test

', 'HTML rendered'; -+ is markdown_to_html('# Test'), '

Test

', 'HTML rendered'; -+ is markdown_to_html('## Test'), '

Test

', 'HTML rendered'; -+ is markdown_to_html('### Test'), '

Test

', 'HTML rendered'; -+ is markdown_to_html('#### Test'), '

Test

', 'HTML rendered'; -+ is markdown_to_html('##### Test'), '
Test
', 'HTML rendered'; -+ is markdown_to_html('###### Test'), '
Test
', 'HTML rendered'; -+ is markdown_to_html("Test\n123\n\n456 789 tset\n"), qq{

Test\n123

\n\n

456 789 tset

}, 'HTML rendered'; -+ is markdown_to_html('*Test*'), '

Test

', 'HTML rendered'; -+ is markdown_to_html('**Test**'), '

Test

', 'HTML rendered'; -+ is markdown_to_html("1. a\n2. b\n3. c\n"), qq{
    \n
  1. a
    2. \n
  2. b
    3. \n
  3. c
    4. \n
}, 'HTML rendered'; -+ is markdown_to_html("* a\n* b\n* c\n"), qq{
    \n
  • a
    • \n
  • b
    • \n
  • c
    • \n
}, 'HTML rendered'; -+ is markdown_to_html('[Test](http://test.com)'), qq{

Test

}, 'HTML rendered'; -+ is markdown_to_html('[Test](/test.html)'), qq{

Test

}, 'HTML rendered'; -+ is markdown_to_html('![Test](http://test.com)'), qq{

Test

}, 'HTML rendered'; -+ is markdown_to_html('Test `123` 123'), '

Test 123 123

', 'HTML rendered'; -+ is markdown_to_html("> test\n> 123"), "
\n

test\n 123

\n
", 'HTML rendered'; -+ is markdown_to_html('---'), '
', 'HTML rendered'; - }; - - subtest 'bugrefs' => sub { - is markdown_to_html('boo#123'), -- qq{

boo#123

\n}, 'bugref expanded'; -+ qq{

boo#123

}, 'bugref expanded'; -+ is markdown_to_html('testing boo#123 123'), -+ qq{

testing boo#123 123

}, -+ 'bugref expanded'; -+ is markdown_to_html('testing boo#123 123 boo#321'), -+ qq{

testing boo#123 123} -+ . qq{ boo#321

}, -+ 'bugref expanded'; -+ is markdown_to_html("testing boo#123 \n123\n boo#321"), -+ qq{

testing boo#123 \n123\n} -+ . qq{ boo#321

}, -+ 'bugref expanded'; -+ is markdown_to_html("boo\ntesting boo#123 123\n123"), -+ qq{

boo\ntesting boo#123 123\n123

}, -+ 'bugref expanded'; - }; - - subtest 'openQA additions' => sub { - is markdown_to_html('https://example.com'), -- qq{

https://example.com

\n}, 'URL turned into a link'; -- is markdown_to_html('t#123'), qq{

t#123

\n}, 'testref expanded'; -+ qq{

https://example.com

}, 'URL turned into a link'; -+ is markdown_to_html('testing https://example.com 123'), -+ qq{

testing https://example.com 123

}, 'URL turned into a link'; -+ is markdown_to_html("t\ntesting https://example.com 123\n123"), -+ qq{

t\ntesting https://example.com 123\n123

}, 'URL turned into a link'; -+ is markdown_to_html('t#123'), qq{

t#123

}, 'testref expanded'; -+ is markdown_to_html('testing t#123 123'), qq{

testing t#123 123

}, 'testref expanded'; -+ is markdown_to_html("t\ntesting t#123 123\n123"), qq{

t\ntesting t#123 123\n123

}, -+ 'testref expanded'; -+}; -+ -+subtest 'unsafe HTML filtered out' => sub { -+ is markdown_to_html('Test 123'), '

Test 123

', 'unsafe HTML filtered'; -+ is markdown_to_html('Test'), '

Test

', 'unsafe HTML filtered'; -+ is markdown_to_html('Test [Boom!](javascript:alert("boom!")) 123'), '

Test Boom! 123

', -+ 'unsafe HTML filtered'; -+ is markdown_to_html('Totally safe'), -+ '

Totally safe

', 'unsafe HTML filtered'; -+ is markdown_to_html(qq{> hello href="javascript:alert('boom!')">*you*}), -+ qq{
\n

hello you

\n
}, 'unsafe HTML filtered'; - }; - - done_testing; - -From 044392c9c4ee12cdeddb7c9c2a301ddd9ac11c4e Mon Sep 17 00:00:00 2001 -From: Sebastian Riedel -Date: Fri, 26 Jul 2019 17:31:15 +0200 -Subject: [PATCH 3/3] Replace Text::Markdown and HTML::Restrict with CommonMark - for much better comment rendering speed and security - ---- - cpanfile | 2 - - docker/travis_test/Dockerfile | 2 - - lib/OpenQA/Markdown.pm | 69 ++++++++--------- - lib/OpenQA/Utils.pm | 1 + - openQA.spec | 2 +- - t/16-markdown.t | 142 +++++++++++++++++++++++++--------- - t/api/10-jobgroups.t | 2 +- - t/fixtures/01-jobs.pl | 2 +- - t/ui/15-comments.t | 49 ++++++------ - 9 files changed, 166 insertions(+), 105 deletions(-) - -diff --git a/cpanfile b/cpanfile -index 8159d6554..5dad876f9 100644 ---- a/cpanfile -+++ b/cpanfile -@@ -76,8 +76,6 @@ requires 'Scalar::Util'; - requires 'Sub::Install'; - requires 'Sub::Name'; - requires 'Text::Diff'; --requires 'Text::Markdown'; --requires 'HTML::Restrict'; - requires 'CommonMark'; - requires 'Time::HiRes'; - requires 'Time::ParseDate'; -diff --git a/docker/travis_test/Dockerfile b/docker/travis_test/Dockerfile -index 1e14c0679..e2a6d53a2 100644 ---- a/docker/travis_test/Dockerfile -+++ b/docker/travis_test/Dockerfile -@@ -121,8 +121,6 @@ RUN zypper in -y -C \ - 'perl(Socket::MsgHdr)' \ - 'perl(Test::Warnings)' \ - 'perl(Text::Diff)' \ -- 'perl(Text::Markdown)' \ -- 'perl(HTML::Restrict)' \ - 'perl(CommonMark)' \ - 'perl(Time::ParseDate)' \ - 'perl(XSLoader) >= 0.24' \ -diff --git a/lib/OpenQA/Markdown.pm b/lib/OpenQA/Markdown.pm -index ddbcf0913..cf6745ce0 100644 ---- a/lib/OpenQA/Markdown.pm -+++ b/lib/OpenQA/Markdown.pm -@@ -17,55 +17,54 @@ use Mojo::Base -strict; - - use Exporter 'import'; - use Regexp::Common 'URI'; --use OpenQA::Utils 'bugref_to_href'; --use Text::Markdown; --use HTML::Restrict; -+use OpenQA::Utils qw(bugref_regex bugurl); -+use CommonMark; - --our @EXPORT_OK = qw(markdown_to_html); -+our @EXPORT_OK = qw(bugref_to_markdown is_light_color markdown_to_html); - --# Limit tags to a safe subset --my $RULES = { -- a => [qw(href)], -- blockquote => [], -- code => [], -- em => [], -- img => [qw(src alt)], -- h1 => [], -- h2 => [], -- h3 => [], -- h4 => [], -- h5 => [], -- h6 => [], -- hr => [], -- li => [], -- ol => [], -- p => [], -- strong => [], -- ul => []}; -+my $RE = bugref_regex; - --# Only allow "href=/...", "href=http://..." and "href=https://..." --my $SCHEMES = [undef, 'http', 'https']; -+sub bugref_to_markdown { -+ my $text = shift; -+ $text =~ s/$RE/"[$+{match}](" . bugurl($+{match}) . ')'/geio; -+ return $text; -+} - --my $RESTRICT = HTML::Restrict->new(rules => $RULES, uri_schemes => $SCHEMES); --my $MARKDOWN = Text::Markdown->new; -+sub is_light_color { -+ my $color = shift; -+ return undef unless $color =~ m/^#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})$/; -+ my ($red, $green, $blue) = ($1, $2, $3); -+ my $sum = hex($red) + hex($green) + hex($blue); -+ return $sum > 380; -+} - - sub markdown_to_html { - my $text = shift; - -- # Replace bugrefs with links -- $text = bugref_to_href($text); -+ $text = bugref_to_markdown($text); - - # Turn all remaining URLs into links -- $text =~ s@(?])($RE{URI})@<$1>@gi; -+ $text =~ s/(?])($RE{URI})/<$1>/gio; - - # Turn references to test modules and needling steps into links -- $text =~ s{\b(t#([\w/]+))}{$1}gi; -+ $text =~ s!\b(t#([\w/]+))![$1](/tests/$2)!gi; - -- # Markdown -> HTML -- my $html = $MARKDOWN->markdown($text); -+ my $html = CommonMark->markdown_to_html($text); -+ -+ # Custom markup "{{color:#ff0000|Some text}}" -+ $html =~ s/(\{\{([^|]+?)\|(.*?)\}\})/_custom($1, $2, $3)/ge; -+ -+ return $html; -+} - -- # Unsafe -> safe -- return $RESTRICT->process($html); -+sub _custom { -+ my ($full, $rules, $text) = @_; -+ if ($rules =~ /^color:(#[a-fA-F0-9]{6})$/) { -+ my $color = $1; -+ my $bg = is_light_color($color) ? 'black' : 'white'; -+ return qq{$text}; -+ } -+ return $full; - } - - 1; -diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm -index d1ac1fa2f..0dc27766f 100644 ---- a/lib/OpenQA/Utils.pm -+++ b/lib/OpenQA/Utils.pm -@@ -63,6 +63,7 @@ our @EXPORT = qw( - &parse_assets_from_settings - &find_bugref - &find_bugrefs -+ bugref_regex - &bugurl - &bugref_to_href - &href_to_bugref -diff --git a/openQA.spec b/openQA.spec -index 1f61418e2..5984ed027 100644 ---- a/openQA.spec -+++ b/openQA.spec -@@ -45,7 +45,7 @@ - %bcond_with tests - %endif - # runtime requirements that also the testsuite needs --%define t_requires perl(DBD::Pg) perl(DBIx::Class) perl(Config::IniFiles) perl(SQL::Translator) perl(Date::Format) perl(File::Copy::Recursive) perl(DateTime::Format::Pg) perl(Net::OpenID::Consumer) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Plugin::AssetPack) perl(aliased) perl(Config::Tiny) perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(IO::Socket::SSL) perl(Data::Dump) perl(DBIx::Class::OptimisticLocking) perl(Module::Pluggable) perl(Text::Diff) perl(Text::Markdown) perl(HTML::Restrict) perl(CommonMark) perl(JSON::Validator) perl(YAML::XS) perl(IPC::Run) perl(Archive::Extract) perl(CSS::Minifier::XS) perl(JavaScript::Minifier::XS) perl(Time::ParseDate) perl(Sort::Versions) perl(Mojo::RabbitMQ::Client) perl(BSD::Resource) perl(Cpanel::JSON::XS) perl(Pod::POM) perl(Mojo::IOLoop::ReadWriteProcess) perl(Minion) perl(Mojo::Pg) perl(Mojo::SQLite) perl(Minion::Backend::SQLite) -+%define t_requires perl(DBD::Pg) perl(DBIx::Class) perl(Config::IniFiles) perl(SQL::Translator) perl(Date::Format) perl(File::Copy::Recursive) perl(DateTime::Format::Pg) perl(Net::OpenID::Consumer) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Plugin::AssetPack) perl(aliased) perl(Config::Tiny) perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(IO::Socket::SSL) perl(Data::Dump) perl(DBIx::Class::OptimisticLocking) perl(Module::Pluggable) perl(Text::Diff) perl(CommonMark) perl(JSON::Validator) perl(YAML::XS) perl(IPC::Run) perl(Archive::Extract) perl(CSS::Minifier::XS) perl(JavaScript::Minifier::XS) perl(Time::ParseDate) perl(Sort::Versions) perl(Mojo::RabbitMQ::Client) perl(BSD::Resource) perl(Cpanel::JSON::XS) perl(Pod::POM) perl(Mojo::IOLoop::ReadWriteProcess) perl(Minion) perl(Mojo::Pg) perl(Mojo::SQLite) perl(Minion::Backend::SQLite) - Name: openQA - Version: 4.6 - Release: 0 -diff --git a/t/16-markdown.t b/t/16-markdown.t -index d6839cd7a..e0804169e 100644 ---- a/t/16-markdown.t -+++ b/t/16-markdown.t -@@ -1,6 +1,6 @@ - #!/usr/bin/env perl -w - --# Copyright (C) 2016 SUSE LLC -+# Copyright (C) 2019 SUSE LLC - # - # This program is free software; you can redistribute it and/or modify - # it under the terms of the GNU General Public License as published by -@@ -18,71 +18,137 @@ - - use Mojo::Base -strict; - -+BEGIN { unshift @INC, 'lib' } -+ - use Test::More; --use OpenQA::Markdown 'markdown_to_html'; -+use OpenQA::Markdown qw(bugref_to_markdown is_light_color markdown_to_html); - - subtest 'standard markdown' => sub { -- is markdown_to_html('Test'), '

Test

', 'HTML rendered'; -- is markdown_to_html('# Test'), '

Test

', 'HTML rendered'; -- is markdown_to_html('## Test'), '

Test

', 'HTML rendered'; -- is markdown_to_html('### Test'), '

Test

', 'HTML rendered'; -- is markdown_to_html('#### Test'), '

Test

', 'HTML rendered'; -- is markdown_to_html('##### Test'), '
Test
', 'HTML rendered'; -- is markdown_to_html('###### Test'), '
Test
', 'HTML rendered'; -- is markdown_to_html("Test\n123\n\n456 789 tset\n"), qq{

Test\n123

\n\n

456 789 tset

}, 'HTML rendered'; -- is markdown_to_html('*Test*'), '

Test

', 'HTML rendered'; -- is markdown_to_html('**Test**'), '

Test

', 'HTML rendered'; -- is markdown_to_html("1. a\n2. b\n3. c\n"), qq{
    \n
  1. a
    2. \n
  2. b
    3. \n
  3. c
    4. \n
}, 'HTML rendered'; -- is markdown_to_html("* a\n* b\n* c\n"), qq{
    \n
  • a
    • \n
  • b
    • \n
  • c
    • \n
}, 'HTML rendered'; -- is markdown_to_html('[Test](http://test.com)'), qq{

Test

}, 'HTML rendered'; -- is markdown_to_html('[Test](/test.html)'), qq{

Test

}, 'HTML rendered'; -- is markdown_to_html('![Test](http://test.com)'), qq{

Test

}, 'HTML rendered'; -- is markdown_to_html('Test `123` 123'), '

Test 123 123

', 'HTML rendered'; -- is markdown_to_html("> test\n> 123"), "
\n

test\n 123

\n
", 'HTML rendered'; -- is markdown_to_html('---'), '
', 'HTML rendered'; -+ is markdown_to_html('Test'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('# Test #'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('# Test'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('## Test'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('### Test'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('#### Test'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('##### Test'), "
Test
\n", 'HTML rendered'; -+ is markdown_to_html('###### Test'), "
Test
\n", 'HTML rendered'; -+ is markdown_to_html("Test\n123\n\n456 789 tset\n"), qq{

Test\n123

\n

456 789 tset

\n}, 'HTML rendered'; -+ is markdown_to_html('*Test*'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html('**Test**'), "

Test

\n", 'HTML rendered'; -+ is markdown_to_html("1. a\n2. b\n3. c\n"), qq{
    \n
  1. a
    2. \n
  2. b
    3. \n
  3. c
    4. \n
\n}, 'HTML rendered'; -+ is markdown_to_html("* a\n* b\n* c\n"), qq{
    \n
  • a
    • \n
  • b
    • \n
  • c
    • \n
\n}, 'HTML rendered'; -+ is markdown_to_html('[Test](http://test.com)'), qq{

Test

\n}, 'HTML rendered'; -+ is markdown_to_html('[Test](/test.html)'), qq{

Test

\n}, 'HTML rendered'; -+ is markdown_to_html('![Test](http://test.com)'), qq{

Test

\n}, -+ 'HTML rendered'; -+ is markdown_to_html('Test `123` 123'), "

Test 123 123

\n", 'HTML rendered'; -+ is markdown_to_html("> test\n> 123"), "
\n

test\n123

\n
\n", 'HTML rendered'; -+ is markdown_to_html('---'), "
\n", 'HTML rendered'; - }; - - subtest 'bugrefs' => sub { - is markdown_to_html('boo#123'), -- qq{

boo#123

}, 'bugref expanded'; -+ qq{

boo#123

\n}, 'bugref expanded'; - is markdown_to_html('testing boo#123 123'), -- qq{

testing boo#123 123

}, -+ qq{

testing boo#123 123

\n}, - 'bugref expanded'; - is markdown_to_html('testing boo#123 123 boo#321'), - qq{

testing boo#123 123} -- . qq{ boo#321

}, -+ . qq{ boo#321

\n}, - 'bugref expanded'; -- is markdown_to_html("testing boo#123 \n123\n boo#321"), -- qq{

testing boo#123 \n123\n} -- . qq{ boo#321

}, -+ is markdown_to_html("testing boo#123\n123\n boo#321"), -+ qq{

testing boo#123\n123\n} -+ . qq{boo#321

\n}, - 'bugref expanded'; - is markdown_to_html("boo\ntesting boo#123 123\n123"), -- qq{

boo\ntesting boo#123 123\n123

}, -+ qq{

boo\ntesting boo#123 123\n123

\n}, - 'bugref expanded'; - }; - - subtest 'openQA additions' => sub { - is markdown_to_html('https://example.com'), -- qq{

https://example.com

}, 'URL turned into a link'; -+ qq{

https://example.com

\n}, 'URL turned into a link'; - is markdown_to_html('testing https://example.com 123'), -- qq{

testing https://example.com 123

}, 'URL turned into a link'; -+ qq{

testing https://example.com 123

\n}, 'URL turned into a link'; - is markdown_to_html("t\ntesting https://example.com 123\n123"), -- qq{

t\ntesting https://example.com 123\n123

}, 'URL turned into a link'; -- is markdown_to_html('t#123'), qq{

t#123

}, 'testref expanded'; -- is markdown_to_html('testing t#123 123'), qq{

testing t#123 123

}, 'testref expanded'; -- is markdown_to_html("t\ntesting t#123 123\n123"), qq{

t\ntesting t#123 123\n123

}, -+ qq{

t\ntesting https://example.com 123\n123

\n}, 'URL turned into a link'; -+ -+ is markdown_to_html('t#123'), qq{

t#123

\n}, 'testref expanded'; -+ is markdown_to_html('testing t#123 123'), qq{

testing t#123 123

\n}, -+ 'testref expanded'; -+ is markdown_to_html("t\ntesting t#123 123\n123"), qq{

t\ntesting t#123 123\n123

\n}, - 'testref expanded'; -+ -+ is markdown_to_html(qq{{{color:#ffffff|"Text"}}}), -+ qq{

"Text"

\n}, -+ 'White text'; -+ is markdown_to_html("test {{color:#ff0000|Text}} 123"), -+ qq{

test Text 123

\n}, 'Red text'; -+ is markdown_to_html("test {{color:#FFFFFF|Text}} 123"), -+ qq{

test Text 123

\n}, 'White text'; -+ is markdown_to_html("test {{color:#00ff00|Some Text}} 123"), -+ qq{

test Some Text 123

\n}, 'Green text'; -+ is markdown_to_html("test {{color:#00ff00|Some Text}} 123 {{color:#0000ff|Also {w}orks}}"), -+ qq{

test Some Text 123} -+ . qq{ Also {w}orks

\n}, -+ 'Green and blue text'; -+ is markdown_to_html("test {{ color: #00ff00 | Some Text }} 123"), -+ "

test {{ color: #00ff00 | Some Text }} 123

\n", 'Extra whitespace is not allowed'; -+ is markdown_to_html("test {{color:javascript|Text}} 123"), -+ qq{

test {{color:javascript|Text}} 123

\n}, 'Invalid custom tag'; -+ is markdown_to_html(qq{test {{javascript:alert("test")|Text}} 123}), -+ qq{

test {{javascript:alert("test")|Text}} 123

\n}, 'Invalid custom tag'; - }; - - subtest 'unsafe HTML filtered out' => sub { -- is markdown_to_html('Test 123'), '

Test 123

', 'unsafe HTML filtered'; -- is markdown_to_html('Test'), '

Test

', 'unsafe HTML filtered'; -- is markdown_to_html('Test [Boom!](javascript:alert("boom!")) 123'), '

Test Boom! 123

', -+ is markdown_to_html('Test 123'), -+ "

Test alert("boom!"); 123

\n", -+ 'unsafe HTML filtered'; -+ is markdown_to_html('Test'), "

Test

\n", -+ 'unsafe HTML filtered'; -+ is markdown_to_html('Test [Boom!](javascript:alert("boom!")) 123'), qq{

Test Boom! 123

\n}, - 'unsafe HTML filtered'; - is markdown_to_html('Totally safe'), -- '

Totally safe

', 'unsafe HTML filtered'; -+ "

Totally safe

\n", 'unsafe HTML filtered'; - is markdown_to_html(qq{> hello href="javascript:alert('boom!')">*you*}), -- qq{
\n

hello you

\n
}, 'unsafe HTML filtered'; -+ qq{
\n

hello you

\n
\n}, -+ 'unsafe HTML filtered'; -+ is markdown_to_html('{{color:#0000ff|Test}}'), -+ qq{

} -+ . qq{Test

\n}, -+ 'unsafe HTML filtered'; -+}; -+ -+subtest 'bugrefs to markdown' => sub { -+ is bugref_to_markdown('bnc#9876'), '[bnc#9876](https://bugzilla.suse.com/show_bug.cgi?id=9876)', 'right markdown'; -+ is bugref_to_markdown('bsc#9876'), '[bsc#9876](https://bugzilla.suse.com/show_bug.cgi?id=9876)', 'right markdown'; -+ is bugref_to_markdown('boo#9876'), '[boo#9876](https://bugzilla.opensuse.org/show_bug.cgi?id=9876)', -+ 'right markdown'; -+ is bugref_to_markdown('bgo#9876'), '[bgo#9876](https://bugzilla.gnome.org/show_bug.cgi?id=9876)', 'right markdown'; -+ is bugref_to_markdown('brc#9876'), '[brc#9876](https://bugzilla.redhat.com/show_bug.cgi?id=9876)', 'right markdown'; -+ is bugref_to_markdown('bko#9876'), '[bko#9876](https://bugzilla.kernel.org/show_bug.cgi?id=9876)', 'right markdown'; -+ is bugref_to_markdown('poo#9876'), '[poo#9876](https://progress.opensuse.org/issues/9876)', 'right markdown'; -+ is bugref_to_markdown('gh#foo/bar#1234'), '[gh#foo/bar#1234](https://github.com/foo/bar/issues/1234)', -+ 'right markdown'; -+ is bugref_to_markdown('kde#9876'), '[kde#9876](https://bugs.kde.org/show_bug.cgi?id=9876)', 'right markdown'; -+ is bugref_to_markdown('fdo#9876'), '[fdo#9876](https://bugs.freedesktop.org/show_bug.cgi?id=9876)', -+ 'right markdown'; -+ is bugref_to_markdown('jsc#9876'), '[jsc#9876](https://jira.suse.de/browse/9876)', 'right markdown'; -+ is bugref_to_markdown("boo#9876\n\ntest boo#211\n"), -+ "[boo#9876](https://bugzilla.opensuse.org/show_bug.cgi?id=9876)\n\n" -+ . "test [boo#211](https://bugzilla.opensuse.org/show_bug.cgi?id=211)\n", -+ 'right markdown'; -+}; -+ -+subtest 'color detection' => sub { -+ ok !is_light_color('#000000'), 'dark'; -+ ok !is_light_color('#ff0000'), 'dark'; -+ ok !is_light_color('#00ff00'), 'dark'; -+ ok !is_light_color('#0000ff'), 'dark'; -+ ok !is_light_color('#0000FF'), 'dark'; -+ ok is_light_color('#ffffff'), 'light'; -+ ok is_light_color('#FFFFFF'), 'light'; -+ ok !is_light_color('test'), 'not a color at all'; - }; - - done_testing; -diff --git a/t/api/10-jobgroups.t b/t/api/10-jobgroups.t -index f45b8fd39..6240638b0 100644 ---- a/t/api/10-jobgroups.t -+++ b/t/api/10-jobgroups.t -@@ -70,7 +70,7 @@ subtest 'list job groups' => sub() { - keep_important_logs_in_days => 120, - default_priority => 50, - carry_over_bugrefs => 1, -- description => "##Test description\n\nwith bugref bsc#1234", -+ description => "## Test description\n\nwith bugref bsc#1234", - template => undef, - keep_results_in_days => 365, - keep_important_results_in_days => 0, -diff --git a/t/fixtures/01-jobs.pl b/t/fixtures/01-jobs.pl -index c7b27b840..f7e07a409 100644 ---- a/t/fixtures/01-jobs.pl -+++ b/t/fixtures/01-jobs.pl -@@ -32,7 +32,7 @@ - id => 1001, - name => 'opensuse', - sort_order => 0, -- description => "##Test description\n\nwith bugref bsc#1234", -+ description => "## Test description\n\nwith bugref bsc#1234", - }, - JobGroups => { - id => 1002, -diff --git a/t/ui/15-comments.t b/t/ui/15-comments.t -index 30aa9aca4..bd841f418 100644 ---- a/t/ui/15-comments.t -+++ b/t/ui/15-comments.t -@@ -194,26 +194,29 @@ subtest 'commenting in the group overview' => sub { - }; - - subtest 'URL auto-replace' => sub { -- $driver->find_element_by_id('text')->send_keys(' -- foo@bar foo#bar should not be detected as bugref -- bsc#2436346bla should not be detected, too -- bsc#2436347bla2 -- https://openqa.example.com/foo/bar: http://localhost:9562 -- https://openqa.example.com/tests/181148 (reference http://localhost/foo/bar ) -- bsc#1234 boo#2345,poo#3456 t#4567 "some quotes suff should not cause problems" -- t#5678/modules/welcome/steps/1 -- https://progress.opensuse.org/issues/6789 -- https://bugzilla.novell.com/show_bug.cgi?id=7890 -- [bsc#1000629](https://bugzilla.suse.com/show_bug.cgi?id=1000629) -- bsc#1000630 -- bnc#1246 -- gh#os-autoinst/openQA#1234 -- https://github.com/os-autoinst/os-autoinst/pull/960 -- bgo#768954 brc#1401123 -- https://bugzilla.gnome.org/show_bug.cgi?id=690345 -- https://bugzilla.redhat.com/show_bug.cgi?id=343098 -- [bsc#1043970](https://bugzilla.suse.com/show_bug.cgi?id=1043970 "Bugref at end of title: bsc#1043760")' -- ); -+ my $build_url = $driver->get_current_url(); -+ $build_url =~ s/\?.*//; -+ OpenQA::Utils::log_debug('build_url: ' . $build_url); -+ $driver->find_element_by_id('text')->send_keys(<<'EOF'); -+foo@bar foo#bar should not be detected as bugref -+bsc#2436346bla should not be detected, too -+bsc#2436347bla2 -+https://openqa.example.com/foo/bar: http://localhost:9562 -+https://openqa.example.com/tests/181148 (reference http://localhost/foo/bar ) -+bsc#1234 boo#2345,poo#3456 t#4567 "some quotes suff should not cause problems" -+t#5678/modules/welcome/steps/1 -+https://progress.opensuse.org/issues/6789 -+https://bugzilla.novell.com/show_bug.cgi?id=7890 -+[bsc#1000629](https://bugzilla.suse.com/show_bug.cgi?id=1000629) -+bsc#1000630 -+bnc#1246 -+gh#os-autoinst/openQA#1234 -+https://github.com/os-autoinst/os-autoinst/pull/960 -+bgo#768954 brc#1401123 -+https://bugzilla.gnome.org/show_bug.cgi?id=690345 -+https://bugzilla.redhat.com/show_bug.cgi?id=343098 -+[bsc#1043970](https://bugzilla.suse.com/show_bug.cgi?id=1043970 "Bugref at end of title: bsc#1043760") -+EOF - $driver->find_element_by_id('submitComment')->click(); - wait_for_ajax; - -@@ -226,8 +229,7 @@ subtest 'URL auto-replace' => sub { - qr(bsc#1234 boo#2345,poo#3456 t#4567 .*poo#6789 bsc#7890 bsc#1000629 bsc#1000630 bnc#1246 gh#os-autoinst/openQA#1234 gh#os-autoinst/os-autoinst#960 bgo#768954 brc#1401123) - ); - my @urls = $driver->find_elements('div.media-comment a', 'css'); -- is(scalar @urls, 21); -- is((shift @urls)->get_text(), 'https://openqa.example.com/foo/bar', "url1"); -+ is(scalar @urls, 19); - is((shift @urls)->get_text(), 'http://localhost:9562', "url2"); - is((shift @urls)->get_text(), 'https://openqa.example.com/tests/181148', "url3"); - is((shift @urls)->get_text(), 'http://localhost/foo/bar', "url4"); -@@ -239,7 +241,6 @@ qr(bsc#1234 boo#2345,poo#3456 t#4567 .*poo#6789 bsc#7890 bsc#1000629 bsc#1000630 - is((shift @urls)->get_text(), 'poo#6789', "url10"); - is((shift @urls)->get_text(), 'bsc#7890', "url11"); - is((shift @urls)->get_text(), 'bsc#1000629', "url12"); -- is((shift @urls)->get_text(), 'bsc#1000630', "url13"); - is((shift @urls)->get_text(), 'bnc#1246', "url14"); - is((shift @urls)->get_text(), 'gh#os-autoinst/openQA#1234', "url15"); - is((shift @urls)->get_text(), 'gh#os-autoinst/os-autoinst#960', "url16"); -@@ -250,7 +251,6 @@ qr(bsc#1234 boo#2345,poo#3456 t#4567 .*poo#6789 bsc#7890 bsc#1000629 bsc#1000630 - is((shift @urls)->get_text(), 'bsc#1043970', "url21"); - - my @urls2 = $driver->find_elements('div.media-comment a', 'css'); -- is((shift @urls2)->get_attribute('href'), 'https://openqa.example.com/foo/bar', "url1-href"); - is((shift @urls2)->get_attribute('href'), 'http://localhost:9562/', "url2-href"); - is((shift @urls2)->get_attribute('href'), 'https://openqa.example.com/tests/181148', "url3-href"); - is((shift @urls2)->get_attribute('href'), 'http://localhost/foo/bar', "url4-href"); -@@ -262,7 +262,6 @@ qr(bsc#1234 boo#2345,poo#3456 t#4567 .*poo#6789 bsc#7890 bsc#1000629 bsc#1000630 - is((shift @urls2)->get_attribute('href'), 'https://progress.opensuse.org/issues/6789', "url10-href"); - is((shift @urls2)->get_attribute('href'), 'https://bugzilla.suse.com/show_bug.cgi?id=7890', "url11-href"); - is((shift @urls2)->get_attribute('href'), 'https://bugzilla.suse.com/show_bug.cgi?id=1000629', "url12-href"); -- is((shift @urls2)->get_attribute('href'), 'https://bugzilla.suse.com/show_bug.cgi?id=1000630', "url13-href"); - is((shift @urls2)->get_attribute('href'), 'https://bugzilla.suse.com/show_bug.cgi?id=1246', "url14-href"); - is((shift @urls2)->get_attribute('href'), 'https://github.com/os-autoinst/openQA/issues/1234', "url15-href"); - is((shift @urls2)->get_attribute('href'), 'https://github.com/os-autoinst/os-autoinst/issues/960', "url16-href"); diff --git a/23-fedora-messaging.t b/23-fedora-messaging.t new file mode 100644 index 0000000..01c392d --- /dev/null +++ b/23-fedora-messaging.t @@ -0,0 +1,302 @@ +#! /usr/bin/perl + +# Copyright (C) 2016-2019 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +BEGIN { + unshift @INC, 'lib'; +} + +use Mojo::Base; +use Mojo::IOLoop; + +use FindBin; +use lib "$FindBin::Bin/lib"; +use OpenQA::Client; +use OpenQA::Jobs::Constants; +use OpenQA::Test::Database; +use Test::MockModule; +use Test::More; +use Test::Mojo; +use Test::Warnings; +use Mojo::File qw(tempdir path); +use OpenQA::WebAPI::Plugin::AMQP; + +my %published; +my $plugin_mock_callcount; + +my $uuid_mock = Test::MockModule->new('UUID::URandom'); +$uuid_mock->mock( + create_uuid_string => sub { + return '9fbba7a5-2402-4f6b-a20e-af478eee05f5'; + } +); + +# we mock the parent class here so we can test the child class +my $plugin_mock = Test::MockModule->new('OpenQA::WebAPI::Plugin::AMQP'); +$plugin_mock->mock( + publish_amqp => sub { + my ($self, $topic, $body, $headerframe) = @_; + # ignore the non-fedoraci messages, makes it easier to + # understand the expected call counts + if ($topic =~ /\.ci\./) { + $plugin_mock_callcount++; + # strip the time-based bits, at least till + # https://github.com/cho45/Test-Time/issues/14 is done + delete $body->{'generated_at'}; + delete $headerframe->{'headers'}->{'sent-at'}; + $published{$topic} = [$body, $headerframe]; + } + } +); + +OpenQA::Test::Database->new->create(); + +# this test also serves to test plugin loading via config file +my $conf = << 'EOF'; +[global] +plugins=FedoraMessaging +base_url=https://openqa.stg.fedoraproject.org +[amqp] +topic_prefix=org.fedoraproject.stg +EOF + +my $tempdir = tempdir; +$ENV{OPENQA_CONFIG} = $tempdir; +path($ENV{OPENQA_CONFIG})->make_path->child("openqa.ini")->spurt($conf); + +my $t = Test::Mojo->new('OpenQA::WebAPI'); + +# XXX: Test::Mojo loses its app when setting a new ua +# https://github.com/kraih/mojo/issues/598 +my $app = $t->app; +$t->ua( + OpenQA::Client->new(apikey => 'PERCIVALKEY02', apisecret => 'PERCIVALSECRET02')->ioloop(Mojo::IOLoop->singleton)); +$t->app($app); + +my $settings = { + DISTRI => 'Unicorn', + FLAVOR => 'pink', + VERSION => '42', + BUILD => 'Fedora-Rawhide-20180129.n.0', + TEST => 'rainbow', + ISO => 'whatever.iso', + DESKTOP => 'DESKTOP', + KVM => 'KVM', + ISO_MAXSIZE => 1, + MACHINE => "RainbowPC", + ARCH => 'x86_64', + SUBVARIANT => 'workstation', + TEST_TARGET => 'ISO', +}; + +my $expected_artifact = { + id => 'Fedora-Rawhide-20180129.n.0', + type => 'productmd-compose', + compose_type => 'nightly', +}; + +my $expected_contact = { + name => 'Fedora openQA', + team => 'Fedora QA', + url => 'https://openqa.stg.fedoraproject.org', + docs => 'https://fedoraproject.org/wiki/OpenQA', + irc => '#fedora-qa', + email => 'qa-devel@lists.fedoraproject.org', +}; + +my $expected_image = { + id => 'whatever.iso', + name => 'whatever.iso', + type => 'ISO', +}; + +my $expected_pipeline = { + id => 'openqa.Fedora-Rawhide-20180129.n.0.rainbow.RainbowPC.pink.x86_64', + name => 'openqa.Fedora-Rawhide-20180129.n.0.rainbow.RainbowPC.pink.x86_64', +}; + +my $expected_run = { + url => '', + log => '', + id => '', +}; + +my $expected_system = { + os => 'fedora-42', + provider => 'openqa', + architecture => 'x86_64', + variant => 'workstation', +}; + +my $expected_test = { + category => 'validation', + type => 'rainbow RainbowPC pink x86_64', + namespace => 'compose', + lifetime => 240, +}; + +my $expected_error; + +my $expected_version = '0.2.1'; + +sub get_expected { + my $expected = { + artifact => $expected_artifact, + contact => $expected_contact, + image => $expected_image, + pipeline => $expected_pipeline, + run => $expected_run, + system => [$expected_system], + test => $expected_test, + version => $expected_version, + }; + $expected->{'error'} = $expected_error if ($expected_error); + return $expected; +} + +# create a job via API +my $job; +my $newjob; +subtest 'create job' => sub { + # reset the call count + $plugin_mock_callcount = 0; + $t->post_ok("/api/v1/jobs" => form => $settings)->status_is(200); + ok($job = $t->tx->res->json->{id}, 'got ID of new job'); + ok($plugin_mock_callcount == 1, 'plugin mock was called'); + my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.queued'}}; + $expected_run = { + url => "https://openqa.stg.fedoraproject.org/tests/$job", + log => "https://openqa.stg.fedoraproject.org/tests/$job/file/autoinst-log.txt", + id => $job, + }; + my $expected = get_expected; + my $expected_headerframe = { + headers => { + fedora_messaging_severity => 20, + fedora_messaging_schema => 'fedora_messaging.message:Message', + }, + content_encoding => 'utf-8', + delivery_mode => 2, + # if this fails, probably means mock broke + message_id => '9fbba7a5-2402-4f6b-a20e-af478eee05f5', + }; + is_deeply($body, $expected, 'job create triggers standardized amqp'); + is_deeply($headerframe, $expected_headerframe, 'amqp header frame is as expected'); +}; + +subtest 'mark job as done' => sub { + $plugin_mock_callcount = 0; + $t->post_ok("/api/v1/jobs/$job/set_done")->status_is(200); + ok($plugin_mock_callcount == 1, 'mock was called'); + my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.complete'}}; + $expected_test->{'result'} = 'failed'; + delete $expected_test->{'lifetime'}; + my $expected = get_expected; + is_deeply($body, $expected, 'job done (failed) triggers standardized amqp'); +}; + +subtest 'duplicate and cancel job' => sub { + $plugin_mock_callcount = 0; + $t->post_ok("/api/v1/jobs/$job/duplicate")->status_is(200); + $newjob = $t->tx->res->json->{id}; + ok($plugin_mock_callcount == 1, 'mock was called'); + my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.queued'}}; + $expected_run = { + clone_of => $job, + url => "https://openqa.stg.fedoraproject.org/tests/$newjob", + log => "https://openqa.stg.fedoraproject.org/tests/$newjob/file/autoinst-log.txt", + id => $newjob, + }; + $expected_test->{'lifetime'} = 240; + delete $expected_test->{'result'}; + my $expected = get_expected; + is_deeply($body, $expected, 'job duplicate triggers standardized amqp'); + + $plugin_mock_callcount = 0; + $t->post_ok("/api/v1/jobs/$newjob/cancel")->status_is(200); + ok($plugin_mock_callcount == 1, 'mock was called'); + my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.error'}}; + $expected_error = {reason => 'user_cancelled',}; + delete $expected_test->{'lifetime'}; + delete $expected_run->{'clone_of'}; + my $expected = get_expected; + is_deeply($body, $expected, 'job cancel triggers standardized amqp'); +}; + +subtest 'duplicate and pass job' => sub { + $plugin_mock_callcount = 0; + $t->post_ok("/api/v1/jobs/$newjob/duplicate")->status_is(200); + my $newerjob = $t->tx->res->json->{id}; + # explicitly set job as passed + $t->post_ok("/api/v1/jobs/$newerjob/set_done?result=passed")->status_is(200); + ok($plugin_mock_callcount == 2, 'mock was called'); + my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.complete'}}; + $expected_run = { + url => "https://openqa.stg.fedoraproject.org/tests/$newerjob", + log => "https://openqa.stg.fedoraproject.org/tests/$newerjob/file/autoinst-log.txt", + id => $newerjob, + }; + $expected_test->{'result'} = 'passed'; + $expected_error = ''; + my $expected = get_expected; + is_deeply($body, $expected, 'job done (passed) triggers standardized amqp'); +}; + +subtest 'create update job' => sub { + $plugin_mock_callcount = 0; + diag("Count: $plugin_mock_callcount"); + $settings->{BUILD} = 'Update-FEDORA-2018-3c876babb9'; + # let's test HDD_* impact on system->{os} here too + $settings->{HDD_1} = 'disk_f40_minimal.qcow2'; + $settings->{BOOTFROM} = 'c'; + # some update tests don't have SUBVARIANT, in which case system + # variant should not be set + delete $settings->{SUBVARIANT}; + $t->post_ok("/api/v1/jobs" => form => $settings)->status_is(200); + ok(my $updatejob = $t->tx->res->json->{id}, 'got ID of update job'); + diag("Count: $plugin_mock_callcount"); + ok($plugin_mock_callcount == 1, 'mock was called'); + my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.fedora-update.test.queued'}}; + $expected_artifact = { + id => 'FEDORA-2018-3c876babb9', + type => 'fedora-update', + release => { + version => '42', + name => 'F42' + }, + }; + $expected_pipeline = { + id => 'openqa.Update-FEDORA-2018-3c876babb9.rainbow.RainbowPC.pink.x86_64', + name => 'openqa.Update-FEDORA-2018-3c876babb9.rainbow.RainbowPC.pink.x86_64', + }; + $expected_run = { + url => "https://openqa.stg.fedoraproject.org/tests/$updatejob", + log => "https://openqa.stg.fedoraproject.org/tests/$updatejob/file/autoinst-log.txt", + id => $updatejob, + }; + $expected_system->{'os'} = 'fedora-40'; + $expected_test->{'namespace'} = 'update'; + $expected_test->{'lifetime'} = 240; + delete $expected_system->{'variant'}; + delete $expected_test->{'result'}; + my $expected = get_expected; + # no image for update message + delete $expected->{'image'}; + is_deeply($body, $expected, 'update job create triggers standardized amqp'); +}; + +done_testing(); diff --git a/FedoraMessaging.pm b/FedoraMessaging.pm new file mode 100644 index 0000000..af0509e --- /dev/null +++ b/FedoraMessaging.pm @@ -0,0 +1,280 @@ +# Copyright (C) Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +# This is a plugin for publishing messages to fedora-messaging, Fedora's +# AMQP message broker. It piggybacks on the upstream AMQP plugin but +# includes headers required by the fedora-messaging spec and publishes +# messages in the "CI Messages" spec: https://pagure.io/fedora-ci/messages +# as well as more 'native' style messages. + +package OpenQA::WebAPI::Plugin::FedoraMessaging; + +use POSIX qw(strftime); + +use Mojo::Base 'OpenQA::WebAPI::Plugin::AMQP'; +use OpenQA::Jobs::Constants; +use OpenQA::Utils; +use UUID::URandom 'create_uuid_string'; + +sub _iso8601_now { + # we do this twice, so factor it out + my $now = strftime("%Y-%m-%dT%H:%M:%S", gmtime()) . 'Z'; + return $now; +} + +sub publish_amqp { + my ($self, $topic, $event_data, $headerframe) = @_; + my $sentat = _iso8601_now; + my $messageid = create_uuid_string; + # default fedora-messaging compliant header frame. Ridiculous + # naming note: AMQP wire format has a header frame and then the + # body. Inside the header frame *is a field called headers*. Yo + # dawg, I heard you liked headers...fedora-messaging has specific + # expectations for some other fields in the header frame, and for + # some of the fields in the 'headers' field in the header frame. + # Mojo::RabbitMQ::Client tends to call things that represent the + # header frame "%headers" and "$headers", so that's fun. + my %fullheaderframe = ( + headers => { + fedora_messaging_severity => 20, + fedora_messaging_schema => 'fedora_messaging.message:Message', + "sent-at" => $sentat, + }, + content_encoding => 'utf-8', + delivery_mode => 2, + message_id => $messageid, + ); + # merge in the passed header frame values to allow overriding + $headerframe //= {}; + %fullheaderframe = (%fullheaderframe, %$headerframe); + # call parent method + $self->SUPER::publish_amqp($topic, $event_data, \%fullheaderframe); +} + +sub log_event_fedora_ci_messages { + # this is for publishing messages in the "CI Messages" format: + # https://pagure.io/fedora-ci/messages + # This is a Fedora/Red Hat-ish thing in a way, but in theory + # anyone could adopt it + my ($self, $event, $job, $baseurl) = @_; + my $stdevent; + my $clone_of; + my $job_id; + # first, get the standard 'state' (from 'queued', 'running', + # 'complete', 'error'; we cannot do 'running' at present + if ($event eq 'openqa_job_create') { + $stdevent = 'queued'; + $job_id = $job->id; + } + elsif ($event eq 'openqa_job_restart' || $event eq 'openqa_job_duplicate') { + $stdevent = 'queued'; + $clone_of = $job->id; + $job_id = $job->clone_id; + } + elsif ($event eq 'openqa_job_cancel') { + $stdevent = 'error'; + $job_id = $job->id; + } + elsif ($event eq 'openqa_job_done') { + $job_id = $job->id; + # lifecycle note: any job cancelled directly via the web API will + # see both job_cancel and job_done with result USER_CANCELLED, so + # we emit duplicate standardized fedmsgs in this case. This is + # kinda unavoidable, though, as it's possible for a job to wind up + # USER_CANCELLED *without* an openqa_job_cancel event happening, + # so we can't just throw away all openqa_job_done USER_CANCELLED + # events... + $stdevent = (grep { $job->result eq $_ } INCOMPLETE_RESULTS) ? 'error' : 'complete'; + } + else { + return undef; + } + + # we need this for the system dict; it should be the release of + # the system-under-test (the VM in which the test runs) at the + # *start* of the test, I think. We're trying to capture info about + # the environment in which the test runs + my $sysrelease = $job->VERSION; + my $hdd1; + my $bootfrom; + $hdd1 = $job->settings_hash->{HDD_1} if ($job->settings_hash->{HDD_1}); + $bootfrom = $job->settings_hash->{BOOTFROM} if ($job->settings_hash->{BOOTFROM}); + if ($hdd1 && $bootfrom) { + $sysrelease = $1 if ($hdd1 =~ /disk_f(\d+)/ && $bootfrom eq 'c'); + } + + # next, get the 'artifact' (type of thing we tested) + my $artifact; + my $artifact_id; + my $artifact_release; + my $compose_type; + my $test_namespace; + # current date/time in ISO 8601 format + my $generated_at = _iso8601_now; + + # this is used as a 'pipeline ID', see + # https://pagure.io/fedora-ci/messages/blob/master/f/schemas/pipeline.yaml + my $pipeid = join('.', "openqa", $job->BUILD, $job->TEST, $job->MACHINE, $job->FLAVOR, $job->ARCH); + + my $build = $job->BUILD; + if ($build =~ /^Fedora/) { + $artifact = 'productmd-compose'; + $artifact_id = $build; + $compose_type = 'production'; + $compose_type = 'nightly' if ($build =~ /\.n\./); + $compose_type = 'test' if ($build =~ /\.t\./); + $test_namespace = 'compose'; + } + elsif ($build =~ /^Update-FEDORA/) { + $artifact = 'fedora-update'; + $artifact_id = $build; + $artifact_id =~ s/^Update-//; + $artifact_release = { + version => $job->VERSION, + name => "F" . $job->VERSION + }; + $test_namespace = 'update'; + } + else { + # unhandled artifact type + return undef; + } + + # finally, construct the message content + my %msg_data = ( + contact => { + name => 'Fedora openQA', + team => 'Fedora QA', + url => $baseurl, + docs => 'https://fedoraproject.org/wiki/OpenQA', + irc => '#fedora-qa', + email => 'qa-devel@lists.fedoraproject.org', + }, + run => { + url => "$baseurl/tests/$job_id", + log => "$baseurl/tests/$job_id/file/autoinst-log.txt", + id => $job_id, + }, + artifact => { + type => $artifact, + id => $artifact_id, + }, + pipeline => { + # per https://pagure.io/fedora-ci/messages/issue/61 this + # is meant to be unique per test scenario *and* artifact, + # so we construct it out of BUILD and the scenario keys. + # 'name' is supposed to be a 'human readable name', well, + # this is human readable, so we'll just use it twice + id => $pipeid, + name => $pipeid, + }, + test => { + # openQA tests are pretty much always validation + category => 'validation', + # test identifier: test name plus scenario keys + type => join(' ', $job->TEST, $job->MACHINE, $job->FLAVOR, $job->ARCH), + namespace => $test_namespace, + }, + system => [ + { + # it's interesting whether we should record info on the + # *worker host itself* or the *SUT* (the VM run on top of + # the worker host environment) here...on the whole I think + # SUT is more in line with expectations, so let's do that + os => "fedora-${sysrelease}", + # openqa provisions itself...we *could* I guess set this + # to 'createhdds' if we booted a disk image, but ehhhh + provider => 'openqa', + architecture => $job->ARCH, + }, + ], + generated_at => $generated_at, + version => "0.2.1", + ); + + # add keys that don't exist in all cases to the message + if ($stdevent eq 'complete') { + $msg_data{test}{result} = $job->result; + $msg_data{test}{result} = 'info' if $job->result eq 'softfailed'; + } + elsif ($stdevent eq 'error') { + $msg_data{error} = {}; + $msg_data{error}{reason} = $job->result; + } + elsif ($stdevent eq 'queued') { + # this is a hint to consumers that the job probably went away + # if they don't get a 'complete' or 'error' in 4 hours + # FIXME: we should set this as 2 hours on 'running', but we + # can't emit running because there is no internal event for + # it, there is no job_running event or anything like it - + # this is part of https://progress.opensuse.org/issues/31069 + $msg_data{test}{lifetime} = 240; + } + $msg_data{run}{clone_of} = $clone_of if ($clone_of); + + $msg_data{artifact}{release} = $artifact_release if ($artifact_release); + + $msg_data{artifact}{compose_type} = $compose_type if ($compose_type); + + my $subvariant = $job->settings_hash->{SUBVARIANT} || ''; + $msg_data{system}[0]{variant} = $subvariant if ($subvariant); + + # record info about the image tested, for compose tests. In theory + # we might test more than one image in a job, which would break + # the schema. But we don't do that yet fortunately + if ($artifact eq 'productmd-compose') { + # this is a handy variable which indicates what the 'thing' + # the test really tests is (also used for resultsdb) + my $target = $job->settings_hash->{TEST_TARGET} || ''; + my $imgname = $job->settings_hash->{"$target"} || ''; + if ($imgname) { + $msg_data{image} = { + id => $imgname, + name => $imgname, + type => $target + }; + } + } + + # create the topic + my $topic = "ci.$artifact.test.$stdevent"; + + # prepend the prefix (kinda duplicated with parent log_event) + my $prefix = $self->{config}->{amqp}{topic_prefix}; + $topic = $prefix . '.' . $topic if ($prefix); + + # finally, send the message + log_debug("Sending CI Messages AMQP message for $event"); + # FIXME: we should set fedora_messaging_schema header here, but the + # ci-messages schemas are not currently provided as fedora-messaging + # Python classes anywhere, so we kinda can't. See: + # https://pagure.io/fedora-ci/messages/issue/33 + $self->publish_amqp($topic, \%msg_data); +} + +sub on_job_event { + # do just enough work to send the 'CI messaging' spec message + # (unfortunately a bit of duplication is inevitable) + my ($self, $args) = @_; + my ($user_id, $connection_id, $event, $event_data) = @$args; + my $jobs = $self->{app}->schema->resultset('Jobs'); + my $job = $jobs->find({id => $event_data->{id}}); + my $baseurl = $self->{config}->{global}->{base_url} || "http://UNKNOWN"; + $self->log_event_fedora_ci_messages($event, $job, $baseurl); + # call parent method to send 'native' message + $self->SUPER::on_job_event($args); +} + +1; diff --git a/openqa.spec b/openqa.spec index 6771e98..1fdb210 100644 --- a/openqa.spec +++ b/openqa.spec @@ -23,13 +23,13 @@ %global github_owner os-autoinst %global github_name openQA %global github_version 4.6 -%global github_commit 5bfa647a930e573a9f01f517ac8c322c3394b7a8 +%global github_commit 1c533908f94cf9f156a250dbf29ca809cb474076 # if set, will be a post-release snapshot build, otherwise a 'normal' build -%global github_date 20190716 +%global github_date 20190806 %global shortcommit %(c=%{github_commit}; echo ${c:0:7}) # can't use linebreaks here! -%global openqa_services openqa-webui.service openqa-gru.service openqa-websockets.service openqa-scheduler.service openqa-livehandler.service +%global openqa_services openqa-webui.service openqa-gru.service openqa-websockets.service openqa-scheduler.service openqa-livehandler.service openqa-enqueue-audit-event-cleanup.service openqa-enqueue-audit-event-cleanup.timer %global openqa_worker_services openqa-worker.target openqa-slirpvde.service openqa-vde_switch.service openqa-worker-cacheservice.service openqa-worker-cacheservice-minion.service %if %{undefined tmpfiles_create} @@ -38,6 +38,7 @@ %{nil} %endif +%define python_scripts_requires python3-requests python3-future # runtime requirements for server, worker and testsuite # these are Requires: in -common # diff from suse: we need perl(Time::Seconds) and perl(Time::Piece) @@ -46,7 +47,7 @@ Name: openqa Version: %{github_version} -Release: 18%{?github_date:.%{github_date}git%{shortcommit}}%{?dist}.2 +Release: 30%{?github_date:.%{github_date}git%{shortcommit}}%{?dist} Summary: OS-level automated testing framework License: GPLv2+ Url: http://os-autoinst.github.io/openQA/ @@ -62,28 +63,17 @@ Source1: assetcache-%{github_commit}.tar.xz # FIXME: this probably doesn't handle jobs with parents/children # properly Source2: FedoraUpdateRestart.pm -# https://github.com/os-autoinst/openQA/pull/2192 -# fixes a test failure -Patch0: 2192.patch -# https://github.com/os-autoinst/openQA/pull/2205 -# fixes another test failure -Patch1: 0001-t-api-03-auth.t-don-t-actually-wipe-asset-files-from.patch -# https://github.com/os-autoinst/openQA/pull/2217 -# Allow sending headers when publishing AMQP -Patch2: 0001-AMQP-allow-passing-headers-to-publish_amqp.patch -# https://github.com/os-autoinst/openQA/pull/2213 -# very unimportant definitely not security fixes -Patch3: 0001-Improve-rendering-summary-in-test-result-overview.patch -# https://github.com/os-autoinst/openQA/commit/acb5d403963eb3e209b95dfb882d3efe278301ed -# Needed for next patch to apply cleanly -# rediffed on the snapshot we're using -Patch4: 0001-Add-dependency-on-CommonMark-which-will-soon-replace-rediffed.patch -# https://github.com/os-autoinst/openQA/pull/2232 -# Switch to CommonMark for markdown rendering: faster, also safer -# rediffed on the snapshot we're using -Patch5: 2232-rediffed.patch +# fedora-messaging publishing plugin (upstream doesn't want this as +# it's too fedora-y for them) +Source3: FedoraMessaging.pm +# tests for the fedora-messaging publishing plugin +Source4: 23-fedora-messaging.t +# https://github.com/os-autoinst/openQA/pull/2244 +# Fix a couple of bugs in load_templates that break our usage +Patch0: 0001-load_templates-fix-clean-and-reading-perl-templates.patch BuildRequires: %{t_requires} +BuildRequires: %{python_scripts_requires} BuildRequires: perl-generators BuildRequires: postgresql-server # Standard for packages that have systemd services @@ -94,11 +84,16 @@ BuildRequires: rubygem(sass) BuildRequires: glibc-langpack-en # versioned requires can't go in the t_requires macro +BuildRequires: perl(DBIx::Class) >= 0.082801 +BuildRequires: perl(Minion) >= 9.09 +BuildRequires: perl(DBI) >= 1.632 BuildRequires: perl(Mojolicious) >= 7.92 BuildRequires: perl(Mojolicious::Plugin::AssetPack) >= 2.01 -# critical bug fix -BuildRequires: perl(DBIx::Class) >= 0.082801 +BuildRequires: perl(Mojo::RabbitMQ::Client) >= 0.2 Requires: perl(Minion) >= 9.09 +Requires: perl(DBI) >= 1.632 +Requires: perl(Mojo::RabbitMQ::Client) >= 0.2 +Requires: perl(YAML::XS) >= 0.67 # things required for the test suite BuildRequires: git @@ -117,8 +112,8 @@ BuildRequires: perl(DBD::SQLite) BuildRequires: perl(SQL::SplitStatement) BuildRequires: perl(IPC::Cmd) BuildRequires: perl(Module::Load::Conditional) -# needed for parallel test execution(?) -BuildRequires: perl(CPAN::Meta::YAML) +# needed for parallel test execution(?), which we don't do ATM +#BuildRequires: perl(CPAN::Meta::YAML) BuildRequires: rsync # These two are not packaged for Fedora yet; without them some tests # are skipped, but we can run a lot @@ -229,18 +224,24 @@ server (rather than being accessed directly). The config snippets in this package help you configure openQA to be reverse proxied by httpd. %package plugin-fedmsg -Summary: openQA plugin for sending fedmsgs +Summary: openQA plugins for sending fedmsgs and fedora-messaging Requires: openqa >= 4.3-20 Requires: /usr/bin/fedmsg-logger-3 Requires: daemonize +Requires: perl(UUID::URandom) +BuildRequires: perl(UUID::URandom) %description plugin-fedmsg -This package contains an openQA plugin which sends fedmsgs for certain -openQA internal events. To enable it, put 'plugins = Fedmsg' in the -global section of /etc/openqa/openqa.ini. The plugin uses fedmsg-logger +This package contains openQA plugins which send fedora-messaging +messages and fedmsgs for certain openQA internal events. To enable the +fedmsg plugin, put 'plugins = Fedmsg' in the global section of +/etc/openqa/openqa.ini; for the fedora-messaging plugin, use +'plugins = FedoraMessing'. The fedmsg plugin uses fedmsg-logger to send messages, so fedmsg configuration determines where the message goes. Note: the certificate prefix is hard coded to 'openqa' for now, -to send signed messages you must have an 'openqa' cert for fedmsg. +to send signed messages you must have an 'openqa' cert for fedmsg. The +fedora-messaging plugin piggybacks on the upstream AMQP plugin, and +follows its configuration (in the 'amqp' section of openqa.ini). %package plugin-fedoraupdaterestart Summary: openQA plugin for restarting failed Fedora update tests @@ -271,6 +272,14 @@ This package contains the openQA client script, along with several other useful tools and support files. The client script is a convenient helper for interacting with the openQA REST API. +%package python-scripts +Summary: Additional scripts in python +Group: Development/Tools/Other +Requires: %python_scripts_requires + +%description python-scripts +Additional scripts for the use of openQA in the python programming language. + %package local-db Summary: Helper package to ease setup of postgresql DB Requires: %name @@ -305,6 +314,13 @@ sed -i -e 's,/etc/apache2/vhosts.d,%{_sysconfdir}/httpd/conf.d,g' etc/apache2/vh # These are the Fedora-y standard TLS cert/key locations. sed -i -e 's,/etc/apache2/ssl.crt,%{_sysconfdir}/pki/tls/certs,g' etc/apache2/vhosts.d/* sed -i -e 's,/etc/apache2/ssl.key,%{_sysconfdir}/pki/tls/private,g' etc/apache2/vhosts.d/* +# Add our downstream plugins to the sources +cp %{SOURCE2} lib/OpenQA/WebAPI/Plugin/ +cp %{SOURCE3} lib/OpenQA/WebAPI/Plugin/ +# we don't really need the tidy test +rm -f t/00-tidy.t +# add the fedora-messaging publishing plugin test to the sources +cp %{SOURCE4} t/ %build @@ -325,6 +341,9 @@ ln -s %{_datadir}/openqa/script/client %{buildroot}%{_bindir}/openqa-client ln -s %{_datadir}/openqa/script/openqa-clone-job %{buildroot}%{_bindir}/openqa-clone-job ln -s %{_datadir}/openqa/script/dump_templates %{buildroot}%{_bindir}/openqa-dump-templates ln -s %{_datadir}/openqa/script/load_templates %{buildroot}%{_bindir}/openqa-load-templates +ln -s %{_datadir}/openqa/script/openqa-clone-custom-git-refspec %{buildroot}%{_bindir}/openqa-clone-custom-git-refspec +ln -s %{_datadir}/openqa/script/openqa-label-all %{buildroot}%{_bindir}/openqa-label-all + cd %{buildroot} grep -rl %{_bindir}/env . | while read file; do @@ -339,14 +358,8 @@ mkdir %{buildroot}%{_localstatedir}/lib/openqa/webui/cache # We don't do AppArmor rm -rf %{buildroot}%{_sysconfdir}/apparmor.d -# Install the FedoraUpdateRestarter plugin -install %{SOURCE2} %{buildroot}%{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/ - %check -# we don't really need the tidy test -rm -f t/00-tidy.t - rm -rf %{buildroot}/DB export LC_ALL=en_US.UTF-8 # seems necessary to make 24-worker-job.t pass - two subtests there @@ -354,7 +367,10 @@ export LC_ALL=en_US.UTF-8 # if they exist. upstream repo does this in .travis.yml. touch openqa-debug.log autoinst-log.txt chmod a+w openqa-debug.log autoinst-log.txt -make test-with-database OBS_RUN=1 PROVE_ARGS='-r' TEST_PG_PATH=%{buildroot}/DB +# GIT_CEILING_DIRECTORIES here avoids a case where git error handling +# can differ when you run the build in mock and cause 16-utils-runcmd +# to fail +make test-with-database GIT_CEILING_DIRECTORIES="/" OBS_RUN=1 PROVE_ARGS='-l -r' TEST_PG_PATH=%{buildroot}/DB rm -rf %{buildroot}/DB @@ -423,6 +439,8 @@ fi %{_unitdir}/openqa-gru.service %{_unitdir}/openqa-scheduler.service %{_unitdir}/openqa-websockets.service +%{_unitdir}/openqa-enqueue-audit-event-cleanup.service +%{_unitdir}/openqa-enqueue-audit-event-cleanup.timer # web libs %{_datadir}/openqa/templates %{_datadir}/openqa/public @@ -460,6 +478,7 @@ fi %ghost %dir %{_datadir}/openqa/packed %{_datadir}/openqa/lib %exclude %{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/Fedmsg.pm +%exclude %{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/FedoraMessaging.pm %exclude %{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/FedoraUpdateRestart.pm %exclude %{_datadir}/openqa/lib/OpenQA/Client.pm %dir %{_localstatedir}/lib/openqa @@ -524,6 +543,11 @@ fi %{_bindir}/openqa-clone-job %{_bindir}/openqa-dump-templates %{_bindir}/openqa-load-templates +%{_bindir}/openqa-clone-custom-git-refspec + +%files python-scripts +%{_datadir}/openqa/script/openqa-label-all +%{_bindir}/openqa-label-all %files doc %doc docs/* @@ -537,11 +561,51 @@ fi %files plugin-fedmsg %{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/Fedmsg.pm +%{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/FedoraMessaging.pm %files plugin-fedoraupdaterestart %{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/FedoraUpdateRestart.pm %changelog +* Fri Aug 09 2019 Adam Williamson - 4.6-30.20190806git1c53390 +- Fix test invocation again + +* Thu Aug 08 2019 Adam Williamson - 4.6-29.20190806git1c53390 +- FedoraMessaging: only include (sub)variant if it's defined + +* Thu Aug 08 2019 Adam Williamson - 4.6-28.20190806git1c53390 +- Fix FedoraMessaging 'system' value to be an array of hashes + +* Thu Aug 08 2019 Adam Williamson - 4.6-27.20190806git1c53390 +- Tweak FedoraMessaging update message 'release' data + +* Thu Aug 08 2019 Adam Williamson - 4.6-26.20190806git1c53390 +- Tweak FedoraMessaging to publish 'image' in compose messages + +* Thu Aug 01 2019 Adam Williamson - 4.6-25.20190806git1c53390 +- Backport PR #2244 to fix some problems in load_templates + +* Thu Aug 01 2019 Adam Williamson - 4.6-24.20190806git1c53390 +- Bump to latest git again, drop all patches (all merged) + +* Thu Aug 01 2019 Adam Williamson - 4.6-23.20190726git92e8f3c +- Prepend the topic_prefix to ci-messages messages also + +* Thu Aug 01 2019 Adam Williamson - 4.6-22.20190726git92e8f3c +- Some fixes for the FedoraMessaging plugin + +* Thu Aug 01 2019 Adam Williamson - 4.6-21.20190726git92e8f3c +- Allow AMQP messages with no prefix (backport PR #2236) + +* Tue Jul 30 2019 Adam Williamson - 4.6-20.20190726git92e8f3c +- Add plugin for publishing fedora-messaging messages +- Backport PR #2232 (faster and safer markdown rendering) + +* Fri Jul 26 2019 Adam Williamson - 4.6-19.20190726git92e8f3c +- Bump to latest git again +- Re-sync spec with upstream (including new python script subpackage) +- Drop merged patches + * Thu Aug 01 2019 Adam Williamson - 4.6-18.20190716git5bfa647.fc30.2 - Backport PR #2232 (faster and safer markdown rendering) - Allow comments by users again (safe with PR #2232) diff --git a/sources b/sources index be04972..4cba8fa 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (openQA-5bfa647a930e573a9f01f517ac8c322c3394b7a8.tar.gz) = 01cb85d32b2efb906c694af7e2755e478d6793918c99d801f6def6f8d9055335eb8169b50ca02e520a5f860e2dcdcda7f8cd7fc038a1c6686e71772f7a3ca288 -SHA512 (assetcache-5bfa647a930e573a9f01f517ac8c322c3394b7a8.tar.xz) = 46d75eac23367dad1e73058d206a3064be030cadb73562a5b570e758c35a4e8eeb427d43c1846cb4f7d61b351515cb7d2399815b13b4937a97adfb5276e43fbf +SHA512 (openQA-1c533908f94cf9f156a250dbf29ca809cb474076.tar.gz) = 799b378aa94aefa94e8877e582cd62eca291c01c8ec0a989471ea405a600d043084b3bb295353a49d4c091f421e0cdf75a3cdad54567c3b00cb6d8469bcdd775 +SHA512 (assetcache-1c533908f94cf9f156a250dbf29ca809cb474076.tar.xz) = 387eefefb87ef04e257260a36b78bddf30617fbded11a976c945eda79b5bd70a913ab24e455f098e2248a5286bfe1f27cca1462901d060303660ea994bdaf495