#1 convert to FMF and fix the tests
Opened 3 years ago by pvalena. Modified 2 years ago
tests/ pvalena/ruby tmt  into  master

file added
+1
@@ -0,0 +1,1 @@ 

+ 1

file modified
+2 -7
@@ -1,9 +1,4 @@ 

  contact: Vít Ondruch <vondruch@redhat.com>

- test: runtest.sh

+ test: ./runtest.sh

  category: integration

- 

- /run-basic-rails-application:

-     component: rubygem-rails

- 

- /systemtap-static-probes-in-ruby:

-     component: systemtap

+ how: beakerlib

@@ -1,64 +0,0 @@ 

- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- #

- #   Makefile of /CoreOS/rubygem-rails/run-basic-rails-application

- #   Description: Test creates new rails application and runs it

- #   Author: Iveta Senfeldova <isenfeld@redhat.com>

- #

- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- #

- #   Copyright (c) 2012 Red Hat, Inc. All rights reserved.

- #

- #   This copyrighted material is made available to anyone wishing

- #   to use, modify, copy, or redistribute it subject to the terms

- #   and conditions of the GNU General Public License version 2.

- #

- #   This program is distributed in the hope that it will be

- #   useful, but WITHOUT ANY WARRANTY; without even the implied

- #   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

- #   PURPOSE. See the GNU General Public License for more details.

- #

- #   You should have received a copy of the GNU General Public

- #   License along with this program; if not, write to the Free

- #   Software Foundation, Inc., 51 Franklin Street, Fifth Floor,

- #   Boston, MA 02110-1301, USA.

- #

- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- 

- export TEST=/CoreOS/rubygem-rails/run-basic-rails-application

- export TESTVERSION=1.0

- 

- BUILT_FILES=

- 

- FILES=$(METADATA) runtest.sh Makefile PURPOSE

- 

- .PHONY: all install download clean

- 

- run: $(FILES) build

- 	./runtest.sh

- 

- build: $(BUILT_FILES)

- 	test -x runtest.sh || chmod a+x runtest.sh

- 

- clean:

- 	rm -f *~ $(BUILT_FILES)

- 

- 

- -include /usr/share/rhts/lib/rhts-make.include

- 

- $(METADATA): Makefile

- 	@echo "Owner:           Iveta Senfeldova <isenfeld@redhat.com>" > $(METADATA)

- 	@echo "Name:            $(TEST)" >> $(METADATA)

- 	@echo "TestVersion:     $(TESTVERSION)" >> $(METADATA)

- 	@echo "Path:            $(TEST_DIR)" >> $(METADATA)

- 	@echo "Description:     Test creates new rails application in collection and runs it" >> $(METADATA)

- 	@echo "Type:            Sanity" >> $(METADATA)

- 	@echo "TestTime:        60m" >> $(METADATA)

- 	@echo "RunFor:          rubygem-rails ruby" >> $(METADATA)

- 	@echo "Requires:        rubygems rubygem-rails ruby-devel sqlite-devel libxml2-devel libxslt-devel gcc-c++ net-tools procps-ng psmisc" >> $(METADATA)

- 	@echo "Priority:        Normal" >> $(METADATA)

- 	@echo "License:         GPLv2" >> $(METADATA)

- 	@echo "Confidential:    no" >> $(METADATA)

- 	@echo "Destructive:     no" >> $(METADATA)

- 	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA)

- 

- 	rhts-lint $(METADATA)

@@ -0,0 +1,28 @@ 

+ summary: Test creates new rails application in collection and runs it

+ description: |

+     Test ensure it is possible to create new rails application and run it.

+ component:

+ - rubygem-rails

+ - ruby

+ recommend:

+ - rubygems

+ - rubygem-rails

+ - rubygem-nokogiri

+ - rubygem-sqlite3

+ - rubygem-sass-rails

+ - rubygem-puma

+ - rubygem-jbuilder

+ - rubygem-byebug

+ - rubygem-web-console

+ - rubygem-listen

+ - ruby-devel

+ - sqlite-devel

+ - libxml2-devel

+ - libxslt-devel

+ - gcc-c++

+ - net-tools

+ - procps-ng

+ - psmisc

+ - wget

+ duration: 60m

+ extra-task: /CoreOS/rubygem-rails/run-basic-rails-application

@@ -65,24 +65,25 @@ 

      rlPhaseEnd

  

      rlPhaseStartTest

-         rlRun "rails new app | tee bundle.log" 0 "Creating new rails application"

+         rlRun "rails new app --skip-bundle --skip-spring --skip-test --skip-bootsnap --skip-webpacker --skip-javascript -f | tee bundle.log" 0 "Creating new rails application"

          [ ! -z "$COLLECTIONS" ] && rlAssertGrep "bundle install --local" bundle.log || rlAssertNotGrep "bundle install --local" bundle.log

          rlRun "pushd app/"

          rlAssertExists "app"

  

-         if [[ ! "$COLLECTIONS" =~ "rh-ror50" ]]

+         if [ -z "$COLLECTIONS" ] && rlIsRHEL '<=7'

          then

-             rlRun "echo \"gem 'therubyracer'\" >> Gemfile" 0 "Adding therubyracer gem into Gemfile"

-             rlRun "bundle install"

+             rlRun "bundle install --local"

          fi

  

-         if [ -z "$COLLECTIONS" ] && rlIsRHEL '<=7'

+         if rlIsFedora

          then

-             rlRun "echo \"gem 'nokogiri', '1.6.8'\" >> Gemfile" 0 "Adding nokogiri gem into Gemfile"

-             sed -i "s/gem 'spring'/#gem 'spring'/" Gemfile

-             rlRun "gem pristine nokogiri" 0 "Restore installed nokogiri to pristine condition"

-             rlRun "bundle config build.nokogiri --use-system-libraries" 0 "Build nokogiri with system libraries"

-             rlRun "bundle install"

+             rlRun "sed -i \"s/\(gem .puma.\).*/\1/\" Gemfile" 0 'Relax puma dependency'

+             rlRun "sed -i \"s/\(gem .listen.\).*/\1/\" Gemfile" 0 'Relax listen dependency'

+             rlRun "sed -i \"/gem .sass-rails./ s/^/#/\" Gemfile" 0 'Drop sass-rails dependency'

+ 

+             rlRun "bundle config set without test" 0 "Set bundle config(1)"

+             rlRun "bundle config set deployment false" 0 "Set bundle config(2)"

+             rlRun "bundle install --local" 0 "Bundle dependencies locally"

          fi

          rlRun "netstat -tulpn | grep 3000" 1 "Check if port 3000 isn't engaged"

          rlRun "rails server &" 0 "Running rails server"

@@ -1,65 +0,0 @@ 

- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- #

- #   Makefile of /CoreOS/ruby/Regression/systemtap-static-probes-in-ruby

- #   Description: It tests there are static probes in ruby package.

- #   Author: Ales Marecek <amarecek@redhat.com>

- #

- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- #

- #   Copyright (c) 2011 Red Hat, Inc. All rights reserved.

- #

- #   This copyrighted material is made available to anyone wishing

- #   to use, modify, copy, or redistribute it subject to the terms

- #   and conditions of the GNU General Public License version 2.

- #

- #   This program is distributed in the hope that it will be

- #   useful, but WITHOUT ANY WARRANTY; without even the implied

- #   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

- #   PURPOSE. See the GNU General Public License for more details.

- #

- #   You should have received a copy of the GNU General Public

- #   License along with this program; if not, write to the Free

- #   Software Foundation, Inc., 51 Franklin Street, Fifth Floor,

- #   Boston, MA 02110-1301, USA.

- #

- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- 

- export TEST=/CoreOS/ruby/Regression/systemtap-static-probes-in-ruby

- export TESTVERSION=1.0

- 

- BUILT_FILES=

- 

- FILES=$(METADATA) runtest.sh Makefile PURPOSE

- 

- .PHONY: all install download clean

- 

- run: $(FILES) build

- 	./runtest.sh

- 

- build: $(BUILT_FILES)

- 	chmod a+x runtest.sh

- 

- clean:

- 	rm -f *~ $(BUILT_FILES)

- 

- 

- -include /usr/share/rhts/lib/rhts-make.include

- 

- $(METADATA): Makefile

- 	@echo "Owner:           Iveta Senfeldova <isenfeld@redhat.com>" > $(METADATA)

- 	@echo "Name:            $(TEST)" >> $(METADATA)

- 	@echo "TestVersion:     $(TESTVERSION)" >> $(METADATA)

- 	@echo "Path:            $(TEST_DIR)" >> $(METADATA)

- 	@echo "Description:     It tests there are static probes in ruby package." >> $(METADATA)

- 	@echo "Type:            Regression" >> $(METADATA)

- 	@echo "TestTime:        5m" >> $(METADATA)

- 	@echo "RunFor:          ruby" >> $(METADATA)

- 	@echo "Requires:        ruby ruby-doc ruby-libs systemtap grep procps-ng" >> $(METADATA)

- 	@echo "Releases:        RHEL6" >> $(METADATA)

- 	@echo "Priority:        Normal" >> $(METADATA)

- 	@echo "License:         GPLv2" >> $(METADATA)

- 	@echo "Confidential:    no" >> $(METADATA)

- 	@echo "Destructive:     no" >> $(METADATA)

- 	@echo "Bug:             673162" >> $(METADATA)

- 

- 	rhts-lint $(METADATA)

@@ -0,0 +1,16 @@ 

+ summary: It tests there are static probes in ruby package.

+ description: |

+     Simple test to exercise ruby's systemtap API.

+ component:

+ - ruby

+ require:

+ - /usr/share/doc/ruby-doc/ruby-exercise.stp

+ recommend:

+ - ruby

+ - ruby-doc

+ - ruby-libs

+ - systemtap

+ - grep

+ - procps-ng

+ duration: 5m

+ extra-task: /CoreOS/ruby/Regression/systemtap-static-probes-in-ruby

@@ -41,10 +41,25 @@ 

  _STAP_FILE="`rpm -ql $PACKAGES | grep 'ruby-exercise.stp' 2>/dev/null`"

  _LOG_FILE="ruby.log"

  _STAP_OUT="stap-out.log"

+ _KERNEL_DEVEL="kernel-devel-`uname -r`"

  

  # for ruby on RHEL7 and in collections

  PRINT="Kernel::printf"

  

+ _run_stap () {

+     local t="${1:-2}"

+     rlRun "( timeout $t stap -v ${_STAP_FILE} -o ${_LOG_FILE} 1>&2 2>${_STAP_OUT} & )" 0 "Running systemtap in background"

+     sleep 1

+     # wait for systemtap to start

+     rlWaitForCmd "grep 'starting run' ${_STAP_OUT}"

+ }

+ 

+ _log_file () {

+     sleep 1

+     echo -e "---${_LOG_FILE}--"

+     grep ' -e:1$' ${_LOG_FILE}

+ }

+ 

  rlJournalStart

      rlPhaseStartSetup

          rlAssertRpm --all
@@ -54,6 +69,9 @@ 

          rlRun "pushd ${TmpDir}"

          _LOG_FILE="${TmpDir}/${_LOG_FILE}"

          touch ${_LOG_FILE}

+         if rlIsRHEL '>=8' || rlIsFedora; then

+           rlRun "rpm -q '$_KERNEL_DEVEL' || dnf install -y '$_KERNEL_DEVEL'" 0 'Installing corresponding kernel-devel: $_KERNEL_DEVEL'

+         fi

      rlPhaseEnd

  

      rlPhaseStartTest
@@ -61,45 +79,50 @@ 

          rlRun "rpm -ql $PACKAGES | grep 'stp' | grep 'tapset'" 0 "Checking tapset file"

          rlRun "rpm -ql $PACKAGES | grep 'stp' | grep 'doc'" 0 "Checking doc file"

  

-         rlRun "{ stap -v ${_STAP_FILE} -o ${_LOG_FILE} |& tee ${_STAP_OUT}; } &" 0 "Running systemtap in background"

-         _STAP_PID=$!

-         echo -e "--_STAP_PID--\n${_STAP_PID}" # info

+         # stap takes long time to start for the first time

+         _run_stap 60

+         rlRun "grep -q 'starting run' ${_STAP_OUT}" 0 "Checking stap output."

+         _log_file

  

-         # wait for systemtap to start

-         rlWaitForCmd "grep 'starting run' ${_STAP_OUT}"

+         if rlIsRHEL '<7' && [ -z "$COLLECTIONS" ]

+         then

+             _run_stap

+             rlRun "$RUBY -e '1.0 + 9.0'" 0 "Running ruby interpreter: Float test"

+             _log_file

+             rlRun "grep -q 'Float::+' ${_LOG_FILE}" 0 "Checking systemtap log: Float test"

+             # for ruby on RHEL6

+             PRINT="Object::printf"

+         fi

  

-         rlRun "$RUBY -e '1.0 + 9.0'" 0 "Running ruby interpreter: Float test"

+         _run_stap

          rlRun "$RUBY -e '[1, 2, 3].push(4)'" 0 "Running ruby interpreter: Array test"

+         _log_file

+         rlRun "grep -q 'Array::push -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: Array test"

+ 

+         _run_stap

          rlRun "$RUBY -e '(1..10).step(1)'" 0 "Running ruby interpreter: Range test"

+         _log_file

+         rlRun "grep -q 'Range::step -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: Range test"

+ 

+         _run_stap

          rlRun "$RUBY -e 'Hash.new()'" 0 "Running ruby interpreter: Hash test"

-         rlRun "$RUBY -e '\"abc\".slice!(\"c\")'" 0 "Running ruby interpreter: String test"

-         rlRun "$RUBY -e '5.times { printf \"Ruby!\n\" }'" 0 "Running ruby interpreter: Cycle and print test"

+         _log_file

+         rlRun "grep -q 'Hash::initialize -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: Hash test"

  

-         # wait for all output

-         rlRun "sleep 10" 0 "Waiting \"10\" seconds"

+         _run_stap

+         rlRun "$RUBY -e '\"abc\".slice!(\"c\")'" 0 "Running ruby interpreter: String test"

+         _log_file

+         rlRun "grep -q 'String::slice! -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: String test"

  

-         rlRun "kill ${_STAP_PID}" 0 "Terminating stap"

-         sleep 1 # No Operation

+         _run_stap

+         rlRun "$RUBY -e '5.times { printf \"Ruby!\n\" }'" 0 "Running ruby interpreter: Cycle and print test"

+         _log_file

+         rlRun "grep -q 'Integer::times -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: Cycle and print test - iteration number"

+         rlRun "grep -q '$PRINT -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: Cycle and print test - printf function"

+         rlRun "grep -q 'IO::write -e:1$' ${_LOG_FILE}" 0 "Checking systemtap log: Cycle and print test - IO write"

  

-         echo -e "---ps-out--" && ps -e | grep stap      # info

-         rlRun "kill -s SIGKILL ${_STAP_PID}" 0,1 "Attempt killing stap"

          rlRun "pkill stapio" 0,1 "Attempt killing all leftover stapios"

          echo -e "---ps-out--" && ps -e | grep stap      # info

- 

-         echo -e "---${_LOG_FILE}--" && cat ${_LOG_FILE} # info

-         if rlIsRHEL '<7' && [ -z "$COLLECTIONS" ]

-         then

-             rlRun "grep -q 'Float::+' ${_LOG_FILE}" 0 "Checking systemtap log: Float test"

-             # for ruby on RHEL6

-             PRINT="Object::printf"

-         fi

-         rlRun "grep -q 'Array::push' ${_LOG_FILE}" 0 "Checking systemtap log: Array test" 

-         rlRun "grep -q 'Range::step' ${_LOG_FILE}" 0 "Checking systemtap log: Range test"

-         rlRun "grep -q 'Hash::initialize' ${_LOG_FILE}" 0 "Checking systemtap log: Hash test"

-         rlRun "grep -q 'String::slice!' ${_LOG_FILE}" 0 "Checking systemtap log: String test"

-         rlRun "grep -q 'Integer::times' ${_LOG_FILE}" 0 "Checking systemtap log: Cycle and print test - iteration number"

-         rlRun "grep -q '$PRINT' ${_LOG_FILE}" 0 "Checking systemtap log: Cycle and print test - printf function"

-         rlRun "grep -q 'IO::write' ${_LOG_FILE}" 0 "Checking systemtap log: Cycle and print test - IO write"

      rlPhaseEnd

  

      rlPhaseStartCleanup

Tests plan is defined in rpms/ruby (PR); if you want to test it it without it, you need to specify you're using beakerlib test suite:

# Runs in VM, Fedora stable by default
$ tmt run -avv execute -h beakerlib

Test run: https://gist.github.com/f2e1fb3d4badae291ed18e5471b0d91c


To run the systemtap test only, in container:

$ tmt run -avv provision -h container -i fedora:rawhide test --name /run-basic-rails-application

Sample run: https://git.io/JJNEH.

1 new commit added

  • `Cleanupbundle install```
3 years ago

1 new commit added

  • Enhance systemtap test and add required kernel-devel.
3 years ago

1 new commit added

  • Add rubygem-puma dependency for rails tests.
3 years ago

1 new commit added

  • Fix rails test.
3 years ago

1 new commit added

  • Fixup kernel install for systemtap tests.
3 years ago

1 new commit added

  • Do not cat the log file.
3 years ago

1 new commit added

  • Fixup systemtap test.
3 years ago

@jaruga @vondruch PTAL.

Both tests succeed now (I will squash later on), please check the result if possible (ping me for options if you want; there's even a container).


@mvadkert please let me know what's the status of Fedora CI (can we have some test run?).

12 new commits added

  • Fixup systemtap test.
  • Do not cat the log file.
  • Fixup kernel install for systemtap tests.
  • Fix rails test.
  • Add rubygem-puma dependency for rails tests.
  • Enhance systemtap test and add required kernel-devel.
  • `Cleanupbundle install```
  • fix rails new app
  • Fix rails dependencies.
  • fixup
  • Convert tests to FMF using tmt.
  • Create fmf metadata.
3 years ago

1 new commit added

  • Upgrade for Ruby on Rails 6.
3 years ago

13 new commits added

  • Upgrade for Ruby on Rails 6.
  • Fixup systemtap test.
  • Do not cat the log file.
  • Fixup kernel install for systemtap tests.
  • Fix rails test.
  • Add rubygem-puma dependency for rails tests.
  • Enhance systemtap test and add required kernel-devel.
  • `Cleanupbundle install```
  • fix rails new app
  • Fix rails dependencies.
  • fixup
  • Convert tests to FMF using tmt.
  • Create fmf metadata.
3 years ago

13 new commits added

  • Upgrade for Ruby on Rails 6.
  • Fixup systemtap test.
  • Do not cat the log file.
  • Fixup kernel install for systemtap tests.
  • Fix rails test.
  • Add rubygem-puma dependency for rails tests.
  • Enhance systemtap test and add required kernel-devel.
  • `Cleanupbundle install```
  • fix rails new app
  • Fix rails dependencies.
  • fixup
  • Convert tests to FMF using tmt.
  • Create fmf metadata.
3 years ago

13 new commits added

  • Upgrade for Ruby on Rails 6.
  • Fixup systemtap test.
  • Do not cat the log file.
  • Fixup kernel install for systemtap tests.
  • Fix rails test.
  • Add rubygem-puma dependency for rails tests.
  • Enhance systemtap test and add required kernel-devel.
  • `Cleanupbundle install```
  • fix rails new app
  • Fix rails dependencies.
  • fixup
  • Convert tests to FMF using tmt.
  • Create fmf metadata.
3 years ago

rebased onto 0e9277b

3 years ago

I've squashed commits and updated to run with Ruby on Rails 6.

@vondruch, @jaruga PTAL.

Sample run: https://git.io/JJNEH


Uses RoR from my copr, to use it, you need to edit plan (ci.fmf from the other PR) metadata:

diff --git a/ci.fmf b/ci.fmf
index 0e0a042..07cf83e 100644
--- a/ci.fmf
+++ b/ci.fmf
@@ -3,5 +3,9 @@ discover:
     how: fmf
     repository: https://src.fedoraproject.org/forks/pvalena/tests/ruby.git
     ref: tmt
+prepare:
+    how: install
+    copr: pvalena/ruby-on-rails
+    package: rubygem-rails
 execute:
     how: beakerlib

Test run: https://gist.github.com/f2e1fb3d4badae291ed18e5471b0d91c

I tried to run the tmt command on your latest tmt branch (32f750b2f542c2f5a61a9677cfccc4b3963b5a83) on my local machine, and I got an error message Unsupported method 'virtual' in '/plans/default'..
Do you know what's a reason of the error?

$ rpm -qf $(command -v tmt)
tmt-0.20-1.fc32.noarch

$ pwd
/home/jaruga/fed/tests/ruby

$ git remote -v
origin  ssh://jaruga@pkgs.fedoraproject.org/tests/ruby (fetch)
origin  ssh://jaruga@pkgs.fedoraproject.org/tests/ruby (push)
p   https://src.fedoraproject.org/forks/pvalena/tests/ruby.git (fetch)
p   https://src.fedoraproject.org/forks/pvalena/tests/ruby.git (push)

$ git branch
  master
* tmt

$ ls
main.fmf  README.md  run-basic-rails-application/  systemtap-static-probes-in-ruby/

$ tmt run -avv execute -h beakerlib
/var/tmp/tmt/run-001
Found 1 plan.

/plans/default
Unsupported method 'virtual' in '/plans/default'.

$ echo $?
2

Test run: https://gist.github.com/f2e1fb3d4badae291ed18e5471b0d91c

I tried to run the tmt command on your latest tmt branch (32f750b2f542c2f5a61a9677cfccc4b3963b5a83) on my local machine, and I got an error message Unsupported method 'virtual' in '/plans/default'..
Do you know what's a reason of the error?

Well, yes. It means you don't have local support (missing packages?) for running VMs. Try installing tmt-virtual (I hope I remember the name correctly) or tmt-all.

https://tmt.readthedocs.io/en/latest/overview.html#install

If you encounter any issues (UX ones; anything), please report them here: https://github.com/psss/tmt/

(It's still a new project and we're trying to improve what we can.)

Alternatively, you can install tmt-container and run the tests in container (please description of this PR).

@pvalena thanks for the info. Let me show my working log, and I still got some issues. If you know something, let me know. And I may also report them to the upstream GitHub.

I installed the tmt command I showed you by sudo dnf install tmt.

I think the package names are not tmt-virtual and tmt-container, but tmt-provision-virtual and tmt-provision-container.

$ sudo dnf install tmt-virtual
...
No match for argument: tmt-virtual
Error: Unable to find a match: tmt-virtual

$ sudo dnf install tmt-provision-virtual
...
Complete!

Run tmt command.

$ pwd
/home/jaruga/fed/tests/ruby

$ tmt run -avv execute -h beakerlib
/var/tmp/tmt/run-003
Found 1 plan.

/plans/default
    discover
        how: fmf
        directory: /home/jaruga/git/pkg/fed/tests/ruby
        summary: 2 tests selected
            /run-basic-rails-application
            /systemtap-static-probes-in-ruby
    provision
        how: virtual
        image: fedora
        user: root
        memory: 2048 MB
        disk: 10 GB
        qcow: Fedora-Cloud-Base-32-1.6.x86_64.qcow2
        progress: downloading...
        name: gehKKKVPNmgrhNSG
        progress: booting...
libvirt: Polkit error : authentication unavailable: no polkit agent available to authenticate action 'org.libvirt.unix.manage'
    finish
        summary: 0 tasks completed
Traceback (most recent call last):
  File "/usr/bin/tmt", line 8, in <module>
    tmt.cli.main()
  File "/usr/lib/python3.8/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/usr/lib/python3.8/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/usr/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/lib/python3.8/site-packages/click/core.py", line 1290, in invoke
    return _process_result(rv)
  File "/usr/lib/python3.8/site-packages/click/core.py", line 1224, in _process_result
    value = ctx.invoke(self.result_callback, value, **ctx.params)
  File "/usr/lib/python3.8/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/usr/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/usr/lib/python3.8/site-packages/tmt/cli.py", line 254, in finito
    context.obj.run.go()
  File "/usr/lib/python3.8/site-packages/tmt/base.py", line 909, in go
    plan.go()
  File "/usr/lib/python3.8/site-packages/tmt/base.py", line 503, in go
    step.go()
  File "/usr/lib/python3.8/site-packages/tmt/steps/provision/__init__.py", line 92, in go
    plugin.go()
  File "/usr/lib/python3.8/site-packages/tmt/steps/provision/testcloud.py", line 228, in go
    self._guest.start()
  File "/usr/lib/python3.8/site-packages/tmt/steps/provision/testcloud.py", line 409, in start
    self.instance.spawn_vm()
  File "/usr/lib/python3.8/site-packages/testcloud/instance.py", line 401, in spawn_vm
    conn = libvirt.open(self.connection)
  File "/usr/lib64/python3.8/site-packages/libvirt.py", line 277, in open
    if ret is None:raise libvirtError('virConnectOpen() failed')
libvirt.libvirtError: authentication unavailable: no polkit agent available to authenticate action 'org.libvirt.unix.manage'

$ echo $?
1

After installing tmt-all, the result is same.

$ sudo dnf install tmt-all
...
Complete!

$ tmt run -avv execute -h beakerlib
/var/tmp/tmt/run-004
Found 1 plan.
...
  File "/usr/lib/python3.8/site-packages/tmt/steps/provision/testcloud.py", line 409, in start
    self.instance.spawn_vm()
  File "/usr/lib/python3.8/site-packages/testcloud/instance.py", line 401, in spawn_vm
    conn = libvirt.open(self.connection)
  File "/usr/lib64/python3.8/site-packages/libvirt.py", line 277, in open
    if ret is None:raise libvirtError('virConnectOpen() failed')
libvirt.libvirtError: authentication unavailable: no polkit agent available to authenticate action 'org.libvirt.unix.manage'

Run the tests in container.

$ sudo dnf install tmt-provision-container
...
Nothing to do.
Complete!

$ tmt run -avv provision -h container -i fedora:rawhide test --name /run-basic-rails-application
/var/tmp/tmt/run-005
Found 1 plan.

/plans/default
    discover
        how: fmf
        directory: /home/jaruga/git/pkg/fed/tests/ruby
        summary: 1 test selected
            /run-basic-rails-application
    provision
        how: container
        image: fedora:rawhide
        name: tmt-var-tmp-tmt-run-005-plans-default
        distro: Fedora release 33 (Thirty Three)
        kernel: 5.6.19-300.fc32.x86_64
        summary: 1 guest provisioned
    prepare
        how: install
        summary: Install recommended packages
        name: recommends
        package: rubygem-web-console, rubygem-byebug, procps-ng and 16 more
        summary: 1 preparation applied
    execute
        how: shell.tmt
            test: Test creates new rails application in collection and runs it
                00:00:10 /run-basic-rails-application
        summary: 1 test executed
    report
        how: display
            errr /run-basic-rails-application
                out.log: /var/tmp/tmt/run-005/plans/default/execute/logs/run-basic-rails-application/out.log
        summary: 1 error
    finish
        container: stopped
        container: removed
        summary: 0 tasks completed

total: 1 error

$ echo $?
2

Here is the out.log.

$ cat /var/tmp/tmt/run-005/plans/default/execute/logs/run-basic-rails-application/out.log
./runtest.sh: line 31: /usr/share/beakerlib/beakerlib.sh: No such file or directory
./runtest.sh: line 42: rlIsRHEL: command not found
./runtest.sh: line 44: rlJournalStart: command not found
./runtest.sh: line 45: rlPhaseStartSetup: command not found
./runtest.sh: line 46: rlAssertRpm: command not found
./runtest.sh: line 47: rlAssertBinaryOrigin: command not found
./runtest.sh: line 48: rlAssertBinaryOrigin: command not found
./runtest.sh: line 49: rlRun: command not found
./runtest.sh: line 50: rlRun: command not found
true
./runtest.sh: line 65: rlPhaseEnd: command not found
./runtest.sh: line 67: rlPhaseStartTest: command not found
./runtest.sh: line 68: rlRun: command not found
./runtest.sh: line 69: rlAssertNotGrep: command not found
./runtest.sh: line 70: rlRun: command not found
./runtest.sh: line 71: rlAssertExists: command not found
./runtest.sh: line 73: rlIsRHEL: command not found
./runtest.sh: line 78: rlIsFedora: command not found
./runtest.sh: line 88: rlRun: command not found
./runtest.sh: line 89: rlRun: command not found
./runtest.sh: line 91: rlRun: command not found
./runtest.sh: line 92: rlRun: command not found
./runtest.sh: line 93: rlPhaseEnd: command not found
./runtest.sh: line 95: rlPhaseStartCleanup: command not found
./runtest.sh: line 96: rlRun: command not found
./runtest.sh: line 98: rlRun: command not found
./runtest.sh: line 99: rlPhaseEnd: command not found
./runtest.sh: line 100: rlJournalPrintText: command not found
./runtest.sh: line 101: rlJournalEnd: command not found
Error: exec session exited with non-zero exit code 127: OCI runtime error

I saw your old comment about vagrant-* packages here.
https://github.com/psss/tmt/issues/167#issuecomment-601726739

My current vagrant-* packages are like this. Do I need to install additional vagrant-* packages?

$ rpm -qa | grep vagrant
vagrant-2.2.9-1.fc32.noarch
vagrant-libvirt-0.0.45-4.fc32.noarch

I opened the issue ticket on the upstream project here.

beakerlib: libvirt.libvirtError: authentication unavailable: no polkit agent available to authenticate action 'org.libvirt.unix.manage'
https://github.com/psss/tmt/issues/329

I succeeded to run the following command.
The reason was that I needed to add my account to libvirt group, and needed to activate the default network on virtualization.

$ tmt run -avv execute -h beakerlib

Here is the worknig log.
https://github.com/psss/tmt/issues/329#issuecomment-683808192

However I am still facing the following error by running tmt with container.

$ tmt run -avv provision -h container -i fedora:rawhide test --name /run-basic-rails-application
/var/tmp/tmt/run-005

$ cat /var/tmp/tmt/run-005/plans/default/execute/logs/run-basic-rails-application/out.log
./runtest.sh: line 31: /usr/share/beakerlib/beakerlib.sh: No such file or directory
./runtest.sh: line 42: rlIsRHEL: command not found
...

However I am still facing the following error by running tmt with container.

Asking about this here.
https://github.com/psss/tmt/issues/336

Try the following:

tmt run -avv provision -h container execute -h beakerlib.tmt

This will both enable container provision and select beakerlib as execution method.

Hi @psss . Thanks for your help.

$ tmt run -avv provision -h container execute -h beakerlib.tmt

When running the command, ./runtest.sh: line NN: rlFoo: command not found errors were disappeared.

The log is here. https://gist.github.com/junaruga/def45ec7f7233a78f887e9c75a6bd2bc

@pvalena I see the following error on the systemtap-static-probes-in-ruby test. Can you fix it?

:: [ 08:14:09 ] :: [   PASS   ] :: Binary 'ruby' should belong to: ruby ruby-libs rubypick ruby-doc 
:: [ 08:14:09 ] :: [   FAIL   ] :: File /usr/share/doc/ruby-doc/ruby-exercise.stp should exist 
:: [ 08:14:09 ] :: [  FATAL   ] :: 
:: [ 08:14:09 ] :: [   FAIL   ] ::  (Assert: expected 0, got 1)

This error also happens on fedora:rawhide.

$ tmt run -avv provision -h container -i fedora:rawhide execute -h beakerlib.tmt 2>&1 | tee tmt_run_container_rawhide.log

The log is here. https://gist.github.com/junaruga/ebb7e58f07ddcf20b5f714fe80f75e69

When running only the rails test, it passes of course.

$ tmt run -avv provision -h container -i fedora:rawhide execute -h beakerlib.tmt test --name /run-basic-rails-application 2>&1 | tee tmt_run_container_rawhide_test_rails.lo
...
total: 1 test passed

@pvalena You are defining the following part in rpms/ruby's ci.fmf. Why not move the definition to testes/ruby's fmf file? It looks useful to run tests/ruby tests independently by tmt on local without rpms/ruby's CI.

https://src.fedoraproject.org/fork/pvalena/rpms/ruby/blob/tmt/f/ci.fmf

execute:    
    how: beakerlib

@pvalena I see the following error on the systemtap-static-probes-in-ruby test. Can you fix it?

:: [ 08:14:09 ] :: [ PASS ] :: Binary 'ruby' should belong to: ruby ruby-libs rubypick ruby-doc :: [ 08:14:09 ] :: [ FAIL ] :: File /usr/share/doc/ruby-doc/ruby-exercise.stp should exist :: [ 08:14:09 ] :: [ FATAL ] :: :: [ 08:14:09 ] :: [ FAIL ] :: (Assert: expected 0, got 1)

This error also happens on fedora:rawhide.

$ tmt run -avv provision -h container -i fedora:rawhide execute -h beakerlib.tmt 2>&1 | tee tmt_run_container_rawhide.log

The log is here. https://gist.github.com/junaruga/ebb7e58f07ddcf20b5f714fe80f75e69

This means not all required packages (artefacts) are installed on the test system (container). Let me check what's missing (probably ruby-doc). It worked for me though, but I may have installed it manually, before.

Note: these tests do not define the environment (there are requires though), which you (CI) should supply (image with rawhide / epel etc.), together with artifacts to test (the ruby RPMs f.e.). More on this bellow.


Hi @psss . Thanks for your help.

$ tmt run -avv provision -h container execute -h beakerlib.tmt

Note: You were missing execute -h beakerlib
More on that:

@pvalena You are defining the following part in rpms/ruby's ci.fmf. Why not move the definition to testes/ruby's fmf file? It looks useful to run tests/ruby tests independently by tmt on local without rpms/ruby's CI.

You're right this could / should be defined in the L1 metadata (= here with the tests). Let me check if it's possible yet.

Running the tests without the plan is however sort of workaround. They're meant to be run from the location of the PLAN (L2), where the tested component / artifacts are stored (like the ci.fmf you've linked).

To elaborate: the test is universal for all of upstream, like any github CI, or downstream, like Fedora or RHEL or CentOS. And for various components as well (rubygem-rails). The location may by anywhere also - f.e. it could be stored in github.com, if we wanted.

This means not all required packages (artefacts) are installed on the test system (container). Let me check what's missing (probably ruby-doc). It worked for me though, but I may have installed it manually, before.

Note: these tests do not define the environment (there are requires though), which you (CI) should supply (image with rawhide / epel etc.), together with artifacts to test (the ruby RPMs f.e.). More on this bellow.

OK. Thanks. I expected the following ruby-doc was used for the test. It's set as recommend in systemtap-static-probes-in-ruby/main.fmf rather than component (= required?). It is a reason of the error?

+ component:
+ - ruby
+ recommend:
+ - ruby
+ - ruby-doc

Note: You were missing execute -h beakerlib

Yes, that's because you showed the command line example tmt run -avv provision -h container -i fedora:rawhide test --name /run-basic-rails-application without execute -h beakerlib at https://src.fedoraproject.org/tests/ruby/pull-request/1#comment-0 . I just tried it at that time.

It might make sense to have plan (L2 metadata) stored in both tests and rpms namespace. In the first one for easy test debugging and development, in the second one so that tests are run in the CI. In the future we also consider to support referencing plans from plans, so there could be just a single config which would be referenced from other places.

2 new commits added

  • Cleanup
  • Fixup
2 years ago