From bc88b5703fb3dd11b2c74fd77873764354b26dcb Mon Sep 17 00:00:00 2001 From: Matus Marhefka Date: Nov 05 2020 11:25:05 +0000 Subject: Sanity/smoke-test: new smoke test which runs upstream test suite --- diff --git a/Sanity/smoke-test/fix_hugepages_err.patch b/Sanity/smoke-test/fix_hugepages_err.patch new file mode 100644 index 0000000..4ca95ee --- /dev/null +++ b/Sanity/smoke-test/fix_hugepages_err.patch @@ -0,0 +1,12 @@ +diff --git a/tests/probes/sysctl/test_sysctl_probe_all.sh b/tests/probes/sysctl/test_sysctl_probe_all.sh +index 732d22ec1..119e92c33 100755 +--- a/tests/probes/sysctl/test_sysctl_probe_all.sh ++++ b/tests/probes/sysctl/test_sysctl_probe_all.sh +@@ -69,6 +69,7 @@ diff "$sysctlNames" "$ourNames" + + # remove oscap error message related to permissions from stderr + sed -i -E "/^E: lt-probe_sysctl: Can't read sysctl value from /d" "$stderr" ++sed -i "/^.*hugepages.*$/d" "$stderr" + [ ! -s $stderr ] + + rm $stderr $result $ourNames $sysctlNames diff --git a/Sanity/smoke-test/main.fmf b/Sanity/smoke-test/main.fmf new file mode 100644 index 0000000..b0f05fc --- /dev/null +++ b/Sanity/smoke-test/main.fmf @@ -0,0 +1,49 @@ +summary: Test runs upstream test suite. +description: | + Runs upstream test suite against installed openscap. Test suite is built from + the openscap source RPM by running %prep and %build stages from the spec file. +contact: Matus Marhefka +component: +- openscap +test: ./runtest.sh +framework: beakerlib +require: +- library(distribution/Cleanup) +- library(distribution/RpmSnapshot) +recommend: +- openscap-scanner +- openscap +- openscap-engine-sce +- python +- python3 +- gcc +- gcc-c++ +- perl-XML-XPath +- yum-utils +- rpm-build +- dnf-plugins-core +- wget +- lua +- which +- procps-ng +- initscripts +- chkconfig +- sendmail +- bzip2 +- strace +duration: 3h +enabled: true +tag: +- CI-Tier-1 +- Fedora-Private +- NoFIPS +- Tier1 +- Tier1security +- rhel8-buildroot +- smoke +tier: '1' +relevancy: | + distro < rhel-5: False +extra-summary: /CoreOS/openscap/Sanity/smoke-test +extra-task: /CoreOS/openscap/Sanity/smoke-test +extra-nitrate: TC#0075476 diff --git a/Sanity/smoke-test/review_notes b/Sanity/smoke-test/review_notes new file mode 100644 index 0000000..737c8e9 --- /dev/null +++ b/Sanity/smoke-test/review_notes @@ -0,0 +1,16 @@ +This document is meant to help review process of openscap upstream test suite +while doing rebases. At the very least the last reviewed version should be +mentioned here. Plus any mandatory items with links to appropriate bugs, so +changes (even innocent looking ones) can be evalued according to the real impact +they might have to our productization efforts. + +For more detail, see https://tcms.engineering.redhat.com/case/571216 + +To obtain a diff between commits/tags use: +$ git log -p REVIEWED_COMMIT^..LATEST_COMMIT -- tests/ +To obtain only names of changed files add `--name-only`. +Example: +$ git log -p 1.3.1^..1.3.2 -- tests/ + +Last reviewed version: openscap-1.3.3-1 +https://github.com/OpenSCAP/openscap/commit/0cb55c55af6be9934d6fd0caf4563b206f289732 diff --git a/Sanity/smoke-test/runtest.sh b/Sanity/smoke-test/runtest.sh new file mode 100755 index 0000000..871a22a --- /dev/null +++ b/Sanity/smoke-test/runtest.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# vim: dict=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# runtest.sh of /CoreOS/openscap/Sanity/smoke-test +# Description: Test runs upstream test suite. +# Author: Ondrej Moris +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Copyright (c) 2010 Red Hat, Inc. All rights reserved. +# +# This copyrighted material is made available to anyone wishing +# to use, modify, copy, or redistribute it subject to the terms +# and conditions of the GNU General Public License version 2. +# +# This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +. /usr/lib/beakerlib/beakerlib.sh + +PACKAGE="openscap" + +rlJournalStart + + rlPhaseStartSetup + rlImport --all || rlDie "Failed to import libraries" + rlAssertRpm "$PACKAGE" + + RpmSnapshotCreate + CleanupRegister "RpmSnapshotRevert" + rlRun "TmpDir=\$(mktemp -d)" 0 + CleanupRegister "rlRun 'rm -r $TmpDir' 0 'Removing tmp directory'" + rlRun "cp fix_hugepages_err.patch $TmpDir" + rlRun "pushd $TmpDir" + CleanupRegister "rlRun 'popd'" + rlServiceStart sendmail + CleanupRegister "rlRun 'rlServiceRestore sendmail'" + + if rlIsRHEL 7; then + rlLog "Build system is using the latest version of python, therefore" + rlLog "python3 packages must not be present on the machine to make" + rlLog "sure that the build system configures with python2." + rlRun "yum remove -y python3*" 0 "python3 packages must not be installed" + elif rlIsRHEL ">=8"; then + rlFileBackup "/etc/yum.repos.d/" + CleanupRegister "rlFileRestore" + # we have to enable buildroot repo + rlRun "dnf config-manager --set-enabled rhel-buildroot{,-debuginfo,-source}" 0-255 + # Some packages needed to build openscap moved from rhel-buildroot to rhel-CRB + rlRun "dnf config-manager --set-enabled rhel-CRB{,-debuginfo,-source}" 0-255 + rlRun "dnf config-manager --set-enabled beaker-CRB{,-debuginfo,-source}" 0-255 + # AppStream source repo is not enabled by default + rlRun "dnf config-manager --set-enabled beaker-AppStream-source" 0-255 + fi + + rlFetchSrcForInstalled $PACKAGE + if rlIsFedora || rlIsRHEL ">=8"; then + # --nobest is used as a workaround for the bz#1677583 + rlRun "dnf --nobest builddep -y $PACKAGE*" + else + rlRun "yum-builddep -y $PACKAGE*" + fi + TOPDIR=`rpm --eval %_topdir` + rlRun "rm -rf ${TOPDIR}/BUILD/${PACKAGE}*" 0-255 + rlRun "rpm -ihv `ls *.rpm`" 0 "Install $PACKAGE source RPM" + rlPhaseEnd + + rlPhaseStartSetup "Prepare upstream test suite (%prep and %build stages from the spec file)" + rlRun "rpmbuild -v -bc ${TOPDIR}/SPECS/${PACKAGE}.spec" + CleanupRegister "rlRun 'rm -rf ${TOPDIR}/BUILD/${PACKAGE}*'" + CleanupRegister "rlRun 'rm -rf ${TOPDIR}/SPECS/${PACKAGE}*'" + CleanupRegister "rlRun 'rm -rf ${TOPDIR}/SOURCES/*'" + if rlIsRHEL 7; then + rlLogInfo "Waive the known issue with hugepages on ppc64/ppc64le" + rlLogInfo "platforms which won't be fixed in RHEL7:" + rlLogInfo "https://bugzilla.redhat.com/show_bug.cgi?id=1607382" + TEST_CWD=$(pwd) + TREE_CWD=$(readlink -f ${TOPDIR}/BUILD/${PACKAGE}*) + rlRun "cp fix_hugepages_err.patch $TREE_CWD" + rlRun "cd $TREE_CWD" + rlRun "patch -p1 < fix_hugepages_err.patch" + rlRun "cd $TEST_CWD" 0 "Restore test default working directory" + fi + rlPhaseEnd + + rlPhaseStartTest "Run upstream test suite against installed $PACKAGE" + rlRun "export OSCAP_FULL_VALIDATION=1" + rlRun "export CUSTOM_OSCAP=$(which oscap)" + if rlIsFedora || rlIsRHEL ">=8"; then + rlRun -s "cmake --build $(readlink -f ${TOPDIR}/BUILD/${PACKAGE}*/build) --target test" + rv=$? + # cmake output does not contain enough information, submit verbose output log + FILE="$TOPDIR/BUILD/openscap*/build/Testing/Temporary/LastTest.log" + rlFileSubmit $(readlink -f $FILE) test_verbose_output.log + else + rlRun -s "make -C $(readlink -f ${TOPDIR}/BUILD/${PACKAGE}*) check" + rv=$? + fi + + # if we got error, submit file with result of particular test for easier debugging + if [ $rv -ne 0 ]; then + if rlIsFedora || rlIsRHEL ">=8"; then + FILE="${TOPDIR}/BUILD/${PACKAGE}*/build/tests" + rlBundleLogs test_outputs_all $(readlink -f $FILE) + elif grep "See tests/" $rlRun_LOG; then + RESULT=`grep "See tests/" $rlRun_LOG | sed -e "s/See tests\(.*\)/tests\1/"` + FILE="${TOPDIR}/BUILD/${PACKAGE}*/${RESULT}" + cat $(readlink -f $FILE) + rlBundleLogs test_outputs $(dirname $FILE) + if grep "See tests/" $FILE; then # we need to go deeper ... + RESULT=`grep "See tests/" $FILE | sed -e "s/See tests\(.*\)/tests\1/"` + FILE="${TOPDIR}/BUILD/${PACKAGE}*/${RESULT}" + cat $(readlink -f $FILE) + rlBundleLogs test_outputs_deep $(dirname $FILE) + fi + fi + fi + rlPhaseEnd + + rlPhaseStartCleanup + CleanupDo + rlPhaseEnd + +rlJournalPrintText +rlJournalEnd