Blame Sanity/Run-internal-BIND-test-suite/runtest.sh

fdd78bd
#!/bin/bash
fdd78bd
# vim: dict=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
fdd78bd
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fdd78bd
#
fdd78bd
#   runtest.sh of /CoreOS/bind/Sanity/Run-internal-BIND-test-suite
fdd78bd
#   Description: Run internal BIND test suite
fdd78bd
#   Author: Martin Cermak <mcermak@redhat.com>
fdd78bd
#
fdd78bd
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fdd78bd
#
fdd78bd
#   Copyright (c) 2010 Red Hat, Inc. All rights reserved.
fdd78bd
#
fdd78bd
#   This copyrighted material is made available to anyone wishing
fdd78bd
#   to use, modify, copy, or redistribute it subject to the terms
fdd78bd
#   and conditions of the GNU General Public License version 2.
fdd78bd
#
fdd78bd
#   This program is distributed in the hope that it will be
fdd78bd
#   useful, but WITHOUT ANY WARRANTY; without even the implied
fdd78bd
#   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
fdd78bd
#   PURPOSE. See the GNU General Public License for more details.
fdd78bd
#
fdd78bd
#   You should have received a copy of the GNU General Public
fdd78bd
#   License along with this program; if not, write to the Free
fdd78bd
#   Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
fdd78bd
#   Boston, MA 02110-1301, USA.
fdd78bd
#
fdd78bd
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
fdd78bd
#
fdd78bd
# Usable variables:
fdd78bd
# override make target to start testing on more threads
fdd78bd
# MAKE_TEST='-j4 test'
fdd78bd
# Do not clean existing build if already built
fdd78bd
# REUSE_BUILD=y
fdd78bd
# Make retest faster, skip build if possible
fdd78bd
# QUICK=y
fdd78bd
fdd78bd
# Include rhts environment
fdd78bd
. /usr/bin/rhts-environment.sh
fdd78bd
. /usr/lib/beakerlib/beakerlib.sh
fdd78bd
fdd78bd
PACKAGE=""
fdd78bd
rpm -q bind && PACKAGE="bind"
fdd78bd
rpm -q bind97 && PACKAGE="bind97"
fdd78bd
fdd78bd
rlJournalStart
fdd78bd
    rlPhaseStartSetup
fdd78bd
        rpm -q perl-Net-DNS-Nameserver || yum install -y perl-Net-DNS-Nameserver
fdd78bd
        rlLog "`rpm -q perl-Net-DNS-Nameserver`"
fdd78bd
        # package assertions
fdd78bd
        rlAssertRpm $PACKAGE 
fdd78bd
        rlAssertRpm rpm-build
fdd78bd
fdd78bd
	#pwd
fdd78bd
	ORIG=`pwd`
fdd78bd
	FOUNDERROR=`mktemp`
fdd78bd
	SETUP_SOFTHSM=`readlink -f setup-named-softhsm.sh`
fdd78bd
	FILTER=`readlink -f bind-systest-filter.sh`
fdd78bd
fdd78bd
	TAG=generic
fdd78bd
	if [ -f /etc/os-release ]; then
fdd78bd
		# extract platform tag
fdd78bd
		VERSION_ID=`(source /etc/os-release && echo ${ID}-${VERSION_ID})`
fdd78bd
		TAG=`(source /etc/os-release && echo ${PLATFORM_ID#platform:})`
fdd78bd
	else
fdd78bd
		rlIsRHEL '6' && TAG=el6
fdd78bd
	fi
fdd78bd
fdd78bd
	if [ -f "knownerror.$VERSION_ID" ]; then
fdd78bd
		KNOWNERROR=`readlink -f knownerror.$VERSION_ID`
fdd78bd
	elif [ -f "knownerror.$TAG" ]; then
fdd78bd
		KNOWNERROR=`readlink -f knownerror.$TAG`
fdd78bd
	elif [ -f "knownerror" ]; then
fdd78bd
		KNOWNERROR=`readlink -f knownerror`
fdd78bd
	fi
fdd78bd
fdd78bd
	if [ -n "$QUICK" ]; then
fdd78bd
		REUSE_BUILD=y
fdd78bd
		MAKE_TEST='test -j4'
fdd78bd
	fi
fdd78bd
fdd78bd
        #tempdir
fdd78bd
        rlRun "TMPDIR=\`mktemp -d\`" 0 "Creating tmp directory"
fdd78bd
        rlRun "pushd $TMPDIR"
fdd78bd
fdd78bd
	if rlIsRHEL 8 && dnf config-manager --help >/dev/null; then
fdd78bd
		# Some build dependencies are not in repositories enabled
fdd78bd
		# by default: libidn2-devel, softshm
fdd78bd
		dnf config-manager --set-enabled rhel-buildroot
fdd78bd
		#not checking return code: 1mt and beaker uses different names for repo
fdd78bd
	fi
fdd78bd
fdd78bd
        # topdir
fdd78bd
        if rlIsRHEL 3 || rlIsRHEL 4 || rlIsRHEL 5; then
fdd78bd
                TOPDIR="/usr/src/redhat"
fdd78bd
        else
fdd78bd
                TOPDIR="/root/rpmbuild"
fdd78bd
        fi
fdd78bd
fdd78bd
        # cleanup in topdir
fdd78bd
        mkdir -p $TOPDIR/{BUILD,SOURCES,SPECS}
fdd78bd
	if [ "$REUSE_BUILD" != y ]
fdd78bd
	then
fdd78bd
	        rlRun "rm -rf $TOPDIR/{BUILD,SOURCES,SPECS}/*"
fdd78bd
	else
fdd78bd
		rlLog "Not cleaning previous build"
fdd78bd
	fi
fdd78bd
fdd78bd
        # download src rpm
fdd78bd
        rlFetchSrcForInstalled "$PACKAGE"
fdd78bd
	if ! ls bind*.src.rpm; then
fdd78bd
		rlRun "yumdownloader --source bind" 0 "Trying alternative fetch from repository"
fdd78bd
		rlRun "rpm -i bind*.src.rpm"
fdd78bd
	fi
fdd78bd
	
fdd78bd
        rlRun "rpm --define '_topdir $TOPDIR' -Uvh *rpm &> $TMPDIR/install.txt"
fdd78bd
        rlRun "cd $TOPDIR/SPECS" 
fdd78bd
fdd78bd
	# softhsm is no longer in normal repository. Enable idm module on RHEL8 to make softhsm module available
fdd78bd
	if rlIsRHEL 8
fdd78bd
	then
fdd78bd
		rlRun "dnf -y module enable idm:DL1"
fdd78bd
	fi
fdd78bd
fdd78bd
	if dnf builddep --help >/dev/null; then
fdd78bd
		rlRun "dnf -y builddep --nobest *.spec"
fdd78bd
	elif which yum-builddep; then
fdd78bd
		rlRun "yum-builddep -y *.spec"
fdd78bd
	else
fdd78bd
		rlWarn "there is nor yum-utils neither dnf-utils for install dependencies, ENJOY!"
fdd78bd
	fi
fdd78bd
fdd78bd
        # stop bind if it is running
fdd78bd
        service named stop
fdd78bd
    rlPhaseEnd
fdd78bd
fdd78bd
    rlPhaseStartTest
fdd78bd
	if [ "$REUSE_BUILD" = y ] && ls -1 "$TOPDIR/BUILD"/bind* > /dev/null
fdd78bd
	then
fdd78bd
		rlLog "Skipping bind build"
fdd78bd
	else
fdd78bd
        	# rebuild from source
fdd78bd
	        rlRun "rpmbuild -ba *.spec &> $TMPDIR/build.txt" 0 "Building bind"
fdd78bd
	fi
fdd78bd
fdd78bd
        # the test
fdd78bd
        rlRun "cd $TOPDIR/BUILD/bind*"
fdd78bd
fdd78bd
        rlLogInfo "Test takes place in `pwd`"
fdd78bd
fdd78bd
        rlRun "chown -R root ."
fdd78bd
fdd78bd
	if [ -x "$SETUP_SOFTHSM" ]; then
fdd78bd
		rlRun "eval $(bash $SETUP_SOFTHSM -A)" 0 "Preparing PKCS#11 token slot"
fdd78bd
		rlRun "pkcs11-tokens" 0 "Testing token slot availability"
fdd78bd
	else
fdd78bd
		rlLog "PKCS#11 not initialized"
fdd78bd
	fi
fdd78bd
fdd78bd
	if [ -d build ]; then
fdd78bd
		BUILD=build
fdd78bd
	else
fdd78bd
		BUILD=.
fdd78bd
	fi
fdd78bd
fdd78bd
        rlRun "./bin/tests/system/ifconfig.sh up" 0 "Setup fake network interfaces."
fdd78bd
fdd78bd
	# required by idna test 
fdd78bd
	export LC_ALL=en_US.UTF-8
fdd78bd
fdd78bd
        rlRun "pushd $BUILD"
fdd78bd
	# keep separate results on 9.11+
fdd78bd
	rlRun "sed -e 's/testsummary.sh/& -n/' -i bin/tests/system/Makefile" 0 "Modify to keep results"
fdd78bd
	# dlz test is broken because specific build way we use. It is supported only by named-sdb
fdd78bd
	# but that is not even tested by testsuite
fdd78bd
	rlRun "sed -e 's/ dlz / /' -i bin/tests/system/Makefile" 0 "Skip always failing dlz test"
fdd78bd
fdd78bd
	# Try to fix tssgsig failures on some machines, do not use system kerberos configuration
fdd78bd
	export KRB5_CONFIG=/dev/null
fdd78bd
	# Allow speedup by defining MAKE_TEST='test -j4' on multiprocessor machines
fdd78bd
        rlRun "make ${MAKE_TEST:-test} &> $TMPDIR/test.txt" 0-255 "Perform the test."
fdd78bd
	export -n KRB5_CONFIG
fdd78bd
fdd78bd
	# This would catch just errors on 9.11+
fdd78bd
	if [ -f bin/tests/system/testsummary.sh ]; then
fdd78bd
		FAILED_TESTS=`grep '^R:[a-z0-9_-][a-z0-9_-]*:FAIL' $TMPDIR/test.txt | cut -d':' -f2 | sort | xargs echo`
fdd78bd
		if [ -n "$FAILED_TESTS" ]; then
fdd78bd
			rlLog "Failed tests: $FAILED_TESTS"
fdd78bd
			rlRun "tar czf $TMPDIR/failed-artifacts.tar.gz -C bin/tests/system $FAILED_TESTS" 0 "Archiving failed artifacts in tests"
fdd78bd
		else
fdd78bd
			rlLog "No failed tests"
fdd78bd
		fi
fdd78bd
	else
fdd78bd
		FAILED_TESTS=:any:
fdd78bd
		rlRun "tar czf $TMPDIR/failed-artifacts.tar.gz bin/tests/system" 0 "Archiving all system tests"
fdd78bd
	fi
fdd78bd
        rlRun "popd"
fdd78bd
fdd78bd
        rlRun "grep -C 10 FAIL $TMPDIR/test.txt" 0-255 "Quickly show the test error (if any)."
fdd78bd
fdd78bd
        rlRun "./bin/tests/system/ifconfig.sh down" 0 "Remove fake network interfaces."
fdd78bd
fdd78bd
fdd78bd
	#list of failures:
fdd78bd
	rlRun "$FILTER $TMPDIR/test.txt" 0 "Showing unsuccessful tests"
fdd78bd
	rlRun "$FILTER -s $TMPDIR/test.txt > $FOUNDERROR" 0
fdd78bd
	rlRun "ls $KNOWNERROR $FOUNDERROR $TMPDIR/test.txt" 0 'check if there are needed files'
fdd78bd
	rlLog "`echo list;cat $FOUNDERROR`"
fdd78bd
fdd78bd
	FAILED_FOUND="$(grep '^FAIL' $FOUNDERROR | wc -l)"
fdd78bd
	FAILED_KNOWN="$(wc -l <$KNOWNERROR)"
fdd78bd
	rlAssertLesserOrEqual "Checking number of found errors is in limits" "$FAILED_FOUND" "$FAILED_KNOWN"
fdd78bd
        cat $FOUNDERROR | while read STATUS TEST ; do
fdd78bd
		if [ "$STATUS" = FAIL ]; then
fdd78bd
			rlRun "grep '$TEST' $KNOWNERROR" 0 "Check $TEST failure is expected"
fdd78bd
		else
fdd78bd
			rlLog "$STATUS $TEST"
fdd78bd
		fi
fdd78bd
        done
fdd78bd
fdd78bd
	if [ "$FAILED_TESTS" = ':any:' ] && [ "$FAILED_FOUND" -le "$FAILED_KNOWN" ]
fdd78bd
	then
fdd78bd
		# Newer version produces archive only when some error occured
fdd78bd
		rlLog "No error found, not uploading artifacts"
fdd78bd
		rm -f "$TMPDIR/failed-artifacts.tar.gz"
fdd78bd
	fi
fdd78bd
    rlPhaseEnd
fdd78bd
fdd78bd
   rlPhaseStartCleanup "`echo RESULT_ ;cat $FOUNDERROR|grep 'FAIL'`"
fdd78bd
	#this phase is only due to report to webUI without needs of open file
fdd78bd
	rlLog "`echo RESULT_ ;cat $FOUNDERROR|grep FAIL`"
fdd78bd
   rlPhaseEnd
fdd78bd
fdd78bd
    rlPhaseStartCleanup
fdd78bd
        rlBundleLogs "TEST_LOGS" "$TMPDIR/install.txt" "$TMPDIR/builddeps.txt" "$TMPDIR/build.txt" "$TMPDIR/test.txt"
fdd78bd
	if [ -r "$TMPDIR/failed-artifacts.tar.gz" ]; then
fdd78bd
		rlFileSubmit "$TMPDIR/failed-artifacts.tar.gz" failed-artifacts.tar.gz
fdd78bd
	fi
fdd78bd
        rlRun "popd"
fdd78bd
        rlRun "rm -r $TMPDIR" 0 "Removing tmp directory"
fdd78bd
	rlRun "rm -rf $FOUNDERROR"
fdd78bd
    rlPhaseEnd
fdd78bd
rlJournalEnd