diff --git a/tests/roles/custom-test-fonts/README.md b/tests/roles/custom-test-fonts/README.md index ac6b267..c2fae1c 100644 --- a/tests/roles/custom-test-fonts/README.md +++ b/tests/roles/custom-test-fonts/README.md @@ -14,6 +14,8 @@ to run testing. You can redefine the following variables: similar as rebuilding an rpm package to be tested) so this should be used with caution on only when necessary. * **path_prefix**: The directory on the system where fonts are installed. + please use one in coverage sub-parameter if having different path_prefix + per sub-packages. * **package**: The package name to test. this is used to find out fontconfig config file. please use one in families sub-parameter if having different config files per sub-packages. @@ -53,11 +55,13 @@ You can redefine the following variables for dictionary format: * **name**: The name to store logs. the test script is trying to make an unique file names to store logs but not perfectly working in some cases. this is optional parameter to make it unique by yourself. + * **path_prefix**: A list of directory names where fonts are installed on system. ## Family test parameters * **lang**: A language to test family name for. * **alias**: An alias name to test. + * **family**: A family name to test, which is supposed to be assinged to the alias. * **package**: The package name to test. this is used to find out fontconfig config file. this is optional. if not specified here, global `package` parameter will be used. diff --git a/tests/roles/custom-test-fonts/files/run-family-test b/tests/roles/custom-test-fonts/files/run-family-test index e87cad0..bbbed73 100755 --- a/tests/roles/custom-test-fonts/files/run-family-test +++ b/tests/roles/custom-test-fonts/files/run-family-test @@ -83,13 +83,13 @@ debug "Family: $OPT_FAMILY" debug "Lang: $OPT_LANG" debug "Artifacts dir: $OPT_ARTIFACTS_DIR" debug "Package name: $OPT_PACKAGE" -STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}" | sed -e 's/\//-/g' -e 's/ /-/g') +STR_TEST_DASHED=$(echo "${OPT_PACKAGE}_${OPT_ALIAS}_${OPT_LANG}" | sed -e 's/\//-/g' -e 's/ /-/g') debug "Log file: $STR_TEST_DASHED.log" clean_exit() { rc=$?; trap - SIGINT SIGTERM SIGABRT EXIT - echo "Run test $OPT_ALIAS: done." + echo "Run test $OPT_ALIAS: done. Test's exit code: $rc" for pid in $(ps -o pid --no-headers --ppid $$); do if [ -n "$(ps -p $pid -o pid=)" ]; then kill -s HUP $pid @@ -114,11 +114,16 @@ clean_exit() { status="ERROR" fi echo "$status $OPT_ALIAS" >> "$OPT_ARTIFACTS_DIR/test.log" - mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}_${log_file_name}" + mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}" local results="$OPT_ARTIFACTS_DIR/results.yml" local result=$(echo $status | tr '[:upper:]' '[:lower:]') test -f "$results" || echo 'results:' > "$results" - echo "- {result: $result, test: $OPT_ALIAS}" >> "$results" + printf '%s\n' '' \ + "- test: $OPT_ALIAS" \ + " result: $result" \ + " logs:" \ + " - ${status}_${log_file_name}" \ + >> "$results" exit 0 } trap clean_exit SIGINT SIGTERM SIGABRT EXIT @@ -139,15 +144,19 @@ debug "Config: `cat $tmpconfd/fonts.conf`" mkdir -p "$OPT_ARTIFACTS_DIR" export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log" -logfile_stdout="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log" -logfile_stderr="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED-err.log" -exec 3>&1 4>&2 1> >(tee -a "$logfile_stdout" >&3) 2> >(tee -a "$logfile_stderr" >&4) +logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log" +logfile="$(realpath "$logfile")" +exec > >(tee -a "$logfile") 2>&1 debug "Check family assignment" res=`FONTCONFIG_FILE=$tmpconfd/fonts.conf fc-match -f "%{family[0]}" :family=$OPT_ALIAS:lang=$OPT_LANG` +ret=0 if [ "x$res" = "x$OPT_FAMILY" ]; then echo "RESULT: PASS: $OPT_FAMILY was assigned to $OPT_ALIAS as expected" else - echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)" + echo "RESULT: FAIL: $OPT_FAMILY wasn't assigned to $OPT_ALIAS (actual result: $res)" + ret=1 fi rm -rf $tmpconfd + +exit $ret diff --git a/tests/roles/custom-test-fonts/files/run-lang-coverage-test b/tests/roles/custom-test-fonts/files/run-lang-coverage-test index 81fefa8..299f06b 100755 --- a/tests/roles/custom-test-fonts/files/run-lang-coverage-test +++ b/tests/roles/custom-test-fonts/files/run-lang-coverage-test @@ -1,13 +1,13 @@ #! /bin/bash -efu debug() { - if [ -n "$DEBUG" ]; then - echo "$*" >&2 - fi + if [ -n "$DEBUG" ]; then + echo "$*" >&2 + fi } msg_usage() { - cat <<_EOF_ + cat <<_EOF_ Run language coverage test. Usage: @@ -28,7 +28,7 @@ _EOF_ PROG="${PROG:-${0##*/}}" DEBUG="${DEBUG:-}" OPT_LANG="${OPT_LANG:-en}" -OPT_PATH="${OPT_PATH:-}" +OPT_PATH=() OPT_ARTIFACTS_DIR="${OPT_ARTIFACTS_DIR:-}" OPT_EXCLUDE=() OPT_INCLUDE=() @@ -37,161 +37,197 @@ OPT_NAME="${OPT_NAME:-}" opt=$(getopt -n "$0" --options "hvl:p:n:a:e:i:" --longoptions "help,verbose,lang:,path:,name:,artifactsdir:,exclude:,include:" -- "$@") eval set -- "$opt" while [[ $# -gt 0 ]]; do - case "$1" in + case "$1" in -n|--name) - OPT_NAME="$2" - shift 2 - ;; + OPT_NAME="$2" + shift 2 + ;; -i|--include) - OPT_INCLUDE+=("$2") - shift 2 - ;; + OPT_INCLUDE+=("$2") + shift 2 + ;; -e|--exclude) - OPT_EXCLUDE+=("$2") - shift 2 - ;; + OPT_EXCLUDE+=("$2") + shift 2 + ;; -a|--artifactsdir) - OPT_ARTIFACTS_DIR="$2" - shift 2 - ;; + OPT_ARTIFACTS_DIR="$2" + shift 2 + ;; -p|--path) - OPT_PATH="$2" - shift 2 - ;; + OPT_PATH+=("$2") + shift 2 + ;; -l|--lang) - OPT_LANG="$2" - shift 2 - ;; + OPT_LANG="$2" + shift 2 + ;; -v|--verbose) - DEBUG="-v" - shift - ;; + DEBUG="-v" + shift + ;; -h|--help) - msg_usage - exit 0 - ;; + msg_usage + exit 0 + ;; --) - shift - ;; + shift + ;; *) - msg_usage - exit 1 - esac + msg_usage + exit 1 + esac done if [ -z "$OPT_ARTIFACTS_DIR" ] || [ -z "$OPT_LANG" ] || [ -z "$OPT_PATH" ]; then - echo "Use: $PROG -h for help." - exit 0 + echo "Use: $PROG -h for help." + exit 0 fi -expand_regex() { - local e ret=() - local path="$1" - shift - (cd $path; - for e; do - debug "$e" - set +f - local x=$(find -regextype posix-egrep -regex "./$e" -print|sed -e 's,^\./,,g') - ret+=($x) - set -f - echo ${ret[@]} - done) - echo ${ret[@]} -} - -OPT_INCLUDE=($(expand_regex $OPT_PATH ${OPT_INCLUDE[@]})) -OPT_EXCLUDE=($(expand_regex $OPT_PATH ${OPT_EXCLUDE[@]})) - -debug "Path: $OPT_PATH" -debug "Lang: $OPT_LANG" -debug "Artifacts dir: $OPT_ARTIFACTS_DIR" -debug "Exclude: ${OPT_EXCLUDE[@]}" -debug "Include: ${OPT_INCLUDE[@]}" -STR_TEST_DASHED=$(echo "${OPT_NAME:-${OPT_LANG}_${OPT_PATH}}" | sed -e 's/\//-/g') +STR_TEST_DASHED=$(echo "${OPT_NAME:-$OPT_LANG}" | sed -e 's/\//-/g') clean_exit() { - rc=$?; - trap - SIGINT SIGTERM SIGABRT EXIT - echo "Run test $OPT_PATH: done." - for pid in $(ps -o pid --no-headers --ppid $$); do - if [ -n "$(ps -p $pid -o pid=)" ]; then - kill -s HUP $pid - fi - done - local log_file_name="$STR_TEST_DASHED.log" - local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name" - local status - if [[ $rc -eq 127 ]]; then - status="ERROR" - elif grep -q "RESULT: WARN" "$log_file_path"; then - status="ERROR" - elif grep -q "RESULT: FAIL" "$log_file_path"; then - status="FAIL" - elif grep -q "RESULT: PASS" "$log_file_path"; then - status="PASS" - elif grep -q "FAIL" "$log_file_path"; then - status="FAIL" - elif grep -q "PASS" "$log_file_path"; then - status="PASS" - else - status="ERROR" + rc=$?; + trap - SIGINT SIGTERM SIGABRT EXIT + echo "Run test $OPT_LANG: done. Test's exit code: $rc" + for pid in $(ps -o pid --no-headers --ppid $$); do + if [ -n "$(ps -p $pid -o pid=)" ]; then + kill -s HUP $pid fi - echo "$status $OPT_PATH" >> "$OPT_ARTIFACTS_DIR/test.log" - mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}_${log_file_name}" - local results="$OPT_ARTIFACTS_DIR/results.yml" - local result=$(echo $status | tr '[:upper:]' '[:lower:]') - test -f "$results" || echo 'results:' > "$results" - echo "- {result: $result, test: $OPT_PATH}" >> "$results" - exit 0 + done + local log_file_name="$STR_TEST_DASHED.log" + local log_file_path="$OPT_ARTIFACTS_DIR/$log_file_name" + local status + if [[ $rc -eq 127 ]]; then + status="ERROR" + elif grep -q "RESULT: WARN" "$log_file_path"; then + status="ERROR" + elif grep -q "RESULT: FAIL" "$log_file_path"; then + status="FAIL" + elif grep -q "RESULT: PASS" "$log_file_path"; then + status="PASS" + elif grep -q "WARN" "$log_file_path"; then + status="ERROR" + elif grep -q "FAIL" "$log_file_path"; then + status="FAIL" + elif grep -q "PASS" "$log_file_path"; then + status="PASS" + else + status="ERROR" + fi + echo "$status $OPT_LANG" >> "$OPT_ARTIFACTS_DIR/test.log" + mv "$log_file_path" "$OPT_ARTIFACTS_DIR/${status}-${log_file_name}" + local results="$OPT_ARTIFACTS_DIR/results.yml" + local result=$(echo $status | tr '[:upper:]' '[:lower:]') + test -f "$results" || echo 'results:' > "$results" + printf '%s\n' '' \ + "- test: $OPT_LANG" \ + " result: $result" \ + " logs:" \ + " - ${status}_${log_file_name}" \ + >> "$results" + exit 0 } trap clean_exit SIGINT SIGTERM SIGABRT EXIT mkdir -p "$OPT_ARTIFACTS_DIR" export OUTPUTFILE="$(realpath "$OPT_ARTIFACTS_DIR")/$STR_TEST_DASHED-out.log" -logfile_stdout="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log" -logfile_stderr="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED-err.log" -exec 3>&1 4>&2 1> >(tee -a "$logfile_stdout" >&3) 2> >(tee -a "$logfile_stderr" >&4) +logfile="$OPT_ARTIFACTS_DIR/$STR_TEST_DASHED.log" +logfile="$(realpath "$logfile")" +exec > >(tee -a "$logfile") 2>&1 + +expand_regex() { + local p ret=() + local regex="$1" + shift + debug "Expanding $regex" + for p; do + set +f + debug "$p: $regex" + (cd $p; + local x=$(find -regextype posix-egrep -regex "./$regex" -print|sed -e 's,^\./,,g') + debug "$x" + ret+=($x) + set -f + echo -n ${ret[@]} + ) + done + echo -n ${ret[@]} +} + +iv=() +ev=() +x=() +for p in ${OPT_INCLUDE[@]}; do + x=$(expand_regex $p ${OPT_PATH[@]}) + if [ "x$x" == "x" ]; then + echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?" + continue + fi + iv=("${iv[@]}" "${x[@]}") +done +for p in ${OPT_EXCLUDE[@]}; do + x=$(expand_regex $p ${OPT_PATH[@]}) + if [ "x$x" == "x" ]; then + echo "RESULT: WARN: No matches on \"$p\". maybe typo or something changed?" + continue + fi + ev=("${ev[@]}" "${x[@]}") +done +OPT_EXCLUDE=(${ev[@]}) +OPT_INCLUDE=(${iv[@]}) + +debug "Path: ${OPT_PATH[@]}" +debug "Lang: $OPT_LANG" +debug "Artifacts dir: $OPT_ARTIFACTS_DIR" +debug "Exclude: ${#OPT_EXCLUDE[@]}: ${OPT_EXCLUDE[@]}" +debug "Include: ${#OPT_INCLUDE[@]}: ${OPT_INCLUDE[@]}" contains() { - local e match="$1" - shift - for e; do [[ "$e" == "$match" ]] && return 1; done - return 0 + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 1; done + return 0 } debug "Check language coverage" +ret=0 set +f -for i in `find $OPT_PATH -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do -set -f +for p in ${OPT_PATH[@]}; do + for i in `find $p -regex '.*/*\.\(t1\)?\(ttf\)?\(otf\)?\(ttc\)?\(pcf.*\)?\(pfa\)?'`; do + set -f debug "$i" if test -f $i; then - n=`basename $i` + n=`basename $i` + set +e + contains "$n" "${OPT_EXCLUDE[@]}" + ret=$? + set -e + if [ $ret -eq 1 ]; then + debug "ignoring $i" + continue + fi + if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then set +e - contains "$n" "${OPT_EXCLUDE[@]}" + contains "$n" "${OPT_INCLUDE[@]}" ret=$? set -e - if [ $ret -eq 1 ]; then - debug "ignoring $i" - continue - fi - if [ ${#OPT_INCLUDE[@]} -ne 0 ]; then - set +e - contains "$n" "${OPT_INCLUDE[@]}" - ret=$? - set -e - if [ $ret -eq 0 ]; then - debug "$i isn't targeted file" - continue - fi - fi - debug " $i" - res=`fc-validate -l $OPT_LANG $i` - if echo $res | grep -q Missing; then - echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage." - else - echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage." + if [ $ret -eq 0 ]; then + debug "$i isn't targeted file" + continue fi + NOT_MATCHED=("${NOT_MATCHED[@]/$n}") + fi + debug " $i" + res=`fc-validate -l $OPT_LANG $i || :` + if echo $res | grep -q Missing; then + echo "RESULT: FAIL: $i doesn't satisfy $OPT_LANG language coverage." + ret=1 + else + echo "RESULT: PASS: $i satisfy $OPT_LANG language coverage." + fi fi + done done +exit $ret diff --git a/tests/roles/custom-test-fonts/tasks/main.yml b/tests/roles/custom-test-fonts/tasks/main.yml index 981d3aa..2665795 100644 --- a/tests/roles/custom-test-fonts/tasks/main.yml +++ b/tests/roles/custom-test-fonts/tasks/main.yml @@ -2,15 +2,23 @@ - block: - name: language coverage - script: run-lang-coverage-test --lang "{{ item }}" --path "{{ path_prefix }}" --artifactsdir "{{ remote_artifacts }}" {{ "" if coverage.values is not defined or coverage[item].name is not defined else "--name " + coverage[item].name }} {{ "" if coverage.values is not defined or coverage[item].exclude is not defined else '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"'}} {{ "" if coverage.values is not defined or coverage[item].include is not defined else '--include "' + (coverage[item].include | join('" --include "')) + '"'}} + environment: + LANG: "en_US.UTF-8" + script: run-lang-coverage-test --lang "{{ item }}" {% if coverage.values is not defined or coverage[item].path_prefix is not defined %} --path {{ path_prefix }} {% else %} {{ '--path "' + (coverage[item].path_prefix | join('" --path "')) + '"' }} {% endif %} --artifactsdir "{{ remote_artifacts }}" {{ "" if coverage.values is not defined or coverage[item].name is not defined else "--name " + coverage[item].name }} {{ "" if coverage.values is not defined or coverage[item].exclude is not defined else '--exclude "' + (coverage[item].exclude | join('" --exclude "')) + '"'}} {{ "" if coverage.values is not defined or coverage[item].include is not defined else '--include "' + (coverage[item].include | join('" --include "')) + '"'}} with_items: - "{{ coverage if coverage.keys is not defined else coverage.keys()|list }}" - name: generic family assignment + environment: + LANG: "en_US.UTF-8" when: families is defined script: run-family-test --lang {{ item.lang }} --family '{{ item.family }}' --alias {{ item.alias }} --artifactsdir {{ remote_artifacts }} --package {{ package if item.package is not defined else item.package }} with_items: - "{{ families }}" + - name: Collect all of failures into one file + shell: | + grep -v PASS "{{ remote_artifacts }}"/FAIL_*.log > "{{ remote_artifacts }}"/FAIL-all.log || : + - name: Check the results shell: | log="{{ remote_artifacts }}/test.log" @@ -38,3 +46,7 @@ - include_role: name: str-common-final + + - name: Validate the result + shell: echo "test_results.stdout" + failed_when: test_results.stdout != 'PASS'