#!/usr/bin/env bash
#------------------------------------------------------------------------------#
#  DFTB+: general package for performing fast atomistic simulations            #
#  Copyright (C) 2006 - 2023  DFTB+ developers group                           #
#                                                                              #
#  See the LICENSE file for terms of usage and distribution.                   #
#------------------------------------------------------------------------------#

############################################################################
#
#  Script for automated testing
#
############################################################################
#
# (merged previous autotest and autotest_collect, with controllable stages)
#
# TODO:
#	- figure out needed stages automatically, making that the default
#	- if $ROOT exists, try printing log output relative to it
#
# NB:	- what if just given a single test dir? -> useless
#		- need refdir to compare, or can do "summary" by ls
#
############################################################################

SCRIPTNAME=`basename $0`
SCRIPTDIR=`dirname $0`
OWD=`pwd`

#---------------------------------------------------------------------
# hardwired file names and message strings
#---------------------------------------------------------------------
TAGDIFF_CONF=tagdiff.conf
TAGDIFF_LOG=tagdiff.log

TEST_SCRIPT=testrun.sh

TAGDIFF_MATCH_OK="OK"
TAGDIFF_MATCH_INCOMPLETE="Not found in new"

#APP_STDIN=input
APP_INPUT=dftb_in.hsd
APP_STDOUT=output
APP_STDERR=stderror.log
APP_TAGDATA=autotest.tag
APP_TAGDATA_REF=_autotest.tag

TIME_CMD=time
# Override time command if not available on the system
($TIME_CMD ls 2>&1) > /dev/null || TIME_CMD=""

SUMMARY_TAGDIFF_LOG="$TAGDIFF_LOG"
SUMMARY_STDERROR="$APP_STDERR"

STATUS_PREFIX=_tagstatus_
STATUS_MATCH=Match
STATUS_MISMATCH=Mismatch
STATUS_INCOMPLETE=Incomplete
STATUS_ERROR=Error
STATUS_TODO=Not_run
TABLE_HEADER="======================================="
TABLE_HEADER="$TABLE_HEADER$TABLE_HEADER"
TABLE_SEP="---------------------------------------"
TABLE_SEP="$TABLE_SEP$TABLE_SEP"

LIST_NONE="None"
ENODIR="Couldn't change to directory"
STAGE_SEP=","


#---------------------------------------------------------------------
# Help output
#---------------------------------------------------------------------
Usage () {
    cat << EOT
Performs automated tests and check results with tagdiff
Usage: ${SCRIPTNAME} [options] [test1 test2 .. ]
Options (default value in parenthesis):
    -r templateroot	path to root of test template directories (SCRIPTDIR/..)
    -w workdir		workdir to run the tests (./)
    -P prerun           command to use before executing binary
    -Q postrun          command on line after binary
    -p program		application program generating tagged output
    -d tagdiff		path to the tagdiff program (SCRIPTDIR/tagdiff)
    -f testspec		file containing test names -- cumulative
    -l			parse testspec file and list tests
    -F			(force) annul previous -f
    -c                  check executable binary is present
    -s stage1,stage2,...
    			Perform given stage[s] -- cumulative.
                        Known stages:
			    P - prepare
			    R - run
			    C - compare
			    S - summary
                            A - all (equal to P,R,C,S)

    -h			print this help
    -v			verbose (show stages)

    test1 ..	individual tests relative to "templateroot"
EOT
    exit 1
}

#---------------------------------------------------------------------
# portability functions for bash/ksh features
#---------------------------------------------------------------------
# NB: if the local shell can't even handle functions, well, tough luck!

# cd with a more informative error message than errno(3) provides.
safecd () {
    cd $1 > /dev/null 2>&1 || {
	echo Error: $ENODIR "\"$1\"" 1>&2
	exit 1
    }
}

# echo without trailing newline; needs initializtion first:
case "`echo -n`" in
    "")	_ECHO_PREFIX="-n"; _ECHO_POSTFIX="" ;;     # BSD
    *)	_ECHO_PREFIX="";   _ECHO_POSTFIX="\\c" ;;  # SysV
esac
echo_n () {
    echo ${_ECHO_PREFIX} ${1+"$@"}${_ECHO_POSTFIX}
}

# cp $1 to $2/$1, maintaining tree depth; exclude some file patterns
deepcopydir_filtered () {
    dst=$2/$1
    [ -d $dst ] || mkdir -p $dst
    [ -d $dst ] || exit 1		# exit if dest. dir cannot be created
    for i in `ls $1/`; do
	case "$i" in CVS*|_*) continue ;; esac	# filter
	cp -p -f $1/$i $dst
    done
}


#---------------------------------------------------------------------
# Defaults
#---------------------------------------------------------------------
templateroot=${SCRIPTDIR}/..
workroot=.
tagdiff_cmd=${SCRIPTDIR}/tagdiff


#---------------------------------------------------------------------
# parse options
#---------------------------------------------------------------------
while :
do
    case "$1" in
	-r)	templateroot=$2; shift 2 ;;
	-w)	workroot=$2; shift 2 ;;

        -P)     app_prerun=$2; shift 2;;
        -Q)     app_postrun=$2; shift 2;;
	-p)	app_cmd=$2; shift 2 ;;
        -d)	tagdiff_cmd=$2; shift 2 ;;

	# the following act cumulatively:
	-f)	testspec_files="$testspec_files $2"; shift 2 ;;
	-F)	testspec_files=""; shift ;;	# countermand -f
	-l)	list_tests=1; shift ;;
	-c)	check_for_bin=1; shift ;;
	-s)	stages="${stages}${stages_sep}$2"
		stages_sep=${STAGE_SEP}; shift 2 ;;

	# standard options
        --)	shift; break ;;		# stop interpreting
	-h|--help) Usage ;;
	-v)	verbose=1; shift ;;
	-q)	QUIET=1; shift ;;	# reserved; not implemented
	-x)	set -x; shift ;;	# undocumented
        -*)	echo "Error: unknown option '$1'" 1>&2
		echo "Type '${SCRIPTNAME} -h' for help" 1>&2
		exit 1
		;;
        *)	break ;;
    esac
done


#---------------------------------------------------------------------
# process option args and main args
#---------------------------------------------------------------------
    # figure out stages requested
case "${STAGE_SEP}$stages" in *${STAGE_SEP}P*) STAGE_PREPARE=1;; esac
case "${STAGE_SEP}$stages" in *${STAGE_SEP}R*) STAGE_RUN=1    ;; esac
case "${STAGE_SEP}$stages" in *${STAGE_SEP}C*) STAGE_COMPARE=1;; esac
case "${STAGE_SEP}$stages" in *${STAGE_SEP}S*) STAGE_SUMMARY=1;; esac
case "${STAGE_SEP}$stages" in
  *${STAGE_SEP}A*)
    STAGE_PREPARE=1
    STAGE_RUN=1
    STAGE_COMPARE=1
    STAGE_SUMMARY=1
    ;;
esac

if [ -n "$check_for_bin" ]; then
    # Check if executable is there:
    if [ ! -x "${app_cmd}" -a -n "${STAGE_RUN}" ]; then
	echo "${SCRIPTNAME}: Error: Executable '${app_cmd}' does not exist" 1>&2
	echo "Type '${SCRIPTNAME} -h' for help" 1>&2
	exit 1
    fi
fi

if [ -n "$STAGE_PREPARE" -o -n "$STAGE_RUN" -o -n "$STAGE_COMPARE" \
    -o -n "$verbose" ]
then
    show_tests=1
elif [ -n "$STAGE_SUMMARY" ]; then
    # TODO: set STAGE_AUTO
    :
fi

    # Append contents of spec file to args
for file in ${testspec_files}; do
  if [ -f $file ]; then
    tests="$tests `sed 's/#.*//' $file`"
  else
    echo "Warning: Test file '$file' does not exist => skipped." 1>&2
  fi
done
set -- ${1+"$@"} $tests
tests=${1+"$@"}

    # For "-l": output contents of parsed file and stop
if [ -n "$list_tests" ]; then
    echo "$tests" | xargs -n1
    exit
fi

if [ -z "$tests" ]; then
  exit
fi

    # Expand reference and work dir paths to absolute.
    # When appropriate and not yet there, create:
if [ -n "$STAGE_PREPARE" ]; then
    mkdir -p $workroot || exit 1
fi
workroot_orig=$workroot
safecd $workroot; workroot=`pwd`; cd $OWD

    # construct relative path to workroot from parent, unless given absolute
case "$workroot_orig" in
    /*)	workroot_parented=$workroot_orig ;;
    *)	workroot_parented=`basename $workroot_orig` ;;
esac


    # always needed, hence enforce:
templateroot_orig=$templateroot
safecd $templateroot; templateroot=`pwd`; cd $OWD

    # Expand $app_cmd and $tagdiff_cmd to absolute path if given relative.
    # Otherwise, they are exected to be found in $PATH.
case "$tagdiff_cmd" in
  /*)	;;
  */*)	tagdiff_cmd=$OWD/$tagdiff_cmd ;;
esac
case "$app_cmd" in
  /*)	;;
  */*)	app_cmd=$OWD/$app_cmd ;;
esac


#------------------------------------------------------------
# Init test environment
#------------------------------------------------------------

if [ -n "$STAGE_PREPARE" ]; then
    safecd $templateroot		# also validate

    # prepare all tests at once -- useful one day for testing in parallel
    for testname in $tests
    do
	deepcopydir_filtered $testname $workroot
    done

    # copy tagdiff config file -- regardless of STAGE_COMPARE
    # (cannot later overwrite possibly user-edited file)
    if [ -e "$TAGDIFF_CONF" ]; then
	cp -p $TAGDIFF_CONF $workroot
    fi
fi

# init summary (harmless parts)
todo_list=""
incomplete_list=""
match_list=""
mismatch_list=""
error_list=""

#---------------------------------------------------------------------
# main work
#---------------------------------------------------------------------
SUMMARY_INITIALIZED=""
for testname in $tests
do
    test -d $workroot/$testname || {
	echo Warning: $ENODIR "\"$testname\"; test skipped." 1>&2
	todo_list="$todo_list $testname"
	continue
    }

    safecd $workroot/$testname

    test -n "$show_tests" && echo_n "${testname}:	"

    #------------------------------------------------------------
    # Prepare
    #------------------------------------------------------------
    if [ -n "$STAGE_PREPARE" ]; then
	test -n "$verbose" && echo_n " preparing .."

	rm -f $APP_STDOUT  $APP_STDERR  \
		$APP_TAGDATA  $TAGDIFF_LOG ${STATUS_PREFIX}* 2> /dev/null
	gunzip -f _dummy_arg *.gz 2> /dev/null

	if [ ! -e "$APP_INPUT" ] && [ ! -e "$TEST_SCRIPT" ]; then
	    echo "Error: Input file \"$APP_INPUT\" missing"
	    touch "${STATUS_PREFIX}${STATUS_ERROR}"
	    continue
	fi
    fi

    #------------------------------------------------------------
    # Run - yeah baby!
    #------------------------------------------------------------
    if [ -n "$STAGE_RUN" ]; then
        rm -f $APP_STDOUT $APP_STDERR
        touch $APP_STDOUT $APP_STDERR
	test -n "$verbose" && echo_n " running .."

	if [ -e "$TEST_SCRIPT" ]; then
	    $TIME_CMD ./$TEST_SCRIPT $app_prerun $app_cmd $app_postrun >> $APP_STDOUT 2>> $APP_STDERR
	else
	    $TIME_CMD $app_prerun $app_cmd $app_postrun  >> $APP_STDOUT 2>> $APP_STDERR
	fi

	if [ -z "$STAGE_COMPARE" ]; then
	    test -n "$verbose" && echo_n " done."
	fi
    fi

    #------------------------------------------------------------
    # Compare
    #------------------------------------------------------------
    if [ -n "$STAGE_COMPARE" ]; then
	test -n "$verbose" && echo_n " comparing .."

	tagstatus=TODO
	if [ -f "$APP_TAGDATA" ]; then
	    rm -f ${STATUS_PREFIX}* 2> /dev/null

	    # gather tagdiff config files
	    tagdiffopt="-c $workroot/$TAGDIFF_CONF"
	    if [ -e "./$TAGDIFF_CONF" ]; then
		tagdiffopt="-c ./$TAGDIFF_CONF ${tagdiffopt}"
	    fi

	    $tagdiff_cmd $tagdiffopt \
		  $templateroot/$testname/$APP_TAGDATA_REF $APP_TAGDATA \
		      > $TAGDIFF_LOG 2>&1

	    # analyse tagdiff output and record status
	    case $? in
	      0)    case "`grep -c -v \"$TAGDIFF_MATCH_OK\" \
					$TAGDIFF_LOG 2> /dev/null`"
		    in
			0)  tagstatus=$STATUS_MATCH ;;
			*)  tagstatus=$STATUS_MISMATCH ;;
		    esac

		    # attempt to soften mismatch
		    case "`grep -c \"$TAGDIFF_MATCH_INCOMPLETE\" \
		    			$TAGDIFF_LOG 2> /dev/null`"
		    in
			0)  ;;
			*)  tagstatus=$STATUS_INCOMPLETE ;;
		    esac
		  ;;
	      *)	tagstatus=$STATUS_ERROR ;;
	    esac
	    touch ${STATUS_PREFIX}${tagstatus}
	fi
	test -n "$show_tests" && echo_n " $tagstatus."
    fi

    #------------------------------------------------------------
    # Gather summary data
    #------------------------------------------------------------
    if [ -n "$STAGE_SUMMARY" ]; then
	if [ -z "$STAGE_COMPARE" ]; then
	    # This makes for lengthy lines and may obscure $tagstatus.
	    # It's fairly fast, so we might as well skip the announcement.
	    test -n "$verbose" && echo_n " gathering .."
	fi

	if [ -z "$SUMMARY_INITIALIZED" ]; then
	    SUMMARY_INITIALIZED=1

	    INIT_OWD=`pwd`
	    safecd $workroot
	    rm -f $SUMMARY_STDERROR $SUMMARY_TAGDIFF_LOG 2> /dev/null
	    exec 3> $SUMMARY_STDERROR
	    exec 4> $SUMMARY_TAGDIFF_LOG
	    cd $INIT_OWD
	fi

	if [ ! -f $APP_TAGDATA ]; then
	    todo_list="$todo_list $testname"
	else
	    if   [ -f ${STATUS_PREFIX}${STATUS_INCOMPLETE} ]; then
		incomplete_list="$incomplete_list $testname"
	    elif [ -f ${STATUS_PREFIX}${STATUS_MATCH} ]; then
		match_list="$match_list $testname"
	    elif [ -f ${STATUS_PREFIX}${STATUS_MISMATCH} ]; then
		mismatch_list="$mismatch_list $testname"
	    #elif [ -f ${STATUS_PREFIX}${STATUS_ERROR} ]; then
	    else
		error_list="$error_list $testname"	# missing status?
	    fi

	    echo "======= $testname ======="	1>&3
	    cat $APP_STDERR			1>&3 2>/dev/null
	    echo ""				1>&3

	    echo "======= $testname ======="	1>&4
	    cat $TAGDIFF_LOG			1>&4 2>/dev/null
	    echo ""				1>&4
	fi
    fi
    test -n "$show_tests" && echo ""

    cd $OWD
done

#------------------------------------------------------------
# Print summary
#------------------------------------------------------------
if [ -n "$STAGE_SUMMARY" ]; then
    problems=""
    test -n "$show_tests" && echo ""
    echo "$TABLE_HEADER"
    echo "TEST SUMMARY"
    echo "$TABLE_SEP"

	echo "Match:";
	echo "${match_list:-$LIST_NONE}"      | xargs -n1 | pr -t2 -o4

    if [ -n "$mismatch_list" ]; then
	problems="$problems $STATUS_MISMATCH"
	echo ""
	echo "Mismatch:";
	echo "${mismatch_list:-$LIST_NONE}"   | xargs -n1 | pr -t2 -o4
    fi

    if [ -n "$incomplete_list" ]; then
	problems="$problems $STATUS_INCOMPLETE"
	echo ""
	echo "Incomplete:";
	echo "${incomplete_list:-$LIST_NONE}" | xargs -n1 | pr -t2 -o4
    fi

    if [ -n "$error_list" ]; then
	problems="$problems $STATUS_ERROR"
	echo ""
	echo "Errors:";
	echo "${error_list:-$LIST_NONE}"      | xargs -n1 | pr -t2 -o4
    fi

    if [ -n "$todo_list" ]; then
	problems="$problems $STATUS_TODO"
	echo ""
	echo "Not run:"
	echo "${todo_list:-$LIST_NONE}"       | xargs -n1 | pr -t2 -o4
    fi

    echo "$TABLE_SEP"
    exit_status=1
    if [ -z "$problems" ]; then
	exit_status=0
	echo "Status: OK"
    else
	echo "Status: FAIL"
	#echo "Problems:$problems"
    fi
    echo "$TABLE_SEP"
    echo Details in:
    echo  $workroot/$SUMMARY_STDERROR
    echo  $workroot/$SUMMARY_TAGDIFF_LOG
    echo "$TABLE_HEADER"
    exit $exit_status
fi

# -*- Mode: shell -*-
