testmistify

#!/bin/bash
#+
# Use this script to run verification tests against Mistify-OS.
#
#-

source scripts/mistify-functions.sh
source test/scripts/mistify-test-defaults.sh

usage () {
    cat << EOF
Usage: $0 [options] [-- test suite options]
  Use this script to execute tests to verify the build of an instance of the
  Mistify-OS and its execution either in a VM or on a target platform.

  NOTE: This script uses the Robot Framework (http://robotframework.org)
  utility, pybot, to execute test cases.

  Options:
    ==== System Under Test ====
    --builddir <dir>
        The path to the directory where Mistify-OS was built (see buildmistify).
        This option is passed to the test suites on the command line in the
        variable BUILDDIR and defaults to the most recent build.
        This option is saved in the file:
        $testmistifystatedir/builddir
        [builddir=$builddirdefault]
    --buildrootversion <version>
        The version of Buildroot to use and can be a branch, tag or a commit ID.
        This option is passed to the test suites on the command line in the
        variable BUILDROOTVERSION and defaults to the version used for the most
        recent build (see buildmistify). This option is saved in the file:
        $testmistifystatedir/buildrootversion
        [buildrootversion=$buildrootversiondefault]
    --toolchainversion <version>
        The version of the toolchain to use and can be a branch, tag or a commit
        ID. This option is passed to the test suites on the command line in the
        variable TOOLCHAINVERSION and defaults to the version used for the most
        recent build (see buildmistify). This option is saved in the file:
        $testmistifystatedir/toolchainversion
        [toolchainversion=$toolchainversiondefault]
    --gotag <tag>
        The tag to use when fetching the GO source code from the repository.
        This is saved in the file $testmistifystatedir/gotag.
        [gotag=$gotagdefault]
    ==== Test Environment ====
    --testlibdir <dir>
        Where the test case and test suite library resides. This option is saved
        in the file:
        $testmistifystatedir/testlibdir
        [testlibdir=$testlibdirdefault]
    --testbeddir <dir>
        Where the test bed definitions are stored. This option is saved in the
        file: $testmistifystatedir/testbeddir
        [testbeddir=$testbeddirdefault]
    --testbed <name>
        The environment in which to test Mistify-OS. This can be a hardware
        platform or a virtual machine. This option is saved in the file:
        $testmistifystatedir/testbed
        [testbed=$testbeddefault]
    ==== Build Testing Options ====
    --testdistrodir <dir>
        Mistify-OS build verification is performed using a Linux Container (lxc)
        constructed specifically for the test. The container can be based upon
        one of a set of possible Linux distributions which are defined in distro
        definition files. This defines the location of the distro definition file
        to use for this test run. This option is saved in the file:
        $testmistifystatedir/testdistrodir
        [testdistrodir=$testdistrodirdefault]
    --testdistro <distro>
        This is the distro definition file to use for a test build run. This
        option is saved in the file:
        $testmistifystatedir/testdistro
        [testdistro=$testdistrodefault]
    ==== Test Execution ====
    --testsuitedir <dir>
        The directory where the test suites reside. This option is saved in the
        file: $testmistifystatedir/testsuitedir
        [testsuitedir=$testsuitedirdefault]
    --testsuite <testsuite>
        The test suite to execute. A test suite is comprised of a number of
        symlinks in a directory. The symlinks are expected to point to testcases
        typically in the test case directory. This option is saved in the file:
        $testmistifystatedir/testsuite.
        [testsuite=$testsuitedefault]
    --testcase <testcase>
        Execute only a single test case. This
        The "testsuite" option is ignored when this option is used.
        This option is NOT saved.
    --exitonfailure
        This option is used to stop test execution on the first failure. Test
        teardown is still executed. This option is NOT saved.
    --tag <tag>
        Run only test cases having this tag. This option is not sticky.
    --testlogdir <dir>
        Where to put the test log.
        This option is saved in the file:
        $testmistifystatedir/testlogdir
        [testlogdir=$testlogdirdefault]
    -- <pybotoptions>
        Anything following the "--" is passed directly to Robot Framework (pybot).
        Most often this is used to pass variables on the command line which are
        specific to a given test. e.g. "-v VAR:varvalue"
    ==== other ====
    --resetdefaults
        Reset options back to their default values.
    --verbose
        Verbose output from this script.
    --dryrun
        Just showing what will happen with this script. Don't run the test.
    --alwaysok
        Return 0 always. Some CI tools have a Robot Framework plugin
        which requires that a 0 return code always be returned after executing
        tests whether they fail or not. This is so the CI tool won't stop the
        run before the plugin has an opportunity to process the test results.
    -h|--help
        Display this usage.

    ==== Variables Passed On the Command Line ====
    PWD = The current directory.
    USER = The current user as indicated in the \$USER environment variable.
    HOME = The current user's home directory as indicated in the \$HOME
        environment variable.
    MISTIFYBRANCH = The active Mistify-OS branch as reported by git.
    BUILDROOTVERSION = The version of buildroot to pass to the test suites. This
        is used to modify test case behavior based upon the version of Buildroot.
    TOOLCHAINVERSION = The version of the toolchain to pass to the test suites.
        This is used to modify test case behavior based upon the version of the
        toolchain used to build Mistify-OS.
    GOTAG = The version of Go to buld. This is used to modify test case behavior
        based upon the version of Go.
    BUILDDIR = The directory in which Mistify-OS was built. Test cases can use
        this to test against different builds of the Mistify-OS.
    TESTDISTRO = The Linux distribution used for container based testing. The
        test container is initialized using this distribution (see mistify.robot).
    TESTBED = The path to the file which describes the environment in which to
        test.
    These can be overridden on the command line using the "-v" option mentioned
    next.

    ==== Test Suite Options ====
        Any options following "--" on the commnand line are passed to pybot.
        Read the Robot Framework documentation for a list of available options.
        Of particular interest is the "-v" option which can be used to pass
        variables to test cases.

  NOTE: This script maintains state in:
  $testmistifystatedir.
EOF
}

#+
# Handle the command line options.
#-
a=`getopt -l "\
builddir:,\
buildrootversion:,\
toolchainversion:,\
gotag:,\
testlibdir:,\
testbeddir:,\
testbed:,\
testdistrodir:,\
testdistro:,\
testsuitedir:,\
testsuite:,\
testcase:,\
exitonfailure,\
tag:,\
testlogdir:,\
resetdefaults,\
verbose,\
dryrun,\
alwaysok,\
help" \
   -o "h" -- "$@"`

if [ $? -gt 0 ]; then
    usage
    exit 1
fi

eval set -- $a

while [ $# -ge 1 ]; do
    case "$1" in
	--)
	    shift
	    pybotoptions=$*
	    break
	    ;;
	--builddir)
	    builddir=$2
	    shift
	    ;;
	--buildrootversion)
	    buildrootversion=$2
	    shift
	    ;;
	--toolchainversion)
	    toolchainversion=$2
	    shift
	    ;;
	--gotag)
	    gotag=$2
	    shift
	    ;;
	--testlibdir)
	    testlibdir=$2
	    shift
	    ;;
	--testbeddir)
	    testbeddir=$2
	    shift
	    ;;
	--testbed)
	    testbed=$2
	    shift
	    ;;
	--testdistrodir)
	    testdistrodir=$2
	    shift
	    ;;
	--testdistro)
	    testdistro=$2
	    shift
	    ;;
	--testsuite)
	    testsuite=$2
	    shift
	    ;;
	--testsuitedir)
	    testsuitedir=$2
	    shift
	    ;;
	--testcase)
	    testcase=$2
	    shift
	    ;;
	--exitonfailure)
	    exitonfailure=y
	    ;;
	--tag)
	    tag=$2
	    shift
	    ;;
	--testlogdir)
	    testlogdir=$2
	    shift
	    ;;
	--resetdefaults)
	    resetdefaults=y
	    ;;
	--verbose)
	    verbose=y
	    ;;
	--dryrun)
	    dryrun=echo
	    ;;
	--alwaysok)
	    alwaysok=y
	    ;;
	-h|--help)
	    showusage=y
	    ;;
	# using getopt should avoid needing this catchall but just in case...
	*)
	    error "Invalid option: $1"
	    usage
	    exit 1
	    ;;
    esac
    shift
done

if [ ! -z "$resetdefaults" ]; then
    reset_test_default builddir
    reset_test_default buildrootversion
    reset_test_default toolchainversion
    reset_test_default gotag
    reset_test_default testlibdir
    reset_test_default testbeddir
    reset_test_default testbed
    reset_test_default testdistrodir
    reset_test_default testdistro
    reset_test_default testsuitedir
    reset_test_default testsuite
    reset_test_default testlogdir
fi

builddirdefault=$(get_test_default builddir \
			$(get_build_default variantbuilddir build/base))
buildrootversiondefault=$(get_test_default buildrootversion \
			$(get_build_default buildrootversion master))
toolchainversiondefault=$(get_test_default toolchainversion \
			$(get_build_default toolchainversion crosstool-ng-1.21.0))
gotagdefault=$(get_test_default gotag \
			$(get_build_default gotag master))

testlibdirdefault=$(get_test_default testlibdir test)

testbeddirdefault=$(get_test_default testbeddir $testlibdirdefault/testbeds)
testbeddefault=$(get_test_default testbed labrat.robot)

testdistrodirdefault=$(get_test_default testdistrodir $testlibdirdefault/distros)
testdistrodefault=$(get_test_default testdistro ubuntu-trusty-amd64.robot)

testsuitedirdefault=$(get_test_default testsuitedir $testlibdirdefault/testsuites)
testsuitedefault=$(get_test_default testsuite defaulttests)

testlogdirdefault=$(get_test_default testlogdir $builddirdefault/testlogs)

#####
#+
# TODO: It's a real possibility that a user will want to run a test against
# a running system which doesn't have a corresponding build (at least locally).
# This means that user will want to skip all this builddir and variant stuff.
# However, that can only be a subset of the available tests. Obviously, tests
# which verify the build itself can't be run.
# This creates an orginization problem for test cases. It's likely TAGs will
# need to be employeed to specify the class of test cases to run.
# This raises other questions such as where to put test logs if no build.
# I don't want to take the time to think this through at the moment.
#
# For now one must build before one can test!!
#-
if [ -z "$builddir" ]; then
    if [ -d "$builddirdefault" ]; then
	builddir=$builddirdefault
    fi
fi

if [ -d "$builddir" ]; then
    echo $builddir >$testmistifystatedir/builddir
    message "Testing the build in $builddir"
else
    warning "The build directory $builddir does not exist."
    message "Defaulting to: $PWD/build"
    builddir=$PWD/build
    mkdir -p $builddir
fi

#####
if [ -z "$buildrootversion" ]; then
    buildrootversion=$buildrootversiondefault
fi
verbose Setting BUILDROOTVERSION to $buildrootversion

#####
if [ -z "$toolchainversion" ]; then
    toolchainversion=$toolchainversiondefault
fi
verbose Setting TOOLCHAINVERSION to $toolchainversion

#####
if [ -z "$gotag" ]; then
    gotag=$gotagdefault
fi
verbose Setting GOTAG to $gotag

#####

if [ -z "$testlibdir" ]; then
    testlibdir=$testlibdirdefault
fi

if [ -d "$testlibdir" ]; then
    echo $testlibdir >$testmistifystatedir/testlibdir
    verbose "Using test library located in: $testlibdir"
else
    die "Test library $testlibdir does not exist"
fi

#####
if [ -z "$testbeddir" ]; then
    testbeddir=$testbeddirdefault
fi

if [ -d "$testbeddir" ]; then
    echo $testbeddir >$testmistifystatedir/testbeddir
    verbose "Using test bed directory in: $testbeddir"
else
    die "Test bed definition directory $testbeddir does not exist"
fi

#####
if [ -z "$testbed" ]; then
    testbed=$testbeddefault
fi

if [ -f "$testbeddir/$testbed" ]; then
    echo $testbed >$testmistifystatedir/testbed
    verbose "Using test bed defintion: $testbed"
else
    die "Test bed definition $testbed does not exist"
fi

#####
if [ -z "$testdistrodir" ]; then
    testdistrodir=$testdistrodirdefault
fi

if [ -d "$testdistrodir" ]; then
    verbose "Using test distro directory in: $testdistrodir"
else
    die "Test distro definition directory $testdistrodir does not exist"
fi

#####
if [ -z "$testdistro" ]; then
    testdistro=$testdistrodefault
fi

if [ -f "$testdistrodir/$testdistro" ]; then
    verbose "Using test distro definition: $testdistro"
else
    die "Test distro definition $testdistro does not exist"
fi

#####
if [ -z "$testsuitedir" ]; then
    testsuitedir=$testsuitedirdefault
fi

if [ -d "$testsuitedir" ]; then
    verbose "Using test suite directory in: $testsuitedir"
else
    die "Test suite definition directory $testsuitedir does not exist"
fi

#####
if [ -z "$testsuite" ]; then
    testsuite=$testsuitedefault
fi

if [ -d "$testsuitedir/$testsuite" ]; then
    tests=$testsuitedir/$testsuite
    verbose "Using test suite: $testsuitedir/$testsuite"
else
    die "Test suite $testsuitedir/$testsuite does not exist"
fi

#####
if [ -n "$testcase" ]; then
    if [ -e $testcase ]; then
        tests=$testlibdir/$testcase
        verbose "Running $testcase ($testsuite ignored)"
    else
        die "Test case $testcase does not exist"
    fi
fi

#####

if [ -z "$testlogdir" ]; then
    testlogdir=$testlogdirdefault
fi

#####

if [ ! -z "$showusage" ]; then
    usage
    exit 0
fi

verbose "Creating the testlog directory if it doesn't exist."
$dryrun mkdir -p $testlogdir
if [ $? -gt 0 ]; then
    die Could not create the test log directory.
fi

#####
verbose "Running test(s): $tests"

#+
# TODO: Verify Robot Framework is properly installed and if not install it.
# TODO: Also need to install in a virtual python environment.
#-

if [[ "$verbose" == "y" ]]; then
    verboseoption="-L DEBUG:INFO"
fi

if [ -n "$tag" ]; then
  pybotoptions+=" --include $tag"
fi

if [ -n "$exitonfailure" ]; then
  pybotoptions+=" --exitonfailure"
fi

mkdir -p $testlogdir
pybotcommand="pybot $verboseoption -d $testlogdir \
    -v PWD:`pwd` \
    -v MISTIFYBRANCH:$mistifybranch \
    -v BUILDROOTVERSION:$buildrootversion \
    -v TOOLCHAINVERSION:$toolchainversion \
    -v GOTAG:$gotag \
    -v BUILDDIR:$builddir \
    -v TESTDISTRO:$testdistrodir/$testdistro \
    -v TESTBED:$testbeddir/$testbed \
    -v USER:$USER \
    -v HOME:$HOME \
    $pybotoptions \
    $@ $tests"
verbose "Command is: $pybotcommand"

if [ -n "$dryrun" ]; then
  message "Just a dry run -- not running tests."
else
  verbose Saving current settings.
  set_test_default builddir $builddir
  set_test_default testlibdir $testlibdir
  set_test_default testdistrodir $testdistrodir
  set_test_default testdistro $testdistro
  set_test_default testbeddir $testbeddir
  set_test_default testbed $testbed
  set_test_default testsuitedir $testsuitedir
  set_test_default testsuite $testsuite
  set_test_default testlogdir $testlogdir
fi

message "Running test case(s): $target"
$dryrun $pybotcommand
if [ $? -gt 0 ]; then
  error "One or more test cases failed."
  if [ -n "$alwaysok" ]; then
        message Returning OK regardless of test results.
        exit 0
  fi
  exit 1
fi