#! /usr/bin/env python

# Author: Steven Smith based on code from : Chris White

import exceptions
import os
import subprocess
import sys
import argparse
import re

def main():
    parser = argparse.ArgumentParser(description='Sundials test runner for use in ctest.')
    parser.add_argument('--testname', dest='testName', type=str, required=True,
                        help='name of the test')
    parser.add_argument('--executablename', dest='executableName', type=str, required=True,
                        help='executable, including path')
    parser.add_argument('--answersdir', dest='answersDir', type=str, required=True,
                        help='path to answers directory')
    parser.add_argument('--answerfile', dest='answerFile', type=str, required=False,
                        help='answer file, filename only specify path using --answersdir')
    parser.add_argument('--outputdir', dest='outputDir', type=str, required=True,
                        help='path to output directory')

    parser.add_argument('--runcommand', dest='runCommand', type=str,
                        help='command used to run test executable, eg. mpirun')
    parser.add_argument('--runargs', dest='runArgs', type=str,
                        help='command line arguments passed to test executable')

    parser.add_argument('--floatprecision', dest='floatPrecision', type=float,
                        help='precision for floating point failure comparison')

    parser.add_argument('--integerpercentage', dest='integerPercentage', type=int,
                        help='integer percentage difference for failure comparison')

    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                        help='run verbosely')

    args = parser.parse_args()

    if(args.verbose):
        print "testName=%s"%(args.testName)
        print "executableName=%s"%(args.executableName)
        print "answersDir=%s"%(args.answersDir)
        print "answerFile=%s"%(args.answerFile)
        print "outputDir=%s"%(args.outputDir)
        print "runCommand=%s"%(args.runCommand)
        print "runArgs=%s"%(args.runArgs)

    testName = args.testName

    testDir = os.path.dirname(sys.argv[0])
    outDir = args.outputDir

    answersDir = os.path.join(testDir, "answers")
    
    if(args.verbose):
        print "testDir %s"%testDir
        print "outDir %s"%outDir
        print "answersDir %s"%answersDir

    if not os.path.exists(outDir):
        os.makedirs(outDir)
    elif not os.path.isdir(outDir):
        error("Output directory exists but is not a directory, it must be deleted.", outDir)
        sys.exit(1)

    testInfos = getInfo(testName, testDir)
    success = True
    for testInfo in testInfos:
        if(args.verbose):
            error("testInfo.arguments = ",testInfo.arguments)

        # If running under another program (e.g. mpirun)
        if(args.runCommand):
            testCommand = args.runCommand.replace("\"", "").split(" ")
        else:
            testCommand = []

        testCommand = testCommand + [args.executableName]

        if(args.runArgs):
            # Remove first and last quote
            userArgs=re.sub(r'^"|"$', '', args.runArgs)
            testCommand = testCommand +  userArgs.split(" ")
            answerFileName = testName
        else:
            if testInfo.arguments != "":
                testCommand = testCommand + testInfo.arguments.split(" ")
                answerFileName = testName + "_" + testInfo.arguments.replace(" ", "_")
            else:
                answerFileName = testName

        # if user supplies filename overide default choices
        if(args.answerFile):
            answerFileName = args.answerFile

        outPath = os.path.join(outDir, args.testName)

        answerPath = os.path.join(answersDir, answerFileName)

        # if user supplies precision info overide default choices
        if(args.floatPrecision):
            testInfo.floatPrecision=args.floatPrecision

        if(args.integerPercentage):
            testInfo.integerPercentage=args.integerPercentage
        
        if(args.verbose):
            print "Starting test..."
            print "  Floating Point Precision = " + str(testInfo.floatPrecision)
            print "  Integer Precision = " + str(testInfo.integerPercentage)
            print "  Command " + ' '.join(testCommand)
            print "  Answer File = " + answerPath
            print "  Output File = " + outPath + "\n"

        outFile = open(outPath, 'w')
        try:
            if(args.verbose):
                print "Running command : " + ' '.join(testCommand)
                        
            process = subprocess.Popen(testCommand, stdout=outFile, stderr=outFile)
            process.wait()
            if process.returncode != 0:
                error("Exited with error code " + str(process.returncode) + ".", testName)
                success = False
            if not os.path.exists(answerPath):
                error("Answer file did not exist.", answerPath)
                success = False
            if not os.path.isfile(answerPath):
                error("Answer file existed but was not a file.", answerPath)
                success = False
            if not compare(outPath, answerPath, testInfo):
                success = False
        finally:
            outFile.close()

    if not success:
        print "Test failed."
        sys.exit(1)
    print "Test passed."
    sys.exit(0)

def compare(outPath, answerPath, testInfo):
    #Note: This functions strips all lines of leading and trailing whitespace and blank lines.  Then it
    #      compares the output against the checked in answer file. It allows fuzziness in floating points
    #      and integers, as described in the testInfo file.
    outFile = open(outPath, 'r')
    try:
        outLines = stripLines(outFile.readlines())
    finally:
        outFile.close()
    answerFile = open(answerPath, 'r')
    try:
        answerLines = stripLines(answerFile.readlines())
    finally:
        answerFile.close()
    diffCount = 0

    if len(outLines) != len(answerLines):
        error("Line count is not equal (blank lines ignored):\n" + str(len(outLines)) + ":" + outPath + "\n" + str(len(answerLines)) + ":" + answerPath)
        return False

    for i, outLine in enumerate(outLines):
        answerLine = answerLines[i]
        if not compareLines(outLine, answerLine, testInfo):
            diffCount += 1
            print "Difference:\n  Output: " + outLine + "\n  Answer: " + answerLine + "\n\n"
    if diffCount != 0:
        error(str(diffCount) + " line differences found.", outPath)
        return False
    return True

def compareLines(outLine, answerLine, testInfo):
    outTokens = tokenizeLine(outLine)
    answerTokens = tokenizeLine(answerLine)
    if len(outTokens) != len(answerTokens):
        print "DEBUG: number of tokens differs"
        print answerTokens
        print "----------"
        print outTokens
        print "----------"
        return False
    for i, outValue in enumerate(outTokens):
        answerValue = answerTokens[i]
        outIsInt, outInt = isInt(outValue)
        if outIsInt:
            answerIsInt, answerInt = isInt(answerValue)
            if answerIsInt and not intCompare(answerInt, outInt, testInfo.integerPercentage):
                print "DEBUG: int"
                return False
        else:
            outIsFloat, outFloat = isFloat(outValue)
            if outIsFloat:
                answerIsFloat, answerFloat = isFloat(answerValue)
                if answerIsFloat and not floatCompare(answerFloat, outFloat, testInfo.floatPrecision):
                    print "DEBUG: float"
                    return False
            elif outValue != answerValue:
                print "DEBUG: str"
                print "outValue <%s>"%(outValue)
                print "answerValue <%s>"%(answerValue)
                return False
    return True

def isInt(value):
    try:
        return True, int(value)
    except exceptions.ValueError:
        return False, 0

def intCompare(answer, value, percentage):
    if answer == value:
        return True
    if percentage != 0:
        percentageOff = abs(int(100 - (float(value)/float(answer) * 100)))
        if percentageOff <= percentage:
            return True
    print "Error: Integer difference found:\n  Value = " + str(value) + ", Answer = " + str(answer) + ", allowance = " + str(percentage) + "%"
    return False

def isFloat(value):
    try:
        return True, float(value)
    except exceptions.ValueError:
        return False, 0

def floatCompare(answer, value, precision):
    if precision == 0:
        if answer == value:
            return True
    elif round(answer, precision) == round(value, precision):
        return True
    print "Error: Floating point difference found:\n Value = " + str(value) + ", Answer = " + str(answer) + ", allowance = " + str(precision) + " decimals"
    return False

def tokenizeLine(line):
    return tokenizeLine.pattern.split(line)

# Precompile a pattern since it will be used many times
tokenizeLine.pattern = re.compile(r';|,|\s+|=+|:|\|\(|\)')


def error(message, path=""):
    if path:
        print "Error: " + path + ": " + message
    else:
        print "Error: " + message

def stripLines(lines):
    strippedLines = []
    for line in lines:
        strippedLine = line.strip()
        if strippedLine != "":
            strippedLines.append(strippedLine)
    return strippedLines

class TestInfo(object):
    def __init__(self, floatPrecision, integerPercentage, arguments):
        self.floatPrecision = floatPrecision
        self.integerPercentage = integerPercentage
        self.arguments = arguments

def getInfo(testName, testDir):
    infoPath = os.path.join(testDir, "testInfo")
    loweredTestName = testName.lower()
    testInfos = []

    if os.path.exists(infoPath):
        infoFile = open(infoPath, 'r')
        try:
            lines = stripLines(infoFile.readlines())
        finally:
            infoFile.close()

        for line in lines:
            if line.startswith('#'):
                continue
            lineInfo = line.split(':', 3)
            if lineInfo[0].strip().lower() == loweredTestName:
                testInfos.append(TestInfo(int(lineInfo[1].strip()), int(lineInfo[2].strip()), lineInfo[3].strip()))

    if len(testInfos) == 0:
        #Note: If no info found in testInfo file then run test without arguments and do default comparisions
        testInfos.append(TestInfo(4,10,""))
    return testInfos

if __name__ == "__main__":
    main()
