run_specs

#!/usr/bin/python
#
# run_specs
#
# Run compiler unit tests
#
# Copyright (c) 2008,2009,2013 Matthias Kramm <kramm@quiss.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

import sys
import os
import re
import time
import subprocess
import marshal
import select
from optparse import OptionParser

class Command:
    def __init__(self, cmd, args):
        self.cmd = cmd
        self.args = args

cmd_run_unsafe = Command("spec/run", ["-u"])
cmd_run_sandbox = Command("spec/run", [])
cmds = [cmd_run_unsafe, cmd_run_sandbox]

EXTENSIONS=[".py", ".rb", ".js", ".lua"]

NON_PRINTABLE = re.compile("((?=[\x00-\x1f])[^\n\r\t])|[\x7f-\xff]")

def check(s):
    row = None
    ok = 0
    for line in s.split("\n"):
        if line.startswith("[") and line.endswith("]"):
            continue
        if not line.strip():
            continue
        if not line.startswith("ok"):
            return 0
        if line.startswith("ok "):
            if "/" not in line:
                return 0
            i = line.index('/')
            try:
                nr,l = int(line[3:i]),int(line[i+1:])
            except ValueError:
                return 0
            if nr<1 or nr>l:
                return 0
            if not row:
                row = [0]*l
            elif l != len(row):
                return 0
            if row[nr-1]:
                return 0
            row[nr-1] = 1
        elif line == "ok":
            ok = 1
    if ok:
        return not row
    if row:
        return 0 not in row
    return 0

def runcmd(cmd, args, wait):
    #fo = os.tmpfile()
    fi,fo = os.pipe()
    fo = os.fdopen(fo, "wb")
    p = subprocess.Popen([cmd] + args, executable=cmd, stdout=fo, stderr=fo)
    ret = -1
    output = ""
    for i in range(wait*10):
        if fi in select.select([fi],[],[], 0.01)[0]:
            output += os.read(fi, 8192)
            if "[exit]" in output:
                break
            if "==    by" in output:
                ret = -33
                break
        ret = p.poll()
        if ret is not None:
            if cmd == "valgrind":
                # valgrind never returns true
                ret = 0
            break
        time.sleep(0.1)
    else:
        os.kill(p.pid, 9)
        os.system("killall -9 %s >/dev/null 2>/dev/null" % cmd)
        ret = -1
    fo.close()
   
    if fi in select.select([fi],[],[], 0.01)[0]:
        output += os.read(fi, 8192)

    os.close(fi)
    return CommandResult(ret, output)

class CommandResult:
    def __init__(self, ret, output):
        self.ret = ret
        self.output = output
        self.compile_error = 0
        self.compile_output = output
        self.exit_status = 0
        if ret:
            self.compile_output += "\nExit status %d" % (-ret)
            self.exit_status = -ret
            self.compile_error = 1
            self.success = False
        else:
            self.success = True

class Cache:
    def __init__(self, filename):
        self.filename = filename
        self.filename_milestone = filename+"_milestone"
        try:
            self.filename2status = marshal.load(open(self.filename, "rb"))
        except IOError:
            self.filename2status = {}
        try:
            self.milestone = marshal.load(open(self.filename_milestone, "rb"))
        except IOError:
            self.milestone = {}

    def parse_args(self):
        parser = OptionParser()
        parser.add_option("-d", "--diff", dest="diff", help="Only run tests that failed the last time",action="store_true")
        parser.add_option("-a", "--all", dest="all", help="Run all tests (also tests expected to fail)",action="store_true")
        parser.add_option("-t", "--tag", dest="tag", help="Mark the current pass/fail statistic as milestone",action="store_true")
        parser.add_option("-m", "--valgrind", dest="valgrind", help="Run compiler through valgrind",action="store_true")
        (options, args) = parser.parse_args()

        if args and args[0]=="add":
            self.all = 1
            self.tag = 1
            self.milestone[args[1]] = "ok"
            self.filename2status = self.milestone
            self.save()
            sys.exit(0)

        self.__dict__.update(options.__dict__)
        self.runtime = 1
        if self.tag: 
            self.all = 1
            self.runtime = 3 # allow more time if we're tagging this state
        
        if self.valgrind:
            global CMD,CMD_ARGS
            CMD_ARGS = [CMD] + CMD_ARGS
            CMD = "valgrind"
            self.runtime = 20 # allow even more time for valgrind

        self.checknum=-1
        self.checkfile=None
        if len(args):
            try:
                self.checknum = int(args[0])
            except ValueError:
                self.checkfile = args[0]

    @staticmethod
    def load(filename):
        return Cache(filename)

    def save(self):
        fi = open(self.filename, "wb")
        marshal.dump(self.filename2status, fi)
        fi.close()
        if self.tag:
            assert(self.all)
            fi = open(self.filename_milestone, "wb")
            marshal.dump(self.filename2status, fi)
            fi.close()

    def highlight(self, nr, filename):
        if self.checkfile and filename==self.checkfile:
            return 1
        return self.checknum==nr

    def skip_file(self, nr, filename):
        if self.checknum>=0 and nr!=self.checknum:
            return 1
        if self.checkfile and filename!=self.checkfile:
            return 1
        if not self.all and self.milestone.get(filename,"new")!="ok":
            return 1
        if self.diff and self.filename2status(filename,"new")=="ok":
            return 1
        return 0

    def file_status(self, filename, status):
        self.filename2status[filename] = status

class TestBase:
    def __init__(self, cache, nr, file, run):
        self.cache = cache
        self.nr = nr
        self.dorun = run
        self.file = file
        self.compile_output = None
        self.compile_error = None
        self.status = []

    def compile(self, cmd):
        command_status = runcmd(cmd.cmd, cmd.args+[self.file], wait=cache.runtime)
        self.status.append(command_status)
        return command_status.success

    def doprint(self):
        print self.r(str(self.nr),3)," ",
        for status in self.status:
            if status.exit_status == 11:
                print "crash",
            elif status.exit_status == 9:
                print "deny ",
            elif not status.success:
                print "err%2d" % -status.exit_status,
            else:
                print "ok   ",
        for _ in range(len(self.status), len(cmds)):
            print " -  ",
        print " ",
        print self.file

    def doprintlong(self):
        print self.nr, self.file
        print "================================"
        status = self.status[-1]
        print "compile:", (status.compile_error and "error" or "ok")
        print NON_PRINTABLE.sub("?", status.compile_output)

    def r(self,s,l):
        if(len(s)>=l):
            return s
        return (" "*(l-len(s))) + s
    def l(self,s,l):
        if(len(s)>=l):
            return s
        return s + (" "*(l-len(s)))

class Test(TestBase):
    def __init__(self, cache, nr, file):
        TestBase.__init__(self, cache, nr, file, run=1)
        ok = True
        for cmd in cmds:
            if not self.compile(cmd):
                ok = False
                break
        if ok:
            cache.file_status(file, "ok")
        else:
            cache.file_status(file, "error")

class ErrTest(TestBase):
    def __init__(self, cache, nr, file):
        TestBase.__init__(self, cache, nr, file, run=0)
        ok = True
        for cmd in cmds:
            if not self.compile(cmd):
                ok = False
                break
        if ok:
            cache.file_status(file, "error")
            self.compile_error = True
        else:
            cache.file_status(file, "ok")
            self.compile_error = False

class Suite:
    def __init__(self, cache, dir, should_fail=False):
        self.dir = dir
        self.cache = cache
        self.errtest = should_fail
    def run(self, nr):
        if not os.path.isdir(self.dir):
            return
        print "-"*40,"directory \""+self.dir+"\"","-"*40
        for file in sorted(os.listdir(self.dir)):
            if not os.path.splitext(file)[1] in EXTENSIONS:
                continue
            nr = nr + 1
            file = os.path.join(self.dir, file)

            if cache.skip_file(nr, file):
                continue

            if self.errtest:
                test = ErrTest(cache, nr, file)
            else:
                test = Test(cache, nr, file)

            if not cache.highlight(nr, file):
                test.doprint()
            else:
                test.doprintlong()
        return nr

cache = Cache.load(".tests.cache")
cache.parse_args()

nr = 0
nr = Suite(cache, "spec").run(nr)
nr = Suite(cache, "spec/err", should_fail=True).run(nr)

cache.save()