commit inicial

GeovanaS · GeovanaS · commit 1231834dc87d · 2018-04-29T21:15:33.000Z
diff --git a/autograder.py b/autograder.py
@@ -0,0 +1,351 @@
+# autograder.py
+# -------------
+# Licensing Information:  You are free to use or extend these projects for
+# educational purposes provided that (1) you do not distribute or publish
+# solutions, (2) you retain this notice, and (3) you provide clear
+# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
+# 
+# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
+# The core projects and autograders were primarily created by John DeNero
+# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
+# Student side autograding was added by Brad Miller, Nick Hay, and
+# Pieter Abbeel (pabbeel@cs.berkeley.edu).
+
+
+# imports from python standard library
+import grading
+import imp
+import optparse
+import os
+import re
+import sys
+import projectParams
+import random
+random.seed(0)
+try: 
+    from pacman import GameState
+except:
+    pass
+
+# register arguments and set default values
+def readCommand(argv):
+    parser = optparse.OptionParser(description = 'Run public tests on student code')
+    parser.set_defaults(generateSolutions=False, edxOutput=False, muteOutput=False, printTestCase=False, noGraphics=False)
+    parser.add_option('--test-directory',
+                      dest = 'testRoot',
+                      default = 'test_cases',
+                      help = 'Root test directory which contains subdirectories corresponding to each question')
+    parser.add_option('--student-code',
+                      dest = 'studentCode',
+                      default = projectParams.STUDENT_CODE_DEFAULT,
+                      help = 'comma separated list of student code files')
+    parser.add_option('--code-directory',
+                    dest = 'codeRoot',
+                    default = "",
+                    help = 'Root directory containing the student and testClass code')
+    parser.add_option('--test-case-code',
+                      dest = 'testCaseCode',
+                      default = projectParams.PROJECT_TEST_CLASSES,
+                      help = 'class containing testClass classes for this project')
+    parser.add_option('--generate-solutions',
+                      dest = 'generateSolutions',
+                      action = 'store_true',
+                      help = 'Write solutions generated to .solution file')
+    parser.add_option('--edx-output',
+                    dest = 'edxOutput',
+                    action = 'store_true',
+                    help = 'Generate edX output files')
+    parser.add_option('--mute',
+                    dest = 'muteOutput',
+                    action = 'store_true',
+                    help = 'Mute output from executing tests')
+    parser.add_option('--print-tests', '-p',
+                    dest = 'printTestCase',
+                    action = 'store_true',
+                    help = 'Print each test case before running them.')
+    parser.add_option('--test', '-t',
+                      dest = 'runTest',
+                      default = None,
+                      help = 'Run one particular test.  Relative to test root.')
+    parser.add_option('--question', '-q',
+                    dest = 'gradeQuestion',
+                    default = None,
+                    help = 'Grade one particular question.')
+    parser.add_option('--no-graphics',
+                    dest = 'noGraphics',
+                    action = 'store_true',
+                    help = 'No graphics display for pacman games.')
+    (options, args) = parser.parse_args(argv)
+    return options
+
+
+# confirm we should author solution files
+def confirmGenerate():
+    print 'WARNING: this action will overwrite any solution files.'
+    print 'Are you sure you want to proceed? (yes/no)'
+    while True:
+        ans = sys.stdin.readline().strip()
+        if ans == 'yes':
+            break
+        elif ans == 'no':
+            sys.exit(0)
+        else:
+            print 'please answer either "yes" or "no"'
+
+
+# TODO: Fix this so that it tracebacks work correctly
+# Looking at source of the traceback module, presuming it works
+# the same as the intepreters, it uses co_filename.  This is,
+# however, a readonly attribute.
+def setModuleName(module, filename):
+    functionType = type(confirmGenerate)
+    classType = type(optparse.Option)
+
+    for i in dir(module):
+        o = getattr(module, i)
+        if hasattr(o, '__file__'): continue
+
+        if type(o) == functionType:
+            setattr(o, '__file__', filename)
+        elif type(o) == classType:
+            setattr(o, '__file__', filename)
+            # TODO: assign member __file__'s?
+        #print i, type(o)
+
+
+#from cStringIO import StringIO
+
+def loadModuleString(moduleSource):
+    # Below broken, imp doesn't believe its being passed a file:
+    #    ValueError: load_module arg#2 should be a file or None
+    #
+    #f = StringIO(moduleCodeDict[k])
+    #tmp = imp.load_module(k, f, k, (".py", "r", imp.PY_SOURCE))
+    tmp = imp.new_module(k)
+    exec moduleCodeDict[k] in tmp.__dict__
+    setModuleName(tmp, k)
+    return tmp
+
+import py_compile
+
+def loadModuleFile(moduleName, filePath):
+    with open(filePath, 'r') as f:
+        return imp.load_module(moduleName, f, "%s.py" % moduleName, (".py", "r", imp.PY_SOURCE))
+
+
+def readFile(path, root=""):
+    "Read file from disk at specified path and return as string"
+    with open(os.path.join(root, path), 'r') as handle:
+        return handle.read()
+
+
+#######################################################################
+# Error Hint Map
+#######################################################################
+
+# TODO: use these
+ERROR_HINT_MAP = {
+  'q1': {
+    "<type 'exceptions.IndexError'>": """
+      We noticed that your project threw an IndexError on q1.
+      While many things may cause this, it may have been from
+      assuming a certain number of successors from a state space
+      or assuming a certain number of actions available from a given
+      state. Try making your code more general (no hardcoded indices)
+      and submit again!
+    """
+  },
+  'q3': {
+      "<type 'exceptions.AttributeError'>": """
+        We noticed that your project threw an AttributeError on q3.
+        While many things may cause this, it may have been from assuming
+        a certain size or structure to the state space. For example, if you have
+        a line of code assuming that the state is (x, y) and we run your code
+        on a state space with (x, y, z), this error could be thrown. Try
+        making your code more general and submit again!
+
+    """
+  }
+}
+
+import pprint
+
+def splitStrings(d):
+    d2 = dict(d)
+    for k in d:
+        if k[0:2] == "__":
+            del d2[k]
+            continue
+        if d2[k].find("\n") >= 0:
+            d2[k] = d2[k].split("\n")
+    return d2
+
+
+def printTest(testDict, solutionDict):
+    pp = pprint.PrettyPrinter(indent=4)
+    print "Test case:"
+    for line in testDict["__raw_lines__"]:
+        print "   |", line
+    print "Solution:"
+    for line in solutionDict["__raw_lines__"]:
+        print "   |", line
+
+
+def runTest(testName, moduleDict, printTestCase=False, display=None):
+    import testParser
+    import testClasses
+    for module in moduleDict:
+        setattr(sys.modules[__name__], module, moduleDict[module])
+
+    testDict = testParser.TestParser(testName + ".test").parse()
+    solutionDict = testParser.TestParser(testName + ".solution").parse()
+    test_out_file = os.path.join('%s.test_output' % testName)
+    testDict['test_out_file'] = test_out_file
+    testClass = getattr(projectTestClasses, testDict['class'])
+
+    questionClass = getattr(testClasses, 'Question')
+    question = questionClass({'max_points': 0}, display)
+    testCase = testClass(question, testDict)
+
+    if printTestCase:
+        printTest(testDict, solutionDict)
+
+    # This is a fragile hack to create a stub grades object
+    grades = grading.Grades(projectParams.PROJECT_NAME, [(None,0)])
+    testCase.execute(grades, moduleDict, solutionDict)
+
+
+# returns all the tests you need to run in order to run question
+def getDepends(testParser, testRoot, question):
+    allDeps = [question]
+    questionDict = testParser.TestParser(os.path.join(testRoot, question, 'CONFIG')).parse()
+    if 'depends' in questionDict:
+        depends = questionDict['depends'].split()
+        for d in depends:
+            # run dependencies first
+            allDeps = getDepends(testParser, testRoot, d) + allDeps
+    return allDeps
+
+# get list of questions to grade
+def getTestSubdirs(testParser, testRoot, questionToGrade):
+    problemDict = testParser.TestParser(os.path.join(testRoot, 'CONFIG')).parse()
+    if questionToGrade != None:
+        questions = getDepends(testParser, testRoot, questionToGrade)
+        if len(questions) > 1:
+            print 'Note: due to dependencies, the following tests will be run: %s' % ' '.join(questions)
+        return questions
+    if 'order' in problemDict:
+        return problemDict['order'].split()
+    return sorted(os.listdir(testRoot))
+
+
+# evaluate student code
+def evaluate(generateSolutions, testRoot, moduleDict, exceptionMap=ERROR_HINT_MAP, edxOutput=False, muteOutput=False,
+            printTestCase=False, questionToGrade=None, display=None):
+    # imports of testbench code.  note that the testClasses import must follow
+    # the import of student code due to dependencies
+    import testParser
+    import testClasses
+    for module in moduleDict:
+        setattr(sys.modules[__name__], module, moduleDict[module])
+
+    questions = []
+    questionDicts = {}
+    test_subdirs = getTestSubdirs(testParser, testRoot, questionToGrade)
+    for q in test_subdirs:
+        subdir_path = os.path.join(testRoot, q)
+        if not os.path.isdir(subdir_path) or q[0] == '.':
+            continue
+
+        # create a question object
+        questionDict = testParser.TestParser(os.path.join(subdir_path, 'CONFIG')).parse()
+        questionClass = getattr(testClasses, questionDict['class'])
+        question = questionClass(questionDict, display)
+        questionDicts[q] = questionDict
+
+        # load test cases into question
+        tests = filter(lambda t: re.match('[^#~.].*\.test\Z', t), os.listdir(subdir_path))
+        tests = map(lambda t: re.match('(.*)\.test\Z', t).group(1), tests)
+        for t in sorted(tests):
+            test_file = os.path.join(subdir_path, '%s.test' % t)
+            solution_file = os.path.join(subdir_path, '%s.solution' % t)
+            test_out_file = os.path.join(subdir_path, '%s.test_output' % t)
+            testDict = testParser.TestParser(test_file).parse()
+            if testDict.get("disabled", "false").lower() == "true":
+                continue
+            testDict['test_out_file'] = test_out_file
+            testClass = getattr(projectTestClasses, testDict['class'])
+            testCase = testClass(question, testDict)
+            def makefun(testCase, solution_file):
+                if generateSolutions:
+                    # write solution file to disk
+                    return lambda grades: testCase.writeSolution(moduleDict, solution_file)
+                else:
+                    # read in solution dictionary and pass as an argument
+                    testDict = testParser.TestParser(test_file).parse()
+                    solutionDict = testParser.TestParser(solution_file).parse()
+                    if printTestCase:
+                        return lambda grades: printTest(testDict, solutionDict) or testCase.execute(grades, moduleDict, solutionDict)
+                    else:
+                        return lambda grades: testCase.execute(grades, moduleDict, solutionDict)
+            question.addTestCase(testCase, makefun(testCase, solution_file))
+
+        # Note extra function is necessary for scoping reasons
+        def makefun(question):
+            return lambda grades: question.execute(grades)
+        setattr(sys.modules[__name__], q, makefun(question))
+        questions.append((q, question.getMaxPoints()))
+
+    grades = grading.Grades(projectParams.PROJECT_NAME, questions, edxOutput=edxOutput, muteOutput=muteOutput)
+    if questionToGrade == None:
+        for q in questionDicts:
+            for prereq in questionDicts[q].get('depends', '').split():
+                grades.addPrereq(q, prereq)
+
+    grades.grade(sys.modules[__name__], bonusPic = projectParams.BONUS_PIC)
+    return grades.points
+
+
+
+def getDisplay(graphicsByDefault, options=None):
+    graphics = graphicsByDefault
+    if options is not None and options.noGraphics:
+        graphics = False
+    if graphics:
+        try:
+            import graphicsDisplay
+            return graphicsDisplay.PacmanGraphics(1, frameTime=.05)
+        except ImportError:
+            pass
+    import textDisplay
+    return textDisplay.NullGraphics()
+
+
+
+
+if __name__ == '__main__':
+    options = readCommand(sys.argv)
+    if options.generateSolutions:
+        confirmGenerate()
+    codePaths = options.studentCode.split(',')
+    # moduleCodeDict = {}
+    # for cp in codePaths:
+    #     moduleName = re.match('.*?([^/]*)\.py', cp).group(1)
+    #     moduleCodeDict[moduleName] = readFile(cp, root=options.codeRoot)
+    # moduleCodeDict['projectTestClasses'] = readFile(options.testCaseCode, root=options.codeRoot)
+    # moduleDict = loadModuleDict(moduleCodeDict)
+
+    moduleDict = {}
+    for cp in codePaths:
+        moduleName = re.match('.*?([^/]*)\.py', cp).group(1)
+        moduleDict[moduleName] = loadModuleFile(moduleName, os.path.join(options.codeRoot, cp))
+    moduleName = re.match('.*?([^/]*)\.py', options.testCaseCode).group(1)
+    moduleDict['projectTestClasses'] = loadModuleFile(moduleName, os.path.join(options.codeRoot, options.testCaseCode))
+
+
+    if options.runTest != None:
+        runTest(options.runTest, moduleDict, printTestCase=options.printTestCase, display=getDisplay(True, options))
+    else:
+        evaluate(options.generateSolutions, options.testRoot, moduleDict,
+            edxOutput=options.edxOutput, muteOutput=options.muteOutput, printTestCase=options.printTestCase,
+            questionToGrade=options.gradeQuestion, display=getDisplay(options.gradeQuestion!=None, options))
