Pass source code location to oracles

tested/oracles/common.py

@@ -37,16 +37,22 @@ def evaluate_text(configs, channel, actual):
 from tested.languages.utils import convert_stacktrace_to_clickable_feedback
 from tested.parsing import fallback_field, get_converter
 from tested.serialisation import Value
-from tested.testsuite import ExceptionOutputChannel, NormalOutputChannel, OutputChannel
+from tested.testsuite import (
+    ExceptionOutputChannel,
+    NormalOutputChannel,
+    OutputChannel,
+    SupportedLanguage,
+)
 
 
 @define
 class OracleContext:
     expected: Value
     actual: Value
-    execution_directory: str
-    evaluation_directory: str
-    programming_language: str
+    execution_directory: Path
+    evaluation_directory: Path
+    submission_path: Path
+    programming_language: SupportedLanguage
     natural_language: str
 
 

tested/oracles/programmed.py

@@ -32,12 +32,18 @@
 
 @define
 class ConvertedOracleContext:
+    """
+    This is the oracle context that is passed to the actual function.
+    It should thus remain backwards compatible.
+    """
+
     expected: Any
     actual: Any
     execution_directory: str
     evaluation_directory: str
     programming_language: str
     natural_language: str
+    submission_path: str
 
     @staticmethod
     def from_context(
@@ -46,10 +52,11 @@ def from_context(
         return ConvertedOracleContext(
             expected=eval(generate_statement(bundle, context.expected)),
             actual=eval(generate_statement(bundle, context.actual)),
-            execution_directory=context.execution_directory,
-            evaluation_directory=context.evaluation_directory,
+            execution_directory=str(context.execution_directory.absolute()),
+            evaluation_directory=str(context.evaluation_directory.absolute()),
             programming_language=context.programming_language,
             natural_language=context.natural_language,
+            submission_path=str(context.submission_path.absolute()),
         )
 
 
@@ -237,10 +244,11 @@ def evaluate(
     context = OracleContext(
         expected=expected,
         actual=actual,
-        execution_directory=str(config.context_dir.absolute()),
-        evaluation_directory=str(config.bundle.config.resources.absolute()),
-        programming_language=str(config.bundle.config.programming_language),
+        execution_directory=config.context_dir,
+        evaluation_directory=config.bundle.config.resources,
+        programming_language=config.bundle.config.programming_language,
         natural_language=config.bundle.config.natural_language,
+        submission_path=config.bundle.config.source,
     )
     result = _evaluate_programmed(config.bundle, channel.oracle, context)
 

tests/exercises/lotto/evaluation/evaluator.py

@@ -1,4 +1,5 @@
 import re
+import ast
 # noinspection PyUnresolvedReferences
 from evaluation_utils import EvaluationResult, Message
 
@@ -48,3 +49,33 @@ def evaluate(context, count, maximum):
     if valid:
         expected = actual
     return EvaluationResult(valid, expected, actual, messages)
+
+
+def check_for_node(search, context, count, maximum):
+    assert context.programming_language == "python", "This exercise only supports Python"
+    # Check if the submission uses a while loop.
+    with open(context.submission_path, "r") as submission_file:
+        submission = submission_file.read()
+
+    # This has no error handling, so it is not ready for production.
+    nodes = ast.walk(ast.parse(submission))
+    has_while = any(isinstance(node, search) for node in nodes)
+    messages = []
+    if not has_while:
+        messages.append("Your code does not use a while loop, which is mandatory.")
+    eval_result = evaluate(context, count, maximum)
+
+    return EvaluationResult(
+        eval_result.result and has_while,
+        eval_result.readable_expected,
+        eval_result.readable_actual,
+        eval_result.messages + messages
+    )
+
+
+def check_for_while(context, count, maximum):
+    return check_for_node(ast.While, context, count, maximum)
+
+
+def check_for_for(context, count, maximum):
+    return check_for_node(ast.For, context, count, maximum)

tests/exercises/lotto/evaluation/one-programmed-analysis-correct.yaml

@@ -0,0 +1,9 @@
+- tab: "Feedback"
+  testcases:
+    - expression: "loterij(18, 172)"
+      return: !oracle
+        oracle: "custom_check"
+        file: "evaluator.py"
+        name: "check_for_while"
+        value: "7 - 37 - 48 - 54 - 70 - 78 - 81 - 90 - 102 - 103 - 113 - 119 - 120 - 137 - 140 - 154 - 157 - 159"
+        arguments: [18, 172]

tests/exercises/lotto/evaluation/one-programmed-analysis-wrong.yaml

@@ -0,0 +1,9 @@
+- tab: "Feedback"
+  testcases:
+    - expression: "loterij(18, 172)"
+      return: !oracle
+        oracle: "custom_check"
+        file: "evaluator.py"
+        name: "check_for_for"
+        value: "7 - 37 - 48 - 54 - 70 - 78 - 81 - 90 - 102 - 103 - 113 - 119 - 120 - 137 - 140 - 154 - 157 - 159"
+        arguments: [18, 172]

tests/test_oracles_programmed.py

@@ -130,3 +130,39 @@ def test_custom_check_function_lotto_wrong(
     assert len(updates.find_all("start-testcase")) == 1
     assert updates.find_status_enum() == ["wrong"]
     assert len(updates.find_all("append-message")) == 1
+
+
+def test_custom_check_function_static_analysis_correct(
+    tmp_path: Path, pytestconfig: pytest.Config
+):
+    conf = configuration(
+        pytestconfig,
+        "lotto",
+        "python",
+        tmp_path,
+        "one-programmed-analysis-correct.yaml",
+        "correct",
+    )
+    result = execute_config(conf)
+    updates = assert_valid_output(result, pytestconfig)
+    assert len(updates.find_all("start-testcase")) == 1
+    assert updates.find_status_enum() == ["correct"]
+    assert len(updates.find_all("append-message")) == 0
+
+
+def test_custom_check_function_static_analysis_wrong(
+    tmp_path: Path, pytestconfig: pytest.Config
+):
+    conf = configuration(
+        pytestconfig,
+        "lotto",
+        "python",
+        tmp_path,
+        "one-programmed-analysis-wrong.yaml",
+        "correct",
+    )
+    result = execute_config(conf)
+    updates = assert_valid_output(result, pytestconfig)
+    assert len(updates.find_all("start-testcase")) == 1
+    assert updates.find_status_enum() == ["wrong"]
+    assert len(updates.find_all("append-message")) == 1