Merge pull request #427 from dodona-edu/unified-return

Unify return values in DSL

tested/dsl/schema.json

@@ -269,10 +269,7 @@
           }
         },
         "return" : {
-          "description" : "Expected return value"
-        },
-        "return_raw" : {
-          "description" : "Value string to parse to the expected return value",
+          "description" : "Expected return value.",
           "$ref" : "#/$defs/advancedValueOutputChannel"
         },
         "stderr" : {
@@ -396,7 +393,8 @@
       ]
     },
     "advancedValueOutputChannel" : {
-      "oneOf" : [
+      "anyOf" : [
+        {},
         {
           "type" : "string",
           "description" : "A 'Python' value to parse and use as the expected type."

tested/dsl/schema_draft7.json

@@ -265,10 +265,7 @@
           }
         },
         "return" : {
-          "description" : "Expected return value"
-        },
-        "return_raw" : {
-          "description" : "Value string to parse to the expected return value",
+          "description" : "Expected return value",
           "$ref" : "#/definitions/advancedValueOutputChannel"
         },
         "stderr" : {
@@ -391,7 +388,8 @@
       ]
     },
     "advancedValueOutputChannel" : {
-      "oneOf" : [
+      "anyOf" : [
+        {},
         {
           "type" : "string",
           "description" : "A 'Python' value to parse and use as the expected type."

tested/dsl/translate_parser.py

@@ -86,26 +86,42 @@ class TestedType:
     type: str | AllTypes
 
 
-def custom_type_constructors(loader: yaml.Loader, node: yaml.Node):
-    tested_tag = node.tag[1:]
+@define
+class YamlValue:
+    value: Any
+
+
+def _parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any:
     if isinstance(node, yaml.MappingNode):
-        base_result = loader.construct_mapping(node)
+        result = loader.construct_mapping(node)
     elif isinstance(node, yaml.SequenceNode):
-        base_result = loader.construct_sequence(node)
+        result = loader.construct_sequence(node)
     else:
         assert isinstance(node, yaml.ScalarNode)
-        base_result = loader.construct_scalar(node)
+        result = loader.construct_scalar(node)
+    return result
+
+
+def _custom_type_constructors(loader: yaml.Loader, node: yaml.Node) -> TestedType:
+    tested_tag = node.tag[1:]
+    base_result = _parse_yaml_value(loader, node)
     return TestedType(type=tested_tag, value=base_result)
 
 
+def _yaml_value_constructor(loader: yaml.Loader, node: yaml.Node) -> YamlValue:
+    return YamlValue(value=_parse_yaml_value(loader, node))
+
+
 def _parse_yaml(yaml_stream: Union[str, TextIO]) -> YamlObject:
     """
     Parse a string or stream to YAML.
     """
     loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.CSafeLoader)
     for types in get_args(AllTypes):
         for actual_type in types:
-            yaml.add_constructor("!" + actual_type, custom_type_constructors, loader)
+            yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
+    yaml.add_constructor("!v", _yaml_value_constructor, loader)
+    yaml.add_constructor("!value", _yaml_value_constructor, loader)
     return yaml.load(yaml_stream, loader)
 
 
@@ -336,11 +352,23 @@ def _convert_text_output_channel(
 
 
 def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel:
-    if isinstance(stream, str):
+    if isinstance(stream, YamlValue):
+        # A normal yaml type tagged explicitly.
+        value = _convert_value(stream.value)
+        assert isinstance(value, Value)
+        return ValueOutputChannel(value=value)
+    if isinstance(stream, (int, float, bool, TestedType, list, set)):
+        # Simple values where no confusion is possible.
+        value = _convert_value(stream)
+        assert isinstance(value, Value)
+        return ValueOutputChannel(value=value)
+    elif isinstance(stream, str):
+        # A normal YAML string is considered a "Python" string.
         value = parse_string(stream, is_return=True)
         assert isinstance(value, Value)
         return ValueOutputChannel(value=value)
     else:
+        # We have an object, which means we have an output channel.
         assert isinstance(stream, dict)
         assert isinstance(stream["value"], str)
         value = parse_string(stream["value"], is_return=True)
@@ -362,18 +390,16 @@ def _validate_testcase_combinations(testcase: YamlDict):
         raise ValueError("A main call cannot contain an expression or a statement.")
     if "statement" in testcase and "expression" in testcase:
         raise ValueError("A statement and expression as input are mutually exclusive.")
-    if "statement" in testcase and ("return" in testcase or "return_raw" in testcase):
+    if "statement" in testcase and "return" in testcase:
         raise ValueError("A statement cannot have an expected return value.")
-    if "return" in testcase and "return_raw" in testcase:
-        raise ValueError("The outputs return and return_raw are mutually exclusive.")
 
 
 def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
     context = context.deepen_context(testcase)
 
     # This is backwards compatability to some extend.
     # TODO: remove this at some point.
-    if "statement" in testcase and ("return" in testcase or "return_raw" in testcase):
+    if "statement" in testcase and "return" in testcase:
         testcase["expression"] = testcase.pop("statement")
 
     _validate_testcase_combinations(testcase)
@@ -430,9 +456,6 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
     if (exit_code := testcase.get("exit_code")) is not None:
         output.exit_code = ExitCodeOutputChannel(value=cast(int, exit_code))
     if (result := testcase.get("return")) is not None:
-        assert not return_channel
-        output.result = ValueOutputChannel(value=_convert_value(result))
-    if (result := testcase.get("return_raw")) is not None:
         assert not return_channel
         output.result = _convert_advanced_value_output_channel(result)
 

tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'echo("input")'
-      return: "input"
+      return: !v "input"

tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'no_echo("input")'
-      return: "input"
+      return: !v "input"

tests/exercises/echo-function/evaluation/one-language-literals.yaml

@@ -9,4 +9,4 @@
         kotlin: "toString(1+1)"
         python: "submission.to_string(1+1)"
         csharp: "Submission.toString(1+1)"
-      return: "2"
+      return: !v "2"

tests/exercises/echo-function/evaluation/one-nested.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'echo(echo("input"))'
-      return: "input"
+      return: !v "input"

tests/exercises/global/evaluation/plan.yaml

@@ -1,4 +1,4 @@
 - tab: "Global variable"
   testcases:
     - expression: "GLOBAL_VAR"
-      return: "GLOBAL"
+      return: !v "GLOBAL"

tests/exercises/objects/evaluation/missing_key_types.yaml

@@ -1,4 +1,4 @@
 - tab: "Feedback"
   testcases:
     - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
-      return_raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
+      return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'

tests/exercises/objects/evaluation/no-test.yaml

@@ -1,11 +1,11 @@
 - tab: "Feedback"
   testcases:
     - statement: '{["a", "b"], ["c"]}'
-      return_raw: '{["a", "b"], ["a"]}'
+      return: '{["a", "b"], ["a"]}'
     - statement: 'x = {{"a"}: [int16(1)], {"b"}: [int16(0)]}'
     - statement: 'x = {{"a"}: [int32(1)], {"b"}: "a"}'
     - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
-      return_raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
+      return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
       files:
         - name: "a.txt"
           url: "a.txt"

tests/test_dsl_yaml.py

@@ -255,15 +255,15 @@ def test_statements():
     - statement: 'safe: Safe = Safe("Ignore whitespace")'
       stdout: "New safe"
     - expression: 'safe.content()'
-      return: "Ignore whitespace"
+      return: !v "Ignore whitespace"
   - testcases:
     - statement: 'safe: Safe = Safe(uint8(5))'
       stdout:
         data: "New safe"
         config:
           ignoreWhitespace: false
     - expression: 'safe.content()'
-      return_raw: 'uint8(5)'
+      return: 'uint8(5)'
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -378,25 +378,12 @@ def test_invalid_yaml():
     stderr: []
     testcases:
     - statement: 'data = () ()'
-      return_raw: '() {}'
+      return: '() {}'
     """
     with pytest.raises(Exception):
         translate_to_test_suite(yaml_str)
 
 
-def test_invalid_mutual_exclusive_return_yaml():
-    yaml_str = """
-- tab: "Tab"
-  contexts:
-  - testcases:
-    - statement: "5"
-      return: 5
-      return_raw: "5"
-    """
-    with pytest.raises(ValueError):
-        translate_to_test_suite(yaml_str)
-
-
 def test_invalid_context_as_testcase():
     yaml_str = """
 - tab: "Tab"
@@ -414,7 +401,7 @@ def test_statement_with_yaml_dict():
 - tab: "Feedback"
   testcases:
   - expression: "get_dict()"
-    return:
+    return: !v
         alpha: 5
         beta: 6
 """
@@ -466,7 +453,7 @@ def test_expression_raw_return():
   contexts:
     - testcases:
         - expression: 'test()'
-          return_raw: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]'
+          return: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -498,7 +485,7 @@ def test_empty_constructor(function_name, result):
   contexts:
     - testcases:
         - expression: 'test()'
-          return_raw: '{function_name}()'
+          return: '{function_name}()'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -528,7 +515,7 @@ def test_empty_constructor_with_param(function_name, result):
   contexts:
     - testcases:
         - expression: 'test()'
-          return_raw: '{function_name}([])'
+          return: '{function_name}([])'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -639,7 +626,7 @@ def test_value_built_in_checks_implied():
       contexts:
         - testcases:
             - expression: 'test()'
-              return_raw:
+              return:
                 value: "'hallo'"
     """
     json_str = translate_to_test_suite(yaml_str)
@@ -664,7 +651,7 @@ def test_value_built_in_checks_explicit():
       contexts:
         - testcases:
             - expression: 'test()'
-              return_raw:
+              return:
                 value: "'hallo'"
                 oracle: "builtin"
     """
@@ -690,7 +677,7 @@ def test_value_custom_checks_correct():
       contexts:
         - testcases:
             - expression: 'test()'
-              return_raw:
+              return:
                 value: "'hallo'"
                 oracle: "custom_check"
                 language: "python"