From e396a754b89c3cd7933df57e953c4535a1b25ba1 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Thu, 21 Sep 2023 10:43:07 +0200 Subject: [PATCH] Use individual arguments in custom check functions --- tested/judge/programmed.py | 24 +++++++------------ tested/languages/generation.py | 12 ++-------- .../echo-function/evaluation/Evaluator.cs | 4 ++-- .../echo-function/evaluation/Evaluator.hs | 6 ++--- .../echo-function/evaluation/Evaluator.java | 4 ++-- .../echo-function/evaluation/Evaluator.kt | 4 ++-- .../echo-function/evaluation/evaluator.js | 4 ++-- .../echo-function/evaluation/evaluator.py | 5 ++-- tests/exercises/echo/evaluation/evaluator.py | 5 ++-- tests/exercises/lotto/evaluation/evaluator.py | 4 ++-- tests/test_functionality.py | 2 +- tests/test_oracles.py | 2 +- 12 files changed, 32 insertions(+), 44 deletions(-) diff --git a/tested/judge/programmed.py b/tested/judge/programmed.py index 68d7acf4..379184e6 100644 --- a/tested/judge/programmed.py +++ b/tested/judge/programmed.py @@ -17,13 +17,9 @@ from tested.internationalization import get_i18n_string from tested.judge.execution import execute_file, filter_files from tested.judge.utils import BaseExecutionResult, copy_from_paths_to_path, run_command -from tested.languages.generation import ( - custom_oracle_arguments, - generate_custom_evaluator, - generate_statement, -) +from tested.languages.generation import generate_custom_evaluator, generate_statement from tested.oracles.common import BooleanEvalResult -from tested.serialisation import Value +from tested.serialisation import FunctionCall, FunctionType, Value from tested.testsuite import CustomCheckOracle from tested.utils import get_identifier @@ -214,17 +210,15 @@ def _evaluate_python( exec(evaluator_code, global_env) # Call the oracle. - literal_expected = generate_statement(eval_bundle, expected) - literal_actual = generate_statement(eval_bundle, actual) - arguments = custom_oracle_arguments(oracle) - literal_arguments = generate_statement(eval_bundle, arguments) + check_function_call = FunctionCall( + type=FunctionType.FUNCTION, + name=oracle.function.name, + arguments=[expected, actual, *oracle.arguments], + ) + literal_function_call = generate_statement(eval_bundle, check_function_call) with _catch_output() as (stdout_, stderr_): - exec( - f"__tested_test__result = {oracle.function.name}(" - f"{literal_expected}, {literal_actual}, {literal_arguments})", - global_env, - ) + exec(f"__tested_test__result = {literal_function_call}", global_env) stdout_ = stdout_.getvalue() stderr_ = stderr_.getvalue() diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 502d9066..3c66d8c0 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -17,7 +17,7 @@ from pygments.lexers import get_lexer_by_name from tested.configs import Bundle -from tested.datatypes import AllTypes, BasicObjectTypes, BasicSequenceTypes +from tested.datatypes import AllTypes, BasicObjectTypes from tested.dodona import ExtendedMessage from tested.internationalization import get_i18n_string from tested.judge.planning import PlannedExecutionUnit @@ -40,7 +40,6 @@ Expression, FunctionType, Identifier, - SequenceType, Statement, Value, VariableType, @@ -294,12 +293,6 @@ def generate_selector( return selector_filename -def custom_oracle_arguments(oracle: CustomCheckOracle) -> Value: - return SequenceType( - type=BasicSequenceTypes.SEQUENCE, data=oracle.arguments # pyright: ignore - ) - - def generate_custom_evaluator( bundle: Bundle, destination: Path, @@ -321,13 +314,12 @@ def generate_custom_evaluator( evaluator_name = conventionalize_namespace( bundle.language, evaluator.function.file.stem ) - arguments = custom_oracle_arguments(evaluator) function = PreparedFunctionCall( type=FunctionType.FUNCTION, namespace=Identifier(evaluator_name), name=evaluator.function.name, - arguments=[expected_value, actual_value, arguments], + arguments=[expected_value, actual_value, *evaluator.arguments], has_root_namespace=False, ) diff --git a/tests/exercises/echo-function/evaluation/Evaluator.cs b/tests/exercises/echo-function/evaluation/Evaluator.cs index d56695a2..17f74c73 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.cs +++ b/tests/exercises/echo-function/evaluation/Evaluator.cs @@ -8,12 +8,12 @@ public static EvaluationResult Evaluate(Object actual) { return new EvaluationResult(correct, "correct", actual != null ? actual.ToString() : "", messages); } - public static EvaluationResult EvaluateValue(Object expected, Object actual, IList arguments) { + public static EvaluationResult EvaluateValue(Object expected, Object actual) { var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(expected == actual, expected.ToString(), actual != null ? actual.ToString() : "", messages); } - public static EvaluationResult EvaluateValueDsl(Object expected, Object actual, IList arguments) { + public static EvaluationResult EvaluateValueDsl(Object expected, Object actual) { var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(expected == actual, null, null, messages, "{5, 5}", "{4, 4}"); } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.hs b/tests/exercises/echo-function/evaluation/Evaluator.hs index f3e76c89..586f84aa 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.hs +++ b/tests/exercises/echo-function/evaluation/Evaluator.hs @@ -15,12 +15,12 @@ evaluate value = } -evaluate_value :: String -> String -> [String] -> EvaluationResult -evaluate_value expected actual arguments = +evaluate_value :: String -> String -> EvaluationResult +evaluate_value expected actual = let correct = if actual == expected then True else False in evaluationResult { result = correct, readableExpected = Just expected, readableActual = Just actual, messages = [message "Hallo"] - } \ No newline at end of file + } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.java b/tests/exercises/echo-function/evaluation/Evaluator.java index ebae008a..f9e9a0a9 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.java +++ b/tests/exercises/echo-function/evaluation/Evaluator.java @@ -11,7 +11,7 @@ public static EvaluationResult evaluate(Object actual) { .build(); } - public static EvaluationResult evaluateValue(Object expected, Object actual, List arguments) { + public static EvaluationResult evaluateValue(Object expected, Object actual) { return EvaluationResult.builder(expected.equals(actual)) .withReadableExpected(expected.toString()) .withReadableActual(actual != null ? actual.toString() : "") @@ -19,7 +19,7 @@ public static EvaluationResult evaluateValue(Object expected, Object actual, Lis .build(); } - public static EvaluationResult evaluateValueDsl(Object expected, Object actual, List arguments) { + public static EvaluationResult evaluateValueDsl(Object expected, Object actual) { return EvaluationResult.builder(expected.equals(actual)) .withDslExpected("{5, 5}") .withDslActual("{4, 4}") diff --git a/tests/exercises/echo-function/evaluation/Evaluator.kt b/tests/exercises/echo-function/evaluation/Evaluator.kt index 777a8721..0f693aa6 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.kt +++ b/tests/exercises/echo-function/evaluation/Evaluator.kt @@ -10,7 +10,7 @@ class Evaluator { } @JvmStatic - fun evaluateValue(expected: Any, actual: Any?, arguments: List?): EvaluationResult { + fun evaluateValue(expected: Any, actual: Any?): EvaluationResult { return EvaluationResult.Builder(result = expected == actual, readableExpected = expected.toString(), readableActual = actual?.toString() ?: "") @@ -19,7 +19,7 @@ class Evaluator { } @JvmStatic - fun evaluateValueDsl(expected: Any, actual: Any?, arguments: List?): EvaluationResult { + fun evaluateValueDsl(expected: Any, actual: Any?): EvaluationResult { return EvaluationResult.Builder(result = expected == actual, dslExpected = "{5, 5}", dslActual = "{4, 4}") diff --git a/tests/exercises/echo-function/evaluation/evaluator.js b/tests/exercises/echo-function/evaluation/evaluator.js index 2a5ffd93..0b283517 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.js +++ b/tests/exercises/echo-function/evaluation/evaluator.js @@ -8,7 +8,7 @@ function evaluate(actual) { } } -function evaluateValue(expected, actual, args) { +function evaluateValue(expected, actual) { return { "result": expected === actual, "readable_expected": expected, @@ -17,7 +17,7 @@ function evaluateValue(expected, actual, args) { } } -function evaluateValueDsl(expected, actual, args) { +function evaluateValueDsl(expected, actual) { return { "result": expected === actual, "dsl_expected": "{5, 5}", diff --git a/tests/exercises/echo-function/evaluation/evaluator.py b/tests/exercises/echo-function/evaluation/evaluator.py index 6f177c1f..e4046066 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.py +++ b/tests/exercises/echo-function/evaluation/evaluator.py @@ -1,3 +1,4 @@ +# noinspection PyUnresolvedReferences from evaluation_utils import EvaluationResult, Message @@ -6,11 +7,11 @@ def evaluate(actual): return EvaluationResult(correct, "correct", actual, [Message("Hallo")]) -def evaluate_value(expected, actual, args): +def evaluate_value(expected, actual): return EvaluationResult(expected == actual, expected, actual, [Message("Hallo")]) -def evaluate_value_dsl(expected, actual, args): +def evaluate_value_dsl(expected, actual): return EvaluationResult( result=expected == actual, messages=[Message("Hallo")], diff --git a/tests/exercises/echo/evaluation/evaluator.py b/tests/exercises/echo/evaluation/evaluator.py index cdcf2dcc..0c99dcb1 100644 --- a/tests/exercises/echo/evaluation/evaluator.py +++ b/tests/exercises/echo/evaluation/evaluator.py @@ -1,9 +1,10 @@ +# noinspection PyUnresolvedReferences from evaluation_utils import EvaluationResult -def evaluate_correct(expected, actual, arguments): +def evaluate_correct(expected, actual): return EvaluationResult(expected.strip() == actual.strip()) -def evaluate_wrong(expected, actual, arguments): +def evaluate_wrong(_expected, _actual): return EvaluationResult(False) diff --git a/tests/exercises/lotto/evaluation/evaluator.py b/tests/exercises/lotto/evaluation/evaluator.py index dcec4923..a2c5c87e 100644 --- a/tests/exercises/lotto/evaluation/evaluator.py +++ b/tests/exercises/lotto/evaluation/evaluator.py @@ -1,4 +1,5 @@ import re +# noinspection PyUnresolvedReferences from evaluation_utils import EvaluationResult, Message @@ -39,8 +40,7 @@ def valid_lottery_numbers(number_str, count=6, maximum=42): return True, None -def evaluate(expected, actual, arguments): - count, maximum = arguments +def evaluate(expected, actual, count, maximum): valid, message = valid_lottery_numbers(actual, count, maximum) messages = [Message(message)] if message else [] if valid: diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 38f265cd..517e1de8 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -921,7 +921,7 @@ def test_expected_no_return_and_got_none(language: str, tmp_path: Path, pytestco result = execute_config(conf) updates = assert_valid_output(result, pytestconfig) - assert updates.find_status_enum() == ["correct"] + assert updates.find_status_enum() == [] @pytest.mark.parametrize("language", ALL_SPECIFIC_LANGUAGES) diff --git a/tests/test_oracles.py b/tests/test_oracles.py index 0d7df04a..db8888c5 100644 --- a/tests/test_oracles.py +++ b/tests/test_oracles.py @@ -6,7 +6,7 @@ from tested.configs import create_bundle from tested.datatypes import BasicObjectTypes, BasicSequenceTypes, BasicStringTypes from tested.dodona import Status -from tested.oracles.common import OracleConfig, OracleResult +from tested.oracles.common import OracleConfig from tested.oracles.exception import evaluate as evaluate_exception from tested.oracles.text import evaluate_file, evaluate_text from tested.oracles.value import evaluate as evaluate_value