diff --git a/WORKSPACE b/WORKSPACE index d0c2c1f13..0428704b0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -21,14 +21,13 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar") # Load license rules. # Must be loaded first due to https://github.com/bazel-contrib/rules_jvm_external/issues/1244 -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "rules_license", + sha256 = "26d4021f6898e23b82ef953078389dd49ac2b5618ac564ade4ef87cced147b38", urls = [ "https://mirror.bazel.build/github.com/bazelbuild/rules_license/releases/download/1.0.0/rules_license-1.0.0.tar.gz", "https://github.com/bazelbuild/rules_license/releases/download/1.0.0/rules_license-1.0.0.tar.gz", ], - sha256 = "26d4021f6898e23b82ef953078389dd49ac2b5618ac564ade4ef87cced147b38", ) http_archive( @@ -48,10 +47,10 @@ bazel_skylib_workspace() http_archive( name = "rules_java", + sha256 = "8daa0e4f800979c74387e4cd93f97e576ec6d52beab8ac94710d2931c57f8d8b", urls = [ "https://github.com/bazelbuild/rules_java/releases/download/8.9.0/rules_java-8.9.0.tar.gz", ], - sha256 = "8daa0e4f800979c74387e4cd93f97e576ec6d52beab8ac94710d2931c57f8d8b", ) http_archive( @@ -76,27 +75,31 @@ http_archive( ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + protobuf_deps() load("@rules_java//java:rules_java_deps.bzl", "rules_java_dependencies") + rules_java_dependencies() load("@rules_java//java:repositories.bzl", "rules_java_toolchains") + rules_java_toolchains() load("@rules_python//python:repositories.bzl", "py_repositories") + py_repositories() load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies") + rules_proto_dependencies() load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains") -rules_proto_toolchains() +rules_proto_toolchains() ### End of Protobuf Setup - ### rules_jvm_external setup RULES_JVM_EXTERNAL_TAG = "6.6" @@ -124,6 +127,7 @@ load("//:maven_utils.bzl", "maven_artifact_compile_only", "maven_artifact_test_o ### end of rules_jvm_external setup ANTLR4_VERSION = "4.13.2" + maven_install( name = "maven", # keep sorted @@ -137,9 +141,11 @@ maven_install( "com.google.re2j:re2j:1.8", "info.picocli:picocli:4.7.6", "org.antlr:antlr4-runtime:" + ANTLR4_VERSION, + "com.google.api.grpc:proto-google-common-protos:2.54.1", "info.picocli:picocli:4.7.6", "org.freemarker:freemarker:2.3.33", "org.jspecify:jspecify:1.0.0", + "org.mockito:mockito-core:4.11.0", "org.threeten:threeten-extra:1.8.0", "org.yaml:snakeyaml:2.3", maven_artifact_test_only("com.google.testparameterinjector", "test-parameter-injector", "1.18"), @@ -178,14 +184,17 @@ http_archive( ) load("@rules_android//:prereqs.bzl", "rules_android_prereqs") + rules_android_prereqs() load("@rules_android//:defs.bzl", "rules_android_workspace") + rules_android_workspace() load("@rules_android//rules:rules.bzl", "android_sdk_repository") + android_sdk_repository( - name = "androidsdk" + name = "androidsdk", ) register_toolchains( @@ -270,4 +279,3 @@ http_jar( sha256 = "eae2dfa119a64327444672aff63e9ec35a20180dc5b8090b7a6ab85125df4d76", urls = ["https://www.antlr.org/download/antlr-" + ANTLR4_VERSION + "-complete.jar"], ) - diff --git a/testing/src/main/java/dev/cel/testing/testrunner/BUILD.bazel b/testing/src/main/java/dev/cel/testing/testrunner/BUILD.bazel new file mode 100644 index 000000000..98b656eb0 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/BUILD.bazel @@ -0,0 +1,206 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_testonly = True, + default_visibility = [ + "//testing/testrunner:__pkg__", + ], +) + +java_library( + name = "test_executor", + srcs = ["TestExecutor.java"], + tags = [ + ], + deps = [ + ":cel_test_suite", + ":cel_test_suite_exception", + ":cel_test_suite_text_proto_parser", + ":cel_test_suite_yaml_parser", + ":junit_xml_reporter", + "@maven//:com_google_guava_guava", + "@maven//:junit_junit", + ], +) + +java_library( + name = "junit_xml_reporter", + srcs = ["JUnitXmlReporter.java"], + tags = [ + ], + deps = ["@maven//:com_google_guava_guava"], +) + +java_library( + name = "cel_user_test_template", + srcs = ["CelUserTestTemplate.java"], + tags = [ + ], + deps = [ + ":cel_test_context", + ":cel_test_suite", + ":test_runner_library", + "@maven//:junit_junit", + ], +) + +java_library( + name = "test_runner_library", + srcs = ["TestRunnerLibrary.java"], + tags = [ + ], + deps = [ + ":cel_test_context", + ":cel_test_suite", + ":registry_utils", + ":result_matcher", + "//:auto_value", + "//bundle:cel", + "//bundle:environment", + "//bundle:environment_exception", + "//bundle:environment_yaml_parser", + "//common:cel_ast", + "//common:cel_descriptors", + "//common:compiler_common", + "//common:options", + "//common:proto_ast", + "//policy", + "//policy:compiler_factory", + "//policy:parser", + "//policy:parser_factory", + "//policy:validation_exception", + "//runtime", + "//testing:expr_value_utils", + "@cel_spec//proto/cel/expr:expr_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + ], +) + +java_library( + name = "cel_test_suite", + srcs = ["CelTestSuite.java"], + tags = [ + ], + deps = [ + "//:auto_value", + "//common:source", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + ], +) + +java_library( + name = "cel_test_suite_yaml_parser", + srcs = ["CelTestSuiteYamlParser.java"], + tags = [ + ], + deps = [ + ":cel_test_suite", + ":cel_test_suite_exception", + "//common:compiler_common", + "//common/formats:file_source", + "//common/formats:parser_context", + "//common/formats:yaml_helper", + "//common/formats:yaml_parser_context_impl", + "//common/internal", + "@maven//:com_google_guava_guava", + "@maven//:org_yaml_snakeyaml", + ], +) + +java_library( + name = "cel_test_suite_exception", + srcs = ["CelTestSuiteException.java"], + tags = [ + ], + deps = ["//common:cel_exception"], +) + +java_library( + name = "cel_test_context", + srcs = ["CelTestContext.java"], + tags = [ + ], + deps = [ + ":default_result_matcher", + ":result_matcher", + "//:auto_value", + "//bundle:cel", + "//common:options", + "//policy:parser", + "//runtime", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "registry_utils", + srcs = ["RegistryUtils.java"], + deps = [ + "//common:cel_descriptors", + "//common/internal:cel_descriptor_pools", + "//common/internal:default_message_factory", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + ], +) + +java_library( + name = "result_matcher", + srcs = ["ResultMatcher.java"], + deps = [ + ":cel_test_suite", + "//:auto_value", + "//bundle:cel", + "//common/types:type_providers", + "//runtime", + "@cel_spec//proto/cel/expr:expr_java_proto", + ], +) + +java_library( + name = "default_result_matcher", + srcs = ["DefaultResultMatcher.java"], + deps = [ + ":cel_test_suite", + ":registry_utils", + ":result_matcher", + "//:java_truth", + "//bundle:cel", + "//common:cel_ast", + "//common:cel_descriptors", + "//runtime", + "//testing:expr_value_utils", + "@cel_spec//proto/cel/expr:expr_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_extensions_truth_proto_extension", + ], +) + +java_library( + name = "cel_test_suite_text_proto_parser", + srcs = ["CelTestSuiteTextProtoParser.java"], + tags = [ + ], + deps = [ + ":cel_test_suite", + ":cel_test_suite_exception", + ":registry_utils", + "//common:cel_descriptors", + "@cel_spec//proto/cel/expr/conformance/test:suite_java_proto", + "@maven//:com_google_api_grpc_proto_google_common_protos", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + ], +) + +filegroup( + name = "test_runner_binary", + srcs = [ + "TestRunnerBinary.java", + ], +) diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelTestContext.java b/testing/src/main/java/dev/cel/testing/testrunner/CelTestContext.java new file mode 100644 index 000000000..6633d0e63 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelTestContext.java @@ -0,0 +1,112 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import dev.cel.bundle.Cel; +import dev.cel.bundle.CelFactory; +import dev.cel.common.CelOptions; +import dev.cel.policy.CelPolicyParser; +import dev.cel.runtime.CelLateFunctionBindings; +import java.util.Map; +import java.util.Optional; + +/** + * The context class for a CEL test, holding configurations needed to create environments and + * evaluate CEL expressions and policies. + */ +@AutoValue +public abstract class CelTestContext { + + private static final Cel DEFAULT_CEL = CelFactory.standardCelBuilder().build(); + + /** + * The CEL environment for the CEL test. + * + *

The CEL environment is created by extending the provided base CEL environment with the + * config file if provided. + */ + public abstract Cel cel(); + + /** + * The CEL policy parser for the CEL test. + * + *

A custom parser to be used for parsing CEL policies in scenarios where custom policy tags + * are used. If not provided, the default CEL policy parser will be used. + */ + public abstract Optional celPolicyParser(); + + /** + * The CEL options for the CEL test. + * + *

The CEL options are used to configure the {@link Cel} environment. + */ + public abstract CelOptions celOptions(); + + /** + * The late function bindings for the CEL test. + * + *

These bindings are used to provide functions which are to be consumed during the eval phase + * directly. + */ + public abstract Optional celLateFunctionBindings(); + + /** + * The variable bindings for the CEL test. + * + *

These bindings are used to provide values for variables for which it is difficult to provide + * a value in the test suite file for example, using proto extensions or fetching the value from + * some other source. + */ + public abstract ImmutableMap variableBindings(); + + /** + * The result matcher for the CEL test. + * + *

This matcher is used to perform assertions on the result of a CEL test case. + */ + public abstract ResultMatcher resultMatcher(); + + /** Returns a builder for {@link CelTestContext} with the current instance's values. */ + public abstract Builder toBuilder(); + + /** Returns a new builder for {@link CelTestContext}. */ + public static CelTestContext.Builder newBuilder() { + return new AutoValue_CelTestContext.Builder() + .setCel(DEFAULT_CEL) + .setCelOptions(CelOptions.DEFAULT) + .setVariableBindings(ImmutableMap.of()) + .setResultMatcher(new DefaultResultMatcher()); + } + + /** Builder for {@link CelTestContext}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setCel(Cel cel); + + public abstract Builder setCelPolicyParser(CelPolicyParser celPolicyParser); + + public abstract Builder setCelOptions(CelOptions celOptions); + + public abstract Builder setCelLateFunctionBindings( + CelLateFunctionBindings celLateFunctionBindings); + + public abstract Builder setVariableBindings(Map variableBindings); + + public abstract Builder setResultMatcher(ResultMatcher resultMatcher); + + public abstract CelTestContext build(); + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuite.java b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuite.java new file mode 100644 index 000000000..cf661bb4f --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuite.java @@ -0,0 +1,233 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import com.google.auto.value.AutoOneOf; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CheckReturnValue; +import com.google.protobuf.Any; +import dev.cel.common.Source; +import java.util.Optional; +import java.util.Set; + +/** Class representing a CEL test suite which is generated post parsing the test suite file. */ +@AutoValue +public abstract class CelTestSuite { + + public abstract String name(); + + /** Test suite source in textual format (ex: textproto, YAML). */ + public abstract Optional source(); + + public abstract String description(); + + public abstract ImmutableSet sections(); + + /** Builder for {@link CelTestSuite}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setName(String name); + + public abstract Builder setDescription(String description); + + public abstract Builder setSections(Set section); + + public abstract Builder setSource(Source source); + + @CheckReturnValue + public abstract CelTestSuite build(); + } + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_CelTestSuite.Builder(); + } + + /** + * Class representing a CEL test section within a test suite following the schema in {@link + * dev.cel.expr.conformance.test.TestSuite}. + */ + @AutoValue + public abstract static class CelTestSection { + + public abstract String name(); + + public abstract String description(); + + public abstract ImmutableSet tests(); + + /** Builder for {@link CelTestSection}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setName(String name); + + public abstract Builder setTests(Set tests); + + public abstract Builder setDescription(String description); + + @CheckReturnValue + public abstract CelTestSection build(); + } + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_CelTestSuite_CelTestSection.Builder(); + } + + /** Class representing a CEL test case within a test section. */ + @AutoValue + public abstract static class CelTestCase { + + public abstract String name(); + + public abstract String description(); + + public abstract Input input(); + + public abstract Output output(); + + /** This class represents the input of a CEL test case. */ + @AutoOneOf(Input.Kind.class) + public abstract static class Input { + /** Kind of input for a CEL test case. */ + public enum Kind { + BINDINGS, + CONTEXT_EXPR, + CONTEXT_MESSAGE, + NO_INPUT + } + + public abstract Input.Kind kind(); + + public abstract ImmutableMap bindings(); + + public abstract String contextExpr(); + + public abstract Any contextMessage(); + + public abstract void noInput(); + + public static Input ofBindings(ImmutableMap bindings) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Input.bindings(bindings); + } + + public static Input ofContextExpr(String contextExpr) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Input.contextExpr(contextExpr); + } + + public static Input ofContextMessage(Any contextMessage) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Input.contextMessage( + contextMessage); + } + + public static Input ofNoInput() { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Input.noInput(); + } + + /** This class represents a binding for a CEL test case. */ + @AutoOneOf(Binding.Kind.class) + public abstract static class Binding { + + /** Kind of binding for a CEL test case. */ + public enum Kind { + VALUE, + EXPR + } + + public abstract Binding.Kind kind(); + + public abstract Object value(); + + public abstract String expr(); + + public static Binding ofValue(Object value) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Input_Binding.value(value); + } + + public static Binding ofExpr(String expr) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Input_Binding.expr(expr); + } + } + } + + /** This class represents the result of a CEL test case. */ + @AutoOneOf(Output.Kind.class) + public abstract static class Output { + /** Kind of result for a CEL test case. */ + public enum Kind { + RESULT_VALUE, + RESULT_EXPR, + EVAL_ERROR, + NO_OUTPUT + } + + public abstract Output.Kind kind(); + + public abstract Object resultValue(); + + public abstract String resultExpr(); + + public abstract void noOutput(); + + public abstract ImmutableList evalError(); + + public static Output ofResultValue(Object resultValue) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Output.resultValue(resultValue); + } + + public static Output ofResultExpr(String resultExpr) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Output.resultExpr(resultExpr); + } + + public static Output ofEvalError(ImmutableList errors) { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Output.evalError(errors); + } + + public static Output ofNoOutput() { + return AutoOneOf_CelTestSuite_CelTestSection_CelTestCase_Output.noOutput(); + } + } + + /** Builder for {@link CelTestCase}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setName(String name); + + public abstract Builder setDescription(String description); + + public abstract Builder setInput(Input input); + + public abstract Builder setOutput(Output output); + + @CheckReturnValue + public abstract CelTestCase build(); + } + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_CelTestSuite_CelTestSection_CelTestCase.Builder() + .setInput(Input.ofNoInput()); // Default input to no input. + } + } + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteException.java b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteException.java new file mode 100644 index 000000000..3b3449df4 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteException.java @@ -0,0 +1,29 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import dev.cel.common.CelException; + +/** Checked exception thrown when a CEL test suite is misconfigured. */ +public final class CelTestSuiteException extends CelException { + + CelTestSuiteException(String message) { + super(message); + } + + CelTestSuiteException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteTextProtoParser.java b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteTextProtoParser.java new file mode 100644 index 000000000..906be0c9d --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteTextProtoParser.java @@ -0,0 +1,171 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.TextFormat; +import com.google.protobuf.TextFormat.ParseException; +import com.google.protobuf.TypeRegistry; +import com.google.rpc.Status; +import dev.cel.common.CelDescriptorUtil; +import dev.cel.expr.conformance.test.InputValue; +import dev.cel.expr.conformance.test.TestCase; +import dev.cel.expr.conformance.test.TestSection; +import dev.cel.expr.conformance.test.TestSuite; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Input.Binding; +import java.io.IOException; +import java.util.Map; + +/** + * CelTestSuiteTextProtoParser intakes a textproto document that describes the structure of a CEL + * test suite, parses it then creates a {@link CelTestSuite}. + */ +final class CelTestSuiteTextProtoParser { + + /** Creates a new instance of {@link CelTestSuiteTextProtoParser}. */ + static CelTestSuiteTextProtoParser newInstance() { + return new CelTestSuiteTextProtoParser(); + } + + CelTestSuite parse(String textProto) throws IOException, CelTestSuiteException { + TestSuite testSuite = parseTestSuite(textProto); + return parseCelTestSuite(testSuite); + } + + private TestSuite parseTestSuite(String textProto) throws IOException { + String fileDescriptorSetPath = System.getProperty("file_descriptor_set_path"); + TypeRegistry typeRegistry = TypeRegistry.getEmptyTypeRegistry(); + ExtensionRegistry extensionRegistry = ExtensionRegistry.getEmptyRegistry(); + if (fileDescriptorSetPath != null) { + ImmutableSet fileDescriptors = + CelDescriptorUtil.getFileDescriptorsFromFileDescriptorSet( + RegistryUtils.getFileDescriptorSet(fileDescriptorSetPath)); + extensionRegistry = RegistryUtils.getExtensionRegistry(fileDescriptors); + typeRegistry = RegistryUtils.getTypeRegistry(fileDescriptors); + } + TextFormat.Parser parser = TextFormat.Parser.newBuilder().setTypeRegistry(typeRegistry).build(); + TestSuite.Builder builder = TestSuite.newBuilder(); + try { + parser.merge(textProto, extensionRegistry, builder); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse test suite", e); + } + return builder.build(); + } + + @VisibleForTesting + static CelTestSuite parseCelTestSuite(TestSuite testSuite) throws CelTestSuiteException { + CelTestSuite.Builder builder = + CelTestSuite.newBuilder() + .setName(testSuite.getName()) + .setDescription(testSuite.getDescription()); + ImmutableSet.Builder sectionSetBuilder = ImmutableSet.builder(); + + for (TestSection section : testSuite.getSectionsList()) { + CelTestSection.Builder sectionBuilder = + CelTestSection.newBuilder() + .setName(section.getName()) + .setDescription(section.getDescription()); + ImmutableSet.Builder testCaseSetBuilder = ImmutableSet.builder(); + + for (TestCase testCase : section.getTestsList()) { + CelTestCase.Builder testCaseBuilder = + CelTestCase.newBuilder() + .setName(testCase.getName()) + .setDescription(testCase.getDescription()); + addInputs(testCaseBuilder, testCase); + addOutputs(testCaseBuilder, testCase); + testCaseSetBuilder.add(testCaseBuilder.build()); + } + + sectionBuilder.setTests(testCaseSetBuilder.build()); + sectionSetBuilder.add(sectionBuilder.build()); + } + return builder.setSections(sectionSetBuilder.build()).build(); + } + + private static void addInputs(CelTestCase.Builder testCaseBuilder, TestCase testCase) + throws CelTestSuiteException { + if (testCase.getInputCount() > 0 && testCase.hasInputContext()) { + throw new CelTestSuiteException( + String.format( + "Test case: %s cannot have both input map and input context.", testCase.getName())); + } else if (testCase.getInputCount() > 0) { + testCaseBuilder.setInput(parseInputMap(testCase)); + } else if (testCase.hasInputContext()) { + testCaseBuilder.setInput(parseInputContext(testCase)); + } else { + testCaseBuilder.setInput(CelTestCase.Input.ofNoInput()); + } + } + + private static CelTestCase.Input parseInputMap(TestCase testCase) { + ImmutableMap.Builder inputMapBuilder = ImmutableMap.builder(); + for (Map.Entry entry : testCase.getInputMap().entrySet()) { + InputValue inputValue = entry.getValue(); + if (inputValue.hasValue()) { + inputMapBuilder.put(entry.getKey(), Binding.ofValue(inputValue.getValue())); + } else if (inputValue.hasExpr()) { + inputMapBuilder.put(entry.getKey(), Binding.ofExpr(inputValue.getExpr())); + } + } + return CelTestCase.Input.ofBindings(inputMapBuilder.buildOrThrow()); + } + + private static CelTestCase.Input parseInputContext(TestCase testCase) { + if (testCase.getInputContext().hasContextMessage()) { + return CelTestCase.Input.ofContextMessage(testCase.getInputContext().getContextMessage()); + } else if (testCase.getInputContext().hasContextExpr()) { + return CelTestCase.Input.ofContextExpr(testCase.getInputContext().getContextExpr()); + } + return CelTestCase.Input.ofNoInput(); + } + + private static void addOutputs(CelTestCase.Builder testCaseBuilder, TestCase testCase) { + if (testCase.hasOutput()) { + switch (testCase.getOutput().getResultKindCase()) { + case RESULT_VALUE: + testCaseBuilder.setOutput( + CelTestCase.Output.ofResultValue(testCase.getOutput().getResultValue())); + break; + case RESULT_EXPR: + testCaseBuilder.setOutput( + CelTestCase.Output.ofResultExpr(testCase.getOutput().getResultExpr())); + break; + case EVAL_ERROR: + testCaseBuilder.setOutput(CelTestCase.Output.ofEvalError(parseEvalError(testCase))); + break; + default: + break; + } + } + } + + private static ImmutableList parseEvalError(TestCase testCase) { + ImmutableList.Builder evalErrorSetBuilder = ImmutableList.builder(); + for (Status error : testCase.getOutput().getEvalError().getErrorsList()) { + evalErrorSetBuilder.add(error.getMessage()); + } + return evalErrorSetBuilder.build(); + } + + private CelTestSuiteTextProtoParser() {} +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteYamlParser.java b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteYamlParser.java new file mode 100644 index 000000000..20f4e1f58 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelTestSuiteYamlParser.java @@ -0,0 +1,360 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import static dev.cel.common.formats.YamlHelper.YamlNodeType.nodeType; +import static dev.cel.common.formats.YamlHelper.assertYamlType; +import static dev.cel.common.formats.YamlHelper.newBoolean; +import static dev.cel.common.formats.YamlHelper.newDouble; +import static dev.cel.common.formats.YamlHelper.newInteger; +import static dev.cel.common.formats.YamlHelper.newString; +import static dev.cel.common.formats.YamlHelper.parseYamlSource; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import dev.cel.common.CelIssue; +import dev.cel.common.formats.CelFileSource; +import dev.cel.common.formats.ParserContext; +import dev.cel.common.formats.YamlHelper.YamlNodeType; +import dev.cel.common.formats.YamlParserContextImpl; +import dev.cel.common.internal.CelCodePointArray; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Input.Binding; +import java.util.Optional; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + +/** + * CelTestSuiteYamlParser intakes a YAML document that describes the structure of a CEL test suite, + * parses it then creates a {@link CelTestSuite}. + */ +final class CelTestSuiteYamlParser { + + /** Creates a new instance of {@link CelTestSuiteYamlParser}. */ + static CelTestSuiteYamlParser newInstance() { + return new CelTestSuiteYamlParser(); + } + + CelTestSuite parse(String celTestSuiteYamlContent) throws CelTestSuiteException { + return parseYaml(celTestSuiteYamlContent, ""); + } + + private CelTestSuite parseYaml(String celTestSuiteYamlContent, String description) + throws CelTestSuiteException { + Node node; + try { + node = + parseYamlSource(celTestSuiteYamlContent) + .orElseThrow( + () -> + new CelTestSuiteException( + String.format( + "YAML document empty or malformed: %s", celTestSuiteYamlContent))); + } catch (RuntimeException e) { + throw new CelTestSuiteException("YAML document is malformed: " + e.getMessage(), e); + } + + CelFileSource testSuiteSource = + CelFileSource.newBuilder(CelCodePointArray.fromString(celTestSuiteYamlContent)) + .setDescription(description) + .build(); + ParserContext ctx = YamlParserContextImpl.newInstance(testSuiteSource); + CelTestSuite.Builder builder = parseTestSuite(ctx, node); + testSuiteSource = testSuiteSource.toBuilder().setPositionsMap(ctx.getIdToOffsetMap()).build(); + + if (!ctx.getIssues().isEmpty()) { + throw new CelTestSuiteException(CelIssue.toDisplayString(ctx.getIssues(), testSuiteSource)); + } + + return builder.setSource(testSuiteSource).build(); + } + + private CelTestSuite.Builder parseTestSuite(ParserContext ctx, Node node) { + CelTestSuite.Builder builder = CelTestSuite.newBuilder(); + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { + ctx.reportError(id, "Unknown test suite type: " + node.getTag()); + return builder; + } + + MappingNode rootNode = (MappingNode) node; + for (NodeTuple nodeTuple : rootNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + if (!assertYamlType(ctx, keyId, keyNode, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "name": + builder.setName(newString(ctx, valueNode)); + break; + case "description": + builder.setDescription(newString(ctx, valueNode)); + break; + case "sections": + builder.setSections(parseSections(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, "Unknown test suite tag: " + fieldName); + break; + } + } + return builder; + } + + private ImmutableSet parseSections(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder celTestSectionSetBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + ctx.reportError(valueId, "Sections is not a list: " + node.getTag()); + return celTestSectionSetBuilder.build(); + } + + SequenceNode sectionListNode = (SequenceNode) node; + for (Node elementNode : sectionListNode.getValue()) { + celTestSectionSetBuilder.add(parseSection(ctx, elementNode)); + } + return celTestSectionSetBuilder.build(); + } + + private CelTestSection parseSection(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { + ctx.reportError(valueId, "Unknown section type: " + node.getTag()); + return CelTestSection.newBuilder().build(); + } + + CelTestSection.Builder celTestSectionBuilder = CelTestSection.newBuilder(); + MappingNode sectionNode = (MappingNode) node; + for (NodeTuple nodeTuple : sectionNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "name": + celTestSectionBuilder.setName(newString(ctx, valueNode)); + break; + case "description": + celTestSectionBuilder.setDescription(newString(ctx, valueNode)); + break; + case "tests": + celTestSectionBuilder.setTests(parseTests(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, "Unknown test section tag: " + fieldName); + break; + } + } + return celTestSectionBuilder.build(); + } + + private ImmutableSet parseTests(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder celTestCaseSetBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + ctx.reportError(valueId, "Tests is not a list: " + node.getTag()); + return celTestCaseSetBuilder.build(); + } + + SequenceNode testCasesListNode = (SequenceNode) node; + for (Node elementNode : testCasesListNode.getValue()) { + celTestCaseSetBuilder.add(parseTestCase(ctx, elementNode)); + } + return celTestCaseSetBuilder.build(); + } + + private CelTestCase parseTestCase(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + CelTestCase.Builder celTestCaseBuilder = CelTestCase.newBuilder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { + ctx.reportError(valueId, "Testcase is not a map: " + node.getTag()); + return celTestCaseBuilder.build(); + } + MappingNode testCaseNode = (MappingNode) node; + for (NodeTuple nodeTuple : testCaseNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "name": + celTestCaseBuilder.setName(newString(ctx, valueNode)); + break; + case "description": + celTestCaseBuilder.setDescription(newString(ctx, valueNode)); + break; + case "input": + celTestCaseBuilder.setInput(parseInput(ctx, valueNode)); + break; + case "context_expr": + celTestCaseBuilder.setInput(parseContextExpr(ctx, valueNode)); + break; + case "output": + celTestCaseBuilder.setOutput(parseOutput(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, "Unknown test case tag: " + fieldName); + break; + } + } + return celTestCaseBuilder.build(); + } + + private CelTestCase.Input parseInput(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { + ctx.reportError(valueId, "Input is not a map: " + node.getTag()); + return CelTestCase.Input.ofNoInput(); + } + MappingNode inputNode = (MappingNode) node; + ImmutableMap.Builder bindingsBuilder = ImmutableMap.builder(); + for (NodeTuple nodeTuple : inputNode.getValue()) { + Node valueNode = nodeTuple.getValueNode(); + Optional binding = parseBindingValueNode(ctx, valueNode); + binding.ifPresent( + b -> bindingsBuilder.put(((ScalarNode) nodeTuple.getKeyNode()).getValue(), b)); + } + return CelTestCase.Input.ofBindings(bindingsBuilder.buildOrThrow()); + } + + private Optional parseBindingValueNode(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { + ctx.reportError(valueId, "Input binding node is not a map: " + node.getTag()); + return Optional.empty(); + } + MappingNode bindingValueNode = (MappingNode) node; + + if (bindingValueNode.getValue().size() != 1) { + ctx.reportError(valueId, "Input binding node must have exactly one value: " + node.getTag()); + return Optional.empty(); + } + + for (NodeTuple nodeTuple : bindingValueNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "value": + return Optional.of(Binding.ofValue(parseNodeValue(ctx, valueNode))); + case "expr": + return Optional.of(Binding.ofExpr(newString(ctx, valueNode))); + default: + ctx.reportError(keyId, "Unknown input binding value tag: " + fieldName); + break; + } + } + return Optional.empty(); + } + + // TODO: Create a CelTestSuiteNodeValue class to represent the value of a test suite + // node. + private Object parseNodeValue(ParserContext ctx, Node node) { + Object value = null; + Optional yamlNodeType = nodeType(node.getTag().getValue()); + if (yamlNodeType.isPresent()) { + switch (yamlNodeType.get()) { + case STRING: + case TEXT: + value = newString(ctx, node); + break; + case BOOLEAN: + value = newBoolean(ctx, node); + break; + case INTEGER: + value = newInteger(ctx, node); + break; + case DOUBLE: + value = newDouble(ctx, node); + break; + case MAP: + value = parseMap(ctx, node); + break; + case LIST: + value = parseList(ctx, node); + break; + } + } + return value; + } + + private ImmutableMap parseMap(ParserContext ctx, Node node) { + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + MappingNode mapNode = (MappingNode) node; + mapNode + .getValue() + .forEach( + nodeTuple -> { + Node keyNode = nodeTuple.getKeyNode(); + Node valueNode = nodeTuple.getValueNode(); + mapBuilder.put(parseNodeValue(ctx, keyNode), parseNodeValue(ctx, valueNode)); + }); + return mapBuilder.buildOrThrow(); + } + + private ImmutableList parseList(ParserContext ctx, Node node) { + ImmutableList.Builder listBuilder = ImmutableList.builder(); + SequenceNode listNode = (SequenceNode) node; + listNode.getValue().forEach(childNode -> listBuilder.add(parseNodeValue(ctx, childNode))); + return listBuilder.build(); + } + + private CelTestCase.Input parseContextExpr(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.STRING)) { + ctx.reportError(valueId, "Input context is not a string: " + node.getTag()); + return CelTestCase.Input.ofNoInput(); + } + return CelTestCase.Input.ofContextExpr(newString(ctx, node)); + } + + private CelTestCase.Output parseOutput(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { + ctx.reportError(valueId, "Output is not a map: " + node.getTag()); + return CelTestCase.Output.ofNoOutput(); + } + MappingNode outputNode = (MappingNode) node; + for (NodeTuple nodeTuple : outputNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "value": + return CelTestCase.Output.ofResultValue(parseNodeValue(ctx, valueNode)); + case "expr": + return CelTestCase.Output.ofResultExpr(newString(ctx, valueNode)); + case "error_set": + return CelTestCase.Output.ofEvalError(parseList(ctx, valueNode)); + default: + ctx.reportError(keyId, "Unknown output tag: " + fieldName); + break; + } + } + return CelTestCase.Output.ofNoOutput(); + } + + private CelTestSuiteYamlParser() {} +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/CelUserTestTemplate.java b/testing/src/main/java/dev/cel/testing/testrunner/CelUserTestTemplate.java new file mode 100644 index 000000000..7cafb3efc --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/CelUserTestTemplate.java @@ -0,0 +1,41 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; + +/** + * Template to be extended by user's test class in order to be parameterized based on individual + * test cases. + */ +@RunWith(Parameterized.class) +public abstract class CelUserTestTemplate { + + @Parameter public CelTestCase testCase; + private final CelTestContext celTestContext; + + public CelUserTestTemplate(CelTestContext celTestContext) { + this.celTestContext = celTestContext; + } + + @Test + public void test() throws Exception { + TestRunnerLibrary.runTest(testCase, celTestContext); + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/DefaultResultMatcher.java b/testing/src/main/java/dev/cel/testing/testrunner/DefaultResultMatcher.java new file mode 100644 index 000000000..c800dc53c --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/DefaultResultMatcher.java @@ -0,0 +1,103 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static dev.cel.testing.utils.ExprValueUtils.toExprValue; + +import dev.cel.expr.ExprValue; +import dev.cel.expr.MapValue; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Descriptors.FileDescriptor; +import dev.cel.bundle.Cel; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelDescriptorUtil; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelRuntime.Program; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Output; +import dev.cel.testing.testrunner.ResultMatcher.ResultMatcherParams; +import dev.cel.testing.testrunner.ResultMatcher.ResultMatcherParams.ComputedOutput; +import java.io.IOException; + +final class DefaultResultMatcher implements ResultMatcher { + + @Override + public void match(ResultMatcherParams params, Cel cel) throws Exception { + Output result = params.expectedOutput().get(); + switch (result.kind()) { + case RESULT_EXPR: + if (params.computedOutput().kind().equals(ComputedOutput.Kind.ERROR)) { + throw new AssertionError( + "Error: " + params.computedOutput().error().getMessage(), + params.computedOutput().error()); + } + CelAbstractSyntaxTree exprAst = cel.compile(result.resultExpr()).getAst(); + Program exprProgram = cel.createProgram(exprAst); + Object evaluationResult = null; + try { + evaluationResult = exprProgram.eval(); + } catch (CelEvaluationException e) { + throw new IllegalArgumentException( + "Failed to evaluate result_expr: " + e.getMessage(), e); + } + ExprValue expectedExprValue = toExprValue(evaluationResult, exprAst.getResultType()); + assertThat(params.computedOutput().exprValue()).isEqualTo(expectedExprValue); + break; + case RESULT_VALUE: + if (params.computedOutput().kind().equals(ComputedOutput.Kind.ERROR)) { + throw new AssertionError( + "Error: " + params.computedOutput().error().getMessage(), + params.computedOutput().error()); + } + assertExprValue( + params.computedOutput().exprValue(), + toExprValue(result.resultValue(), params.resultType())); + break; + case EVAL_ERROR: + if (params.computedOutput().kind().equals(ComputedOutput.Kind.EXPR_VALUE)) { + throw new AssertionError( + "Evaluation was successful but no value was provided. Computed output: " + + params.computedOutput().exprValue()); + } + assertThat(params.computedOutput().error().toString()) + .contains(result.evalError().get(0).toString()); + break; + default: + throw new IllegalArgumentException("Unexpected output type: " + result.kind()); + } + } + + private static void assertExprValue(ExprValue exprValue, ExprValue expectedExprValue) + throws IOException { + String fileDescriptorSetPath = System.getProperty("file_descriptor_set_path"); + if (fileDescriptorSetPath != null) { + ImmutableSet fileDescriptors = + CelDescriptorUtil.getFileDescriptorsFromFileDescriptorSet( + RegistryUtils.getFileDescriptorSet(fileDescriptorSetPath)); + assertThat(exprValue) + .ignoringRepeatedFieldOrderOfFieldDescriptors( + MapValue.getDescriptor().findFieldByName("entries")) + .unpackingAnyUsing( + RegistryUtils.getTypeRegistry(fileDescriptors), + RegistryUtils.getExtensionRegistry(fileDescriptors)) + .isEqualTo(expectedExprValue); + } else { + assertThat(exprValue) + .ignoringRepeatedFieldOrderOfFieldDescriptors( + MapValue.getDescriptor().findFieldByName("entries")) + .isEqualTo(expectedExprValue); + } + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/JUnitXmlReporter.java b/testing/src/main/java/dev/cel/testing/testrunner/JUnitXmlReporter.java new file mode 100644 index 000000000..df45f4c2c --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/JUnitXmlReporter.java @@ -0,0 +1,230 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Text; + +/** Reporter class to generate an xml report in junit format. */ +final class JUnitXmlReporter { + private String outputFileName = null; + private File outputFile = null; + private TestContext testContext = null; + // NOMUTANTS -- To be fixed in b/394771693 and when more failure tests are added. + private int numFailed = 0; + + private final List allTests = Lists.newArrayList(); + + /** Creates an instance that will write to {@code outputFileName}. */ + JUnitXmlReporter(String outputFileName) { + this.outputFileName = outputFileName; + } + + /** Called for each test case */ + void onTestStart(TestResult result) {} + + /** Called on test success */ + void onTestSuccess(TestResult tr) { + allTests.add(tr); + } + + /** Called when the test fails */ + void onTestFailure(TestResult tr) { + allTests.add(tr); + numFailed++; + } + + /** Called in the beginning of test suite. */ + void onStart(TestContext context) { + outputFile = new File(outputFileName); + testContext = context; + } + + /** Called after all tests are run */ + void onFinish() { + generateReport(); + } + + /** Returns the number of failed tests */ + int getNumFailed() { + return numFailed; + } + + /** + * Generates junit equivalent xml report that sponge/fusion can understand. Called after all tests + * are run + */ + void generateReport() { + try { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder(); + Document doc = docBuilder.newDocument(); + + Element rootElement = doc.createElement(XmlConstants.TESTSUITE); + rootElement.setAttribute(XmlConstants.ATTR_NAME, testContext.getSuiteName()); + + rootElement.setAttribute(XmlConstants.ATTR_TESTS, "" + allTests.size()); + rootElement.setAttribute(XmlConstants.ATTR_FAILURES, "" + numFailed); + rootElement.setAttribute(XmlConstants.ATTR_ERRORS, "0"); + + long elapsedTimeMillis = testContext.getEndTime() - testContext.getStartTime(); + + rootElement.setAttribute(XmlConstants.ATTR_TIME, "" + (elapsedTimeMillis / 1000.0)); + + String prevClassName = null; + String currentClassName = null; + Element prevSuite = null; + Element currentSuite = null; + int testsInSuite = 0; + int failedTests = 0; + // NOMUTANTS -- Need not to be fixed. + long startTime = 0; + // NOMUTANTS -- Need not to be fixed. + long endTime = 0; + + // go through each test result + for (TestResult tr : allTests) { + prevClassName = currentClassName; + currentClassName = tr.getTestClassName(); + + // as all results are in single array this will create + // testsuite element as in junit. + if (!currentClassName.equals(prevClassName)) { + prevSuite = currentSuite; + currentSuite = doc.createElement(XmlConstants.TESTSUITE); + rootElement.appendChild(currentSuite); + currentSuite.setAttribute(XmlConstants.ATTR_NAME, tr.getTestClassName()); + if (prevSuite != null) { + prevSuite.setAttribute(XmlConstants.ATTR_TESTS, "" + testsInSuite); + prevSuite.setAttribute(XmlConstants.ATTR_FAILURES, "" + failedTests); + prevSuite.setAttribute(XmlConstants.ATTR_ERRORS, "0"); + prevSuite.setAttribute(XmlConstants.ATTR_TIME, "" + (endTime - startTime) / 1000.0); + testsInSuite = 0; + failedTests = 0; + } + startTime = tr.getStartMillis(); + } + endTime = tr.getEndMillis(); + + Element testCaseElement = doc.createElement(XmlConstants.TESTCASE); + elapsedTimeMillis = tr.getEndMillis() - tr.getStartMillis(); + testCaseElement.setAttribute(XmlConstants.ATTR_NAME, tr.getName()); + testCaseElement.setAttribute(XmlConstants.ATTR_CLASSNAME, tr.getTestClassName()); + testCaseElement.setAttribute( + XmlConstants.ATTR_TIME, "" + ((double) elapsedTimeMillis) / 1000); + + // for failure add fail message + if (tr.getStatus() == TestResult.FAILURE) { + failedTests++; + Element nested = doc.createElement(XmlConstants.FAILURE); + testCaseElement.appendChild(nested); + Throwable t = tr.getThrowable(); + if (t != null) { + nested.setAttribute(XmlConstants.ATTR_TYPE, t.getClass().getName()); + String message = t.getMessage(); + if ((message != null) && (message.length() > 0)) { + nested.setAttribute(XmlConstants.ATTR_MESSAGE, message); + } + Text trace = doc.createTextNode(Throwables.getStackTraceAsString(t)); + nested.appendChild(trace); + } + } + currentSuite.appendChild(testCaseElement); + testsInSuite++; + } + + currentSuite.setAttribute(XmlConstants.ATTR_TESTS, "" + testsInSuite); + currentSuite.setAttribute(XmlConstants.ATTR_FAILURES, "" + failedTests); + currentSuite.setAttribute(XmlConstants.ATTR_ERRORS, "0"); + currentSuite.setAttribute(XmlConstants.ATTR_TIME, "" + (endTime - startTime) / 1000.0); + + // Writes to a file + try (BufferedWriter fw = Files.newBufferedWriter(outputFile.toPath(), UTF_8)) { + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.transform(new DOMSource(rootElement), new StreamResult(fw)); + } catch (TransformerException te) { + te.printStackTrace(); + System.err.println("Error while writing out JUnitXML because of " + te); + } catch (IOException ioe) { + ioe.printStackTrace(); + System.err.println("failed to create JUnitXML because of " + ioe); + } + + } catch (ParserConfigurationException pce) { + pce.printStackTrace(); + System.err.println("failed to create JUnitXML because of " + pce); + } + } + + /** Description of a test suite execution. */ + static interface TestContext { + String getSuiteName(); + + long getEndTime(); + + long getStartTime(); + } + + /** Description of a single test result. */ + static interface TestResult { + String getTestClassName(); + + String getName(); + + long getStartMillis(); + + long getEndMillis(); + + Throwable getThrowable(); + + int getStatus(); + + public static int FAILURE = 0; + public static int SUCCESS = 1; + } + + /** Elements and attributes for JUnit-style XML doc. */ + private static final class XmlConstants { + static final String TESTSUITE = "testsuite"; + static final String TESTCASE = "testcase"; + static final String FAILURE = "failure"; + static final String ATTR_NAME = "name"; + static final String ATTR_TIME = "time"; + static final String ATTR_ERRORS = "errors"; + static final String ATTR_FAILURES = "failures"; + static final String ATTR_TESTS = "tests"; + static final String ATTR_TYPE = "type"; + static final String ATTR_MESSAGE = "message"; + static final String ATTR_CLASSNAME = "classname"; + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/RegistryUtils.java b/testing/src/main/java/dev/cel/testing/testrunner/RegistryUtils.java new file mode 100644 index 000000000..34bb01e52 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/RegistryUtils.java @@ -0,0 +1,99 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import com.google.common.io.Files; +import com.google.protobuf.DescriptorProtos.FileDescriptorSet; +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.TypeRegistry; +import dev.cel.common.CelDescriptorUtil; +import dev.cel.common.CelDescriptors; +import dev.cel.common.internal.CelDescriptorPool; +import dev.cel.common.internal.DefaultDescriptorPool; +import dev.cel.common.internal.DefaultMessageFactory; +import java.io.File; +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.Set; + +/** Utility class for creating registries from a file descriptor set. */ +final class RegistryUtils { + + private RegistryUtils() {} + + /** Returns the {@link FileDescriptorSet} for the given file descriptor set path. */ + static FileDescriptorSet getFileDescriptorSet(String fileDescriptorSetPath) throws IOException { + return FileDescriptorSet.parseFrom( + Files.toByteArray(new File(fileDescriptorSetPath)), ExtensionRegistry.newInstance()); + } + + /** Returns the {@link TypeRegistry} for the given file descriptor set. */ + static TypeRegistry getTypeRegistry(Set fileDescriptors) throws IOException { + return createTypeRegistry(fileDescriptors); + } + + /** Returns the {@link ExtensionRegistry} for the given file descriptor set. */ + static ExtensionRegistry getExtensionRegistry(Set fileDescriptors) + throws IOException { + return createExtensionRegistry(fileDescriptors); + } + + private static TypeRegistry createTypeRegistry(Set fileDescriptors) { + CelDescriptors allDescriptors = + CelDescriptorUtil.getAllDescriptorsFromFileDescriptor(fileDescriptors); + return TypeRegistry.newBuilder().add(allDescriptors.messageTypeDescriptors()).build(); + } + + private static ExtensionRegistry createExtensionRegistry(Set fileDescriptors) { + ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance(); + + CelDescriptors allDescriptors = + CelDescriptorUtil.getAllDescriptorsFromFileDescriptor(fileDescriptors); + + CelDescriptorPool pool = DefaultDescriptorPool.create(allDescriptors); + + // We need to create a default message factory because there would always be difference in + // reference between the default instance's descriptor and the descriptor in the pool if the + // file descriptor set is created at runtime, therefore + // we need to create a default message factory to get the default instance for each descriptor + // because it falls back to the DynamicMessages. + // + // For more details, see: b/292174333 + DefaultMessageFactory defaultMessageFactory = DefaultMessageFactory.create(pool); + + allDescriptors + .extensionDescriptors() + .forEach( + (descriptorName, descriptor) -> { + if (descriptor.getType().equals(FieldDescriptor.Type.MESSAGE)) { + Message.Builder defaultInstance = + defaultMessageFactory + .newBuilder(descriptor.getMessageType().getFullName()) + .orElseThrow( + () -> + new NoSuchElementException( + "Could not find a default message for: " + + descriptor.getFullName())); + extensionRegistry.add(descriptor, defaultInstance.build()); + } else { + extensionRegistry.add(descriptor); + } + }); + + return extensionRegistry; + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/ResultMatcher.java b/testing/src/main/java/dev/cel/testing/testrunner/ResultMatcher.java new file mode 100644 index 000000000..b308860a2 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/ResultMatcher.java @@ -0,0 +1,81 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import dev.cel.expr.ExprValue; +import com.google.auto.value.AutoOneOf; +import com.google.auto.value.AutoValue; +import dev.cel.bundle.Cel; +import dev.cel.common.types.CelType; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Output; +import java.util.Optional; + +/** Custom result matcher for performing assertions on the result of a CEL test case. */ +public interface ResultMatcher { + + /** Parameters for the result matcher. */ + @AutoValue + public abstract class ResultMatcherParams { + public abstract Optional expectedOutput(); + + public abstract ComputedOutput computedOutput(); + + /** Computed output of the CEL test case. */ + @AutoOneOf(ComputedOutput.Kind.class) + public abstract static class ComputedOutput { + /** Kind of the computed output. */ + public enum Kind { + EXPR_VALUE, + ERROR + } + + public abstract Kind kind(); + + public abstract ExprValue exprValue(); + + public abstract CelEvaluationException error(); + + public static ComputedOutput ofExprValue(ExprValue exprValue) { + return AutoOneOf_ResultMatcher_ResultMatcherParams_ComputedOutput.exprValue(exprValue); + } + + public static ComputedOutput ofError(CelEvaluationException error) { + return AutoOneOf_ResultMatcher_ResultMatcherParams_ComputedOutput.error(error); + } + } + + public abstract CelType resultType(); + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_ResultMatcher_ResultMatcherParams.Builder(); + } + + /** Builder for {@link ResultMatcherParams}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setExpectedOutput(Optional result); + + public abstract Builder setResultType(CelType resultType); + + public abstract Builder setComputedOutput(ComputedOutput computedOutput); + + public abstract ResultMatcherParams build(); + } + } + + void match(ResultMatcherParams params, Cel cel) throws Exception; +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/TestExecutor.java b/testing/src/main/java/dev/cel/testing/testrunner/TestExecutor.java new file mode 100644 index 000000000..386f4e1e5 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/TestExecutor.java @@ -0,0 +1,219 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.ZoneId.systemDefault; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase; +import java.io.File; +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import org.junit.runner.Description; +import org.junit.runner.JUnitCore; +import org.junit.runner.Request; +import org.junit.runner.Result; +import org.junit.runner.Runner; +import org.junit.runner.manipulation.Filter; +import org.junit.runners.model.TestClass; +import org.junit.runners.parameterized.BlockJUnit4ClassRunnerWithParametersFactory; +import org.junit.runners.parameterized.ParametersRunnerFactory; +import org.junit.runners.parameterized.TestWithParameters; + +/** Test executor for running tests using custom runner. */ +public final class TestExecutor { + + private TestExecutor() {} + + private static CelTestSuite readTestSuite(String testSuitePath) + throws IOException, CelTestSuiteException { + switch (testSuitePath.substring(testSuitePath.lastIndexOf(".") + 1)) { + case "textproto": + return CelTestSuiteTextProtoParser.newInstance() + .parse(Files.asCharSource(new File(testSuitePath), UTF_8).read()); + case "yaml": + return CelTestSuiteYamlParser.newInstance() + .parse(Files.asCharSource(new File(testSuitePath), UTF_8).read()); + default: + throw new IllegalArgumentException( + "Unsupported test suite file type: " + testSuitePath); + } + } + + private static class TestContext implements JUnitXmlReporter.TestContext { + + final LocalDate startDate; + LocalDate endDate; + + TestContext() { + startDate = Instant.now().atZone(systemDefault()).toLocalDate(); + } + + @Override + public long getEndTime() { + return endDate.atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli(); + } + + @Override + public long getStartTime() { + return startDate.atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli(); + } + + @Override + public String getSuiteName() { + return "test_suite"; + } + + void done() { + endDate = Instant.now().atZone(systemDefault()).toLocalDate(); + } + } + + private static class TestResult implements JUnitXmlReporter.TestResult { + + long endMillis; + + long startMillis; + + int status; + + final String testName; + + Throwable throwable; + + final String testClassName; + + TestResult(String testName, String testClassName) { + this.testName = testName; + this.startMillis = Instant.now().toEpochMilli(); + this.testClassName = testClassName; + } + + @Override + public String getTestClassName() { + return testClassName; + } + + @Override + public long getEndMillis() { + return endMillis; + } + + @Override + public String getName() { + return testName; + } + + @Override + public long getStartMillis() { + return startMillis; + } + + @Override + public int getStatus() { + return status; + } + + @Override + public Throwable getThrowable() { + return throwable; + } + + private void setEndMillis(long endMillis) { + this.endMillis = endMillis; + } + + private void setStatus(int status) { + this.status = status; + } + + private void setThrowable(Throwable throwable) { + this.throwable = throwable; + } + + private void setStartMillis(long startMillis) { + this.startMillis = startMillis; + } + } + + /** Runs test cases for a given test class and test suite. */ + public static void runTests(Class testClass, String testSuitePath) throws Exception { + + String envXmlFile = System.getenv("XML_OUTPUT_FILE"); + JUnitXmlReporter testReporter = new JUnitXmlReporter(envXmlFile); + TestContext testContext = new TestContext(); + testReporter.onStart(testContext); + + boolean allTestsPassed = true; + + CelTestSuite testSuite = readTestSuite(testSuitePath); + for (CelTestSection testSection : testSuite.sections()) { + for (CelTestCase testCase : testSection.tests()) { + String testName = testSection.name() + "." + testCase.name(); + + Object[] parameter = new Object[] {testCase}; + TestWithParameters test = + new TestWithParameters( + testName, new TestClass(testClass), ImmutableList.copyOf(parameter)); + + TestResult testResult = new TestResult(testName, testClass.getName()); + testReporter.onTestStart(testResult); + testResult.setStartMillis(Instant.now().toEpochMilli()); + + ParametersRunnerFactory factory = new BlockJUnit4ClassRunnerWithParametersFactory(); + Runner runner = factory.createRunnerForTestWithParameters(test); + + JUnitCore junitCore = new JUnitCore(); + Request request = + Request.runner(runner) + .filterWith( + new Filter() { + @Override + public boolean shouldRun(Description description) { + return true; + } + + @Override + public String describe() { + return "Filter to run only test method"; + } + }); + Result result = junitCore.run(request); + testResult.setEndMillis(Instant.now().toEpochMilli()); + + if (result.wasSuccessful()) { + testResult.setStatus(JUnitXmlReporter.TestResult.SUCCESS); + testReporter.onTestSuccess(testResult); + } else { + allTestsPassed = false; + testResult.setStatus(JUnitXmlReporter.TestResult.FAILURE); + testResult.setThrowable(result.getFailures().get(0).getException()); + testReporter.onTestFailure(testResult); + } + } + } + + testContext.done(); + testReporter.onFinish(); + if (!allTestsPassed) { + throw new RuntimeException(testReporter.getNumFailed() + " tests failed"); + } + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/TestRunnerBinary.java b/testing/src/main/java/dev/cel/testing/testrunner/TestRunnerBinary.java new file mode 100644 index 000000000..10d9fa999 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/TestRunnerBinary.java @@ -0,0 +1,32 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +/** Main class for the CEL test runner. */ +public final class TestRunnerBinary { + + private TestRunnerBinary() {} + + public static void main(String[] args) throws Exception { + + String testSuitePath = System.getProperty("test_suite_path"); + String userTestClassName = System.getProperty("user_test_class_name"); + + Class userTestClass = Class.forName(userTestClassName); + + // NOMUTANTS -- To be fixed in b/394771693. Since no assertions result in false positive. + TestExecutor.runTests(userTestClass, testSuitePath); + } +} diff --git a/testing/src/main/java/dev/cel/testing/testrunner/TestRunnerLibrary.java b/testing/src/main/java/dev/cel/testing/testrunner/TestRunnerLibrary.java new file mode 100644 index 000000000..a1591f280 --- /dev/null +++ b/testing/src/main/java/dev/cel/testing/testrunner/TestRunnerLibrary.java @@ -0,0 +1,356 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import static com.google.common.io.Files.asCharSource; +import static dev.cel.testing.utils.ExprValueUtils.fromValue; +import static dev.cel.testing.utils.ExprValueUtils.toExprValue; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.readAllBytes; + +import dev.cel.expr.CheckedExpr; +import dev.cel.expr.ExprValue; +import dev.cel.expr.Value; +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.TextFormat; +import dev.cel.bundle.Cel; +import dev.cel.bundle.CelEnvironment; +import dev.cel.bundle.CelEnvironment.ExtensionConfig; +import dev.cel.bundle.CelEnvironmentException; +import dev.cel.bundle.CelEnvironmentYamlParser; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelDescriptorUtil; +import dev.cel.common.CelOptions; +import dev.cel.common.CelProtoAbstractSyntaxTree; +import dev.cel.common.CelValidationException; +import dev.cel.policy.CelPolicy; +import dev.cel.policy.CelPolicyCompilerFactory; +import dev.cel.policy.CelPolicyParser; +import dev.cel.policy.CelPolicyParserFactory; +import dev.cel.policy.CelPolicyValidationException; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelRuntime.Program; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Input.Binding; +import dev.cel.testing.testrunner.ResultMatcher.ResultMatcherParams; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; + +/** Runner library for creating the environment and running the assertions. */ +public final class TestRunnerLibrary { + TestRunnerLibrary() {} + + private static final Logger logger = Logger.getLogger(TestRunnerLibrary.class.getName()); + + private static final String CEL_EXPR_SYSTEM_PROPERTY = "cel_expr"; + + /** + * Run the assertions for a given raw/checked expression test case. + * + * @param testCase The test case to run. + * @param celTestContext The test context containing the {@link Cel} bundle and other + * configurations. + */ + public static void runTest(CelTestCase testCase, CelTestContext celTestContext) throws Exception { + String celExpression = System.getProperty(CEL_EXPR_SYSTEM_PROPERTY); + CelExprFileSource celExprFileSource = CelExprFileSource.fromFile(celExpression); + evaluateTestCase(testCase, celTestContext, celExprFileSource); + } + + @VisibleForTesting + static void evaluateTestCase( + CelTestCase testCase, CelTestContext celTestContext, CelExprFileSource celExprFileSource) + throws Exception { + celTestContext = extendCelTestContext(celTestContext, celExprFileSource); + CelAbstractSyntaxTree ast; + switch (celExprFileSource.type()) { + case POLICY: + ast = + compilePolicy( + celTestContext.cel(), + celTestContext.celPolicyParser().get(), + readFile(celExprFileSource.value())); + break; + case TEXTPROTO: + case BINARYPB: + ast = readAstFromCheckedExpression(celExprFileSource); + break; + case CEL: + ast = celTestContext.cel().compile(readFile(celExprFileSource.value())).getAst(); + break; + case RAW_EXPR: + ast = celTestContext.cel().compile(celExprFileSource.value()).getAst(); + break; + default: + throw new IllegalArgumentException( + "Unsupported expression type: " + celExprFileSource.type()); + } + evaluate(ast, testCase, celTestContext); + } + + private static CelAbstractSyntaxTree readAstFromCheckedExpression( + CelExprFileSource celExprFileSource) throws IOException { + switch (celExprFileSource.type()) { + case BINARYPB: + byte[] bytes = readAllBytes(Paths.get(celExprFileSource.value())); + CheckedExpr checkedExpr = + CheckedExpr.parseFrom(bytes, ExtensionRegistry.getEmptyRegistry()); + return CelProtoAbstractSyntaxTree.fromCheckedExpr(checkedExpr).getAst(); + case TEXTPROTO: + String content = readFile(celExprFileSource.value()); + CheckedExpr.Builder builder = CheckedExpr.newBuilder(); + TextFormat.merge(content, builder); + return CelProtoAbstractSyntaxTree.fromCheckedExpr(builder.build()).getAst(); + default: + throw new IllegalArgumentException( + "Unsupported expression type: " + celExprFileSource.type()); + } + } + + private static CelTestContext extendCelTestContext( + CelTestContext celTestContext, CelExprFileSource celExprFileSource) + throws CelEnvironmentException, IOException { + CelOptions celOptions = celTestContext.celOptions(); + CelTestContext.Builder celTestContextBuilder = + celTestContext.toBuilder().setCel(extendCel(celTestContext.cel(), celOptions)); + if (celExprFileSource.type().equals(ExpressionFileType.POLICY)) { + celTestContextBuilder.setCelPolicyParser( + celTestContext + .celPolicyParser() + .orElse(CelPolicyParserFactory.newYamlParserBuilder().build())); + } + + return celTestContextBuilder.build(); + } + + private static Cel extendCel(Cel cel, CelOptions celOptions) + throws IOException, CelEnvironmentException { + Cel extendedCel = cel; + + // Add the file descriptor set to the cel object if provided. + // + // Note: This needs to be added first because the config file may contain type information + // regarding proto messages that need to be added to the cel object. + String fileDescriptorSetPath = System.getProperty("file_descriptor_set_path"); + if (fileDescriptorSetPath != null) { + extendedCel = + cel.toCelBuilder() + .addFileTypes(RegistryUtils.getFileDescriptorSet(fileDescriptorSetPath)) + .build(); + } + + CelEnvironment environment = CelEnvironment.newBuilder().build(); + + // Extend the cel object with the config file if provided. + String configPath = System.getProperty("config_path"); + if (configPath != null) { + String configContent = readFile(configPath); + environment = CelEnvironmentYamlParser.newInstance().parse(configContent); + } + + // Policy compiler requires optional support. Add the optional library by default to the + // environment. + return environment.toBuilder() + .addExtensions(ExtensionConfig.of("optional")) + .build() + .extend(extendedCel, celOptions); + } + + /** + * CelExprFileSource is an encapsulation around cel_expr file format argument accepted in + * cel_java_test bzl macro. It either holds a {@link CheckedExpr} in binarypb/textproto format, a + * serialized {@link CelPolicy} file in yaml/celpolicy format or a raw cel expression in cel file + * format or string format. + */ + @AutoValue + abstract static class CelExprFileSource { + + abstract String value(); + + abstract ExpressionFileType type(); + + static CelExprFileSource fromFile(String value) { + return new AutoValue_TestRunnerLibrary_CelExprFileSource( + value, ExpressionFileType.fromFile(value)); + } + } + + enum ExpressionFileType { + BINARYPB, + TEXTPROTO, + POLICY, + CEL, + RAW_EXPR; + + static ExpressionFileType fromFile(String filePath) { + if (filePath.endsWith(".binarypb")) { + return BINARYPB; + } + if (filePath.endsWith(".textproto")) { + return TEXTPROTO; + } + if (filePath.endsWith(".yaml") || filePath.endsWith(".celpolicy")) { + return POLICY; + } + if (filePath.endsWith(".cel")) { + return CEL; + } + if (System.getProperty("is_raw_expr").equals("True")) { + return RAW_EXPR; + } + throw new IllegalArgumentException("Unsupported expression type: " + filePath); + } + } + + private static String readFile(String path) throws IOException { + return asCharSource(new File(path), UTF_8).read(); + } + + private static CelAbstractSyntaxTree compilePolicy( + Cel cel, CelPolicyParser celPolicyParser, String policyContent) + throws CelPolicyValidationException { + CelPolicy celPolicy = celPolicyParser.parse(policyContent); + return CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(celPolicy); + } + + private static void evaluate( + CelAbstractSyntaxTree ast, CelTestCase testCase, CelTestContext celTestContext) + throws Exception { + Cel cel = celTestContext.cel(); + Program program = cel.createProgram(ast); + ExprValue exprValue = null; + CelEvaluationException error = null; + Object evaluationResult = null; + try { + evaluationResult = getEvaluationResult(testCase, celTestContext, program); + exprValue = toExprValue(evaluationResult, ast.getResultType()); + } catch (CelEvaluationException e) { + String errorMessage = + String.format( + "Evaluation failed for test case: %s. Error: %s", testCase.name(), e.getMessage()); + error = new CelEvaluationException(errorMessage, e); + logger.severe(errorMessage); + } + + // Perform the assertion on the result of the evaluation. + ResultMatcherParams.Builder paramsBuilder = + ResultMatcherParams.newBuilder() + .setExpectedOutput(Optional.ofNullable(testCase.output())) + .setResultType(ast.getResultType()); + + if (exprValue != null) { + paramsBuilder.setComputedOutput(ResultMatcherParams.ComputedOutput.ofExprValue(exprValue)); + } else { + paramsBuilder.setComputedOutput(ResultMatcherParams.ComputedOutput.ofError(error)); + } + + celTestContext.resultMatcher().match(paramsBuilder.build(), cel); + } + + private static Object getEvaluationResult( + CelTestCase testCase, CelTestContext celTestContext, Program program) throws Exception { + if (celTestContext.celLateFunctionBindings().isPresent()) { + return program.eval( + getBindings(testCase, celTestContext), celTestContext.celLateFunctionBindings().get()); + } + switch (testCase.input().kind()) { + case CONTEXT_MESSAGE: + return program.eval( + unpackAny( + testCase.input().contextMessage().getTypeUrl(), + testCase.input().contextMessage().getValue(), + System.getProperty("file_descriptor_set_path"))); + case CONTEXT_EXPR: + return program.eval(getEvaluatedContextExpr(testCase, celTestContext)); + case BINDINGS: + return program.eval(getBindings(testCase, celTestContext)); + case NO_INPUT: + return program.eval(celTestContext.variableBindings()); + } + throw new IllegalArgumentException("Unexpected input type: " + testCase.input().kind()); + } + + // TODO: Remove DynamicMessage parsing once default instance generation is fixed. + // + // Dynamic Message parsing is added here to make the default instance generation code OSS + // compatible. Otherwise, we'd have to depend on AnyUtil which is not available in OSS. + // However, the functionality fails in OSS as the generated DynamicMessage descriptor is different + // from the message descriptor. + private static Message unpackAny(String typeUrl, ByteString value, String fileDescriptorSetPath) + throws IOException { + Preconditions.checkNotNull( + fileDescriptorSetPath, + "File descriptor set path is required to unpack Any of type: %s.", + typeUrl); + ImmutableSet fileDescriptors = + CelDescriptorUtil.getFileDescriptorsFromFileDescriptorSet( + RegistryUtils.getFileDescriptorSet(fileDescriptorSetPath)); + Descriptor descriptor = + RegistryUtils.getTypeRegistry(fileDescriptors).getDescriptorForTypeUrl(typeUrl); + return DynamicMessage.parseFrom( + descriptor, value, RegistryUtils.getExtensionRegistry(fileDescriptors)); + } + + private static Message getEvaluatedContextExpr( + CelTestCase testCase, CelTestContext celTestContext) + throws CelEvaluationException, CelValidationException { + try { + return (Message) evaluateInput(celTestContext.cel(), testCase.input().contextExpr()); + } catch (ClassCastException e) { + throw new IllegalArgumentException("Context expression must evaluate to a proto message.", e); + } + } + + private static ImmutableMap getBindings( + CelTestCase testCase, CelTestContext celTestContext) + throws CelEvaluationException, CelValidationException, IOException { + Cel cel = celTestContext.cel(); + ImmutableMap.Builder inputBuilder = ImmutableMap.builder(); + for (Map.Entry entry : testCase.input().bindings().entrySet()) { + if (entry.getValue().kind().equals(Binding.Kind.EXPR)) { + inputBuilder.put(entry.getKey(), evaluateInput(cel, entry.getValue().expr())); + } else { + Object value; + if (entry.getValue().value() instanceof Value) { + value = fromValue((Value) entry.getValue().value()); + } else { + value = entry.getValue().value(); + } + inputBuilder.put(entry.getKey(), value); + } + } + return inputBuilder.buildOrThrow(); + } + + private static Object evaluateInput(Cel cel, String expr) + throws CelEvaluationException, CelValidationException { + CelAbstractSyntaxTree exprInputAst = cel.compile(expr).getAst(); + return cel.createProgram(exprInputAst).eval(); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/BUILD.bazel b/testing/src/test/java/dev/cel/testing/testrunner/BUILD.bazel new file mode 100644 index 000000000..9512c1614 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/BUILD.bazel @@ -0,0 +1,293 @@ +load("@rules_java//java:java_library.bzl", "java_library") +load("@rules_java//java:java_test.bzl", "java_test") +load("@rules_proto//proto:defs.bzl", "proto_descriptor_set") +load("//testing/testrunner:cel_java_test.bzl", "cel_java_test") + +package( + default_applicable_licenses = ["//:license"], + default_testonly = 1, + default_visibility = [ + "//testing/testrunner:__pkg__", + ], +) + +proto_descriptor_set( + name = "test_all_types_fds", + deps = [ + "@cel_spec//proto/cel/expr/conformance/proto2:test_all_types_proto", + "@cel_spec//proto/cel/expr/conformance/proto3:test_all_types_proto", + ], +) + +# Since the user test class is triggered by the cel_test_runner rule, we should not add it to the +# junit4_test_suite. +# This is just a sample test class for the cel_test_runner rule. +java_library( + name = "user_test", + srcs = ["UserTest.java"], + deps = [ + "//bundle:cel", + "//common/types", + "//testing/testrunner:cel_test_context", + "//testing/testrunner:cel_user_test_template", + "@maven//:junit_junit", + ], +) + +# This is just a sample test class for the cel_test_runner rule. +java_library( + name = "env_config_user_test", + srcs = ["EnvConfigUserTest.java"], + deps = [ + "//bundle:cel", + "//testing/testrunner:cel_test_context", + "//testing/testrunner:cel_user_test_template", + "@maven//:junit_junit", + ], +) + +java_library( + name = "late_function_binding_user_test", + srcs = ["LateFunctionBindingUserTest.java"], + deps = [ + "//runtime", + "//runtime:function_binding", + "//testing/testrunner:cel_test_context", + "//testing/testrunner:cel_user_test_template", + "@maven//:junit_junit", + ], +) + +java_library( + name = "custom_variable_binding_user_test", + srcs = ["CustomVariableBindingUserTest.java"], + deps = [ + "//testing/testrunner:cel_test_context", + "//testing/testrunner:cel_user_test_template", + "@cel_spec//proto/cel/expr/conformance/proto2:test_all_types_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:junit_junit", + ], +) + +java_library( + name = "context_pb_user_test", + srcs = ["ContextPbUserTest.java"], + deps = [ + "//bundle:cel", + "//checker:proto_type_mask", + "//testing/testrunner:cel_test_context", + "//testing/testrunner:cel_user_test_template", + "@cel_spec//proto/cel/expr/conformance/proto3:test_all_types_java_proto", + "@maven//:junit_junit", + ], +) + +java_test( + name = "junit_xml_reporter_test", + srcs = ["JUnitXmlReporterTest.java"], + test_class = "dev.cel.testing.testrunner.JUnitXmlReporterTest", + deps = [ + "//:java_truth", + "//testing/testrunner:junit_xml_reporter", + "@maven//:junit_junit", + "@maven//:org_mockito_mockito_core", + ], +) + +java_test( + name = "default_result_matcher_test", + srcs = ["DefaultResultMatcherTest.java"], + test_class = "dev.cel.testing.testrunner.DefaultResultMatcherTest", + deps = [ + "//:java_truth", + "//bundle:cel", + "//common/types", + "//runtime", + "//testing/testrunner:cel_test_suite", + "//testing/testrunner:default_result_matcher", + "//testing/testrunner:result_matcher", + "@cel_spec//proto/cel/expr:expr_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_testparameterinjector_test_parameter_injector", + "@maven//:junit_junit", + ], +) + +java_test( + name = "cel_test_suite_yaml_parser_test", + srcs = ["CelTestSuiteYamlParserTest.java"], + test_class = "dev.cel.testing.testrunner.CelTestSuiteYamlParserTest", + deps = [ + "//:java_truth", + "//testing/testrunner:cel_test_suite", + "//testing/testrunner:cel_test_suite_exception", + "//testing/testrunner:cel_test_suite_yaml_parser", + "@maven//:com_google_guava_guava", + "@maven//:com_google_testparameterinjector_test_parameter_injector", + "@maven//:junit_junit", + ], +) + +java_test( + name = "cel_test_suite_textproto_parser_test", + srcs = ["CelTestSuiteTextprotoParserTest.java"], + test_class = "dev.cel.testing.testrunner.CelTestSuiteTextprotoParserTest", + deps = [ + "//:java_truth", + "//testing/testrunner:cel_test_suite_exception", + "//testing/testrunner:cel_test_suite_text_proto_parser", + "@cel_spec//proto/cel/expr/conformance/test:suite_java_proto", + "@maven//:junit_junit", + ], +) + +cel_java_test( + name = "late_function_binding_test_runner_sample", + cel_expr = "late_function_binding/policy.yaml", + config = "late_function_binding/config.yaml", + test_class = "dev.cel.testing.testrunner.LateFunctionBindingUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":late_function_binding_user_test", + test_suite = "late_function_binding/tests.yaml", +) + +cel_java_test( + name = "test_runner_sample_yaml", + cel_expr = "nested_rule/policy.yaml", + file_descriptor_set = "//testing/src/test/java/dev/cel/testing/testrunner:test_all_types_fds", + test_class = "dev.cel.testing.testrunner.UserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":user_test", + test_suite = "nested_rule/testrunner_tests.yaml", +) + +# TODO: Add this testcase back to OSS once the default instance issue is fixed. + +cel_java_test( + name = "test_runner_yaml_sample_with_eval_error", + cel_expr = "nested_rule/eval_error_policy.yaml", + config = "nested_rule/eval_error_config.yaml", + file_descriptor_set = ":test_all_types_fds", + test_class = "dev.cel.testing.testrunner.EnvConfigUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":env_config_user_test", + test_suite = "nested_rule/eval_error_tests.yaml", +) + +cel_java_test( + name = "context_pb_user_test_runner_sample", + cel_expr = "context_pb/policy.yaml", + config = "context_pb/config.yaml", + file_descriptor_set = ":test_all_types_fds", + test_class = "dev.cel.testing.testrunner.ContextPbUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":context_pb_user_test", + test_suite = "context_pb/tests.yaml", +) + +cel_java_test( + name = "additional_config_test_runner_sample", + cel_expr = "nested_rule/policy.yaml", + config = "nested_rule/config.yaml", + file_descriptor_set = ":test_all_types_fds", + # TODO: Derive the test class name. + test_class = "dev.cel.testing.testrunner.EnvConfigUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":env_config_user_test", + test_suite = "nested_rule/testrunner_tests.textproto", +) + +cel_java_test( + name = "test_runner_sample", + cel_expr = "nested_rule/policy.yaml", + file_descriptor_set = ":test_all_types_fds", + # TODO: Derive the test class name. + test_class = "dev.cel.testing.testrunner.UserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":user_test", + test_suite = "nested_rule/testrunner_tests.textproto", +) + +cel_java_test( + name = "test_runner_sample_with_expr_value_output", + cel_expr = "expr_value_output/policy.yaml", + test_class = "dev.cel.testing.testrunner.UserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":user_test", + test_suite = "expr_value_output/tests.textproto", +) + +cel_java_test( + name = "test_runner_sample_with_eval_error", + cel_expr = "nested_rule/eval_error_policy.yaml", + config = "nested_rule/eval_error_config.yaml", + file_descriptor_set = ":test_all_types_fds", + test_class = "dev.cel.testing.testrunner.EnvConfigUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":env_config_user_test", + test_suite = "nested_rule/eval_error_tests.textproto", +) + +cel_java_test( + name = "late_function_binding_test_runner_textproto_sample", + cel_expr = "late_function_binding/policy.yaml", + config = "late_function_binding/config.yaml", + test_class = "dev.cel.testing.testrunner.LateFunctionBindingUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":late_function_binding_user_test", + test_suite = "late_function_binding/tests.textproto", +) + +cel_java_test( + name = "context_message_user_test_runner_textproto_sample", + cel_expr = "context_pb/policy.yaml", + config = "context_pb/config.yaml", + file_descriptor_set = ":test_all_types_fds", + test_class = "dev.cel.testing.testrunner.ContextPbUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":context_pb_user_test", + test_suite = "context_pb/context_msg_tests.textproto", +) + +cel_java_test( + name = "context_pb_user_test_runner_textproto_sample", + cel_expr = "context_pb/policy.yaml", + config = "context_pb/config.yaml", + file_descriptor_set = ":test_all_types_fds", + test_class = "dev.cel.testing.testrunner.ContextPbUserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":context_pb_user_test", + test_suite = "context_pb/tests.textproto", +) + +cel_java_test( + name = "raw_expression_test", + cel_expr = "2 + 2 == 4", + is_raw_expr = True, + test_class = "dev.cel.testing.testrunner.UserTest", + test_data_path = "//testing/src/test/resources/expressions", + test_src = ":user_test", + test_suite = "simple_test_case/tests.textproto", +) + +cel_java_test( + name = "extension_as_input_test", + cel_expr = "2 + 2 == 4", + file_descriptor_set = ":test_all_types_fds", + is_raw_expr = True, + test_class = "dev.cel.testing.testrunner.UserTest", + test_data_path = "//testing/src/test/resources/policy", + test_src = ":user_test", + test_suite = "protoextension_value_as_input/tests.textproto", +) + +cel_java_test( + name = "expression_cel_file_test", + cel_expr = "simple_test_case/simple_expression.cel", + test_class = "dev.cel.testing.testrunner.UserTest", + test_data_path = "//testing/src/test/resources/expressions", + test_src = ":user_test", + test_suite = "simple_test_case/tests.textproto", +) diff --git a/testing/src/test/java/dev/cel/testing/testrunner/CelTestSuiteTextprotoParserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/CelTestSuiteTextprotoParserTest.java new file mode 100644 index 000000000..4603cc845 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/CelTestSuiteTextprotoParserTest.java @@ -0,0 +1,60 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import dev.cel.expr.conformance.test.InputContext; +import dev.cel.expr.conformance.test.InputValue; +import dev.cel.expr.conformance.test.TestCase; +import dev.cel.expr.conformance.test.TestSection; +import dev.cel.expr.conformance.test.TestSuite; +import java.io.IOException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class CelTestSuiteTextprotoParserTest { + + @Test + public void parseTestSuite_illegalInput_failure() throws IOException { + TestSuite testSuite = + TestSuite.newBuilder() + .setName("some_test_suite") + .setDescription("Some test suite") + .addSections( + TestSection.newBuilder() + .setName("section_name") + .addTests( + TestCase.newBuilder() + .setName("test_case_name") + .setDescription("Some test case") + .putInput("test_key", InputValue.getDefaultInstance()) + .setInputContext(InputContext.getDefaultInstance()) + .build()) + .build()) + .build(); + + CelTestSuiteException exception = + assertThrows( + CelTestSuiteException.class, + () -> CelTestSuiteTextProtoParser.parseCelTestSuite(testSuite)); + + assertThat(exception) + .hasMessageThat() + .contains("Test case: test_case_name cannot have both input map and input context."); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/CelTestSuiteYamlParserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/CelTestSuiteYamlParserTest.java new file mode 100644 index 000000000..359c1cee3 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/CelTestSuiteYamlParserTest.java @@ -0,0 +1,470 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package dev.cel.testing.testrunner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Input.Binding; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelTestSuiteYamlParserTest { + + private static final CelTestSuiteYamlParser CEL_TEST_SUITE_YAML_PARSER = + CelTestSuiteYamlParser.newInstance(); + + @Test + public void parseTestSuite_withBindingsAsInput_success() throws CelTestSuiteException { + String testSuiteYamlContent = + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value:\n" + + " - nested_key: true\n" + + " output:\n" + + " value: 'test_result_value'\n" + + " - name: 'test_case_name_2'\n" + + " description: 'test_case_description_2'\n" + + " input:\n" + + " test_key:\n" + + " value: 1\n" + + " output:\n" + + " value: 2.20\n"; + + CelTestSuite testSuite = CEL_TEST_SUITE_YAML_PARSER.parse(testSuiteYamlContent); + CelTestSuite expectedTestSuite = + CelTestSuite.newBuilder() + .setSource(testSuite.source().get()) + .setName("test_suite_name") + .setDescription("test_suite_description") + .setSections( + ImmutableSet.of( + CelTestSuite.CelTestSection.newBuilder() + .setName("test_section_name") + .setDescription("test_section_description") + .setTests( + ImmutableSet.of( + CelTestSuite.CelTestSection.CelTestCase.newBuilder() + .setName("test_case_name") + .setDescription("test_case_description") + .setInput( + CelTestSuite.CelTestSection.CelTestCase.Input.ofBindings( + ImmutableMap.of( + "test_key", + Binding.ofValue( + ImmutableList.of( + ImmutableMap.of("nested_key", true)))))) + .setOutput( + CelTestSuite.CelTestSection.CelTestCase.Output + .ofResultValue("test_result_value")) + .build(), + CelTestSuite.CelTestSection.CelTestCase.newBuilder() + .setName("test_case_name_2") + .setDescription("test_case_description_2") + .setInput( + CelTestSuite.CelTestSection.CelTestCase.Input.ofBindings( + ImmutableMap.of("test_key", Binding.ofValue(1)))) + .setOutput( + CelTestSuite.CelTestSection.CelTestCase.Output + .ofResultValue(2.20)) + .build())) + .build())) + .build(); + + assertThat(testSuite).isEqualTo(expectedTestSuite); + assertThat(testSuite.source().get().getPositionsMap()).isNotEmpty(); + } + + @Test + public void parseTestSuite_withExprAsOutput_success() throws CelTestSuiteException { + String testSuiteYamlContent = + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " expr: 'some_value'\n" + + " output:\n" + + " expr: '1 == 1'\n"; + + CelTestSuite testSuite = CEL_TEST_SUITE_YAML_PARSER.parse(testSuiteYamlContent); + CelTestSuite expectedTestSuite = + CelTestSuite.newBuilder() + .setSource(testSuite.source().get()) + .setName("test_suite_name") + .setDescription("test_suite_description") + .setSections( + ImmutableSet.of( + CelTestSuite.CelTestSection.newBuilder() + .setName("test_section_name") + .setDescription("test_section_description") + .setTests( + ImmutableSet.of( + CelTestSuite.CelTestSection.CelTestCase.newBuilder() + .setName("test_case_name") + .setDescription("test_case_description") + .setInput( + CelTestSuite.CelTestSection.CelTestCase.Input.ofBindings( + ImmutableMap.of( + "test_key", Binding.ofExpr("some_value")))) + .setOutput( + CelTestSuite.CelTestSection.CelTestCase.Output.ofResultExpr( + "1 == 1")) + .build())) + .build())) + .build(); + + assertThat(testSuite).isEqualTo(expectedTestSuite); + assertThat(testSuite.source().get().getPositionsMap()).isNotEmpty(); + } + + @Test + public void parseTestSuite_failure_throwsException( + @TestParameter TestSuiteYamlParsingErrorTestCase testCase) throws CelTestSuiteException { + CelTestSuiteException celTestSuiteException = + assertThrows( + CelTestSuiteException.class, + () -> CEL_TEST_SUITE_YAML_PARSER.parse(testCase.testSuiteYamlContent)); + + assertThat(celTestSuiteException).hasMessageThat().contains(testCase.expectedErrorMessage); + } + + private enum TestSuiteYamlParsingErrorTestCase { + TEST_SUITE_WITH_MISALIGNED_NAME_TAG( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + "name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key: 'test_value'\n" + + " value: 'test_result_value'\n", + "YAML document is malformed: while parsing a block mapping\n" + + " in 'reader', line 1, column 1:\n" + + " name: 'test_suite_name'\n" + + " ^"), + TEST_SUITE_WITH_ILLEGAL_TEST_SUITE_TAG( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n" + + "unknown_tag: 'test_value'\n", + "ERROR: :14:1: Unknown test suite tag: unknown_tag\n" + + " | unknown_tag: 'test_value'\n" + + " | ^"), + TEST_SUITE_WITH_ILLEGAL_TEST_SECTION_TAG( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " unknown_tag: 'test_value'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :5:3: Unknown test section tag: unknown_tag\n" + + " | unknown_tag: 'test_value'\n" + + " | ..^"), + TEST_SUITE_WITH_ILLEGAL_TEST_CASE_OUTPUT_TAG( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " unknown_tag: 'test_result_value'\n", + "ERROR: :13:7: Unknown output tag: unknown_tag\n" + + " | unknown_tag: 'test_result_value'\n" + + " | ......^"), + TEST_SUITE_WITH_ILLEGAL_TEST_CASE_TAG( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n" + + " unknown_tag: 'test_value'\n", + "ERROR: :14:5: Unknown test case tag: unknown_tag\n" + + " | unknown_tag: 'test_value'\n" + + " | ....^"), + ILLEGAL_TEST_SUITE_WITH_SECTION_NOT_LIST( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + " name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :4:3: Got yaml node type tag:yaml.org,2002:map, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | name: 'test_section_name'\n" + + " | ..^\n" + + "ERROR: :4:3: Sections is not a list: tag:yaml.org,2002:map\n" + + " | name: 'test_section_name'\n" + + " | ..^"), + ILLEGAL_TEST_SUITE_WITH_TEST_SUITE_NOT_MAP( + "- name: 'test_suite_name'\n" + + "- description: 'test_suite_description'\n" + + "- sections:\n" + + " name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :1:1: Got yaml node type tag:yaml.org,2002:seq, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - name: 'test_suite_name'\n" + + " | ^\n" + + "ERROR: :1:1: Unknown test suite type: tag:yaml.org,2002:seq\n" + + " | - name: 'test_suite_name'\n" + + " | ^"), + ILLEGAL_TEST_SUITE_WITH_OUTPUT_NOT_MAP( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " - value: 'test_result_value'\n", + "ERROR: :13:7: Got yaml node type tag:yaml.org,2002:seq, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - value: 'test_result_value'\n" + + " | ......^\n" + + "ERROR: :13:7: Output is not a map: tag:yaml.org,2002:seq\n" + + " | - value: 'test_result_value'\n" + + " | ......^"), + ILLEGAL_TEST_SUITE_WITH_MORE_THAN_ONE_INPUT_VALUES_AGAINST_KEY( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " value: 'test_value_2'\n" + + " expr: 'test_expr'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :11:11: Input binding node must have exactly one value:" + + " tag:yaml.org,2002:map\n" + + " | value: 'test_value'\n" + + " | ..........^"), + ILLEGAL_TEST_SUITE_WITH_UNKNOWN_INPUT_BINDING_VALUE_TAG( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " something: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :11:11: Unknown input binding value tag: something\n" + + " | something: 'test_value'\n" + + " | ..........^"), + ILLEGAL_TEST_SUITE_WITH_BINDINGS_NOT_MAP( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " - test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :10:10: Got yaml node type tag:yaml.org,2002:seq, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - test_key:\n" + + " | .........^\n" + + "ERROR: :10:10: Input is not a map: tag:yaml.org,2002:seq\n" + + " | - test_key:\n" + + " | .........^"), + ILLEGAL_TEST_SUITE_WITH_ILLEGAL_BINDINGS_VALUE( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " - value\n" + + " - 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :11:13: Got yaml node type tag:yaml.org,2002:seq, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - value\n" + + " | ............^\n" + + "ERROR: :11:13: Input binding node is not a map: tag:yaml.org,2002:seq\n" + + " | - value\n" + + " | ............^"), + ILLEGAL_TEST_SUITE_WITH_ILLEGAL_CONTEXT_EXPR_VALUE( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " context_expr:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :10:9: Got yaml node type tag:yaml.org,2002:map, wanted type(s)" + + " [tag:yaml.org,2002:str]\n" + + " | test_key:\n" + + " | ........^\n" + + "ERROR: :10:9: Input context is not a string: tag:yaml.org,2002:map\n" + + " | test_key:\n" + + " | ........^"), + ILLEGAL_TEST_SUITE_WITH_TESTS_NOT_LIST( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "sections:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :7:5: Got yaml node type tag:yaml.org,2002:map, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | name: 'test_case_name'\n" + + " | ....^\n" + + "ERROR: :7:5: Tests is not a list: tag:yaml.org,2002:map\n" + + " | name: 'test_case_name'\n" + + " | ....^"), + TEST_SUITE_WITH_ILLEGAL_TEST_SUITE_FORMAT( + "- name: 'test_suite_name'\n" + + "- name: 'test_section_name'\n" + + "- name: 'test_section_name_2'\n", + "ERROR: :1:1: Unknown test suite type: tag:yaml.org,2002:seq\n" + + " | - name: 'test_suite_name'\n" + + " | ^"), + TEST_SUITE_WITH_ILLEGAL_SECTION_TYPE( + "name: 'test_suite_name'\n" + + "description: 'test_suite_description'\n" + + "1:\n" + + "- name: 'test_section_name'\n" + + " description: 'test_section_description'\n" + + " tests:\n" + + " - name: 'test_case_name'\n" + + " description: 'test_case_description'\n" + + " input:\n" + + " test_key:\n" + + " value: 'test_value'\n" + + " output:\n" + + " value: 'test_result_value'\n", + "ERROR: :3:1: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | 1:\n" + + " | ^"), + ; + + private final String testSuiteYamlContent; + private final String expectedErrorMessage; + + TestSuiteYamlParsingErrorTestCase(String testSuiteYamlContent, String expectedErrorMessage) { + this.testSuiteYamlContent = testSuiteYamlContent; + this.expectedErrorMessage = expectedErrorMessage; + } + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/ContextPbUserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/ContextPbUserTest.java new file mode 100644 index 000000000..0270cc52d --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/ContextPbUserTest.java @@ -0,0 +1,44 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import dev.cel.bundle.CelFactory; +import dev.cel.checker.ProtoTypeMask; +import dev.cel.expr.conformance.proto3.TestAllTypes; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * This test demonstrates the use case where the fields of a proto are used as variables in the CEL + * expression i.e. context_expr. + */ +@RunWith(Parameterized.class) +public class ContextPbUserTest extends CelUserTestTemplate { + + // TODO: Add support for context_expr and remove the need to add the proto type masks + // explicitly. + public ContextPbUserTest() { + super( + CelTestContext.newBuilder() + .setCel( + CelFactory.standardCelBuilder() + .addFileTypes(TestAllTypes.getDescriptor().getFile()) + .addProtoTypeMasks( + ProtoTypeMask.ofAllFields(TestAllTypes.getDescriptor().getFullName()) + .withFieldsAsVariableDeclarations()) + .build()) + .build()); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/CustomVariableBindingUserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/CustomVariableBindingUserTest.java new file mode 100644 index 000000000..707b5eef9 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/CustomVariableBindingUserTest.java @@ -0,0 +1,43 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.Any; +import dev.cel.expr.conformance.proto2.TestAllTypes; +import dev.cel.expr.conformance.proto2.TestAllTypesExtensions; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * This test demonstrates the use case where the custom variable bindings are being provided + * programmatically for using extensions. + */ +@RunWith(Parameterized.class) +public class CustomVariableBindingUserTest extends CelUserTestTemplate { + + public CustomVariableBindingUserTest() { + super( + CelTestContext.newBuilder() + .setVariableBindings( + ImmutableMap.of( + "spec", + Any.pack( + TestAllTypes.newBuilder() + .setExtension(TestAllTypesExtensions.int32Ext, 1) + .build()))) + .build()); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/DefaultResultMatcherTest.java b/testing/src/test/java/dev/cel/testing/testrunner/DefaultResultMatcherTest.java new file mode 100644 index 000000000..41677c16c --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/DefaultResultMatcherTest.java @@ -0,0 +1,131 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import dev.cel.expr.ExprValue; +import dev.cel.expr.Value; +import com.google.common.collect.ImmutableList; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.bundle.CelFactory; +import dev.cel.common.types.SimpleType; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase.Output; +import dev.cel.testing.testrunner.ResultMatcher.ResultMatcherParams; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public class DefaultResultMatcherTest { + + private static final DefaultResultMatcher MATCHER = new DefaultResultMatcher(); + + @Test + public void match_resultExprEvaluationError_failure() throws Exception { + ResultMatcherParams params = + ResultMatcherParams.newBuilder() + .setExpectedOutput(Optional.of(Output.ofResultExpr("2 / 0"))) + .setComputedOutput( + ResultMatcherParams.ComputedOutput.ofExprValue( + ExprValue.newBuilder() + .setValue(Value.newBuilder().setInt64Value(0).build()) + .build())) + .setResultType(SimpleType.INT) + .build(); + + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> MATCHER.match(params, CelFactory.standardCelBuilder().build())); + + assertThat(thrown) + .hasMessageThat() + .contains("Failed to evaluate result_expr: evaluation error at :2: / by zero"); + } + + @Test + public void match_expectedExprValueForResultExprOutputAndComputedEvalError_failure() + throws Exception { + ResultMatcherParams params = + ResultMatcherParams.newBuilder() + .setExpectedOutput(Optional.of(Output.ofResultExpr("x + y"))) + .setComputedOutput( + ResultMatcherParams.ComputedOutput.ofError( + new CelEvaluationException("evaluation error"))) + .setResultType(SimpleType.INT) + .build(); + + AssertionError thrown = + assertThrows( + AssertionError.class, + () -> MATCHER.match(params, CelFactory.standardCelBuilder().build())); + + assertThat(thrown).hasMessageThat().contains("Error: evaluation error"); + } + + @Test + public void match_expectedExprValueAndComputedEvalError_failure() throws Exception { + ResultMatcherParams params = + ResultMatcherParams.newBuilder() + .setExpectedOutput( + Optional.of( + Output.ofResultValue( + ExprValue.newBuilder() + .setValue(Value.newBuilder().setInt64Value(3).build()) + .build()))) + .setComputedOutput( + ResultMatcherParams.ComputedOutput.ofError( + new CelEvaluationException("evaluation error"))) + .setResultType(SimpleType.INT) + .build(); + + AssertionError thrown = + assertThrows( + AssertionError.class, + () -> MATCHER.match(params, CelFactory.standardCelBuilder().build())); + + assertThat(thrown).hasMessageThat().contains("Error: evaluation error"); + } + + @Test + public void match_expectedEvalErrorAndComputedExprValue_failure() throws Exception { + ResultMatcherParams params = + ResultMatcherParams.newBuilder() + .setExpectedOutput( + Optional.of(Output.ofEvalError(ImmutableList.of("evaluation error")))) + .setComputedOutput( + ResultMatcherParams.ComputedOutput.ofExprValue( + ExprValue.newBuilder() + .setValue(Value.newBuilder().setInt64Value(3).build()) + .build())) + .setResultType(SimpleType.INT) + .build(); + + AssertionError thrown = + assertThrows( + AssertionError.class, + () -> MATCHER.match(params, CelFactory.standardCelBuilder().build())); + + assertThat(thrown) + .hasMessageThat() + .contains( + "Evaluation was successful but no value was provided. Computed output: value {\n" + + " int64_value: 3\n" + + "}"); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/EnvConfigUserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/EnvConfigUserTest.java new file mode 100644 index 000000000..99b2d1f79 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/EnvConfigUserTest.java @@ -0,0 +1,31 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import dev.cel.bundle.CelFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * This test demonstrates the use case where the declarations are provided in the environment config + * file. + */ +@RunWith(Parameterized.class) +public class EnvConfigUserTest extends CelUserTestTemplate { + + public EnvConfigUserTest() { + super(CelTestContext.newBuilder().setCel(CelFactory.standardCelBuilder().build()).build()); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/JUnitXmlReporterTest.java b/testing/src/test/java/dev/cel/testing/testrunner/JUnitXmlReporterTest.java new file mode 100644 index 000000000..efdd9fedc --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/JUnitXmlReporterTest.java @@ -0,0 +1,126 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.when; + +import dev.cel.testing.testrunner.JUnitXmlReporter.TestContext; +import dev.cel.testing.testrunner.JUnitXmlReporter.TestResult; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +@RunWith(JUnit4.class) +public class JUnitXmlReporterTest { + + private static final String SUITE_NAME = "TestSuiteName"; + private static final String TEST_CLASS_NAME = "TestClass1"; + private static final String TEST_METHOD_NAME = "testMethod1"; + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + @Mock private TestContext context; + @Mock private TestResult result1; + @Mock private TestResult result2; + @Mock private TestResult resultFailure; + + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testGenerateReport_success() throws IOException { + String outputFileName = "test-report.xml"; + File subFolder = tempFolder.newFolder("subFolder"); + File outputFile = new File(subFolder.getAbsolutePath(), outputFileName); + JUnitXmlReporter reporter = new JUnitXmlReporter(outputFile.getAbsolutePath()); + long startTime = 100L; + long test1EndTime = startTime + 400; + long endTime = startTime + 900; + + when(context.getSuiteName()).thenReturn(SUITE_NAME); + when(context.getStartTime()).thenReturn(startTime); + when(context.getEndTime()).thenReturn(endTime); + reporter.onStart(context); + + when(result1.getTestClassName()).thenReturn(TEST_CLASS_NAME); + when(result1.getName()).thenReturn(TEST_METHOD_NAME); + when(result1.getStartMillis()).thenReturn(startTime); + when(result1.getEndMillis()).thenReturn(test1EndTime); + when(result1.getStatus()).thenReturn(JUnitXmlReporter.TestResult.SUCCESS); + reporter.onTestSuccess(result1); + + when(result2.getTestClassName()).thenReturn("TestClass2"); + when(result2.getName()).thenReturn("testMethod2"); + when(result2.getStartMillis()).thenReturn(test1EndTime); + when(result2.getEndMillis()).thenReturn(endTime); + when(result2.getStatus()).thenReturn(JUnitXmlReporter.TestResult.SUCCESS); + reporter.onTestSuccess(result2); + + reporter.onFinish(); + assertThat(outputFile.exists()).isTrue(); + String concatenatedFileContent = String.join("\n", Files.readAllLines(outputFile.toPath())); + + assertThat(concatenatedFileContent) + .contains( + ""); + + outputFile.delete(); + } + + @Test + public void testGenerateReport_failure() throws IOException { + String outputFileName = "test-report.xml"; + File subFolder = tempFolder.newFolder("subFolder"); + File outputFile = new File(subFolder.getAbsolutePath(), outputFileName); + JUnitXmlReporter reporter = new JUnitXmlReporter(outputFile.getAbsolutePath()); + + when(context.getSuiteName()).thenReturn(SUITE_NAME); + when(context.getStartTime()).thenReturn(0L); + when(context.getEndTime()).thenReturn(1000L); + reporter.onStart(context); + + when(resultFailure.getTestClassName()).thenReturn(TEST_CLASS_NAME); + when(resultFailure.getName()).thenReturn(TEST_METHOD_NAME); + when(resultFailure.getStartMillis()).thenReturn(0L); + when(resultFailure.getEndMillis()).thenReturn(500L); + when(resultFailure.getStatus()).thenReturn(JUnitXmlReporter.TestResult.FAILURE); + Throwable throwable = new RuntimeException("Test Exception"); + when(resultFailure.getThrowable()).thenReturn(throwable); + reporter.onTestFailure(resultFailure); + reporter.onFinish(); + + assertThat(reporter.getNumFailed()).isEqualTo(1); + assertThat(outputFile.exists()).isTrue(); + + String concatenatedFileContent = String.join("\n", Files.readAllLines(outputFile.toPath())); + + assertThat(concatenatedFileContent).contains("failures=\"1\""); + assertThat(concatenatedFileContent).contains("failure message=\"Test Exception\""); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/LateFunctionBindingUserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/LateFunctionBindingUserTest.java new file mode 100644 index 000000000..72992be3f --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/LateFunctionBindingUserTest.java @@ -0,0 +1,35 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import dev.cel.runtime.CelFunctionBinding; +import dev.cel.runtime.CelLateFunctionBindings; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** This test demonstrates the use case where the function bindings are provided at eval stage. */ +@RunWith(Parameterized.class) +public class LateFunctionBindingUserTest extends CelUserTestTemplate { + + public LateFunctionBindingUserTest() { + super( + CelTestContext.newBuilder() + .setCelLateFunctionBindings( + CelLateFunctionBindings.from( + CelFunctionBinding.from("foo_id", String.class, (String a) -> a.equals("foo")), + CelFunctionBinding.from("bar_id", String.class, (String a) -> a.equals("bar")))) + .build()); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/TestRunnerLibraryTest.java b/testing/src/test/java/dev/cel/testing/testrunner/TestRunnerLibraryTest.java new file mode 100644 index 000000000..f79cb7067 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/TestRunnerLibraryTest.java @@ -0,0 +1,231 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.Any; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import com.google.testing.util.TestUtil; +import dev.cel.bundle.CelFactory; +import dev.cel.checker.ProtoTypeMask; +import dev.cel.expr.conformance.proto3.TestAllTypes; +import dev.cel.testing.testrunner.CelTestSuite.CelTestSection.CelTestCase; +import dev.cel.testing.testrunner.TestRunnerLibrary.CelExprFileSource; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public class TestRunnerLibraryTest { + + @Before + public void setUp() { + System.setProperty("is_raw_expr", "False"); + } + + @Test + public void runPolicyTest_simpleBooleanOutput() throws Exception { + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("simple_output_test_case") + .setDescription("simple_output_test_case_description") + .setInput(CelTestSuite.CelTestSection.CelTestCase.Input.ofBindings(ImmutableMap.of())) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofResultValue(false)) + .build(); + CelExprFileSource celExprFileSource = + CelExprFileSource.fromFile( + TestUtil.getSrcDir() + + "/google3/third_party/java/cel/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml"); + + TestRunnerLibrary.evaluateTestCase( + simpleOutputTestCase, CelTestContext.newBuilder().build(), celExprFileSource); + } + + @Test + public void runPolicyTest_outputMismatch_failureAssertion() throws Exception { + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("output_mismatch_test_case") + .setDescription("output_mismatch_test_case_description") + .setInput(CelTestSuite.CelTestSection.CelTestCase.Input.ofBindings(ImmutableMap.of())) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofResultValue(true)) + .build(); + CelExprFileSource celExprFileSource = + CelExprFileSource.fromFile( + TestUtil.getSrcDir() + + "/google3/third_party/java/cel/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml"); + + AssertionError thrown = + assertThrows( + AssertionError.class, + () -> + TestRunnerLibrary.evaluateTestCase( + simpleOutputTestCase, CelTestContext.newBuilder().build(), celExprFileSource)); + + assertThat(thrown).hasMessageThat().contains("modified: value.bool_value: true -> false"); + } + + @Test + public void runPolicyTest_fileDescriptorSetPathNotSet_failureInUnpackAny() throws Exception { + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("fileDescriptorSetPathNotSet_test") + .setDescription("fileDescriptorSetPathNotSet_test_description") + .setInput( + CelTestSuite.CelTestSection.CelTestCase.Input.ofContextMessage( + Any.pack(TestAllTypes.getDefaultInstance()))) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofResultValue(true)) + .build(); + CelExprFileSource celExprFileSource = + CelExprFileSource.fromFile( + TestUtil.getSrcDir() + + "/google3/third_party/java/cel/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml"); + + NullPointerException thrown = + assertThrows( + NullPointerException.class, + () -> + TestRunnerLibrary.evaluateTestCase( + simpleOutputTestCase, CelTestContext.newBuilder().build(), celExprFileSource)); + + assertThat(thrown) + .hasMessageThat() + .contains( + "File descriptor set path is required to unpack Any of type:" + + " type.googleapis.com/cel.expr.conformance.proto3.TestAllTypes."); + } + + @Test + public void runPolicyTest_evaluatedContextExprNotProtoMessage_failure() throws Exception { + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("output_mismatch_test_case") + .setDescription("output_mismatch_test_case_description") + .setInput(CelTestSuite.CelTestSection.CelTestCase.Input.ofContextExpr("1 > 2")) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofResultValue(false)) + .build(); + CelExprFileSource celExprFileSource = + CelExprFileSource.fromFile( + TestUtil.getSrcDir() + + "/google3/third_party/java/cel/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml"); + + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + TestRunnerLibrary.evaluateTestCase( + simpleOutputTestCase, + CelTestContext.newBuilder() + .setCel( + CelFactory.standardCelBuilder() + .addFileTypes(TestAllTypes.getDescriptor().getFile()) + .addProtoTypeMasks( + ProtoTypeMask.ofAllFields( + TestAllTypes.getDescriptor().getFullName()) + .withFieldsAsVariableDeclarations()) + .build()) + .build(), + celExprFileSource)); + + assertThat(thrown) + .hasMessageThat() + .contains("Context expression must evaluate to a proto message."); + } + + @Test + public void runPolicyTest_evaluationError_failureAssertion() throws Exception { + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("evaluation_error_test_case") + .setDescription("evaluation_error_test_case_description") + .setInput(CelTestSuite.CelTestSection.CelTestCase.Input.ofNoInput()) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofResultValue(false)) + .build(); + CelExprFileSource celExprFileSource = + CelExprFileSource.fromFile( + TestUtil.getSrcDir() + + "/google3/third_party/java/cel/testing/src/test/java/dev/cel/testing/testrunner/resources/eval_error_policy.yaml"); + + AssertionError thrown = + assertThrows( + AssertionError.class, + () -> + TestRunnerLibrary.evaluateTestCase( + simpleOutputTestCase, + CelTestContext.newBuilder() + .setCel(CelFactory.standardCelBuilder().build()) + .build(), + celExprFileSource)); + + assertThat(thrown) + .hasMessageThat() + .contains( + "Error: Evaluation failed for test case: evaluation_error_test_case." + + " Error: evaluation error: / by zero"); + } + + @Test + public void runExpressionTest_outputMismatch_failureAssertion() throws Exception { + System.setProperty( + "cel_expr", + TestUtil.getSrcDir() + + "/google3/third_party/java/cel/testing/src/test/java/dev/cel/testing/testrunner/output.textproto"); + + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("output_mismatch_test_case") + .setDescription("output_mismatch_test_case_description") + .setInput(CelTestSuite.CelTestSection.CelTestCase.Input.ofNoInput()) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofResultValue(true)) + .build(); + + AssertionError thrown = + assertThrows( + AssertionError.class, + () -> + TestRunnerLibrary.runTest( + simpleOutputTestCase, + CelTestContext.newBuilder() + .setCel(CelFactory.standardCelBuilder().build()) + .build())); + + assertThat(thrown).hasMessageThat().contains("modified: value.bool_value: true -> false"); + } + + @Test + public void runTest_illegalFileType_failure() throws Exception { + System.setProperty("cel_expr", "output.txt"); + + CelTestCase simpleOutputTestCase = + CelTestCase.newBuilder() + .setName("illegal_file_type_test_case") + .setDescription("illegal_file_type_test_case_description") + .setInput(CelTestSuite.CelTestSection.CelTestCase.Input.ofNoInput()) + .setOutput(CelTestSuite.CelTestSection.CelTestCase.Output.ofNoOutput()) + .build(); + + IllegalArgumentException thrown = + assertThrows( + IllegalArgumentException.class, + () -> + TestRunnerLibrary.runTest( + simpleOutputTestCase, CelTestContext.newBuilder().build())); + + assertThat(thrown).hasMessageThat().contains("Unsupported expression type: output.txt"); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/UserTest.java b/testing/src/test/java/dev/cel/testing/testrunner/UserTest.java new file mode 100644 index 000000000..6f4c838dd --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/UserTest.java @@ -0,0 +1,39 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.testing.testrunner; + +import dev.cel.bundle.CelFactory; +import dev.cel.common.types.MapType; +import dev.cel.common.types.SimpleType; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * A sample test class for demonstrating the use of the CEL test runner when no config file is + * provided. + */ +@RunWith(Parameterized.class) +public class UserTest extends CelUserTestTemplate { + + public UserTest() { + super( + CelTestContext.newBuilder() + .setCel( + CelFactory.standardCelBuilder() + .addVar("resource", MapType.create(SimpleType.STRING, SimpleType.ANY)) + .build()) + .build()); + } +} diff --git a/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml b/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml new file mode 100644 index 000000000..ea6863dfd --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/resources/empty_policy.yaml @@ -0,0 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rule: + match: + - output: 'false' diff --git a/testing/src/test/java/dev/cel/testing/testrunner/resources/eval_error_policy.yaml b/testing/src/test/java/dev/cel/testing/testrunner/resources/eval_error_policy.yaml new file mode 100644 index 000000000..ef0598b67 --- /dev/null +++ b/testing/src/test/java/dev/cel/testing/testrunner/resources/eval_error_policy.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "evaluation_error_policy" +rule: + match: + - condition: "1/0 == 0" + output: "false" + - output: "true" diff --git a/testing/testrunner/BUILD.bazel b/testing/testrunner/BUILD.bazel new file mode 100644 index 000000000..8febe2b05 --- /dev/null +++ b/testing/testrunner/BUILD.bazel @@ -0,0 +1,84 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_testonly = True, + default_visibility = ["//visibility:public"], +) + +java_library( + name = "cel_user_test_template", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:cel_user_test_template"], +) + +java_library( + name = "junit_xml_reporter", + visibility = ["//:internal"], + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:junit_xml_reporter"], +) + +java_library( + name = "test_runner_library", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:test_runner_library"], +) + +java_library( + name = "test_executor", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:test_executor"], +) + +java_library( + name = "cel_test_suite", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:cel_test_suite"], +) + +java_library( + name = "cel_test_context", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:cel_test_context"], +) + +java_library( + name = "cel_test_suite_yaml_parser", + visibility = ["//:internal"], + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:cel_test_suite_yaml_parser"], +) + +java_library( + name = "cel_test_suite_text_proto_parser", + visibility = ["//:internal"], + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:cel_test_suite_text_proto_parser"], +) + +java_library( + name = "cel_test_suite_exception", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:cel_test_suite_exception"], +) + +java_library( + name = "result_matcher", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:result_matcher"], +) + +java_library( + name = "default_result_matcher", + exports = ["//testing/src/main/java/dev/cel/testing/testrunner:default_result_matcher"], +) + +alias( + name = "test_runner_binary", + actual = "//testing/src/main/java/dev/cel/testing/testrunner:test_runner_binary", +) + +exports_files( + srcs = ["run_testrunner_binary.sh"], +) + +bzl_library( + name = "cel_java_test", + srcs = ["cel_java_test.bzl"], + deps = [ + "@bazel_skylib//lib:paths", + "@rules_java//java:core_rules", + ], +) diff --git a/testing/testrunner/cel_java_test.bzl b/testing/testrunner/cel_java_test.bzl new file mode 100644 index 000000000..a255f5fdc --- /dev/null +++ b/testing/testrunner/cel_java_test.bzl @@ -0,0 +1,152 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rules for triggering the java impl of the CEL test runner.""" + +load("@rules_java//java:java_binary.bzl", "java_binary") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("@bazel_skylib//lib:paths.bzl", "paths") + +def cel_java_test( + name, + test_class, + test_suite, + cel_expr, + test_src, + is_raw_expr = False, + filegroup = "", + config = "", + deps = [], + test_data_path = "", + data = [], + file_descriptor_set = None): + """trigger the java impl of the CEL test runner. + + This rule will generate a java_binary and a run_test rule. This rule will be used to trigger + the java impl of the cel_test rule. + + Args: + name: str name for the generated artifact + test_class: str fully qualified user's test class name. + test_suite: str label of a file containing a test suite. The file should have a .yaml or a + .textproto extension. + cel_expr: cel expression to be evaluated. This could be a raw expression or a compiled + expression or cel policy. + is_raw_expr: bool whether the cel_expr is a raw expression or not. If true, the cel_expr + will be used as is and would not be treated as a file path. + filegroup: str label of a filegroup containing the test suite, the config and the checked + expression. + config: str label of a file containing a google.api.expr.conformance.Environment message. + The file should have the .textproto extension. + test_src: user's test class build target. + deps: list of dependencies for the java_binary rule. + data: list of data dependencies for the java_binary rule. + file_descriptor_set: str label or filename pointing to a file_descriptor_set message. Note: + this must be in binary format. If you need to support a textformat file_descriptor_set, + embed it in the environment file. (default None) + test_data_path: absolute path of the directory containing the test files. This is needed only + if the test files are not located in the same directory as the BUILD file. This + would be of the form "//foo/bar". + """ + + # TODO: File path computation will be removed once the cel_java_test + # rule is updated to be integrated with the cel_test rule in order to avoid repetition. + _, cel_expr_format = paths.split_extension(cel_expr) + if filegroup != "": + data = data + [filegroup] + elif test_data_path != "" and test_data_path != native.package_name(): + data = data + [test_data_path + ":" + test_suite] + if config != "": + data = data + [test_data_path + ":" + config] + if is_valid_cel_file_format(file_extension = cel_expr_format): + data = data + [test_data_path + ":" + cel_expr] + else: + test_data_path = native.package_name() + data = data + [test_suite] + if config != "": + data = data + [config] + if is_valid_cel_file_format(file_extension = cel_expr_format): + data = data + [cel_expr] + + # Since the test_data_path is of the form "//foo/bar", we need to strip the leading "/" to get + # the absolute path. + test_data_path = test_data_path.lstrip("/") + test_suite = test_data_path + "/" + test_suite + + jvm_flags = [ + "-Dtest_suite_path=%s" % test_suite, + "-Duser_test_class_name=%s" % test_class, + ] + + if config != "": + config = test_data_path + "/" + config + jvm_flags.append("-Dconfig_path=%s" % config) + + if file_descriptor_set != None: + data.append(file_descriptor_set) + jvm_flags.append("-Dfile_descriptor_set_path=$(location {})".format(file_descriptor_set)) + + if is_valid_cel_file_format(file_extension = cel_expr_format) == True: + jvm_flags.append("-Dcel_expr=%s" % test_data_path + "/" + cel_expr) + elif is_raw_expr == True: + jvm_flags.append("-Dcel_expr='%s'" % cel_expr) + else: + jvm_flags.append("-Dcel_expr=$(location {})".format(cel_expr)) + data = data + [cel_expr] + + jvm_flags.append("-Dis_raw_expr=%s" % is_raw_expr) + + java_binary( + name = name + "_test_runner_binary", + srcs = ["//testing/testrunner:test_runner_binary"], + data = data, + jvm_flags = jvm_flags, + testonly = True, + main_class = "dev.cel.testing.testrunner.TestRunnerBinary", + runtime_deps = [ + test_src, + ], + deps = [ + "//testing/testrunner:test_executor", + "@maven//:com_google_guava_guava", + "@bazel_tools//tools/java/runfiles:runfiles", + ] + deps, + ) + + sh_test( + name = name, + tags = ["nomsan"], + srcs = ["//testing/testrunner:run_testrunner_binary.sh"], + data = [ + ":%s_test_runner_binary" % name, + ], + args = [ + name, + ], + ) + +def is_valid_cel_file_format(file_extension): + """Checks if the file extension is a valid CEL file format. + + Args: + file_extension: The file extension to check. + + Returns: + True if the file extension is a valid CEL file format, False otherwise. + """ + return file_extension in [ + ".cel", + ".celpolicy", + ".yaml", + ] diff --git a/testing/testrunner/run_testrunner_binary.sh b/testing/testrunner/run_testrunner_binary.sh new file mode 100755 index 000000000..551a41cc9 --- /dev/null +++ b/testing/testrunner/run_testrunner_binary.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Path: //third_party/java/cel/testing/testrunner/run_testrunner_binary.sh + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +# Get the name passed to the sh_test rule from the arguments. +# This is the name passed to the sh_test macro, +# and it's also the name of the java_binary +NAME=$1 + +# Find the test_runner_binary executable (wrapper script), using the NAME +TEST_RUNNER_BINARY="$(find -L "${TEST_SRCDIR}" -name "${NAME}_test_runner_binary" -type f -executable)" + +# This would have also worked but the above is more strict in finding +# a file that's a symlink to the executable. +# TEST_RUNNER_BINARY="$(find "${TEST_SRCDIR}" -name "${NAME}_test_runner_binary")" + +if [ -z "$TEST_RUNNER_BINARY" ]; then + die "Test runner binary (wrapper script) $TEST_RUNNER_BINARY not found in runfiles." +fi + +#Execute the symlink to the executable. +"$TEST_RUNNER_BINARY" || die "Some or all the tests failed." + +echo "PASS"