From e1b6c1168d9343cbeccb685930c96d12c4be2cce Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 18 Aug 2025 13:08:09 -0700 Subject: [PATCH] Evaluate CEL's null and bytes to their native equivalent types PiperOrigin-RevId: 796545163 --- .../src/test/java/dev/cel/bundle/BUILD.bazel | 3 +- .../test/java/dev/cel/bundle/CelImplTest.java | 40 +++++++-- .../src/main/java/dev/cel/common/BUILD.bazel | 2 + .../main/java/dev/cel/common/CelOptions.java | 16 ++++ .../dev/cel/common/CelProtoJsonAdapter.java | 8 +- .../main/java/dev/cel/common/ast/BUILD.bazel | 15 +++- .../java/dev/cel/common/ast/CelConstant.java | 66 ++++++++++----- .../dev/cel/common/ast/CelExprConverter.java | 15 +++- .../dev/cel/common/ast/CelExprFactory.java | 17 ++-- .../dev/cel/common/ast/CelExprFormatter.java | 8 +- .../common/ast/CelExprV1Alpha1Converter.java | 15 +++- .../java/dev/cel/common/internal/BUILD.bazel | 9 ++ .../dev/cel/common/internal/Constants.java | 5 +- .../dev/cel/common/internal/ProtoAdapter.java | 42 ++++++++-- .../cel/common/internal/ProtoLiteAdapter.java | 17 ++-- .../java/dev/cel/common/values/BUILD.bazel | 3 + .../values/BaseProtoCelValueConverter.java | 4 - .../dev/cel/common/values/CelByteString.java | 65 +++++++++++++++ .../cel/common/values/CelValueConverter.java | 2 + .../java/dev/cel/common/values/NullValue.java | 5 ++ .../values/ProtoLiteCelValueConverter.java | 5 +- .../values/ProtoMessageValueProvider.java | 10 ++- .../src/test/java/dev/cel/common/BUILD.bazel | 2 +- .../cel/common/CelProtoJsonAdapterTest.java | 4 +- .../test/java/dev/cel/common/ast/BUILD.bazel | 2 + .../dev/cel/common/ast/CelConstantTest.java | 25 +++--- .../cel/common/ast/CelExprConverterTest.java | 5 +- .../ast/CelExprV1Alpha1ConverterTest.java | 5 +- .../ast/CelMutableExprConverterTest.java | 8 +- .../java/dev/cel/common/internal/BUILD.bazel | 3 +- .../cel/common/internal/ConstantsTest.java | 6 +- .../cel/common/internal/ProtoAdapterTest.java | 39 ++++----- .../values/ProtoCelValueConverterTest.java | 23 ++--- .../values/ProtoMessageValueProviderTest.java | 18 ++-- .../cel/common/values/StructValueTest.java | 3 +- common/values/BUILD.bazel | 2 +- .../dev/cel/conformance/ConformanceTest.java | 29 +++++-- .../main/java/dev/cel/extensions/BUILD.bazel | 4 + .../cel/extensions/CelEncoderExtensions.java | 83 ++++++++++++------- .../dev/cel/extensions/CelExtensions.java | 18 +++- .../cel/extensions/CelOptionalLibrary.java | 12 +-- .../test/java/dev/cel/extensions/BUILD.bazel | 3 +- .../extensions/CelEncoderExtensionsTest.java | 30 ++++--- .../extensions/CelOptionalLibraryTest.java | 12 ++- .../src/main/java/dev/cel/parser/BUILD.bazel | 2 +- .../dev/cel/parser/CelUnparserVisitor.java | 4 +- .../src/test/java/dev/cel/parser/BUILD.bazel | 2 +- .../cel/parser/CelMacroExprFactoryTest.java | 4 +- .../src/main/java/dev/cel/runtime/BUILD.bazel | 10 ++- .../dev/cel/runtime/CelRuntimeLegacyImpl.java | 2 +- .../dev/cel/runtime/DefaultInterpreter.java | 14 +++- .../runtime/DescriptorMessageProvider.java | 12 ++- .../java/dev/cel/runtime/LiteRuntimeImpl.java | 6 +- .../ProtoMessageActivationFactory.java | 3 +- .../runtime/ProtoMessageRuntimeHelpers.java | 3 +- .../java/dev/cel/runtime/RuntimeHelpers.java | 4 +- .../java/dev/cel/runtime/TypeResolver.java | 7 +- .../dev/cel/runtime/standard/AddOperator.java | 13 ++- .../java/dev/cel/runtime/standard/BUILD.bazel | 16 ++++ .../cel/runtime/standard/BytesFunction.java | 13 ++- .../standard/GreaterEqualsOperator.java | 17 +++- .../cel/runtime/standard/GreaterOperator.java | 17 +++- .../runtime/standard/LessEqualsOperator.java | 17 +++- .../cel/runtime/standard/LessOperator.java | 17 +++- .../cel/runtime/standard/SizeFunction.java | 25 ++++-- .../cel/runtime/standard/StringFunction.java | 23 ++++- .../java/dev/cel/runtime/ActivationTest.java | 2 +- .../src/test/java/dev/cel/runtime/BUILD.bazel | 3 + .../runtime/CelLiteRuntimeAndroidTest.java | 8 +- .../dev/cel/runtime/CelLiteRuntimeTest.java | 5 +- .../DescriptorMessageProviderTest.java | 5 +- .../ProtoMessageRuntimeEqualityTest.java | 9 +- .../ProtoMessageRuntimeHelpersTest.java | 7 +- .../src/main/java/dev/cel/testing/BUILD.bazel | 1 + .../dev/cel/testing/BaseInterpreterTest.java | 22 ++--- .../java/dev/cel/testing/utils/BUILD.bazel | 2 + .../dev/cel/testing/utils/ExprValueUtils.java | 22 +++-- 77 files changed, 727 insertions(+), 298 deletions(-) diff --git a/bundle/src/test/java/dev/cel/bundle/BUILD.bazel b/bundle/src/test/java/dev/cel/bundle/BUILD.bazel index b1b352b09..a4c6f513d 100644 --- a/bundle/src/test/java/dev/cel/bundle/BUILD.bazel +++ b/bundle/src/test/java/dev/cel/bundle/BUILD.bazel @@ -41,6 +41,8 @@ java_library( "//common/types:cel_proto_types", "//common/types:message_type_provider", "//common/types:type_providers", + "//common/values", + "//common/values:cel_byte_string", "//compiler", "//compiler:compiler_builder", "//extensions", @@ -64,7 +66,6 @@ java_library( "@maven//:com_google_truth_extensions_truth_proto_extension", "@maven//:junit_junit", "@maven//:org_jspecify_jspecify", - "@maven_android//:com_google_protobuf_protobuf_javalite", ], ) diff --git a/bundle/src/test/java/dev/cel/bundle/CelImplTest.java b/bundle/src/test/java/dev/cel/bundle/CelImplTest.java index 1e82719fb..2a702538a 100644 --- a/bundle/src/test/java/dev/cel/bundle/CelImplTest.java +++ b/bundle/src/test/java/dev/cel/bundle/CelImplTest.java @@ -48,7 +48,6 @@ import com.google.protobuf.Empty; import com.google.protobuf.FieldMask; import com.google.protobuf.Message; -import com.google.protobuf.NullValue; import com.google.protobuf.Struct; import com.google.protobuf.TextFormat; import com.google.protobuf.Timestamp; @@ -86,6 +85,8 @@ import dev.cel.common.types.ProtoMessageTypeProvider; import dev.cel.common.types.SimpleType; import dev.cel.common.types.StructTypeReference; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import dev.cel.compiler.CelCompiler; import dev.cel.compiler.CelCompilerFactory; import dev.cel.compiler.CelCompilerImpl; @@ -108,6 +109,7 @@ import dev.cel.runtime.UnknownContext; import dev.cel.testing.testdata.proto3.StandaloneGlobalEnum; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -1019,7 +1021,10 @@ public void program_enumTypeReferenceResolution(boolean resolveTypeDependencies) Cel cel = standardCelBuilderWithMacros() .setOptions( - CelOptions.current().resolveTypeDependencies(resolveTypeDependencies).build()) + CelOptions.current() + .evaluateCanonicalTypesToNativeValues(true) + .resolveTypeDependencies(resolveTypeDependencies) + .build()) .addMessageTypes(Struct.getDescriptor()) .setResultType(StructTypeReference.create("google.protobuf.NullValue")) .setContainer(CelContainer.ofName("google.protobuf")) @@ -1037,7 +1042,11 @@ public void program_enumTypeReferenceResolution(boolean resolveTypeDependencies) public void program_enumTypeTransitiveResolution() throws Exception { Cel cel = standardCelBuilderWithMacros() - .setOptions(CelOptions.current().resolveTypeDependencies(true).build()) + .setOptions( + CelOptions.current() + .evaluateCanonicalTypesToNativeValues(true) + .resolveTypeDependencies(true) + .build()) .addMessageTypes(Proto2ExtensionScopedMessage.getDescriptor()) .setResultType(StructTypeReference.create("google.protobuf.NullValue")) .setContainer(CelContainer.ofName("google.protobuf")) @@ -1626,7 +1635,11 @@ public void programAdvanceEvaluation_indexOnUnknownContainer() throws Exception public void programAdvanceEvaluation_unsupportedIndexIgnored() throws Exception { Cel cel = standardCelBuilderWithMacros() - .setOptions(CelOptions.current().enableUnknownTracking(true).build()) + .setOptions( + CelOptions.current() + .evaluateCanonicalTypesToNativeValues(true) + .enableUnknownTracking(true) + .build()) .addVar("unk", MapType.create(SimpleType.STRING, SimpleType.BOOL)) .setContainer(CelContainer.ofName("")) .addFunctionBindings() @@ -1654,7 +1667,7 @@ public void programAdvanceEvaluation_unsupportedIndexIgnored() throws Exception UnknownContext.create( fromMap( ImmutableMap.of( - "unk", ImmutableMap.of(ByteString.copyFromUtf8("a"), false))), + "unk", ImmutableMap.of(CelByteString.copyFromUtf8("a"), false))), ImmutableList.of()))) .isEqualTo(false); } @@ -2077,6 +2090,23 @@ public void program_regexProgramSizeExceedsLimit_throws() throws Exception { assertThat(e.getErrorCode()).isEqualTo(CelErrorCode.INVALID_ARGUMENT); } + @Test + @SuppressWarnings("unchecked") // test only + public void program_evaluateCanonicalTypesToNativeTypesDisabled_producesProtoValues() + throws Exception { + Cel cel = + standardCelBuilderWithMacros() + .setOptions(CelOptions.current().evaluateCanonicalTypesToNativeValues(false).build()) + .build(); + CelAbstractSyntaxTree ast = cel.compile("[null, {b'abc': null}]").getAst(); + Map expectedNestedMap = new LinkedHashMap<>(); + expectedNestedMap.put(ByteString.copyFromUtf8("abc"), com.google.protobuf.NullValue.NULL_VALUE); + + List result = (List) cel.createProgram(ast).eval(); + + assertThat(result).containsExactly(com.google.protobuf.NullValue.NULL_VALUE, expectedNestedMap); + } + @Test public void toBuilder_isImmutable() { CelBuilder celBuilder = CelFactory.standardCelBuilder(); diff --git a/common/src/main/java/dev/cel/common/BUILD.bazel b/common/src/main/java/dev/cel/common/BUILD.bazel index e64faf7b9..255ae5d8d 100644 --- a/common/src/main/java/dev/cel/common/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/BUILD.bazel @@ -206,6 +206,8 @@ java_library( ], deps = [ "//common/internal:proto_time_utils", + "//common/values", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", diff --git a/common/src/main/java/dev/cel/common/CelOptions.java b/common/src/main/java/dev/cel/common/CelOptions.java index b05b21867..23654235d 100644 --- a/common/src/main/java/dev/cel/common/CelOptions.java +++ b/common/src/main/java/dev/cel/common/CelOptions.java @@ -109,6 +109,8 @@ public enum ProtoUnsetFieldOptions { public abstract int comprehensionMaxIterations(); + public abstract boolean evaluateCanonicalTypesToNativeValues(); + public abstract boolean unwrapWellKnownTypesOnFunctionDispatch(); public abstract ProtoUnsetFieldOptions fromProtoUnsetFieldOption(); @@ -150,6 +152,7 @@ public static Builder newBuilder() { .enableNamespacedDeclarations(true) // Evaluation options .disableCelStandardEquality(true) + .evaluateCanonicalTypesToNativeValues(false) .enableShortCircuiting(true) .enableRegexPartialMatch(false) .enableUnsignedComparisonAndArithmeticIsUnsigned(false) @@ -450,6 +453,19 @@ public abstract static class Builder { */ public abstract Builder comprehensionMaxIterations(int value); + /** + * If set, canonical CEL types such as bytes and CEL null will return their native value + * equivalents instead of protobuf based values. Specifically: + * + *
    + *
  • Bytes: {@code dev.cel.common.values.CelByteString} instead of {@code + * com.google.protobuf.ByteString}. + *
  • CEL null: {@code dev.cel.common.values.NullValue} instead of {@code + * com.google.protobuf.NullValue}. + *
+ */ + public abstract Builder evaluateCanonicalTypesToNativeValues(boolean value); + /** * If disabled, CEL runtime will no longer adapt the function dispatch results for protobuf's * well known types to other types. This option is enabled by default. diff --git a/common/src/main/java/dev/cel/common/CelProtoJsonAdapter.java b/common/src/main/java/dev/cel/common/CelProtoJsonAdapter.java index 6002e00e3..e0afb8c4e 100644 --- a/common/src/main/java/dev/cel/common/CelProtoJsonAdapter.java +++ b/common/src/main/java/dev/cel/common/CelProtoJsonAdapter.java @@ -18,7 +18,6 @@ import com.google.common.base.Joiner; import com.google.common.primitives.UnsignedLong; import com.google.errorprone.annotations.Immutable; -import com.google.protobuf.ByteString; import com.google.protobuf.Duration; import com.google.protobuf.Empty; import com.google.protobuf.FieldMask; @@ -28,6 +27,7 @@ import com.google.protobuf.Timestamp; import com.google.protobuf.Value; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -71,7 +71,7 @@ public static Struct adaptToJsonStructValue(Map map) @SuppressWarnings("unchecked") public static Value adaptValueToJsonValue(Object value) { Value.Builder json = Value.newBuilder(); - if (value == null || value instanceof NullValue) { + if (value == null || value instanceof dev.cel.common.values.NullValue) { return json.setNullValue(NullValue.NULL_VALUE).build(); } if (value instanceof Boolean) { @@ -93,9 +93,9 @@ public static Value adaptValueToJsonValue(Object value) { if (value instanceof Float || value instanceof Double) { return json.setNumberValue(((Number) value).doubleValue()).build(); } - if (value instanceof ByteString) { + if (value instanceof CelByteString) { return json.setStringValue( - Base64.getEncoder().encodeToString(((ByteString) value).toByteArray())) + Base64.getEncoder().encodeToString(((CelByteString) value).toByteArray())) .build(); } if (value instanceof String) { diff --git a/common/src/main/java/dev/cel/common/ast/BUILD.bazel b/common/src/main/java/dev/cel/common/ast/BUILD.bazel index 368b90c8c..3fc709a07 100644 --- a/common/src/main/java/dev/cel/common/ast/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/ast/BUILD.bazel @@ -49,6 +49,8 @@ java_library( deps = [ "//:auto_value", "//common/annotations", + "//common/values", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", @@ -62,9 +64,12 @@ java_library( ], deps = [ ":ast", + "//common/values", + "//common/values:cel_byte_string", "@cel_spec//proto/cel/expr:checked_java_proto", "@cel_spec//proto/cel/expr:syntax_java_proto", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", ], ) @@ -75,9 +80,12 @@ cel_android_library( ], deps = [ ":ast_android", + "//common/values:cel_byte_string", + "//common/values:values_android", "@cel_spec//proto/cel/expr:checked_java_proto_lite", "@cel_spec//proto/cel/expr:syntax_java_proto_lite", "@maven_android//:com_google_guava_guava", + "@maven_android//:com_google_protobuf_protobuf_javalite", ], ) @@ -88,8 +96,11 @@ java_library( ], deps = [ ":ast", + "//common/values", + "//common/values:cel_byte_string", "@com_google_googleapis//google/api/expr/v1alpha1:expr_java_proto", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", ], ) @@ -112,8 +123,8 @@ java_library( deps = [ ":ast", "//common/annotations", + "//common/values:cel_byte_string", "@maven//:com_google_guava_guava", - "@maven//:com_google_protobuf_protobuf_java", ], ) @@ -136,6 +147,8 @@ cel_android_library( deps = [ "//:auto_value", "//common/annotations", + "//common/values:cel_byte_string", + "//common/values:values_android", "@maven//:com_google_errorprone_error_prone_annotations", "@maven_android//:com_google_guava_guava", "@maven_android//:com_google_protobuf_protobuf_javalite", diff --git a/common/src/main/java/dev/cel/common/ast/CelConstant.java b/common/src/main/java/dev/cel/common/ast/CelConstant.java index f981225e3..c27f4ff8f 100644 --- a/common/src/main/java/dev/cel/common/ast/CelConstant.java +++ b/common/src/main/java/dev/cel/common/ast/CelConstant.java @@ -19,11 +19,13 @@ import com.google.common.collect.ImmutableSet; import com.google.common.primitives.UnsignedLong; import com.google.errorprone.annotations.Immutable; +import com.google.errorprone.annotations.InlineMe; import com.google.protobuf.ByteString; import com.google.protobuf.Duration; -import com.google.protobuf.NullValue; import com.google.protobuf.Timestamp; import dev.cel.common.annotations.Internal; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; /** * Represents a primitive literal. @@ -42,7 +44,7 @@ public abstract class CelConstant { UnsignedLong.class, Double.class, String.class, - ByteString.class); + CelByteString.class); /** Represents the type of the Constant */ public enum Kind { @@ -92,7 +94,7 @@ public abstract static class CelConstantNotSet {} public abstract String stringValue(); - public abstract ByteString bytesValue(); + public abstract CelByteString bytesValue(); /** * @deprecated Do not use. Timestamp is no longer built-in CEL type. @@ -134,10 +136,46 @@ public static CelConstant ofValue(String value) { return AutoOneOf_CelConstant.stringValue(value); } - public static CelConstant ofValue(ByteString value) { + public static CelConstant ofValue(CelByteString value) { return AutoOneOf_CelConstant.bytesValue(value); } + /** + * @deprecated Use native type equivalent {@link #ofValue(NullValue)} instead. + */ + @InlineMe( + replacement = "CelConstant.ofValue(NullValue.NULL_VALUE)", + imports = {"dev.cel.common.ast.CelConstant", "dev.cel.common.values.NullValue"}) + @Deprecated + public static CelConstant ofValue(com.google.protobuf.NullValue unused) { + return ofValue(NullValue.NULL_VALUE); + } + + /** + * @deprecated Use native type equivalent {@link #ofValue(CelByteString)} instead. + */ + @Deprecated + public static CelConstant ofValue(ByteString value) { + CelByteString celByteString = CelByteString.of(value.toByteArray()); + return ofValue(celByteString); + } + + /** + * @deprecated Do not use. Duration is no longer built-in CEL type. + */ + @Deprecated + public static CelConstant ofValue(Duration value) { + return AutoOneOf_CelConstant.durationValue(value); + } + + /** + * @deprecated Do not use. Timestamp is no longer built-in CEL type. + */ + @Deprecated + public static CelConstant ofValue(Timestamp value) { + return AutoOneOf_CelConstant.timestampValue(value); + } + /** Checks whether the provided Java object is a valid CelConstant value. */ public static boolean isConstantValue(Object value) { return CONSTANT_CLASSES.contains(value.getClass()); @@ -163,26 +201,10 @@ public static CelConstant ofObjectValue(Object value) { return ofValue((double) value); } else if (value instanceof String) { return ofValue((String) value); - } else if (value instanceof ByteString) { - return ofValue((ByteString) value); + } else if (value instanceof CelByteString) { + return ofValue((CelByteString) value); } throw new IllegalArgumentException("Value is not a CelConstant: " + value); } - - /** - * @deprecated Do not use. Duration is no longer built-in CEL type. - */ - @Deprecated - public static CelConstant ofValue(Duration value) { - return AutoOneOf_CelConstant.durationValue(value); - } - - /** - * @deprecated Do not use. Timestamp is no longer built-in CEL type. - */ - @Deprecated - public static CelConstant ofValue(Timestamp value) { - return AutoOneOf_CelConstant.timestampValue(value); - } } diff --git a/common/src/main/java/dev/cel/common/ast/CelExprConverter.java b/common/src/main/java/dev/cel/common/ast/CelExprConverter.java index 3f8675546..6037d901d 100644 --- a/common/src/main/java/dev/cel/common/ast/CelExprConverter.java +++ b/common/src/main/java/dev/cel/common/ast/CelExprConverter.java @@ -30,6 +30,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.UnsignedLong; +import com.google.protobuf.ByteString; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import java.util.Map; import java.util.Optional; @@ -179,7 +182,7 @@ public static CelConstant exprConstantToCelConstant(Constant constExpr) { case CONSTANTKIND_NOT_SET: return CelConstant.ofNotSet(); case NULL_VALUE: - return CelConstant.ofValue(constExpr.getNullValue()); + return CelConstant.ofValue(NullValue.NULL_VALUE); case BOOL_VALUE: return CelConstant.ofValue(constExpr.getBoolValue()); case INT64_VALUE: @@ -191,7 +194,8 @@ public static CelConstant exprConstantToCelConstant(Constant constExpr) { case STRING_VALUE: return CelConstant.ofValue(constExpr.getStringValue()); case BYTES_VALUE: - return CelConstant.ofValue(constExpr.getBytesValue()); + ByteString bytesValue = constExpr.getBytesValue(); + return CelConstant.ofValue(CelByteString.of(bytesValue.toByteArray())); case DURATION_VALUE: return CelConstant.ofValue(constExpr.getDurationValue()); case TIMESTAMP_VALUE: @@ -250,7 +254,7 @@ public static Constant celConstantToExprConstant(CelConstant celConstant) { case NOT_SET: return Constant.getDefaultInstance(); case NULL_VALUE: - return Constant.newBuilder().setNullValue(celConstant.nullValue()).build(); + return Constant.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); case BOOLEAN_VALUE: return Constant.newBuilder().setBoolValue(celConstant.booleanValue()).build(); case INT64_VALUE: @@ -262,7 +266,10 @@ public static Constant celConstantToExprConstant(CelConstant celConstant) { case STRING_VALUE: return Constant.newBuilder().setStringValue(celConstant.stringValue()).build(); case BYTES_VALUE: - return Constant.newBuilder().setBytesValue(celConstant.bytesValue()).build(); + CelByteString celByteString = celConstant.bytesValue(); + return Constant.newBuilder() + .setBytesValue(ByteString.copyFrom(celByteString.toByteArray())) + .build(); case DURATION_VALUE: return Constant.newBuilder().setDurationValue(celConstant.durationValue()).build(); case TIMESTAMP_VALUE: diff --git a/common/src/main/java/dev/cel/common/ast/CelExprFactory.java b/common/src/main/java/dev/cel/common/ast/CelExprFactory.java index 2ee2701be..f009e67a4 100644 --- a/common/src/main/java/dev/cel/common/ast/CelExprFactory.java +++ b/common/src/main/java/dev/cel/common/ast/CelExprFactory.java @@ -18,8 +18,8 @@ import static com.google.common.base.Strings.isNullOrEmpty; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import dev.cel.common.annotations.Internal; +import dev.cel.common.values.CelByteString; import java.util.Arrays; /** Factory for generating expression nodes. */ @@ -42,23 +42,18 @@ public final CelExpr newBoolLiteral(boolean value) { } /** Creates a new constant {@link CelExpr} for a bytes value. */ - public final CelExpr newBytesLiteral(ByteString value) { - return newConstant(CelConstant.ofValue(value)); + public final CelExpr newBytesLiteral(String value) { + return newBytesLiteral(CelByteString.copyFromUtf8(value)); } /** Creates a new constant {@link CelExpr} for a bytes value. */ public final CelExpr newBytesLiteral(byte[] value) { - return newBytesLiteral(value, 0, value.length); + return newBytesLiteral(CelByteString.of(value)); } /** Creates a new constant {@link CelExpr} for a bytes value. */ - public final CelExpr newBytesLiteral(byte[] value, int offset, int size) { - return newBytesLiteral(ByteString.copyFrom(value, offset, size)); - } - - /** Creates a new constant {@link CelExpr} for a bytes value. */ - public final CelExpr newBytesLiteral(String value) { - return newBytesLiteral(ByteString.copyFromUtf8(value)); + public final CelExpr newBytesLiteral(CelByteString value) { + return newConstant(CelConstant.ofValue(value)); } /** Creates a new constant {@link CelExpr} for a double value. */ diff --git a/common/src/main/java/dev/cel/common/ast/CelExprFormatter.java b/common/src/main/java/dev/cel/common/ast/CelExprFormatter.java index 0faf2835c..d0eee2f50 100644 --- a/common/src/main/java/dev/cel/common/ast/CelExprFormatter.java +++ b/common/src/main/java/dev/cel/common/ast/CelExprFormatter.java @@ -15,6 +15,8 @@ package dev.cel.common.ast; import com.google.common.collect.ImmutableSet; +import dev.cel.common.values.CelByteString; +import java.nio.charset.StandardCharsets; import java.util.Locale; /** Provides string formatting support for {@link CelExpr}. */ @@ -104,8 +106,12 @@ private void appendConst(CelConstant celConstant) { appendWithoutIndent("\"" + celConstant.stringValue() + "\""); break; case BYTES_VALUE: + CelByteString byteString = celConstant.bytesValue(); appendWithoutIndent( - String.format(Locale.getDefault(), "b\"%s\"", celConstant.bytesValue().toStringUtf8())); + String.format( + Locale.getDefault(), + "b\"%s\"", + new String(byteString.toByteArray(), StandardCharsets.UTF_8))); break; default: append("Unknown kind: " + celConstant.getKind()); diff --git a/common/src/main/java/dev/cel/common/ast/CelExprV1Alpha1Converter.java b/common/src/main/java/dev/cel/common/ast/CelExprV1Alpha1Converter.java index 593957739..0e755e7a2 100644 --- a/common/src/main/java/dev/cel/common/ast/CelExprV1Alpha1Converter.java +++ b/common/src/main/java/dev/cel/common/ast/CelExprV1Alpha1Converter.java @@ -30,6 +30,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.UnsignedLong; +import com.google.protobuf.ByteString; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import java.util.Map; import java.util.Optional; @@ -179,7 +182,7 @@ public static CelConstant exprConstantToCelConstant(Constant constExpr) { case CONSTANTKIND_NOT_SET: return CelConstant.ofNotSet(); case NULL_VALUE: - return CelConstant.ofValue(constExpr.getNullValue()); + return CelConstant.ofValue(NullValue.NULL_VALUE); case BOOL_VALUE: return CelConstant.ofValue(constExpr.getBoolValue()); case INT64_VALUE: @@ -191,7 +194,8 @@ public static CelConstant exprConstantToCelConstant(Constant constExpr) { case STRING_VALUE: return CelConstant.ofValue(constExpr.getStringValue()); case BYTES_VALUE: - return CelConstant.ofValue(constExpr.getBytesValue()); + ByteString bytesValue = constExpr.getBytesValue(); + return CelConstant.ofValue(CelByteString.of(bytesValue.toByteArray())); case DURATION_VALUE: return CelConstant.ofValue(constExpr.getDurationValue()); case TIMESTAMP_VALUE: @@ -250,7 +254,7 @@ public static Constant celConstantToExprConstant(CelConstant celConstant) { case NOT_SET: return Constant.getDefaultInstance(); case NULL_VALUE: - return Constant.newBuilder().setNullValue(celConstant.nullValue()).build(); + return Constant.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); case BOOLEAN_VALUE: return Constant.newBuilder().setBoolValue(celConstant.booleanValue()).build(); case INT64_VALUE: @@ -262,7 +266,10 @@ public static Constant celConstantToExprConstant(CelConstant celConstant) { case STRING_VALUE: return Constant.newBuilder().setStringValue(celConstant.stringValue()).build(); case BYTES_VALUE: - return Constant.newBuilder().setBytesValue(celConstant.bytesValue()).build(); + CelByteString celByteString = celConstant.bytesValue(); + return Constant.newBuilder() + .setBytesValue(ByteString.copyFrom(celByteString.toByteArray())) + .build(); case DURATION_VALUE: return Constant.newBuilder().setDurationValue(celConstant.durationValue()).build(); case TIMESTAMP_VALUE: diff --git a/common/src/main/java/dev/cel/common/internal/BUILD.bazel b/common/src/main/java/dev/cel/common/internal/BUILD.bazel index 0db2d1862..c508649a3 100644 --- a/common/src/main/java/dev/cel/common/internal/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/internal/BUILD.bazel @@ -43,6 +43,8 @@ java_library( "//:auto_value", "//common/annotations", "//common/ast", + "//common/values", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", @@ -59,6 +61,8 @@ cel_android_library( "//:auto_value", "//common/annotations", "//common/ast:ast_android", + "//common/values:cel_byte_string", + "//common/values:values_android", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:org_antlr_antlr4_runtime", @@ -171,8 +175,11 @@ java_library( ":well_known_proto", "//:auto_value", "//common:error_codes", + "//common:options", "//common:runtime_exception", "//common/annotations", + "//common/values", + "//common/values:cel_byte_string", "@maven//:com_google_code_findbugs_annotations", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", @@ -191,6 +198,8 @@ java_library( "//common:proto_json_adapter", "//common:runtime_exception", "//common/annotations", + "//common/values", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", diff --git a/common/src/main/java/dev/cel/common/internal/Constants.java b/common/src/main/java/dev/cel/common/internal/Constants.java index 6a661fc41..7a9ffa1ec 100644 --- a/common/src/main/java/dev/cel/common/internal/Constants.java +++ b/common/src/main/java/dev/cel/common/internal/Constants.java @@ -19,9 +19,10 @@ import com.google.common.primitives.UnsignedLong; import com.google.protobuf.ByteString; -import com.google.protobuf.NullValue; import dev.cel.common.annotations.Internal; import dev.cel.common.ast.CelConstant; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import java.text.ParseException; import java.util.PrimitiveIterator; @@ -154,7 +155,7 @@ public static CelConstant parseBytes(String text) throws ParseException { text = text.substring(0, text.length() - quote.length()); DecodeBuffer buffer = new DecodeByteStringBuffer(text.length()); decodeString(offset, text, buffer, isRawLiteral, true); - return CelConstant.ofValue(buffer.toDecodedValue()); + return CelConstant.ofValue(CelByteString.of(buffer.toDecodedValue().toByteArray())); } public static CelConstant parseString(String text) throws ParseException { diff --git a/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java b/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java index c1a096930..3461106ae 100644 --- a/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java +++ b/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java @@ -22,6 +22,7 @@ import com.google.errorprone.annotations.CheckReturnValue; import com.google.errorprone.annotations.Immutable; import com.google.protobuf.Any; +import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; @@ -30,10 +31,12 @@ import com.google.protobuf.MapEntry; import com.google.protobuf.Message; import com.google.protobuf.MessageOrBuilder; -import com.google.protobuf.NullValue; import dev.cel.common.CelErrorCode; +import dev.cel.common.CelOptions; import dev.cel.common.CelRuntimeException; import dev.cel.common.annotations.Internal; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -117,13 +120,13 @@ public final class ProtoAdapter { BidiConverter.of(Number::doubleValue, Number::floatValue); private final ProtoLiteAdapter protoLiteAdapter; + private final CelOptions celOptions; private final DynamicProto dynamicProto; - private final boolean enableUnsignedLongs; - public ProtoAdapter(DynamicProto dynamicProto, boolean enableUnsignedLongs) { + public ProtoAdapter(DynamicProto dynamicProto, CelOptions celOptions) { this.dynamicProto = checkNotNull(dynamicProto); - this.enableUnsignedLongs = enableUnsignedLongs; - this.protoLiteAdapter = new ProtoLiteAdapter(enableUnsignedLongs); + this.protoLiteAdapter = new ProtoLiteAdapter(celOptions.enableUnsignedLongs()); + this.celOptions = celOptions; } /** @@ -192,6 +195,7 @@ public Optional adaptFieldToValue(FieldDescriptor fieldDescriptor, Objec } return Optional.of(AdaptingTypes.adaptingList((List) fieldValue, bidiConverter)); } + return Optional.of( fieldToValueConverter(fieldDescriptor).forwardConverter().convert(fieldValue)); } @@ -230,6 +234,7 @@ public Optional adaptValueToFieldType( AdaptingTypes.adaptingList( (List) fieldValue, fieldToValueConverter(fieldDescriptor).reverse())); } + return Optional.of( fieldToValueConverter(fieldDescriptor).backwardConverter().convert(fieldValue)); } @@ -243,18 +248,25 @@ private BidiConverter fieldToValueConverter(FieldDescriptor fieldDescriptor) { return INT_CONVERTER; case FIXED32: case UINT32: - if (enableUnsignedLongs) { + if (celOptions.enableUnsignedLongs()) { return UNSIGNED_UINT32_CONVERTER; } return SIGNED_UINT32_CONVERTER; case FIXED64: case UINT64: - if (enableUnsignedLongs) { + if (celOptions.enableUnsignedLongs()) { return UNSIGNED_UINT64_CONVERTER; } return BidiConverter.IDENTITY; case FLOAT: return DOUBLE_CONVERTER; + case BYTES: + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return BidiConverter.of( + ProtoAdapter::adaptProtoByteStringToValue, ProtoAdapter::adaptCelByteStringToProto); + } + + return BidiConverter.IDENTITY; case ENUM: return BidiConverter.of( value -> (long) ((EnumValueDescriptor) value).getNumber(), @@ -271,6 +283,22 @@ private BidiConverter fieldToValueConverter(FieldDescriptor fieldDescriptor) { } } + private static CelByteString adaptProtoByteStringToValue(Object proto) { + if (proto instanceof CelByteString) { + return (CelByteString) proto; + } + + return CelByteString.of(((ByteString) proto).toByteArray()); + } + + private static ByteString adaptCelByteStringToProto(Object value) { + if (value instanceof ByteString) { + return (ByteString) value; + } + + return ByteString.copyFrom(((CelByteString) value).toByteArray()); + } + /** * Adapt the Java object {@code value} to the given protobuf {@code protoTypeName} if possible. * diff --git a/common/src/main/java/dev/cel/common/internal/ProtoLiteAdapter.java b/common/src/main/java/dev/cel/common/internal/ProtoLiteAdapter.java index a8d47d359..eb8c42862 100644 --- a/common/src/main/java/dev/cel/common/internal/ProtoLiteAdapter.java +++ b/common/src/main/java/dev/cel/common/internal/ProtoLiteAdapter.java @@ -46,6 +46,7 @@ import dev.cel.common.CelProtoJsonAdapter; import dev.cel.common.CelRuntimeException; import dev.cel.common.annotations.Internal; +import dev.cel.common.values.CelByteString; import java.util.Map; import java.util.Map.Entry; @@ -76,7 +77,8 @@ public MessageLite adaptValueToWellKnownProto(Object value, WellKnownProto wellK case BOOL_VALUE: return BoolValue.of((Boolean) value); case BYTES_VALUE: - return BytesValue.of((ByteString) value); + CelByteString byteString = (CelByteString) value; + return BytesValue.of(ByteString.copyFrom(byteString.toByteArray())); case DOUBLE_VALUE: return adaptValueToDouble(value); case FLOAT_VALUE: @@ -112,7 +114,8 @@ public Any adaptValueToAny(Object value, String typeName) { return packAnyMessage((MessageLite) value, typeName); } - if (value instanceof NullValue) { + // if (value instanceof NullValue) { + if (value instanceof dev.cel.common.values.NullValue) { return packAnyMessage( Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(), WellKnownProto.JSON_VALUE); } @@ -121,7 +124,7 @@ public Any adaptValueToAny(Object value, String typeName) { if (value instanceof Boolean) { wellKnownProto = WellKnownProto.BOOL_VALUE; - } else if (value instanceof ByteString) { + } else if (value instanceof CelByteString) { wellKnownProto = WellKnownProto.BYTES_VALUE; } else if (value instanceof String) { wellKnownProto = WellKnownProto.STRING_VALUE; @@ -159,7 +162,8 @@ public Object adaptWellKnownProtoToValue( case BOOL_VALUE: return ((BoolValue) proto).getValue(); case BYTES_VALUE: - return ((BytesValue) proto).getValue(); + ByteString byteString = ((BytesValue) proto).getValue(); + return CelByteString.of(byteString.toByteArray()); case DOUBLE_VALUE: return ((DoubleValue) proto).getValue(); case FLOAT_VALUE: @@ -191,7 +195,8 @@ private Object adaptJsonToValue(Value value) { case BOOL_VALUE: return value.getBoolValue(); case NULL_VALUE: - return value.getNullValue(); + case KIND_NOT_SET: + return dev.cel.common.values.NullValue.NULL_VALUE; case NUMBER_VALUE: return value.getNumberValue(); case STRING_VALUE: @@ -200,8 +205,6 @@ private Object adaptJsonToValue(Value value) { return adaptJsonListToValue(value.getListValue()); case STRUCT_VALUE: return adaptJsonStructToValue(value.getStructValue()); - case KIND_NOT_SET: - return NullValue.NULL_VALUE; } throw new IllegalArgumentException("unexpected value kind: " + value.getKindCase()); } diff --git a/common/src/main/java/dev/cel/common/values/BUILD.bazel b/common/src/main/java/dev/cel/common/values/BUILD.bazel index 17c8da00d..9ffa5bad3 100644 --- a/common/src/main/java/dev/cel/common/values/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/values/BUILD.bazel @@ -234,6 +234,7 @@ java_library( ":base_proto_message_value_provider", ":cel_value", ":proto_message_value", + "//common:options", "//common/annotations", "//common/internal:dynamic_proto", "//common/internal:proto_message_factory", @@ -261,6 +262,7 @@ java_library( "//common/internal:well_known_proto", "//common/types", "//common/types:type_providers", + "//common/values:cel_byte_string", "//protobuf:cel_lite_descriptor", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", @@ -288,6 +290,7 @@ cel_android_library( "//common/internal:well_known_proto_android", "//common/types:type_providers_android", "//common/types:types_android", + "//common/values:cel_byte_string", "//protobuf:cel_lite_descriptor", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", diff --git a/common/src/main/java/dev/cel/common/values/BaseProtoCelValueConverter.java b/common/src/main/java/dev/cel/common/values/BaseProtoCelValueConverter.java index 2335e2553..38741f555 100644 --- a/common/src/main/java/dev/cel/common/values/BaseProtoCelValueConverter.java +++ b/common/src/main/java/dev/cel/common/values/BaseProtoCelValueConverter.java @@ -65,10 +65,6 @@ public Object fromCelValueToJavaObject(CelValue celValue) { return ProtoTimeUtils.toProtoTimestamp(((TimestampValue) celValue).value()); } else if (celValue instanceof DurationValue) { return ProtoTimeUtils.toProtoDuration(((DurationValue) celValue).value()); - } else if (celValue instanceof BytesValue) { - return ByteString.copyFrom(((BytesValue) celValue).value().toByteArray()); - } else if (celValue.equals(NullValue.NULL_VALUE)) { - return com.google.protobuf.NullValue.NULL_VALUE; } return super.fromCelValueToJavaObject(celValue); diff --git a/common/src/main/java/dev/cel/common/values/CelByteString.java b/common/src/main/java/dev/cel/common/values/CelByteString.java index d8a50949e..b37fc778d 100644 --- a/common/src/main/java/dev/cel/common/values/CelByteString.java +++ b/common/src/main/java/dev/cel/common/values/CelByteString.java @@ -15,7 +15,13 @@ package dev.cel.common.values; import com.google.errorprone.annotations.Immutable; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Comparator; /** CelByteString is an immutable sequence of a byte array. */ @Immutable @@ -38,6 +44,32 @@ public static CelByteString of(byte[] buffer) { return new CelByteString(buffer); } + public static CelByteString copyFromUtf8(String utf8String) { + return new CelByteString(utf8String.getBytes(StandardCharsets.UTF_8)); + } + + public static Comparator unsignedLexicographicalComparator() { + return UNSIGNED_LEXICOGRAPHICAL_COMPARATOR; + } + + public String toStringUtf8() { + return new String(data, StandardCharsets.UTF_8); + } + + /** Checks if the byte array is a valid utf-8 encoded text. */ + public boolean isValidUtf8() { + CharsetDecoder charsetDecoder = StandardCharsets.UTF_8.newDecoder(); + charsetDecoder.onMalformedInput(CodingErrorAction.REPORT); + charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); + + try { + charsetDecoder.decode(ByteBuffer.wrap(data)); + } catch (CharacterCodingException unused) { + return false; + } + return true; + } + public int size() { return data.length; } @@ -55,6 +87,14 @@ public byte[] toByteArray() { return Arrays.copyOf(data, size); } + public CelByteString concat(CelByteString other) { + byte[] result = new byte[data.length + other.data.length]; + System.arraycopy(data, 0, result, 0, data.length); + System.arraycopy(other.data, 0, result, data.length, other.data.length); + + return CelByteString.of(result); + } + private CelByteString(byte[] buffer) { data = Arrays.copyOf(buffer, buffer.length); } @@ -91,4 +131,29 @@ public int hashCode() { return hash; } + + @Override + public String toString() { + return toStringUtf8(); + } + + private static final Comparator UNSIGNED_LEXICOGRAPHICAL_COMPARATOR = + (former, latter) -> { + // Once we're on Java 9+, we can replace this whole thing with Arrays.compareUnsigned + byte[] formerBytes = former.toByteArray(); + byte[] latterBytes = latter.toByteArray(); + int minLength = Math.min(formerBytes.length, latterBytes.length); + + for (int i = 0; i < minLength; i++) { + int formerUnsigned = Byte.toUnsignedInt(formerBytes[i]); + int latterUnsigned = Byte.toUnsignedInt(latterBytes[i]); + int result = Integer.compare(formerUnsigned, latterUnsigned); + + if (result != 0) { + return result; + } + } + + return Integer.compare(formerBytes.length, latterBytes.length); + }; } diff --git a/common/src/main/java/dev/cel/common/values/CelValueConverter.java b/common/src/main/java/dev/cel/common/values/CelValueConverter.java index 1489d7b70..a281318f8 100644 --- a/common/src/main/java/dev/cel/common/values/CelValueConverter.java +++ b/common/src/main/java/dev/cel/common/values/CelValueConverter.java @@ -112,6 +112,8 @@ protected CelValue fromJavaPrimitiveToCelValue(Object value) { return DoubleValue.create(Double.valueOf((Float) value)); } else if (value instanceof UnsignedLong) { return UintValue.create((UnsignedLong) value); + } else if (value instanceof CelByteString) { + return BytesValue.create((CelByteString) value); } // Fall back to an Opaque value, as a custom class was supplied in the runtime. The legacy diff --git a/common/src/main/java/dev/cel/common/values/NullValue.java b/common/src/main/java/dev/cel/common/values/NullValue.java index 320d85b9c..ca35c1550 100644 --- a/common/src/main/java/dev/cel/common/values/NullValue.java +++ b/common/src/main/java/dev/cel/common/values/NullValue.java @@ -43,5 +43,10 @@ public boolean isZeroValue() { return true; } + @Override + public String toString() { + return "NULL_VALUE"; + } + private NullValue() {} } diff --git a/common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java b/common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java index 60f020eeb..c23ad54de 100644 --- a/common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java +++ b/common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java @@ -25,7 +25,6 @@ import com.google.common.collect.Multimaps; import com.google.common.primitives.UnsignedLong; import com.google.errorprone.annotations.Immutable; -import com.google.protobuf.ByteString; import com.google.protobuf.CodedInputStream; import com.google.protobuf.ExtensionRegistryLite; import com.google.protobuf.MessageLite; @@ -198,10 +197,10 @@ private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) { case STRING: return ""; case BYTE_STRING: - return ByteString.EMPTY; + return CelByteString.EMPTY; case MESSAGE: if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) { - return com.google.protobuf.NullValue.NULL_VALUE; + return NullValue.NULL_VALUE; } return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build(); diff --git a/common/src/main/java/dev/cel/common/values/ProtoMessageValueProvider.java b/common/src/main/java/dev/cel/common/values/ProtoMessageValueProvider.java index 1b2bb112d..6dc9dfed2 100644 --- a/common/src/main/java/dev/cel/common/values/ProtoMessageValueProvider.java +++ b/common/src/main/java/dev/cel/common/values/ProtoMessageValueProvider.java @@ -18,6 +18,7 @@ import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Message; +import dev.cel.common.CelOptions; import dev.cel.common.annotations.Internal; import dev.cel.common.internal.DynamicProto; import dev.cel.common.internal.ProtoAdapter; @@ -82,14 +83,15 @@ private FieldDescriptor findField(Descriptor descriptor, String fieldName) { fieldName, descriptor.getFullName()))); } - public static ProtoMessageValueProvider newInstance(DynamicProto dynamicProto) { - return new ProtoMessageValueProvider(dynamicProto); + public static ProtoMessageValueProvider newInstance( + CelOptions celOptions, DynamicProto dynamicProto) { + return new ProtoMessageValueProvider(celOptions, dynamicProto); } - private ProtoMessageValueProvider(DynamicProto dynamicProto) { + private ProtoMessageValueProvider(CelOptions celOptions, DynamicProto dynamicProto) { this.protoMessageFactory = dynamicProto.getProtoMessageFactory(); this.protoCelValueConverter = ProtoCelValueConverter.newInstance(protoMessageFactory.getDescriptorPool(), dynamicProto); - this.protoAdapter = new ProtoAdapter(dynamicProto, true); + this.protoAdapter = new ProtoAdapter(dynamicProto, celOptions); } } diff --git a/common/src/test/java/dev/cel/common/BUILD.bazel b/common/src/test/java/dev/cel/common/BUILD.bazel index 703a1c840..c2ce1efdb 100644 --- a/common/src/test/java/dev/cel/common/BUILD.bazel +++ b/common/src/test/java/dev/cel/common/BUILD.bazel @@ -26,6 +26,7 @@ java_library( "//common/types", "//common/types:cel_proto_types", "//common/types:cel_v1alpha1_types", + "//common/values:cel_byte_string", "//compiler", "//compiler:compiler_builder", "//parser:macro", @@ -40,7 +41,6 @@ java_library( "@maven//:com_google_truth_extensions_truth_proto_extension", "@maven//:junit_junit", "@maven//:org_antlr_antlr4_runtime", - "@maven_android//:com_google_protobuf_protobuf_javalite", ], ) diff --git a/common/src/test/java/dev/cel/common/CelProtoJsonAdapterTest.java b/common/src/test/java/dev/cel/common/CelProtoJsonAdapterTest.java index e7ccd5233..bc845e5ab 100644 --- a/common/src/test/java/dev/cel/common/CelProtoJsonAdapterTest.java +++ b/common/src/test/java/dev/cel/common/CelProtoJsonAdapterTest.java @@ -19,11 +19,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import com.google.protobuf.ListValue; import com.google.protobuf.Struct; import com.google.protobuf.Value; import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.common.values.CelByteString; import java.util.Arrays; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,7 +41,7 @@ public void adaptValueToJsonValue_asymmetricJsonConversion() { .isEqualTo(Value.newBuilder().setNumberValue(1).build()); assertThat(CelProtoJsonAdapter.adaptValueToJsonValue(Long.MAX_VALUE)) .isEqualTo(Value.newBuilder().setStringValue(Long.toString(Long.MAX_VALUE)).build()); - assertThat(CelProtoJsonAdapter.adaptValueToJsonValue(ByteString.copyFromUtf8("foo"))) + assertThat(CelProtoJsonAdapter.adaptValueToJsonValue(CelByteString.copyFromUtf8("foo"))) .isEqualTo(Value.newBuilder().setStringValue("Zm9v").build()); } diff --git a/common/src/test/java/dev/cel/common/ast/BUILD.bazel b/common/src/test/java/dev/cel/common/ast/BUILD.bazel index 7e2fea525..a2bb62719 100644 --- a/common/src/test/java/dev/cel/common/ast/BUILD.bazel +++ b/common/src/test/java/dev/cel/common/ast/BUILD.bazel @@ -25,6 +25,8 @@ java_library( "//common/ast:expr_v1alpha1_converter", "//common/ast:mutable_expr", "//common/types", + "//common/values", + "//common/values:cel_byte_string", "//compiler", "//compiler:compiler_builder", "//extensions:optional_library", diff --git a/common/src/test/java/dev/cel/common/ast/CelConstantTest.java b/common/src/test/java/dev/cel/common/ast/CelConstantTest.java index 462267ba7..947073220 100644 --- a/common/src/test/java/dev/cel/common/ast/CelConstantTest.java +++ b/common/src/test/java/dev/cel/common/ast/CelConstantTest.java @@ -15,16 +15,17 @@ package dev.cel.common.ast; import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertThrows; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import com.google.protobuf.Duration; -import com.google.protobuf.NullValue; import com.google.protobuf.Timestamp; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import dev.cel.common.ast.CelConstant.Kind; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,8 +43,8 @@ public void equality_objectsAreValueEqual_success() { .isEqualTo(CelConstant.ofValue(UnsignedLong.valueOf(2))); assertThat(CelConstant.ofValue(2.1)).isEqualTo(CelConstant.ofValue(2.1)); assertThat(CelConstant.ofValue("Hello world!")).isEqualTo(CelConstant.ofValue("Hello world!")); - assertThat(CelConstant.ofValue(ByteString.copyFromUtf8("Test"))) - .isEqualTo(CelConstant.ofValue(ByteString.copyFromUtf8("Test"))); + assertThat(CelConstant.ofValue(CelByteString.of("Test".getBytes(UTF_8)))) + .isEqualTo(CelConstant.ofValue(CelByteString.of("Test".getBytes(UTF_8)))); assertThat(CelConstant.ofValue(Duration.newBuilder().setSeconds(100L).build())) .isEqualTo(CelConstant.ofValue(Duration.newBuilder().setSeconds(100L).build())); assertThat(CelConstant.ofValue(Timestamp.newBuilder().setSeconds(100L).build())) @@ -52,8 +53,6 @@ public void equality_objectsAreValueEqual_success() { @Test public void equality_valueEqualityUnsatisfied_fails() { - assertThat(CelConstant.ofValue(NullValue.NULL_VALUE)) - .isNotEqualTo(CelConstant.ofValue(NullValue.UNRECOGNIZED)); assertThat(CelConstant.ofValue(true)).isNotEqualTo(CelConstant.ofValue(false)); assertThat(CelConstant.ofValue(false)).isNotEqualTo(CelConstant.ofValue(true)); assertThat(CelConstant.ofValue(3)).isNotEqualTo(CelConstant.ofValue(2)); @@ -63,8 +62,8 @@ public void equality_valueEqualityUnsatisfied_fails() { assertThat(CelConstant.ofValue(3)).isNotEqualTo(CelConstant.ofValue(3.0)); assertThat(CelConstant.ofValue(3.1)).isNotEqualTo(CelConstant.ofValue(2.1)); assertThat(CelConstant.ofValue("world!")).isNotEqualTo(CelConstant.ofValue("Hello world!")); - assertThat(CelConstant.ofValue(ByteString.copyFromUtf8("T"))) - .isNotEqualTo(CelConstant.ofValue(ByteString.copyFromUtf8("Test"))); + assertThat(CelConstant.ofValue(CelByteString.of("T".getBytes(UTF_8)))) + .isNotEqualTo(CelConstant.ofValue(CelByteString.of("Test".getBytes(UTF_8)))); assertThat(CelConstant.ofValue(Duration.newBuilder().setSeconds(100L).build())) .isNotEqualTo(CelConstant.ofValue(Duration.newBuilder().setSeconds(50).build())); assertThat(CelConstant.ofValue(Timestamp.newBuilder().setSeconds(100L).build())) @@ -117,9 +116,9 @@ public void constructStringValue() { @Test public void constructBytesValue() { - CelConstant constant = CelConstant.ofValue(ByteString.copyFromUtf8("Test")); + CelConstant constant = CelConstant.ofValue(CelByteString.of("Test".getBytes(UTF_8))); - assertThat(constant.bytesValue()).isEqualTo(ByteString.copyFromUtf8("Test")); + assertThat(constant.bytesValue()).isEqualTo(CelByteString.of("Test".getBytes(UTF_8))); } @Test @@ -152,7 +151,7 @@ private enum CelConstantTestCase { UINT64(CelConstant.ofValue(UnsignedLong.valueOf(2))), DOUBLE(CelConstant.ofValue(2.1)), STRING(CelConstant.ofValue("Hello world!")), - BYTES(CelConstant.ofValue(ByteString.copyFromUtf8("Test"))), + BYTES(CelConstant.ofValue(CelByteString.of("Test".getBytes(UTF_8)))), DURATION(CelConstant.ofValue(Duration.newBuilder().setSeconds(100L).build())), TIMESTAMP(CelConstant.ofValue(Timestamp.newBuilder().setSeconds(100L).build())); @@ -207,8 +206,8 @@ public void getObjectValue_success() { .isEqualTo(CelConstant.ofValue(UnsignedLong.valueOf(3L))); assertThat(CelConstant.ofObjectValue(3.0d)).isEqualTo(CelConstant.ofValue(3.0d)); assertThat(CelConstant.ofObjectValue("test")).isEqualTo(CelConstant.ofValue("test")); - assertThat(CelConstant.ofObjectValue(ByteString.copyFromUtf8("hello"))) - .isEqualTo(CelConstant.ofValue(ByteString.copyFromUtf8("hello"))); + assertThat(CelConstant.ofObjectValue(CelByteString.of("hello".getBytes(UTF_8)))) + .isEqualTo(CelConstant.ofValue(CelByteString.of("hello".getBytes(UTF_8)))); } @Test diff --git a/common/src/test/java/dev/cel/common/ast/CelExprConverterTest.java b/common/src/test/java/dev/cel/common/ast/CelExprConverterTest.java index 8cac69522..ec064b4bb 100644 --- a/common/src/test/java/dev/cel/common/ast/CelExprConverterTest.java +++ b/common/src/test/java/dev/cel/common/ast/CelExprConverterTest.java @@ -33,6 +33,7 @@ import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import com.google.testing.junit.testparameterinjector.TestParameters; +import dev.cel.common.values.CelByteString; import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,7 +50,7 @@ private enum ConstantTestCase { .setId(1) .setConstExpr(Constant.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) .build(), - CelExpr.ofConstant(1, CelConstant.ofValue(NullValue.NULL_VALUE))), + CelExpr.ofConstant(1, CelConstant.ofValue(dev.cel.common.values.NullValue.NULL_VALUE))), BOOLEAN( Expr.newBuilder() .setId(1) @@ -86,7 +87,7 @@ private enum ConstantTestCase { .setConstExpr( Constant.newBuilder().setBytesValue(ByteString.copyFromUtf8("TEST")).build()) .build(), - CelExpr.ofConstant(1, CelConstant.ofValue(ByteString.copyFromUtf8("TEST")))); + CelExpr.ofConstant(1, CelConstant.ofValue(CelByteString.copyFromUtf8("TEST")))); final Expr protoExpr; final CelExpr celExpr; diff --git a/common/src/test/java/dev/cel/common/ast/CelExprV1Alpha1ConverterTest.java b/common/src/test/java/dev/cel/common/ast/CelExprV1Alpha1ConverterTest.java index 28df2285d..193bfa2df 100644 --- a/common/src/test/java/dev/cel/common/ast/CelExprV1Alpha1ConverterTest.java +++ b/common/src/test/java/dev/cel/common/ast/CelExprV1Alpha1ConverterTest.java @@ -33,6 +33,7 @@ import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import com.google.testing.junit.testparameterinjector.TestParameters; +import dev.cel.common.values.CelByteString; import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,7 +50,7 @@ private enum ConstantTestCase { .setId(1) .setConstExpr(Constant.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) .build(), - CelExpr.ofConstant(1, CelConstant.ofValue(NullValue.NULL_VALUE))), + CelExpr.ofConstant(1, CelConstant.ofValue(dev.cel.common.values.NullValue.NULL_VALUE))), BOOLEAN( Expr.newBuilder() .setId(1) @@ -86,7 +87,7 @@ private enum ConstantTestCase { .setConstExpr( Constant.newBuilder().setBytesValue(ByteString.copyFromUtf8("TEST")).build()) .build(), - CelExpr.ofConstant(1, CelConstant.ofValue(ByteString.copyFromUtf8("TEST")))); + CelExpr.ofConstant(1, CelConstant.ofValue(CelByteString.copyFromUtf8("TEST")))); final Expr protoExpr; final CelExpr celExpr; diff --git a/common/src/test/java/dev/cel/common/ast/CelMutableExprConverterTest.java b/common/src/test/java/dev/cel/common/ast/CelMutableExprConverterTest.java index 47b07552f..d58d336ae 100644 --- a/common/src/test/java/dev/cel/common/ast/CelMutableExprConverterTest.java +++ b/common/src/test/java/dev/cel/common/ast/CelMutableExprConverterTest.java @@ -18,8 +18,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; -import com.google.protobuf.NullValue; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import dev.cel.common.ast.CelExpr.CelCall; @@ -31,6 +29,8 @@ import dev.cel.common.ast.CelMutableExpr.CelMutableMap; import dev.cel.common.ast.CelMutableExpr.CelMutableSelect; import dev.cel.common.ast.CelMutableExpr.CelMutableStruct; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import org.junit.Test; import org.junit.runner.RunWith; @@ -60,8 +60,8 @@ private enum ConstantTestCase { CelMutableExpr.ofConstant(1, CelConstant.ofValue("Test")), CelExpr.ofConstant(1, CelConstant.ofValue("Test"))), BYTES( - CelMutableExpr.ofConstant(1, CelConstant.ofValue(ByteString.copyFromUtf8("TEST"))), - CelExpr.ofConstant(1, CelConstant.ofValue(ByteString.copyFromUtf8("TEST")))); + CelMutableExpr.ofConstant(1, CelConstant.ofValue(CelByteString.copyFromUtf8("TEST"))), + CelExpr.ofConstant(1, CelConstant.ofValue(CelByteString.copyFromUtf8("TEST")))); final CelMutableExpr mutableExpr; final CelExpr celExpr; diff --git a/common/src/test/java/dev/cel/common/internal/BUILD.bazel b/common/src/test/java/dev/cel/common/internal/BUILD.bazel index 333f6b58c..d46607118 100644 --- a/common/src/test/java/dev/cel/common/internal/BUILD.bazel +++ b/common/src/test/java/dev/cel/common/internal/BUILD.bazel @@ -21,7 +21,6 @@ java_library( "//common/internal:comparison_functions", "//common/internal:converter", "//common/internal:default_instance_message_factory", - "//common/internal:default_instance_message_lite_factory", "//common/internal:default_lite_descriptor_pool", "//common/internal:default_message_factory", "//common/internal:dynamic_proto", @@ -32,6 +31,8 @@ java_library( "//common/src/test/resources:default_instance_message_test_protos_java_proto", "//common/src/test/resources:service_conflicting_name_java_proto", "//common/testing", + "//common/values", + "//common/values:cel_byte_string", "//protobuf:cel_lite_descriptor", "//testing/protos:multi_file_java_proto", "//testing/protos:single_file_java_proto", diff --git a/common/src/test/java/dev/cel/common/internal/ConstantsTest.java b/common/src/test/java/dev/cel/common/internal/ConstantsTest.java index d222f9df5..f4a9fc32e 100644 --- a/common/src/test/java/dev/cel/common/internal/ConstantsTest.java +++ b/common/src/test/java/dev/cel/common/internal/ConstantsTest.java @@ -18,11 +18,11 @@ import static org.junit.Assert.assertThrows; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import com.google.testing.junit.testparameterinjector.TestParameters; import dev.cel.common.ast.CelConstant; import dev.cel.common.ast.CelConstant.Kind; +import dev.cel.common.values.CelByteString; import java.text.ParseException; import org.junit.Test; import org.junit.runner.RunWith; @@ -336,7 +336,7 @@ public void parseBytes_multibyteCodePoints() throws Exception { private static void testBytes(String actual, String expected) throws Exception { CelConstant constant = Constants.parseBytes(actual); assertThat(constant.getKind()).isEqualTo(Kind.BYTES_VALUE); - assertThat(constant.bytesValue()).isEqualTo(ByteString.copyFromUtf8(expected)); + assertThat(constant.bytesValue()).isEqualTo(CelByteString.copyFromUtf8(expected)); } private static void testQuotedBytes(String actual, String expected) throws Exception { @@ -380,7 +380,7 @@ public void parseRawBytes_escapeSequence() throws Exception { private static void testRawBytes(String actual, String expected) throws Exception { CelConstant constant = Constants.parseBytes(actual); assertThat(constant.getKind()).isEqualTo(Kind.BYTES_VALUE); - assertThat(constant.bytesValue()).isEqualTo(ByteString.copyFromUtf8(expected)); + assertThat(constant.bytesValue()).isEqualTo(CelByteString.copyFromUtf8(expected)); } private static void testRawQuotedBytes(String actual, String expected) throws Exception { diff --git a/common/src/test/java/dev/cel/common/internal/ProtoAdapterTest.java b/common/src/test/java/dev/cel/common/internal/ProtoAdapterTest.java index 253132377..3172258f4 100644 --- a/common/src/test/java/dev/cel/common/internal/ProtoAdapterTest.java +++ b/common/src/test/java/dev/cel/common/internal/ProtoAdapterTest.java @@ -40,6 +40,7 @@ import com.google.protobuf.Value; import com.google.type.Expr; import dev.cel.common.CelOptions; +import dev.cel.common.values.CelByteString; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -70,17 +71,17 @@ public static List data() { return Arrays.asList( new Object[][] { { - NullValue.NULL_VALUE, + dev.cel.common.values.NullValue.NULL_VALUE, Any.pack(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()), }, {true, BoolValue.of(true)}, {true, Any.pack(BoolValue.of(true))}, {true, Value.newBuilder().setBoolValue(true).build()}, { - ByteString.copyFromUtf8("hello"), BytesValue.of(ByteString.copyFromUtf8("hello")), + CelByteString.copyFromUtf8("hello"), BytesValue.of(ByteString.copyFromUtf8("hello")), }, { - ByteString.copyFromUtf8("hello"), + CelByteString.copyFromUtf8("hello"), Any.pack(BytesValue.of(ByteString.copyFromUtf8("hello"))), }, {1.5D, DoubleValue.of(1.5D)}, @@ -116,7 +117,9 @@ public static List data() { .build()), }, { - ImmutableMap.of("list_value", ImmutableList.of(false, NullValue.NULL_VALUE)), + ImmutableMap.of( + "list_value", + ImmutableList.of(false, dev.cel.common.values.NullValue.NULL_VALUE)), Struct.newBuilder() .putFields( "list_value", @@ -149,8 +152,7 @@ public static List data() { @Test public void adaptValueToProto_bidirectionalConversion() { DynamicProto dynamicProto = DynamicProto.create(DefaultMessageFactory.INSTANCE); - ProtoAdapter protoAdapter = - new ProtoAdapter(dynamicProto, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(dynamicProto, CelOptions.DEFAULT); assertThat(protoAdapter.adaptValueToProto(value, proto.getDescriptorForType().getFullName())) .isEqualTo(proto); assertThat(protoAdapter.adaptProtoToValue(proto)).isEqualTo(value); @@ -169,7 +171,7 @@ public void adaptAnyValue_hermeticTypes_bidirectionalConversion() { typeName.equals(Expr.getDescriptor().getFullName()) ? Optional.of(Expr.newBuilder()) : Optional.empty()), - CelOptions.DEFAULT.enableUnsignedLongs()); + CelOptions.DEFAULT); assertThat(protoAdapter.adaptValueToProto(expr, Any.getDescriptor().getFullName())) .isEqualTo(Any.pack(expr)); assertThat(protoAdapter.adaptProtoToValue(Any.pack(expr))).isEqualTo(expr); @@ -180,8 +182,7 @@ public void adaptAnyValue_hermeticTypes_bidirectionalConversion() { public static class AsymmetricConversionTest { @Test public void adaptValueToProto_asymmetricFloatConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThat(protoAdapter.adaptValueToProto(1.5F, Any.getDescriptor().getFullName())) .isEqualTo(Any.pack(FloatValue.of(1.5F))); assertThat(protoAdapter.adaptProtoToValue(Any.pack(FloatValue.of(1.5F)))).isEqualTo(1.5D); @@ -189,8 +190,7 @@ public void adaptValueToProto_asymmetricFloatConversion() { @Test public void adaptValueToProto_asymmetricDoubleFloatConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThat(protoAdapter.adaptValueToProto(1.5D, FloatValue.getDescriptor().getFullName())) .isEqualTo(FloatValue.of(1.5F)); assertThat(protoAdapter.adaptProtoToValue(FloatValue.of(1.5F))).isEqualTo(1.5D); @@ -198,16 +198,14 @@ public void adaptValueToProto_asymmetricDoubleFloatConversion() { @Test public void adaptValueToProto_asymmetricFloatDoubleConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThat(protoAdapter.adaptValueToProto(1.5F, DoubleValue.getDescriptor().getFullName())) .isEqualTo(DoubleValue.of(1.5D)); } @Test public void adaptValueToProto_asymmetricJsonConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThat( protoAdapter.adaptValueToProto( UnsignedLong.valueOf(1L), Value.getDescriptor().getFullName())) @@ -223,14 +221,13 @@ public void adaptValueToProto_asymmetricJsonConversion() { .isEqualTo(Value.newBuilder().setStringValue(Long.toString(Long.MAX_VALUE)).build()); assertThat( protoAdapter.adaptValueToProto( - ByteString.copyFromUtf8("foo"), Value.getDescriptor().getFullName())) + CelByteString.copyFromUtf8("foo"), Value.getDescriptor().getFullName())) .isEqualTo(Value.newBuilder().setStringValue("Zm9v").build()); } @Test public void adaptValueToProto_unsupportedJsonConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThrows( ClassCastException.class, @@ -241,8 +238,7 @@ public void adaptValueToProto_unsupportedJsonConversion() { @Test public void adaptValueToProto_unsupportedJsonListConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThrows( ClassCastException.class, @@ -253,8 +249,7 @@ public void adaptValueToProto_unsupportedJsonListConversion() { @Test public void adaptValueToProto_unsupportedConversion() { - ProtoAdapter protoAdapter = - new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT.enableUnsignedLongs()); + ProtoAdapter protoAdapter = new ProtoAdapter(DYNAMIC_PROTO, CelOptions.DEFAULT); assertThrows( IllegalStateException.class, diff --git a/common/src/test/java/dev/cel/common/values/ProtoCelValueConverterTest.java b/common/src/test/java/dev/cel/common/values/ProtoCelValueConverterTest.java index f5dd756e2..b7e72c6b8 100644 --- a/common/src/test/java/dev/cel/common/values/ProtoCelValueConverterTest.java +++ b/common/src/test/java/dev/cel/common/values/ProtoCelValueConverterTest.java @@ -16,7 +16,6 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.protobuf.ByteString; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; import dev.cel.common.internal.DefaultDescriptorPool; @@ -56,21 +55,25 @@ public void fromCelValueToJavaObject_returnsDurationValue() { } @Test - public void fromCelValueToJavaObject_returnsBytesValue() { - ByteString byteString = - (ByteString) + public void fromCelValueToJavaObject_returnsCelBytesValue() { + CelByteString byteString = + (CelByteString) PROTO_CEL_VALUE_CONVERTER.fromCelValueToJavaObject( BytesValue.create(CelByteString.of(new byte[] {0x1, 0x5, 0xc}))); - assertThat(byteString).isEqualTo(ByteString.copyFrom(new byte[] {0x1, 0x5, 0xc})); + // Note: No conversion is attempted. CelByteString is the native Java type equivalent for CEL's + // bytes. + assertThat(byteString).isEqualTo(CelByteString.of(new byte[] {0x1, 0x5, 0xc})); } @Test - public void fromCelValueToJavaObject_returnsProtobufNullValue() { - com.google.protobuf.NullValue nullValue = - (com.google.protobuf.NullValue) - PROTO_CEL_VALUE_CONVERTER.fromCelValueToJavaObject(NullValue.NULL_VALUE); + public void fromCelValueToJavaObject_returnsCelNullValue() { + NullValue nullValue = + (NullValue) PROTO_CEL_VALUE_CONVERTER.fromCelValueToJavaObject(NullValue.NULL_VALUE); - assertThat(nullValue).isEqualTo(com.google.protobuf.NullValue.NULL_VALUE); + // Note: No conversion is attempted. We're using dev.cel.common.values.NullValue.NULL_VALUE as + // the + // sentinel type for CEL's `null`. + assertThat(nullValue).isEqualTo(NullValue.NULL_VALUE); } } diff --git a/common/src/test/java/dev/cel/common/values/ProtoMessageValueProviderTest.java b/common/src/test/java/dev/cel/common/values/ProtoMessageValueProviderTest.java index e8273dfae..891a7d0e2 100644 --- a/common/src/test/java/dev/cel/common/values/ProtoMessageValueProviderTest.java +++ b/common/src/test/java/dev/cel/common/values/ProtoMessageValueProviderTest.java @@ -22,6 +22,7 @@ import com.google.common.primitives.UnsignedLong; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import dev.cel.common.CelDescriptorUtil; +import dev.cel.common.CelOptions; import dev.cel.common.internal.CelDescriptorPool; import dev.cel.common.internal.DefaultDescriptorPool; import dev.cel.common.internal.DefaultMessageFactory; @@ -46,13 +47,12 @@ public class ProtoMessageValueProviderTest { TestAllTypes.getDescriptor().getFile(), TestAllTypesExtensions.getDescriptor()))); private static final ProtoMessageFactory MESSAGE_FACTORY = DefaultMessageFactory.create(DESCRIPTOR_POOL); - private static final DynamicProto DYNAMIC_PROTO = DynamicProto.create(MESSAGE_FACTORY); @Test public void newValue_createEmptyProtoMessage() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); ProtoMessageValue protoMessageValue = (ProtoMessageValue) @@ -66,7 +66,7 @@ public void newValue_createEmptyProtoMessage() { @Test public void newValue_createProtoMessage_fieldsPopulated() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); ProtoMessageValue protoMessageValue = (ProtoMessageValue) @@ -118,7 +118,7 @@ public void newValue_createProtoMessage_fieldsPopulated() { @Test public void newValue_createProtoMessage_unsignedLongFieldsPopulated() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); ProtoMessageValue protoMessageValue = (ProtoMessageValue) @@ -139,7 +139,7 @@ public void newValue_createProtoMessage_unsignedLongFieldsPopulated() { @Test public void newValue_createProtoMessage_wrappersPopulated() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); ProtoMessageValue protoMessageValue = (ProtoMessageValue) @@ -185,7 +185,7 @@ public void newValue_createProtoMessage_wrappersPopulated() { @Test public void newValue_createProtoMessage_extensionFieldsPopulated() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); ProtoMessageValue protoMessageValue = (ProtoMessageValue) @@ -206,7 +206,7 @@ public void newValue_createProtoMessage_extensionFieldsPopulated() { @Test public void newValue_invalidMessageName_returnsEmpty() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); assertThat(protoMessageValueProvider.newValue("bogus", ImmutableMap.of())).isEmpty(); } @@ -214,7 +214,7 @@ public void newValue_invalidMessageName_returnsEmpty() { @Test public void newValue_invalidField_throws() { ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); IllegalArgumentException e = assertThrows( @@ -234,7 +234,7 @@ public void newValue_invalidField_throws() { public void newValue_onCombinedProvider() { CelValueProvider celValueProvider = (structType, fields) -> Optional.empty(); ProtoMessageValueProvider protoMessageValueProvider = - ProtoMessageValueProvider.newInstance(DYNAMIC_PROTO); + ProtoMessageValueProvider.newInstance(CelOptions.DEFAULT, DYNAMIC_PROTO); CelValueProvider combinedProvider = CombinedCelValueProvider.combine(celValueProvider, protoMessageValueProvider); diff --git a/common/src/test/java/dev/cel/common/values/StructValueTest.java b/common/src/test/java/dev/cel/common/values/StructValueTest.java index acb2d6d56..1ab1137e0 100644 --- a/common/src/test/java/dev/cel/common/values/StructValueTest.java +++ b/common/src/test/java/dev/cel/common/values/StructValueTest.java @@ -187,7 +187,7 @@ public void evaluate_usingMultipleProviders_selectFieldFromCustomClass() throws .setValueProvider( CombinedCelValueProvider.combine( ProtoMessageValueProvider.newInstance( - DynamicProto.create(typeName -> Optional.empty())), + CelOptions.DEFAULT, DynamicProto.create(typeName -> Optional.empty())), CUSTOM_STRUCT_VALUE_PROVIDER)) .build(); CelAbstractSyntaxTree ast = cel.compile("custom_struct{data: 5}.data").getAst(); @@ -207,6 +207,7 @@ public void evaluate_usingMultipleProviders_selectFieldFromProtobufMessage() thr .setValueProvider( CombinedCelValueProvider.combine( ProtoMessageValueProvider.newInstance( + CelOptions.DEFAULT, // Note: this is unideal. Future iterations should make DynamicProto // completely an internal concern, and not expose it at all. DynamicProto.create( diff --git a/common/values/BUILD.bazel b/common/values/BUILD.bazel index 08bad9298..f1fa107b6 100644 --- a/common/values/BUILD.bazel +++ b/common/values/BUILD.bazel @@ -3,7 +3,7 @@ load("//:cel_android_rules.bzl", "cel_android_library") package( default_applicable_licenses = ["//:license"], - default_visibility = ["//:internal"], # TODO: Expose to public when ready + default_visibility = ["//visibility:public"], ) java_library( diff --git a/conformance/src/test/java/dev/cel/conformance/ConformanceTest.java b/conformance/src/test/java/dev/cel/conformance/ConformanceTest.java index b5ab8f533..c4cf8193d 100644 --- a/conformance/src/test/java/dev/cel/conformance/ConformanceTest.java +++ b/conformance/src/test/java/dev/cel/conformance/ConformanceTest.java @@ -54,6 +54,7 @@ public final class ConformanceTest extends Statement { private static final CelOptions OPTIONS = CelOptions.current() + .evaluateCanonicalTypesToNativeValues(true) .enableTimestampEpoch(true) .enableHeterogeneousNumericComparisons(true) .enableProtoDifferencerEquality(true) @@ -61,26 +62,36 @@ public final class ConformanceTest extends Statement { .enableQuotedIdentifierSyntax(true) .build(); + private static final CelParser PARSER_WITH_MACROS = + CelParserFactory.standardCelParserBuilder() + .setOptions(OPTIONS) + .addLibraries( + CelExtensions.bindings(), + CelExtensions.encoders(OPTIONS), + CelExtensions.math(OPTIONS), + CelExtensions.protos(), + CelExtensions.sets(OPTIONS), + CelExtensions.strings(), + CelOptionalLibrary.INSTANCE) + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .build(); + private static final CelParser PARSER_WITHOUT_MACROS = CelParserFactory.standardCelParserBuilder() .setOptions(OPTIONS) .addLibraries( CelExtensions.bindings(), - CelExtensions.encoders(), + CelExtensions.encoders(OPTIONS), CelExtensions.math(OPTIONS), CelExtensions.protos(), CelExtensions.sets(OPTIONS), CelExtensions.strings(), CelOptionalLibrary.INSTANCE) + .setStandardMacros() .build(); private static CelParser getParser(SimpleTest test) { - return test.getDisableMacros() - ? PARSER_WITHOUT_MACROS - : PARSER_WITHOUT_MACROS - .toParserBuilder() - .setStandardMacros(CelStandardMacro.STANDARD_MACROS) - .build(); + return test.getDisableMacros() ? PARSER_WITHOUT_MACROS : PARSER_WITH_MACROS; } private static CelChecker getChecker(SimpleTest test) throws Exception { @@ -96,7 +107,7 @@ private static CelChecker getChecker(SimpleTest test) throws Exception { .addFileTypes(dev.cel.expr.conformance.proto2.TestAllTypesExtensions.getDescriptor()) .addLibraries( CelExtensions.bindings(), - CelExtensions.encoders(), + CelExtensions.encoders(OPTIONS), CelExtensions.math(OPTIONS), CelExtensions.sets(OPTIONS), CelExtensions.strings(), @@ -110,7 +121,7 @@ private static CelChecker getChecker(SimpleTest test) throws Exception { CelRuntimeFactory.standardCelRuntimeBuilder() .setOptions(OPTIONS) .addLibraries( - CelExtensions.encoders(), + CelExtensions.encoders(OPTIONS), CelExtensions.math(OPTIONS), CelExtensions.sets(OPTIONS), CelExtensions.strings(), diff --git a/extensions/src/main/java/dev/cel/extensions/BUILD.bazel b/extensions/src/main/java/dev/cel/extensions/BUILD.bazel index e75043641..e32ca9a65 100644 --- a/extensions/src/main/java/dev/cel/extensions/BUILD.bazel +++ b/extensions/src/main/java/dev/cel/extensions/BUILD.bazel @@ -155,7 +155,9 @@ java_library( deps = [ "//checker:checker_builder", "//common:compiler_common", + "//common:options", "//common/types", + "//common/values:cel_byte_string", "//compiler:compiler_builder", "//extensions:extension_library", "//runtime", @@ -177,6 +179,8 @@ java_library( "//common:options", "//common/ast", "//common/types", + "//common/values", + "//common/values:cel_byte_string", "//compiler:compiler_builder", "//extensions:extension_library", "//parser:macro", diff --git a/extensions/src/main/java/dev/cel/extensions/CelEncoderExtensions.java b/extensions/src/main/java/dev/cel/extensions/CelEncoderExtensions.java index 4422d6710..a98f9db41 100644 --- a/extensions/src/main/java/dev/cel/extensions/CelEncoderExtensions.java +++ b/extensions/src/main/java/dev/cel/extensions/CelEncoderExtensions.java @@ -21,8 +21,10 @@ import com.google.protobuf.ByteString; import dev.cel.checker.CelCheckerBuilder; import dev.cel.common.CelFunctionDecl; +import dev.cel.common.CelOptions; import dev.cel.common.CelOverloadDecl; import dev.cel.common.types.SimpleType; +import dev.cel.common.values.CelByteString; import dev.cel.compiler.CelCompilerLibrary; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.CelRuntimeBuilder; @@ -41,6 +43,7 @@ public class CelEncoderExtensions private static final Decoder BASE64_DECODER = Base64.getDecoder(); private final ImmutableSet functions; + private final CelOptions celOptions; enum Function { DECODE( @@ -48,53 +51,67 @@ enum Function { "base64.decode", CelOverloadDecl.newGlobalOverload( "base64_decode_string", SimpleType.BYTES, SimpleType.STRING)), - ImmutableSet.of( - CelFunctionBinding.from( - "base64_decode_string", - String.class, - str -> ByteString.copyFrom(BASE64_DECODER.decode(str))))), + CelFunctionBinding.from( + "base64_decode_string", + String.class, + str -> CelByteString.of(BASE64_DECODER.decode(str))), + CelFunctionBinding.from( + "base64_decode_string", + String.class, + str -> ByteString.copyFrom(BASE64_DECODER.decode(str)))), ENCODE( CelFunctionDecl.newFunctionDeclaration( "base64.encode", CelOverloadDecl.newGlobalOverload( "base64_encode_bytes", SimpleType.STRING, SimpleType.BYTES)), - ImmutableSet.of( - CelFunctionBinding.from( - "base64_encode_bytes", - ByteString.class, - bytes -> BASE64_ENCODER.encodeToString(bytes.toByteArray())))), + CelFunctionBinding.from( + "base64_encode_bytes", + CelByteString.class, + bytes -> BASE64_ENCODER.encodeToString(bytes.toByteArray())), + CelFunctionBinding.from( + "base64_encode_bytes", + ByteString.class, + bytes -> BASE64_ENCODER.encodeToString(bytes.toByteArray()))), ; private final CelFunctionDecl functionDecl; - private final ImmutableSet functionBindings; + private final CelFunctionBinding nativeBytesFunctionBinding; + private final CelFunctionBinding protoBytesFunctionBinding; String getFunction() { return functionDecl.name(); } - Function(CelFunctionDecl functionDecl, ImmutableSet functionBindings) { + Function( + CelFunctionDecl functionDecl, + CelFunctionBinding nativeBytesFunctionBinding, + CelFunctionBinding protoBytesFunctionBinding) { this.functionDecl = functionDecl; - this.functionBindings = functionBindings; + this.nativeBytesFunctionBinding = nativeBytesFunctionBinding; + this.protoBytesFunctionBinding = protoBytesFunctionBinding; } } - private static final CelExtensionLibrary LIBRARY = - new CelExtensionLibrary() { - private final CelEncoderExtensions version0 = new CelEncoderExtensions(); + private static final class Library implements CelExtensionLibrary { + private final CelEncoderExtensions version0; - @Override - public String name() { - return "encoders"; - } + @Override + public String name() { + return "encoders"; + } + + @Override + public ImmutableSet versions() { + return ImmutableSet.of(version0); + } - @Override - public ImmutableSet versions() { - return ImmutableSet.of(version0); - } - }; + private Library(CelOptions celOptions) { + this.version0 = new CelEncoderExtensions(celOptions); + } + } - static CelExtensionLibrary library() { - return LIBRARY; + static CelExtensionLibrary library(CelOptions celOptions) { + return new Library(celOptions); } @Override @@ -115,10 +132,18 @@ public void setCheckerOptions(CelCheckerBuilder checkerBuilder) { @SuppressWarnings("Immutable") // Instances of java.util.Base64 are immutable @Override public void setRuntimeOptions(CelRuntimeBuilder runtimeBuilder) { - functions.forEach(function -> runtimeBuilder.addFunctionBindings(function.functionBindings)); + functions.forEach( + function -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + runtimeBuilder.addFunctionBindings(function.nativeBytesFunctionBinding); + } else { + runtimeBuilder.addFunctionBindings(function.protoBytesFunctionBinding); + } + }); } - public CelEncoderExtensions() { + CelEncoderExtensions(CelOptions celOptions) { + this.celOptions = celOptions; this.functions = ImmutableSet.copyOf(Function.values()); } } diff --git a/extensions/src/main/java/dev/cel/extensions/CelExtensions.java b/extensions/src/main/java/dev/cel/extensions/CelExtensions.java index a8dc56ce4..48bb99451 100644 --- a/extensions/src/main/java/dev/cel/extensions/CelExtensions.java +++ b/extensions/src/main/java/dev/cel/extensions/CelExtensions.java @@ -33,7 +33,6 @@ public final class CelExtensions { private static final CelStringExtensions STRING_EXTENSIONS_ALL = new CelStringExtensions(); private static final CelProtoExtensions PROTO_EXTENSIONS = new CelProtoExtensions(); private static final CelBindingsExtensions BINDINGS_EXTENSIONS = new CelBindingsExtensions(); - private static final CelEncoderExtensions ENCODER_EXTENSIONS = new CelEncoderExtensions(); private static final CelRegexExtensions REGEX_EXTENSIONS = new CelRegexExtensions(); /** @@ -192,14 +191,25 @@ public static CelBindingsExtensions bindings() { return BINDINGS_EXTENSIONS; } + /** + * @deprecated Use {@link #encoders(CelOptions) instead.} + */ + @Deprecated + public static CelEncoderExtensions encoders() { + return new CelEncoderExtensions(CelOptions.DEFAULT); + } + /** * Extended functions for string, byte and object encodings. * *

This adds {@code base64.encode} and {@code base64.decode} functions. See README.md for their * documentation. + * + * @param celOptions This should be the same {@link CelOptions} object used to configure + * compilation/runtime environments. */ - public static CelEncoderExtensions encoders() { - return ENCODER_EXTENSIONS; + public static CelEncoderExtensions encoders(CelOptions celOptions) { + return new CelEncoderExtensions(celOptions); } /** @@ -324,7 +334,7 @@ public static CelExtensionLibrary getE case "bindings": return CelBindingsExtensions.library(); case "encoders": - return CelEncoderExtensions.library(); + return CelEncoderExtensions.library(options); case "lists": return CelListsExtensions.library(); case "math": diff --git a/extensions/src/main/java/dev/cel/extensions/CelOptionalLibrary.java b/extensions/src/main/java/dev/cel/extensions/CelOptionalLibrary.java index a1e850a55..a384b09e2 100644 --- a/extensions/src/main/java/dev/cel/extensions/CelOptionalLibrary.java +++ b/extensions/src/main/java/dev/cel/extensions/CelOptionalLibrary.java @@ -23,10 +23,8 @@ import com.google.common.collect.Iterables; import com.google.common.primitives.Ints; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import com.google.protobuf.Duration; import com.google.protobuf.Message; -import com.google.protobuf.NullValue; import com.google.protobuf.Timestamp; import dev.cel.checker.CelCheckerBuilder; import dev.cel.common.CelFunctionDecl; @@ -41,6 +39,8 @@ import dev.cel.common.types.SimpleType; import dev.cel.common.types.TypeParamType; import dev.cel.common.types.TypeType; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import dev.cel.compiler.CelCompilerLibrary; import dev.cel.parser.CelMacro; import dev.cel.parser.CelMacroExprFactory; @@ -237,7 +237,9 @@ static CelExtensionLibrary library() { private final ImmutableSet variables; CelOptionalLibrary( - int version, ImmutableSet functions, ImmutableSet macros, + int version, + ImmutableSet functions, + ImmutableSet macros, ImmutableSet variables) { this.version = version; this.functions = functions; @@ -381,8 +383,8 @@ private static boolean isZeroValue(Object val) { return ((Collection) val).isEmpty(); } else if (val instanceof Map) { return ((Map) val).isEmpty(); - } else if (val instanceof ByteString) { - return ((ByteString) val).size() == 0; + } else if (val instanceof CelByteString) { + return ((CelByteString) val).isEmpty(); } else if (val instanceof Duration) { return val.equals(((Duration) val).getDefaultInstanceForType()); } else if (val instanceof Timestamp) { diff --git a/extensions/src/test/java/dev/cel/extensions/BUILD.bazel b/extensions/src/test/java/dev/cel/extensions/BUILD.bazel index cd2a0e42e..4d3e99e41 100644 --- a/extensions/src/test/java/dev/cel/extensions/BUILD.bazel +++ b/extensions/src/test/java/dev/cel/extensions/BUILD.bazel @@ -17,6 +17,8 @@ java_library( "//common/internal:proto_time_utils", "//common/types", "//common/types:type_providers", + "//common/values", + "//common/values:cel_byte_string", "//compiler", "//compiler:compiler_builder", "//extensions", @@ -40,7 +42,6 @@ java_library( "@maven//:com_google_protobuf_protobuf_java", "@maven//:com_google_testparameterinjector_test_parameter_injector", "@maven//:junit_junit", - "@maven_android//:com_google_protobuf_protobuf_javalite", ], ) diff --git a/extensions/src/test/java/dev/cel/extensions/CelEncoderExtensionsTest.java b/extensions/src/test/java/dev/cel/extensions/CelEncoderExtensionsTest.java index bbb24950f..ece060011 100644 --- a/extensions/src/test/java/dev/cel/extensions/CelEncoderExtensionsTest.java +++ b/extensions/src/test/java/dev/cel/extensions/CelEncoderExtensionsTest.java @@ -19,13 +19,13 @@ import static org.junit.Assert.assertThrows; import com.google.common.collect.ImmutableMap; -import com.google.protobuf.ByteString; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelFunctionDecl; import dev.cel.common.CelOptions; import dev.cel.common.CelValidationException; import dev.cel.common.types.SimpleType; +import dev.cel.common.values.CelByteString; import dev.cel.compiler.CelCompiler; import dev.cel.compiler.CelCompilerFactory; import dev.cel.runtime.CelEvaluationException; @@ -36,14 +36,19 @@ @RunWith(TestParameterInjector.class) public class CelEncoderExtensionsTest { + private static final CelOptions CEL_OPTIONS = + CelOptions.current().evaluateCanonicalTypesToNativeValues(true).build(); private static final CelCompiler CEL_COMPILER = CelCompilerFactory.standardCelCompilerBuilder() .addVar("stringVar", SimpleType.STRING) - .addLibraries(CelExtensions.encoders()) + .addLibraries(CelExtensions.encoders(CEL_OPTIONS)) .build(); private static final CelRuntime CEL_RUNTIME = - CelRuntimeFactory.standardCelRuntimeBuilder().addLibraries(CelExtensions.encoders()).build(); + CelRuntimeFactory.standardCelRuntimeBuilder() + .setOptions(CEL_OPTIONS) + .addLibraries(CelExtensions.encoders(CEL_OPTIONS)) + .build(); @Test public void library() { @@ -69,27 +74,27 @@ public void encode_success() throws Exception { @Test public void decode_success() throws Exception { - ByteString decodedBytes = - (ByteString) + CelByteString decodedBytes = + (CelByteString) CEL_RUNTIME .createProgram(CEL_COMPILER.compile("base64.decode('aGVsbG8=')").getAst()) .eval(); assertThat(decodedBytes.size()).isEqualTo(5); - assertThat(decodedBytes.toString(ISO_8859_1)).isEqualTo("hello"); + assertThat(new String(decodedBytes.toByteArray(), ISO_8859_1)).isEqualTo("hello"); } @Test public void decode_withoutPadding_success() throws Exception { - ByteString decodedBytes = - (ByteString) + CelByteString decodedBytes = + (CelByteString) CEL_RUNTIME // RFC2045 6.8, padding can be ignored. .createProgram(CEL_COMPILER.compile("base64.decode('aGVsbG8')").getAst()) .eval(); assertThat(decodedBytes.size()).isEqualTo(5); - assertThat(decodedBytes.toString(ISO_8859_1)).isEqualTo("hello"); + assertThat(new String(decodedBytes.toByteArray(), ISO_8859_1)).isEqualTo("hello"); } @Test @@ -99,13 +104,13 @@ public void roundTrip_success() throws Exception { CEL_RUNTIME .createProgram(CEL_COMPILER.compile("base64.encode(b'Hello World!')").getAst()) .eval(); - ByteString decodedBytes = - (ByteString) + CelByteString decodedBytes = + (CelByteString) CEL_RUNTIME .createProgram(CEL_COMPILER.compile("base64.decode(stringVar)").getAst()) .eval(ImmutableMap.of("stringVar", encodedString)); - assertThat(decodedBytes.toString(ISO_8859_1)).isEqualTo("Hello World!"); + assertThat(new String(decodedBytes.toByteArray(), ISO_8859_1)).isEqualTo("Hello World!"); } @Test @@ -141,4 +146,3 @@ public void decode_malformedBase64Char_throwsEvaluationException() throws Except assertThat(e).hasCauseThat().hasMessageThat().contains("Illegal base64 character"); } } - diff --git a/extensions/src/test/java/dev/cel/extensions/CelOptionalLibraryTest.java b/extensions/src/test/java/dev/cel/extensions/CelOptionalLibraryTest.java index 1e6bbded6..a806037da 100644 --- a/extensions/src/test/java/dev/cel/extensions/CelOptionalLibraryTest.java +++ b/extensions/src/test/java/dev/cel/extensions/CelOptionalLibraryTest.java @@ -20,9 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import com.google.protobuf.DoubleValue; -import com.google.protobuf.NullValue; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import com.google.testing.junit.testparameterinjector.TestParameters; @@ -44,6 +42,8 @@ import dev.cel.common.types.SimpleType; import dev.cel.common.types.StructTypeReference; import dev.cel.common.types.TypeType; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import dev.cel.expr.conformance.proto3.TestAllTypes; import dev.cel.expr.conformance.proto3.TestAllTypes.NestedMessage; import dev.cel.parser.CelMacro; @@ -69,7 +69,7 @@ private enum ConstantTestCases { DOUBLE("5.2", "0.0", SimpleType.DOUBLE, 5.2d), UINT("5u", "0u", SimpleType.UINT, UnsignedLong.valueOf(5)), BOOL("true", "false", SimpleType.BOOL, true), - BYTES("b'abc'", "b''", SimpleType.BYTES, ByteString.copyFromUtf8("abc")), + BYTES("b'abc'", "b''", SimpleType.BYTES, CelByteString.copyFromUtf8("abc")), DURATION( "duration('180s')", "duration('0s')", @@ -101,7 +101,11 @@ private static CelBuilder newCelBuilder() { private static CelBuilder newCelBuilder(int version) { return CelFactory.standardCelBuilder() - .setOptions(CelOptions.current().enableTimestampEpoch(true).build()) + .setOptions( + CelOptions.current() + .evaluateCanonicalTypesToNativeValues(true) + .enableTimestampEpoch(true) + .build()) .setStandardMacros(CelStandardMacro.STANDARD_MACROS) .setContainer(CelContainer.ofName("cel.expr.conformance.proto3")) .addMessageTypes(TestAllTypes.getDescriptor()) diff --git a/parser/src/main/java/dev/cel/parser/BUILD.bazel b/parser/src/main/java/dev/cel/parser/BUILD.bazel index d4ffb0c5c..2bbc8c0e2 100644 --- a/parser/src/main/java/dev/cel/parser/BUILD.bazel +++ b/parser/src/main/java/dev/cel/parser/BUILD.bazel @@ -139,8 +139,8 @@ java_library( "//common:cel_source", "//common/ast", "//common/ast:cel_expr_visitor", + "//common/values:cel_byte_string", "@maven//:com_google_guava_guava", - "@maven//:com_google_protobuf_protobuf_java", "@maven//:com_google_re2j_re2j", ], ) diff --git a/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java b/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java index 11683618b..398a554ca 100644 --- a/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java +++ b/parser/src/main/java/dev/cel/parser/CelUnparserVisitor.java @@ -14,7 +14,6 @@ package dev.cel.parser; import com.google.common.collect.ImmutableSet; -import com.google.protobuf.ByteString; import com.google.re2j.Pattern; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelSource; @@ -29,6 +28,7 @@ import dev.cel.common.ast.CelExpr.CelStruct; import dev.cel.common.ast.CelExpr.ExprKind.Kind; import dev.cel.common.ast.CelExprVisitor; +import dev.cel.common.values.CelByteString; import java.util.HashSet; import java.util.Optional; @@ -360,7 +360,7 @@ private boolean isComplexOperatorWithRespectTo(CelExpr expr, String op) { // bytesToOctets converts byte sequences to a string using a three digit octal encoded value // per byte. - private String bytesToOctets(ByteString bytes) { + private String bytesToOctets(CelByteString bytes) { StringBuilder sb = new StringBuilder(); for (byte b : bytes.toByteArray()) { sb.append(String.format("\\%03o", b)); diff --git a/parser/src/test/java/dev/cel/parser/BUILD.bazel b/parser/src/test/java/dev/cel/parser/BUILD.bazel index f56c080ce..b6a89ff66 100644 --- a/parser/src/test/java/dev/cel/parser/BUILD.bazel +++ b/parser/src/test/java/dev/cel/parser/BUILD.bazel @@ -21,6 +21,7 @@ java_library( "//common:source_location", "//common/ast", "//common/internal", + "//common/values:cel_byte_string", "//extensions:optional_library", "//parser", "//parser:macro", @@ -37,7 +38,6 @@ java_library( "@maven//:com_google_protobuf_protobuf_java", "@maven//:com_google_testparameterinjector_test_parameter_injector", "@maven//:junit_junit", - "@maven_android//:com_google_protobuf_protobuf_javalite", ], ) diff --git a/parser/src/test/java/dev/cel/parser/CelMacroExprFactoryTest.java b/parser/src/test/java/dev/cel/parser/CelMacroExprFactoryTest.java index 249b752c5..72abf81a4 100644 --- a/parser/src/test/java/dev/cel/parser/CelMacroExprFactoryTest.java +++ b/parser/src/test/java/dev/cel/parser/CelMacroExprFactoryTest.java @@ -17,7 +17,6 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.ByteString; import dev.cel.common.CelIssue; import dev.cel.common.CelSourceLocation; import dev.cel.common.ast.CelConstant; @@ -26,6 +25,7 @@ import dev.cel.common.ast.CelExpr.CelStruct.Entry; import dev.cel.common.ast.CelExpr.ExprKind.Kind; import dev.cel.common.internal.Constants; +import dev.cel.common.values.CelByteString; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -103,7 +103,7 @@ public void newBytesLiteral_returnsBytesConstant() { assertThat(expr.id()).isEqualTo(1L); assertThat(expr.exprKind().getKind()).isEqualTo(Kind.CONSTANT); assertThat(expr.constant().getKind()).isEqualTo(CelConstant.Kind.BYTES_VALUE); - assertThat(expr.constant().bytesValue()).isEqualTo(ByteString.copyFromUtf8("foo")); + assertThat(expr.constant().bytesValue()).isEqualTo(CelByteString.copyFromUtf8("foo")); } @Test diff --git a/runtime/src/main/java/dev/cel/runtime/BUILD.bazel b/runtime/src/main/java/dev/cel/runtime/BUILD.bazel index b594a1753..7c0539d00 100644 --- a/runtime/src/main/java/dev/cel/runtime/BUILD.bazel +++ b/runtime/src/main/java/dev/cel/runtime/BUILD.bazel @@ -110,6 +110,7 @@ java_library( "//common/internal:dynamic_proto", "//common/internal:proto_message_factory", "//common/types:cel_types", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", @@ -206,6 +207,7 @@ java_library( deps = [ "//common/types", "//common/types:type_providers", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", @@ -219,6 +221,7 @@ cel_android_library( deps = [ "//common/types:type_providers_android", "//common/types:types_android", + "//common/values:cel_byte_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven_android//:com_google_guava_guava", @@ -301,9 +304,11 @@ java_library( "//common/ast", "//common/types", "//common/types:type_providers", + "//common/values:cel_byte_string", "@maven//:com_google_code_findbugs_annotations", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", "@maven//:org_jspecify_jspecify", ], ) @@ -337,10 +342,12 @@ cel_android_library( "//common/ast:ast_android", "//common/types:type_providers_android", "//common/types:types_android", + "//common/values:cel_byte_string", "@maven//:com_google_code_findbugs_annotations", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven_android//:com_google_guava_guava", + "@maven_android//:com_google_protobuf_protobuf_javalite", ], ) @@ -414,6 +421,7 @@ cel_android_library( "//common:runtime_exception", "//common/annotations", "//common/internal:converter", + "//common/values:values_android", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_re2j_re2j", "@maven//:org_threeten_threeten_extra", @@ -436,6 +444,7 @@ java_library( "//common:runtime_exception", "//common/annotations", "//common/internal:converter", + "//common/values", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", @@ -843,7 +852,6 @@ java_library( "//common/types:cel_types", "//common/values:cel_value_provider", "//common/values:proto_message_value_provider", - "//runtime/standard:standard_function", "@maven//:com_google_code_findbugs_annotations", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", diff --git a/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java b/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java index 2a6131723..2947445d1 100644 --- a/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java +++ b/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java @@ -301,7 +301,7 @@ public CelRuntimeLegacyImpl build() { CelValueProvider messageValueProvider = celValueProvider; if (messageValueProvider == null) { - messageValueProvider = ProtoMessageValueProvider.newInstance(dynamicProto); + messageValueProvider = ProtoMessageValueProvider.newInstance(options, dynamicProto); } runtimeTypeProvider = CelValueRuntimeTypeProvider.newInstance(messageValueProvider); diff --git a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java index f029b2d3b..f0b606e98 100644 --- a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java +++ b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java @@ -25,6 +25,8 @@ import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.Immutable; import javax.annotation.concurrent.ThreadSafe; +import com.google.protobuf.ByteString; +import com.google.protobuf.NullValue; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelErrorCode; import dev.cel.common.CelOptions; @@ -42,6 +44,7 @@ import dev.cel.common.types.CelKind; import dev.cel.common.types.CelType; import dev.cel.common.types.TypeType; +import dev.cel.common.values.CelByteString; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; @@ -288,7 +291,9 @@ private Object evalConstant( ExecutionFrame unusedFrame, CelExpr unusedExpr, CelConstant constExpr) { switch (constExpr.getKind()) { case NULL_VALUE: - return constExpr.nullValue(); + return celOptions.evaluateCanonicalTypesToNativeValues() + ? constExpr.nullValue() + : NullValue.NULL_VALUE; case BOOLEAN_VALUE: return constExpr.booleanValue(); case INT64_VALUE: @@ -304,7 +309,12 @@ private Object evalConstant( case STRING_VALUE: return constExpr.stringValue(); case BYTES_VALUE: - return constExpr.bytesValue(); + CelByteString celByteString = constExpr.bytesValue(); + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return celByteString; + } + + return ByteString.copyFrom(celByteString.toByteArray()); default: throw new IllegalStateException("unsupported constant case: " + constExpr.getKind()); } diff --git a/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java b/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java index aefc8eb96..ab6934e01 100644 --- a/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java +++ b/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java @@ -15,6 +15,7 @@ package dev.cel.runtime; import com.google.errorprone.annotations.Immutable; +import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Message; @@ -28,6 +29,7 @@ import dev.cel.common.internal.ProtoAdapter; import dev.cel.common.internal.ProtoMessageFactory; import dev.cel.common.types.CelTypes; +import dev.cel.common.values.CelByteString; import java.util.Map; import java.util.Optional; import org.jspecify.annotations.Nullable; @@ -45,6 +47,7 @@ @Internal public final class DescriptorMessageProvider implements RuntimeTypeProvider { private final ProtoMessageFactory protoMessageFactory; + private final CelOptions celOptions; @SuppressWarnings("Immutable") private final ProtoAdapter protoAdapter; @@ -65,9 +68,8 @@ public DescriptorMessageProvider(MessageFactory messageFactory) { */ public DescriptorMessageProvider(ProtoMessageFactory protoMessageFactory, CelOptions celOptions) { this.protoMessageFactory = protoMessageFactory; - this.protoAdapter = - new ProtoAdapter( - DynamicProto.create(protoMessageFactory), celOptions.enableUnsignedLongs()); + this.celOptions = celOptions; + this.protoAdapter = new ProtoAdapter(DynamicProto.create(protoMessageFactory), celOptions); } @Override @@ -144,6 +146,10 @@ public Object adapt(String messageName, Object message) { return protoAdapter.adaptProtoToValue((Message) message); } + if (celOptions.evaluateCanonicalTypesToNativeValues() && message instanceof ByteString) { + return CelByteString.of(((ByteString) message).toByteArray()); + } + return message; } diff --git a/runtime/src/main/java/dev/cel/runtime/LiteRuntimeImpl.java b/runtime/src/main/java/dev/cel/runtime/LiteRuntimeImpl.java index 6836b2254..152c96160 100644 --- a/runtime/src/main/java/dev/cel/runtime/LiteRuntimeImpl.java +++ b/runtime/src/main/java/dev/cel/runtime/LiteRuntimeImpl.java @@ -201,7 +201,11 @@ public CelLiteRuntime build() { } private Builder() { - this.celOptions = CelOptions.current().enableCelValue(true).build(); + this.celOptions = + CelOptions.current() + .enableCelValue(true) + .evaluateCanonicalTypesToNativeValues(true) + .build(); this.celValueProvider = (structType, fields) -> Optional.empty(); this.customFunctionBindings = new HashMap<>(); this.standardFunctionBuilder = ImmutableSet.builder(); diff --git a/runtime/src/main/java/dev/cel/runtime/ProtoMessageActivationFactory.java b/runtime/src/main/java/dev/cel/runtime/ProtoMessageActivationFactory.java index 6833db43a..fb42a1964 100644 --- a/runtime/src/main/java/dev/cel/runtime/ProtoMessageActivationFactory.java +++ b/runtime/src/main/java/dev/cel/runtime/ProtoMessageActivationFactory.java @@ -41,8 +41,7 @@ public static Activation fromProto(Message message, CelOptions celOptions) { Map msgFieldValues = message.getAllFields(); ProtoAdapter protoAdapter = - new ProtoAdapter( - DynamicProto.create(DefaultMessageFactory.INSTANCE), celOptions.enableUnsignedLongs()); + new ProtoAdapter(DynamicProto.create(DefaultMessageFactory.INSTANCE), celOptions); boolean skipUnsetFields = celOptions.fromProtoUnsetFieldOption().equals(CelOptions.ProtoUnsetFieldOptions.SKIP); diff --git a/runtime/src/main/java/dev/cel/runtime/ProtoMessageRuntimeHelpers.java b/runtime/src/main/java/dev/cel/runtime/ProtoMessageRuntimeHelpers.java index b18310b5b..85acd373c 100644 --- a/runtime/src/main/java/dev/cel/runtime/ProtoMessageRuntimeHelpers.java +++ b/runtime/src/main/java/dev/cel/runtime/ProtoMessageRuntimeHelpers.java @@ -35,8 +35,7 @@ public final class ProtoMessageRuntimeHelpers extends RuntimeHelpers { @Internal public static ProtoMessageRuntimeHelpers create( DynamicProto dynamicProto, CelOptions celOptions) { - return new ProtoMessageRuntimeHelpers( - new ProtoAdapter(dynamicProto, celOptions.enableUnsignedLongs())); + return new ProtoMessageRuntimeHelpers(new ProtoAdapter(dynamicProto, celOptions)); } /** diff --git a/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java b/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java index a95f0b831..37247f3d3 100644 --- a/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java +++ b/runtime/src/main/java/dev/cel/runtime/RuntimeHelpers.java @@ -22,13 +22,13 @@ import com.google.errorprone.annotations.Immutable; import com.google.protobuf.Duration; import com.google.protobuf.MessageLiteOrBuilder; -import com.google.protobuf.NullValue; import com.google.re2j.Pattern; import dev.cel.common.CelErrorCode; import dev.cel.common.CelOptions; import dev.cel.common.CelRuntimeException; import dev.cel.common.annotations.Internal; import dev.cel.common.internal.Converter; +import dev.cel.common.values.NullValue; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.List; @@ -332,7 +332,7 @@ static Converter identity() { /** Adapts a plain old Java object into a CEL value. */ public Object adaptValue(Object value) { - if (value == null) { + if (value == null || value.equals(com.google.protobuf.NullValue.NULL_VALUE)) { return NullValue.NULL_VALUE; } if (value instanceof Number) { diff --git a/runtime/src/main/java/dev/cel/runtime/TypeResolver.java b/runtime/src/main/java/dev/cel/runtime/TypeResolver.java index a996ed151..b9cc42cc1 100644 --- a/runtime/src/main/java/dev/cel/runtime/TypeResolver.java +++ b/runtime/src/main/java/dev/cel/runtime/TypeResolver.java @@ -34,6 +34,7 @@ import dev.cel.common.types.StructType; import dev.cel.common.types.StructTypeReference; import dev.cel.common.types.TypeType; +import dev.cel.common.values.CelByteString; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -68,12 +69,16 @@ static TypeResolver create() { .put(ArrayList.class, TypeType.create(ListType.create(SimpleType.DYN))) .put(HashMap.class, TypeType.create(MapType.create(SimpleType.DYN, SimpleType.DYN))) .put(Optional.class, TypeType.create(OptionalType.create(SimpleType.DYN))) + .put(CelByteString.class, TypeType.create(SimpleType.BYTES)) .buildOrThrow(); private static final ImmutableMap, TypeType> EXTENDABLE_TYPES = ImmutableMap., TypeType>builder() + .put( + ByteString.class, + TypeType.create( + SimpleType.BYTES)) // TODO: Remove once clients have been migrated .put(Collection.class, TypeType.create(ListType.create(SimpleType.DYN))) - .put(ByteString.class, TypeType.create(SimpleType.BYTES)) .put(Map.class, TypeType.create(MapType.create(SimpleType.DYN, SimpleType.DYN))) .buildOrThrow(); diff --git a/runtime/src/main/java/dev/cel/runtime/standard/AddOperator.java b/runtime/src/main/java/dev/cel/runtime/standard/AddOperator.java index 813d19522..9d75b15ce 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/AddOperator.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/AddOperator.java @@ -24,6 +24,7 @@ import dev.cel.common.CelOptions; import dev.cel.common.CelRuntimeException; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import dev.cel.runtime.RuntimeHelpers; @@ -90,9 +91,15 @@ public enum AddOverload implements CelStandardOverload { } }), ADD_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( - "add_bytes", ByteString.class, ByteString.class, ByteString::concat)), + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "add_bytes", CelByteString.class, CelByteString.class, CelByteString::concat); + } else { + return CelFunctionBinding.from( + "add_bytes", ByteString.class, ByteString.class, ByteString::concat); + } + }), ADD_DOUBLE( (celOptions, runtimeEquality) -> CelFunctionBinding.from("add_double", Double.class, Double.class, Double::sum)), diff --git a/runtime/src/main/java/dev/cel/runtime/standard/BUILD.bazel b/runtime/src/main/java/dev/cel/runtime/standard/BUILD.bazel index 7284041f9..bac80475d 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/BUILD.bazel +++ b/runtime/src/main/java/dev/cel/runtime/standard/BUILD.bazel @@ -51,6 +51,7 @@ java_library( "//common:options", "//common:runtime_exception", "//common/internal:proto_time_utils", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime:runtime_helpers", @@ -71,6 +72,7 @@ cel_android_library( "//common:options", "//common:runtime_exception", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "//runtime:runtime_helpers_android", @@ -163,6 +165,7 @@ java_library( deps = [ ":standard_overload", "//common:options", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime/standard:standard_function", @@ -180,6 +183,7 @@ cel_android_library( ":standard_function_android", ":standard_overload_android", "//common:options", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "@maven_android//:com_google_guava_guava", @@ -737,6 +741,7 @@ java_library( "//common:options", "//common/internal:comparison_functions", "//common/internal:proto_time_utils", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime:runtime_helpers", @@ -757,6 +762,7 @@ cel_android_library( "//common:options", "//common/internal:comparison_functions_android", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "//runtime:runtime_helpers_android", @@ -775,6 +781,7 @@ java_library( "//common:options", "//common/internal:comparison_functions", "//common/internal:proto_time_utils", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime:runtime_helpers", @@ -795,6 +802,7 @@ cel_android_library( "//common:options", "//common/internal:comparison_functions_android", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "//runtime:runtime_helpers_android", @@ -915,6 +923,7 @@ java_library( "//common:options", "//common/internal:comparison_functions", "//common/internal:proto_time_utils", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime:runtime_helpers", @@ -935,6 +944,7 @@ cel_android_library( "//common:options", "//common/internal:comparison_functions_android", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "//runtime:runtime_helpers_android", @@ -953,6 +963,7 @@ java_library( "//common:options", "//common/internal:comparison_functions", "//common/internal:proto_time_utils", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime:runtime_helpers", @@ -973,6 +984,7 @@ cel_android_library( "//common:options", "//common/internal:comparison_functions_android", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "//runtime:runtime_helpers_android", @@ -1229,6 +1241,7 @@ java_library( deps = [ ":standard_overload", "//common:options", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime/standard:standard_function", @@ -1246,6 +1259,7 @@ cel_android_library( ":standard_function_android", ":standard_overload_android", "//common:options", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "@maven_android//:com_google_guava_guava", @@ -1294,6 +1308,7 @@ java_library( "//common:options", "//common:runtime_exception", "//common/internal:proto_time_utils", + "//common/values:cel_byte_string", "//runtime:function_binding", "//runtime:runtime_equality", "//runtime/standard:standard_function", @@ -1314,6 +1329,7 @@ cel_android_library( "//common:options", "//common:runtime_exception", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//runtime:function_binding_android", "//runtime:runtime_equality_android", "@maven_android//:com_google_guava_guava", diff --git a/runtime/src/main/java/dev/cel/runtime/standard/BytesFunction.java b/runtime/src/main/java/dev/cel/runtime/standard/BytesFunction.java index 64ebe3e75..b45d6d65b 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/BytesFunction.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/BytesFunction.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableSet; import com.google.protobuf.ByteString; import dev.cel.common.CelOptions; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import java.util.Arrays; @@ -40,11 +41,17 @@ public static BytesFunction create(Iterable overloa /** Overloads for the standard function. */ public enum BytesOverload implements CelStandardOverload { BYTES_TO_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from("bytes_to_bytes", ByteString.class, (ByteString x) -> x)), + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "bytes_to_bytes", CelByteString.class, (CelByteString x) -> x); + } else { + return CelFunctionBinding.from("bytes_to_bytes", ByteString.class, (ByteString x) -> x); + } + }), STRING_TO_BYTES( (celOptions, runtimeEquality) -> - CelFunctionBinding.from("string_to_bytes", String.class, ByteString::copyFromUtf8)), + CelFunctionBinding.from("string_to_bytes", String.class, CelByteString::copyFromUtf8)), ; private final FunctionBindingCreator bindingCreator; diff --git a/runtime/src/main/java/dev/cel/runtime/standard/GreaterEqualsOperator.java b/runtime/src/main/java/dev/cel/runtime/standard/GreaterEqualsOperator.java index 6cca22da3..1eaced251 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/GreaterEqualsOperator.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/GreaterEqualsOperator.java @@ -22,6 +22,7 @@ import dev.cel.common.CelOptions; import dev.cel.common.internal.ComparisonFunctions; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import dev.cel.runtime.RuntimeHelpers; @@ -55,13 +56,23 @@ public enum GreaterEqualsOverload implements CelStandardOverload { Boolean.class, (Boolean x, Boolean y) -> x || !y)), GREATER_EQUALS_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "greater_equals_bytes", + CelByteString.class, + CelByteString.class, + (CelByteString x, CelByteString y) -> + CelByteString.unsignedLexicographicalComparator().compare(x, y) >= 0); + } else { + return CelFunctionBinding.from( "greater_equals_bytes", ByteString.class, ByteString.class, (ByteString x, ByteString y) -> - ByteString.unsignedLexicographicalComparator().compare(x, y) >= 0)), + ByteString.unsignedLexicographicalComparator().compare(x, y) >= 0); + } + }), GREATER_EQUALS_DOUBLE( (celOptions, runtimeEquality) -> CelFunctionBinding.from( diff --git a/runtime/src/main/java/dev/cel/runtime/standard/GreaterOperator.java b/runtime/src/main/java/dev/cel/runtime/standard/GreaterOperator.java index 8b051ce81..9eed5747b 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/GreaterOperator.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/GreaterOperator.java @@ -22,6 +22,7 @@ import dev.cel.common.CelOptions; import dev.cel.common.internal.ComparisonFunctions; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import dev.cel.runtime.RuntimeHelpers; @@ -50,13 +51,23 @@ public enum GreaterOverload implements CelStandardOverload { CelFunctionBinding.from( "greater_bool", Boolean.class, Boolean.class, (Boolean x, Boolean y) -> x && !y)), GREATER_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "greater_bytes", + CelByteString.class, + CelByteString.class, + (CelByteString x, CelByteString y) -> + CelByteString.unsignedLexicographicalComparator().compare(x, y) > 0); + } else { + return CelFunctionBinding.from( "greater_bytes", ByteString.class, ByteString.class, (ByteString x, ByteString y) -> - ByteString.unsignedLexicographicalComparator().compare(x, y) > 0)), + ByteString.unsignedLexicographicalComparator().compare(x, y) > 0); + } + }), GREATER_DOUBLE( (celOptions, runtimeEquality) -> CelFunctionBinding.from( diff --git a/runtime/src/main/java/dev/cel/runtime/standard/LessEqualsOperator.java b/runtime/src/main/java/dev/cel/runtime/standard/LessEqualsOperator.java index d695cfafa..3da852f8a 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/LessEqualsOperator.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/LessEqualsOperator.java @@ -22,6 +22,7 @@ import dev.cel.common.CelOptions; import dev.cel.common.internal.ComparisonFunctions; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import dev.cel.runtime.RuntimeHelpers; @@ -54,13 +55,23 @@ public enum LessEqualsOverload implements CelStandardOverload { Boolean.class, (Boolean x, Boolean y) -> !x || y)), LESS_EQUALS_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "less_equals_bytes", + CelByteString.class, + CelByteString.class, + (CelByteString x, CelByteString y) -> + CelByteString.unsignedLexicographicalComparator().compare(x, y) <= 0); + } else { + return CelFunctionBinding.from( "less_equals_bytes", ByteString.class, ByteString.class, (ByteString x, ByteString y) -> - ByteString.unsignedLexicographicalComparator().compare(x, y) <= 0)), + ByteString.unsignedLexicographicalComparator().compare(x, y) <= 0); + } + }), LESS_EQUALS_DOUBLE( (celOptions, runtimeEquality) -> CelFunctionBinding.from( diff --git a/runtime/src/main/java/dev/cel/runtime/standard/LessOperator.java b/runtime/src/main/java/dev/cel/runtime/standard/LessOperator.java index c96f3ec7a..8bae06a85 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/LessOperator.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/LessOperator.java @@ -22,6 +22,7 @@ import dev.cel.common.CelOptions; import dev.cel.common.internal.ComparisonFunctions; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import dev.cel.runtime.RuntimeHelpers; @@ -70,13 +71,23 @@ public enum LessOverload implements CelStandardOverload { } }), LESS_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "less_bytes", + CelByteString.class, + CelByteString.class, + (CelByteString x, CelByteString y) -> + CelByteString.unsignedLexicographicalComparator().compare(x, y) < 0); + } else { + return CelFunctionBinding.from( "less_bytes", ByteString.class, ByteString.class, (ByteString x, ByteString y) -> - ByteString.unsignedLexicographicalComparator().compare(x, y) < 0)), + ByteString.unsignedLexicographicalComparator().compare(x, y) < 0); + } + }), LESS_DOUBLE( (celOptions, runtimeEquality) -> CelFunctionBinding.from( diff --git a/runtime/src/main/java/dev/cel/runtime/standard/SizeFunction.java b/runtime/src/main/java/dev/cel/runtime/standard/SizeFunction.java index b31f8aa3b..206e4bdd4 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/SizeFunction.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/SizeFunction.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableSet; import com.google.protobuf.ByteString; import dev.cel.common.CelOptions; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import java.util.Arrays; @@ -43,13 +44,25 @@ public static SizeFunction create(Iterable overloads) @SuppressWarnings("rawtypes") public enum SizeOverload implements CelStandardOverload { SIZE_BYTES( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( - "size_bytes", ByteString.class, (ByteString bytes) -> (long) bytes.size())), + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "size_bytes", CelByteString.class, (CelByteString bytes) -> (long) bytes.size()); + } else { + return CelFunctionBinding.from( + "size_bytes", ByteString.class, (ByteString bytes) -> (long) bytes.size()); + } + }), BYTES_SIZE( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( - "bytes_size", ByteString.class, (ByteString bytes) -> (long) bytes.size())), + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "bytes_size", CelByteString.class, (CelByteString bytes) -> (long) bytes.size()); + } else { + return CelFunctionBinding.from( + "bytes_size", ByteString.class, (ByteString bytes) -> (long) bytes.size()); + } + }), SIZE_LIST( (celOptions, runtimeEquality) -> CelFunctionBinding.from("size_list", List.class, (List list1) -> (long) list1.size())), diff --git a/runtime/src/main/java/dev/cel/runtime/standard/StringFunction.java b/runtime/src/main/java/dev/cel/runtime/standard/StringFunction.java index 0229205e5..3525ba5e0 100644 --- a/runtime/src/main/java/dev/cel/runtime/standard/StringFunction.java +++ b/runtime/src/main/java/dev/cel/runtime/standard/StringFunction.java @@ -24,6 +24,7 @@ import dev.cel.common.CelOptions; import dev.cel.common.CelRuntimeException; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelFunctionBinding; import dev.cel.runtime.RuntimeEquality; import java.util.Arrays; @@ -59,8 +60,22 @@ public enum StringOverload implements CelStandardOverload { (celOptions, runtimeEquality) -> CelFunctionBinding.from("bool_to_string", Boolean.class, Object::toString)), BYTES_TO_STRING( - (celOptions, runtimeEquality) -> - CelFunctionBinding.from( + (celOptions, runtimeEquality) -> { + if (celOptions.evaluateCanonicalTypesToNativeValues()) { + return CelFunctionBinding.from( + "bytes_to_string", + CelByteString.class, + (byteStr) -> { + if (!byteStr.isValidUtf8()) { + throw new CelRuntimeException( + new IllegalArgumentException( + "invalid UTF-8 in bytes, cannot convert to string"), + CelErrorCode.BAD_FORMAT); + } + return byteStr.toStringUtf8(); + }); + } else { + return CelFunctionBinding.from( "bytes_to_string", ByteString.class, (byteStr) -> { @@ -71,7 +86,9 @@ public enum StringOverload implements CelStandardOverload { CelErrorCode.BAD_FORMAT); } return byteStr.toStringUtf8(); - })), + }); + } + }), TIMESTAMP_TO_STRING( (celOptions, runtimeEquality) -> CelFunctionBinding.from( diff --git a/runtime/src/test/java/dev/cel/runtime/ActivationTest.java b/runtime/src/test/java/dev/cel/runtime/ActivationTest.java index 0f7ba0193..5e3f3f1fe 100644 --- a/runtime/src/test/java/dev/cel/runtime/ActivationTest.java +++ b/runtime/src/test/java/dev/cel/runtime/ActivationTest.java @@ -17,8 +17,8 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.primitives.UnsignedLong; -import com.google.protobuf.NullValue; import dev.cel.common.CelOptions; +import dev.cel.common.values.NullValue; import dev.cel.expr.conformance.proto3.NestedTestAllTypes; import dev.cel.expr.conformance.proto3.TestAllTypes; import dev.cel.expr.conformance.proto3.TestAllTypes.NestedMessage; diff --git a/runtime/src/test/java/dev/cel/runtime/BUILD.bazel b/runtime/src/test/java/dev/cel/runtime/BUILD.bazel index 2bdfbcad4..5d083cdc9 100644 --- a/runtime/src/test/java/dev/cel/runtime/BUILD.bazel +++ b/runtime/src/test/java/dev/cel/runtime/BUILD.bazel @@ -48,6 +48,8 @@ java_library( "//common/types", "//common/types:cel_v1alpha1_types", "//common/types:message_type_provider", + "//common/values", + "//common/values:cel_byte_string", "//common/values:cel_value_provider", "//common/values:proto_message_lite_value_provider", "//compiler", @@ -137,6 +139,7 @@ cel_android_local_test( "//common:cel_ast_android", "//common:options", "//common/internal:proto_time_utils_android", + "//common/values:cel_byte_string", "//common/values:cel_value_provider_android", "//common/values:proto_message_lite_value_provider_android", "//extensions:lite_extensions_android", diff --git a/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeAndroidTest.java b/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeAndroidTest.java index 61ce0db8f..2896e36f4 100644 --- a/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeAndroidTest.java +++ b/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeAndroidTest.java @@ -38,6 +38,7 @@ import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelOptions; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import dev.cel.common.values.CelValueProvider; import dev.cel.common.values.ProtoMessageLiteValueProvider; import dev.cel.expr.conformance.proto3.NestedTestAllTypes; @@ -245,7 +246,7 @@ public void eval_primitiveVariables() throws Exception { "bool_var", true, "bytes_var", - ByteString.copyFromUtf8("abc"), + CelByteString.copyFromUtf8("abc"), "double_var", 1.0, "int_var", @@ -603,7 +604,8 @@ public void eval_protoMessage_repeatedFields(String checkedExpr) throws Exceptio ImmutableList.of(23.3d, 24.4d), ImmutableList.of(true, false), ImmutableList.of("alpha", "beta"), - ImmutableList.of(ByteString.copyFromUtf8("gamma"), ByteString.copyFromUtf8("delta"))) + ImmutableList.of( + CelByteString.copyFromUtf8("gamma"), CelByteString.copyFromUtf8("delta"))) .inOrder(); } @@ -695,7 +697,7 @@ public void eval_protoMessage_mapFields(String checkedExpr) throws Exception { ImmutableMap.of(true, false, false, true), ImmutableMap.of(true, "foo", false, "bar"), ImmutableMap.of( - true, ByteString.copyFromUtf8("baz"), false, ByteString.copyFromUtf8("qux")), + true, CelByteString.copyFromUtf8("baz"), false, CelByteString.copyFromUtf8("qux")), ImmutableMap.of(true, 1L, false, 2L), ImmutableMap.of(true, 3L, false, 4L), ImmutableMap.of(true, UnsignedLong.valueOf(5), false, UnsignedLong.valueOf(6)), diff --git a/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeTest.java b/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeTest.java index e71ae718c..4083ae494 100644 --- a/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeTest.java +++ b/runtime/src/test/java/dev/cel/runtime/CelLiteRuntimeTest.java @@ -30,7 +30,6 @@ import com.google.protobuf.Int32Value; import com.google.protobuf.Int64Value; import com.google.protobuf.ListValue; -import com.google.protobuf.NullValue; import com.google.protobuf.StringValue; import com.google.protobuf.Struct; import com.google.protobuf.Timestamp; @@ -47,6 +46,8 @@ import dev.cel.common.internal.ProtoTimeUtils; import dev.cel.common.types.SimpleType; import dev.cel.common.types.StructTypeReference; +import dev.cel.common.values.CelByteString; +import dev.cel.common.values.NullValue; import dev.cel.common.values.ProtoMessageLiteValueProvider; import dev.cel.compiler.CelCompiler; import dev.cel.compiler.CelCompilerFactory; @@ -550,7 +551,7 @@ private enum DefaultValueTestCase { DOUBLE("msg.single_double", 0.0d), BOOL("msg.single_bool", false), STRING("msg.single_string", ""), - BYTES("msg.single_bytes", ByteString.EMPTY), + BYTES("msg.single_bytes", CelByteString.EMPTY), ENUM("msg.standalone_enum", 0L), NESTED_MESSAGE("msg.single_nested_message", NestedMessage.getDefaultInstance()), OPTIONAL_BOOL("msg.optional_bool", false), diff --git a/runtime/src/test/java/dev/cel/runtime/DescriptorMessageProviderTest.java b/runtime/src/test/java/dev/cel/runtime/DescriptorMessageProviderTest.java index 5dcab8c10..8866d1fa6 100644 --- a/runtime/src/test/java/dev/cel/runtime/DescriptorMessageProviderTest.java +++ b/runtime/src/test/java/dev/cel/runtime/DescriptorMessageProviderTest.java @@ -124,7 +124,8 @@ public void createMessage_unsetWrapperField() { (dev.cel.expr.conformance.proto3.TestAllTypes) provider.createMessage( dev.cel.expr.conformance.proto3.TestAllTypes.getDescriptor().getFullName(), - ImmutableMap.of("single_int64_wrapper", NullValue.NULL_VALUE)); + ImmutableMap.of( + "single_int64_wrapper", dev.cel.common.values.NullValue.NULL_VALUE)); assertThat(message).isEqualToDefaultInstance(); } @@ -135,7 +136,7 @@ public void createMessage_badFieldError() { () -> provider.createMessage( dev.cel.expr.conformance.proto3.TestAllTypes.getDescriptor().getFullName(), - ImmutableMap.of("bad_field", NullValue.NULL_VALUE))); + ImmutableMap.of("bad_field", dev.cel.common.values.NullValue.NULL_VALUE))); } @Test diff --git a/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeEqualityTest.java b/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeEqualityTest.java index bb4b839e1..16806c856 100644 --- a/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeEqualityTest.java +++ b/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeEqualityTest.java @@ -49,6 +49,7 @@ import dev.cel.common.internal.DefaultMessageFactory; import dev.cel.common.internal.DynamicProto; import dev.cel.common.internal.ProtoTimeUtils; +import dev.cel.common.values.CelByteString; import java.util.Arrays; import java.util.List; import org.jspecify.annotations.Nullable; @@ -276,14 +277,14 @@ public static List data() { // Bytes tests. {ByteString.copyFromUtf8("h¢"), ByteString.copyFromUtf8("h¢"), Result.alwaysTrue()}, {ByteString.copyFromUtf8("hello"), ByteString.EMPTY, Result.alwaysFalse()}, - {BytesValue.of(ByteString.EMPTY), ByteString.EMPTY, Result.alwaysTrue()}, + {BytesValue.of(ByteString.EMPTY), CelByteString.EMPTY, Result.alwaysTrue()}, { BytesValue.of(ByteString.copyFromUtf8("h¢")), - ByteString.copyFromUtf8("h¢"), + CelByteString.copyFromUtf8("h¢"), Result.alwaysTrue() }, - {Any.pack(BytesValue.of(ByteString.EMPTY)), ByteString.EMPTY, Result.alwaysTrue()}, - {"h¢", ByteString.copyFromUtf8("h¢"), Result.alwaysFalse()}, + {Any.pack(BytesValue.of(ByteString.EMPTY)), CelByteString.EMPTY, Result.alwaysTrue()}, + {"h¢", CelByteString.copyFromUtf8("h¢"), Result.alwaysFalse()}, // Double tests. {1.0, 1.0, Result.alwaysTrue()}, diff --git a/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeHelpersTest.java b/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeHelpersTest.java index 985fbcc42..f6d701741 100644 --- a/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeHelpersTest.java +++ b/runtime/src/test/java/dev/cel/runtime/ProtoMessageRuntimeHelpersTest.java @@ -39,6 +39,7 @@ import dev.cel.common.CelRuntimeException; import dev.cel.common.internal.DefaultMessageFactory; import dev.cel.common.internal.DynamicProto; +import dev.cel.common.values.CelByteString; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -340,7 +341,7 @@ public void maybeAdaptPrimitive_optionalValues() { public void adaptProtoToValue_wrapperValues() throws Exception { assertThat(RUNTIME_HELPER.adaptProtoToValue(BoolValue.of(true))).isEqualTo(true); assertThat(RUNTIME_HELPER.adaptProtoToValue(BytesValue.of(ByteString.EMPTY))) - .isEqualTo(ByteString.EMPTY); + .isEqualTo(CelByteString.EMPTY); assertThat(RUNTIME_HELPER.adaptProtoToValue(DoubleValue.of(1.5d))).isEqualTo(1.5d); assertThat(RUNTIME_HELPER.adaptProtoToValue(FloatValue.of(1.5f))).isEqualTo(1.5d); assertThat(RUNTIME_HELPER.adaptProtoToValue(Int32Value.of(12))).isEqualTo(12L); @@ -372,7 +373,7 @@ public void adaptProtoToValue_jsonValues() throws Exception { .isEqualTo(ImmutableList.of(1.2d)); Map mp = new HashMap<>(); - mp.put("list_value", ImmutableList.of(false, NullValue.NULL_VALUE)); + mp.put("list_value", ImmutableList.of(false, dev.cel.common.values.NullValue.NULL_VALUE)); assertThat( RUNTIME_HELPER.adaptProtoToValue( Struct.newBuilder() @@ -404,7 +405,7 @@ public void adaptProtoToValue_anyValues() throws Exception { .build()) .build(); Any anyJsonValue = Any.pack(jsonValue); - mp.put("list_value", ImmutableList.of(false, NullValue.NULL_VALUE)); + mp.put("list_value", ImmutableList.of(false, dev.cel.common.values.NullValue.NULL_VALUE)); assertThat(RUNTIME_HELPER.adaptProtoToValue(anyJsonValue)).isEqualTo(mp); } diff --git a/testing/src/main/java/dev/cel/testing/BUILD.bazel b/testing/src/main/java/dev/cel/testing/BUILD.bazel index 1ee08bfce..27286829b 100644 --- a/testing/src/main/java/dev/cel/testing/BUILD.bazel +++ b/testing/src/main/java/dev/cel/testing/BUILD.bazel @@ -84,6 +84,7 @@ java_library( "//common/resources/testdata/proto3:standalone_global_enum_java_proto", "//common/types", "//common/types:type_providers", + "//common/values:cel_byte_string", "//extensions:optional_library", "//runtime", "//runtime:function_binding", diff --git a/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java b/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java index 07847ec0d..cd0da5b80 100644 --- a/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java +++ b/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java @@ -32,7 +32,6 @@ import com.google.protobuf.Any; import com.google.protobuf.BoolValue; import com.google.protobuf.ByteString; -import com.google.protobuf.ByteString.ByteIterator; import com.google.protobuf.BytesValue; import com.google.protobuf.DescriptorProtos.FileDescriptorSet; import com.google.protobuf.Descriptors.Descriptor; @@ -70,6 +69,7 @@ import dev.cel.common.types.SimpleType; import dev.cel.common.types.StructTypeReference; import dev.cel.common.types.TypeParamType; +import dev.cel.common.values.CelByteString; import dev.cel.expr.conformance.proto3.TestAllTypes; import dev.cel.expr.conformance.proto3.TestAllTypes.NestedEnum; import dev.cel.expr.conformance.proto3.TestAllTypes.NestedMessage; @@ -110,6 +110,7 @@ public abstract class BaseInterpreterTest extends CelBaselineTestCase { .enableTimestampEpoch(true) .enableHeterogeneousNumericComparisons(true) .enableOptionalSyntax(true) + .evaluateCanonicalTypesToNativeValues(true) .comprehensionMaxIterations(1_000) .build(); private CelRuntime celRuntime; @@ -208,10 +209,10 @@ private Object runTestInternal( ? program.eval(((Map) input)) : program.eval((CelVariableResolver) input); } - if (result instanceof ByteString) { + if (result instanceof CelByteString) { // Note: this call may fail for printing byte sequences that are not valid UTF-8, but works // pretty well for test purposes. - result = ((ByteString) result).toStringUtf8(); + result = ((CelByteString) result).toStringUtf8(); } println("result: " + UnredactedDebugFormatForTest.unredactedToString(result)); } catch (CelEvaluationException e) { @@ -1396,7 +1397,7 @@ public void sizeTests() { declareVariable("b", SimpleType.BYTES); source = "size(b) == 5 && b.size() == 5"; - runTest(ImmutableMap.of("b", ByteString.copyFromUtf8("happy"))); + runTest(ImmutableMap.of("b", CelByteString.copyFromUtf8("happy"))); source = "size(str) == 5 && str.size() == 5"; runTest(ImmutableMap.of("str", "happy")); @@ -2320,8 +2321,8 @@ private void printBinding(Object input) { sb.append(entry.getKey()); sb.append("="); Object value = entry.getValue(); - if (value instanceof ByteString) { - sb.append(getHumanReadableString((ByteString) value)); + if (value instanceof CelByteString) { + sb.append(getHumanReadableString((CelByteString) value)); } else { sb.append(UnredactedDebugFormatForTest.unredactedToString(entry.getValue())); } @@ -2333,14 +2334,15 @@ private void printBinding(Object input) { } } - private static String getHumanReadableString(ByteString byteString) { + private static String getHumanReadableString(CelByteString byteString) { // Very unfortunate we have to do this at all StringBuilder sb = new StringBuilder(); sb.append("["); - for (ByteIterator i = byteString.iterator(); i.hasNext(); ) { - byte b = i.nextByte(); + byte[] bytes = byteString.toByteArray(); + for (int i = 0; i < bytes.length; i++) { + byte b = bytes[i]; sb.append(b); - if (i.hasNext()) { + if (i < bytes.length - 1) { sb.append(", "); } } diff --git a/testing/src/main/java/dev/cel/testing/utils/BUILD.bazel b/testing/src/main/java/dev/cel/testing/utils/BUILD.bazel index d6646fd32..a7e4c628c 100644 --- a/testing/src/main/java/dev/cel/testing/utils/BUILD.bazel +++ b/testing/src/main/java/dev/cel/testing/utils/BUILD.bazel @@ -19,6 +19,8 @@ java_library( "//common/internal:default_instance_message_factory", "//common/types", "//common/types:type_providers", + "//common/values", + "//common/values:cel_byte_string", "//runtime:unknown_attributes", "//testing/testrunner:registry_utils", "@cel_spec//proto/cel/expr:expr_java_proto", diff --git a/testing/src/main/java/dev/cel/testing/utils/ExprValueUtils.java b/testing/src/main/java/dev/cel/testing/utils/ExprValueUtils.java index 10a502537..0b27fa452 100644 --- a/testing/src/main/java/dev/cel/testing/utils/ExprValueUtils.java +++ b/testing/src/main/java/dev/cel/testing/utils/ExprValueUtils.java @@ -37,6 +37,7 @@ import dev.cel.common.types.OptionalType; import dev.cel.common.types.SimpleType; import dev.cel.common.types.TypeType; +import dev.cel.common.values.CelByteString; import dev.cel.runtime.CelUnknownSet; import dev.cel.testing.testrunner.RegistryUtils; import java.io.IOException; @@ -92,7 +93,7 @@ public static Object fromValue(Value value) throws IOException { private static Object toNativeObject(Value value) throws IOException { switch (value.getKindCase()) { case NULL_VALUE: - return value.getNullValue(); + return dev.cel.common.values.NullValue.NULL_VALUE; case BOOL_VALUE: return value.getBoolValue(); case INT64_VALUE: @@ -104,7 +105,8 @@ private static Object toNativeObject(Value value) throws IOException { case STRING_VALUE: return value.getStringValue(); case BYTES_VALUE: - return value.getBytesValue(); + ByteString byteString = value.getBytesValue(); + return CelByteString.of(byteString.toByteArray()); case ENUM_VALUE: return value.getEnumValue(); case MAP_VALUE: @@ -170,8 +172,8 @@ public static Value toValue(Object object, CelType type) throws Exception { if (!(object instanceof Optional) && type instanceof OptionalType) { return toValue(object, type.parameters().get(0)); } - if (object == null) { - object = NullValue.NULL_VALUE; + if (object == null || object.equals(NullValue.NULL_VALUE)) { + object = dev.cel.common.values.NullValue.NULL_VALUE; } if (object instanceof dev.cel.expr.Value) { object = @@ -181,8 +183,8 @@ public static Value toValue(Object object, CelType type) throws Exception { if (object instanceof Value) { return (Value) object; } - if (object instanceof NullValue) { - return Value.newBuilder().setNullValue((NullValue) object).build(); + if (object instanceof dev.cel.common.values.NullValue) { + return Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build(); } if (object instanceof Boolean) { return Value.newBuilder().setBoolValue((Boolean) object).build(); @@ -224,8 +226,10 @@ public static Value toValue(Object object, CelType type) throws Exception { throw new IllegalArgumentException(String.format("Unexpected result type: %s", type)); } } - if (object instanceof ByteString) { - return Value.newBuilder().setBytesValue((ByteString) object).build(); + if (object instanceof CelByteString) { + return Value.newBuilder() + .setBytesValue(ByteString.copyFrom(((CelByteString) object).toByteArray())) + .build(); } if (object instanceof List) { CelType elemType = type instanceof ListType ? ((ListType) type).elemType() : SimpleType.DYN; @@ -280,7 +284,7 @@ private static Message unpackAny( return defaultInstance.getParserForType().parseFrom(value.getValue(), extensionRegistry); } - private static Message getDefaultInstance(Descriptor descriptor) throws IOException { + private static Message getDefaultInstance(Descriptor descriptor) { return DefaultInstanceMessageFactory.getInstance() .getPrototype(descriptor) .orElseThrow(