test: improve gherkin test suite only relying on the newest version, with data loading

pom.xml

@@ -168,6 +168,31 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>dev.cel</groupId>
+            <artifactId>cel</artifactId>
+            <version>0.10.1</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <dependencyManagement>
@@ -191,6 +216,14 @@
             </dependency>
             <!-- End mockito workaround-->
 
+            <dependency>
+                <groupId>com.fasterxml.jackson</groupId>
+                <artifactId>jackson-bom</artifactId>
+                <version>2.16.1</version> <!-- Use the desired version -->
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+
             <dependency>
                 <groupId>io.cucumber</groupId>
                 <artifactId>cucumber-bom</artifactId>

src/main/java/dev/openfeature/sdk/providers/memory/InMemoryProvider.java

@@ -138,11 +138,21 @@ private <T> ProviderEvaluation<T> getEvaluation(
         }
         Flag<?> flag = flags.get(key);
         if (flag == null) {
-            throw new FlagNotFoundError("flag " + key + "not found");
+            throw new FlagNotFoundError("flag " + key + " not found");
         }
         T value;
+        Reason reason = Reason.STATIC;
         if (flag.getContextEvaluator() != null) {
-            value = (T) flag.getContextEvaluator().evaluate(flag, evaluationContext);
+            try {
+                value = (T) flag.getContextEvaluator().evaluate(flag, evaluationContext);
+                reason = Reason.TARGETING_MATCH;
+            } catch (Exception e) {
+                value = null;
+            }
+            if (value == null) {
+                value = (T) flag.getVariants().get(flag.getDefaultVariant());
+                reason = Reason.DEFAULT;
+            }
         } else if (!expectedType.isInstance(flag.getVariants().get(flag.getDefaultVariant()))) {
             throw new TypeMismatchError("flag " + key + "is not of expected type");
         } else {
@@ -151,7 +161,7 @@ private <T> ProviderEvaluation<T> getEvaluation(
         return ProviderEvaluation.<T>builder()
                 .value(value)
                 .variant(flag.getDefaultVariant())
-                .reason(Reason.STATIC.toString())
+                .reason(reason.toString())
                 .flagMetadata(flag.getFlagMetadata())
                 .build();
     }

src/test/java/dev/openfeature/sdk/e2e/EvaluationTest.java → src/test/java/dev/openfeature/sdk/e2e/GherkinSpecTest.java

@@ -5,6 +5,7 @@
 import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME;
 
 import org.junit.platform.suite.api.ConfigurationParameter;
+import org.junit.platform.suite.api.ExcludeTags;
 import org.junit.platform.suite.api.IncludeEngines;
 import org.junit.platform.suite.api.SelectDirectories;
 import org.junit.platform.suite.api.Suite;
@@ -15,4 +16,5 @@
 @ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty")
 @ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.sdk.e2e.steps")
 @ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory")
-public class EvaluationTest {}
+@ExcludeTags({"deprecated", "reason-codes", "async", "immutability", "evaluation-options"})
+public class GherkinSpecTest {}

src/test/java/dev/openfeature/sdk/e2e/Utils.java

@@ -1,9 +1,14 @@
 package dev.openfeature.sdk.e2e;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import dev.openfeature.sdk.Value;
 import java.util.Objects;
 
 public final class Utils {
 
+    public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
     private Utils() {}
 
     public static Object convert(String value, String type) {
@@ -22,6 +27,12 @@ public static Object convert(String value, String type) {
                 return Double.parseDouble(value);
             case "long":
                 return Long.parseLong(value);
+            case "object":
+                try {
+                    return Value.objectToValue(OBJECT_MAPPER.readValue(value, Object.class));
+                } catch (JsonProcessingException e) {
+                    throw new RuntimeException(e);
+                }
         }
         throw new RuntimeException("Unknown config type: " + type);
     }

src/test/java/dev/openfeature/sdk/e2e/steps/ContextSteps.java

@@ -8,11 +8,14 @@
 import dev.openfeature.sdk.Hook;
 import dev.openfeature.sdk.HookContext;
 import dev.openfeature.sdk.ImmutableContext;
+import dev.openfeature.sdk.ImmutableStructure;
+import dev.openfeature.sdk.MutableContext;
 import dev.openfeature.sdk.OpenFeatureAPI;
 import dev.openfeature.sdk.ThreadLocalTransactionContextPropagator;
 import dev.openfeature.sdk.Value;
 import dev.openfeature.sdk.e2e.ContextStoringProvider;
 import dev.openfeature.sdk.e2e.State;
+import dev.openfeature.sdk.e2e.Utils;
 import io.cucumber.datatable.DataTable;
 import io.cucumber.java.en.And;
 import io.cucumber.java.en.Given;
@@ -101,4 +104,29 @@ public void contextEntriesForEachLevelFromAPILevelDownToTheLevelWithKeyAndValue(
             }
         }
     }
+
+    @Given("a context containing a key {string} with null value")
+    public void a_context_containing_a_key_with_null_value(String key) {
+        a_context_containing_a_key_with_type_and_with_value(key, "String", null);
+    }
+
+    @Given("a context containing a key {string}, with type {string} and with value {string}")
+    public void a_context_containing_a_key_with_type_and_with_value(String key, String type, String value) {
+        Map<String, Value> map = state.context.asMap();
+        map.put(key, Value.objectToValue(Utils.convert(value, type)));
+        state.context = new MutableContext(state.context.getTargetingKey(), map);
+    }
+
+    @Given("a context containing a targeting key with value {string}")
+    public void a_context_containing_a_targeting_key_with_value(String string) {
+        state.context.setTargetingKey(string);
+    }
+
+    @Given("a context containing a nested property with outer key {string} and inner key {string}, with value {string}")
+    public void a_context_containing_a_nested_property_with_outer_key_and_inner_key_with_value(
+            String outer, String inner, String value) {
+        Map<String, Value> innerMap = new HashMap<>();
+        innerMap.put(inner, new Value(value));
+        state.context.add(outer, new ImmutableStructure(innerMap));
+    }
 }

src/test/java/dev/openfeature/sdk/e2e/steps/FlagStepDefinitions.java

@@ -2,6 +2,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import dev.openfeature.sdk.ErrorCode;
 import dev.openfeature.sdk.FlagEvaluationDetails;
 import dev.openfeature.sdk.ImmutableMetadata;
 import dev.openfeature.sdk.Value;
@@ -60,7 +61,20 @@ public void the_flag_was_evaluated_with_details() {
 
     @Then("the resolved details value should be {string}")
     public void the_resolved_details_value_should_be(String value) {
-        assertThat(state.evaluation.getValue()).isEqualTo(Utils.convert(value, state.flag.type));
+        Object evaluationValue = state.evaluation.getValue();
+        if (state.flag.type.equalsIgnoreCase("object")) {
+            assertThat(((Value) evaluationValue).asStructure().asObjectMap())
+                    .isEqualTo(((Value) Utils.convert(value, state.flag.type))
+                            .asStructure()
+                            .asObjectMap());
+        } else {
+            assertThat(evaluationValue).isEqualTo(Utils.convert(value, state.flag.type));
+        }
+    }
+
+    @Then("the flag key should be {string}")
+    public void the_flag_key_should_be(String key) {
+        assertThat(state.evaluation.getFlagKey()).isEqualTo(key);
     }
 
     @Then("the reason should be {string}")
@@ -73,6 +87,20 @@ public void the_variant_should_be(String variant) {
         assertThat(state.evaluation.getVariant()).isEqualTo(variant);
     }
 
+    @Then("the error-code should be {string}")
+    public void the_error_code_should_be(String errorCode) {
+        if (errorCode.isEmpty()) {
+            assertThat(state.evaluation.getErrorCode()).isNull();
+        } else {
+            assertThat(state.evaluation.getErrorCode()).isEqualTo(ErrorCode.valueOf(errorCode));
+        }
+    }
+
+    @Then("the error message should contain {string}")
+    public void the_error_message_should_contain(String messageSubstring) {
+        assertThat(state.evaluation.getErrorMessage()).contains(messageSubstring);
+    }
+
     @Then("the resolved metadata value \"{}\" with type \"{}\" should be \"{}\"")
     public void theResolvedMetadataValueShouldBe(String key, String type, String value)
             throws NoSuchFieldException, IllegalAccessException {

src/test/java/dev/openfeature/sdk/e2e/steps/ProviderSteps.java

@@ -1,13 +1,31 @@
 package dev.openfeature.sdk.e2e.steps;
 
 import static dev.openfeature.sdk.testutils.TestFlagsUtils.buildFlags;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
+import dev.openfeature.sdk.ErrorCode;
+import dev.openfeature.sdk.EventProvider;
+import dev.openfeature.sdk.FeatureProvider;
 import dev.openfeature.sdk.OpenFeatureAPI;
+import dev.openfeature.sdk.ProviderEvaluation;
+import dev.openfeature.sdk.ProviderEventDetails;
+import dev.openfeature.sdk.ProviderState;
+import dev.openfeature.sdk.Reason;
+import dev.openfeature.sdk.Value;
 import dev.openfeature.sdk.e2e.State;
+import dev.openfeature.sdk.exceptions.FatalError;
 import dev.openfeature.sdk.providers.memory.Flag;
 import dev.openfeature.sdk.providers.memory.InMemoryProvider;
 import io.cucumber.java.en.Given;
+import io.cucumber.java.en.Then;
 import java.util.Map;
+import org.awaitility.Awaitility;
 
 public class ProviderSteps {
     private final State state;
@@ -16,11 +34,127 @@ public ProviderSteps(State state) {
         this.state = state;
     }
 
-    @Given("a stable provider")
-    public void aStableProvider() {
+    @Given("a {} provider")
+    public void a_provider_with_status(String providerType) throws Exception {
+        // Normalize input to handle both single word and quoted strings
+        String normalizedType =
+                providerType.toLowerCase().replaceAll("[\"\\s]+", " ").trim();
+
+        switch (normalizedType) {
+            case "not ready":
+                setupMockProvider(ErrorCode.PROVIDER_NOT_READY, "Provider in not ready state", ProviderState.NOT_READY);
+                break;
+            case "stable":
+            case "ready":
+                setupStableProvider();
+                break;
+            case "fatal":
+                setupMockProvider(ErrorCode.PROVIDER_FATAL, "Provider in fatal state", ProviderState.FATAL);
+                break;
+            case "error":
+                setupMockProvider(ErrorCode.GENERAL, "Provider in error state", ProviderState.ERROR);
+                break;
+            case "stale":
+                setupMockProvider(null, null, ProviderState.STALE);
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported provider type: " + providerType);
+        }
+    }
+
+    // ===============================
+    // Provider Status Assertion Steps
+    // ===============================
+
+    @Then("the provider status should be {string}")
+    public void the_provider_status_should_be(String expectedStatus) {
+        ProviderState actualStatus = state.client.getProviderState();
+        ProviderState expected = ProviderState.valueOf(expectedStatus);
+        assertThat(actualStatus).isEqualTo(expected);
+    }
+
+    // ===============================
+    // Helper Methods
+    // ===============================
+
+    private void setupStableProvider() throws Exception {
         Map<String, Flag<?>> flags = buildFlags();
         InMemoryProvider provider = new InMemoryProvider(flags);
         OpenFeatureAPI.getInstance().setProviderAndWait(provider);
         state.client = OpenFeatureAPI.getInstance().getClient();
     }
+
+    private void setupMockProvider(ErrorCode errorCode, String errorMessage, ProviderState providerState)
+            throws Exception {
+        EventProvider mockProvider = spy(EventProvider.class);
+
+        switch (providerState) {
+            case NOT_READY:
+                doAnswer(invocationOnMock -> {
+                            while (true) {}
+                        })
+                        .when(mockProvider)
+                        .initialize(any());
+                break;
+            case FATAL:
+                doThrow(new FatalError(errorMessage)).when(mockProvider).initialize(any());
+                break;
+        }
+        // Configure all evaluation methods with a single helper
+        configureMockEvaluations(mockProvider, errorCode, errorMessage);
+
+        OpenFeatureAPI.getInstance().setProvider(providerState.name(), mockProvider);
+        state.client = OpenFeatureAPI.getInstance().getClient(providerState.name());
+
+        ProviderEventDetails details =
+                ProviderEventDetails.builder().errorCode(errorCode).build();
+        switch (providerState) {
+            case FATAL:
+            case ERROR:
+                mockProvider.emitProviderReady(details);
+                mockProvider.emitProviderError(details);
+                break;
+            case STALE:
+                mockProvider.emitProviderReady(details);
+                mockProvider.emitProviderStale(details);
+                break;
+            default:
+        }
+        Awaitility.await().until(() -> {
+            ProviderState providerState1 = state.client.getProviderState();
+            return providerState1 == providerState;
+        });
+    }
+
+    private void configureMockEvaluations(FeatureProvider mockProvider, ErrorCode errorCode, String errorMessage) {
+        // Configure Boolean evaluation
+        when(mockProvider.getBooleanEvaluation(anyString(), any(Boolean.class), any()))
+                .thenAnswer(invocation -> createProviderEvaluation(invocation.getArgument(1), errorCode, errorMessage));
+
+        // Configure String evaluation
+        when(mockProvider.getStringEvaluation(anyString(), any(String.class), any()))
+                .thenAnswer(invocation -> createProviderEvaluation(invocation.getArgument(1), errorCode, errorMessage));
+
+        // Configure Integer evaluation
+        when(mockProvider.getIntegerEvaluation(anyString(), any(Integer.class), any()))
+                .thenAnswer(invocation -> createProviderEvaluation(invocation.getArgument(1), errorCode, errorMessage));
+
+        // Configure Double evaluation
+        when(mockProvider.getDoubleEvaluation(anyString(), any(Double.class), any()))
+                .thenAnswer(invocation -> createProviderEvaluation(invocation.getArgument(1), errorCode, errorMessage));
+
+        // Configure Object evaluation
+        when(mockProvider.getObjectEvaluation(anyString(), any(Value.class), any()))
+                .thenAnswer(invocation -> createProviderEvaluation(invocation.getArgument(1), errorCode, errorMessage));
+    }
+
+    private <T> ProviderEvaluation<T> createProviderEvaluation(
+            T defaultValue, ErrorCode errorCode, String errorMessage) {
+        return ProviderEvaluation.<T>builder()
+                .value(defaultValue)
+                .errorCode(errorCode)
+                .errorMessage(errorMessage)
+                .reason(Reason.ERROR.toString())
+                .build();
+    }
 }

src/test/java/dev/openfeature/sdk/e2e/steps/StepDefinitions.java

@@ -21,6 +21,7 @@
 import java.util.Map;
 import lombok.SneakyThrows;
 
+@Deprecated
 public class StepDefinitions {
 
     private static Client client;