Handle incompatible OpenFeature SDK event APIs gracefully

.gitlab-ci.yml

@@ -1070,6 +1070,16 @@ test_smoke:
   parallel:
     matrix: *test_matrix_8
 
+test_openfeature_compatibility:
+  extends: .test_job_amd64_with_test_agent
+  needs: *needs_build_tests_smoke
+  variables:
+    <<: *tier_l_variables
+    GRADLE_TARGET: "stageMainDist :dd-smoke-tests:openfeature:openFeatureCompatibilityTest"
+    GRADLE_PARAMS: "-PskipFlakyTests"
+    CACHE_TYPE: "smoke"
+    testJvm: "11"
+
 test_smoke_arm64:
   extends: .test_job_arm64_with_test_agent
   variables:

dd-smoke-tests/openfeature/application/build.gradle

@@ -16,6 +16,16 @@ if (hasProperty('appBuildDir')) {
 
 version = ""
 
+configurations.configureEach {
+  resolutionStrategy.componentSelection.all { selection ->
+    if (selection.candidate.group == 'dev.openfeature' &&
+      selection.candidate.module == 'sdk' &&
+      selection.candidate.version.endsWith('-SNAPSHOT')) {
+      selection.reject('Compatibility tests require the latest stable release')
+    }
+  }
+}
+
 // Java 11 (not Java 8 like the other Spring Boot 2.x smoke tests): the OpenFeature SDK
 // requires Java 11+. This matches what the outer (pre-nested) build was doing before
 // the application was extracted into this nested build.
@@ -24,15 +34,25 @@ java {
   targetCompatibility = 11
 }
 
+if (hasProperty('featureFlaggingApiJar') && hasProperty('ddOpenFeatureVersion')) {
+  throw new GradleException('Specify either featureFlaggingApiJar or ddOpenFeatureVersion, not both')
+}
+
 if (hasProperty('featureFlaggingApiJar')) {
   dependencies {
     implementation files(property('featureFlaggingApiJar'))
   }
+} else if (hasProperty('ddOpenFeatureVersion')) {
+  dependencies {
+    implementation "com.datadoghq:dd-openfeature:${property('ddOpenFeatureVersion')}"
+  }
+} else {
+  throw new GradleException('featureFlaggingApiJar or ddOpenFeatureVersion is required')
 }
 
 dependencies {
   // OpenFeature SDK is an API dependency of feature-flagging-api but is not
   // transitively resolved when the jar is passed as a files() dependency.
-  implementation 'dev.openfeature:sdk:1.20.1'
+  implementation "dev.openfeature:sdk:${findProperty('openFeatureSdkVersion') ?: '1.20.1'}"
   implementation 'org.springframework.boot:spring-boot-starter-web'
 }

dd-smoke-tests/openfeature/build.gradle

@@ -1,10 +1,30 @@
+import datadog.buildlogic.smoketest.NestedGradleBuild
+import org.gradle.process.CommandLineArgumentProvider
+
 plugins {
   id 'dd-trace-java.smoke-test-app'
   id 'dd-trace-java.module.smoke-test'
 }
 
 description = 'Open Feature provider Smoke Tests.'
 
+def minimumSupportedOpenFeatureSdkVersion = '1.20.1'
+def latestOpenFeatureSdkVersion = '+'
+def minimumSupportedAgentVersion = '1.65.0'
+
+configurations {
+  register('developmentProvider') {
+    canBeConsumed = false
+    canBeResolved = true
+    transitive = false
+  }
+  register('minimumSupportedAgent') {
+    canBeConsumed = false
+    canBeResolved = true
+    transitive = false
+  }
+}
+
 testJvmConstraints {
   minJavaVersion = JavaVersion.VERSION_11
 }
@@ -21,6 +41,118 @@ smokeTestApp {
 dependencies {
   testImplementation project(':dd-smoke-tests')
   testImplementation project(':products:feature-flagging:feature-flagging-lib')
+
+  developmentProvider project(':products:feature-flagging:feature-flagging-api')
+  minimumSupportedAgent "com.datadoghq:dd-java-agent:${minimumSupportedAgentVersion}"
+}
+
+def developmentProviderJar = configurations.developmentProvider.elements.map { files ->
+  objects.fileProperty().fileValue(files.iterator().next().asFile).get()
+}
+
+def registerCompatibilityApp = { String name, String providerVersion, String sdkVersion ->
+  def appBuildDir = layout.buildDirectory.dir("application-${name}")
+  def appTask = tasks.register("${name}App", NestedGradleBuild) {
+    applicationDir = layout.projectDirectory.dir('application')
+    applicationBuildDir = appBuildDir
+    tasksToRun = ['bootJar']
+    buildArguments.add("-PopenFeatureSdkVersion=${sdkVersion}")
+    if (providerVersion == null) {
+      projectJar('featureFlaggingApiJar', developmentProviderJar)
+    } else {
+      buildArguments.add("-PddOpenFeatureVersion=${providerVersion}")
+    }
+  }
+  return [
+    task: appTask,
+    jar: appBuildDir.map {
+      it.file('libs/openfeature-smoketest.jar')
+    }
+  ]
+}
+
+def registerCompatibilityTest = { String name, Map app, Configuration agent, String testClass ->
+  tasks.register(name, Test) {
+    group = LifecycleBasePlugin.VERIFICATION_GROUP
+    description = "Runs the OpenFeature smoke suite with ${name - 'Test'}."
+    useJUnitPlatform()
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+    include "**/${testClass}.*"
+    dependsOn app.task
+    inputs.file(app.jar)
+    if (agent != null) {
+      inputs.files(agent)
+    }
+    jvmArgumentProviders.add(new CommandLineArgumentProvider() {
+        @Override
+        Iterable<String> asArguments() {
+          final arguments = [
+            "-Ddatadog.smoketest.openfeature.application.path=${app.jar.get().asFile.absolutePath}"
+          ]
+          if (agent != null) {
+            arguments.add("-Ddatadog.smoketest.openfeature.agent.path=${agent.singleFile.absolutePath}")
+          }
+          return arguments
+        }
+      })
+  }
+}
+
+tasks.named('test', Test) {
+  exclude '**/OpenFeatureCompatibilitySmokeTest.*'
+}
+
+def minimumSdkApp = registerCompatibilityApp(
+  'minimumSdk',
+  null,
+  minimumSupportedOpenFeatureSdkVersion
+  )
+def latestSdkMinimumAgentApp = registerCompatibilityApp(
+  'latestSdkMinimumAgent',
+  // Keep the provider aligned with the minimum agent's internal API.
+  minimumSupportedAgentVersion,
+  latestOpenFeatureSdkVersion
+  )
+def latestDevelopmentVersionsApp = registerCompatibilityApp(
+  'latestDevelopmentVersions',
+  null,
+  latestOpenFeatureSdkVersion
+  )
+
+def minimumSdkLatestAgentTest = registerCompatibilityTest(
+  'minimumSdkLatestAgentTest',
+  minimumSdkApp,
+  null,
+  'OpenFeatureCompatibilitySmokeTest'
+  )
+def latestSdkMinimumAgentTest = registerCompatibilityTest(
+  'latestSdkMinimumAgentTest',
+  latestSdkMinimumAgentApp,
+  configurations.minimumSupportedAgent,
+  'OpenFeatureCompatibilitySmokeTest'
+  )
+def latestDevelopmentVersionsTest = registerCompatibilityTest(
+  'latestDevelopmentVersionsTest',
+  latestDevelopmentVersionsApp,
+  null,
+  'OpenFeatureProviderSmokeTest'
+  )
+
+minimumSdkLatestAgentTest.configure {
+  mustRunAfter tasks.named('test')
+}
+latestSdkMinimumAgentTest.configure {
+  mustRunAfter minimumSdkLatestAgentTest
+}
+latestDevelopmentVersionsTest.configure {
+  mustRunAfter latestSdkMinimumAgentTest
+}
+
+tasks.register('openFeatureCompatibilityTest') {
+  group = LifecycleBasePlugin.VERIFICATION_GROUP
+  description = 'Tests supported OpenFeature SDK versions with minimum and development agents.'
+  dependsOn minimumSdkLatestAgentTest, latestSdkMinimumAgentTest, latestDevelopmentVersionsTest
 }
 
 spotless {

dd-smoke-tests/openfeature/gradle.lockfile

@@ -10,6 +10,7 @@ com.blogspot.mydailyjava:weak-lock-free:0.17=testCompileClasspath,testRuntimeCla
 com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath
 com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath
 com.datadoghq:dd-instrument-java:0.0.4=testCompileClasspath,testRuntimeClasspath
+com.datadoghq:dd-java-agent:1.65.0=minimumSupportedAgent
 com.datadoghq:dd-javac-plugin-client:0.2.2=testCompileClasspath,testRuntimeClasspath
 com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath
 com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath

dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy

@@ -16,26 +16,29 @@ import spock.lang.Stepwise
 import spock.lang.Unroll
 import spock.util.concurrent.PollingConditions
 
-/** Due to the exposure cache it's important to run the tests in the specified order */
-@Stepwise
-class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest {
+abstract class AbstractOpenFeatureProviderSmokeTest extends AbstractServerSmokeTest {
 
   @Shared
-  private final rcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map<String, Object>
+  protected final rcConfig = new JsonSlurper().parse(fetchResource('ffe-system-test-data/ufc-config.json')) as Map<String, Object>
 
   @Shared
-  private final rcPayload = JsonOutput.toJson(rcConfig)
-
-  @Shared
-  private final loggedAllocations = buildLoggedAllocations(rcConfig)
+  protected final rcPayload = JsonOutput.toJson(rcConfig)
 
   @Override
   ProcessBuilder createProcessBuilder() {
-    setRemoteConfig("datadog/2/FFE_FLAGS/1/config", rcPayload)
+    setRemoteConfig('datadog/2/FFE_FLAGS/1/config', rcPayload)
 
-    final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path")
+    final springBootShadowJar = System.getProperty(
+      'datadog.smoketest.openfeature.application.path',
+      System.getProperty('datadog.smoketest.springboot.shadowJar.path')
+      )
+    final agentJar = System.getProperty('datadog.smoketest.openfeature.agent.path', shadowJarPath)
+    assert Files.isRegularFile(Paths.get(springBootShadowJar))
+    assert Files.isRegularFile(Paths.get(agentJar))
     final command = [javaPath()]
-    command.addAll(defaultJavaProperties)
+    command.addAll(defaultJavaProperties.collect {
+      it.startsWith('-javaagent:') ? "-javaagent:${agentJar}".toString() : it
+    })
     command.add('-Ddd.trace.debug=true')
     command.add('-Ddd.remote_config.enabled=true')
     command.add("-Ddd.remote_config.url=http://localhost:${server.address.port}/v0.7/config".toString())
@@ -62,6 +65,35 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest {
     }
   }
 
+  protected static URL fetchResource(final String name) {
+    return Thread.currentThread().getContextClassLoader().getResource(name)
+  }
+
+  protected static Set<Product> decodeProducts(final Map<String, Object> request) {
+    return request.client.products.collect { Product.valueOf(it) }
+  }
+
+  protected static long decodeCapabilities(final Map<String, Object> request) {
+    final clientCapabilities = request.client.capabilities as byte[]
+    long capabilities = 0l
+    for (int i = 0; i < clientCapabilities.length; i++) {
+      capabilities |= (clientCapabilities[i] & 0xFFL) << ((clientCapabilities.length - i - 1) * 8)
+    }
+    return capabilities
+  }
+
+  protected static boolean hasCapability(final long capabilities, final long test) {
+    return (capabilities & test) > 0
+  }
+}
+
+/** Due to the exposure cache it's important to run the tests in the specified order */
+@Stepwise
+class OpenFeatureProviderSmokeTest extends AbstractOpenFeatureProviderSmokeTest {
+
+  @Shared
+  private final loggedAllocations = buildLoggedAllocations(rcConfig)
+
   void 'test first remote config poll asks agent for feature flags'() {
     when:
     final firstRcRequest = waitForRcClientRequest { req ->
@@ -179,10 +211,6 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest {
     testCase << parseTestCases()
   }
 
-  private static URL fetchResource(final String name) {
-    return Thread.currentThread().getContextClassLoader().getResource(name)
-  }
-
   private static List<Map<String, Object>> parseTestCases() {
     final folder = fetchResource('ffe-system-test-data/evaluation-cases')
     final uri = folder.toURI()
@@ -250,21 +278,64 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest {
     }
     return logged
   }
+}
 
-  private static Set<Product> decodeProducts(final Map<String, Object> request) {
-    return request.client.products.collect { Product.valueOf(it) }
-  }
+/** Stable contract shared by all supported OpenFeature SDK and dd-java-agent versions. */
+@Stepwise
+class OpenFeatureCompatibilitySmokeTest extends AbstractOpenFeatureProviderSmokeTest {
 
-  private static long decodeCapabilities(final Map<String, Object> request) {
-    final clientCapabilities = request.client.capabilities as byte[]
-    long capabilities = 0l
-    for (int i = 0; i < clientCapabilities.length; i++) {
-      capabilities |= (clientCapabilities[i] & 0xFFL) << ((clientCapabilities.length - i - 1) * 8)
+  void 'test agent advertises feature flag remote configuration'() {
+    when:
+    final firstRcRequest = waitForRcClientRequest { req ->
+      return true
     }
-    return capabilities
+
+    then:
+    firstRcRequest == rcClientMessages.first()
+    decodeProducts(firstRcRequest).contains(Product.FFE_FLAGS)
+    hasCapability(
+    decodeCapabilities(firstRcRequest),
+    Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES
+    )
   }
 
-  private static boolean hasCapability(final long capabilities, final long test) {
-    return (capabilities & test) > 0
+  void 'test provider evaluates a flag and reports its exposure'() {
+    setup:
+    setRemoteConfig('datadog/2/FFE_FLAGS/1/config', rcPayload)
+    final request = new Request.Builder()
+    .url("http://localhost:${httpPort}/openfeature/evaluate")
+    .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson([
+      flag: 'boolean-false-assignment',
+      variationType: 'BOOLEAN',
+      defaultValue: true,
+      targetingKey: 'compatibility-test',
+      attributes: [should_disable_feature: true]
+    ])))
+    .build()
+
+    when:
+    final response = client.newCall(request).execute()
+    final responseBody = new JsonSlurper().parse(response.body().byteStream())
+
+    then:
+    response.code() == 200
+    responseBody.value == false
+    responseBody.reason == 'TARGETING_MATCH'
+    responseBody.variant == 'false-variation'
+    responseBody.flagMetadata.allocationKey == 'disable-feature'
+    new PollingConditions(timeout: 10).eventually {
+      final requests = evpProxyMessages*.getV2() as List<Map<String, Object>>
+      final events = requests*.exposures.flatten()
+      assert events.find {
+        event ->
+        event.flag.key == 'boolean-false-assignment' &&
+        event.allocation.key == 'disable-feature' &&
+        event.variant.key == 'false-variation' &&
+        event.subject.id == 'compatibility-test'
+      } != null
+    }
+
+    cleanup:
+    response.close()
   }
 }