From b3f1c3716f97b45e6962a1c9b7a284100fb4b739 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Tue, 17 Nov 2026 13:13:34 -0500 Subject: [PATCH 1/4] build: compile for Kotlin API 2.3, the last two Kotlin versions Bootable supports the last two Kotlin versions, 2.3 and 2.4, so it compiles for API 2.3 rather than 2.1. 2.3 is also what Metro needs, so the Metro flavors and examples no longer raise it themselves, and where kotlin.time.Clock is stable, so boot no longer opts in to ExperimentalTime. --- CHANGELOG.adoc | 5 +++++ build.gradle.kts | 17 +++-------------- examples/di-bench/build.gradle.kts | 6 ------ examples/server-ktor-db/metro/build.gradle.kts | 7 ------- examples/wiring-metro/build.gradle.kts | 10 ---------- 5 files changed, 8 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 715b5b0d..90d02626 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,11 @@ == 3.1.0 (unreleased) +=== Kotlin 2.3 or later + +Bootable's modules are compiled for Kotlin API 2.3, rather than 2.1, so an application needs Kotlin 2.3 or later: Bootable supports the last two Kotlin versions. +The Metro flavors already required it. + === Service start order: phases and milestones replace priorities Services no longer order themselves with a global `priority()` number. diff --git a/build.gradle.kts b/build.gradle.kts index 8101c2fc..7b1b22a2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,7 +5,6 @@ import com.vanniktech.maven.publish.SourcesJar import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion -import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension import org.jetbrains.kotlin.powerassert.gradle.PowerAssertGradleExtension import dev.zacsweers.metro.gradle.MetroPluginExtension import org.gradle.api.artifacts.ExternalModuleDependency @@ -121,8 +120,9 @@ configure(subprojects.filter { it.path !in multiplatformProjects }) { // target 17 for max compatibility jvmTarget.set(JvmTarget.JVM_17) freeCompilerArgs.addAll(jvmTarget.map { listOf("-Xjdk-release=${it.target}") }) - apiVersion.set(KotlinVersion.KOTLIN_2_1) - languageVersion.set(KotlinVersion.KOTLIN_2_1) + // the last two Kotlin versions: 2.3, which Metro needs, and kotlin.time.Clock is stable from, and 2.4 + apiVersion.set(KotlinVersion.KOTLIN_2_3) + languageVersion.set(KotlinVersion.KOTLIN_2_3) } } @@ -218,11 +218,6 @@ project("boot") { runtimeClasspath += sourceSets["kodein"].output + sourceSets["metro"].output } - kotlin { - compilerOptions { - optIn.add("kotlin.time.ExperimentalTime") - } - } } project("boot-common") { @@ -738,12 +733,6 @@ fun Project.configureDiVariants() { // but it stays off the published/runtime classpath of the non-metro artifacts. dependencies.add("compileOnly", metroRuntime) dependencies.add("testCompileOnly", metroRuntime) - extensions.configure { - compilerOptions { - apiVersion.set(KotlinVersion.KOTLIN_2_3) - languageVersion.set(KotlinVersion.KOTLIN_2_3) - } - } } val main = sourceSets["main"] diff --git a/examples/di-bench/build.gradle.kts b/examples/di-bench/build.gradle.kts index f57443ee..016aec0b 100644 --- a/examples/di-bench/build.gradle.kts +++ b/examples/di-bench/build.gradle.kts @@ -1,5 +1,3 @@ -import org.jetbrains.kotlin.gradle.dsl.KotlinVersion - plugins { kotlin("jvm") id("dev.zacsweers.metro") @@ -8,10 +6,6 @@ plugins { kotlin { jvmToolchain(25) - compilerOptions { - apiVersion.set(KotlinVersion.KOTLIN_2_3) - languageVersion.set(KotlinVersion.KOTLIN_2_3) - } } dependencies { diff --git a/examples/server-ktor-db/metro/build.gradle.kts b/examples/server-ktor-db/metro/build.gradle.kts index 59436a84..28a543ad 100644 --- a/examples/server-ktor-db/metro/build.gradle.kts +++ b/examples/server-ktor-db/metro/build.gradle.kts @@ -1,5 +1,3 @@ -import org.jetbrains.kotlin.gradle.dsl.KotlinVersion - plugins { kotlin("jvm") id("dev.zacsweers.metro") @@ -8,11 +6,6 @@ plugins { kotlin { jvmToolchain(25) - compilerOptions { - // Metro requires Kotlin 2.3; the shared subprojects config targets 2.1 - apiVersion.set(KotlinVersion.KOTLIN_2_3) - languageVersion.set(KotlinVersion.KOTLIN_2_3) - } } dependencies { diff --git a/examples/wiring-metro/build.gradle.kts b/examples/wiring-metro/build.gradle.kts index 86ac8a91..d6080b8c 100644 --- a/examples/wiring-metro/build.gradle.kts +++ b/examples/wiring-metro/build.gradle.kts @@ -1,19 +1,9 @@ -import org.jetbrains.kotlin.gradle.dsl.KotlinVersion - plugins { kotlin("jvm") id("dev.zacsweers.metro") application } -kotlin { - compilerOptions { - // Metro requires Kotlin 2.3; the shared subprojects config targets 2.1 - apiVersion.set(KotlinVersion.KOTLIN_2_3) - languageVersion.set(KotlinVersion.KOTLIN_2_3) - } -} - dependencies { // consume each Bootable module's metro flavor. In-repo that's the module's `metroElements` configuration; // a real downstream consumer writes implementation("com.github.rocketraman.bootable:boot-metro:") etc. From 503f027252f234054389144baeb2e492c4cda032 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Tue, 17 Nov 2026 16:13:02 -0500 Subject: [PATCH 2/4] telemetry: time the SDK and the start's root span with a Clock ServiceStartTracing dated the start's root span with Instant.now(), while the SDK took the time of every other span from its own clock, so neither could be fixed in a test, nor be sure to agree. - OpenTelemetryService takes a kotlin.time.Clock, Clock.System by default, which becomes the SDK's clock for its tracer and meter providers. With Clock.System, the SDK keeps its own, which measures durations with System.nanoTime. - ServiceStartTracing takes the same Clock, and dates the root span at its time less the time since the startup mark. - Kodein and Metro pass a Clock binding if there is one, or else Clock.System, as Bootable does for StartOrder. --- CHANGELOG.adoc | 1 + README.adoc | 1 + .../telemetry/otel/OpenTelemetryModule.kt | 9 +- .../otel/ServiceStartTracingModule.kt | 7 +- .../telemetry/otel/OpenTelemetryService.kt | 24 +++- .../telemetry/otel/ServiceStartTracing.kt | 15 ++- .../telemetry/otel/OpenTelemetryBindings.kt | 11 +- .../otel/ServiceStartTracingBindings.kt | 6 +- .../telemetry/otel/ServiceStartTracingTest.kt | 109 ++++++++++++++++++ 9 files changed, 171 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 90d02626..eadf8e4d 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -62,6 +62,7 @@ Its exemplars carry only the trace and span ids, within OpenMetrics's 128-charac * `boot-db-pool-hikari-otel`: records HikariCP's pool metrics through the application's OpenTelemetry, as a `HikariConfigCustomizer` the Kodein and Metro flavors add to the pool's customizers. * `boot-telemetry-otel` adds `ServiceStartTracing`, a lifecycle controller that traces the application's start as one trace: a root span, from when `boot` was called until the application is running, with a span for each service's start, and what it does as it starts, such as a migration's queries, within that. Kodein's `serviceStartTracingModule` and Metro's `ServiceStartTracingBindings`, named in the graph, add it. +* `OpenTelemetryService` and `ServiceStartTracing` take a `kotlin.time.Clock`, `Clock.System` by default, which the SDK and the root span take their time from; with Kodein and Metro, a `Clock` binding, if there is one. * `boot-db-jdbc-otel`: traces each JDBC query as a client span, under the span current where it runs, such as a request's, as a `DataSourceWrapper` the Kodein and Metro flavors add to `boot-db-jdbc`'s wrappers. `jdbcTracing`'s `configure` lambda, Kodein's `jdbcTracingModule { ... }`, or a Metro `JdbcTelemetryConfigurer` binding changes the instrumentation's defaults. `JdbcDataSourceService` takes the `DataSourceWrapper`, as its third argument; Kodein's `jdbcDataSourceModule` and Metro's `JdbcDataSourceBindings`, now an interface, collect a set of them. diff --git a/README.adoc b/README.adoc index 9eb23cf6..de205dd4 100644 --- a/README.adoc +++ b/README.adoc @@ -176,6 +176,7 @@ The root span begins when `boot` was called, so it covers building the applicati A start that fails records the exception, and an error status, on the service's span and on the root span. Add it to the lifecycle controllers: `defaultLifecycleControllers() + ServiceStartTracing(openTelemetry)` with vanilla wiring, `serviceStartTracingModule` with Kodein, and `ServiceStartTracingBindings` in the graph's `bindingContainers` with Metro. The span is current on the thread that calls `start`: a `start` that runs coroutines carries it into them with `runBlocking(Context.current().asContextElement()) { … }`. +The SDK times its spans and measurements, and `ServiceStartTracing` its root span, with a `kotlin.time.Clock`: `Clock.System` unless one is given to `OpenTelemetryService` and `ServiceStartTracing`, or bound, with Kodein or Metro, such as a fixed clock in a test. For SDK settings the config doesn't cover, such as a span processor, a metric view, or another sampler, add an `OpenTelemetryCustomizer` to its set binding (Kodein `inBindSet`, Metro `@IntoSet`); this is also how the modules below plug in. `excludeUrlPathsFromTraces(paths)` is one, which keeps requests such as health checks out of the traces. diff --git a/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryModule.kt b/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryModule.kt index 92babc72..fb98b968 100644 --- a/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryModule.kt +++ b/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryModule.kt @@ -7,7 +7,9 @@ import org.kodein.di.bindSet import org.kodein.di.bindSingleton import org.kodein.di.inBindSet import org.kodein.di.instance +import org.kodein.di.instanceOrNull import org.kodein.di.singleton +import kotlin.time.Clock /** * A Kodein module that binds the application's [OpenTelemetry] and registers its [OpenTelemetryService] into the @@ -15,6 +17,9 @@ import org.kodein.di.singleton * [TelemetryConfig] is decoded by [telemetryConfig], from the `telemetry.sdk` config section. Requires a * `ConfigBinder` binding (e.g. from boot-config-hoplite's `hopliteConfigModule`). * + * A `kotlin.time.Clock` binding, if there is one, is the SDK's clock; otherwise it is `Clock.System`. Bind one, e.g. a + * fixed clock in a test, to set the time of the spans and measurements. + * * It opens two sets, so import it before the modules that add to them: * - `Set`: how other modules, such as `prometheusTelemetryModule` from * `boot-telemetry-otel-prometheus`, plug into the SDK. Add your own for what [TelemetryConfig] doesn't cover: @@ -27,7 +32,9 @@ val openTelemetryModule by DI.Module { bindSet() bindSet() bindSingleton { telemetryConfig(instance()) } - bindSingleton { OpenTelemetryService(instance(), instance>()) } + bindSingleton { + OpenTelemetryService(instance(), instance>(), instanceOrNull() ?: Clock.System) + } bindSingleton { instance().openTelemetry } inBindSet { add { singleton { instance() } } } } diff --git a/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingModule.kt b/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingModule.kt index 62b5b92e..a15e5166 100644 --- a/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingModule.kt +++ b/boot-telemetry-otel/src/kodein/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingModule.kt @@ -4,13 +4,16 @@ import com.github.rocketraman.bootable.boot.LifecycleController import org.kodein.di.DI import org.kodein.di.inBindSet import org.kodein.di.instance +import org.kodein.di.instanceOrNull import org.kodein.di.singleton +import kotlin.time.Clock /** * A Kodein module that traces the application's start as one trace, with a span for each service's start, by adding * [ServiceStartTracing] to the [LifecycleController] set of `bootableModule` (from boot-kodein). Import it alongside - * [openTelemetryModule], which provides the `OpenTelemetry`. + * [openTelemetryModule], which provides the `OpenTelemetry`. Like that module, it uses the `kotlin.time.Clock` binding, + * if there is one, or else `Clock.System`. */ val serviceStartTracingModule by DI.Module { - inBindSet { add { singleton { ServiceStartTracing(instance()) } } } + inBindSet { add { singleton { ServiceStartTracing(instance(), instanceOrNull() ?: Clock.System) } } } } diff --git a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt index 4014edf1..62f048e9 100644 --- a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt +++ b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt @@ -15,6 +15,7 @@ import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetry import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.common.Clock as OtelClock import io.opentelemetry.sdk.metrics.SdkMeterProvider import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader import io.opentelemetry.sdk.resources.Resource @@ -24,6 +25,7 @@ import io.opentelemetry.sdk.trace.samplers.Sampler import org.apache.logging.log4j.kotlin.logger import java.util.UUID import java.util.concurrent.TimeUnit +import kotlin.time.Clock import kotlin.time.toJavaDuration /** @@ -37,12 +39,18 @@ import kotlin.time.toJavaDuration * When the OpenTelemetry Java agent is attached, it has already built and registered the process's SDK, and owns its * lifecycle; [openTelemetry] is then the agent's, and [config] and the [customizers] are ignored. * + * The SDK takes the time of its spans and measurements from [clock], so a test can fix it, and [ServiceStartTracing], + * given the same clock, dates its spans consistently with them. With [Clock.System], the default, the SDK keeps its + * own clock, which measures durations with the JVM's monotonic `System.nanoTime`. Any other clock is the SDK's + * source of both the time and durations. + * * The service is in the [DefaultPhase.Telemetry] phase, so every other service, the database pool included, starts * after the SDK and shuts down before it, and its last spans and measurements are in the final export. */ class OpenTelemetryService( private val config: TelemetryConfig = TelemetryConfig(), customizers: Collection = emptyList(), + clock: Clock = Clock.System, ) : AppService { // the agent has already built and registered the process's SDK @@ -62,7 +70,7 @@ class OpenTelemetryService( private val sdk: OpenTelemetrySdk? = if (agentAttached) null - else buildSdk(config, customizers, resource).also { if (config.registerGlobal) registerGlobal(it) } + else buildSdk(config, customizers, resource, clock).also { if (config.registerGlobal) registerGlobal(it) } /** The application's OpenTelemetry: get tracers and meters from it, and hand it to instrumentation. */ val openTelemetry: OpenTelemetry = sdk ?: GlobalOpenTelemetry.get() @@ -128,7 +136,10 @@ private fun buildSdk( config: TelemetryConfig, customizers: Collection, resource: Resource, + clock: Clock, ): OpenTelemetrySdk { + // the SDK's own clock, unless another was given + val sdkClock = if (clock === Clock.System) null else SdkClock(clock) val endpoint = config.otlp.endpoint?.trimEnd('/') val headers = config.otlp.headers val builder = OpenTelemetrySdk.builder().setPropagators( @@ -142,6 +153,7 @@ private fun buildSdk( customizer.customizeSampler(sampler) } val tracerProvider = SdkTracerProvider.builder().setResource(resource).setSampler(sampler) + sdkClock?.let { tracerProvider.setClock(it) } if (endpoint != null && config.traces.otlp) { val url = "$endpoint/v1/traces" val exporter = OtlpHttpSpanExporter.builder().setEndpoint(url).apply { @@ -162,6 +174,7 @@ private fun buildSdk( if (config.metrics.enabled) { val meterProvider = SdkMeterProvider.builder().setResource(resource) + sdkClock?.let { meterProvider.setClock(it) } if (endpoint != null && config.metrics.otlp) { val url = "$endpoint/v1/metrics" val exporter = OtlpHttpMetricExporter.builder().setEndpoint(url).apply { @@ -191,3 +204,12 @@ private fun exportHeaders( if (suppliers.isEmpty()) return null return { suppliers.fold(emptyMap()) { headers, supplier -> headers + supplier() } } } + +/** [clock] as the SDK's clock, for both the time and durations, in nanoseconds. */ +private class SdkClock(private val clock: Clock) : OtelClock { + override fun now(): Long = clock.now().let { it.epochSeconds * NANOS_PER_SECOND + it.nanosecondsOfSecond } + + override fun nanoTime(): Long = now() +} + +private const val NANOS_PER_SECOND = 1_000_000_000L diff --git a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt index 1e743bf0..e050c195 100644 --- a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt +++ b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt @@ -8,9 +8,9 @@ import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.SpanBuilder import io.opentelemetry.api.trace.StatusCode import io.opentelemetry.api.trace.Tracer -import java.time.Instant +import kotlin.time.Clock import kotlin.time.TimeMark -import kotlin.time.toJavaDuration +import kotlin.time.toJavaInstant /** * A [LifecycleController] that traces the application's start as one trace: a root span, `start application`, with a @@ -24,6 +24,10 @@ import kotlin.time.toJavaDuration * services, as well as starting them, the telemetry phase's included. It ends once every service has started, when * the application is running. It doesn't cover the JVM's own startup, before `main`. * + * The root span's start time is [clock]'s time, less how long the application has been booting, as measured by the + * JVM's monotonic clock. Give it the clock [OpenTelemetryService] has, so its time is consistent with the SDK's time + * for the other spans; both default to [Clock.System]. + * * Add it to the application's lifecycle controllers: * * ``` @@ -44,11 +48,14 @@ import kotlin.time.toJavaDuration * A service started later, outside the application's start, such as by a controller that starts services on command, * gets a span of its own, a child of whatever span is current then. */ -class ServiceStartTracing(openTelemetry: OpenTelemetry) : LifecycleController { +class ServiceStartTracing( + openTelemetry: OpenTelemetry, + private val clock: Clock = Clock.System, +) : LifecycleController { private val tracer: Tracer = openTelemetry.getTracer(INSTRUMENTATION_SCOPE) override fun aroundAppStart(serviceContext: ServiceContext, startupMark: TimeMark, start: () -> Unit) { - val startedAt = Instant.now().minus(startupMark.elapsedNow().toJavaDuration()) + val startedAt = (clock.now() - startupMark.elapsedNow()).toJavaInstant() tracer.spanBuilder(APP_START_SPAN).setNoParent().setStartTimestamp(startedAt).runInSpan(start) } diff --git a/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryBindings.kt b/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryBindings.kt index 650a37af..80704de8 100644 --- a/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryBindings.kt +++ b/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryBindings.kt @@ -10,12 +10,16 @@ import dev.zacsweers.metro.Multibinds import dev.zacsweers.metro.Provides import dev.zacsweers.metro.SingleIn import io.opentelemetry.api.OpenTelemetry +import kotlin.time.Clock /** * Provides the application's [OpenTelemetry] and registers its [OpenTelemetryService] into the app-service set, so * the last telemetry is exported, and the SDK shut down, when the application stops. Its [TelemetryConfig] is decoded * by [telemetryConfig], from the `telemetry.sdk` config section. * + * A `kotlin.time.Clock` binding, if the graph has one, is the SDK's clock; otherwise it is `Clock.System`. Bind one, e.g. a + * fixed clock in a test, to set the time of the spans and measurements. + * * It declares two sets, each of which may be empty: * - `Set`: how other modules, such as `boot-telemetry-otel-prometheus-metro`, plug into the * SDK. Contribute your own for what [TelemetryConfig] doesn't cover: @@ -34,8 +38,11 @@ interface OpenTelemetryBindings { fun telemetryConfig(binder: ConfigBinder): TelemetryConfig = decodeTelemetryConfig(binder) @Provides @SingleIn(BootScope::class) - fun openTelemetryService(config: TelemetryConfig, customizers: Set): OpenTelemetryService = - OpenTelemetryService(config, customizers) + fun openTelemetryService( + config: TelemetryConfig, + customizers: Set, + clock: Clock = Clock.System, + ): OpenTelemetryService = OpenTelemetryService(config, customizers, clock) @Provides fun openTelemetry(service: OpenTelemetryService): OpenTelemetry = service.openTelemetry diff --git a/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingBindings.kt b/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingBindings.kt index 7eb5f697..ad31da8f 100644 --- a/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingBindings.kt +++ b/boot-telemetry-otel/src/metro/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingBindings.kt @@ -5,11 +5,12 @@ import dev.zacsweers.metro.BindingContainer import dev.zacsweers.metro.IntoSet import dev.zacsweers.metro.Provides import io.opentelemetry.api.OpenTelemetry +import kotlin.time.Clock /** * Traces the application's start as one trace, with a span for each service's start, by contributing * [ServiceStartTracing] to the [LifecycleController] set of `BootableGraph` (from boot-metro), over the `OpenTelemetry` - * of [OpenTelemetryBindings]. + * of [OpenTelemetryBindings], and, like those, the graph's `kotlin.time.Clock`, if it has one, or else `Clock.System`. * * Deliberately not contributed, so that depending on boot-telemetry-otel-metro doesn't change how the application * starts. Name it in your graph: @@ -21,5 +22,6 @@ import io.opentelemetry.api.OpenTelemetry @BindingContainer object ServiceStartTracingBindings { @Provides @IntoSet - fun serviceStartTracing(openTelemetry: OpenTelemetry): LifecycleController = ServiceStartTracing(openTelemetry) + fun serviceStartTracing(openTelemetry: OpenTelemetry, clock: Clock = Clock.System): LifecycleController = + ServiceStartTracing(openTelemetry, clock) } diff --git a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt index 511763ce..2bd6ab4e 100644 --- a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt +++ b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt @@ -10,16 +10,26 @@ import com.github.rocketraman.bootable.boot.ServiceContext import com.github.rocketraman.bootable.boot.bootableModule import com.github.rocketraman.bootable.config.hopliteConfigModule import de.infix.testBalloon.framework.core.testSuite +import dev.zacsweers.metro.BindingContainer import dev.zacsweers.metro.DependencyGraph +import dev.zacsweers.metro.IntoSet +import dev.zacsweers.metro.Provides +import dev.zacsweers.metro.SingleIn import dev.zacsweers.metro.createGraph import io.opentelemetry.api.OpenTelemetry import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.sdk.trace.samplers.Sampler import org.kodein.di.DI +import org.kodein.di.bindInstance +import org.kodein.di.inBindSet +import org.kodein.di.singleton import org.kodein.di.direct import org.kodein.di.instance import java.time.Duration import java.time.Instant import kotlin.time.Duration.Companion.seconds +import kotlin.time.Clock +import kotlin.time.Instant as KotlinInstant import kotlin.time.TimeSource @DependencyGraph( @@ -29,6 +39,30 @@ import kotlin.time.TimeSource ) interface StartTracingGraph : BootableGraph +sealed interface ClockedTestScope + +@DependencyGraph( + ClockedTestScope::class, + additionalScopes = [BootScope::class], + bindingContainers = [ClockedTestBindings::class, ServiceStartTracingBindings::class], +) +interface ClockedStartTracingGraph : BootableGraph { + val clock: Clock + val inMemoryTelemetry: InMemoryTelemetry + val openTelemetryService: OpenTelemetryService +} + +@BindingContainer +object ClockedTestBindings { + @Provides @SingleIn(ClockedTestScope::class) fun clock(): Clock = FixedClock() + + @Provides @SingleIn(ClockedTestScope::class) fun inMemoryTelemetry(): InMemoryTelemetry = InMemoryTelemetry() + + @Provides @IntoSet fun inMemory(telemetry: InMemoryTelemetry): OpenTelemetryCustomizer = telemetry + + @Provides @IntoSet fun sampleAll(): OpenTelemetryCustomizer = SampleAll +} + /** Does traced work as it starts, as a migration's queries would be, and fails to start if [fail]. */ private class WorkingService( private val name: String, @@ -45,7 +79,52 @@ private class WorkingService( } } +/** A clock stopped at [at], far from now, so a span timed by it can't be mistaken for one timed by the system clock. */ +private class FixedClock(private val at: KotlinInstant = KotlinInstant.parse("2030-01-01T00:00:00Z")) : Clock { + override fun now() = at +} + +/** Samples every trace, overriding the test config's `sampleRatio`, so a test's spans are always recorded. */ +private object SampleAll : OpenTelemetryCustomizer { + override fun customizeSampler(sampler: Sampler): Sampler = Sampler.alwaysOn() +} + +private val Long.startInstant: KotlinInstant get() = KotlinInstant.fromEpochSeconds(0, this) + +/** Runs a start with one service, traced by [tracing], and returns the root span's start and the service span's. */ +private fun traceStart(tracing: ServiceStartTracing, telemetry: InMemoryTelemetry): Pair { + tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), TimeSource.Monotonic.markNow() - 1.seconds) { + val pool = object : AppService { + override fun name() = "pool" + override fun start() = Unit + } + tracing.aroundServiceStart(pool, pool::start) + } + val spans = telemetry.spans.finishedSpanItems + return spans.single { it.name == ServiceStartTracing.APP_START_SPAN }.startEpochNanos.startInstant to + spans.single { it.name == "start pool" }.startEpochNanos.startInstant +} + +/** Checks the start was timed by [clock]: the root a second or so before its time, and the service's span at it. */ +private fun assertTimedBy(clock: FixedClock, times: Pair) { + val (root, service) = times + val now = clock.now() + assert(root <= now - 1.seconds && root > now - 10.seconds) + assert(service == now) +} + val ServiceStartTracingTest by testSuite { + test("the SDK and the root span take their time from the clock they are given") { + val clock = FixedClock() + val telemetry = InMemoryTelemetry() + val service = OpenTelemetryService(testConfig(), listOf(telemetry), clock) + try { + assertTimedBy(clock, traceStart(ServiceStartTracing(service.openTelemetry, clock), telemetry)) + } finally { + service.shutdown() + } + } + test("the start is one trace, from the startup mark, with a span for each service's start and the work within it") { val telemetry = InMemoryTelemetry() val service = OpenTelemetryService(testConfig(), listOf(telemetry)) @@ -128,6 +207,36 @@ val ServiceStartTracingTest by testSuite { assert(di.instance>().filterIsInstance().size == 1) } + test("kodein, with a clock") { + val clock = FixedClock() + val telemetry = InMemoryTelemetry() + val di = DI { + import(bootableModule) + import(hopliteConfigModule) + import(openTelemetryModule) + import(serviceStartTracingModule) + bindInstance { clock } + inBindSet { add { singleton { telemetry } } } + inBindSet { add { singleton { SampleAll } } } + }.direct + val tracing = di.instance>().filterIsInstance().single() + try { + assertTimedBy(clock, traceStart(tracing, telemetry)) + } finally { + di.instance().shutdown() + } + } + + test("metro, with a clock") { + val graph = createGraph() + val tracing = graph.lifecycleControllers.filterIsInstance().single() + try { + assertTimedBy(graph.clock as FixedClock, traceStart(tracing, graph.inMemoryTelemetry)) + } finally { + graph.openTelemetryService.shutdown() + } + } + test("metro, only when named in the graph") { assert(createGraph().lifecycleControllers.filterIsInstance().size == 1) assert(createGraph().lifecycleControllers.none { it is ServiceStartTracing }) From dbbf3b4cce17add4f5a2a230ed8d52b06d4a734f Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Tue, 17 Nov 2026 16:13:34 -0500 Subject: [PATCH 3/4] boot: trace the boot's stages before the services start aroundAppStart got only the startup mark, so a controller couldn't tell where the time before the first service went. It now gets a StartupTimeline: the mark, and the stages since, each ending where the next begins. boot adds "initialize JVM classes for logging and configuration" (setting up logging, then the bootable lambda), and Bootable adds "plan start" and "initialize". The first stage is named for what dominates it on a cold JVM: loading and initializing the classes of the logging, config, telemetry and database libraries, which a JVM AOT cache cuts sharply. Setting up logging isn't a stage of its own, since most of its time is class loading the application would do without logging too. ServiceStartTracing records each stage as a span under the start's root span, at the times the timeline marked, since the SDK doesn't exist yet when the early stages run. --- CHANGELOG.adoc | 4 +- README.adoc | 2 + .../telemetry/otel/ServiceStartTracing.kt | 26 +++++++++-- .../telemetry/otel/ServiceStartTracingTest.kt | 37 ++++++++++++++-- .../rocketraman/bootable/boot/Bootable.kt | 31 +++++++++---- .../bootable/boot/LifecycleController.kt | 7 ++- .../bootable/boot/StartupTimeline.kt | 44 +++++++++++++++++++ .../rocketraman/bootable/boot/BootableTest.kt | 33 +++++++++++++- 8 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 boot/src/main/kotlin/com/github/rocketraman/bootable/boot/StartupTimeline.kt diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index eadf8e4d..c8f5e733 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -44,7 +44,8 @@ With Kodein, `bootableModule` uses `StartOrder` and `StartOrder.Tiebreaker` bind `LifecycleController.priority()` is unchanged. -`LifecycleController.aroundAppStart(serviceContext, startupMark, start)` runs code around the application's start, which starts every service, and `aroundServiceStart(appService, start)` around each service's start, the highest-priority controller's outermost; the defaults run the start alone. +`LifecycleController.aroundAppStart(serviceContext, timeline, start)` runs code around the application's start, which starts every service, and `aroundServiceStart(appService, start)` around each service's start, the highest-priority controller's outermost; the defaults run the start alone. +The `StartupTimeline` marks when the boot began and its stages before the services start: with `boot`, initializing the JVM classes for logging and configuration (setting up logging and running the `boot` lambda), planning the start, and initializing. === Telemetry with OpenTelemetry @@ -62,6 +63,7 @@ Its exemplars carry only the trace and span ids, within OpenMetrics's 128-charac * `boot-db-pool-hikari-otel`: records HikariCP's pool metrics through the application's OpenTelemetry, as a `HikariConfigCustomizer` the Kodein and Metro flavors add to the pool's customizers. * `boot-telemetry-otel` adds `ServiceStartTracing`, a lifecycle controller that traces the application's start as one trace: a root span, from when `boot` was called until the application is running, with a span for each service's start, and what it does as it starts, such as a migration's queries, within that. Kodein's `serviceStartTracingModule` and Metro's `ServiceStartTracingBindings`, named in the graph, add it. +* `ServiceStartTracing` records each stage of the `StartupTimeline` as a span under the root. * `OpenTelemetryService` and `ServiceStartTracing` take a `kotlin.time.Clock`, `Clock.System` by default, which the SDK and the root span take their time from; with Kodein and Metro, a `Clock` binding, if there is one. * `boot-db-jdbc-otel`: traces each JDBC query as a client span, under the span current where it runs, such as a request's, as a `DataSourceWrapper` the Kodein and Metro flavors add to `boot-db-jdbc`'s wrappers. `jdbcTracing`'s `configure` lambda, Kodein's `jdbcTracingModule { ... }`, or a Metro `JdbcTelemetryConfigurer` binding changes the instrumentation's defaults. diff --git a/README.adoc b/README.adoc index de205dd4..4772628c 100644 --- a/README.adoc +++ b/README.adoc @@ -173,6 +173,8 @@ A span from `withSpan` is expected to have a parent: without one, it starts a tr To start a trace on purpose, such as for a scheduled job or a consumed message, use `Tracer.withRootSpan(name) { span -> … }`, which can also link the trace to the operation that kicked it off. `ServiceStartTracing(openTelemetry)` is a lifecycle controller that traces the application's start as one trace, so its total time, and each service's share of it, can be seen: a root span, `start application`, with a child span for each service's start, such as `start FlywayMigrationService`, within which is what the service does as it starts, such as its queries. The root span begins when `boot` was called, so it covers building the application, such as decoding its config, as well as starting its services, and ends when the application is running; it doesn't cover the JVM's own startup, before `main`. +The stages before the first service starts are spans under it too: `initialize JVM classes for logging and configuration`, `plan start`, and `initialize`. +The first covers setting up logging and the `boot` lambda, including the config and the DI graph: on a cold JVM, most of its time is loading and initializing the classes of the libraries these use, which a JVM AOT cache can cut sharply. A start that fails records the exception, and an error status, on the service's span and on the root span. Add it to the lifecycle controllers: `defaultLifecycleControllers() + ServiceStartTracing(openTelemetry)` with vanilla wiring, `serviceStartTracingModule` with Kodein, and `ServiceStartTracingBindings` in the graph's `bindingContainers` with Metro. The span is current on the thread that calls `start`: a `start` that runs coroutines carries it into them with `runBlocking(Context.current().asContextElement()) { … }`. diff --git a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt index e050c195..1a18c4a7 100644 --- a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt +++ b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt @@ -3,13 +3,16 @@ package com.github.rocketraman.bootable.telemetry.otel import com.github.rocketraman.bootable.boot.AppService import com.github.rocketraman.bootable.boot.LifecycleController import com.github.rocketraman.bootable.boot.ServiceContext +import com.github.rocketraman.bootable.boot.StartupTimeline import io.opentelemetry.api.OpenTelemetry import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.SpanBuilder import io.opentelemetry.api.trace.StatusCode import io.opentelemetry.api.trace.Tracer +import java.time.Instant import kotlin.time.Clock -import kotlin.time.TimeMark +import kotlin.time.ComparableTimeMark +import kotlin.time.TimeSource import kotlin.time.toJavaInstant /** @@ -19,6 +22,11 @@ import kotlin.time.toJavaInstant * `boot-db-jdbc-otel`. A start that throws has the exception recorded on its span, with an error status, and so does * the root span, when that stops the application's start. * + * The root span has a child span for each stage of the boot before the services start, from the [StartupTimeline]: + * `initialize JVM classes for logging and configuration`, `plan start`, and `initialize`, so the time before the first + * service starts is accounted for too. The SDK doesn't exist yet in those stages, so their spans are recorded + * afterwards, with the times the timeline marked. + * * The root span begins when the application began booting: with Bootable's `boot`, before logging is initialized and * the application built, so it covers building the application, such as decoding its config and creating its * services, as well as starting them, the telemetry phase's included. It ends once every service has started, when @@ -54,9 +62,19 @@ class ServiceStartTracing( ) : LifecycleController { private val tracer: Tracer = openTelemetry.getTracer(INSTRUMENTATION_SCOPE) - override fun aroundAppStart(serviceContext: ServiceContext, startupMark: TimeMark, start: () -> Unit) { - val startedAt = (clock.now() - startupMark.elapsedNow()).toJavaInstant() - tracer.spanBuilder(APP_START_SPAN).setNoParent().setStartTimestamp(startedAt).runInSpan(start) + override fun aroundAppStart(serviceContext: ServiceContext, timeline: StartupTimeline, start: () -> Unit) { + // the wall-clock time of each mark, relative to one reading of the clock, so the spans are consistent + val reference = TimeSource.Monotonic.markNow() + val now = clock.now() + fun at(mark: ComparableTimeMark): Instant = (now - (reference - mark)).toJavaInstant() + + tracer.spanBuilder(APP_START_SPAN).setNoParent().setStartTimestamp(at(timeline.start)) + .runInSpan { + timeline.stages.forEach { stage -> + tracer.spanBuilder(stage.name).setStartTimestamp(at(stage.start)).startSpan().end(at(stage.end)) + } + start() + } } override fun aroundServiceStart(appService: AppService, start: () -> Unit) { diff --git a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt index 2bd6ab4e..e3162d43 100644 --- a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt +++ b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt @@ -7,6 +7,8 @@ import com.github.rocketraman.bootable.boot.DefaultPhase import com.github.rocketraman.bootable.boot.LifecycleController import com.github.rocketraman.bootable.boot.Ordering import com.github.rocketraman.bootable.boot.ServiceContext +import com.github.rocketraman.bootable.boot.StartupStage +import com.github.rocketraman.bootable.boot.StartupTimeline import com.github.rocketraman.bootable.boot.bootableModule import com.github.rocketraman.bootable.config.hopliteConfigModule import de.infix.testBalloon.framework.core.testSuite @@ -89,11 +91,14 @@ private object SampleAll : OpenTelemetryCustomizer { override fun customizeSampler(sampler: Sampler): Sampler = Sampler.alwaysOn() } +/** A timeline that began a second ago, with no stages, as of a [Bootable] run without [boot] that took no time. */ +private fun aSecondAgo() = StartupTimeline(TimeSource.Monotonic.markNow() - 1.seconds) + private val Long.startInstant: KotlinInstant get() = KotlinInstant.fromEpochSeconds(0, this) /** Runs a start with one service, traced by [tracing], and returns the root span's start and the service span's. */ private fun traceStart(tracing: ServiceStartTracing, telemetry: InMemoryTelemetry): Pair { - tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), TimeSource.Monotonic.markNow() - 1.seconds) { + tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), aSecondAgo()) { val pool = object : AppService { override fun name() = "pool" override fun start() = Unit @@ -133,7 +138,7 @@ val ServiceStartTracingTest by testSuite { val context = ServiceContext(emptySet(), emptyList(), {}, {}) try { val beforeBoot = Instant.now().minusSeconds(2) - tracing.aroundAppStart(context, TimeSource.Monotonic.markNow() - 1.seconds) { + tracing.aroundAppStart(context, aSecondAgo()) { val pool = WorkingService("pool", DefaultPhase.Infrastructure, openTelemetry) tracing.aroundServiceStart(pool, pool::start) val app = WorkingService("app", DefaultPhase.Application, openTelemetry) @@ -159,6 +164,32 @@ val ServiceStartTracingTest by testSuite { } } + test("each stage of the boot before the services start is a span, at the times the timeline marked") { + val clock = FixedClock() + val telemetry = InMemoryTelemetry() + val service = OpenTelemetryService(testConfig(), listOf(telemetry), clock) + val tracing = ServiceStartTracing(service.openTelemetry, clock) + try { + val began = TimeSource.Monotonic.markNow() - 3.seconds + val stages = listOf( + StartupStage(StartupTimeline.INITIALIZE_JVM_CLASSES, began, began + 1.seconds), + StartupStage(StartupTimeline.PLAN_START, began + 1.seconds, began + 2.seconds), + ) + tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), StartupTimeline(began, stages)) {} + val spans = telemetry.spans.finishedSpanItems + val root = spans.single { it.name == ServiceStartTracing.APP_START_SPAN } + val classes = spans.single { it.name == StartupTimeline.INITIALIZE_JVM_CLASSES } + val plan = spans.single { it.name == StartupTimeline.PLAN_START } + assert(classes.parentSpanId == root.spanId && plan.parentSpanId == root.spanId) + // the first stage begins with the root, and each stage where the one before it ended + assert(classes.startEpochNanos == root.startEpochNanos) + assert(plan.startEpochNanos == classes.endEpochNanos) + assert(plan.endEpochNanos - plan.startEpochNanos == 1.seconds.inWholeNanoseconds) + } finally { + service.shutdown() + } + } + test("a service whose start fails has an error status, as does the application's start") { val telemetry = InMemoryTelemetry() val service = OpenTelemetryService(testConfig(), listOf(telemetry)) @@ -166,7 +197,7 @@ val ServiceStartTracingTest by testSuite { val tracing = ServiceStartTracing(openTelemetry) try { val failure = runCatching { - tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), TimeSource.Monotonic.markNow()) { + tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), StartupTimeline(TimeSource.Monotonic.markNow())) { val migrate = WorkingService("migrate", DefaultPhase.Initialization, openTelemetry, fail = true) tracing.aroundServiceStart(migrate, migrate::start) } diff --git a/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/Bootable.kt b/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/Bootable.kt index 156165da..30ce2adb 100644 --- a/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/Bootable.kt +++ b/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/Bootable.kt @@ -50,21 +50,33 @@ class Bootable( /** * This is internal as it is intended to be used only from the [com.github.rocketraman.bootable.boot] helper function. */ - internal fun boot(startupMark: TimeMark = TimeSource.Monotonic.markNow(), maxShutdownTime: Long = 30) { - exit(run(startupMark, maxShutdownTime)) + internal fun boot( + timeline: StartupTimeline = StartupTimeline(TimeSource.Monotonic.markNow()), + maxShutdownTime: Long = 30, + ) { + exit(run(timeline, maxShutdownTime)) } - /** Runs the application until it terminates, and returns the exit code. */ - internal fun run(startupMark: TimeMark = TimeSource.Monotonic.markNow(), maxShutdownTime: Long = 30): Int { + /** + * Runs the application until it terminates, and returns the exit code. [boot] is the timeline so far: with the + * [boot] function, its first stages; this adds the planning and initialization. + */ + internal fun run( + boot: StartupTimeline = StartupTimeline(TimeSource.Monotonic.markNow()), + maxShutdownTime: Long = 30, + ): Int { + val startupMark = boot.start appPlan() + val planned = boot.then(StartupTimeline.PLAN_START) log.info("=> Initializing (${startupMark.elapsedNow()})…") appInit() + val timeline = planned.then(StartupTimeline.INITIALIZE) log.info("=> Starting (${startupMark.elapsedNow()})…") @Suppress("TooGenericExceptionCaught") try { - appStart(startupMark) + appStart(timeline) } catch (e: Exception) { log.fatal(e) { "Failed to start application, stopping the services that started…" } running = false @@ -109,9 +121,9 @@ class Bootable( } } - private fun appStart(startupMark: TimeMark) { + private fun appStart(timeline: StartupTimeline) { val start = lifecycleControllers.sortedBy { it.priority() }.fold Unit>(::startControllers) { inner, controller -> - { controller.aroundAppStart(serviceContext, startupMark, inner) } + { controller.aroundAppStart(serviceContext, timeline, inner) } } start() } @@ -236,7 +248,7 @@ fun boot( loggingType: LoggingType? = null, bootable: () -> Bootable, ) { - val startupMark = TimeSource.Monotonic.markNow() + val booting = StartupTimeline(TimeSource.Monotonic.markNow()) loggingInit(loggingType = loggingType) val log = logger("Bootable") @@ -244,7 +256,8 @@ fun boot( @Suppress("TooGenericExceptionCaught") try { log.info("=> Creating application") - bootable().boot(startupMark, maxShutdownTime) + val application = bootable() + application.boot(booting.then(StartupTimeline.INITIALIZE_JVM_CLASSES), maxShutdownTime) } catch (e: Exception) { log.fatal(e) { "Failed to start application. Exiting..." } exitProcess(-1) diff --git a/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/LifecycleController.kt b/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/LifecycleController.kt index 4b405eb7..51f6107d 100644 --- a/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/LifecycleController.kt +++ b/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/LifecycleController.kt @@ -1,7 +1,6 @@ package com.github.rocketraman.bootable.boot import com.github.rocketraman.bootable.common.Ignore -import kotlin.time.TimeMark /** * The application's services, for a [LifecycleController] to start and stop. @@ -53,13 +52,13 @@ interface LifecycleController { * is running, or throws if a service failed to start. An implementation must call [start] once, on the calling * thread, and let what it throws propagate. * - * [startupMark] marks when the application began booting: with [boot], before logging was initialized and the - * [Bootable] built, so the time since it includes building the application, such as decoding its config. + * [timeline] is the application's boot so far: when it began, with [boot] before logging was initialized and the + * [Bootable] built, and the stages since, such as building the application, which includes decoding its config. * * Each controller wraps the start of the controllers of lower [priority], so the highest priority's code is * outermost. The default runs [start] alone. */ - fun aroundAppStart(serviceContext: ServiceContext, startupMark: TimeMark, start: () -> Unit) = start() + fun aroundAppStart(serviceContext: ServiceContext, timeline: StartupTimeline, start: () -> Unit) = start() /** * When a service is started, lifecycle controllers may vote on whether it is allowed to start or not. This may diff --git a/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/StartupTimeline.kt b/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/StartupTimeline.kt new file mode 100644 index 00000000..15c63b6d --- /dev/null +++ b/boot/src/main/kotlin/com/github/rocketraman/bootable/boot/StartupTimeline.kt @@ -0,0 +1,44 @@ +package com.github.rocketraman.bootable.boot + +import kotlin.time.ComparableTimeMark +import kotlin.time.TimeSource + +/** + * The application's boot before its services start: when it began, [start], and the [stages] it went through since, + * in order, each ending where the next begins. A [LifecycleController] gets it in + * [LifecycleController.aroundAppStart], for example to record the stages as spans of the start's trace. + * + * With [boot], [start] is when `boot` was called, and the stages are: + * - [INITIALIZE_JVM_CLASSES]: initializing logging, then the `bootable` lambda, which builds the application, such as + * decoding its config, building its DI graph, and constructing its services. On a cold JVM, most of this stage is + * loading and initializing the classes of the logging, config, telemetry and database libraries, which a JVM AOT + * cache can cut sharply; + * - [PLAN_START]: computing the start plan; + * - [INITIALIZE]: every controller's [LifecycleController.appInit]. + * + * A [Bootable] run without [boot] has only the last two, from when it began running. + * + * The marks are of the JVM's monotonic clock, so the stages' durations aren't affected by changes to the wall clock. + */ +class StartupTimeline( + val start: ComparableTimeMark, + val stages: List = emptyList(), +) { + /** Where the next stage begins: the end of the last, or else [start]. */ + private val last: ComparableTimeMark get() = stages.lastOrNull()?.end ?: start + + /** This timeline, with a stage named [name] from the end of the last until now. */ + internal fun then(name: String, end: ComparableTimeMark = TimeSource.Monotonic.markNow()) = + StartupTimeline(start, stages + StartupStage(name, last, end)) + + companion object { + const val INITIALIZE_JVM_CLASSES = "initialize JVM classes for logging and configuration" + const val PLAN_START = "plan start" + const val INITIALIZE = "initialize" + } +} + +/** A stage of the application's boot, named [name], from [start] to [end]. */ +class StartupStage(val name: String, val start: ComparableTimeMark, val end: ComparableTimeMark) { + override fun toString() = "$name (${end - start})" +} diff --git a/boot/src/test/kotlin/com/github/rocketraman/bootable/boot/BootableTest.kt b/boot/src/test/kotlin/com/github/rocketraman/bootable/boot/BootableTest.kt index e2ecc7be..4931a8ba 100644 --- a/boot/src/test/kotlin/com/github/rocketraman/bootable/boot/BootableTest.kt +++ b/boot/src/test/kotlin/com/github/rocketraman/bootable/boot/BootableTest.kt @@ -5,7 +5,8 @@ import com.github.rocketraman.bootable.boot.DefaultPhase.Infrastructure import com.github.rocketraman.bootable.boot.DefaultPhase.Initialization import com.github.rocketraman.bootable.boot.DefaultPhase.Serving import de.infix.testBalloon.framework.core.testSuite -import kotlin.time.TimeMark +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeSource private class RecordingService( private val name: String, @@ -43,7 +44,7 @@ private class Around( ) : LifecycleController { override fun priority() = priority - override fun aroundAppStart(serviceContext: ServiceContext, startupMark: TimeMark, start: () -> Unit) { + override fun aroundAppStart(serviceContext: ServiceContext, timeline: StartupTimeline, start: () -> Unit) { events += "$name{" try { start() @@ -103,4 +104,32 @@ val BootableTest by testSuite { ) ) } + + test("the controllers get the timeline of the boot, with the planning and initialization added to its stages") { + var captured: StartupTimeline? = null + val capture = object : LifecycleController { + override fun aroundAppStart(serviceContext: ServiceContext, timeline: StartupTimeline, start: () -> Unit) { + captured = timeline + start() + } + } + val started = TimeSource.Monotonic.markNow() - 1.seconds + val booted = StartupTimeline(started).then(StartupTimeline.INITIALIZE_JVM_CLASSES, started + 0.5.seconds) + val bootable = Bootable( + setOf(RecordingService("pool", Infrastructure, mutableListOf(), failStart = true)), + setOf(AppStartStopLifecycleController(), capture), + ) + + assert(bootable.run(booted, maxShutdownTime = 1_000_000) == -1) + val timeline = captured!! + val stages = timeline.stages + assert(timeline.start == started) + assert( + stages.map { it.name } == + listOf(StartupTimeline.INITIALIZE_JVM_CLASSES, StartupTimeline.PLAN_START, StartupTimeline.INITIALIZE), + ) + // each stage begins where the one before it ended + assert(stages.zipWithNext().all { (before, after) -> before.end == after.start }) + assert(stages.first().start == started) + } } From 8cbbc1cf4830be38bdc88888206da216267fa744 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Tue, 17 Nov 2026 16:13:46 -0500 Subject: [PATCH 4/4] telemetry: always sample the start's trace, and withRootSpan's on ask With a sampleRatio below 1, the start's trace was dropped as often as any other, though it happens once per process and is the trace of a slow or failed start. - OpenTelemetryService's sampler always samples a root span marked with ALWAYS_SAMPLE, bootable.sample.always, and leaves every other span to the sampler it wraps. - ServiceStartTracing marks the start's root span with it. - withRootSpan(name, sample = Sample.Always) marks its root span with it too: for a trace too rare or valuable to lose, such as a nightly job's. --- CHANGELOG.adoc | 3 +- README.adoc | 2 ++ .../telemetry/otel/OpenTelemetryService.kt | 35 +++++++++++++++++-- .../telemetry/otel/ServiceStartTracing.kt | 6 +++- .../bootable/telemetry/otel/Tracing.kt | 26 ++++++++++++++ .../telemetry/otel/ServiceStartTracingTest.kt | 27 +++++++++----- .../bootable/telemetry/otel/TracingTest.kt | 12 +++++++ 7 files changed, 97 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index c8f5e733..c587838b 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -63,7 +63,8 @@ Its exemplars carry only the trace and span ids, within OpenMetrics's 128-charac * `boot-db-pool-hikari-otel`: records HikariCP's pool metrics through the application's OpenTelemetry, as a `HikariConfigCustomizer` the Kodein and Metro flavors add to the pool's customizers. * `boot-telemetry-otel` adds `ServiceStartTracing`, a lifecycle controller that traces the application's start as one trace: a root span, from when `boot` was called until the application is running, with a span for each service's start, and what it does as it starts, such as a migration's queries, within that. Kodein's `serviceStartTracingModule` and Metro's `ServiceStartTracingBindings`, named in the graph, add it. -* `ServiceStartTracing` records each stage of the `StartupTimeline` as a span under the root. +* `ServiceStartTracing` records each stage of the `StartupTimeline` as a span under the root, and `OpenTelemetryService`'s sampler always samples the start's trace, whatever `sampleRatio` is. +* `withRootSpan(name, sample = Sample.Always)` always samples the trace it starts too: a root span with the `ALWAYS_SAMPLE` attribute, `bootable.sample.always`, is always sampled. * `OpenTelemetryService` and `ServiceStartTracing` take a `kotlin.time.Clock`, `Clock.System` by default, which the SDK and the root span take their time from; with Kodein and Metro, a `Clock` binding, if there is one. * `boot-db-jdbc-otel`: traces each JDBC query as a client span, under the span current where it runs, such as a request's, as a `DataSourceWrapper` the Kodein and Metro flavors add to `boot-db-jdbc`'s wrappers. `jdbcTracing`'s `configure` lambda, Kodein's `jdbcTracingModule { ... }`, or a Metro `JdbcTelemetryConfigurer` binding changes the instrumentation's defaults. diff --git a/README.adoc b/README.adoc index 4772628c..057a2c22 100644 --- a/README.adoc +++ b/README.adoc @@ -171,10 +171,12 @@ Ship them with the platform's log collection, such as Cloud Logging's, or an Ope `Tracer.withSpan(name) { span -> … }` runs a suspending block in a child span of the current one, which stays current across dispatchers, and records an exception as the span's error. A span from `withSpan` is expected to have a parent: without one, it starts a trace of its own and logs a warning, once for each span name, since its caller's trace context usually didn't reach it, such as across a new `CoroutineScope` or a Java callback. To start a trace on purpose, such as for a scheduled job or a consumed message, use `Tracer.withRootSpan(name) { span -> … }`, which can also link the trace to the operation that kicked it off. +`withRootSpan(name, sample = Sample.Always)` samples that trace whatever `sampleRatio` is, for a trace too rare or valuable to lose, such as a nightly job's; the application's start is always sampled the same way. `ServiceStartTracing(openTelemetry)` is a lifecycle controller that traces the application's start as one trace, so its total time, and each service's share of it, can be seen: a root span, `start application`, with a child span for each service's start, such as `start FlywayMigrationService`, within which is what the service does as it starts, such as its queries. The root span begins when `boot` was called, so it covers building the application, such as decoding its config, as well as starting its services, and ends when the application is running; it doesn't cover the JVM's own startup, before `main`. The stages before the first service starts are spans under it too: `initialize JVM classes for logging and configuration`, `plan start`, and `initialize`. The first covers setting up logging and the `boot` lambda, including the config and the DI graph: on a cold JVM, most of its time is loading and initializing the classes of the libraries these use, which a JVM AOT cache can cut sharply. +The start's trace is always sampled, whatever `sampleRatio` is, so a slow or failed start is never missing from the traces. A start that fails records the exception, and an error status, on the service's span and on the root span. Add it to the lifecycle controllers: `defaultLifecycleControllers() + ServiceStartTracing(openTelemetry)` with vanilla wiring, `serviceStartTracingModule` with Kodein, and `ServiceStartTracingBindings` in the graph's `bindingContainers` with Metro. The span is current on the thread that calls `start`: a `start` that runs coroutines carries it into them with `runBlocking(Context.current().asContextElement()) { … }`. diff --git a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt index 62f048e9..fbe47375 100644 --- a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt +++ b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/OpenTelemetryService.kt @@ -8,7 +8,9 @@ import io.opentelemetry.api.OpenTelemetry import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.common.Attributes +import io.opentelemetry.api.trace.SpanKind import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator +import io.opentelemetry.context.Context import io.opentelemetry.context.propagation.ContextPropagators import io.opentelemetry.context.propagation.TextMapPropagator import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter @@ -21,7 +23,9 @@ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader import io.opentelemetry.sdk.resources.Resource import io.opentelemetry.sdk.trace.SdkTracerProvider import io.opentelemetry.sdk.trace.export.BatchSpanProcessor +import io.opentelemetry.sdk.trace.data.LinkData import io.opentelemetry.sdk.trace.samplers.Sampler +import io.opentelemetry.sdk.trace.samplers.SamplingResult import org.apache.logging.log4j.kotlin.logger import java.util.UUID import java.util.concurrent.TimeUnit @@ -39,6 +43,10 @@ import kotlin.time.toJavaDuration * When the OpenTelemetry Java agent is attached, it has already built and registered the process's SDK, and owns its * lifecycle; [openTelemetry] is then the agent's, and [config] and the [customizers] are ignored. * + * Traces are sampled at [TracesConfig.sampleRatio], or as the customizers' sampler decides, except a trace whose root + * span carries [ALWAYS_SAMPLE], which is always sampled: the application's start, as [ServiceStartTracing] traces it, + * which happens once per process, and a [withRootSpan] with [Sample.Always]. + * * The SDK takes the time of its spans and measurements from [clock], so a test can fix it, and [ServiceStartTracing], * given the same clock, dates its spans consistently with them. With [Clock.System], the default, the SDK keeps its * own clock, which measures durations with the JVM's monotonic `System.nanoTime`. Any other clock is the SDK's @@ -149,9 +157,11 @@ private fun buildSdk( ) if (config.traces.enabled) { - val sampler = customizers.fold(Sampler.parentBased(Sampler.traceIdRatioBased(config.traces.sampleRatio))) { sampler, customizer -> - customizer.customizeSampler(sampler) - } + val sampler = SampleMarked( + customizers.fold(Sampler.parentBased(Sampler.traceIdRatioBased(config.traces.sampleRatio))) { sampler, customizer -> + customizer.customizeSampler(sampler) + } + ) val tracerProvider = SdkTracerProvider.builder().setResource(resource).setSampler(sampler) sdkClock?.let { tracerProvider.setClock(it) } if (endpoint != null && config.traces.otlp) { @@ -213,3 +223,22 @@ private class SdkClock(private val clock: Clock) : OtelClock { } private const val NANOS_PER_SECOND = 1_000_000_000L + +/** + * Samples a span marked [ALWAYS_SAMPLE], such as the application's start, and leaves every other span to [delegate]. + * The spans within its trace follow it if [delegate] samples by the parent, as the default sampler does. + */ +private class SampleMarked(private val delegate: Sampler) : Sampler { + override fun shouldSample( + parentContext: Context, + traceId: String, + name: String, + spanKind: SpanKind, + attributes: Attributes, + parentLinks: List, + ): SamplingResult = + if (attributes.get(ALWAYS_SAMPLE) == true) SamplingResult.recordAndSample() + else delegate.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks) + + override fun getDescription(): String = "SampleMarked{${delegate.description}}" +} diff --git a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt index 1a18c4a7..b6056d2d 100644 --- a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt +++ b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracing.kt @@ -27,6 +27,9 @@ import kotlin.time.toJavaInstant * service starts is accounted for too. The SDK doesn't exist yet in those stages, so their spans are recorded * afterwards, with the times the timeline marked. * + * [OpenTelemetryService]'s sampler always samples the start's trace, whatever the configured ratio, so a start is + * never missing from the traces: the root span carries [ALWAYS_SAMPLE], which the sampler looks for. + * * The root span begins when the application began booting: with Bootable's `boot`, before logging is initialized and * the application built, so it covers building the application, such as decoding its config and creating its * services, as well as starting them, the telemetry phase's included. It ends once every service has started, when @@ -68,7 +71,8 @@ class ServiceStartTracing( val now = clock.now() fun at(mark: ComparableTimeMark): Instant = (now - (reference - mark)).toJavaInstant() - tracer.spanBuilder(APP_START_SPAN).setNoParent().setStartTimestamp(at(timeline.start)) + tracer.spanBuilder(APP_START_SPAN).setNoParent().setAttribute(ALWAYS_SAMPLE, true) + .setStartTimestamp(at(timeline.start)) .runInSpan { timeline.stages.forEach { stage -> tracer.spanBuilder(stage.name).setStartTimestamp(at(stage.start)).startSpan().end(at(stage.end)) diff --git a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/Tracing.kt b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/Tracing.kt index 5ca61611..4fb52c4c 100644 --- a/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/Tracing.kt +++ b/boot-telemetry-otel/src/main/kotlin/com/github/rocketraman/bootable/telemetry/otel/Tracing.kt @@ -1,5 +1,6 @@ package com.github.rocketraman.bootable.telemetry.otel +import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.SpanContext import io.opentelemetry.api.trace.SpanKind @@ -64,6 +65,9 @@ suspend fun Tracer.withSpan( * A trace started for work that another operation kicked off but doesn't wait for, such as a background task started * by a request, can say so with [links] to that operation's span, rather than being its child. * + * [sample] is whether the trace is sampled as the SDK's sampler decides, such as at the configured `sampleRatio`, or + * always, with [Sample.Always], for a trace too rare or too valuable to lose, such as a nightly job's. + * * ``` * suspend fun purge() = tracer.withRootSpan("orders.purge") { * repository.purgeExpired() @@ -74,14 +78,36 @@ suspend fun Tracer.withRootSpan( name: String, kind: SpanKind = SpanKind.INTERNAL, links: Collection = emptyList(), + sample: Sample = Sample.BySampler, block: suspend (Span) -> T, ): T { val span = spanBuilder(name).setNoParent().setSpanKind(kind) .apply { links.forEach { addLink(it) } } + .apply { if (sample == Sample.Always) setAttribute(ALWAYS_SAMPLE, true) } .startSpan() return runInSpan(currentContext(), span, block) } +/** How a trace that [withRootSpan] starts is sampled. */ +enum class Sample { + /** As the SDK's sampler decides, such as at the configured `sampleRatio`. */ + BySampler, + + /** + * Always, whatever the SDK's sampler would decide. The root span carries [ALWAYS_SAMPLE], which the sampler of + * [OpenTelemetryService]'s SDK honors; the spans within the trace follow the root, as long as the sampler samples by + * the parent, as the default does. Under the OpenTelemetry Java agent, whose SDK is its own, it has no effect. + */ + Always, +} + +/** + * Marks a root span whose trace [OpenTelemetryService]'s sampler always samples, as [withRootSpan] does with + * [Sample.Always], and [ServiceStartTracing] does for the application's start. Set it on a span builder, before the + * span starts: the sampler sees only the attributes the span starts with. + */ +val ALWAYS_SAMPLE: AttributeKey = AttributeKey.booleanKey("bootable.sample.always") + /** The coroutine's context, which a traced server's request span is in, falling back to the thread's. */ private suspend fun currentContext(): Context = currentCoroutineContext().getOpenTelemetryContext().takeIf { it != Context.root() } ?: Context.current() diff --git a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt index e3162d43..c4e8bb6b 100644 --- a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt +++ b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/ServiceStartTracingTest.kt @@ -20,7 +20,6 @@ import dev.zacsweers.metro.SingleIn import dev.zacsweers.metro.createGraph import io.opentelemetry.api.OpenTelemetry import io.opentelemetry.api.trace.StatusCode -import io.opentelemetry.sdk.trace.samplers.Sampler import org.kodein.di.DI import org.kodein.di.bindInstance import org.kodein.di.inBindSet @@ -61,8 +60,6 @@ object ClockedTestBindings { @Provides @SingleIn(ClockedTestScope::class) fun inMemoryTelemetry(): InMemoryTelemetry = InMemoryTelemetry() @Provides @IntoSet fun inMemory(telemetry: InMemoryTelemetry): OpenTelemetryCustomizer = telemetry - - @Provides @IntoSet fun sampleAll(): OpenTelemetryCustomizer = SampleAll } /** Does traced work as it starts, as a migration's queries would be, and fails to start if [fail]. */ @@ -86,11 +83,6 @@ private class FixedClock(private val at: KotlinInstant = KotlinInstant.parse("20 override fun now() = at } -/** Samples every trace, overriding the test config's `sampleRatio`, so a test's spans are always recorded. */ -private object SampleAll : OpenTelemetryCustomizer { - override fun customizeSampler(sampler: Sampler): Sampler = Sampler.alwaysOn() -} - /** A timeline that began a second ago, with no stages, as of a [Bootable] run without [boot] that took no time. */ private fun aSecondAgo() = StartupTimeline(TimeSource.Monotonic.markNow() - 1.seconds) @@ -190,6 +182,24 @@ val ServiceStartTracingTest by testSuite { } } + test("the start's trace is always sampled, whatever the configured ratio") { + val telemetry = InMemoryTelemetry() + val service = OpenTelemetryService(testConfig(traces = TracesConfig(sampleRatio = 0.0)), listOf(telemetry)) + val openTelemetry = service.openTelemetry + val tracing = ServiceStartTracing(openTelemetry) + try { + tracing.aroundAppStart(ServiceContext(emptySet(), emptyList(), {}, {}), aSecondAgo()) { + val pool = WorkingService("pool", DefaultPhase.Infrastructure, openTelemetry) + tracing.aroundServiceStart(pool, pool::start) + } + openTelemetry.getTracer("test").spanBuilder("request").startSpan().end() + val names = telemetry.spans.finishedSpanItems.map { it.name }.toSet() + assert(names == setOf(ServiceStartTracing.APP_START_SPAN, "start pool", "pool.work")) + } finally { + service.shutdown() + } + } + test("a service whose start fails has an error status, as does the application's start") { val telemetry = InMemoryTelemetry() val service = OpenTelemetryService(testConfig(), listOf(telemetry)) @@ -248,7 +258,6 @@ val ServiceStartTracingTest by testSuite { import(serviceStartTracingModule) bindInstance { clock } inBindSet { add { singleton { telemetry } } } - inBindSet { add { singleton { SampleAll } } } }.direct val tracing = di.instance>().filterIsInstance().single() try { diff --git a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/TracingTest.kt b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/TracingTest.kt index 1a946466..e5be2005 100644 --- a/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/TracingTest.kt +++ b/boot-telemetry-otel/src/test/kotlin/com/github/rocketraman/bootable/telemetry/otel/TracingTest.kt @@ -138,6 +138,18 @@ val TracingTest by testSuite { assert(spans.getValue("job.step").parentSpanId == spans.getValue("job").spanId) } + test("with Sample.Always, samples the trace whatever the ratio, and only then") { + val (service, telemetry) = tracing(sampleRatio = 0.0) + val tracer = service.openTelemetry.getTracer("test") + tracer.withRootSpan("nightly", sample = Sample.Always) { + tracer.withSpan("nightly.step") { } + } + tracer.withRootSpan("routine") { + tracer.withSpan("routine.step") { } + } + assert(telemetry.spans.finishedSpanItems.map { it.name }.toSet() == setOf("nightly", "nightly.step")) + } + test("links the trace to the spans it is given") { val (service, telemetry) = tracing() val tracer = service.openTelemetry.getTracer("test")