Collect metrics from every inference cluster

apis/metricmappings/composition.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.crossplane.io/v1
+kind: Composition
+metadata:
+  name: metricmappings.modelplane.ai
+spec:
+  compositeTypeRef:
+    apiVersion: modelplane.ai/v1alpha1
+    kind: MetricMapping
+  mode: Pipeline
+  pipeline:
+  - functionRef:
+      name: modelplane-modelplanecompose-metric-mapping
+    step: compose-metric-mapping

apis/metricmappings/definition.yaml

@@ -0,0 +1,179 @@
+apiVersion: apiextensions.crossplane.io/v2
+kind: CompositeResourceDefinition
+metadata:
+  name: metricmappings.modelplane.ai
+spec:
+  group: modelplane.ai
+  names:
+    categories: [crossplane, modelplane, platform]
+    kind: MetricMapping
+    plural: metricmappings
+    shortNames: [mm]
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    served: true
+    referenceable: true
+    additionalPrinterColumns:
+    - name: CLUSTERS
+      type: integer
+      jsonPath: .status.clusters
+    - name: AGE
+      type: date
+      jsonPath: .metadata.creationTimestamp
+    schema:
+      openAPIV3Schema:
+        type: object
+        required: [spec]
+        properties:
+          spec:
+            type: object
+            required: [metrics]
+            description: >-
+              How one component's metrics become part of the modelplane_*
+              surface. Modelplane renders every MetricMapping into each
+              inference cluster's collector, so a mapping is written once on
+              the control plane and reaches the whole fleet.
+
+              A mapping naming a component Modelplane already provides
+              renames for is additive: its renames run after the built-in
+              ones, on whatever those left behind. A metric a built-in
+              already renamed no longer answers to the name it was emitted
+              under, so a second mapping selecting on that name matches
+              nothing and the built-in stands. Select on the `modelplane_`
+              name instead to rename one of Modelplane's own.
+            properties:
+              metrics:
+                type: array
+                minItems: 1
+                maxItems: 128
+                description: >-
+                  The metrics to rename. Give two components' metrics the same
+                  name only if they measure the same thing, and histograms only
+                  if their buckets match too. A quantile across mismatched
+                  buckets is wrong.
+                items:
+                  type: object
+                  required: [from, to]
+                  properties:
+                    from:
+                      type: string
+                      maxLength: 255
+                      pattern: '^[a-zA-Z_:][a-zA-Z0-9_:]*$'
+                      description: >-
+                        The metric's name as the component exposes it, matched
+                        exactly wherever it appears in the fleet.
+
+                        A histogram is named by its base name, without the
+                        _count, _sum or _bucket a Prometheus query would use:
+                        the collector holds it as one metric, and `part` is
+                        what reaches into it.
+                    to:
+                      type: string
+                      maxLength: 255
+                      pattern: '^modelplane_[a-z0-9_]*[a-z0-9]$'
+                      description: >-
+                        The name to export the metric under. Only modelplane_*
+                        metrics leave a cluster, so a metric no mapping renames
+                        never leaves its cluster.
+
+                        Name it in base units - seconds, bytes, joules, a ratio
+                        from nought to one - because that is what `fromUnit`
+                        converts to.
+                    part:
+                      type: string
+                      enum: [Count, Sum]
+                      description: >-
+                        Take a part of a histogram as a counter of its own,
+                        rather than the histogram itself. Count is how many
+                        observations it holds, which is a request count where
+                        the histogram measures request duration. Sum is their
+                        total.
+
+                        The extraction leaves the histogram alone, but the
+                        collector exports only what a mapping renames, so the
+                        histogram itself is dropped unless another mapping
+                        gives it a `modelplane_` name of its own. Write that
+                        second mapping to keep both.
+                    labels:
+                      type: array
+                      maxItems: 16
+                      description: >-
+                        Labels to set on this metric's series. To fold several
+                        metrics into one name, give each its own entry with
+                        the same `to` and a different fixed value, such as
+                        direction: input and direction: output on
+                        modelplane_tokens_total.
+                      items:
+                        type: object
+                        required: [name]
+                        x-kubernetes-validations:
+                        - rule: "has(self.value) != has(self.from)"
+                          message: set either value, for a fixed label, or from, to carry one the component already emits.
+                        - rule: "!has(self.values) || has(self.from)"
+                          message: values remaps what from carries, so it needs from. A fixed value has nothing to remap.
+                        properties:
+                          name:
+                            type: string
+                            maxLength: 63
+                            pattern: '^[a-zA-Z_][a-zA-Z0-9_]*$'
+                            description: The label to set.
+                          value:
+                            type: string
+                            maxLength: 253
+                            description: >-
+                              A fixed value, the same on every series this
+                              mapping produces. This is what tells two folded
+                              metrics apart.
+                          from:
+                            type: string
+                            maxLength: 63
+                            pattern: '^[a-zA-Z_][a-zA-Z0-9_]*$'
+                            description: >-
+                              A label the component already emits. Modelplane
+                              copies its value into this label and removes the
+                              original.
+
+                              Naming the label itself keeps it: that is how
+                              `values` rewrites what a component writes without
+                              renaming the label.
+                          values:
+                            type: object
+                            maxProperties: 32
+                            additionalProperties:
+                              type: string
+                              maxLength: 253
+                            description: >-
+                              What each of that label's values becomes, for
+                              putting an engine's own vocabulary into
+                              Modelplane's. A value with no entry here is left
+                              as the component wrote it.
+                    fromUnit:
+                      type: string
+                      enum: [Millijoules, Mebibytes, Milliseconds, Nanoseconds, Percent]
+                      description: >-
+                        What the component measures this in, when that isn't
+                        the unit the name claims. Modelplane converts to the
+                        base unit: millijoules and milliseconds are divided by
+                        a thousand, nanoseconds by a billion, percent by a
+                        hundred, and mebibytes multiplied out to bytes. A
+                        histogram is converted whole - its sum, its bounds and
+                        its bucket boundaries - so its quantiles come out in
+                        the target unit too.
+
+                        Percent is for a component that counts a saturation
+                        from nought to a hundred where the name says a ratio.
+                        Check rather than assume: vLLM publishes
+                        kv_cache_usage_perc and the value is a fraction, so a
+                        name is no guide.
+          status:
+            type: object
+            properties:
+              clusters:
+                type: integer
+                description: >-
+                  How many inference clusters apply this mapping.
+              conditions:
+                type: array
+                items:
+                  type: object

apis/telemetrydestinations/composition.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.crossplane.io/v1
+kind: Composition
+metadata:
+  name: telemetrydestinations.modelplane.ai
+spec:
+  compositeTypeRef:
+    apiVersion: modelplane.ai/v1alpha1
+    kind: TelemetryDestination
+  mode: Pipeline
+  pipeline:
+  - functionRef:
+      name: modelplane-modelplanecompose-telemetry-destination
+    step: compose-telemetry-destination

apis/telemetrydestinations/definition.yaml

@@ -0,0 +1,160 @@
+apiVersion: apiextensions.crossplane.io/v2
+kind: CompositeResourceDefinition
+metadata:
+  name: telemetrydestinations.modelplane.ai
+spec:
+  group: modelplane.ai
+  names:
+    categories: [crossplane, modelplane, platform]
+    kind: TelemetryDestination
+    plural: telemetrydestinations
+    shortNames: [td]
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    served: true
+    referenceable: true
+    additionalPrinterColumns:
+    - name: AGE
+      type: date
+      jsonPath: .metadata.creationTimestamp
+    schema:
+      openAPIV3Schema:
+        type: object
+        required: [spec]
+        properties:
+          spec:
+            type: object
+            required: [sinks]
+            description: >-
+              Where the fleet's metrics go. Modelplane runs no collectors
+              until a TelemetryDestination exists, and creating one turns on
+              collection on every inference cluster.
+
+              Several can exist. Their sinks are concatenated into one
+              collector configuration, so adding a backend is a new object
+              rather than an edit to one somebody else owns. A sink's name is
+              the collector's name for its exporter, so it has to be unique
+              across destinations.
+            properties:
+              sinks:
+                type: array
+                minItems: 1
+                maxItems: 16
+                description: >-
+                  Where to send it. Every sink gets the whole stream, so two
+                  sinks is two copies of the fleet's metrics, billed twice.
+                x-kubernetes-list-type: map
+                x-kubernetes-list-map-keys: [name]
+                items:
+                  type: object
+                  required: [name, type]
+                  x-kubernetes-validations:
+                  - rule: "!has(self.auth) || has(self.secretRef)"
+                    message: secretRef is required when auth is set, because the credential lives in it.
+                  properties:
+                    name:
+                      type: string
+                      maxLength: 63
+                      pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$'
+                      description: >-
+                        This sink's name, unique within the destination. It
+                        names the collector's exporter instance, the
+                        authenticator Modelplane composes for it, and the
+                        directory its credential mounts at, so renaming one
+                        restarts the collector.
+                    type:
+                      type: string
+                      maxLength: 63
+                      description: >-
+                        The collector exporter to send with, by the name
+                        OpenTelemetry gives it: otlphttp, otlp,
+                        prometheus_remote_write, kafka, and every other one the
+                        collector provides.
+
+                        Not an enum: the collector already refuses to start
+                        on a name it doesn't have, so repeating the list here
+                        would only add a second place for it to go stale.
+                    endpoint:
+                      type: string
+                      maxLength: 2048
+                      description: >-
+                        Where this sink writes. Leave it unset for an exporter
+                        that doesn't take an endpoint, such as kafka or debug,
+                        and configure it in `config` instead.
+
+                        Set here, it wins: Modelplane applies it over
+                        `config`, so a sink cannot be quietly redirected by
+                        the configuration passed through beside it.
+                    auth:
+                      type: object
+                      description: >-
+                        Authentication Modelplane sets up for this sink, using
+                        a credential from `secretRef`. For another scheme,
+                        define an authenticator under spec.extensions and
+                        reference it from `config`.
+
+                        Set here, it wins: Modelplane applies it over an
+                        `auth` block in `config`, so a sink's credential
+                        cannot be quietly unpicked.
+                      properties:
+                        bearerTokenKey:
+                          type: string
+                          maxLength: 253
+                          description: >-
+                            The key in this sink's Secret holding the bearer
+                            token. Modelplane mounts it as a file and points
+                            the authenticator at it, so a rotated token is
+                            picked up without restarting the collector.
+                    config:
+                      type: object
+                      x-kubernetes-preserve-unknown-fields: true
+                      description: >-
+                        The rest of the exporter's configuration, passed
+                        through as written: TLS, retries, queueing,
+                        compression, headers.
+
+                        Modelplane sets one default, for the exporters that
+                        flatten a series into labels: prometheus and
+                        prometheus_remote_write get
+                        resource_to_telemetry_conversion, or they would
+                        receive every series stripped of the cluster,
+                        deployment, engine and role it belongs to. Setting it
+                        here overrides that.
+                    secretRef:
+                      type: object
+                      required: [name]
+                      description: >-
+                        A Secret holding this sink's credentials. Modelplane
+                        mounts each key as a file under
+                        /etc/modelplane/telemetry/<sink name>/ and sets it as
+                        an environment variable for ${env:KEY} references in
+                        `config`. All sinks share one environment, so where
+                        two Secrets have the same key, refer to the file.
+                      properties:
+                        name:
+                          type: string
+                          maxLength: 253
+                          description: >-
+                            Name of the Secret, in modelplane-system on the
+                            control plane. Modelplane copies it to every
+                            cluster running a collector, so it doesn't have
+                            to exist on each of them already.
+              extensions:
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
+                description: >-
+                  Collector extensions, passed through as written. Use it to
+                  define an authenticator that a sink's `config` references.
+
+                  An exporter needs a client authenticator - basicauth,
+                  oauth2client, sigv4auth, headers_setter. The oidc extension
+                  authenticates callers of a receiver, so it is not one of
+                  these.
+          status:
+            type: object
+            properties:
+              conditions:
+                type: array
+                items:
+                  type: object

crossplane-project.yaml

@@ -75,6 +75,14 @@ spec:
     tarball:
       name: compose-model-service
       pathPrefix: _output/functions/compose-model-service
+  - source: Tarball
+    tarball:
+      name: compose-metric-mapping
+      pathPrefix: _output/functions/compose-metric-mapping
+  - source: Tarball
+    tarball:
+      name: compose-telemetry-destination
+      pathPrefix: _output/functions/compose-telemetry-destination
   - source: Tarball
     tarball:
       name: compose-usages