Add secure opt-in fleet metrics and Git synchronization reporting

.github/workflows/docker-image.yml

@@ -1489,6 +1489,9 @@ jobs:
       - name: Reconcile a commit from an isolated Git repository
         run: sudo bash tests/integration/live-update/test-live-update.sh
 
+      - name: Validate metrics through Git failure and recovery
+        run: sudo bash tests/integration/metrics/test-metrics.sh
+
   loader-validate:
     runs-on: ubuntu-26.04
     needs: [ build, pull-and-archive ]

.github/workflows/unit-tests.yml

@@ -36,7 +36,7 @@ jobs:
           AGE_TEST_BINARY: /usr/bin/age-keygen
         run: go test -mod=readonly -tags "$BUILDTAGS" ./...
       - name: Engine concurrency race checks
-        run: go test -race -mod=readonly -tags "$BUILDTAGS" ./pkg/engine -run 'TestStatus|TestMirror|TestCommonGit|TestHelperContainer|TestRetirement|TestConfigReloadCan|TestRollback|TestDisconnected|TestFileTransfer|TestShutdown'
+        run: go test -race -mod=readonly -tags "$BUILDTAGS" ./pkg/engine -run 'TestStatus|TestMetrics|TestMirror|TestCommonGit|TestHelperContainer|TestRetirement|TestConfigReloadCan|TestRollback|TestDisconnected|TestFileTransfer|TestShutdown'
 
   fedora:
     runs-on: ubuntu-26.04

docs/index.rst

@@ -25,6 +25,7 @@ images containing those changes; an older published image may lack new options.
    mirrors
    quadlet
    status
+   metrics
    release_notes
    running
    documentation

docs/metrics.rst

@@ -0,0 +1,206 @@
+Fleet metrics
+=============
+
+FetchIt provides an optional, read-only Prometheus endpoint for
+`issue #418 <https://github.com/containers/fetchit/issues/418>`_. An existing
+remote Prometheus server scrapes FetchIt; FetchIt does not push metrics or need
+a Prometheus server URL. Deploying Prometheus is outside FetchIt's scope.
+
+Enable the endpoint
+-------------------
+
+Set ``FETCHIT_METRICS_ADDR`` to an explicit listen address. Unset means no
+metrics listener. Port ``8080`` is the convention used in these examples;
+you can choose another port. For a native process, begin with loopback:
+
+.. code-block:: bash
+
+   FETCHIT_METRICS_ADDR=127.0.0.1:8080 FETCHIT_HOSTNAME=edge-01.example.invalid fetchit start
+   curl -i http://127.0.0.1:8080/metrics
+
+``FETCHIT_HOSTNAME`` defaults to the operating system hostname. For containers,
+set it to the underlying host's unique hostname, since the container hostname
+usually identifies the container. Allowed values contain 1–253 ASCII letters,
+digits, dots, underscores or hyphens, starting with a letter or digit. An invalid
+value fails startup. Host identity comes from configuration, never request
+headers. ``/metrics`` includes the ``hostname`` label and an
+``X-Fetchit-Hostname`` response header. The separate :doc:`status` listener
+also includes that header, and ``/status`` adds a top-level ``hostname`` field.
+
+For an existing container deployment, add these options to ``podman run``:
+
+.. code-block:: bash
+
+   -e FETCHIT_METRICS_ADDR=:8080 \
+   -e FETCHIT_HOSTNAME=edge-01.example.invalid \
+   -p 127.0.0.1:8080:8080
+
+Keep the deployment's existing config, state volume, and Podman socket mounts.
+These options listen inside the container and publish only the host's loopback
+port. With host networking, omit ``-p`` and bind ``127.0.0.1:8080`` directly.
+
+Remote collection and security
+------------------------------
+
+To scrape remotely, publish on the host's monitoring-network address instead
+of loopback, or bind that address directly with host networking. For example,
+if ``10.0.1.20`` is an address assigned to the host:
+
+.. code-block:: bash
+
+   -e FETCHIT_METRICS_ADDR=:8080 \
+   -e FETCHIT_HOSTNAME=edge-01.example.invalid \
+   -p 10.0.1.20:8080:8080
+
+Permit inbound TCP 8080 only from the collector using the host firewall and
+network security rules. In AWS, restrict the instance security group's rule to
+the collector's security group or a specific private source address. Verify
+both intended access and denial from other hosts. A private address alone is
+not access control. Loopback does not authenticate local processes; published
+loopback ports do not isolate a service from container-network peers.
+
+The listener serves plain HTTP with no authentication or native TLS. Use only
+on a deliberately restricted trusted monitoring network. For traffic crossing
+untrusted networks, use an authenticated TLS proxy or tunnel, with certificate
+verification at the collector. This endpoint supplies neither encryption nor
+credentials. Do not publish it on the public internet.
+
+The metrics listener serves only ``GET /metrics`` (also ``HEAD``). It does not
+expose ``/status``, ``/healthz``, debug, or administration routes. Leave
+``FETCHIT_STATUS_ADDR`` unset unless its separate endpoint is needed:
+``/status`` contains raw method names and repository URLs. The two listeners
+must use different addresses/ports if both are enabled.
+
+Scrapes read an in-memory snapshot. They do not fetch Git, invoke Podman, or
+change workloads. The output contains bounded kinds/states and the configured
+hostname; no repository URLs, method names, paths, SHAs, credentials, decrypted
+contents, or free-form errors become labels. A Git SHA identifies a revision
+shared by many hosts, not a host identity. Keep hostnames unique within the
+fleet; Prometheus also attaches its own ``job`` and ``instance`` target labels.
+
+Configure an existing Prometheus server:
+
+.. code-block:: yaml
+
+   scrape_configs:
+     - job_name: fetchit
+       metrics_path: /metrics
+       scrape_interval: 30s
+       static_configs:
+         - targets: ['edge-01.example.invalid:8080']
+
+Replace the target with the reachable, access-controlled endpoint. Prometheus's
+own port ``9090`` does not determine FetchIt's listening port.
+
+Metrics and synchronization
+---------------------------
+
+All metrics are gauges. A configuration means a **method declaration**, not a
+repository, file, or deployed resource. One repository may contain several
+independently scheduled methods. Counts describe the current configuration;
+removed methods disappear and reloads reset synchronization to unknown until
+startup replay and reconciliation complete.
+
+.. list-table:: Metric families
+   :header-rows: 1
+   :widths: 45 55
+
+   * - Metric
+     - Meaning and labels
+   * - ``fetchit_host_info``
+     - Value 1; ``hostname``.
+   * - ``fetchit_configured_methods``
+     - All configured method declarations; ``hostname``, ``kind``.
+   * - ``fetchit_git_methods``
+     - Connected Git methods; ``hostname``, ``kind``, ``state``.
+   * - ``fetchit_git_oldest_observation_timestamp_seconds``
+     - Oldest successful Git observation among methods of a kind, in Unix
+       seconds; ``hostname``, ``kind``. Zero if any has never observed Git.
+
+``kind`` is one of ``raw``, ``kube``, ``ansible``, ``filetransfer``, ``systemd``,
+``quadlet``, ``image``, ``prune``, ``config`` or ``other``. Only kinds
+present in configuration are emitted. For connected Git methods, all three
+state series are emitted, including zero values:
+
+* ``synced``: the latest successfully fetched and accepted branch revision was
+  applied successfully, and the saved Git baseline matches it. Startup replay
+  must also succeed. An unchanged revision is confirmed by a successful check.
+* ``out_of_sync``: a desired revision was observed but its application or
+  baseline update has not completed successfully. Partial application, failed
+  application, skipped bad commits, and rollback do not establish sync.
+* ``unknown``: no completed observation/application establishes the current
+  state, including startup, checks in progress, clone/replay failure, or source
+  access failure. A cancelled run cannot report success.
+
+Image, prune, config reload, auto-update, and disconnected archive/device
+methods count toward configured declarations, but not connected Git sync
+counts. Disconnected operation cannot establish whether a remote Git branch
+is current. Mirror-enabled targets observe the accepted configured source
+using FetchIt's existing mirror selection and commit verification rules.
+
+Git is checked on each method's schedule. ``synced`` means up to date with the
+**last observed** repository/branch revision; commits published afterward are
+unknown until the next check. Freshness advances only after a successful Git
+fetch/reference checkout and any required commit verification. An apply failure
+can have fresh Git knowledge while remaining out of sync. Source failure keeps
+the previous timestamp, or zero if none exists. This is configuration
+synchronization, not workload readiness or detection of manual workload drift.
+
+Example output for one successful raw method:
+
+.. code-block:: text
+
+   fetchit_host_info{hostname="edge-01.example.invalid"} 1
+   fetchit_configured_methods{hostname="edge-01.example.invalid",kind="raw"} 1
+   fetchit_git_methods{hostname="edge-01.example.invalid",kind="raw",state="synced"} 1
+   fetchit_git_methods{hostname="edge-01.example.invalid",kind="raw",state="out_of_sync"} 0
+   fetchit_git_methods{hostname="edge-01.example.invalid",kind="raw",state="unknown"} 0
+   fetchit_git_oldest_observation_timestamp_seconds{hostname="edge-01.example.invalid",kind="raw"} 1791700000.000
+
+Queries and alert examples
+--------------------------
+
+Total configured declarations per host:
+
+.. code-block:: promql
+
+   sum by (hostname) (fetchit_configured_methods)
+
+Pending or unknown connected Git methods:
+
+.. code-block:: promql
+
+   sum by (hostname) (fetchit_git_methods{state=~"out_of_sync|unknown"}) > 0
+
+For methods checking every minute, a five-minute observation deadline can be a
+starting point. Adjust for the longest configured interval, skew, and expected
+operation duration. Require the condition to persist to avoid alerts during
+normal checks or startup:
+
+.. code-block:: yaml
+
+   groups:
+     - name: fetchit
+       rules:
+         - alert: FetchitGitNotSynced
+           expr: sum by (hostname) (fetchit_git_methods{state=~"out_of_sync|unknown"}) > 0
+           for: 5m
+         - alert: FetchitGitObservationStale
+           expr: time() - fetchit_git_oldest_observation_timestamp_seconds > 300
+           for: 5m
+         - alert: FetchitScrapeUnavailable
+           expr: up{job="fetchit"} == 0
+           for: 2m
+
+A successful scrape (``up == 1``) proves endpoint availability, not successful
+Git observation or application. Zero freshness is deliberately stale. Missing
+series after removal are different from a zero-valued state series; the
+``up`` alert detects a failed scrape, while removal of a target from Prometheus
+requires inventory monitoring outside FetchIt.
+
+References
+----------
+
+See `Prometheus scraping <https://prometheus.io/docs/prometheus/latest/getting_started/>`_,
+the `security model <https://prometheus.io/docs/operating/security/>`_, and
+`jobs and instances <https://prometheus.io/docs/concepts/jobs_instances/>`_.

docs/status.rst

@@ -1,6 +1,11 @@
 Health and status endpoint
 ==========================
 
+For Prometheus configuration synchronization metrics, see :doc:`metrics`.
+The separate metrics listener exposes no repository metadata. ``/healthz`` and
+``/status`` describe liveness and invocation activity, not successful
+configuration application, Git synchronization, or workload health.
+
 FetchIt can expose an optional HTTP endpoint for process liveness and scheduled
 method activity, including Quadlet methods. Set ``FETCHIT_STATUS_ADDR`` to enable
 it; leaving the variable unset opens no listener. For a host installation, use
@@ -18,12 +23,25 @@ and exposes method names and repository URLs; use trusted access or an
 authenticated reverse proxy for remote monitoring. Repository URLs should not
 contain credentials.
 
+The loopback publication above restricts the published host port; it does not
+authenticate local clients or prevent access from peers that can reach the
+container's network. Keep the engine off networks shared with untrusted
+containers. With host networking, bind the listener itself to
+``127.0.0.1:8080``. Host-local processes that can connect to the listener can
+read its output. Do not expose ``/status`` remotely merely to collect metrics:
+the current response includes configured names and URLs without redaction.
+
 .. code-block:: shell
 
    curl --fail http://127.0.0.1:8080/healthz
    curl --fail http://127.0.0.1:8080/status
 
 ``GET /healthz`` returns ``200 OK`` with ``ok`` while the HTTP server can respond.
+Responses include an ``X-Fetchit-Hostname`` header; ``/status`` also reports
+``hostname`` in JSON. Set ``FETCHIT_HOSTNAME`` explicitly to the underlying
+hostname when running in a container; otherwise it defaults to the operating
+system hostname. See :doc:`metrics` for validation rules.
+
 It is a liveness check: it does not assert Podman connectivity, successful
 reconciliation, or workload readiness.
 
@@ -33,6 +51,7 @@ reconciliation, or workload readiness.
 
    {
      "status": "running",
+     "hostname": "edge-01.example.invalid",
      "startedAt": "2026-10-08T12:00:00Z",
      "uptimeSeconds": 312,
      "methods": [

pkg/engine/common.go

@@ -103,11 +103,13 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[
 		return fmt.Errorf("Failed to get latest commit: %v", err)
 	}
 
+	observeGit(ctx, "unknown", true)
 	current, err := getCurrent(target, m.GetKind(), m.GetName())
 	if err != nil {
 		return fmt.Errorf("Failed to get current commit: %v", err)
 	}
 
+	observeGit(ctx, "out_of_sync", false)
 	if latest != current {
 		if err := applyWithRecovery(ctx, conn, m, current, latest, tag); err != nil {
 			return fmt.Errorf("Failed to apply changes: %w", err)
@@ -120,6 +122,10 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[
 		logger.Infof("No changes applied to git target %s this run, %s currently at %s", directory, m.GetKind(), current.String()[:hashReportLen])
 	}
 
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	observeGit(ctx, "synced", false)
 	return nil
 }
 

pkg/engine/fetchit.go

@@ -137,12 +137,12 @@ func (f *Fetchit) runMethod(m Method, ctx, conn context.Context, skew int) {
 			return
 		}
 		status.recordRun(m)
-		m.Process(ctx, conn, skew)
+		m.Process(metrics.begin(ctx, m), conn, skew)
 		return
 	}
 	defer f.runMu.RUnlock()
 	status.recordRun(m)
-	m.Process(ctx, conn, skew)
+	m.Process(metrics.begin(ctx, m), conn, skew)
 }
 
 func readConfig(v *viper.Viper) (*FetchitConfig, bool, error) {
@@ -426,6 +426,10 @@ func (f *Fetchit) startTargets() error {
 	if err := parent.Err(); err != nil {
 		return err
 	}
+	hostname, err := configuredHostname()
+	if err != nil {
+		return err
+	}
 	store, err := loadRemovalStore(defaultRemovalsPath)
 	if err != nil {
 		return err
@@ -439,7 +443,12 @@ func (f *Fetchit) startTargets() error {
 	}
 	ctx, cancel := context.WithCancel(parent)
 	f.runCancel = cancel
+	metrics.replace(f.methodTargetScheds, hostname)
 	status.replace(f.methodTargetScheds)
+	status.setHostname(hostname)
+	if err := startMetricsServer(); err != nil {
+		return err
+	}
 	startStatusServer()
 	for method := range f.methodTargetScheds {
 		if err := ctx.Err(); err != nil {