Private/mayuresh/pcd 10432

Dockerfile.pf9

@@ -0,0 +1,46 @@
+# syntax=docker/dockerfile:1
+#
+# Platform9 build of openstack-exporter.
+#
+# Unlike the upstream Dockerfile (which expects promu to have already produced
+# .build/$OS-$ARCH/openstack-exporter), this builds the binary itself so CI
+# agents need nothing but docker. The runtime stage is deliberately the same
+# base upstream publishes, so a PF9-built image differs from the upstream
+# release only by the code in this repo.
+
+ARG GO_VERSION=1.24
+
+FROM golang:${GO_VERSION}-alpine AS builder
+
+ARG GOOS=linux
+ARG GOARCH=amd64
+ARG CGO_ENABLED=0
+
+WORKDIR /src
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN mkdir -p /out && \
+	GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=${CGO_ENABLED} \
+	go build -trimpath -ldflags="-s -w" -o /out/openstack-exporter .
+
+FROM quay.io/prometheus/busybox:latest@sha256:7379672d53c28113469275f2eb48adffa19954f39f8ef20c1e2992278f658e6c
+
+ARG VERSION=1.0.0
+ARG BUILD_ID=0
+ARG BRANCH=local
+ARG APP_METADATA=
+
+COPY --from=builder /out/openstack-exporter /bin/openstack-exporter
+
+ENTRYPOINT ["/bin/openstack-exporter"]
+EXPOSE     9180
+
+LABEL com.platform9.pf9_version=${VERSION}
+LABEL com.platform9.build=${BUILD_ID}
+LABEL com.platform9.version="${VERSION}-${BUILD_ID}"
+LABEL com.platform9.branch=${BRANCH}
+LABEL com.platform9.app_metadata=${APP_METADATA}

Makefile.pf9

@@ -0,0 +1,68 @@
+# Platform9 container build for openstack-exporter.
+#
+# Kept separate from the upstream Makefile so rebases against
+# openstack-exporter/openstack-exporter stay conflict-free.
+#
+#   make -f Makefile.pf9 all
+#
+# TeamCity supplies PF9_VERSION and BUILD_NUMBER; the defaults below are only
+# for local runs.
+
+APP_NAME ?= openstack-exporter
+DOCKER ?= docker
+
+GOOS ?= linux
+GOARCH ?= amd64
+CGO_ENABLED ?= 0
+GO_VERSION ?= 1.24
+
+PF9_VERSION ?= 1.0.0
+BUILD_NUMBER ?= 0
+BRANCH ?= $(or $(TEAMCITY_BUILD_BRANCH),$(shell git symbolic-ref --short HEAD 2>/dev/null || echo local))
+APP_METADATA ?=
+
+IMAGE_NAME ?= quay.io/platform9/$(APP_NAME)
+ECR_IMAGE_NAME ?= public.ecr.aws/platform9/$(APP_NAME)
+IMAGE_TAG ?= $(PF9_VERSION)-$(BUILD_NUMBER)
+FULL_IMAGE ?= $(IMAGE_NAME):$(IMAGE_TAG)
+ECR_FULL_IMAGE ?= $(ECR_IMAGE_NAME):$(IMAGE_TAG)
+
+DOCKERFILE ?= Dockerfile.pf9
+
+.PHONY: all build push print clean
+
+all: build push print
+
+build:
+	@echo "Building Docker image: $(FULL_IMAGE)"
+	$(DOCKER) build \
+		--build-arg GO_VERSION=$(GO_VERSION) \
+		--build-arg GOOS=$(GOOS) \
+		--build-arg GOARCH=$(GOARCH) \
+		--build-arg CGO_ENABLED=$(CGO_ENABLED) \
+		--build-arg VERSION=$(PF9_VERSION) \
+		--build-arg BUILD_ID=$(BUILD_NUMBER) \
+		--build-arg BRANCH=$(BRANCH) \
+		--build-arg APP_METADATA=$(APP_METADATA) \
+		-f $(DOCKERFILE) -t $(FULL_IMAGE) .
+	$(DOCKER) tag $(FULL_IMAGE) $(ECR_FULL_IMAGE)
+	echo $(IMAGE_TAG) > container-tag
+
+push:
+	@echo "Pushing Docker image: $(FULL_IMAGE) and $(ECR_FULL_IMAGE)"
+	$(DOCKER) push $(FULL_IMAGE)
+	$(DOCKER) push $(ECR_FULL_IMAGE)
+
+print:
+	@echo "GOOS=$(GOOS)"
+	@echo "GOARCH=$(GOARCH)"
+	@echo "CGO_ENABLED=$(CGO_ENABLED)"
+	@echo "PF9_VERSION=$(PF9_VERSION)"
+	@echo "BUILD_NUMBER=$(BUILD_NUMBER)"
+	@echo "BRANCH=$(BRANCH)"
+	@echo "FULL_IMAGE=$(FULL_IMAGE)"
+	@echo "ECR_FULL_IMAGE=$(ECR_FULL_IMAGE)"
+
+clean:
+	rm -f container-tag
+	-$(DOCKER) rmi $(FULL_IMAGE) $(ECR_FULL_IMAGE) 2>/dev/null || true

README.md

@@ -108,6 +108,9 @@ Flags:
                                  Disable UUID generation for Cinder agents
       --[no-]multi-cloud         Toggle the multiple cloud scraping mode under /probe?cloud=
       --domain-id=DOMAIN-ID      Gather metrics only for the given Domain ID (defaults to all domains)
+      --[no-]cache                Collect metrics in the background and serve every scrape from the last collection (legacy mode only)
+      --cache-ttl=300s           TTL for cached metrics (eg. 10s, 11m, 1h). Cached metrics older than cache-ttl are not served
+      --cache-interval=150s      How often the background collection runs when --cache is enabled (eg. 30s, 5m). Must be less than --cache-ttl
       --[no-]disable-service.network  
                                  Disable the network service exporter
       --[no-]disable-service.compute  
@@ -174,6 +177,26 @@ curl "https://localhost:9180/probe?cloud=test.cloud&include_services=network,com
 curl "https://localhost:9180/probe?cloud=test.cloud&exclude_services=load-balancer,dns"
 ```
 
+### Cache mechanism
+
+`--cache` applies only to legacy mode (`/metrics`, single cloud). It has no
+effect in multi-cloud mode (`/probe?cloud=`), since each probe already
+targets a specific cloud and is not swept in the background.
+
+Without `--cache`, every scrape of `/metrics` runs a full sweep of the OpenStack
+APIs, so two scrapers cause two sweeps. With `--cache`:
+
+* A background loop sweeps once at start-up and then every `cache-interval`.
+  Sweeps never overlap; a sweep slower than `cache-interval` is followed
+  immediately by the next one. `cache-interval` must be less than `cache-ttl`.
+* The stored result is replaced only when a sweep completes, and every scrape
+  is served from it without contacting OpenStack.
+* If the stored result is older than `cache-ttl`, scrapes return no OpenStack
+  metrics until the next sweep completes. Set `cache-ttl` comfortably above
+  `cache-interval` plus the slowest sweep time.
+* Until the first sweep completes after start-up, scrapes return no OpenStack
+  metrics.
+
 ### OpenStack configuration
 
 The cloud credentials and identity configuration

cache/cache.go

@@ -0,0 +1,97 @@
+// Package cache decouples Prometheus scrapes from OpenStack API sweeps.
+//
+// Without the cache every scrape of /metrics runs a full sweep of every
+// enabled OpenStack service, so N scrapers mean N concurrent sweeps. With the
+// cache a single background loop runs the sweep and stores the result, and
+// every scrape is served from that stored result.
+//
+// Behaviour (ported from upstream openstack-exporter's --cache):
+//   - a sweep runs at start-up and then every interval;
+//   - sweeps never overlap: if a sweep takes longer than interval the next one
+//     starts as soon as it finishes;
+//   - the stored result is replaced only once a sweep has fully completed, so
+//     a scrape never sees a mix of old and new values;
+//   - a stored result older than ttl is treated as expired and scrapes get no
+//     metrics until the next sweep completes.
+package cache
+
+import (
+	"sync"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+// Gatherer is a prometheus.Gatherer that serves the last result gathered from
+// source instead of gathering on every call.
+type Gatherer struct {
+	source   prometheus.Gatherer
+	ttl      time.Duration
+	interval time.Duration
+	now      func() time.Time
+	logger   log.Logger
+
+	mu       sync.RWMutex
+	families []*dto.MetricFamily
+	updated  time.Time
+}
+
+// New returns a Gatherer that sweeps source every interval and serves each
+// sweep's result for up to ttl.
+func New(source prometheus.Gatherer, ttl, interval time.Duration, logger log.Logger) *Gatherer {
+	return &Gatherer{source: source, ttl: ttl, interval: interval, now: time.Now, logger: logger}
+}
+
+// Refresh runs one sweep against source and stores the result.
+//
+// prometheus.Gatherer may return partial results together with an error; the
+// partial results are stored, matching what an uncached scrape would serve.
+func (c *Gatherer) Refresh() error {
+	families, err := c.source.Gather()
+	if families != nil {
+		c.mu.Lock()
+		c.families = families
+		c.updated = c.now()
+		c.mu.Unlock()
+	}
+	return err
+}
+
+// Gather implements prometheus.Gatherer. It never contacts OpenStack.
+func (c *Gatherer) Gather() ([]*dto.MetricFamily, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.updated.IsZero() || c.now().Sub(c.updated) > c.ttl {
+		return nil, nil
+	}
+	return c.families, nil
+}
+
+// Run refreshes the cache immediately and then every interval until stop is
+// closed. It blocks, so call it in its own goroutine.
+func (c *Gatherer) Run(stop <-chan struct{}) {
+	level.Info(c.logger).Log("msg", "Cache: starting background collection", "ttl", c.ttl, "interval", c.interval)
+	ticker := time.NewTicker(c.interval)
+	defer ticker.Stop()
+
+	for {
+		c.refreshAndLog()
+		select {
+		case <-ticker.C:
+		case <-stop:
+			level.Info(c.logger).Log("msg", "Cache: background collection stopped")
+			return
+		}
+	}
+}
+
+func (c *Gatherer) refreshAndLog() {
+	start := c.now()
+	if err := c.Refresh(); err != nil {
+		level.Error(c.logger).Log("msg", "Cache: collection finished with errors", "err", err)
+	}
+	level.Info(c.logger).Log("msg", "Cache: collection finished", "duration", c.now().Sub(start))
+}

cache/cache_test.go

@@ -0,0 +1,122 @@
+package cache
+
+import (
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+)
+
+// countingGatherer records how many sweeps were run against it.
+type countingGatherer struct {
+	calls    int32
+	families []*dto.MetricFamily
+	err      error
+}
+
+func (g *countingGatherer) Gather() ([]*dto.MetricFamily, error) {
+	atomic.AddInt32(&g.calls, 1)
+	return g.families, g.err
+}
+
+func family(name string) *dto.MetricFamily {
+	return &dto.MetricFamily{Name: &name}
+}
+
+func TestGatherBeforeFirstRefreshIsEmpty(t *testing.T) {
+	src := &countingGatherer{families: []*dto.MetricFamily{family("a")}}
+	c := New(src, time.Minute, 30*time.Second, log.NewNopLogger())
+
+	mfs, err := c.Gather()
+	assert.NoError(t, err)
+	assert.Empty(t, mfs)
+	assert.Equal(t, int32(0), src.calls, "Gather must not sweep")
+}
+
+func TestManyScrapesShareOneSweep(t *testing.T) {
+	src := &countingGatherer{families: []*dto.MetricFamily{family("a"), family("b")}}
+	c := New(src, time.Minute, 30*time.Second, log.NewNopLogger())
+
+	assert.NoError(t, c.Refresh())
+	for i := 0; i < 10; i++ {
+		mfs, err := c.Gather()
+		assert.NoError(t, err)
+		assert.Equal(t, src.families, mfs)
+	}
+	assert.Equal(t, int32(1), src.calls)
+}
+
+func TestExpiredCacheIsEmpty(t *testing.T) {
+	src := &countingGatherer{families: []*dto.MetricFamily{family("a")}}
+	c := New(src, time.Minute, 30*time.Second, log.NewNopLogger())
+	now := time.Unix(1000, 0)
+	c.now = func() time.Time { return now }
+
+	assert.NoError(t, c.Refresh())
+
+	now = now.Add(time.Minute)
+	mfs, _ := c.Gather()
+	assert.Len(t, mfs, 1, "exactly ttl old is still valid")
+
+	now = now.Add(time.Second)
+	mfs, _ = c.Gather()
+	assert.Empty(t, mfs, "older than ttl is expired")
+}
+
+func TestPartialResultsAreStored(t *testing.T) {
+	src := &countingGatherer{families: []*dto.MetricFamily{family("a")}, err: errors.New("boom")}
+	c := New(src, time.Minute, 30*time.Second, log.NewNopLogger())
+
+	assert.Error(t, c.Refresh())
+	mfs, _ := c.Gather()
+	assert.Len(t, mfs, 1)
+}
+
+func TestFailedRefreshKeepsPreviousResult(t *testing.T) {
+	src := &countingGatherer{families: []*dto.MetricFamily{family("a")}}
+	c := New(src, time.Minute, 30*time.Second, log.NewNopLogger())
+	assert.NoError(t, c.Refresh())
+
+	src.families, src.err = nil, errors.New("boom")
+	assert.Error(t, c.Refresh())
+
+	mfs, _ := c.Gather()
+	assert.Len(t, mfs, 1)
+}
+
+func TestRunRefreshesUntilStopped(t *testing.T) {
+	src := &countingGatherer{families: []*dto.MetricFamily{family("a")}}
+	c := New(src, time.Minute, 10*time.Millisecond, log.NewNopLogger())
+	stop := make(chan struct{})
+	done := make(chan struct{})
+
+	go func() { c.Run(stop); close(done) }()
+	time.Sleep(55 * time.Millisecond)
+	close(stop)
+	<-done
+
+	// One sweep at start-up plus roughly one every 10ms.
+	calls := atomic.LoadInt32(&src.calls)
+	assert.True(t, calls >= 3, "expected at least 3 sweeps, got %d", calls)
+}
+
+func TestServesRealRegistry(t *testing.T) {
+	reg := prometheus.NewPedanticRegistry()
+	g := prometheus.NewGauge(prometheus.GaugeOpts{Name: "openstack_test_gauge", Help: "h"})
+	reg.MustRegister(g)
+	g.Set(1)
+
+	c := New(reg, time.Minute, 30*time.Second, log.NewNopLogger())
+	assert.NoError(t, c.Refresh())
+
+	g.Set(2) // a scrape between sweeps must still see the swept value
+	mfs, err := c.Gather()
+	assert.NoError(t, err)
+	assert.Len(t, mfs, 1)
+	assert.Equal(t, 1.0, mfs[0].Metric[0].GetGauge().GetValue())
+}