diff --git a/.changeset/home-use-alignment.md b/.changeset/home-use-alignment.md new file mode 100644 index 000000000..73c5e51ec --- /dev/null +++ b/.changeset/home-use-alignment.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Calculate Home use from timestamped charger session energy where available and integrate other sources over the same period. Adapt the period to source reporting and show the estimated mean, period and age. Cloud polls do not refresh source time. Retain signed daily energy only for bounded balance failures that recover, while preserving raw quality, offline gaps and control inputs. diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 836bb755a..c732193ea 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -2463,11 +2463,13 @@ func (s *Server) handleEnergyDaily(w http.ResponseWriter, r *http.Request) { for i := days - 1; i >= 0; i-- { dayStart := todayMidnight.AddDate(0, 0, -i) dayKey := dayStart.Format("2006-01-02") - isToday := i == 0 + // A bounded balance episode can close just after midnight. Let its + // final intervals arrive before treating the previous day as immutable. + cacheReady := dailyEnergyCacheReady(dayStart, now) var de state.DayEnergy - if isToday { - d, err := s.deps.State.DailyEnergy(dayStart.UnixMilli(), now.UnixMilli()) + if !cacheReady { + d, err := s.deps.State.DailyEnergy(dayStart.UnixMilli(), min(now.UnixMilli(), dayStart.AddDate(0, 0, 1).UnixMilli())) if err != nil { slog.Error("handleEnergyDaily: DailyEnergy failed", "err", err, "day", dayKey) http.Error(w, "history load failed", http.StatusInternalServerError) diff --git a/go/internal/api/api_energy_today.go b/go/internal/api/api_energy_today.go index 57ea58739..de8e5cc58 100644 --- a/go/internal/api/api_energy_today.go +++ b/go/internal/api/api_energy_today.go @@ -23,6 +23,11 @@ func energyTodayFields(d state.DayEnergy) map[string]any { out[key] = d.Missing.Value(field.flag, field.wh) measured[key] = d.MeasuredValue(field.flag, field.wh) } + if d.Missing&state.EstimatedLoad != 0 { + // The load total includes inferred intervals. It cannot also appear + // in the measured-only fallback used by clients after a data gap. + measured["load_wh"] = nil + } out["measured"] = measured return out } diff --git a/go/internal/api/api_history_quality.go b/go/internal/api/api_history_quality.go index 08e6a6d18..29a8670fd 100644 --- a/go/internal/api/api_history_quality.go +++ b/go/internal/api/api_history_quality.go @@ -1,11 +1,17 @@ package api import ( + "strings" + "time" + "github.com/srcfl/ftw/go/internal/state" "github.com/srcfl/ftw/go/internal/telemetry" - "strings" ) +func dailyEnergyCacheReady(dayStart, now time.Time) bool { + return !now.Before(dayStart.AddDate(0, 0, 1).Add(state.LoadEstimateRecoveryWindow + time.Minute)) +} + func (s *Server) qualifyStatusHistory(resp map[string]any, siteMeter string) { if s.deps.HistoryMeasurementSources == nil { return @@ -13,6 +19,7 @@ func (s *Server) qualifyStatusHistory(resp map[string]any, siteMeter string) { opts := s.deps.HistoryMeasurementSources() opts.UseWatchdogTimeout = true r := s.deps.Tel.ForecastMeasurementNow(siteMeter, opts) + resp["load_estimate"] = s.deps.Tel.HomeUseEstimate(time.Now(), siteMeter, opts) var missing state.HistoryMissing for kind, flag := range map[telemetry.DerType]state.HistoryMissing{telemetry.DerMeter: state.MissingGrid, telemetry.DerPV: state.MissingPV, telemetry.DerBattery: state.MissingBattery, telemetry.DerEV: state.MissingEV, telemetry.DerV2X: state.MissingV2X} { if !r.Known[kind] { diff --git a/go/internal/api/api_home_estimate_test.go b/go/internal/api/api_home_estimate_test.go new file mode 100644 index 000000000..dbd52632a --- /dev/null +++ b/go/internal/api/api_home_estimate_test.go @@ -0,0 +1,68 @@ +package api + +import ( + "encoding/json" + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +func TestStatusHomeEstimateIsSeparateFromInstantaneousPower(t *testing.T) { + tel := telemetry.NewStore() + tel.EnsureDriverHealth("site") + base := time.Now().Add(-20 * time.Second) + for i := 0; i <= 10; i++ { + data, _ := json.Marshal(map[string]any{"power_observed_at": base.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)}) + tel.Update("site", telemetry.DerMeter, 1000+float64(i)*100, nil, data) + } + srv := New(&Deps{Tel: tel, HistoryMeasurementSources: func() telemetry.ForecastOptions { return telemetry.ForecastOptions{} }}) + resp := map[string]any{} + for i := 0; i < 3; i++ { + srv.qualifyStatusHistory(resp, "site") + e, ok := resp["load_estimate"].(telemetry.HomeEstimate) + if !ok || e.Watts == nil || math.Abs(*e.Watts-1500) > 1e-6 || e.AgeMS < 15000 || e.AgeMS > 20000 || e.Method != "reported_energy_balance_v3" || e.TimingBasis != "source" || e.EnergyBasis != "power_integral" || e.EnergyWh == nil || e.EndAgeMS < 10000 { + t.Fatal(e) + } + if resp["load_w"] != float64(2000) || resp["grid_w"] != float64(2000) || resp["history_quality"] != "complete" { + t.Fatal(resp) + } + } + tel.DriverHealthMut("site").SetOffline() + srv.qualifyStatusHistory(resp, "site") + if resp["load_w"] != nil || resp["load_estimate"].(telemetry.HomeEstimate).Watts != nil { + t.Fatal(resp) + } +} + +func TestEnergyTodayKeepsEstimatedLoadOutOfMeasuredFields(t *testing.T) { + got := energyTodayFields(state.DayEnergy{Intervals: 2, Missing: state.EstimatedLoad, LoadWh: 22, ImportWh: 10}) + if got["history_quality"] != "estimated" || got["load_wh"] != float64(22) { + t.Fatal(got) + } + measured := got["measured"].(map[string]any) + if measured["load_wh"] != nil || measured["import_wh"] != float64(10) { + t.Fatal(measured) + } +} + +func TestDailyEnergyCacheWaitsForMidnightRecovery(t *testing.T) { + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatal(err) + } + // Include the 23/25-hour days: a local day is not always 24 hours. + for _, day := range []time.Time{time.Date(2026, 10, 17, 0, 0, 0, 0, loc), time.Date(2026, 10, 25, 0, 0, 0, 0, loc), time.Date(2026, 3, 29, 0, 0, 0, 0, loc)} { + end := day.AddDate(0, 0, 1) + for _, elapsed := range []time.Duration{-time.Second, 0, 30 * time.Second, 59 * time.Second, 2 * time.Minute, 179 * time.Second} { + if dailyEnergyCacheReady(day, end.Add(elapsed)) { + t.Fatal("cached before recovery can finish", day, elapsed) + } + } + if !dailyEnergyCacheReady(day, end.Add(3*time.Minute)) { + t.Fatal("closed day never became cacheable", day) + } + } +} diff --git a/go/internal/state/history_dashboard.go b/go/internal/state/history_dashboard.go index 8e1bfbce0..d21b48b94 100644 --- a/go/internal/state/history_dashboard.go +++ b/go/internal/state/history_dashboard.go @@ -42,6 +42,9 @@ func (s *Store) recordDashboardTx(ctx context.Context, tx *sql.Tx, p *HistoryPoi } } if p == nil { + if err := clearLoadEstimatePendingTx(ctx, tx); err != nil { + return err + } if _, err := tx.ExecContext(ctx, `INSERT INTO history_site_cursor VALUES('last',0) ON CONFLICT(key) DO UPDATE SET value=0`); err != nil { return err } @@ -78,6 +81,11 @@ func (s *Store) recordDashboardTx(ctx context.Context, tx *sql.Tx, p *HistoryPoi } } if p.TsMs > previous { + if previous == 0 || p.TsMs-previous > maxLoadEstimateTickMS { + if err := clearLoadEstimatePendingTx(ctx, tx); err != nil { + return err + } + } if previous > 0 && p.TsMs-previous <= maxCostIntegrationGap.Milliseconds() { var previousMissing HistoryMissing err := tx.QueryRowContext(ctx, `SELECT value FROM history_partial_cursor WHERE key='missing'`).Scan(&previousMissing) @@ -85,13 +93,11 @@ func (s *Store) recordDashboardTx(ctx context.Context, tx *sql.Tx, p *HistoryPoi return err } var previousEligible int - if previousMissing != 0 { - err := tx.QueryRowContext(ctx, `SELECT value FROM history_partial_cursor WHERE key='residual_eligible'`).Scan(&previousEligible) - if err != nil && !errors.Is(err, sql.ErrNoRows) { - return err - } + err = tx.QueryRowContext(ctx, `SELECT value FROM history_partial_cursor WHERE key='residual_eligible'`).Scan(&previousEligible) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err } - if err := addSiteIntervalEstimateTx(ctx, tx, previous, p, previousMissing, previousMissing == 0 || previousEligible == 1); err != nil { + if err := addSiteIntervalEstimateTx(ctx, tx, previous, p, previousMissing, previousEligible == 1); err != nil { return err } if err := addSiteIntervalTx(ctx, tx, previous, p, previousMissing); err != nil { @@ -158,7 +164,9 @@ func addSiteIntervalTx(ctx context.Context, tx *sql.Tx, previous int64, p *Histo point.ClearMissingValues() p = &point table, kind := "history_site_energy", "energy" - if p.Missing != 0 { + // SoC does not enter the power balance. Keep its quality bit, but do + // not mix otherwise known power into a failed-balance energy bucket. + if p.Missing&^MissingSoC != 0 { table, kind = "history_site_energy_partial", "partial_energy" } for from := previous; from < p.TsMs; { @@ -217,7 +225,7 @@ type siteEnergyRow struct { func (s *Store) walkSiteEnergy(ctx context.Context, from, to int64, visit func(siteEnergyRow) error) error { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - rows, err := s.history.QueryContext(ctx, `WITH energy AS (SELECT *, 'energy' AS kind FROM history_site_energy UNION ALL SELECT *, 'partial_energy' AS kind FROM history_site_energy_partial) SELECT e.start_ms,e.resolution_ms,e.first_ms,e.last_ms,e.covered_ms,e.n,e.import_wh,e.export_wh,e.pv_wh,e.bat_charge_wh,e.bat_discharge_wh,e.load_wh,e.ev_wh,COALESCE(q.missing,0),CASE WHEN e.kind='partial_energy' AND q.missing IN (136,40) THEN COALESCE((SELECT SUM(load_wh) FROM history_load_estimates x WHERE x.start_ms>=e.start_ms AND x.start_ms=e.start_ms AND x.start_ms=? AND e.start_ms=? AND last_ms<=? ORDER BY e.start_ms`, bucketStart(from, EnergyLedgerDailyBucketMS), to, from, to) + rows, err := s.history.QueryContext(ctx, `WITH energy AS (SELECT *, 'energy' AS kind FROM history_site_energy UNION ALL SELECT *, 'partial_energy' AS kind FROM history_site_energy_partial) SELECT e.start_ms,e.resolution_ms,e.first_ms,e.last_ms,e.covered_ms,e.n,e.import_wh,e.export_wh,e.pv_wh,e.bat_charge_wh,e.bat_discharge_wh,e.load_wh,e.ev_wh,COALESCE(q.missing,0),CASE WHEN e.kind='partial_energy' AND (q.missing & ~16) IN (136,40) THEN COALESCE((SELECT SUM(load_wh) FROM history_load_estimates x WHERE x.start_ms>=e.start_ms AND x.start_ms=e.start_ms AND x.start_ms=? AND e.start_ms=? AND last_ms<=? ORDER BY e.start_ms`, bucketStart(from, EnergyLedgerDailyBucketMS), to, from, to) if err != nil { return err } @@ -248,9 +256,9 @@ func (s *Store) aggregateDayEnergy(ctx context.Context, from, to int64) (DayEner originalLoad += r.load originalMissing |= r.missing coveredMS += r.covered - if r.missing == MissingLoad|BalanceMismatch || r.missing == MissingLoad|MissingEV { + if r.missing&^MissingSoC == MissingLoad|BalanceMismatch || r.missing&^MissingSoC == MissingLoad|MissingEV { if r.estimateCovered > 0 && r.estimateCovered == r.covered { - r.load, r.missing = r.estimateLoad, EstimatedLoad + r.load, r.missing = r.estimateLoad, (r.missing&MissingSoC)|EstimatedLoad estimatedMS += r.covered } } diff --git a/go/internal/state/history_load_estimate.go b/go/internal/state/history_load_estimate.go index 9a88bd36d..13d18e0ad 100644 --- a/go/internal/state/history_load_estimate.go +++ b/go/internal/state/history_load_estimate.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "math" + "time" ) // Estimates supplement, never overwrite, the original measurements and quality. @@ -15,8 +16,20 @@ const loadEstimateSchema = `CREATE TABLE IF NOT EXISTS history_load_estimates ( start_ms INTEGER NOT NULL,resolution_ms INTEGER NOT NULL,covered_ms INTEGER NOT NULL, load_wh REAL NOT NULL,method TEXT NOT NULL,PRIMARY KEY(start_ms,resolution_ms)) WITHOUT ROWID` +// Pending evidence never enters totals. A valid balance must close a short +// episode before any of its intervals can supplement the original history. +const loadEstimatePendingSchema = `CREATE TABLE IF NOT EXISTS history_load_estimate_pending ( + from_ms INTEGER PRIMARY KEY,to_ms INTEGER NOT NULL,load_w REAL NOT NULL) WITHOUT ROWID` + +// LoadEstimateRecoveryWindow bounds a recovered balance episode, including a +// minute-scale cloud lag. It is a history estimate budget, not a sensor delay. +const LoadEstimateRecoveryWindow = 2 * time.Minute +const maxLoadEstimateEpisodeMS = int64(LoadEstimateRecoveryWindow / time.Millisecond) +const maxLoadEstimateTickMS int64 = 15_000 + func residualHouseholdW(p *HistoryPoint) (float64, bool) { - if p.Missing != 0 && p.Missing != MissingLoad|BalanceMismatch { + missing := p.Missing &^ MissingSoC + if missing != 0 && missing != MissingLoad|BalanceMismatch { return 0, false } var evidence struct { @@ -27,7 +40,7 @@ func residualHouseholdW(p *HistoryPoint) (float64, bool) { if json.Unmarshal([]byte(p.JSON), &evidence) != nil || evidence.EV == nil || evidence.V2X == nil { return 0, false } - if p.Missing != 0 && evidence.Reason != "invalid_balance" { + if missing != 0 && evidence.Reason != "invalid_balance" { return 0, false } w := p.GridW - p.PVW - p.BatW - *evidence.EV - *evidence.V2X @@ -37,16 +50,72 @@ func residualHouseholdW(p *HistoryPoint) (float64, bool) { func addSiteIntervalEstimateTx(ctx context.Context, tx *sql.Tx, previous int64, p *HistoryPoint, previousMissing HistoryMissing, previousEligible bool) error { // An offline, duplicate, skewed or otherwise missing source is not repaired // by interpreting it as zero. Both endpoints need usable source evidence. - if p.Missing|previousMissing != MissingLoad|BalanceMismatch || !previousEligible { - return nil - } + missing := (p.Missing | previousMissing) &^ MissingSoC w, ok := residualHouseholdW(p) - if !ok { + if missing != MissingLoad|BalanceMismatch || !previousEligible || !ok || p.TsMs-previous > maxLoadEstimateTickMS { + return clearLoadEstimatePendingTx(ctx, tx) + } + var start int64 + if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MIN(from_ms),?) FROM history_load_estimate_pending`, previous).Scan(&start); err != nil { + return err + } + if p.TsMs-start > maxLoadEstimateEpisodeMS { + // Keep a marker until a valid snapshot arrives. Discarding only the + // rows would let a long failure look like repeated short episodes. + if err := clearLoadEstimatePendingTx(ctx, tx); err != nil { + return err + } + if p.Missing&MissingLoad == 0 { + return nil + } + _, err := tx.ExecContext(ctx, `INSERT INTO history_load_estimate_pending VALUES(?, ?, 0)`, start, p.TsMs) + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO history_load_estimate_pending VALUES(?,?,?)`, previous, p.TsMs, w); err != nil { + return err + } + if p.Missing&MissingLoad != 0 { return nil } - for from := previous; from < p.TsMs; { + rows, err := tx.QueryContext(ctx, `SELECT from_ms,to_ms,load_w FROM history_load_estimate_pending ORDER BY from_ms`) + if err != nil { + return err + } + type interval struct { + from, to int64 + watts float64 + } + var intervals []interval + for rows.Next() { + var v interval + if err := rows.Scan(&v.from, &v.to, &v.watts); err != nil { + rows.Close() + return err + } + intervals = append(intervals, v) + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + for _, v := range intervals { + if err := writeLoadEstimateTx(ctx, tx, v.from, v.to, v.watts); err != nil { + return err + } + } + return clearLoadEstimatePendingTx(ctx, tx) +} + +func clearLoadEstimatePendingTx(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `DELETE FROM history_load_estimate_pending`) + return err +} + +func writeLoadEstimateTx(ctx context.Context, tx *sql.Tx, previous, until int64, w float64) error { + for from := previous; from < until; { start := bucketStart(from, ArchiveResolutionMS) - to := min(p.TsMs, start+ArchiveResolutionMS) + to := min(until, start+ArchiveResolutionMS) covered := to - from _, err := tx.ExecContext(ctx, `INSERT INTO history_load_estimates VALUES(?,?,?,?,?) ON CONFLICT(start_ms,resolution_ms) DO UPDATE SET covered_ms=covered_ms+excluded.covered_ms,load_wh=load_wh+excluded.load_wh`, start, ArchiveResolutionMS, covered, w*float64(covered)/3600000, "signed_source_balance_v1") diff --git a/go/internal/state/history_load_replay_test.go b/go/internal/state/history_load_replay_test.go new file mode 100644 index 000000000..3c267cf54 --- /dev/null +++ b/go/internal/state/history_load_replay_test.go @@ -0,0 +1,228 @@ +package state + +import ( + "context" + "fmt" + "math" + "path/filepath" + "testing" + "time" +) + +func replayLoadPoint(at int64, w float64) *HistoryPoint { + p := &HistoryPoint{TsMs: at, GridW: 1000, BatW: 1000 - w, LoadW: w, JSON: `{"ev_w":0,"v2x_w":0}`} + if w < 0 { + p.LoadW = 0 + p.Missing = MissingLoad | BalanceMismatch + p.JSON = `{"ev_w":0,"v2x_w":0,"history_reason":"invalid_balance"}` + } + return p +} + +func TestLoadEstimateWaitsForShortRecovery(t *testing.T) { + s := freshStore(t) + if err := s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + dashboardTick(t, s, "first", replayLoadPoint(base, 1000)) + dashboardTick(t, s, "negative", replayLoadPoint(base+4000, -3946)) + d, err := s.DailyEnergy(base, base+60000) + if err != nil || d.Missing&MissingLoad == 0 { + t.Fatal("unclosed episode supplied a total", d, err) + } + dashboardTick(t, s, "recovered", replayLoadPoint(base+8000, 5768)) + d, err = s.DailyEnergy(base, base+60000) + // Preserve signed balance: (-3946 + 5768) W * 4 s. Averaging the + // surrounding valid 1000/5768 W endpoints would invent 7.52 Wh. + want := float64(-3946+5768) * 4 / 3600 + if err != nil || d.Missing != EstimatedLoad || math.Abs(d.LoadWh-want) > 1e-9 { + t.Fatal(d, err, want) + } + // A retried tick must not count the recovered estimate twice. + dashboardTick(t, s, "recovered", replayLoadPoint(base+8000, 5768)) + d2, err := s.DailyEnergy(base, base+60000) + if err != nil || d2.LoadWh != d.LoadWh { + t.Fatal(d2, err) + } +} + +func TestLoadEstimateRejectsLongUncertainAndOfflineEpisodes(t *testing.T) { + for _, scenario := range []string{"long_episode", "long_tick", "missing_source", "offline_tick", "restart", "out_of_order"} { + t.Run(scenario, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { s.Close() }() + if err = s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + dashboardTick(t, s, "start", replayLoadPoint(base, 1000)) + dashboardTick(t, s, "bad", replayLoadPoint(base+5000, -1000)) + recovery := int64(10000) + switch scenario { + case "long_episode": + for sec := 10; sec <= 130; sec += 5 { + dashboardTick(t, s, fmt.Sprint(sec), replayLoadPoint(base+int64(sec)*1000, -1000)) + } + recovery = 135000 + case "long_tick": + recovery = 25000 + case "missing_source": + p := replayLoadPoint(base+7000, 1000) + p.Missing = MissingLoad | MissingEV + p.JSON = `{"ev_w":null,"v2x_w":0,"history_reason":"offline:ev"}` + dashboardTick(t, s, "offline", p) + case "offline_tick": + dashboardTick(t, s, "nil", nil) + case "restart": + if err = s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + case "out_of_order": + // An older valid tick cannot close the pending failure. + dashboardTick(t, s, "old", replayLoadPoint(base+2000, 2000)) + d, err := s.DailyEnergy(base, base+60000) + if err != nil || d.Missing&MissingLoad == 0 { + t.Fatal(d, err) + } + return + } + dashboardTick(t, s, "recovery", replayLoadPoint(base+recovery, 3000)) + d, err := s.DailyEnergy(base, base+60000) + if err != nil || d.Missing&MissingLoad == 0 { + t.Fatal("unsafe episode supplied a total", d, err) + } + }) + } +} + +func TestLoadEstimateSplitsAtLocalMidnightAndRetainsSignedEnergy(t *testing.T) { + s := freshStore(t) + if err := s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatal(err) + } + midnight := time.Date(2026, 10, 18, 0, 0, 0, 0, loc).UnixMilli() + for i, w := range []float64{1000, -1000, 3000, 1000} { + p := replayLoadPoint(midnight-15000+int64(i)*10000, w) + p.Missing |= MissingSoC // An unrelated SoC gap must not block power evidence. + dashboardTick(t, s, fmt.Sprint(i), p) + } + a, err := s.DailyEnergy(midnight-86400000, midnight) + if err != nil { + t.Fatal(err) + } + b, err := s.DailyEnergy(midnight, midnight+60000) + if err != nil { + t.Fatal(err) + } + // The first half has a positive signed total: -10 kWs + 15 kWs. + near(t, "before midnight", a.LoadWh, 5000.0/3600) + near(t, "after midnight", b.LoadWh, 25000.0/3600) + if a.Missing&EstimatedLoad == 0 || b.Missing&EstimatedLoad == 0 { + t.Fatal(a, b) + } + coverage, err := s.HistoryEnergyCoverage(context.Background(), midnight-60000, midnight+60000) + if err != nil || coverage != 30000 { + t.Fatal(coverage, err) + } +} + +func TestHomeEnergyRestartLeavesStoppedTimeUncovered(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err = s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + for i := 0; i < 2; i++ { + dashboardTick(t, s, fmt.Sprint(i), replayLoadPoint(base+int64(i)*5000, 1000)) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err = s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + for i := 2; i < 4; i++ { + dashboardTick(t, s, fmt.Sprint(i), replayLoadPoint(base+int64(i)*5000, 1000)) + } + coverage, err := s.HistoryEnergyCoverage(context.Background(), base, base+60000) + if err != nil || coverage != 10000 { + t.Fatal("restart interval counted as measured", coverage, err) + } + d, err := s.DailyEnergy(base, base+60000) + if err != nil { + t.Fatal(err) + } + near(t, "recorded home energy", d.LoadWh, 1000*10.0/3600) +} + +func TestMissingSoCDoesNotInflateEstimatedPowerCoverage(t *testing.T) { + s := freshStore(t) + if err := s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + for sec := 0; sec <= 600; sec += 5 { + w := 1000.0 + if sec%120 == 65 { + w = -1000 + } + if sec%120 == 70 { + w = 3000 + } + p := replayLoadPoint(base+int64(sec)*1000, w) + p.Missing |= MissingSoC + dashboardTick(t, s, fmt.Sprint(sec), p) + } + d, err := s.DailyEnergy(base, base+600000) + if err != nil || d.Missing != EstimatedLoad|MissingSoC { + t.Fatal(d, err) + } + near(t, "signed energy without SoC", d.LoadWh, 1000*600.0/3600) +} + +func TestLoadEstimateCanCloseOneMinuteImbalance(t *testing.T) { + s := freshStore(t) + if err := s.EnableHistoryAggregation(); err != nil { + t.Fatal(err) + } + base := time.Now().UTC().Truncate(24 * time.Hour).UnixMilli() + var signedWS float64 + for sec := 0; sec <= 375; sec += 5 { + w := 4000.0 + if sec >= 305 && sec <= 360 { + w = -4000 + } + p := replayLoadPoint(base+int64(sec)*1000, w) + dashboardTick(t, s, fmt.Sprint(sec), p) + if sec > 0 { + signedWS += w * 5 + } + } + d, err := s.DailyEnergy(base, base+420000) + if err != nil || d.Missing != EstimatedLoad { + t.Fatal(d, err) + } + near(t, "signed minute-lag energy", d.LoadWh, signedWS/3600) +} diff --git a/go/internal/state/history_plain.go b/go/internal/state/history_plain.go index b3bd3b57a..2e1656a27 100644 --- a/go/internal/state/history_plain.go +++ b/go/internal/state/history_plain.go @@ -37,7 +37,7 @@ var plainHistoryTables = []string{ "ts_drivers", "ts_metrics", "ts_latest", "energy_daily", "energy_ledger_meta", "energy_assets", "energy_ledger_entries", "energy_ledger_cursors", "ts_buckets", "ts_aggregate_hours", "ts_series_hour", "ts_archive_days", - "history_load_estimates", "history_dashboard", "history_site_energy", "history_site_cursor", "history_quality", "history_daily_quality", "history_dashboard_partial", "history_site_energy_partial", "history_partial_cursor", "energy_daily_partial", + "history_load_estimates", "history_load_estimate_pending", "history_dashboard", "history_site_energy", "history_site_cursor", "history_quality", "history_daily_quality", "history_dashboard_partial", "history_site_energy_partial", "history_partial_cursor", "energy_daily_partial", "history_migrations", "history_sqlite_progress", } diff --git a/go/internal/state/history_schema.go b/go/internal/state/history_schema.go index 521971b5c..19d5a048c 100644 --- a/go/internal/state/history_schema.go +++ b/go/internal/state/history_schema.go @@ -4,7 +4,7 @@ import "strings" var historySchema = append(append([]string{}, sqliteLegacyHistoryStmts...), `CREATE TABLE IF NOT EXISTS ts_latest (driver_id INTEGER NOT NULL,metric_id INTEGER NOT NULL,ts_ms INTEGER NOT NULL,value REAL NOT NULL,PRIMARY KEY(driver_id,metric_id)) WITHOUT ROWID`, - loadEstimateSchema, dashboardSchema, siteEnergySchema, siteEnergyCursorSchema, historyQualitySchema, historyDailyQualitySchema, + loadEstimateSchema, loadEstimatePendingSchema, dashboardSchema, siteEnergySchema, siteEnergyCursorSchema, historyQualitySchema, historyDailyQualitySchema, strings.Replace(dashboardSchema, "history_dashboard", "history_dashboard_partial", 1), strings.Replace(siteEnergySchema, "history_site_energy", "history_site_energy_partial", 1), strings.Replace(siteEnergyCursorSchema, "history_site_cursor", "history_partial_cursor", 1), `CREATE TABLE IF NOT EXISTS energy_daily_partial(day TEXT PRIMARY KEY,import_wh REAL NOT NULL,export_wh REAL NOT NULL,pv_wh REAL NOT NULL,bat_charged_wh REAL NOT NULL,bat_discharged_wh REAL NOT NULL,load_wh REAL NOT NULL,computed_at_ms INTEGER NOT NULL) STRICT`, `CREATE INDEX IF NOT EXISTS idx_partial_dashboard_last ON history_dashboard_partial(last_ms)`, diff --git a/go/internal/state/history_sqlite.go b/go/internal/state/history_sqlite.go index f0ed9cbf1..b0a8e3a6c 100644 --- a/go/internal/state/history_sqlite.go +++ b/go/internal/state/history_sqlite.go @@ -26,7 +26,7 @@ var historyTables = []string{ } var sqliteHistoryBaseTables = append(append([]string{}, historyTables...), "ts_series_hour", "ts_archive_days", "ts_buckets", "ts_bucket_days", "ts_legacy_bucket_days", "ts_aggregate_hours", "ts_latest", "history_dashboard", "history_site_energy", "history_site_cursor") var historyQualityTables = []string{"history_quality", "history_daily_quality", "history_dashboard_partial", "history_site_energy_partial", "history_partial_cursor", "energy_daily_partial"} -var sqliteHistoryTables = append(append(append([]string{}, sqliteHistoryBaseTables...), historyQualityTables...), "history_load_estimates") +var sqliteHistoryTables = append(append(append([]string{}, sqliteHistoryBaseTables...), historyQualityTables...), "history_load_estimates", "history_load_estimate_pending") func ensureHistorySchema(exec func(string) error) error { for _, stmt := range historySchema { @@ -146,6 +146,13 @@ func (s *Store) openHistory() error { return err } } + // A process stop has no continuous sampling proof, even when it was brief. + // Keep completed energy and quality, but never bridge the stopped process. + if _, err := s.history.Exec(`DELETE FROM history_load_estimate_pending; + UPDATE history_partial_cursor SET value=0 WHERE key IN ('last','residual_eligible'); + UPDATE history_site_cursor SET value=0 WHERE key='last'`); err != nil { + return err + } ok = true return nil } @@ -368,6 +375,8 @@ func historyOrder(table string) string { return " ORDER BY driver_id, metric_id" case "history_load_estimates", "history_dashboard", "history_site_energy", "history_dashboard_partial", "history_site_energy_partial": return " ORDER BY start_ms, resolution_ms" + case "history_load_estimate_pending": + return " ORDER BY from_ms" case "ts_buckets": return " ORDER BY driver_id, metric_id, start_ms, resolution_ms" case "ts_bucket_days", "ts_legacy_bucket_days": @@ -494,12 +503,14 @@ func (s *Store) exportHistoryToSQLite(ctx context.Context, path string) error { return errors.New("incomplete history quality tables; keep the source and restore a verified full backup") } // Estimates are additive and optional in published older backups. - var estimatesExist int - if err := src.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='history_load_estimates'`).Scan(&estimatesExist); err != nil { - return err - } - if estimatesExist != 0 { - tables = append(append([]string{}, tables...), "history_load_estimates") + for _, table := range []string{"history_load_estimates", "history_load_estimate_pending"} { + var exists int + if err := src.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&exists); err != nil { + return err + } + if exists != 0 { + tables = append(append([]string{}, tables...), table) + } } // This is an unpublished scratch file. Flush each small commit, then // pause, so the next goal/session sync does not inherit a dirty backlog. diff --git a/go/internal/telemetry/home_cloud_timing_test.go b/go/internal/telemetry/home_cloud_timing_test.go new file mode 100644 index 000000000..c7a0f7aa5 --- /dev/null +++ b/go/internal/telemetry/home_cloud_timing_test.go @@ -0,0 +1,78 @@ +package telemetry + +import ( + "math" + "testing" + "time" +) + +func homeReceiptSample(s *Store, name string, kind DerType, watts float64, received time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + s.recordHomeObservation(name, kind, watts, nil, received) + s.readings[key(name, kind)] = &DerReading{Driver: name, DerType: kind, RawW: watts, UpdatedAt: received} + s.health[name] = &DriverHealth{Name: name} +} + +func TestHomeEstimateCloudPollDoesNotAdvancePowerTime(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 120; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + power := 6000.0 + if sec >= 10 && sec <= 20 { + power += float64(sec-10) * 76.8 + } else if sec > 20 { + power = 6768 + } + homeReceiptSample(s, "site", DerMeter, 1000+power, at) + if sec == 10 { + homeSample(s, "cloud", DerEV, 6000, at, at, true) + } + if sec >= 20 { + homeSample(s, "cloud", DerEV, 6768, base.Add(20*time.Second), at, true) + } + } + got := s.HomeUseEstimate(base.Add(120*time.Second), "site", ForecastOptions{}) + if got.Watts == nil || got.ToMS != base.Add(20*time.Second).UnixMilli() || got.AgeMS != 105000 || got.TimingBasis != "mixed" { + t.Fatalf("fresh HTTP replies manufactured fresh power evidence: %+v", got) + } +} + +func TestHomeEstimateOneMinuteCloudDelayUsesPastWindow(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 150; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + ev := func(sec int) float64 { + if sec >= 30 && sec < 70 { + return 9714 + } + return 0 + } + homeReceiptSample(s, "site", DerMeter, 1000+ev(sec), at) + if sec >= 60 { + homeSample(s, "cloud", DerEV, ev(sec-60), at.Add(-time.Minute), at, true) + } + if sec < 70 { + continue + } + got := s.HomeUseEstimate(at, "site", ForecastOptions{}) + if got.Watts == nil || got.ToMS != at.Add(-time.Minute).UnixMilli() || got.AgeMS != 65000 || math.Abs(*got.Watts-1000) > 1e-8 || got.SourceAgeMS != 60000 { + t.Fatalf("second %d: cannot compare the reported past window: %+v", sec, got) + } + } +} + +func TestHomeEstimateRejectsChargerWithoutSourceTime(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 20; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 8000, at) + homeReceiptSample(s, "cloud", DerEV, 7000, at) + } + if got := s.HomeUseEstimate(base.Add(20*time.Second), "site", ForecastOptions{}); got.Watts != nil { + t.Fatalf("a charger receipt time was presented as measurement alignment: %+v", got) + } +} diff --git a/go/internal/telemetry/home_energy.go b/go/internal/telemetry/home_energy.go new file mode 100644 index 000000000..9bab53d3f --- /dev/null +++ b/go/internal/telemetry/home_energy.go @@ -0,0 +1,101 @@ +package telemetry + +import ( + "encoding/json" + "time" +) + +type homeEnergyPoint struct { + At time.Time + Wh float64 +} + +type homeEnergySeries struct { + Session string + Points []homeEnergyPoint +} + +// Easee already supplies these three fields together. A counter without its +// observation time cannot distinguish a fresh read from a cached slow register. +// In particular, do not receipt-stamp local counters or mix battery DC energy +// with the AC power used in the site balance. Those flows use sampled power. +func (s *Store) recordHomeEnergy(driver string, kind DerType, data json.RawMessage, now time.Time) { + if kind != DerEV { + return + } + k := key(driver, kind) + var d struct { + Wh *float64 `json:"session_wh"` + At string `json:"energy_observed_at"` + Session string `json:"session_id"` + } + err := json.Unmarshal(data, &d) + series := s.homeEnergy[k] + if series == nil { + if d.At == "" { + return + } + if s.homeEnergy == nil { + s.homeEnergy = map[string]*homeEnergySeries{} + } + series = &homeEnergySeries{} + s.homeEnergy[k] = series + } + at, timeErr := time.Parse(time.RFC3339Nano, d.At) + if err != nil || timeErr != nil || at.After(now) || d.Wh == nil || !finite(*d.Wh) || *d.Wh < 0 || d.Session == "" { + series.Points = nil + return + } + if !s.health[driver].TelemetryLive() || series.Session != d.Session { + series.Points = nil + } + series.Session = d.Session + p := homeEnergyPoint{At: at, Wh: *d.Wh} + points := series.Points + if len(points) > 0 { + last := points[len(points)-1] + if at.Before(last.At) || (at.Equal(last.At) && p.Wh != last.Wh) { + series.Points = nil + return + } + if at.Equal(last.At) { + return // A poll of the same observation adds neither energy nor time. + } + if p.Wh < last.Wh || at.Sub(last.At) > homeMaxWindow { + points = nil // Reset, missed session boundary, or unsupported long gap. + } + } + points = append(points, p) + cutoff := now.Add(-homeLookback - homeMaxWindow) + for len(points) > 1 && points[1].At.Before(cutoff) { + points = points[1:] + } + if len(points) > 1024 { + points = points[len(points)-1024:] + } + series.Points = points +} + +// Counter differences give energy over their own interval. At an interior +// boundary, linear cumulative-energy interpolation is an estimate, not evidence +// of constant instantaneous power. Never extend beyond the reported endpoints. +func homeCounterEnergy(points []homeEnergyPoint, from, to time.Time) (float64, bool) { + energyAt := func(at time.Time) (float64, bool) { + for i, p := range points { + if p.At.Equal(at) { + return p.Wh, true + } + if i+1 < len(points) { + next := points[i+1] + if p.At.Before(at) && next.At.After(at) && next.At.Sub(p.At) <= homeMaxWindow { + f := at.Sub(p.At).Seconds() / next.At.Sub(p.At).Seconds() + return p.Wh + (next.Wh-p.Wh)*f, true + } + } + } + return 0, false + } + a, okA := energyAt(from) + b, okB := energyAt(to) + return b - a, okA && okB && to.After(from) && finite(b-a) && b >= a +} diff --git a/go/internal/telemetry/home_energy_test.go b/go/internal/telemetry/home_energy_test.go new file mode 100644 index 000000000..c30bf556c --- /dev/null +++ b/go/internal/telemetry/home_energy_test.go @@ -0,0 +1,275 @@ +package telemetry + +import ( + "encoding/json" + "math" + "testing" + "time" +) + +func homeSessionSample(s *Store, wh float64, session string, measured, received time.Time) { + data, _ := json.Marshal(map[string]any{ + "session_wh": wh, "session_id": session, "energy_observed_at": measured.Format(time.RFC3339Nano), + // Simulate a change-only power report that has not moved for an hour. + "control_power_observed_at": received.Add(-time.Hour).Format(time.RFC3339Nano), + "control_power_confirmed": true, + }) + homeDataSample(s, "cloud", DerEV, 9714, data, received) +} + +func homeDataSample(s *Store, driver string, kind DerType, w float64, data json.RawMessage, at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + s.recordHomeObservation(driver, kind, w, data, at) + s.readings[key(driver, kind)] = &DerReading{Driver: driver, DerType: kind, RawW: w, Data: data, UpdatedAt: at} + s.health[driver] = &DriverHealth{Name: driver} +} + +func TestHomeEstimatePrefersSessionEnergyWithMinuteDeliveryDelay(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 180; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + // A triangular charge ramp: exactly 50 Wh in each full minute. + phase := sec % 60 + ev := float64(phase) * 200 + if phase > 30 { + ev = float64(60-phase) * 200 + } + homeReceiptSample(s, "site", DerMeter, 1000+ev, at) + if sec >= 60 && sec%60 == 0 { + homeSessionSample(s, float64(sec/60-1)*50, "session-a", at.Add(-time.Minute), at) + } + if sec < 120 { + continue + } + got := s.HomeUseEstimate(at, "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 || got.EnergyWh == nil || math.Abs(*got.EnergyWh-1000.0/60) > 1e-8 || got.EnergyBasis != "mixed" { + t.Fatalf("second %d: energy balance failed: %+v", sec, got) + } + if got.ToMS-got.FromMS != 60000 || got.EndAgeMS < 60000 || got.MinW != nil || got.MaxW != nil { + t.Fatalf("counter energy presented as instantaneous power/range: %+v", got) + } + } + // Display energy must not repair the bad raw power or change control evidence. + raw := s.ForecastMeasurement(base.Add(180*time.Second), "site", ForecastOptions{}) + if raw.Valid || raw.Reason != "invalid_balance" || raw.HouseholdW != -8714 { + t.Fatal(raw) + } +} + +func TestHomeEstimateAdaptsToCounterReportsAndDoesNotRefreshOnPoll(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 250; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + stamp := 0 + switch { + case sec >= 180: + stamp = 180 + case sec >= 60: + stamp = 60 + } + homeSessionSample(s, float64(stamp)*6000/3600, "session-a", base.Add(time.Duration(stamp)*time.Second), at) + if sec != 60 && sec != 180 && sec != 250 { + continue + } + got := s.HomeUseEstimate(at, "site", ForecastOptions{}) + period := int64(60000) + if sec >= 180 { + period = 120000 + } + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 || got.ToMS-got.FromMS != period || got.EndAgeMS != int64(sec-stamp)*1000 { + t.Fatal(sec, got) + } + } +} + +func TestHomeSessionCounterBreaksAtInvalidEvidence(t *testing.T) { + base := time.Unix(100000, 0) + for _, scenario := range []string{"session", "reset", "missing_session", "missing_time", "missing_counter", "backward", "conflict", "future", "long_gap", "offline"} { + t.Run(scenario, func(t *testing.T) { + s := NewStore() + for sec := 0; sec <= 60; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + if sec%60 == 0 { + homeSessionSample(s, float64(sec)*6000/3600, "session-a", at, at) + } + } + now := base.Add(61 * time.Second) + d := map[string]any{"session_wh": 101, "session_id": "session-a", "energy_observed_at": now.Format(time.RFC3339Nano)} + switch scenario { + case "session": + d["session_id"] = "session-b" + case "reset": + d["session_wh"] = 0 + case "missing_session": + delete(d, "session_id") + case "missing_time": + delete(d, "energy_observed_at") + case "missing_counter": + delete(d, "session_wh") + case "backward": + d["energy_observed_at"] = base.Add(59 * time.Second).Format(time.RFC3339Nano) + case "conflict": + d["energy_observed_at"] = base.Add(60 * time.Second).Format(time.RFC3339Nano) + case "future": + d["energy_observed_at"] = now.Add(time.Second).Format(time.RFC3339Nano) + case "long_gap": + now = base.Add(361 * time.Second) + d["energy_observed_at"] = now.Format(time.RFC3339Nano) + case "offline": + s.health["cloud"].SetOffline() + } + data, _ := json.Marshal(d) + homeDataSample(s, "cloud", DerEV, 6000, data, now) + homeReceiptSample(s, "site", DerMeter, 7000, now) + if got := s.HomeUseEstimate(now, "site", ForecastOptions{}); got.Watts != nil { + t.Fatalf("crossed %s: %+v", scenario, got) + } + }) + } +} + +func TestHomeEnergyCannotFillLocalPowerGap(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 120; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + if sec < 30 || sec > 90 { + homeReceiptSample(s, "site", DerMeter, 7000, at) + } + if sec%60 == 0 { + homeSessionSample(s, float64(sec)*6000/3600, "session-a", at, at) + } + } + if got := s.HomeUseEstimate(base.Add(120*time.Second), "site", ForecastOptions{}); got.Watts != nil { + t.Fatal("counter filled missing meter energy", got) + } +} + +func TestHomePowerPeriodFollowsDistinctSourceReports(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 60; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + if sec%20 == 0 { + homeSample(s, "cloud", DerEV, 6000, at, at, false) + } + } + got := s.HomeUseEstimate(base.Add(time.Minute), "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 || got.ToMS-got.FromMS != 20000 || got.EnergyBasis != "power_integral" { + t.Fatal(got) + } +} + +func TestHomeEnergyKeepsWholeReferenceCounterInterval(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 90; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + if sec == 0 || sec == 60 || sec == 90 { + homeSessionSample(s, float64(sec)*6000/3600, "session-a", at, at) + } + } + got := s.HomeUseEstimate(base.Add(90*time.Second), "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 || got.FromMS != base.UnixMilli() || got.ToMS-got.FromMS != 90000 { + t.Fatal("split the reference counter interval after a cadence change", got) + } +} + +func TestHomeEnergyUsesCountersWithoutPowerTimeAndDoesNotMixDCEnergy(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 60; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 5000, at) + // Pixii: raw DC and cabinet counters do not describe the AC site flow. + homeDataSample(s, "battery", DerBattery, -2500, json.RawMessage(`{"control_power_w":-2000,"charge_wh":0,"discharge_wh":9000}`), at) + if sec%60 == 0 { + data, _ := json.Marshal(map[string]any{"session_wh": float64(sec) * 6000 / 3600, "session_id": "session-a", "energy_observed_at": at.Format(time.RFC3339Nano)}) + homeDataSample(s, "cloud", DerEV, 12345, data, at) + } + } + got := s.HomeUseEstimate(base.Add(time.Minute), "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 { + t.Fatal("counter path required power time or used DC energy", got) + } +} + +func TestHomeEnergyBalancesTwoCounterSchedules(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 135; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 11000, at) + for name, interval := range map[string]int{"charger-a": 60, "charger-b": 45} { + if sec%interval != 0 { + continue + } + w := 6000.0 + if name == "charger-b" { + w = 4000 + } + data, _ := json.Marshal(map[string]any{"session_wh": float64(sec) * w / 3600, "session_id": name, "energy_observed_at": at.Format(time.RFC3339Nano)}) + homeDataSample(s, name, DerEV, w, data, at) + } + } + for i := 0; i < 20; i++ { + got := s.HomeUseEstimate(base.Add(135*time.Second), "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 || got.ToMS != base.Add(120*time.Second).UnixMilli() || got.FromMS != base.Add(time.Minute).UnixMilli() { + t.Fatal("counter schedules did not share a stable period", got) + } + } +} + +func TestHomeEstimateDropsSessionEnergyWhenDriverIsReplaced(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 60; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + if sec%60 == 0 { + homeSessionSample(s, float64(sec)*6000/3600, "session-a", at, at) + } + } + s.Remove("cloud") + // The replacement has source power but does not expose session energy. + for sec := 61; sec <= 81; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + homeSample(s, "cloud", DerEV, 6000, at, at, false) + } + got := s.HomeUseEstimate(base.Add(81*time.Second), "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-1000) > 1e-8 || got.EnergyBasis != "power_integral" { + t.Fatal("replacement inherited the old counter requirement", got) + } +} + +func TestHomeEnergyRepeatedPollsCannotKeepAnOldIntervalFresh(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 241; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeReceiptSample(s, "site", DerMeter, 7000, at) + stamp := 0 + if sec >= 60 { + stamp = 60 + } + homeSessionSample(s, float64(stamp)*6000/3600, "session-a", base.Add(time.Duration(stamp)*time.Second), at) + if sec < 240 { + continue + } + got := s.HomeUseEstimate(at, "site", ForecastOptions{}) + if sec == 240 && (got.Watts == nil || got.EndAgeMS != 180000) { + t.Fatal("lost supported past interval", got) + } + if sec == 241 && got.Watts != nil { + t.Fatal("fresh HTTP reply extended energy age", got) + } + } +} diff --git a/go/internal/telemetry/home_estimate.go b/go/internal/telemetry/home_estimate.go new file mode 100644 index 000000000..333fef8ce --- /dev/null +++ b/go/internal/telemetry/home_estimate.go @@ -0,0 +1,279 @@ +package telemetry + +import ( + "encoding/json" + "math" + "strings" + "time" +) + +const homeWindow = 10 * time.Second +const homeMaxGap = 30 * time.Second +const homeLookback = 3 * time.Minute +const homeMaxWindow = 5 * time.Minute + +// HomeEstimate describes a past interval, never a new instantaneous measurement. +// MinW/MaxW describe variation in the reported window, not error bounds. +// Reported source clocks and local receipt clocks are not verified as aligned. +type HomeEstimate struct { + Watts *float64 `json:"watts"` + EnergyWh *float64 `json:"energy_wh"` + EnergyBasis string `json:"energy_basis"` + MinW *float64 `json:"window_min_w"` + MaxW *float64 `json:"window_max_w"` + FromMS int64 `json:"from_ms"` + ToMS int64 `json:"to_ms"` + AgeMS int64 `json:"age_ms"` + EndAgeMS int64 `json:"end_age_ms"` + Method string `json:"method"` + TimingBasis string `json:"timing_basis"` + SourceAgeMS int64 `json:"source_age_ms"` + Reason string `json:"reason,omitempty"` +} + +// Keep a separate bounded buffer: neither control evidence nor raw readings may +// depend on how often a client asks for an estimate. Caller holds s.mu. +func (s *Store) recordHomeObservation(driver string, kind DerType, w float64, data json.RawMessage, now time.Time) { + if kind == DerVehicle { + return + } + s.recordHomeEnergy(driver, kind, data, now) + k := key(driver, kind) + if s.homeObservations == nil { + s.homeObservations = map[string][]ControlObservation{} + s.homeSourceTime = map[string]bool{} + } + p, ok := ControlPowerObservation(w, data, now) + var timing struct { + ControlPowerObservedAt string `json:"control_power_observed_at"` + PowerObservedAt string `json:"power_observed_at"` + ForecastPower *ForecastPowerSample `json:"forecast_power"` + } + _ = json.Unmarshal(data, &timing) + sourceTime := timing.ControlPowerObservedAt != "" || timing.PowerObservedAt != "" || timing.ForecastPower != nil + if !ok || ((kind == DerEV || kind == DerV2X) && !sourceTime) { + delete(s.homeObservations, k) + delete(s.homeSourceTime, k) + return + } + // A successful cloud request can return an old observation. The driver's + // control confirmation is not a new power timestamp and cannot extend it. + p.ConfirmedAt = time.Time{} + points := s.homeObservations[k] + if !s.health[driver].TelemetryLive() || s.homeSourceTime[k] != sourceTime { + points = nil + } + s.homeSourceTime[k] = sourceTime + if len(points) > 0 { + last := &points[len(points)-1] + if p.At.Before(last.At) || p.At.Sub(last.At) > homeMaxGap { + points = nil + } else if p.At.Equal(last.At) { + if p.PowerW != last.PowerW { + delete(s.homeObservations, k) + return + } + // A repeated change-only timestamp adds no time, even when a + // fresh HTTP reply carries a control confirmation. + s.homeObservations[k] = points + return + } + } + points = append(points, p) + cutoff := now.Add(-homeLookback - homeMaxWindow) + for len(points) > 1 && points[1].At.Before(cutoff) { + points = points[1:] + } + if len(points) > 1024 { + points = points[len(points)-1024:] + } + s.homeObservations[k] = points +} + +// HomeUseEstimate divides signed energy by a common reported interval. Prefer +// timestamped session counters; otherwise integrate bracketed power. The period +// is at least ten seconds and at least the largest of each source's last three +// distinct reporting intervals. This is observed coverage, not a cadence promise. +// Clocks and sampling delay remain unverified. This path serves display only. +func (s *Store) HomeUseEstimate(now time.Time, siteMeter string, opts ForecastOptions) HomeEstimate { + out := HomeEstimate{Method: "reported_energy_balance_v3"} + if s == nil { + out.Reason = "no_telemetry" + return out + } + s.mu.RLock() + defer s.mu.RUnlock() + r := s.forecastMeasurementLocked(now, siteMeter, opts) + if !r.Valid && r.Reason != "invalid_balance" && r.Reason != "time_skew" && !strings.HasPrefix(r.Reason, "stale:") && !strings.HasPrefix(r.Reason, "unknown_power:") { + out.Reason = r.Reason + return out + } + to := now + window := homeWindow + counterCount := 0 + var reference *homeEnergySeries + var referenceKey string + var referenceGap time.Duration + var sourceClocks, receiptClocks bool + for k, source := range r.Sources { + points := s.homeObservations[k] + counter := s.homeEnergy[k] + if counter == nil && (source.Flow.DerType == DerEV || source.Flow.DerType == DerV2X) && !s.homeSourceTime[k] { + out.Reason = "charger_source_time_unavailable" + return out + } + if !s.health[source.Flow.Driver].TelemetryLive() { + out.Reason = "missing_source_window" + return out + } + var times []time.Time + usesSourceTime := s.homeSourceTime[k] + if counter != nil { + for _, p := range counter.Points { + times = append(times, p.At) + } + usesSourceTime = true + counterCount++ + } else { + for _, p := range points { + times = append(times, p.At) + } + } + if len(times) < 2 { + out.Reason = "incomplete_source_window" + return out + } + last := times[len(times)-1] + if usesSourceTime && now.Sub(last).Milliseconds() > out.SourceAgeMS { + out.SourceAgeMS = now.Sub(last).Milliseconds() + } + if last.After(now) || now.Sub(last) > homeLookback { + out.Reason = "stale_source_window" + return out + } + if last.Before(to) { + to = last + } + var reportGap time.Duration + for i := len(times) - 1; i > 0 && i >= len(times)-3; i-- { + if gap := times[i].Sub(times[i-1]); gap > reportGap { + reportGap = gap + } + } + if reportGap > window { + window = reportGap + } + if counter != nil && (reference == nil || reportGap > referenceGap || (reportGap == referenceGap && k < referenceKey)) { + reference, referenceKey, referenceGap = counter, k, reportGap + } + sourceClocks = sourceClocks || usesSourceTime + receiptClocks = receiptClocks || !usesSourceTime + } + out.TimingBasis = "source" + if receiptClocks { + out.TimingBasis = "receipt" + if sourceClocks { + out.TimingBasis = "mixed" + } + } + from := to.Add(-window) + if reference != nil { + // Keep the slowest counter's complete interval wherever possible. + // Other counter clocks may require cumulative-energy interpolation. + endFound, startFound := false, false + for i := len(reference.Points) - 1; i >= 0; i-- { + at := reference.Points[i].At + if !endFound && !at.After(to) { + to, endFound = at, true + from = to.Add(-window) + } + if endFound && !at.After(from) { + from, startFound = at, true + break + } + } + if !startFound || to.Sub(from) > homeMaxWindow || now.Sub(to) > homeLookback { + out.Reason = "incomplete_source_window" + return out + } + window = to.Sub(from) + } + out.FromMS, out.ToMS = from.UnixMilli(), to.UnixMilli() + out.AgeMS = now.Sub(from.Add(window / 2)).Milliseconds() + out.EndAgeMS = now.Sub(to).Milliseconds() + out.EnergyBasis = "power_integral" + if counterCount > 0 { + out.EnergyBasis = "counters" + if counterCount < len(r.Sources) { + out.EnergyBasis = "mixed" + } + } + var energy, low, high float64 + for k, source := range r.Sources { + var wh, minW, maxW float64 + var ok bool + if counter := s.homeEnergy[k]; counter != nil { + wh, ok = homeCounterEnergy(counter.Points, from, to) + } else { + var avg float64 + avg, minW, maxW, ok = homeWindowPower(s.homeObservations[k], from, to) + wh = avg * window.Hours() + } + if !ok { + out.Reason = "incomplete_source_window" + return out + } + if source.Flow.DerType == DerMeter { + energy += wh + low += minW + high += maxW + } else { + energy -= wh + low -= maxW + high -= minW + } + } + mean := energy / window.Hours() + if !finite(mean) || !finite(low) || !finite(high) || mean < 0 { + out.Reason = "invalid_balance" + return out + } + out.Watts, out.EnergyWh = &mean, &energy + if counterCount == 0 { + out.MinW, out.MaxW = &low, &high + } + return out +} + +func homeWindowPower(points []ControlObservation, from, to time.Time) (mean, low, high float64, ok bool) { + covered := time.Duration(0) + low, high = math.Inf(1), math.Inf(-1) + add := func(a, b time.Time, wa, wb float64) { + left, right := a, b + if left.Before(from) { + left = from + } + if right.After(to) { + right = to + } + if !right.After(left) { + return + } + span := b.Sub(a).Seconds() + v0 := wa + (wb-wa)*left.Sub(a).Seconds()/span + v1 := wa + (wb-wa)*right.Sub(a).Seconds()/span + mean += (v0 + v1) / 2 * right.Sub(left).Seconds() + low = math.Min(low, math.Min(v0, v1)) + high = math.Max(high, math.Max(v0, v1)) + covered += right.Sub(left) + } + for i, p := range points { + if i+1 < len(points) { + next := points[i+1] + if next.At.After(p.At) && next.At.Sub(p.At) <= homeMaxGap { + add(p.At, next.At, p.PowerW, next.PowerW) + } + } + } + return mean / to.Sub(from).Seconds(), low, high, covered == to.Sub(from) +} diff --git a/go/internal/telemetry/home_estimate_test.go b/go/internal/telemetry/home_estimate_test.go new file mode 100644 index 000000000..423713e02 --- /dev/null +++ b/go/internal/telemetry/home_estimate_test.go @@ -0,0 +1,201 @@ +package telemetry + +import ( + "encoding/json" + "fmt" + "math" + "testing" + "time" +) + +func homeSample(s *Store, name string, kind DerType, w float64, measured, received time.Time, confirmed bool) { + data, _ := json.Marshal(map[string]any{"power_observed_at": measured.Format(time.RFC3339Nano), "control_power_confirmed": confirmed}) + s.mu.Lock() + defer s.mu.Unlock() + s.recordHomeObservation(name, kind, w, data, received) + s.readings[key(name, kind)] = &DerReading{Driver: name, DerType: kind, RawW: w, UpdatedAt: received, Data: data} + s.health[name] = &DriverHealth{Name: name} +} + +// This is a synthetic, timestamped replay of the reported failure class, not +// Göran's missing raw 19:32 trace. A four-second delivery delay is a test input. +func TestHomeEstimateAlignedReplayVersusIndependentKalman(t *testing.T) { + s := NewStore() + base := time.Date(2026, 10, 17, 17, 32, 0, 0, time.UTC) + gridFilter, evFilter := NewKalman(100, 50), NewKalman(100, 50) + var alignedError, filteredError float64 + var count int + for second := 0; second <= 100; second++ { + at := base.Add(time.Duration(second) * time.Second) + ev := func(sec int) float64 { + if sec >= 10 && sec < 40 || sec >= 60 && sec < 75 { + return 9714 + } + return 0 + } + homeSample(s, "site", DerMeter, 1000+ev(second), at, at, false) + grid := gridFilter.Update(1000 + ev(second)) + if second >= 4 { + measured := at.Add(-4 * time.Second) + homeSample(s, "ev", DerEV, ev(second-4), measured, at, false) + evFilter.Update(ev(second - 4)) + } + if second < 14 { + continue + } + out := s.HomeUseEstimate(at, "site", ForecastOptions{}) + if out.Watts == nil { + t.Fatalf("second %d: %+v", second, out) + } + alignedError += math.Abs(*out.Watts - 1000) + filteredError += math.Abs(grid - evFilter.Estimate - 1000) + if out.ToMS != at.Add(-4*time.Second).UnixMilli() || out.AgeMS != 9000 { + t.Fatal(out) + } + count++ + } + if alignedError > 1e-8 || filteredError/float64(count) < 500 { + t.Fatal(alignedError, filteredError, count) + } + t.Logf("synthetic replay MAE: aligned %.3f W, independent scalar Kalman %.1f W", alignedError/float64(count), filteredError/float64(count)) +} + +func TestHomeWindowTimeWeightingAndSignedEnergy(t *testing.T) { + base := time.Unix(1000, 0) + for _, offsets := range [][]int{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, {0, 2, 9, 10}} { + var p []ControlObservation + for _, sec := range offsets { + p = append(p, ControlObservation{PowerW: float64(sec) * 100, At: base.Add(time.Duration(sec) * time.Second)}) + } + mean, low, high, ok := homeWindowPower(p, base, base.Add(10*time.Second)) + if !ok || math.Abs(mean-500) > 1e-9 || low != 0 || high != 1000 { + t.Fatal(mean, low, high, ok) + } + } + // Integrating then subtracting must equal subtracting the signed energies. + p := []ControlObservation{{PowerW: -1000, At: base}, {PowerW: 3000, At: base.Add(10 * time.Second)}} + mean, _, _, ok := homeWindowPower(p, base, base.Add(10*time.Second)) + if !ok || mean != 1000 { + t.Fatal(mean, ok) + } +} + +func TestHomeEstimateChangeOnlyConfirmationAndCachedRepeats(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + original := base.Add(-time.Hour) + for sec := 0; sec <= 20; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeSample(s, "site", DerMeter, 1500, at, at, false) + homeSample(s, "ev", DerEV, 500, original, at, true) + } + got := s.HomeUseEstimate(base.Add(20*time.Second), "site", ForecastOptions{}) + if got.Watts != nil || len(s.homeObservations["ev:ev"]) != 1 { + t.Fatal(got) + } + // Asking repeatedly must not advance the interval or modify its source. + for i := 0; i < 100; i++ { + again := s.HomeUseEstimate(base.Add(25*time.Second), "site", ForecastOptions{}) + if again.Watts != nil || again.ToMS != got.ToMS { + t.Fatal(again) + } + } + if got := s.HomeUseEstimate(base.Add(51*time.Second), "site", ForecastOptions{}); got.Watts != nil { + t.Fatal(got) + } + // An unconfirmed duplicate cannot extend a plateau. + u := NewStore() + for sec := 0; sec <= 20; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeSample(u, "site", DerMeter, 1000, base, at, false) + } + if got := u.HomeUseEstimate(base.Add(20*time.Second), "site", ForecastOptions{}); got.Watts != nil { + t.Fatal(got) + } +} + +func TestHomeEstimateRejectsGapsAndConflicts(t *testing.T) { + base := time.Unix(100000, 0) + for _, scenario := range []string{"offline", "long_gap", "backward", "conflict", "future", "unknown", "restart", "duplicate_topology"} { + t.Run(scenario, func(t *testing.T) { + s := NewStore() + for sec := 0; sec <= 10; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + homeSample(s, "site", DerMeter, 1000, at, at, false) + } + now := base.Add(10 * time.Second) + opts := ForecastOptions{} + switch scenario { + case "offline": + s.health["site"].SetOffline() + case "long_gap": + now = base.Add(50 * time.Second) + homeSample(s, "site", DerMeter, 1000, now, now, false) + case "backward": + homeSample(s, "site", DerMeter, 1000, base.Add(5*time.Second), now, false) + case "conflict": + homeSample(s, "site", DerMeter, 2000, now, now, false) + case "future": + homeSample(s, "site", DerMeter, 1000, now.Add(time.Second), now, false) + case "unknown": + s.recordHomeObservation("site", DerMeter, 1000, json.RawMessage(`{"control_power_available":false}`), now) + case "restart": + s = NewStore() + homeSample(s, "site", DerMeter, 1000, now, now, false) + case "duplicate_topology": + opts.ExpectedFlows = []ForecastFlow{{Driver: "site", DerType: DerMeter, FlowID: "x"}, {Driver: "site", DerType: DerMeter, FlowID: "x"}} + } + if got := s.HomeUseEstimate(now, "site", opts); got.Watts != nil { + t.Fatal(got) + } + }) + } +} + +func TestHomeEstimateDoesNotRepairRawNegativeBalance(t *testing.T) { + s := NewStore() + base := time.Unix(100000, 0) + for sec := 0; sec <= 20; sec++ { + at := base.Add(time.Duration(sec) * time.Second) + ev := 9714.0 + grid := 10600.0 + bat := 0.0 + if sec >= 20 { + grid = -267 + bat = -6035 + } + homeSample(s, "site", DerMeter, grid, at, at, false) + homeSample(s, "bat", DerBattery, bat, at, at, false) + // The slow source has not reported the stop yet. + if sec <= 16 { + homeSample(s, "ev", DerEV, ev, at, at, false) + } + } + now := base.Add(20 * time.Second) + for i := 0; i < 5; i++ { + raw := s.ForecastMeasurement(now, "site", ForecastOptions{}) + if raw.Valid || raw.Reason != "invalid_balance" || raw.HouseholdW != -3946 { + t.Fatal(raw) + } + got := s.HomeUseEstimate(now, "site", ForecastOptions{}) + if got.Watts == nil || math.Abs(*got.Watts-886) > 1e-8 || got.AgeMS != 9000 { + t.Fatal(fmt.Sprintf("%+v", got)) + } + } +} + +func TestHomeWindowCannotExtendThroughControlConfirmations(t *testing.T) { + base := time.Unix(100000, 0) + points := []ControlObservation{ + {At: base, ConfirmedAt: base.Add(20 * time.Second), PowerW: 1000}, + {At: base.Add(19 * time.Second), ConfirmedAt: base.Add(40 * time.Second), PowerW: 2000}, + } + // Control confirmations are HTTP/status evidence, not source brackets. + if _, _, _, ok := homeWindowPower(points, base.Add(15*time.Second), base.Add(25*time.Second)); ok { + t.Fatal("overlap treated as certain") + } + mean, _, _, ok := homeWindowPower(points, base.Add(30*time.Second), base.Add(40*time.Second)) + if ok { + t.Fatal("confirmation extended source evidence", mean, ok) + } +} diff --git a/go/internal/telemetry/store.go b/go/internal/telemetry/store.go index 243160c01..ceaeb4c84 100644 --- a/go/internal/telemetry/store.go +++ b/go/internal/telemetry/store.go @@ -296,6 +296,9 @@ type Store struct { health map[string]*DriverHealth commands map[string]CommandEvidence controlObservations map[string][]ControlObservation + homeObservations map[string][]ControlObservation // display estimates only + homeSourceTime map[string]bool // reported source time, otherwise receipt time + homeEnergy map[string]*homeEnergySeries // timestamped session counters, display only processNoise float64 measurementNoise float64 @@ -479,6 +482,7 @@ func (s *Store) Update(driver string, t DerType, rawW float64, soc *float64, dat s.observeCommand(driver, t, rawW, data, now) s.recordControlObservation(driver, t, rawW, data, now) + s.recordHomeObservation(driver, t, rawW, data, now) s.freezeStepWindows(now) // Auto-buffer the standard fields (raw, not smoothed — we store ground @@ -785,6 +789,9 @@ func (s *Store) Remove(driver string) { delete(s.health, driver) for _, kind := range allDerTypes { delete(s.controlObservations, key(driver, kind)) + delete(s.homeObservations, key(driver, kind)) + delete(s.homeSourceTime, key(driver, kind)) + delete(s.homeEnergy, key(driver, kind)) } for k, c := range s.commands { c.Baseline, c.StepAfter = nil, nil diff --git a/web/app.js b/web/app.js index 0ed9c6673..2122eca90 100644 --- a/web/app.js +++ b/web/app.js @@ -475,10 +475,11 @@ out.ev_charging_w = sums.ev_w; } if (out.grid_w != null && data.load_w != null) { - out.load_w = Math.max(0, (out.grid_w || 0) - (out.bat_w || 0) - (out.pv_w || 0) - (out.ev_w || 0)); + out.load_w = smoothDisplayNumber("load_w", data.load_w, now); } else { // No live meter: do not keep a previous load or treat JSON null as 0 W. out.load_w = null; + delete statusDisplayState.load_w; } return out; } @@ -563,6 +564,15 @@ } var opts = energyFlowOpts(data); opts.live = lastFlowLive; lastFlowReadings = window.ftwFlowReadingsFromStatus(data, opts); + var homeNote = document.getElementById("home-use-estimate"); + if (!homeNote) { + homeNote = document.createElement("p"); + homeNote.id = "home-use-estimate"; + homeNote.className = "muted"; + flowEl.insertAdjacentElement("afterend", homeNote); + } + homeNote.textContent = lastFlowReadings.homeEstimateNote || ""; + homeNote.hidden = !homeNote.textContent; (lastFlowReadings.planets || []).forEach(function (p) { if (p.role !== "ev" || p.placeholder || !p.name) return; var lpEv = loadpointsByDriver && loadpointsByDriver[p.name]; diff --git a/web/components/energy-flow-readings.js b/web/components/energy-flow-readings.js index 443e1012d..d7f94dddc 100644 --- a/web/components/energy-flow-readings.js +++ b/web/components/energy-flow-readings.js @@ -39,6 +39,36 @@ export function fmtKwhShort(kwh) { return kwh.toFixed(2); } +function usableHomeEstimate(estimate) { + const from = num(estimate?.from_ms), to = num(estimate?.to_ms); + const age = num(estimate?.end_age_ms); + return num(estimate?.watts) !== null && estimate.watts >= 0 && + from !== null && to !== null && from > 0 && to > from && to - from <= 300000 && + to < 8640000000000000 && age !== null && age >= 0 && age <= 180000; +} + +export function homeEstimateNote(estimate, live = true) { + if (!estimate) return ""; + if (!live) return "Home use estimate paused · no live data"; + if (num(estimate.watts) === null) { + if (estimate.reason === "charger_source_time_unavailable") return "Home use estimate unavailable · charger source time unknown"; + const sourceAge = num(estimate.source_age_ms); + return "Home use estimate unavailable · waiting for source observations" + + (sourceAge > 0 ? ` · last source report ${Math.ceil(sourceAge / 1000)} s old` : ""); + } + if (!usableHomeEstimate(estimate)) { + return "Home use estimate unavailable · readings are too old"; + } + const from = new Date(estimate.from_ms), to = new Date(estimate.to_ms); + const options = {hour:"2-digit", minute:"2-digit", second:"2-digit", hourCycle:"h23", timeZoneName:"short"}; + if (from.toDateString() !== to.toDateString()) Object.assign(options, {year:"numeric", month:"short", day:"numeric"}); + const period = new Intl.DateTimeFormat("en-GB", options).formatRange(from, to); + const age = estimate.end_age_ms === 0 ? "ended now" : `ended ${Math.ceil(estimate.end_age_ms / 1000)} s ago`; + const basis = {counters:"energy counters", power_integral:"sampled power", mixed:"charger energy + sampled power"}[estimate.energy_basis] || "energy estimate"; + const timing = estimate.timing_basis === "source" ? "Device clocks may differ." : "Includes arrival times; device delay is unknown."; + return `Estimated home use ${(estimate.watts / 1000).toFixed(1)} kW · ${Math.round((estimate.to_ms - estimate.from_ms) / 1000)} s mean · ${period} · ${age} · ${basis}. ${timing}`; +} + function planetColor(role, watts, thresholdW) { if (watts === undefined || watts === null) { if (role === "grid" || role === "pv") return "var(--fg-muted)"; @@ -207,17 +237,20 @@ export function flowReadingsFromStatus(status, opts) { if (!live[role]) planets.push(...offline[role]); } - const loadW = num(status && status.load_w); + const hasEstimate = status?.load_estimate != null; + const loadW = hasEstimate ? (usableHomeEstimate(status.load_estimate) ? status.load_estimate.watts : null) : num(status && status.load_w); // Today's share is kWh already in the box, not the live meter. A // quiet meter blanks % SELF-POWERED NOW (unknown instantaneous load) // but must not hide a day that already happened. let selfPoweredPctToday = null; - if (loadWh > 1 && importWh !== null) { + if (loadWh > 1 && importWh !== null && today.history_quality !== "estimated") { selfPoweredPctToday = Math.max(0, Math.min(100, (1 - importWh / loadWh) * 100)); } return { load: loadW === null ? null : loadW / 1000, + loadEstimated: hasEstimate, + homeEstimateNote: homeEstimateNote(status?.load_estimate, opts?.live !== false), planets: withControlMarks(planets, status?.control_feedback, opts?.live !== false), selfPoweredPctToday, }; diff --git a/web/components/ftw-energy-flow.js b/web/components/ftw-energy-flow.js index e529839cc..da490d98a 100644 --- a/web/components/ftw-energy-flow.js +++ b/web/components/ftw-energy-flow.js @@ -574,6 +574,7 @@ class FtwEnergyFlow extends FtwElement { // a previous number. `!= null` would keep drawing the last 0 W as if // the house were idle. if ("load" in r) this._readings.load = r.load; + if ("loadEstimated" in r) this._readings.loadEstimated = r.loadEstimated; if (Array.isArray(r.planets)) this._readings.planets = r.planets; // Optional today's-totals payload pushed through to the central // hub render. selfPoweredPctToday is the share of consumption @@ -876,7 +877,7 @@ class FtwEnergyFlow extends FtwElement { } render() { - const { load } = this._readings; + const { load, loadEstimated } = this._readings; const loadKnown = load != null && Number.isFinite(Number(load)); // Self-powered % for the visible site demand — house load plus any @@ -887,7 +888,7 @@ class FtwEnergyFlow extends FtwElement { // Unknown load (stale meter) is not 0 % — that would claim the house // is fully self-powered while we cannot see it. let selfPoweredPct = null; - if (loadKnown) { + if (loadKnown && !loadEstimated) { let gridImport = 0; for (const p of (this._readings.planets || [])) { if (p.role === "grid" && !p.placeholder && p.toHub) gridImport += Math.max(0, p.kw || 0); @@ -1189,14 +1190,11 @@ class FtwEnergyFlow extends FtwElement { ${aggLayerSvg} ${indLayerSvg} - + + tabindex="0" role="button" aria-label="${loadKnown ? (loadEstimated ? "House load, estimated mean" : "House load, live") : "House load, no data"}"> @@ -1212,7 +1210,7 @@ class FtwEnergyFlow extends FtwElement { - ${loadKnown ? fmtKw(load) : "—"} + ${loadKnown ? (loadEstimated ? "≈ " : "") + fmtKw(load) : "—"} ${selfPoweredPct !== null ? ` { + const estimate = {watts: 886, from_ms: Date.UTC(2026,9,18,10), to_ms: Date.UTC(2026,9,18,10,1), age_ms: 90000, end_age_ms:60000, energy_basis:'mixed', timing_basis: 'mixed'}; + const status = {load_w: 1800, load_estimate: estimate, energy:{today:{load_wh:2000,import_wh:500,history_quality:'estimated'}}}; + const result = flowReadingsFromStatus(status); + assert.equal(result.load, 0.886); + assert.equal(result.loadEstimated, true); + assert.equal(status.load_w, 1800); + assert.equal(result.selfPoweredPctToday, null); + assert.match(result.homeEstimateNote, /Estimated home use 0.9 kW/); + assert.match(result.homeEstimateNote, /60 s mean/); + assert.match(result.homeEstimateNote, /\d{2}:00:00.+\d{2}:01:00/); + assert.match(result.homeEstimateNote, /ended 60 s ago/); + assert.match(result.homeEstimateNote, /charger energy \+ sampled power/); + assert.match(result.homeEstimateNote, /Includes arrival times/); + assert.match(homeEstimateNote(estimate, false), /paused/); + assert.match(homeEstimateNote({...estimate, end_age_ms: 181000}), /too old/); + assert.match(homeEstimateNote({...estimate, from_ms:estimate.to_ms-300000,age_ms:210000}), /300 s mean/); + assert.match(homeEstimateNote({watts:null,reason:'charger_source_time_unavailable'}), /source time unknown/); + assert.match(homeEstimateNote({watts: null}), /unavailable/); + assert.equal(flowReadingsFromStatus({...status,load_estimate:{watts:null}}).load, null); + assert.equal(flowReadingsFromStatus({...status,load_estimate:{...estimate,end_age_ms:181000}}).load, null); +}); const LIVE = { grid_w: 500, diff --git a/web/energy-today.js b/web/energy-today.js index 36e10d511..4b37e2ed0 100644 --- a/web/energy-today.js +++ b/web/energy-today.js @@ -16,27 +16,29 @@ function knownWh(value) { export function energyTodayReading(today, key) { const total = today?.[key]; - if (knownWh(total)) return {wh: total, partial: false}; + if (knownWh(total)) return key === "load_wh" && today?.history_quality === "estimated" + ? {wh: total, partial: false, estimated: true} : {wh: total, partial: false}; const measured = today?.measured?.[key]; if (knownWh(measured)) return {wh: measured, partial: true}; return {wh: null, partial: false}; } export function formatEnergyToday(today, key, withUnit = true) { - const {wh, partial} = energyTodayReading(today, key); + const {wh, partial, estimated} = energyTodayReading(today, key); if (wh === null) return "Unknown"; const kwh = wh / 1000; const digits = kwh >= 100 ? 0 : kwh >= 10 ? 1 : 2; - return `${partial ? "≥ " : ""}${kwh.toFixed(digits)}${withUnit ? " kWh" : ""}`; + return `${estimated ? "≈ " : partial ? "≥ " : ""}${kwh.toFixed(digits)}${withUnit ? " kWh" : ""}`; } export function updateEnergyToday(today, root = document) { for (const [id, key] of Object.entries(ENERGY_FIELDS)) { const el = root.getElementById(id); if (!el) continue; - const {partial} = energyTodayReading(today, key); + const {partial, estimated} = energyTodayReading(today, key); el.textContent = formatEnergyToday(today, key); - el.title = partial ? "Recorded energy only; missing readings are excluded." : ""; + el.title = estimated ? "Estimated from source energy during brief balance failures." + : partial ? "Recorded energy only; missing readings are excluded." : ""; } const group = root.getElementById("e-pv")?.closest(".energy-group"); if (!group) return; @@ -49,8 +51,10 @@ export function updateEnergyToday(today, root = document) { } const keys = new Set(Object.values(ENERGY_FIELDS)); const incomplete = [...keys].some(key => !knownWh(today?.[key])); - note.hidden = !incomplete; + const estimated = today?.history_quality === "estimated"; + note.hidden = !incomplete && !estimated; note.textContent = today ? "Gaps in today's readings. ≥ shows recorded energy only." : "Today's energy is not available yet."; + if (estimated) note.textContent = (incomplete ? note.textContent + " " : "") + "Home use includes estimates for brief balance failures."; } diff --git a/web/energy-today.test.mjs b/web/energy-today.test.mjs index 382d9bd53..8ae893228 100644 --- a/web/energy-today.test.mjs +++ b/web/energy-today.test.mjs @@ -31,6 +31,12 @@ test("the energy cards and overview mark subtotals and clear the mark after a co assert.equal(elements.get("e-pv").textContent, "1.00 kWh"); assert.equal(elements.get("e-pv").title, ""); assert.equal(elements.get("energy-today-quality").hidden, true); + updateEnergyToday({...complete, history_quality: "estimated"}, root); + assert.equal(elements.get("e-load").textContent, "≈ 2.00 kWh"); + assert.equal(elements.get("e-pv").textContent, "1.00 kWh"); + assert.match(elements.get("e-load").title, /Estimated/); + assert.equal(elements.get("energy-today-quality").hidden, false); + assert.match(elements.get("energy-today-quality").textContent, /Home use includes estimates/); updateEnergyToday(null, root); assert.equal(elements.get("e-pv").textContent, "Unknown"); assert.equal(elements.get("energy-today-quality").textContent, "Today's energy is not available yet."); diff --git a/web/home-estimate.test.mjs b/web/home-estimate.test.mjs new file mode 100644 index 000000000..cee0f1867 --- /dev/null +++ b/web/home-estimate.test.mjs @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import vm from 'node:vm'; + +test('display smooths the supplied household balance and resets across gaps', () => { + const app = readFileSync(new URL('./app.js', import.meta.url), 'utf8'); + const source = app.slice(app.indexOf(' function smoothDisplayNumber('), app.indexOf(' function smoothSeriesForChart(')); + const c = {statusDisplayState:{}, STATUS_DISPLAY_TAU_MS:400, Date:{now:()=>1000}}; + vm.createContext(c); vm.runInContext(source,c); + // V2X is already part of Core's raw residual. Re-deriving Home use + // from separately smoothed display fields dropped it and broke balance. + const status = {grid_w:2500, bat_w:0, pv_w:0, ev_w:0, v2x_w:1000, load_w:1500}; + assert.equal(c.smoothStatusForDisplay(status).load_w,1500); + assert.equal(status.load_w,1500); + assert.equal(c.smoothStatusForDisplay({...status,load_w:null}).load_w,null); + assert.equal(c.smoothStatusForDisplay({...status,load_w:500}).load_w,500); +}); + +test('the rendered house marks a mean and never combines it with live self-powered share', () => { + const module = readFileSync(new URL('./components/ftw-energy-flow.js', import.meta.url), 'utf8') + .replace(/^import .+;$/m, '').replace(/^export \{[^\n]+\};$/m, ''); + let Flow; + const c = { + FtwElement: class { + constructor() { this.style = {setProperty(){}}; } + hasAttribute() { return false; } + toggleAttribute() {} + update() { this.html = this.render(); } + }, + ftwDebugDelay: () => 0, + document: {addEventListener(){}}, + performance: {now:()=>0}, + customElements: {define:(_, type)=>{Flow=type;}}, + }; + vm.createContext(c); vm.runInContext(module, c); + const flow = new Flow(); + const planets = [{id:'grid',role:'grid',corner:'bottom-left',kw:1,toHub:true,title:'GRID'}]; + flow.setReadings({load:2,loadEstimated:true,planets,selfPoweredPctToday:70}); + assert.match(flow.html, /≈ 2.0/); + assert.match(flow.html, /House load, estimated mean/); + assert.doesNotMatch(flow.html, /SELF-POWERED NOW/); + assert.match(flow.html, /70% SELF-POWERED TODAY/); + flow.setReadings({load:2,loadEstimated:false,planets}); + assert.match(flow.html, /50% SELF-POWERED NOW/); + flow.setReadings({load:null,loadEstimated:true,planets}); + assert.match(flow.html, /House load, no data/); + assert.doesNotMatch(flow.html, /SELF-POWERED NOW/); +});