From daed653b55ab0812204321b4663ced738328a7d7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 15 Nov 2026 19:12:58 +0000 Subject: [PATCH 1/9] feat(forecast): keep issued forecasts in a local archive Copy the 30-day forecast tables into forecast-archive.db after startup. The file stays off the boot and planning path, keeps full detail for 30 days, then one run per hour for two years, and stops before a 512 MiB cap or a full data volume. Health reports it without changing box status. Full backups store a consistent copy. Signed-off-by: Cursor Agent Co-authored-by: Fredrik Ahlgren --- .changeset/forecast-long-archive.md | 7 + docs/architecture.md | 4 + go/cmd/ftw/forecast_long_archive.go | 106 ++ go/cmd/ftw/forecast_long_archive_test.go | 122 ++ go/cmd/ftw/main.go | 4 + go/internal/api/api.go | 5 + go/internal/api/api_health_test.go | 26 + go/internal/api/api_storage_test.go | 26 + go/internal/backup/archive.go | 61 +- .../backup/forecast_archive_backup_test.go | 190 +++ go/internal/state/backup_state.go | 34 + go/internal/state/forecast_issues.go | 133 ++ go/internal/state/forecast_long_archive.go | 1263 +++++++++++++++++ .../state/forecast_long_archive_bench_test.go | 28 + .../state/forecast_long_archive_copy_test.go | 133 ++ .../forecast_long_archive_coverage_test.go | 51 + .../state/forecast_long_archive_heal_test.go | 58 + .../state/forecast_long_archive_prune_test.go | 123 ++ .../state/forecast_long_archive_test.go | 312 ++++ .../state/forecast_long_archive_thin_test.go | 122 ++ go/internal/state/forecast_long_codec.go | 374 +++++ go/internal/state/forecast_long_codec_test.go | 86 ++ go/internal/state/forecast_long_copy.go | 324 +++++ go/internal/state/forecast_long_maintain.go | 644 +++++++++ go/internal/state/snapshot_state_test.go | 24 + go/internal/state/storage_inventory.go | 22 +- go/internal/state/store.go | 13 +- 27 files changed, 4288 insertions(+), 7 deletions(-) create mode 100644 .changeset/forecast-long-archive.md create mode 100644 go/cmd/ftw/forecast_long_archive.go create mode 100644 go/cmd/ftw/forecast_long_archive_test.go create mode 100644 go/internal/backup/forecast_archive_backup_test.go create mode 100644 go/internal/state/forecast_long_archive.go create mode 100644 go/internal/state/forecast_long_archive_bench_test.go create mode 100644 go/internal/state/forecast_long_archive_copy_test.go create mode 100644 go/internal/state/forecast_long_archive_coverage_test.go create mode 100644 go/internal/state/forecast_long_archive_heal_test.go create mode 100644 go/internal/state/forecast_long_archive_prune_test.go create mode 100644 go/internal/state/forecast_long_archive_test.go create mode 100644 go/internal/state/forecast_long_archive_thin_test.go create mode 100644 go/internal/state/forecast_long_codec.go create mode 100644 go/internal/state/forecast_long_codec_test.go create mode 100644 go/internal/state/forecast_long_copy.go create mode 100644 go/internal/state/forecast_long_maintain.go diff --git a/.changeset/forecast-long-archive.md b/.changeset/forecast-long-archive.md new file mode 100644 index 000000000..919e5ce67 --- /dev/null +++ b/.changeset/forecast-long-archive.md @@ -0,0 +1,7 @@ +--- +"ftw": patch +--- + +Keep issued forecasts and quarter-hour load and PV measurements in a local archive for up to two years, separate from planning. + +The archive is informational. A problem in it does not mark the box degraded, and a full backup stores a consistent copy rather than the live file. diff --git a/docs/architecture.md b/docs/architecture.md index 2a5dd2d57..93fe750e6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -149,6 +149,10 @@ SQLite history.db owns bucketed history, dashboard summaries and the energy ledger. A separate state.db owns goals, device identity, learned state and planner diagnostics. Prices and forecasts live in cache.db; full backups keep its past prices, which savings history needs. +Issued forecasts and quarter-hour measurements also accumulate in +forecast-archive.db, opened after startup and kept out of the control path. +A full backup stores a consistent copy of that file. A problem there does +not mark the box degraded. Database access stays in [`go/internal/state`](../go/internal/state). diff --git a/go/cmd/ftw/forecast_long_archive.go b/go/cmd/ftw/forecast_long_archive.go new file mode 100644 index 000000000..9e4132bfa --- /dev/null +++ b/go/cmd/ftw/forecast_long_archive.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// forecastLongArchiver copies the 30-day forecast tables into +// forecast-archive.db. It does not share the tracker's queue or its lock. +type forecastLongArchiver struct { + store *state.Store + cancel context.CancelFunc + wg sync.WaitGroup + startDelay time.Duration + interval time.Duration + now func() time.Time +} + +func startForecastLongArchive(ctx context.Context, st *state.Store) *forecastLongArchiver { + a := &forecastLongArchiver{store: st, startDelay: 2 * time.Minute, interval: time.Hour} + a.Start(ctx) + return a +} + +func (a *forecastLongArchiver) Start(ctx context.Context) { + if a == nil || a.store == nil || a.cancel != nil { + return + } + workerCtx, cancel := context.WithCancel(ctx) + a.cancel = cancel + a.wg.Add(1) + go func() { + defer a.wg.Done() + a.run(workerCtx) + }() +} + +func (a *forecastLongArchiver) Stop() { + if a == nil || a.cancel == nil { + return + } + a.cancel() + a.wg.Wait() +} + +func (a *forecastLongArchiver) run(ctx context.Context) { + a.store.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ + State: "starting", CapBytes: state.ForecastLongMaxBytes, + }) + delay := a.startDelay + if delay < 0 { + delay = 0 + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + a.runOnce(ctx) + interval := a.interval + if interval <= 0 { + interval = time.Hour + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.runOnce(ctx) + } + } +} + +func (a *forecastLongArchiver) runOnce(ctx context.Context) { + if a == nil || a.store == nil || ctx.Err() != nil { + return + } + openCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + arch, err := a.store.OpenForecastLongArchive(openCtx) + cancel() + if err != nil { + if ctx.Err() != nil { + return + } + slog.Warn("forecast long archive unavailable", "err", err) + a.store.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ + State: "unavailable", CapBytes: state.ForecastLongMaxBytes, LastError: err.Error(), + }) + return + } + now := time.Now().UTC() + if a.now != nil { + now = a.now().UTC() + } + if err := arch.Sync(ctx, a.store, now); err != nil && ctx.Err() == nil { + slog.Warn("forecast long archive copy", "err", err) + } +} diff --git a/go/cmd/ftw/forecast_long_archive_test.go b/go/cmd/ftw/forecast_long_archive_test.go new file mode 100644 index 000000000..cb031f5a2 --- /dev/null +++ b/go/cmd/ftw/forecast_long_archive_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +func forecastingIssueForArchive(origin time.Time) forecasting.Issue { + start := origin.UnixMilli() + band := forecasting.Band{Method: forecasting.BandMethodColdStart} + point := forecasting.Point{ + StartMS: start, EndMS: start + int64(time.Hour/time.Millisecond), + PVW: 1, LoadW: 2, PVKnown: true, LoadKnown: true, + PVQuality: "ready", LoadQuality: "ready", + PVBand: band, LoadBand: band, NetBand: band, + } + return forecasting.Issue{ + Schema: forecasting.Schema, ID: "issued", DecisionID: "decision", + OriginMS: start, IssuedAtMS: start, LatestInputMS: start, ConfigVersion: "cfg", + Series: []forecasting.Series{{Name: "champion", ModelVersion: "v1", Points: []forecasting.Point{point}}}, + } +} + +func TestForecastLongArchiverStops(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + before := runtime.NumGoroutine() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + archiver := &forecastLongArchiver{store: st, startDelay: 0, interval: time.Hour} + archiver.Start(ctx) + archiver.Stop() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && runtime.NumGoroutine() > before { + time.Sleep(20 * time.Millisecond) + } + if got := runtime.NumGoroutine(); got > before { + t.Fatalf("goroutines leaked: before %d after %d", before, got) + } +} + +func TestForecastLongArchiverPausesWhenDiskIsFull(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err := st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Add(-time.Hour).Truncate(time.Hour) + issue := forecastingIssueForArchive(origin) + if err := st.SaveForecastIssue(context.Background(), issue); err != nil { + t.Fatal(err) + } + restore := state.ForecastLongDiskAvailForTest(func(string) (int64, error) { return 0, nil }) + defer restore() + archiver := &forecastLongArchiver{store: st, startDelay: time.Hour, interval: time.Hour} + archiver.runOnce(context.Background()) + status, ok := st.ForecastLongArchiveStatus() + if !ok || status.State != "paused_disk_full" || status.Runs != 0 { + t.Fatalf("status = %+v ok=%v", status, ok) + } +} + +func TestForecastRecordDoesNotWaitOnLongArchive(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + origin := time.Now().UTC().Add(-time.Hour).Truncate(time.Hour) + if err := st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + if err := st.SaveForecastIssue(context.Background(), forecastingIssueForArchive(origin)); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + release := make(chan struct{}) + restore := state.ForecastLongDiskAvailForTest(func(string) (int64, error) { + select { + case <-entered: + default: + close(entered) + } + <-release + return 1 << 40, nil + }) + defer restore() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + archiver := &forecastLongArchiver{store: st, startDelay: 0, interval: time.Hour} + archiver.Start(ctx) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("copier did not reach a write") + } + f := trackerFixture(origin) + f.store = st + in := f.Snapshot(origin, trackerWeather(origin.Add(time.Minute), origin)) + slots := trackerSlots(origin.Add(time.Minute), 1) + started := time.Now() + for i := 0; i < 30; i++ { + in.Record(slots, slots, nil, "decision", origin.UnixMilli()) + } + if time.Since(started) > time.Second { + t.Fatal("planner record waited on the long archive") + } + close(release) + archiver.Stop() +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index e5bef56c7..6144f9273 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1667,6 +1667,10 @@ func main() { } else { defer forecastTrackerSvc.Stop() } + // The long-term archive copies the 30-day tables on its own. It starts + // even when the tracker did not, and it never shares the tracker's queue. + forecastLongArchiveSvc := startForecastLongArchive(ctx, st) + defer forecastLongArchiveSvc.Stop() // A small live-identity poll also protects legacy learners when MPC is off. // Config/schema/script work stays in Configure; this only reads host identity. diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 733cdbc29..a78163bef 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -803,6 +803,11 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { if writer.LastError != "" || writer.MaintenanceError != "" || s.deps.State.HistoryMaintenanceStatus().LastError != "" || (writer.LastRejectMS > 0 && time.Now().UnixMilli()-writer.LastRejectMS < time.Minute.Milliseconds()) { resp["status"] = "degraded" } + // The long-term forecast archive is informational. A full disk or a + // rebuilt file must not mark the box degraded. + if archive, ok := s.deps.State.ForecastLongArchiveStatus(); ok { + resp["forecast_archive"] = archive + } } writeJSON(w, 200, resp) } diff --git a/go/internal/api/api_health_test.go b/go/internal/api/api_health_test.go index fcff17835..bb699bf7d 100644 --- a/go/internal/api/api_health_test.go +++ b/go/internal/api/api_health_test.go @@ -83,6 +83,32 @@ func TestHealthStorageReportsCacheRebuild(t *testing.T) { } } +func TestHealthForecastArchiveIsInformational(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + body := getHealth(t, st) + if _, ok := body["forecast_archive"]; ok { + t.Fatal("health published an archive block before the copier started") + } + if body["status"] != "ok" { + t.Fatalf("status = %v", body["status"]) + } + st.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ + State: "unavailable", LastError: "forecast archive rebuilt", CapBytes: state.ForecastLongMaxBytes, + }) + body = getHealth(t, st) + if body["status"] != "ok" { + t.Fatalf("archive failure changed box status to %v", body["status"]) + } + block, ok := body["forecast_archive"].(map[string]any) + if !ok || block["state"] != "unavailable" { + t.Fatalf("forecast_archive = %#v", body["forecast_archive"]) + } +} + func checkpoint(t *testing.T, path string) { t.Helper() db, err := sql.Open("sqlite", path) diff --git a/go/internal/api/api_storage_test.go b/go/internal/api/api_storage_test.go index 06446f975..d769ae324 100644 --- a/go/internal/api/api_storage_test.go +++ b/go/internal/api/api_storage_test.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -54,6 +55,31 @@ func TestHandleStorageInventoryReportsCoreStateAndDataHeadroom(t *testing.T) { } } +func TestHandleStorageInventoryIncludesForecastArchive(t *testing.T) { + dataDir := t.TempDir() + st, err := state.Open(filepath.Join(dataDir, "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := st.OpenForecastLongArchive(context.Background()); err != nil { + t.Fatal(err) + } + srv := New(&Deps{State: st, DataDir: dataDir}) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/api/storage/inventory", nil)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rr.Code, rr.Body.String()) + } + var got storageInventoryResponse + if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Databases.ForecastArchive == nil || got.Databases.ForecastArchive.AllocatedPages == 0 { + t.Fatalf("forecast archive inventory = %+v", got.Databases.ForecastArchive) + } +} + func TestHandleStorageInventoryRequiresPersistentWiring(t *testing.T) { srv := New(&Deps{}) rr := httptest.NewRecorder() diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index d51752584..1621d5abe 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -183,6 +183,9 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { if err != nil { return Info{}, err } + // The live archive is excluded above. The staged consistent copy still + // needs room, so count the file that copy is made from. + extraBytes += forecastArchiveBytes(statePath) if err := ensureBackupDiskSpace(outputDir, state.BackupArchiveScratch(opts.State.BackupSourceBytes(), extraBytes)); err != nil { return Info{}, err } @@ -259,6 +262,21 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { } else if found { sources = append(sources, sourceEntry{archivePath: "data/" + path.Join(path.Dir(databaseFile), "cache.db"), sourcePath: stagedCache}) } + // Same treatment as cache.db: a consistent copy, never the live file or + // its WAL. A damaged analytics file must not cost the owner the backup. + stagedArchive := filepath.Join(stageDir, state.ForecastLongArchiveFilename) + if found, err := opts.State.BackupForecastLongArchive(ctx, stagedArchive); err != nil || (found && verifyDatabaseContext(ctx, stagedArchive) != nil) { + if ctx.Err() != nil { + return Info{}, ctx.Err() + } + if err == nil { + err = verifyDatabaseContext(ctx, stagedArchive) + } + _ = os.Remove(stagedArchive) + slog.Warn("full backup continues without forecast-archive.db", "err", err) + } else if found { + sources = append(sources, sourceEntry{archivePath: "data/" + path.Join(path.Dir(databaseFile), state.ForecastLongArchiveFilename), sourcePath: stagedArchive}) + } sources = append(sources, sourceEntry{ archivePath: databaseEntry, sourcePath: databaseGzip, @@ -346,6 +364,19 @@ func backupExtraBytes(sources []sourceEntry) (int64, error) { return total, nil } +func forecastArchiveBytes(statePath string) int64 { + path := state.ForecastLongArchivePath(statePath) + var n int64 + for _, suffix := range []string{"", "-wal", "-shm"} { + info, err := os.Lstat(path + suffix) + if err != nil || !info.Mode().IsRegular() { + continue + } + n += info.Size() + } + return n +} + func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) { stateRel, _ := filepath.Rel(dataDir, statePath) cacheRel, _ := filepath.Rel(dataDir, filepath.Join(filepath.Dir(statePath), "cache.db")) @@ -410,6 +441,13 @@ func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) if strings.HasPrefix(rel, stateRel+".snapshot") || rel == cacheRel || strings.HasPrefix(rel, cacheRel+"-") { return nil } + // The long-term forecast archive is staged as a consistent copy above. + // Copying the live file, its WAL, or a quarantined sibling would be + // inconsistent and can vanish during a backup. + archiveRel, _ := filepath.Rel(dataDir, state.ForecastLongArchivePath(statePath)) + if rel == archiveRel || strings.HasPrefix(rel, archiveRel+"-") || strings.HasPrefix(rel, archiveRel+".") { + return nil + } // FTW Cloud's key never goes into a backup. A box restored elsewhere // must not speak for the old box's home: its grant reads as off until // an owner turns FTW Cloud on again, under a new home id. @@ -704,8 +742,12 @@ func verifyInWorkspaceProgressContext(ctx context.Context, archivePath, workspac var writer io.Writer = h var dbFile *os.File parquetPath := "" + sqliteSide := "" if entry.Path == manifest.DatabaseEntry { dbFile, err = os.OpenFile(dbGzip, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + } else if path.Base(entry.Path) == state.ForecastLongArchiveFilename { + sqliteSide = filepath.Join(tmpDir, state.ForecastLongArchiveFilename) + dbFile, err = os.OpenFile(sqliteSide, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) } else if strings.HasSuffix(strings.ToLower(entry.Path), ".parquet") { // Stage and remove one day at a time, not the entire cold archive. parquetPath = filepath.Join(tmpDir, "parquet.tmp") @@ -739,6 +781,14 @@ func verifyInWorkspaceProgressContext(ctx context.Context, archivePath, workspac return Manifest{}, err } } + if sqliteSide != "" { + if err := verifyDatabaseContext(ctx, sqliteSide); err != nil { + return Manifest{}, fmt.Errorf("backup: forecast archive: %w", err) + } + if err := os.Remove(sqliteSide); err != nil { + return Manifest{}, err + } + } } if len(seen) != len(want) { return Manifest{}, errors.New("backup: archive is missing one or more manifest files") @@ -1222,7 +1272,16 @@ func extractArchive(archivePath, staging string) error { if len(seen) != len(want) { return errors.New("backup: archive is missing one or more manifest files during extraction") } - return verifyDatabase(filepath.Join(staging, filepath.FromSlash(manifest.DatabaseFile))) + if err := verifyDatabase(filepath.Join(staging, filepath.FromSlash(manifest.DatabaseFile))); err != nil { + return err + } + archived := filepath.Join(staging, filepath.Dir(filepath.FromSlash(manifest.DatabaseFile)), state.ForecastLongArchiveFilename) + if _, err := os.Stat(archived); err == nil { + if err := verifyDatabase(archived); err != nil { + return fmt.Errorf("backup: forecast archive: %w", err) + } + } + return nil } func verifyCompressedDatabase(src, workDir string) error { diff --git a/go/internal/backup/forecast_archive_backup_test.go b/go/internal/backup/forecast_archive_backup_test.go new file mode 100644 index 000000000..547b4b04a --- /dev/null +++ b/go/internal/backup/forecast_archive_backup_test.go @@ -0,0 +1,190 @@ +package backup + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +func backupForecastIssue(origin time.Time) forecasting.Issue { + start := origin.UnixMilli() + band := forecasting.Band{Method: forecasting.BandMethodColdStart} + point := forecasting.Point{ + StartMS: start, EndMS: start + int64(time.Hour/time.Millisecond), + PVW: 12, LoadW: 34, PVKnown: true, LoadKnown: true, + PVQuality: "ready", LoadQuality: "ready", + PVBand: band, LoadBand: band, NetBand: band, + } + return forecasting.Issue{ + Schema: forecasting.Schema, ID: "backed-up", DecisionID: "decision", + OriginMS: start, IssuedAtMS: start, LatestInputMS: start, ConfigVersion: "cfg", + Series: []forecasting.Series{{Name: "champion", ModelVersion: "v1", Points: []forecasting.Point{point}}}, + } +} + +func TestFullBackupIncludesConsistentForecastArchive(t *testing.T) { + for _, offline := range []bool{false, true} { + t.Run(map[bool]string{false: "live", true: "offline"}[offline], func(t *testing.T) { + root := t.TempDir() + dataDir := filepath.Join(root, "data") + if err := os.MkdirAll(dataDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dataDir, "state.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + arch, err := st.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + origin := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if err := arch.ArchiveIssues(context.Background(), []state.ForecastArchiveIssue{{Issue: backupForecastIssue(origin)}}); err != nil { + t.Fatal(err) + } + src := st + if offline { + if err := st.Close(); err != nil { + t.Fatal(err) + } + src, err = state.OpenBackupSource(statePath) + if err != nil { + t.Fatal(err) + } + } + defer src.Close() + info, err := Create(context.Background(), CreateOptions{ + State: src, StatePath: statePath, DataDir: dataDir, + OutputDir: filepath.Join(root, "backups"), + }) + if err != nil { + t.Fatal(err) + } + manifest, _, err := Inspect(context.Background(), info.Path) + if err != nil { + t.Fatal(err) + } + found := false + for _, file := range manifest.Files { + if file.Path == "data/forecast-archive.db" { + found = true + } + if file.Path == "data/forecast-archive.db-wal" || file.Path == "data/forecast-archive.db-shm" { + t.Fatalf("backup contains a live sidecar: %s", file.Path) + } + } + if !found { + t.Fatal("backup is missing forecast-archive.db") + } + restoredDir := filepath.Join(root, "restored") + if _, err := restoreFresh(info.Path, restoredDir, time.Time{}); err != nil { + t.Fatal(err) + } + restored, err := state.Open(filepath.Join(restoredDir, "state.db")) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + restoredArch, err := restored.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + loaded, err := restoredArch.LoadIssue(context.Background(), "backed-up") + if err != nil { + t.Fatal(err) + } + if loaded.Series[0].Points[0].LoadW != 34 { + t.Fatalf("restored issue = %+v", loaded.Series[0].Points[0]) + } + }) + } +} + +func TestFullBackupSkipsDamagedForecastArchive(t *testing.T) { + root := t.TempDir() + dataDir := filepath.Join(root, "data") + if err := os.MkdirAll(dataDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dataDir, "state.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err := os.WriteFile(state.ForecastLongArchivePath(statePath), []byte("not a database"), 0o600); err != nil { + t.Fatal(err) + } + info, err := Create(context.Background(), CreateOptions{ + State: st, StatePath: statePath, DataDir: dataDir, + OutputDir: filepath.Join(root, "backups"), + }) + if err != nil { + t.Fatal(err) + } + f, err := os.Open(info.Path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + zr, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer zr.Close() + tr := tar.NewReader(zr) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + if header.Name == "data/forecast-archive.db" || header.Name == "data/forecast-archive.db-wal" { + t.Fatalf("damaged archive was copied: %s", header.Name) + } + } +} + +func TestBackupPreflightCountsForecastArchive(t *testing.T) { + root := t.TempDir() + st, err := state.Open(filepath.Join(root, "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + archivePath := state.ForecastLongArchivePath(filepath.Join(root, "state.db")) + const archiveBytes = 1 << 20 + if err := os.WriteFile(archivePath, make([]byte, archiveBytes), 0o600); err != nil { + t.Fatal(err) + } + old := ensureBackupDiskSpace + t.Cleanup(func() { ensureBackupDiskSpace = old }) + var needed int64 + ensureBackupDiskSpace = func(dir string, n int64) error { + needed = n + return os.ErrPermission + } + _, err = Create(context.Background(), CreateOptions{ + State: st, StatePath: filepath.Join(root, "state.db"), DataDir: root, + OutputDir: filepath.Join(root, "backups"), + }) + if err == nil { + t.Fatal("preflight did not run") + } + minimum := state.BackupArchiveScratch(st.BackupSourceBytes(), archiveBytes) + if needed < minimum { + t.Fatalf("space check %d omits the %d byte forecast archive (minimum %d)", needed, archiveBytes, minimum) + } +} diff --git a/go/internal/state/backup_state.go b/go/internal/state/backup_state.go index 16dcff2d3..00a11c3bc 100644 --- a/go/internal/state/backup_state.go +++ b/go/internal/state/backup_state.go @@ -99,6 +99,40 @@ func (s *Store) BackupCache(ctx context.Context, dstPath string) (bool, error) { return true, nil } +// BackupForecastLongArchive writes a consistent, compact copy of +// forecast-archive.db. It reports false when the archive has not been +// created. The caller stages that copy and leaves the live file out of the +// archive, including its WAL. +func (s *Store) BackupForecastLongArchive(ctx context.Context, dstPath string) (bool, error) { + if s == nil { + return false, nil + } + s.forecastLongMu.Lock() + arch := s.forecastLong + s.forecastLongMu.Unlock() + var db *sql.DB + if arch != nil { + db = arch.db + } else { + path := ForecastLongArchivePath(s.mainDBPath) + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + ro, err := sql.Open("sqlite", ReadOnlyDatabaseURI(path)) + if err != nil { + return false, err + } + defer ro.Close() + db = ro + } + if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, dstPath); err != nil { + return false, fmt.Errorf("backup forecast archive: %w", err) + } + return true, nil +} + // BackupSourceBytes is the on-disk size of state and history files, including // WAL and SHM. Used to preflight scratch space before a portable export. func (s *Store) BackupSourceBytes() int64 { diff --git a/go/internal/state/forecast_issues.go b/go/internal/state/forecast_issues.go index e8a4af11e..f674c1efb 100644 --- a/go/internal/state/forecast_issues.go +++ b/go/internal/state/forecast_issues.go @@ -703,3 +703,136 @@ func (s *Store) LoadForecastErrors(ctx context.Context, since, until int64, hour } return out, nil } + +// ForecastIssueCopy is one row from the 30-day issue archive, in issued order. +// Err is set when the row cannot be decoded; the copier skips it and moves on. +type ForecastIssueCopy struct { + Issue forecasting.Issue + IssuedAtMS int64 + ID string + JSONBytes int + GzipBytes int + Err error +} + +// VisitForecastIssuesAfter walks issues strictly after the (issued_at_ms, id) +// cursor, oldest first, at most limit rows. The long-term copier uses it so a +// read snapshot stays short. +func (s *Store) VisitForecastIssuesAfter(ctx context.Context, cursorIssuedMS int64, cursorID string, limit int, visit func(ForecastIssueCopy) error) error { + if visit == nil { + return errors.New("forecast issue visitor is nil") + } + if limit <= 0 || limit > 64 { + limit = 16 + } + rows, err := s.db.QueryContext(ctx, `SELECT issued_at_ms, id, payload FROM forecast_issues + WHERE issued_at_ms > ? OR (issued_at_ms = ? AND id > ?) + ORDER BY issued_at_ms, id LIMIT ?`, cursorIssuedMS, cursorIssuedMS, cursorID, limit) + if err != nil { + return err + } + defer rows.Close() + type rawIssue struct { + issued int64 + id string + data []byte + } + var page []rawIssue + for rows.Next() { + var row rawIssue + if err = rows.Scan(&row.issued, &row.id, &row.data); err != nil { + return err + } + page = append(page, row) + } + if err = rows.Err(); err != nil { + return err + } + if err = rows.Close(); err != nil { + return err + } + for _, row := range page { + item := ForecastIssueCopy{IssuedAtMS: row.issued, ID: row.id, GzipBytes: len(row.data)} + issue, size, decodeErr := decodeForecastIssue(row.data) + if decodeErr != nil { + item.Err = decodeErr + } else { + item.Issue = issue + item.JSONBytes = size + } + if err = visit(item); err != nil { + return err + } + } + return nil +} + +// LoadForecastModelStateRaw returns the gzip bytes stored for a model state. +// The long-term archive copies them unchanged. +func (s *Store) LoadForecastModelStateRaw(ctx context.Context, stateID string) (expanded int, payload []byte, err error) { + decodedID, err := hex.DecodeString(stateID) + if err != nil || len(decodedID) != sha256.Size { + return 0, nil, errors.New("invalid forecast model state ID") + } + err = s.db.QueryRowContext(ctx, `SELECT expanded_bytes, payload FROM forecast_model_states WHERE id=?`, stateID).Scan(&expanded, &payload) + return expanded, payload, err +} + +// VisitForecastObservationsAfter walks quarter-hour truth strictly after the +// (start_ms, end_ms, config_version) cursor. +func (s *Store) VisitForecastObservationsAfter(ctx context.Context, startMS, endMS int64, config string, limit int, visit func(forecasting.Observation) error) error { + if visit == nil { + return errors.New("forecast observation visitor is nil") + } + if limit <= 0 || limit > 512 { + limit = 128 + } + rows, err := s.db.QueryContext(ctx, `SELECT payload FROM forecast_observations + WHERE start_ms > ? OR (start_ms = ? AND end_ms > ?) OR (start_ms = ? AND end_ms = ? AND config_version > ?) + ORDER BY start_ms, end_ms, config_version LIMIT ?`, startMS, startMS, endMS, startMS, endMS, config, limit) + if err != nil { + return err + } + defer rows.Close() + var payloads []string + for rows.Next() { + var data string + if err = rows.Scan(&data); err != nil { + return err + } + payloads = append(payloads, data) + } + if err = rows.Err(); err != nil { + return err + } + if err = rows.Close(); err != nil { + return err + } + for _, data := range payloads { + if len(data) > maxForecastRecordBytes { + return errors.New("oversized forecast observation") + } + var observation forecasting.Observation + if err = json.Unmarshal([]byte(data), &observation); err != nil { + return err + } + if err = observation.Validate(); err != nil { + return err + } + if err = visit(observation); err != nil { + return err + } + } + return nil +} + +// NewestForecastIssueMS is the latest issued_at_ms in the 30-day archive, or +// zero when it is empty. The copier uses it only to describe lag. +func (s *Store) NewestForecastIssueMS(ctx context.Context) (int64, error) { + var newest sql.NullInt64 + err := s.db.QueryRowContext(ctx, `SELECT MAX(issued_at_ms) FROM forecast_issues`).Scan(&newest) + if err != nil || !newest.Valid { + return 0, err + } + return newest.Int64, nil +} diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go new file mode 100644 index 000000000..2e44c6b7e --- /dev/null +++ b/go/internal/state/forecast_long_archive.go @@ -0,0 +1,1263 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "math" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "modernc.org/sqlite" +) + +const ( + // ForecastLongArchiveFilename is the long-term forecast archive beside state.db. + ForecastLongArchiveFilename = "forecast-archive.db" + + // ForecastLongFullDetail keeps every issued run. Older runs are thinned + // to one per UTC hour. It matches the 30-day issue archive in state.db. + ForecastLongFullDetail = ForecastIssueRetention + // ForecastLongRetention is two 365-day years. It is a fixed storage policy, + // not a setting. + ForecastLongRetention = 2 * 365 * 24 * time.Hour + // ForecastLongMaxBytes is the live-page cap. Past it, the oldest days + // outside the full-detail window are deleted. The figure is a margin + // around a size estimate, not a measurement from a box. + ForecastLongMaxBytes = 512 << 20 + // ForecastLongMinFreeBytes is the free-space floor on the data volume. + // Below it the copier pauses instead of filling the card. + ForecastLongMinFreeBytes = 1 << 30 + + forecastLongVacuumPages = 256 + forecastLongVacuumBudget = 30 * time.Second + forecastLongFreeTarget = 1 << 20 + forecastLongPageTimeout = 6 * time.Second + forecastLongWarnGap = time.Hour +) + +// ForecastLongArchiveStatus is the cached view published for /api/health. +// Readers never scan the archive. +type ForecastLongArchiveStatus struct { + State string `json:"state"` + LiveBytes int64 `json:"live_bytes"` + FileBytes int64 `json:"file_bytes"` + WALBytes int64 `json:"wal_bytes"` + CapBytes int64 `json:"cap_bytes"` + OldestOriginMS int64 `json:"oldest_origin_ms"` + NewestOriginMS int64 `json:"newest_origin_ms"` + Runs int64 `json:"runs"` + Coverage7d float64 `json:"coverage_7d"` + Coverage30d float64 `json:"coverage_30d"` + CoverageAll float64 `json:"coverage_all"` + TruthCoverage30d float64 `json:"truth_coverage_30d"` + CopyLagMS int64 `json:"copy_lag_ms"` + LastCopyMS int64 `json:"last_copy_ms"` + LastError string `json:"last_error"` + IssueBytesP50 int64 `json:"issue_bytes_p50"` + IssueBytesP95 int64 `json:"issue_bytes_p95"` + IssueGzipBytesP50 int64 `json:"issue_gzip_bytes_p50"` +} + +// ForecastArchiveIssue is one issued forecast plus the gzip model states +// already stored in state.db. Passing those bytes avoids compressing them again. +type ForecastArchiveIssue struct { + Issue forecasting.Issue + JSONBytes int + GzipBytes int + States map[string]ForecastModelBlob +} + +// ForecastModelBlob is the gzip payload stored for one model-state id. +type ForecastModelBlob struct { + Expanded int + Gzip []byte +} + +// ForecastLongArchive is the long-term forecast file. It is opened by the +// background copier, not during boot. +type ForecastLongArchive struct { + db *sql.DB + path string + store *Store + + mu sync.Mutex + status ForecastLongArchiveStatus + rebuilt bool + warned map[string]time.Time + + maxBytes int64 + retention time.Duration + fullDetail time.Duration + minFree int64 + maintainEvery time.Duration +} + +var forecastLongDiskAvail = diskAvail + +// forecastLongBeforeWrite is a test hook. A SQLITE_FULL error pauses copying. +var forecastLongBeforeWrite func() error + +// ForecastLongDiskAvailForTest replaces the free-space probe. The returned +// function restores it. +func ForecastLongDiskAvailForTest(fn func(string) (int64, error)) func() { + prev := forecastLongDiskAvail + if fn == nil { + forecastLongDiskAvail = diskAvail + } else { + forecastLongDiskAvail = fn + } + return func() { forecastLongDiskAvail = prev } +} + +// ForecastLongArchivePath is forecast-archive.db beside a state.db path. +func ForecastLongArchivePath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), ForecastLongArchiveFilename) +} + +// ReportForecastLongArchive publishes the informational health block. It does +// not change control, dispatch or the top-level box status. +func (s *Store) ReportForecastLongArchive(st ForecastLongArchiveStatus) { + if s == nil { + return + } + s.forecastLongMu.Lock() + s.forecastLongStatus = &st + s.forecastLongMu.Unlock() +} + +// ForecastLongArchiveStatus returns the last published block. The second +// result is false until the copier has published one, so health can omit it. +func (s *Store) ForecastLongArchiveStatus() (ForecastLongArchiveStatus, bool) { + if s == nil { + return ForecastLongArchiveStatus{}, false + } + s.forecastLongMu.Lock() + defer s.forecastLongMu.Unlock() + if s.forecastLongStatus == nil { + return ForecastLongArchiveStatus{}, false + } + return *s.forecastLongStatus, true +} + +// OpenForecastLongArchive opens or creates the archive after startup. A +// damaged file is moved aside and replaced with an empty one. Boot does not +// call this. +func (s *Store) OpenForecastLongArchive(ctx context.Context) (*ForecastLongArchive, error) { + if s == nil || s.mainDBPath == "" { + return nil, errors.New("forecast archive: state store unavailable") + } + s.forecastLongMu.Lock() + defer s.forecastLongMu.Unlock() + if s.forecastLong != nil { + return s.forecastLong, nil + } + path := ForecastLongArchivePath(s.mainDBPath) + arch, ev, err := openForecastLongFile(ctx, path) + if err != nil { + return nil, err + } + arch.store = s + if ev != nil { + s.RecordHealEvent(*ev) + arch.rebuilt = true + arch.status.State = "rebuilt" + } + s.forecastLong = arch + return arch, nil +} + +func (s *Store) closeForecastLongArchive() error { + s.forecastLongMu.Lock() + arch := s.forecastLong + s.forecastLong = nil + s.forecastLongMu.Unlock() + if arch == nil { + return nil + } + return arch.Close() +} + +func openForecastLongFile(ctx context.Context, path string) (*ForecastLongArchive, *HealEvent, error) { + db, damaged, err := openForecastLongDB(ctx, path) + if err != nil && !damaged { + return nil, nil, err + } + var ev *HealEvent + if damaged { + if db != nil { + _ = db.Close() + } + nowMs := time.Now().UnixMilli() + if err := quarantine(path, nowMs); err != nil { + return nil, nil, err + } + db, _, err = openForecastLongDB(ctx, path) + if err != nil { + return nil, nil, err + } + ev = &HealEvent{ + Tier: "forecast_archive", Action: healRebuilt, AtMs: nowMs, + Detail: "forecast-archive.db was damaged — moved aside and started an empty forecast archive", + } + } + if err := initForecastLongSchema(ctx, db); err != nil { + _ = db.Close() + return nil, nil, err + } + arch := &ForecastLongArchive{ + db: db, path: path, + status: ForecastLongArchiveStatus{State: "starting", CapBytes: ForecastLongMaxBytes}, + warned: map[string]time.Time{}, + maxBytes: ForecastLongMaxBytes, + retention: ForecastLongRetention, + fullDetail: ForecastLongFullDetail, + minFree: ForecastLongMinFreeBytes, + maintainEvery: 24 * time.Hour, + } + if ev != nil { + arch.rebuilt = true + arch.status.State = "rebuilt" + } + return arch, ev, nil +} + +func openForecastLongDB(ctx context.Context, path string) (*sql.DB, bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + db, err := openRawPragmas(path, forecastLongPragmas) + if err != nil { + return nil, false, err + } + if err := prepareForecastLongFile(ctx, db); err != nil { + _ = db.Close() + return nil, false, err + } + return db, false, nil + } + db, err := openRawPragmas(path, forecastLongPragmas) + if err != nil { + if sqliteCorrupt(err) { + return nil, true, err + } + return nil, false, err + } + ok, qerr := quickCheckContext(ctx, db) + if qerr == nil && ok { + if err := prepareForecastLongFile(ctx, db); err != nil { + _ = db.Close() + return nil, false, err + } + return db, false, nil + } + if ctx.Err() != nil { + _ = db.Close() + return nil, false, ctx.Err() + } + if sqliteBusy(qerr) { + _ = db.Close() + return nil, false, qerr + } + return db, true, qerr +} + +const forecastLongPragmas = sqlitePragmas + "&_pragma=journal_size_limit(4194304)&_pragma=auto_vacuum(INCREMENTAL)" + +func prepareForecastLongFile(ctx context.Context, db *sql.DB) error { + if _, err := db.ExecContext(ctx, `PRAGMA journal_size_limit=4194304`); err != nil { + return err + } + var mode, tables int + if err := db.QueryRowContext(ctx, `PRAGMA auto_vacuum`).Scan(&mode); err != nil { + return err + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`).Scan(&tables); err != nil { + return err + } + if tables == 0 { + // A connection-string pragma can report incremental vacuum before the + // file has a pointer map. Vacuuming the empty file makes later + // incremental_vacuum calls actually return pages. + if mode != 2 { + if _, err := db.ExecContext(ctx, `PRAGMA auto_vacuum=INCREMENTAL`); err != nil { + return err + } + } + if _, err := db.ExecContext(ctx, `VACUUM`); err != nil { + return err + } + } + return nil +} + +func initForecastLongSchema(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS fa_migrations(name TEXT PRIMARY KEY, completed_at TEXT DEFAULT current_timestamp) STRICT; +CREATE TABLE IF NOT EXISTS fa_meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) STRICT; +CREATE TABLE IF NOT EXISTS fa_dict(code INTEGER PRIMARY KEY, kind TEXT NOT NULL, value TEXT NOT NULL, UNIQUE(kind, value)) STRICT; +CREATE TABLE IF NOT EXISTS fa_runs( + run_id TEXT PRIMARY KEY, + decision_id TEXT NOT NULL, + origin_ms INTEGER NOT NULL, issued_at_ms INTEGER NOT NULL, latest_input_ms INTEGER NOT NULL, + config_version TEXT NOT NULL, + site_id TEXT, learning_revision TEXT, timezone TEXT, + has_location INTEGER NOT NULL DEFAULT 0, latitude REAL, longitude REAL, + models TEXT NOT NULL, + occupancy BLOB, occupancy_start_ms INTEGER, occupancy_n INTEGER, + weather_avail TEXT, + input_hash TEXT NOT NULL, + thinned INTEGER NOT NULL DEFAULT 0, + json_bytes INTEGER NOT NULL DEFAULT 0, + gzip_bytes INTEGER NOT NULL DEFAULT 0 +) STRICT; +CREATE INDEX IF NOT EXISTS fa_runs_origin ON fa_runs(origin_ms); +CREATE INDEX IF NOT EXISTS fa_runs_issued ON fa_runs(issued_at_ms); +CREATE TABLE IF NOT EXISTS fa_series( + run_id TEXT NOT NULL, series TEXT NOT NULL, model_version TEXT NOT NULL, + start_ms INTEGER NOT NULL, slot_ms INTEGER NOT NULL, n INTEGER NOT NULL, + starts BLOB, prediction_start_ms INTEGER, + pv_w BLOB NOT NULL, load_w BLOB NOT NULL, + flags BLOB NOT NULL, + pv_quality BLOB NOT NULL, load_quality BLOB NOT NULL, + pv_source BLOB, load_source BLOB, + pv_lo BLOB, pv_hi BLOB, load_lo BLOB, load_hi BLOB, net_lo BLOB, net_hi BLOB, + band_method BLOB, band_stats BLOB, + PRIMARY KEY(run_id, series) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_weather( + source TEXT NOT NULL, available_at_ms INTEGER NOT NULL, start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, + ghi REAL, cloud REAL, temp REAL, direct_pv REAL, est_pv REAL, + PRIMARY KEY(source, available_at_ms, start_ms) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_run_weather( + run_id TEXT NOT NULL, available_at_ms INTEGER NOT NULL, + PRIMARY KEY(run_id, available_at_ms) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_observations_15m( + start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, config_version TEXT NOT NULL, + available_at_ms INTEGER NOT NULL, pv_w REAL NOT NULL, load_w REAL NOT NULL, + pv_known INTEGER NOT NULL, load_known INTEGER NOT NULL, quality TEXT NOT NULL, + PRIMARY KEY(start_ms, end_ms, config_version) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_model_states( + id TEXT PRIMARY KEY, created_at_ms INTEGER NOT NULL, expanded_bytes INTEGER NOT NULL, payload BLOB NOT NULL +) STRICT; +CREATE TABLE IF NOT EXISTS fa_run_states( + run_id TEXT NOT NULL, state_id TEXT NOT NULL, + PRIMARY KEY(run_id, state_id) +) STRICT, WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS fa_run_states_ref ON fa_run_states(state_id); +INSERT OR IGNORE INTO fa_migrations(name) VALUES('fa-v1'); +`) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, `INSERT OR IGNORE INTO fa_meta(key, value) VALUES('created_at_ms', ?)`, fmt.Sprint(time.Now().UnixMilli())) + return err +} + +func sqliteBusy(err error) bool { + var e *sqlite.Error + return errors.As(err, &e) && (e.Code()&0xff == 5 || e.Code()&0xff == 6) +} + +func sqliteFull(err error) bool { + var code interface{ Code() int } + return errors.As(err, &code) && code.Code()&0xff == 13 +} + +// Close releases the archive file. Store.Close does this before cache.db. +func (a *ForecastLongArchive) Close() error { + if a == nil || a.db == nil { + return nil + } + return a.db.Close() +} + +func (a *ForecastLongArchive) warn(kind string, err error) { + if err == nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.status.LastError = err.Error() + now := time.Now() + if a.warned == nil { + a.warned = map[string]time.Time{} + } + if last, ok := a.warned[kind]; ok && now.Sub(last) < forecastLongWarnGap { + return + } + a.warned[kind] = now + slog.Warn("forecast long archive", "kind", kind, "err", err) +} + +func (a *ForecastLongArchive) publish() { + a.mu.Lock() + st := a.status + a.mu.Unlock() + if a.store != nil { + a.store.ReportForecastLongArchive(st) + } +} + +func (a *ForecastLongArchive) setState(state, lastError string) { + a.mu.Lock() + a.status.State = state + if lastError != "" { + a.status.LastError = lastError + } + if state == "ok" || state == "rebuilt" { + a.status.LastError = lastError + } + a.mu.Unlock() + a.publish() +} + +// ArchiveIssues stores one page of issued forecasts. An identical run is left +// untouched. A different payload with the same id is rejected and the stored +// run is kept. Baselines are not stored; they can be rebuilt from observations +// and weather. Quarter-hour truth copied from state.db has load and PV only. +func (a *ForecastLongArchive) ArchiveIssues(ctx context.Context, batch []ForecastArchiveIssue) error { + if len(batch) == 0 { + return nil + } + if err := a.beforeWrite(); err != nil { + return err + } + tx, err := a.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + codes := map[string]byte{} + var immutable error + for _, item := range batch { + if err := a.archiveOne(ctx, tx, codes, item); err != nil { + if errors.Is(err, errForecastArchiveImmutable) { + a.warn("immutable", err) + immutable = err + continue + } + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + return immutable +} + +func (a *ForecastLongArchive) beforeWrite() error { + if forecastLongBeforeWrite != nil { + if err := forecastLongBeforeWrite(); err != nil { + return err + } + } + avail, err := forecastLongDiskAvail(filepath.Dir(a.path)) + if err != nil { + return nil + } + if avail < a.minFree { + return errForecastArchiveDiskFull + } + return nil +} + +var errForecastArchiveDiskFull = errors.New("forecast archive paused: data volume is below the free-space floor") +var errForecastArchiveImmutable = errors.New("forecast long archive run is immutable") + +func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes map[string]byte, item ForecastArchiveIssue) error { + issue := item.Issue + if err := issue.Validate(); err != nil { + return err + } + kept := make([]forecasting.Series, 0, 4) + for _, series := range issue.Series { + if forecastLongKeepSeries(series.Name) { + kept = append(kept, series) + } + } + if len(kept) == 0 { + a.warn("skip-series", errors.New("issue "+issue.ID+" has none of the stored series")) + return nil + } + states, models, err := forecastLongModelBlobs(issue, item.States) + if err != nil { + return err + } + issue.Models = models + hash, err := forecastLongInputHash(issue) + if err != nil { + return err + } + encoded, err := encodeKeptSeries(kept, func(kind, value string) (byte, error) { + return forecastLongDict(ctx, tx, codes, kind, value) + }) + if err != nil { + return err + } + var decision, config, existingHash string + var origin, issued, latest int64 + err = tx.QueryRowContext(ctx, `SELECT decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, input_hash + FROM fa_runs WHERE run_id=?`, issue.ID).Scan(&decision, &origin, &issued, &latest, &config, &existingHash) + if err == nil { + same, cmpErr := forecastLongSameSeries(ctx, tx, issue.ID, encoded) + if cmpErr != nil { + return cmpErr + } + if existingHash != hash || decision != issue.DecisionID || origin != issue.OriginMS || issued != issue.IssuedAtMS || + latest != issue.LatestInputMS || config != issue.ConfigVersion || !same { + return fmt.Errorf("%w: %s", errForecastArchiveImmutable, issue.ID) + } + return nil + } + if !errors.Is(err, sql.ErrNoRows) { + return err + } + for id, blob := range states { + if len(blob.Gzip) == 0 { + continue + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_model_states(id, created_at_ms, expanded_bytes, payload) VALUES(?,?,?,?)`, + id, issue.IssuedAtMS, blob.Expanded, blob.Gzip); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_run_states(run_id, state_id) VALUES(?,?)`, issue.ID, id); err != nil { + return err + } + } + if err := insertForecastWeather(ctx, tx, issue); err != nil { + if !strings.Contains(err.Error(), "is immutable") { + return err + } + a.warn("weather", err) + } + occ, occStart, occN, err := encodeOccupancy(issue.Occupancy) + if err != nil { + return err + } + avail := forecastWeatherAvail(issue.Weather) + modelsJSON, err := json.Marshal(issue.Models) + if err != nil { + return err + } + var siteID, learning, tz sql.NullString + var lat, lon sql.NullFloat64 + hasLoc := 0 + if issue.Site != nil { + siteID = sql.NullString{String: issue.Site.SiteID, Valid: true} + learning = sql.NullString{String: issue.Site.LearningRevision, Valid: true} + tz = sql.NullString{String: issue.Site.Timezone, Valid: true} + lat = sql.NullFloat64{Float64: issue.Site.Latitude, Valid: true} + lon = sql.NullFloat64{Float64: issue.Site.Longitude, Valid: true} + if issue.Site.HasLocation { + hasLoc = 1 + } + } + _, err = tx.ExecContext(ctx, `INSERT INTO fa_runs( + run_id, decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, + site_id, learning_revision, timezone, has_location, latitude, longitude, + models, occupancy, occupancy_start_ms, occupancy_n, weather_avail, input_hash, + thinned, json_bytes, gzip_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0,?,?)`, + issue.ID, issue.DecisionID, issue.OriginMS, issue.IssuedAtMS, issue.LatestInputMS, issue.ConfigVersion, + siteID, learning, tz, hasLoc, lat, lon, string(modelsJSON), nullableBlob(occ), nullMS(occStart, occN > 0), nullCount(occN), + nullText(avail), hash, item.JSONBytes, item.GzipBytes) + if err != nil { + return err + } + for _, series := range encoded { + if err := insertEncodedSeries(ctx, tx, issue.ID, series); err != nil { + return err + } + } + return nil +} + +func nullMS(v int64, ok bool) any { + if !ok { + return nil + } + return v +} + +func nullCount(n int) any { + if n == 0 { + return nil + } + return n +} + +func nullText(s string) any { + if s == "" { + return nil + } + return s +} + +func nullableBlob(b []byte) any { + if len(b) == 0 { + return nil + } + return b +} + +func forecastLongModelBlobs(issue forecasting.Issue, provided map[string]ForecastModelBlob) (map[string]ForecastModelBlob, []forecasting.ModelState, error) { + out := map[string]ForecastModelBlob{} + for id, blob := range provided { + out[id] = blob + } + models := append([]forecasting.ModelState(nil), issue.Models...) + for i := range models { + model := &models[i] + if len(model.State) == 0 { + continue + } + sum := sha256.Sum256(model.State) + id := hex.EncodeToString(sum[:]) + if model.StateID != "" && model.StateID != id { + return nil, nil, errors.New("forecast model state hash mismatch") + } + if _, ok := out[id]; !ok { + gz, err := gzipForecastModelState(model.State) + if err != nil { + return nil, nil, err + } + out[id] = ForecastModelBlob{Expanded: len(model.State), Gzip: gz} + } + model.StateID = id + model.State = nil + } + return out, models, nil +} + +type encodedSeries struct { + name, version string + layout seriesLayout + pv, load []byte + flags []byte + pvQuality []byte + loadQuality []byte + pvSource []byte + loadSource []byte + pvLo, pvHi []byte + loadLo, loadHi []byte + netLo, netHi []byte + bandMethod []byte + bandStats []byte +} + +func encodeKeptSeries(series []forecasting.Series, code func(kind, value string) (byte, error)) ([]encodedSeries, error) { + out := make([]encodedSeries, 0, len(series)) + for _, s := range series { + layout, err := layoutSeries(s.Points) + if err != nil { + return nil, err + } + enc := encodedSeries{name: s.Name, version: s.ModelVersion, layout: layout} + pv := make([]float64, len(s.Points)) + load := make([]float64, len(s.Points)) + flags := make([]byte, len(s.Points)) + pvQ := make([]byte, len(s.Points)) + loadQ := make([]byte, len(s.Points)) + pvS := make([]byte, len(s.Points)) + loadS := make([]byte, len(s.Points)) + var pvLo, pvHi, loadLo, loadHi, netLo, netHi []float64 + methods := make([]byte, len(s.Points)) + sourceUsed := false + if s.Name == forecastLongSeriesChampion { + pvLo = make([]float64, len(s.Points)) + pvHi = make([]float64, len(s.Points)) + loadLo = make([]float64, len(s.Points)) + loadHi = make([]float64, len(s.Points)) + netLo = make([]float64, len(s.Points)) + netHi = make([]float64, len(s.Points)) + } + for i, p := range s.Points { + pv[i], load[i] = p.PVW, p.LoadW + if p.PVKnown { + flags[i] |= 1 + } + if p.LoadKnown { + flags[i] |= 2 + } + if pvQ[i], err = code("quality", p.PVQuality); err != nil { + return nil, err + } + if loadQ[i], err = code("quality", p.LoadQuality); err != nil { + return nil, err + } + if p.PVSource != "" { + sourceUsed = true + if pvS[i], err = code("source", p.PVSource); err != nil { + return nil, err + } + } + if p.LoadSource != "" { + sourceUsed = true + if loadS[i], err = code("source", p.LoadSource); err != nil { + return nil, err + } + } + if s.Name == forecastLongSeriesChampion { + pvLo[i], pvHi[i] = p.PVBand.LowW, p.PVBand.HighW + loadLo[i], loadHi[i] = p.LoadBand.LowW, p.LoadBand.HighW + netLo[i], netHi[i] = p.NetBand.LowW, p.NetBand.HighW + methods[i] = packBandMethod(p) + } + } + if enc.pv, err = encodeF32(pv); err != nil { + return nil, err + } + if enc.load, err = encodeF32(load); err != nil { + return nil, err + } + if enc.flags, err = encodeU8(flags); err != nil { + return nil, err + } + if enc.pvQuality, err = encodeU8(pvQ); err != nil { + return nil, err + } + if enc.loadQuality, err = encodeU8(loadQ); err != nil { + return nil, err + } + if sourceUsed { + if enc.pvSource, err = encodeU8(pvS); err != nil { + return nil, err + } + if enc.loadSource, err = encodeU8(loadS); err != nil { + return nil, err + } + } + if s.Name == forecastLongSeriesChampion { + for _, pair := range []struct { + vals []float64 + dst *[]byte + }{ + {pvLo, &enc.pvLo}, {pvHi, &enc.pvHi}, {loadLo, &enc.loadLo}, {loadHi, &enc.loadHi}, + {netLo, &enc.netLo}, {netHi, &enc.netHi}, + } { + b, err := encodeF32(pair.vals) + if err != nil { + return nil, err + } + *pair.dst = b + } + if enc.bandMethod, err = encodeU8(methods); err != nil { + return nil, err + } + if enc.bandStats, err = encodeBandStats(s.Points); err != nil { + return nil, err + } + } + out = append(out, enc) + } + return out, nil +} + +func forecastLongDict(ctx context.Context, tx *sql.Tx, cache map[string]byte, kind, value string) (byte, error) { + if value == "" { + return 0, nil + } + key := kind + "\x00" + value + if code, ok := cache[key]; ok { + return code, nil + } + if _, err := tx.ExecContext(ctx, `INSERT INTO fa_dict(kind, value) VALUES(?, ?) ON CONFLICT(kind, value) DO NOTHING`, kind, value); err != nil { + return 0, err + } + var code int + if err := tx.QueryRowContext(ctx, `SELECT code FROM fa_dict WHERE kind=? AND value=?`, kind, value).Scan(&code); err != nil { + return 0, err + } + if code < 1 || code > 255 { + return 0, fmt.Errorf("forecast archive dictionary is full for %s", kind) + } + cache[key] = byte(code) + return byte(code), nil +} + +func insertEncodedSeries(ctx context.Context, tx *sql.Tx, runID string, s encodedSeries) error { + var pred any + if s.layout.predictionStartMS != 0 { + pred = s.layout.predictionStartMS + } + _, err := tx.ExecContext(ctx, `INSERT INTO fa_series( + run_id, series, model_version, start_ms, slot_ms, n, starts, prediction_start_ms, + pv_w, load_w, flags, pv_quality, load_quality, pv_source, load_source, + pv_lo, pv_hi, load_lo, load_hi, net_lo, net_hi, band_method, band_stats) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + runID, s.name, s.version, s.layout.startMS, s.layout.slotMS, s.layout.n, nullableBlob(s.layout.starts), pred, + s.pv, s.load, s.flags, s.pvQuality, s.loadQuality, nullableBlob(s.pvSource), nullableBlob(s.loadSource), + nullableBlob(s.pvLo), nullableBlob(s.pvHi), nullableBlob(s.loadLo), nullableBlob(s.loadHi), + nullableBlob(s.netLo), nullableBlob(s.netHi), nullableBlob(s.bandMethod), nullableBlob(s.bandStats)) + return err +} + +func forecastLongSameSeries(ctx context.Context, tx *sql.Tx, runID string, want []encodedSeries) (bool, error) { + rows, err := tx.QueryContext(ctx, `SELECT series, model_version, start_ms, slot_ms, n, starts, prediction_start_ms, + pv_w, load_w, flags, pv_quality, load_quality, pv_source, load_source, + pv_lo, pv_hi, load_lo, load_hi, net_lo, net_hi, band_method, band_stats + FROM fa_series WHERE run_id=?`, runID) + if err != nil { + return false, err + } + defer rows.Close() + got := map[string]encodedSeries{} + for rows.Next() { + var s encodedSeries + var starts, pvSource, loadSource, pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats []byte + var pred sql.NullInt64 + if err := rows.Scan(&s.name, &s.version, &s.layout.startMS, &s.layout.slotMS, &s.layout.n, &starts, &pred, + &s.pv, &s.load, &s.flags, &s.pvQuality, &s.loadQuality, &pvSource, &loadSource, + &pvLo, &pvHi, &loadLo, &loadHi, &netLo, &netHi, &method, &stats); err != nil { + return false, err + } + s.layout.starts = starts + if pred.Valid { + s.layout.predictionStartMS = pred.Int64 + } + s.pvSource, s.loadSource = pvSource, loadSource + s.pvLo, s.pvHi, s.loadLo, s.loadHi, s.netLo, s.netHi = pvLo, pvHi, loadLo, loadHi, netLo, netHi + s.bandMethod, s.bandStats = method, stats + got[s.name] = s + } + if err := rows.Err(); err != nil { + return false, err + } + if len(got) != len(want) { + return false, nil + } + for _, s := range want { + if !encodedSeriesEqual(got[s.name], s) { + return false, nil + } + } + return true, nil +} + +func encodedSeriesEqual(a, b encodedSeries) bool { + if a.name != b.name || a.version != b.version || a.layout.startMS != b.layout.startMS || a.layout.slotMS != b.layout.slotMS || + a.layout.n != b.layout.n || a.layout.predictionStartMS != b.layout.predictionStartMS { + return false + } + pairs := [][2][]byte{ + {a.layout.starts, b.layout.starts}, {a.pv, b.pv}, {a.load, b.load}, {a.flags, b.flags}, + {a.pvQuality, b.pvQuality}, {a.loadQuality, b.loadQuality}, {a.pvSource, b.pvSource}, {a.loadSource, b.loadSource}, + {a.pvLo, b.pvLo}, {a.pvHi, b.pvHi}, {a.loadLo, b.loadLo}, {a.loadHi, b.loadHi}, + {a.netLo, b.netLo}, {a.netHi, b.netHi}, {a.bandMethod, b.bandMethod}, {a.bandStats, b.bandStats}, + } + for _, pair := range pairs { + if !bytesEqualNil(pair[0], pair[1]) { + return false + } + } + return true +} + +func bytesEqualNil(a, b []byte) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func forecastWeatherAvail(rows []forecasting.Weather) string { + if len(rows) == 0 { + return "" + } + seen := map[int64]bool{} + var ids []int64 + for _, row := range rows { + if seen[row.AvailableAtMS] { + continue + } + seen[row.AvailableAtMS] = true + ids = append(ids, row.AvailableAtMS) + } + data, err := json.Marshal(ids) + if err != nil { + return "" + } + return string(data) +} + +func insertForecastWeather(ctx context.Context, tx *sql.Tx, issue forecasting.Issue) error { + seen := map[int64]bool{} + for _, row := range issue.Weather { + if !seen[row.AvailableAtMS] { + seen[row.AvailableAtMS] = true + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_run_weather(run_id, available_at_ms) VALUES(?,?)`, issue.ID, row.AvailableAtMS); err != nil { + return err + } + } + res, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_weather( + source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv) VALUES(?,?,?,?,?,?,?,?,?)`, + row.Source, row.AvailableAtMS, row.StartMS, row.EndMS, + nullFloat(row.GHIWm2), nullFloat(row.CloudPct), nullFloat(row.TempC), nullFloat(row.DirectPVW), nullFloat(row.EstimatedPVW)) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n > 0 { + continue + } + var end int64 + var ghi, cloud, temp, direct, est sql.NullFloat64 + if err := tx.QueryRowContext(ctx, `SELECT end_ms, ghi, cloud, temp, direct_pv, est_pv FROM fa_weather + WHERE source=? AND available_at_ms=? AND start_ms=?`, row.Source, row.AvailableAtMS, row.StartMS).Scan(&end, &ghi, &cloud, &temp, &direct, &est); err != nil { + return err + } + if end != row.EndMS || !nullFloatEqual(ghi, row.GHIWm2) || !nullFloatEqual(cloud, row.CloudPct) || !nullFloatEqual(temp, row.TempC) || + !nullFloatEqual(direct, row.DirectPVW) || !nullFloatEqual(est, row.EstimatedPVW) { + return fmt.Errorf("forecast archive weather at %d is immutable", row.AvailableAtMS) + } + } + return nil +} + +func nullFloat(v *float64) any { + if v == nil || math.IsNaN(*v) { + return nil + } + return *v +} + +func nullFloatEqual(got sql.NullFloat64, want *float64) bool { + if want == nil || math.IsNaN(*want) { + return !got.Valid + } + return got.Valid && got.Float64 == *want +} + +// LoadIssue rebuilds one stored run. Champion bands are restored. Other series +// get an explicit cold-start band, because only the champion interval was kept. +func (a *ForecastLongArchive) LoadIssue(ctx context.Context, runID string) (forecasting.Issue, error) { + var issue forecasting.Issue + var siteID, learning, tz, modelsJSON, weatherAvail sql.NullString + var lat, lon sql.NullFloat64 + var hasLoc int + var occ []byte + var occStart sql.NullInt64 + var occN sql.NullInt64 + err := a.db.QueryRowContext(ctx, `SELECT decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, + site_id, learning_revision, timezone, has_location, latitude, longitude, models, + occupancy, occupancy_start_ms, occupancy_n, weather_avail + FROM fa_runs WHERE run_id=?`, runID).Scan( + &issue.DecisionID, &issue.OriginMS, &issue.IssuedAtMS, &issue.LatestInputMS, &issue.ConfigVersion, + &siteID, &learning, &tz, &hasLoc, &lat, &lon, &modelsJSON, &occ, &occStart, &occN, &weatherAvail) + if err != nil { + return forecasting.Issue{}, err + } + issue.Schema = forecasting.Schema + issue.ID = runID + if siteID.Valid && siteID.String != "" { + issue.Site = &forecasting.SiteContext{ + SiteID: siteID.String, LearningRevision: learning.String, Timezone: tz.String, + HasLocation: hasLoc != 0, Latitude: lat.Float64, Longitude: lon.Float64, + } + } + if modelsJSON.Valid && modelsJSON.String != "" { + if err := json.Unmarshal([]byte(modelsJSON.String), &issue.Models); err != nil { + return forecasting.Issue{}, err + } + } + nOcc := 0 + if occN.Valid { + nOcc = int(occN.Int64) + } + startOcc := int64(0) + if occStart.Valid { + startOcc = occStart.Int64 + } + if issue.Occupancy, err = decodeOccupancy(occ, startOcc, nOcc); err != nil { + return forecasting.Issue{}, err + } + dict, err := a.loadDict(ctx) + if err != nil { + return forecasting.Issue{}, err + } + issue.Series, err = a.loadSeries(ctx, runID, dict) + if err != nil { + return forecasting.Issue{}, err + } + var spanStart, spanEnd int64 + for _, series := range issue.Series { + for _, p := range series.Points { + if spanStart == 0 || p.StartMS < spanStart { + spanStart = p.StartMS + } + if p.EndMS > spanEnd { + spanEnd = p.EndMS + } + } + } + if issue.Weather, err = a.loadWeather(ctx, weatherAvail.String, spanStart, spanEnd); err != nil { + return forecasting.Issue{}, err + } + if err := issue.Validate(); err != nil { + return forecasting.Issue{}, err + } + return issue, nil +} + +func (a *ForecastLongArchive) loadDict(ctx context.Context) (map[byte]string, error) { + rows, err := a.db.QueryContext(ctx, `SELECT code, value FROM fa_dict`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[byte]string{} + for rows.Next() { + var code int + var value string + if err := rows.Scan(&code, &value); err != nil { + return nil, err + } + if code >= 0 && code <= 255 { + out[byte(code)] = value + } + } + return out, rows.Err() +} + +func (a *ForecastLongArchive) loadSeries(ctx context.Context, runID string, dict map[byte]string) ([]forecasting.Series, error) { + rows, err := a.db.QueryContext(ctx, `SELECT series, model_version, start_ms, slot_ms, n, starts, prediction_start_ms, + pv_w, load_w, flags, pv_quality, load_quality, pv_source, load_source, + pv_lo, pv_hi, load_lo, load_hi, net_lo, net_hi, band_method, band_stats + FROM fa_series WHERE run_id=?`, runID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []forecasting.Series + for rows.Next() { + var name, version string + var layout seriesLayout + var starts, pv, load, flags, pvQ, loadQ, pvS, loadS []byte + var pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats []byte + var pred sql.NullInt64 + if err := rows.Scan(&name, &version, &layout.startMS, &layout.slotMS, &layout.n, &starts, &pred, + &pv, &load, &flags, &pvQ, &loadQ, &pvS, &loadS, &pvLo, &pvHi, &loadLo, &loadHi, &netLo, &netHi, &method, &stats); err != nil { + return nil, err + } + layout.starts = starts + if pred.Valid { + layout.predictionStartMS = pred.Int64 + } + series, err := decodeStoredSeries(name, version, layout, pv, load, flags, pvQ, loadQ, pvS, loadS, pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats, dict) + if err != nil { + return nil, err + } + out = append(out, series) + } + return out, rows.Err() +} + +func decodeStoredSeries(name, version string, layout seriesLayout, pvB, loadB, flags, pvQ, loadQ, pvS, loadS, pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats []byte, dict map[byte]string) (forecasting.Series, error) { + starts, ends, err := expandLayout(layout) + if err != nil { + return forecasting.Series{}, err + } + pv, err := decodeF32(pvB, layout.n) + if err != nil { + return forecasting.Series{}, err + } + load, err := decodeF32(loadB, layout.n) + if err != nil { + return forecasting.Series{}, err + } + flagB, err := decodeU8(flags, layout.n) + if err != nil { + return forecasting.Series{}, err + } + pvQB, err := decodeU8(pvQ, layout.n) + if err != nil { + return forecasting.Series{}, err + } + loadQB, err := decodeU8(loadQ, layout.n) + if err != nil { + return forecasting.Series{}, err + } + var pvSB, loadSB, methodB []byte + if len(pvS) > 0 { + if pvSB, err = decodeU8(pvS, layout.n); err != nil { + return forecasting.Series{}, err + } + } + if len(loadS) > 0 { + if loadSB, err = decodeU8(loadS, layout.n); err != nil { + return forecasting.Series{}, err + } + } + var loPV, hiPV, loLoad, hiLoad, loNet, hiNet []float64 + var samples, days [][3]int + champion := name == forecastLongSeriesChampion + if champion { + if methodB, err = decodeU8(method, layout.n); err != nil { + return forecasting.Series{}, err + } + for _, pair := range []struct { + b []byte + dst *[]float64 + }{{pvLo, &loPV}, {pvHi, &hiPV}, {loadLo, &loLoad}, {loadHi, &hiLoad}, {netLo, &loNet}, {netHi, &hiNet}} { + vals, err := decodeF32(pair.b, layout.n) + if err != nil { + return forecasting.Series{}, err + } + *pair.dst = vals + } + if samples, days, err = decodeBandStats(stats, layout.n); err != nil { + return forecasting.Series{}, err + } + } + points := make([]forecasting.Point, layout.n) + for i := 0; i < layout.n; i++ { + p := forecasting.Point{ + StartMS: starts[i], EndMS: ends[i], PVW: pv[i], LoadW: load[i], + PVKnown: flagB[i]&1 != 0, LoadKnown: flagB[i]&2 != 0, + PVQuality: dict[pvQB[i]], LoadQuality: dict[loadQB[i]], + } + if i == 0 { + p.PredictionStartMS = layout.predictionStartMS + } + if len(pvSB) > 0 { + p.PVSource = dict[pvSB[i]] + } + if len(loadSB) > 0 { + p.LoadSource = dict[loadSB[i]] + } + if champion { + pvM, loadM, netM := unpackBandMethod(methodB[i]) + p.PVBand = forecasting.Band{LowW: loPV[i], HighW: hiPV[i], Method: pvM, Samples: samples[i][0], Days: days[i][0]} + p.LoadBand = forecasting.Band{LowW: loLoad[i], HighW: hiLoad[i], Method: loadM, Samples: samples[i][1], Days: days[i][1]} + p.NetBand = forecasting.Band{LowW: loNet[i], HighW: hiNet[i], Method: netM, Samples: samples[i][2], Days: days[i][2]} + if p.PVBand.LowW > p.PVBand.HighW { + p.PVBand.HighW = p.PVBand.LowW + } + if p.LoadBand.LowW > p.LoadBand.HighW { + p.LoadBand.HighW = p.LoadBand.LowW + } + if p.NetBand.LowW > p.NetBand.HighW { + p.NetBand.HighW = p.NetBand.LowW + } + } else { + cold := forecasting.Band{Method: forecasting.BandMethodColdStart} + p.PVBand, p.LoadBand, p.NetBand = cold, cold, cold + } + points[i] = p + } + return forecasting.Series{Name: name, ModelVersion: version, Points: points}, nil +} + +func (a *ForecastLongArchive) loadWeather(ctx context.Context, availJSON string, spanStart, spanEnd int64) ([]forecasting.Weather, error) { + if availJSON == "" || spanEnd <= spanStart { + return nil, nil + } + var ids []int64 + if err := json.Unmarshal([]byte(availJSON), &ids); err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, nil + } + query := `SELECT source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv FROM fa_weather + WHERE available_at_ms IN (` + placeholders(len(ids)) + `) AND start_ms < ? AND end_ms > ? ORDER BY start_ms, source` + args := make([]any, 0, len(ids)+2) + for _, id := range ids { + args = append(args, id) + } + args = append(args, spanEnd, spanStart) + rows, err := a.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []forecasting.Weather + for rows.Next() { + var row forecasting.Weather + var ghi, cloud, temp, direct, est sql.NullFloat64 + if err := rows.Scan(&row.Source, &row.AvailableAtMS, &row.StartMS, &row.EndMS, &ghi, &cloud, &temp, &direct, &est); err != nil { + return nil, err + } + row.GHIWm2 = floatPtr(ghi) + row.CloudPct = floatPtr(cloud) + row.TempC = floatPtr(temp) + row.DirectPVW = floatPtr(direct) + row.EstimatedPVW = floatPtr(est) + out = append(out, row) + } + return out, rows.Err() +} + +func placeholders(n int) string { + if n <= 0 { + return "" + } + b := make([]byte, 0, n*2) + for i := 0; i < n; i++ { + if i > 0 { + b = append(b, ',') + } + b = append(b, '?') + } + return string(b) +} + +func floatPtr(v sql.NullFloat64) *float64 { + if !v.Valid || math.IsNaN(v.Float64) { + return nil + } + f := v.Float64 + return &f +} + +// ArchiveObservations stores quarter-hour truth. The 30-day source records +// household load and PV, not grid power, so grid is not invented here. +func (a *ForecastLongArchive) ArchiveObservations(ctx context.Context, rows []forecasting.Observation) error { + if len(rows) == 0 { + return nil + } + if err := a.beforeWrite(); err != nil { + return err + } + tx, err := a.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, row := range rows { + if err := row.Validate(); err != nil { + return err + } + pv, load := 0, 0 + if row.PVKnown { + pv = 1 + } + if row.LoadKnown { + load = 1 + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_observations_15m( + start_ms, end_ms, config_version, available_at_ms, pv_w, load_w, pv_known, load_known, quality) + VALUES(?,?,?,?,?,?,?,?,?)`, row.StartMS, row.EndMS, row.ConfigVersion, row.AvailableAtMS, row.PVW, row.LoadW, pv, load, row.Quality); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/go/internal/state/forecast_long_archive_bench_test.go b/go/internal/state/forecast_long_archive_bench_test.go new file mode 100644 index 000000000..ea56da3ba --- /dev/null +++ b/go/internal/state/forecast_long_archive_bench_test.go @@ -0,0 +1,28 @@ +package state + +import ( + "context" + "testing" +) + +func BenchmarkForecastLongArchiveCopy(b *testing.B) { + store, err := Open(b.TempDir() + "/state.db") + if err != nil { + b.Fatal(err) + } + defer store.Close() + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + b.Fatal(err) + } + base := longIssue("bench", longOrigin()) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + item := base + item.ID = "bench-" + forecastLongItoa(i) + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: item, JSONBytes: 8000, GzipBytes: 2000}}); err != nil { + b.Fatal(err) + } + } +} diff --git a/go/internal/state/forecast_long_archive_copy_test.go b/go/internal/state/forecast_long_archive_copy_test.go new file mode 100644 index 000000000..d09c12812 --- /dev/null +++ b/go/internal/state/forecast_long_archive_copy_test.go @@ -0,0 +1,133 @@ +package state + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongCopyPagesAndResumes(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Add(-6 * time.Hour).Truncate(time.Hour) + const n = 20 + for i := 0; i < n; i++ { + issue := longIssue("run-"+forecastLongItoa(i), origin.Add(time.Duration(i)*15*time.Minute)) + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + } + obs := forecasting.Observation{ + StartMS: origin.UnixMilli(), EndMS: origin.Add(15 * time.Minute).UnixMilli(), + AvailableAtMS: origin.Add(15 * time.Minute).UnixMilli(), PVW: 10, LoadW: 20, + PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg-v1", + } + if err := store.SaveForecastObservation(ctx, obs); err != nil { + t.Fatal(err) + } + pages := 0 + forecastLongAfterIssuePage = func(rows int) error { + pages++ + if pages == 1 { + return errors.New("stop after the first page") + } + return nil + } + t.Cleanup(func() { forecastLongAfterIssuePage = nil }) + if err := arch.Sync(ctx, store, time.Now().UTC()); err == nil { + t.Fatal("interrupted copy reported success") + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil { + t.Fatal(err) + } + if runs != forecastLongIssuePage { + t.Fatalf("first page stored %d runs, want %d", runs, forecastLongIssuePage) + } + forecastLongAfterIssuePage = nil + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != n { + t.Fatalf("resumed runs = %d err=%v, want %d", runs, err, n) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != n { + t.Fatalf("second copy duplicated runs: %d err=%v", runs, err) + } + var observations int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_observations_15m`).Scan(&observations); err != nil || observations != 1 { + t.Fatalf("observations = %d err=%v", observations, err) + } + var srcGzip, dstGzip []byte + if err := store.db.QueryRow(`SELECT payload FROM forecast_model_states LIMIT 1`).Scan(&srcGzip); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT payload FROM fa_model_states LIMIT 1`).Scan(&dstGzip); err != nil { + t.Fatal(err) + } + if !bytes.Equal(srcGzip, dstGzip) { + t.Fatal("model state was recompressed") + } + var states int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_model_states`).Scan(&states); err != nil || states != 1 { + t.Fatalf("model states = %d err=%v, want the one shared snapshot", states, err) + } +} + +func TestForecastLongCopySkipsPrunedSource(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + issue := longIssue("kept", time.Now().UTC().Add(-time.Hour).Truncate(time.Hour)) + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + if _, err := store.db.Exec(`DELETE FROM forecast_model_states`); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + var runs, states int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("runs = %d err=%v", runs, err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_model_states`).Scan(&states); err != nil || states != 0 { + t.Fatalf("pruned model state was invented: %d err=%v", states, err) + } + if _, err := store.db.Exec(`DELETE FROM forecast_issues`); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("archive dropped a copied run after the source was pruned: %d err=%v", runs, err) + } +} + +func forecastLongItoa(n int) string { + if n == 0 { + return "0" + } + var b [8]byte + i := len(b) + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + return string(b[i:]) +} diff --git a/go/internal/state/forecast_long_archive_coverage_test.go b/go/internal/state/forecast_long_archive_coverage_test.go new file mode 100644 index 000000000..f9824176a --- /dev/null +++ b/go/internal/state/forecast_long_archive_coverage_test.go @@ -0,0 +1,51 @@ +package state + +import ( + "context" + "testing" + "time" +) + +func TestForecastLongCoverageClipsToArchiveStart(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + start := now.Add(-4 * time.Hour) + for _, hour := range []int{0, 1, 3} { + at := start.Add(time.Duration(hour) * time.Hour).UnixMilli() + if _, err := arch.db.ExecContext(ctx, `INSERT INTO fa_runs( + run_id, decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, models, input_hash) + VALUES(?, '', ?, ?, 0, 'cfg', '[]', 'hash')`, "h"+string(rune('0'+hour)), at, at); err != nil { + t.Fatal(err) + } + } + quarter := int64(15 * 60 * 1000) + // Four known quarters and four unknown ones inside the clipped window. + for i := 0; i < 8; i++ { + known := 0 + if i < 4 { + known = 1 + } + at := start.UnixMilli() + int64(i)*quarter + if _, err := arch.db.ExecContext(ctx, `INSERT INTO fa_observations_15m( + start_ms, end_ms, config_version, available_at_ms, pv_w, load_w, pv_known, load_known, quality) + VALUES(?, ?, 'cfg', ?, 1, 1, 1, ?, 'complete')`, at, at+quarter, at+quarter, known); err != nil { + t.Fatal(err) + } + } + got := arch.hourCoverage(ctx, now, 7*24*time.Hour, start.UnixMilli()) + // Hours 08:00 through 12:00 inclusive once clipped to 08:00, with 08, 09 + // and 11 present. 12:00 itself has no run. + if got != 0.6 { + t.Fatalf("coverage = %v, want 0.6", got) + } + truth := arch.truthCoverage(ctx, now, start.UnixMilli()) + span := (now.Truncate(15*time.Minute).UnixMilli() - start.UnixMilli()) / quarter + if truth != float64(4)/float64(span) { + t.Fatalf("truth coverage = %v, want %v", truth, float64(4)/float64(span)) + } + at := start.Add(3 * time.Hour) + if all := arch.hourCoverage(ctx, at, time.Hour, at.UnixMilli()); all != 1 { + t.Fatalf("coverage clipped to a later start = %v, want 1", all) + } +} diff --git a/go/internal/state/forecast_long_archive_heal_test.go b/go/internal/state/forecast_long_archive_heal_test.go new file mode 100644 index 000000000..df607d69e --- /dev/null +++ b/go/internal/state/forecast_long_archive_heal_test.go @@ -0,0 +1,58 @@ +package state + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestForecastLongArchiveRebuildsCorruption(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + archivePath := ForecastLongArchivePath(path) + if err := os.WriteFile(archivePath, []byte("this is not a sqlite database"), 0o600); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatalf("state.Open failed because of the analytics file: %v", err) + } + defer reopened.Close() + if events := reopened.HealEvents(); len(events) != 0 { + t.Fatalf("opening state healed the archive on the boot path: %+v", events) + } + arch, err := reopened.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + events := reopened.HealEvents() + if len(events) != 1 || events[0].Tier != "forecast_archive" || events[0].Action != healRebuilt { + t.Fatalf("heal events = %+v", events) + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 0 { + t.Fatalf("rebuilt archive runs = %d err=%v", runs, err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + quarantined := false + for _, entry := range entries { + if strings.Contains(entry.Name(), "forecast-archive.db.corrupt-") { + quarantined = true + } + } + if !quarantined { + t.Fatal("damaged archive was not moved aside") + } +} diff --git a/go/internal/state/forecast_long_archive_prune_test.go b/go/internal/state/forecast_long_archive_prune_test.go new file mode 100644 index 000000000..f56041dad --- /dev/null +++ b/go/internal/state/forecast_long_archive_prune_test.go @@ -0,0 +1,123 @@ +package state + +import ( + "context" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongRetentionAndCap(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + arch.retention = 48 * time.Hour + arch.fullDetail = 24 * time.Hour + old := longIssue("old", now.Add(-72*time.Hour)) + recent := longIssue("recent", now.Add(-time.Hour)) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: old}, {Issue: recent}}); err != nil { + t.Fatal(err) + } + obs := func(at time.Time) forecasting.Observation { + return forecasting.Observation{ + StartMS: at.UnixMilli(), EndMS: at.Add(15 * time.Minute).UnixMilli(), + AvailableAtMS: at.Add(15 * time.Minute).UnixMilli(), PVW: 1, LoadW: 2, + PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg-v1", + } + } + if err := arch.ArchiveObservations(ctx, []forecasting.Observation{obs(now.Add(-72 * time.Hour)), obs(now.Add(-time.Hour))}); err != nil { + t.Fatal(err) + } + if err := arch.pruneExpired(ctx, now); err != nil { + t.Fatal(err) + } + assertRuns(t, arch, map[string]bool{"recent": true}) + var oldObs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_observations_15m WHERE end_ms= infoBefore && free > 0 { + t.Fatalf("vacuum left the file at %d (was %d) with %d free pages; live before delete %d", after, infoBefore, free, before) + } +} + +func assertRuns(t *testing.T, arch *ForecastLongArchive, want map[string]bool) { + t.Helper() + rows, err := arch.db.Query(`SELECT run_id FROM fa_runs`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + got := map[string]bool{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + got[id] = true + } + if len(got) != len(want) { + t.Fatalf("runs = %v, want %v", got, want) + } + for id := range want { + if !got[id] { + t.Fatalf("missing %s in %v", id, got) + } + } +} diff --git a/go/internal/state/forecast_long_archive_test.go b/go/internal/state/forecast_long_archive_test.go new file mode 100644 index 000000000..d1ddb7309 --- /dev/null +++ b/go/internal/state/forecast_long_archive_test.go @@ -0,0 +1,312 @@ +package state + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "math" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func openLongArchive(t *testing.T) (*Store, *ForecastLongArchive) { + t.Helper() + store, err := Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + return store, arch +} + +func longOrigin() time.Time { + return time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) +} + +func longBand(low, high float64, method string) forecasting.Band { + b := forecasting.Band{LowW: low, HighW: high, Method: method} + if method == forecasting.BandMethodEmpirical { + b.Samples, b.Days = 48, 7 + } + return b +} + +func longPoint(start int64, pv, load float64, partial bool) forecasting.Point { + p := forecasting.Point{ + StartMS: start, EndMS: start + 15*60*1000, PVW: pv, LoadW: load, + PVKnown: true, LoadKnown: load >= 0, PVQuality: "ready", LoadQuality: "learning", + PVSource: "energyplan", LoadSource: "legacy", + PVBand: longBand(0, pv+10, forecasting.BandMethodEmpirical), + LoadBand: longBand(0, 0, forecasting.BandMethodColdStart), + NetBand: longBand(0, 0, forecasting.BandMethodColdStart), + } + if partial { + p.PredictionStartMS = start + 60*1000 + } + return p +} + +func longIssue(id string, origin time.Time) forecasting.Issue { + start := origin.UnixMilli() + ghi := 120.5 + temp := 4.25 + quarter := int64(15 * 60 * 1000) + return forecasting.Issue{ + Schema: forecasting.Schema, ID: id, DecisionID: "decision-" + id, + OriginMS: start, IssuedAtMS: start + 1000, LatestInputMS: start, + ConfigVersion: "cfg-v1", + Site: &forecasting.SiteContext{ + SiteID: "site-1", LearningRevision: "learn-1", Timezone: "UTC", + HasLocation: true, Latitude: 59.3293, Longitude: 18.0686, + }, + Models: []forecasting.ModelState{{ + Name: "load", Version: "load-v3", Quality: forecasting.ModelQualityWarm, + UpdatedAtMS: start, State: []byte(`{"weights":[1,2,3]}`), + }}, + Weather: []forecasting.Weather{{ + StartMS: start, EndMS: start + quarter, AvailableAtMS: start, + Source: "open_meteo", GHIWm2: &ghi, TempC: &temp, + }}, + Occupancy: []forecasting.Occupancy{ + {StartMS: start - 2*quarter, EndMS: start - quarter, AvailableAtMS: start - quarter, Home: true}, + {StartMS: start - quarter, EndMS: start, AvailableAtMS: start, Home: false}, + }, + Series: []forecasting.Series{ + {Name: "champion", ModelVersion: "champ-1", Points: []forecasting.Point{longPoint(start, 1500.5, 800.25, true)}}, + {Name: "planning", ModelVersion: "plan-1", Points: []forecasting.Point{longPoint(start, 1400, 900, false)}}, + {Name: "legacy_shadow", ModelVersion: "legacy-1", Points: []forecasting.Point{longPoint(start, 1000, 700, false)}}, + {Name: "energyplan", ModelVersion: "ep-1", Points: []forecasting.Point{ + longPoint(start, 1500.5, 800.25, false), + func() forecasting.Point { + p := longPoint(start+30*60*1000, 400, 650, false) + p.EndMS = start + 90*60*1000 + return p + }(), + }}, + {Name: "previous_day", ModelVersion: "baseline", Points: []forecasting.Point{longPoint(start, 1, 1, false)}}, + }, + } +} + +func TestForecastLongArchiveOpensAndReopens(t *testing.T) { + store, arch := openLongArchive(t) + var marker string + if err := arch.db.QueryRow(`SELECT name FROM fa_migrations`).Scan(&marker); err != nil || marker != "fa-v1" { + t.Fatalf("migration = %q err=%v", marker, err) + } + var mode int + if err := arch.db.QueryRow(`PRAGMA auto_vacuum`).Scan(&mode); err != nil || mode != 2 { + t.Fatalf("auto_vacuum = %d err=%v, want incremental", mode, err) + } + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: longIssue("one", longOrigin())}}); err != nil { + t.Fatal(err) + } + store.Close() + again, err := Open(store.mainDBPath) + if err != nil { + t.Fatal(err) + } + defer again.Close() + arch, err = again.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + var n int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_migrations`).Scan(&n); err != nil || n != 1 { + t.Fatalf("migrations = %d err=%v", n, err) + } + if _, err := arch.LoadIssue(context.Background(), "one"); err != nil { + t.Fatal(err) + } +} + +func TestForecastLongArchiveRoundTrip(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + issue := longIssue("round", longOrigin()) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue, JSONBytes: 1200, GzipBytes: 400}}); err != nil { + t.Fatal(err) + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue, JSONBytes: 1200, GzipBytes: 400}}); err != nil { + t.Fatal(err) + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("runs = %d err=%v", runs, err) + } + loaded, err := arch.LoadIssue(ctx, "round") + if err != nil { + t.Fatal(err) + } + if err := loaded.Validate(); err != nil { + t.Fatal(err) + } + byName := map[string]forecasting.Series{} + for _, series := range loaded.Series { + byName[series.Name] = series + } + if len(byName) != 4 { + t.Fatalf("stored series = %d, baselines should be omitted", len(byName)) + } + if _, ok := byName["previous_day"]; ok { + t.Fatal("baseline series was stored") + } + want := map[string]forecasting.Series{} + for _, series := range issue.Series { + if forecastLongKeepSeries(series.Name) { + want[series.Name] = series + } + } + for name, series := range want { + got := byName[name] + if got.ModelVersion != series.ModelVersion || len(got.Points) != len(series.Points) { + t.Fatalf("%s identity = %+v", name, got) + } + for i, point := range series.Points { + gp := got.Points[i] + if float32(gp.PVW) != float32(point.PVW) || float32(gp.LoadW) != float32(point.LoadW) || + gp.PVKnown != point.PVKnown || gp.LoadKnown != point.LoadKnown || + gp.PVQuality != point.PVQuality || gp.LoadQuality != point.LoadQuality || + gp.PVSource != point.PVSource || gp.LoadSource != point.LoadSource || + gp.StartMS != point.StartMS || gp.EndMS != point.EndMS { + t.Fatalf("%s point %d = %+v want %+v", name, i, gp, point) + } + } + } + champ := byName["champion"].Points[0] + if champ.PredictionStartMS != issue.Series[0].Points[0].PredictionStartMS || + champ.PVBand.Method != forecasting.BandMethodEmpirical || champ.PVBand.Samples != 48 || champ.PVBand.Days != 7 || + float32(champ.PVBand.HighW) != float32(issue.Series[0].Points[0].PVBand.HighW) { + t.Fatalf("champion band = %+v", champ) + } + if byName["planning"].Points[0].PVBand.Method != forecasting.BandMethodColdStart { + t.Fatal("non-champion band was stored") + } + if loaded.Site == nil || loaded.Site.SiteID != "site-1" || loaded.Site.Latitude != issue.Site.Latitude { + t.Fatalf("site = %+v", loaded.Site) + } + if len(loaded.Occupancy) != 2 || loaded.Occupancy[0].Home != true || loaded.Occupancy[1].Home != false || + loaded.Occupancy[0].AvailableAtMS != issue.Occupancy[0].AvailableAtMS { + t.Fatalf("occupancy = %+v", loaded.Occupancy) + } + if len(loaded.Weather) != 1 || loaded.Weather[0].GHIWm2 == nil || *loaded.Weather[0].GHIWm2 != *issue.Weather[0].GHIWm2 || + loaded.Weather[0].CloudPct != nil { + t.Fatalf("weather = %+v", loaded.Weather) + } + sum := sha256.Sum256(issue.Models[0].State) + if len(loaded.Models) != 1 || loaded.Models[0].StateID != hex.EncodeToString(sum[:]) || len(loaded.Models[0].State) != 0 || + loaded.Models[0].Version != "load-v3" { + t.Fatalf("models = %+v", loaded.Models) + } + var jsonBytes, gzipBytes int + if err := arch.db.QueryRow(`SELECT json_bytes, gzip_bytes FROM fa_runs WHERE run_id='round'`).Scan(&jsonBytes, &gzipBytes); err != nil || jsonBytes != 1200 || gzipBytes != 400 { + t.Fatalf("measured sizes %d %d err=%v", jsonBytes, gzipBytes, err) + } +} + +func TestForecastLongArchiveRejectsChangedIdentity(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + issue := longIssue("same", longOrigin()) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + changed := issue + changed.Series = append([]forecasting.Series(nil), issue.Series...) + changed.Series[0].Points = append([]forecasting.Point(nil), issue.Series[0].Points...) + changed.Series[0].Points[0].LoadW = 42 + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: changed}}); err == nil { + t.Fatal("changed payload was accepted") + } + loaded, err := arch.LoadIssue(ctx, "same") + if err != nil { + t.Fatal(err) + } + var champ forecasting.Point + for _, series := range loaded.Series { + if series.Name == "champion" { + champ = series.Points[0] + } + } + if float32(champ.LoadW) != float32(issue.Series[0].Points[0].LoadW) { + t.Fatalf("stored load changed to %v", champ.LoadW) + } +} + +func TestForecastLongArchiveDeduplicatesWeather(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + first := longIssue("a", longOrigin()) + second := longIssue("b", longOrigin().Add(time.Hour)) + second.Weather = first.Weather + second.Weather[0].AvailableAtMS = first.OriginMS + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: first}, {Issue: second}}); err != nil { + t.Fatal(err) + } + var n int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_weather`).Scan(&n); err != nil || n != 1 { + t.Fatalf("weather rows = %d err=%v", n, err) + } +} + +func TestForecastLongEncodedHorizonSize(t *testing.T) { + _, arch := openLongArchive(t) + origin := longOrigin() + issue := longIssue("horizon", origin) + quarter := int64(15 * 60 * 1000) + points := make([]forecasting.Point, forecasting.MaxPoints) + for i := range points { + points[i] = longPoint(origin.UnixMilli()+int64(i)*quarter, float64(100+i), float64(500+i), i == 0) + } + for i := range issue.Series { + if forecastLongKeepSeries(issue.Series[i].Name) { + issue.Series[i].Points = points + } + } + // The last baseline series is not stored. Drop the irregular second point + // by replacing every kept series with the regular horizon. + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + var blobs int + if err := arch.db.QueryRow(`SELECT COALESCE(SUM(length(pv_w)+length(load_w)+COALESCE(length(pv_lo),0)+COALESCE(length(pv_hi),0)),0) FROM fa_series`).Scan(&blobs); err != nil { + t.Fatal(err) + } + // Four series of 193 float32 pairs, plus champion bands. This is a fixture + // measurement, not a box measurement. + t.Logf("encoded power and champion band bytes=%d", blobs) + if blobs < 4000 || blobs > 20000 { + t.Fatalf("encoded horizon = %d bytes, outside the expected fixture range", blobs) + } + if math.IsNaN(float64(blobs)) { + t.Fatal("size was not a number") + } +} + +func TestForecastLongGappedOccupancy(t *testing.T) { + _, arch := openLongArchive(t) + issue := longIssue("gap", longOrigin()) + quarter := int64(15 * 60 * 1000) + start := issue.OriginMS + issue.Occupancy = []forecasting.Occupancy{ + {StartMS: start - 4*quarter, EndMS: start - 3*quarter, AvailableAtMS: start - 3*quarter, Home: true}, + {StartMS: start - quarter, EndMS: start, AvailableAtMS: start, Home: false}, + } + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + loaded, err := arch.LoadIssue(context.Background(), "gap") + if err != nil { + t.Fatal(err) + } + if len(loaded.Occupancy) != 2 || loaded.Occupancy[0].StartMS != issue.Occupancy[0].StartMS || loaded.Occupancy[1].Home { + t.Fatalf("gapped occupancy = %+v", loaded.Occupancy) + } +} diff --git a/go/internal/state/forecast_long_archive_thin_test.go b/go/internal/state/forecast_long_archive_thin_test.go new file mode 100644 index 000000000..2124232ce --- /dev/null +++ b/go/internal/state/forecast_long_archive_thin_test.go @@ -0,0 +1,122 @@ +package state + +import ( + "context" + "testing" + "time" +) + +func TestForecastLongThinningKeepsOneRunPerHour(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + old := now.Add(-40 * 24 * time.Hour).Truncate(time.Hour) + young := now.Add(-24 * time.Hour).Truncate(time.Hour) + issues := []struct { + id string + origin time.Time + issued time.Time + }{ + {"hour-keep", old, old.Add(time.Second)}, + {"hour-later", old, old.Add(2 * time.Second)}, + {"hour-quarter", old.Add(15 * time.Minute), old}, + {"next-hour", old.Add(time.Hour), old.Add(time.Hour)}, + {"young-a", young, young}, + {"young-b", young.Add(15 * time.Minute), young}, + } + for _, item := range issues { + issue := longIssue(item.id, item.origin) + issue.IssuedAtMS = item.issued.UnixMilli() + if issue.IssuedAtMS < issue.OriginMS { + issue.IssuedAtMS = issue.OriginMS + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatalf("%s: %v", item.id, err) + } + } + // The quarter-hour run is issued at the same instant as its origin above + // when that would otherwise precede the origin. Give it a later issue time + // that is still earlier than the :00 run's issue time only if the :00 + // preference is what keeps hour-keep. + if err := arch.thin(ctx, now); err != nil { + t.Fatal(err) + } + kept := map[string]bool{} + rows, err := arch.db.Query(`SELECT run_id, thinned FROM fa_runs`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + thinned := map[string]int{} + for rows.Next() { + var id string + var flag int + if err := rows.Scan(&id, &flag); err != nil { + t.Fatal(err) + } + kept[id] = true + thinned[id] = flag + } + if kept["hour-later"] || kept["hour-quarter"] || !kept["hour-keep"] || !kept["next-hour"] { + t.Fatalf("old hours = %v", kept) + } + if !kept["young-a"] || !kept["young-b"] || thinned["young-a"] != 0 || thinned["young-b"] != 0 { + t.Fatalf("young runs were thinned: %v flags %v", kept, thinned) + } + if thinned["hour-keep"] != 1 || thinned["next-hour"] != 1 { + t.Fatalf("kept old runs not marked thinned: %v", thinned) + } + var states int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_run_states`).Scan(&states); err != nil { + t.Fatal(err) + } + // One reference for the first kept run of each local day. Both old hours + // are the same local day, so one reference remains. Young runs are not thinned. + if states != 3 { + t.Fatalf("state references = %d, want the daily old reference plus two young runs", states) + } + var weather int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_weather`).Scan(&weather); err != nil || weather != 4 { + t.Fatalf("weather rows = %d err=%v, want one per kept run", weather, err) + } +} + +func TestForecastLongDeletesAreBatched(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + old := now.Add(-40 * 24 * time.Hour).Truncate(time.Hour) + var batches []int + forecastLongAfterDeleteBatch = func(table string, n int) { + if table == "fa_runs" { + batches = append(batches, n) + } + } + t.Cleanup(func() { forecastLongAfterDeleteBatch = nil }) + for i := 0; i < 70; i++ { + // Same UTC hour. The smallest id is kept; the other 69 are one full delete batch plus a remainder. + issue := longIssue("b-"+forecastLongItoa(i), old) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatalf("issue %d: %v", i, err) + } + } + if err := arch.thin(ctx, now); err != nil { + t.Fatal(err) + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("runs after thinning = %d err=%v", runs, err) + } + sawFull := false + for _, n := range batches { + if n > forecastPruneBatch { + t.Fatalf("delete batch %d exceeds %d", n, forecastPruneBatch) + } + if n == forecastPruneBatch { + sawFull = true + } + } + if !sawFull { + t.Fatalf("batches = %v, want a full batch of %d", batches, forecastPruneBatch) + } +} diff --git a/go/internal/state/forecast_long_codec.go b/go/internal/state/forecast_long_codec.go new file mode 100644 index 000000000..ef5fcf7f9 --- /dev/null +++ b/go/internal/state/forecast_long_codec.go @@ -0,0 +1,374 @@ +package state + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "sort" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +const ( + forecastLongSeriesChampion = "champion" + forecastLongSeriesPlanning = "planning" + forecastLongSeriesLegacy = "legacy_shadow" + forecastLongSeriesEnergyplan = "energyplan" + forecastLongQuarterMS = int64(15 * 60 * 1000) + forecastLongMaxVector = forecasting.MaxPoints + forecastLongBandCold byte = 0 + forecastLongBandEmpirical byte = 1 +) + +func forecastLongKeepSeries(name string) bool { + switch name { + case forecastLongSeriesChampion, forecastLongSeriesPlanning, forecastLongSeriesLegacy, forecastLongSeriesEnergyplan: + return true + default: + return false + } +} + +// encodeF32 writes little-endian float32 values. NaN is kept for an unknown +// weather or band value. Infinity and magnitudes float32 cannot hold are +// rejected; contract points are already finite. +func encodeF32(vals []float64) ([]byte, error) { + if len(vals) > forecastLongMaxVector { + return nil, errors.New("forecast archive vector is too long") + } + out := make([]byte, 4*len(vals)) + for i, v := range vals { + if math.IsNaN(v) { + binary.LittleEndian.PutUint32(out[i*4:], math.Float32bits(float32(math.NaN()))) + continue + } + if math.IsInf(v, 0) || v > math.MaxFloat32 || v < -math.MaxFloat32 { + return nil, fmt.Errorf("forecast archive value %v is outside float32", v) + } + binary.LittleEndian.PutUint32(out[i*4:], math.Float32bits(float32(v))) + } + return out, nil +} + +func decodeF32(b []byte, n int) ([]float64, error) { + if n < 0 || n > forecastLongMaxVector || len(b) != 4*n { + return nil, errors.New("forecast archive float32 length mismatch") + } + out := make([]float64, n) + for i := 0; i < n; i++ { + out[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(b[i*4:]))) + } + return out, nil +} + +func encodeU8(vals []byte) ([]byte, error) { + if len(vals) > forecastLongMaxVector { + return nil, errors.New("forecast archive vector is too long") + } + return append([]byte(nil), vals...), nil +} + +func decodeU8(b []byte, n int) ([]byte, error) { + if n < 0 || n > forecastLongMaxVector || len(b) != n { + return nil, errors.New("forecast archive byte vector length mismatch") + } + return append([]byte(nil), b...), nil +} + +// seriesLayout is the time grid of one stored series. A regular grid leaves +// starts empty. An irregular grid stores little-endian int32 offsets, in +// seconds, from startMS. When durations differ, each point is an offset and +// a duration (2n values) so gaps still round-trip. +type seriesLayout struct { + startMS int64 + slotMS int64 + n int + starts []byte + predictionStartMS int64 +} + +func layoutSeries(points []forecasting.Point) (seriesLayout, error) { + if len(points) == 0 || len(points) > forecastLongMaxVector { + return seriesLayout{}, errors.New("forecast archive series length is invalid") + } + for i, p := range points { + if i > 0 && p.PredictionStartMS != 0 { + return seriesLayout{}, errors.New("forecast archive partial point is only stored on the first interval") + } + } + start := points[0].StartMS + slot := points[0].EndMS - points[0].StartMS + regular := true + sameSlot := true + for i, p := range points { + d := p.EndMS - p.StartMS + if d <= 0 || d > int64(60*60*1000) { + return seriesLayout{}, errors.New("forecast archive interval is invalid") + } + if d != slot { + sameSlot = false + regular = false + } + if p.StartMS != start+int64(i)*slot { + regular = false + } + if (p.StartMS-start)%1000 != 0 || d%1000 != 0 { + return seriesLayout{}, errors.New("forecast archive interval is not a whole number of seconds") + } + } + out := seriesLayout{startMS: start, n: len(points), predictionStartMS: points[0].PredictionStartMS} + if regular { + out.slotMS = slot + return out, nil + } + if sameSlot { + out.slotMS = slot + out.starts = make([]byte, 4*len(points)) + for i, p := range points { + off := (p.StartMS - start) / 1000 + if off > math.MaxInt32 || off < math.MinInt32 { + return seriesLayout{}, errors.New("forecast archive time offset overflows") + } + binary.LittleEndian.PutUint32(out.starts[i*4:], uint32(int32(off))) + } + return out, nil + } + out.starts = make([]byte, 8*len(points)) + for i, p := range points { + off := (p.StartMS - start) / 1000 + dur := (p.EndMS - p.StartMS) / 1000 + if off > math.MaxInt32 || off < math.MinInt32 || dur > math.MaxInt32 || dur <= 0 { + return seriesLayout{}, errors.New("forecast archive time offset overflows") + } + binary.LittleEndian.PutUint32(out.starts[i*8:], uint32(int32(off))) + binary.LittleEndian.PutUint32(out.starts[i*8+4:], uint32(int32(dur))) + } + return out, nil +} + +func expandLayout(l seriesLayout) (starts, ends []int64, err error) { + if l.n <= 0 || l.startMS <= 0 { + return nil, nil, errors.New("forecast archive series grid is empty") + } + starts = make([]int64, l.n) + ends = make([]int64, l.n) + switch { + case len(l.starts) == 0: + if l.slotMS <= 0 { + return nil, nil, errors.New("forecast archive regular grid has no slot") + } + for i := 0; i < l.n; i++ { + starts[i] = l.startMS + int64(i)*l.slotMS + ends[i] = starts[i] + l.slotMS + } + case len(l.starts) == 4*l.n: + if l.slotMS <= 0 { + return nil, nil, errors.New("forecast archive irregular grid has no slot") + } + for i := 0; i < l.n; i++ { + off := int32(binary.LittleEndian.Uint32(l.starts[i*4:])) + starts[i] = l.startMS + int64(off)*1000 + ends[i] = starts[i] + l.slotMS + } + case len(l.starts) == 8*l.n: + for i := 0; i < l.n; i++ { + off := int32(binary.LittleEndian.Uint32(l.starts[i*8:])) + dur := int32(binary.LittleEndian.Uint32(l.starts[i*8+4:])) + if dur <= 0 { + return nil, nil, errors.New("forecast archive irregular duration is empty") + } + starts[i] = l.startMS + int64(off)*1000 + ends[i] = starts[i] + int64(dur)*1000 + } + default: + return nil, nil, errors.New("forecast archive time grid length mismatch") + } + return starts, ends, nil +} + +func packBandMethod(p forecasting.Point) byte { + code := func(method string) byte { + if method == forecasting.BandMethodEmpirical { + return forecastLongBandEmpirical + } + return forecastLongBandCold + } + return code(p.PVBand.Method) | code(p.LoadBand.Method)<<2 | code(p.NetBand.Method)<<4 +} + +func unpackBandMethod(v byte) (pv, load, net string) { + name := func(code byte) string { + if code&0x3 == forecastLongBandEmpirical { + return forecasting.BandMethodEmpirical + } + return forecasting.BandMethodColdStart + } + return name(v), name(v >> 2), name(v >> 4) +} + +func encodeBandStats(points []forecasting.Point) ([]byte, error) { + out := make([]byte, len(points)*6*4) + put := func(i, field, v int) error { + if v < 0 || v > math.MaxUint32 { + return errors.New("forecast archive band statistic overflows") + } + binary.LittleEndian.PutUint32(out[(i*6+field)*4:], uint32(v)) + return nil + } + for i, p := range points { + stats := []int{p.PVBand.Samples, p.PVBand.Days, p.LoadBand.Samples, p.LoadBand.Days, p.NetBand.Samples, p.NetBand.Days} + for field, v := range stats { + if err := put(i, field, v); err != nil { + return nil, err + } + } + } + return out, nil +} + +func decodeBandStats(b []byte, n int) (samples, days [][3]int, err error) { + if len(b) != n*6*4 { + return nil, nil, errors.New("forecast archive band statistic length mismatch") + } + samples = make([][3]int, n) + days = make([][3]int, n) + for i := 0; i < n; i++ { + for field := 0; field < 3; field++ { + samples[i][field] = int(binary.LittleEndian.Uint32(b[(i*6+field*2)*4:])) + days[i][field] = int(binary.LittleEndian.Uint32(b[(i*6+field*2+1)*4:])) + } + } + return samples, days, nil +} + +// encodeOccupancy stores a bit per quarter when the rows are contiguous, and +// the available-at time of each quarter so a restored issue still validates. +// Gaps use an explicit record. An empty slice returns a nil blob. +func encodeOccupancy(rows []forecasting.Occupancy) (blob []byte, startMS int64, n int, err error) { + if len(rows) == 0 { + return nil, 0, 0, nil + } + if len(rows) > 512 { + return nil, 0, 0, errors.New("forecast archive occupancy is too long") + } + n = len(rows) + startMS = rows[0].StartMS + contiguous := true + for i, row := range rows { + if row.EndMS-row.StartMS != forecastLongQuarterMS { + contiguous = false + break + } + if i > 0 && row.StartMS != rows[i-1].EndMS { + contiguous = false + break + } + } + if contiguous { + bits := (n + 7) / 8 + blob = make([]byte, 1+bits+n*8) + blob[0] = 0 + for i, row := range rows { + if row.Home { + blob[1+i/8] |= 1 << (uint(i) % 8) + } + binary.LittleEndian.PutUint64(blob[1+bits+i*8:], uint64(row.AvailableAtMS)) + } + return blob, startMS, n, nil + } + blob = make([]byte, 1+n*25) + blob[0] = 1 + off := 1 + for _, row := range rows { + binary.LittleEndian.PutUint64(blob[off:], uint64(row.StartMS)) + binary.LittleEndian.PutUint64(blob[off+8:], uint64(row.EndMS)) + binary.LittleEndian.PutUint64(blob[off+16:], uint64(row.AvailableAtMS)) + if row.Home { + blob[off+24] = 1 + } + off += 25 + } + return blob, startMS, n, nil +} + +func decodeOccupancy(blob []byte, startMS int64, n int) ([]forecasting.Occupancy, error) { + if n == 0 { + return nil, nil + } + if n < 0 || n > 512 || len(blob) == 0 { + return nil, errors.New("forecast archive occupancy is invalid") + } + switch blob[0] { + case 0: + bits := (n + 7) / 8 + if len(blob) != 1+bits+n*8 || startMS <= 0 { + return nil, errors.New("forecast archive occupancy bitset length mismatch") + } + out := make([]forecasting.Occupancy, n) + for i := 0; i < n; i++ { + out[i] = forecasting.Occupancy{ + StartMS: startMS + int64(i)*forecastLongQuarterMS, + EndMS: startMS + int64(i+1)*forecastLongQuarterMS, + AvailableAtMS: int64(binary.LittleEndian.Uint64(blob[1+bits+i*8:])), + Home: blob[1+i/8]&(1<<(uint(i)%8)) != 0, + } + } + return out, nil + case 1: + if len(blob) != 1+n*25 { + return nil, errors.New("forecast archive occupancy record length mismatch") + } + out := make([]forecasting.Occupancy, n) + off := 1 + for i := 0; i < n; i++ { + out[i] = forecasting.Occupancy{ + StartMS: int64(binary.LittleEndian.Uint64(blob[off:])), + EndMS: int64(binary.LittleEndian.Uint64(blob[off+8:])), + AvailableAtMS: int64(binary.LittleEndian.Uint64(blob[off+16:])), + Home: blob[off+24] != 0, + } + off += 25 + } + return out, nil + default: + return nil, errors.New("forecast archive occupancy version is unknown") + } +} + +// forecastLongInputHash is the SHA-256 of the frozen site, occupancy, weather +// and model references. Series watts are stored beside it and compared on +// their own, so a changed forecast with the same inputs is still rejected. +func forecastLongInputHash(issue forecasting.Issue) (string, error) { + models := append([]forecasting.ModelState(nil), issue.Models...) + sort.Slice(models, func(i, j int) bool { return models[i].Name < models[j].Name }) + for i := range models { + models[i].State = nil + } + weather := append([]forecasting.Weather(nil), issue.Weather...) + sort.Slice(weather, func(i, j int) bool { + if weather[i].AvailableAtMS != weather[j].AvailableAtMS { + return weather[i].AvailableAtMS < weather[j].AvailableAtMS + } + if weather[i].Source != weather[j].Source { + return weather[i].Source < weather[j].Source + } + return weather[i].StartMS < weather[j].StartMS + }) + occupancy := append([]forecasting.Occupancy(nil), issue.Occupancy...) + sort.Slice(occupancy, func(i, j int) bool { return occupancy[i].StartMS < occupancy[j].StartMS }) + payload, err := json.Marshal(struct { + Site *forecasting.SiteContext `json:"site,omitempty"` + Occupancy []forecasting.Occupancy `json:"occupancy,omitempty"` + Weather []forecasting.Weather `json:"weather,omitempty"` + Models []forecasting.ModelState `json:"models,omitempty"` + }{issue.Site, occupancy, weather, models}) + if err != nil { + return "", err + } + sum := sha256.Sum256(payload) + return hex.EncodeToString(sum[:]), nil +} diff --git a/go/internal/state/forecast_long_codec_test.go b/go/internal/state/forecast_long_codec_test.go new file mode 100644 index 000000000..8460c69e1 --- /dev/null +++ b/go/internal/state/forecast_long_codec_test.go @@ -0,0 +1,86 @@ +package state + +import ( + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongCodecRoundTrip(t *testing.T) { + values := []float64{0, 1, 1500.5, math.MaxFloat32, -math.MaxFloat32, math.NaN()} + encoded, err := encodeF32(values) + if err != nil { + t.Fatal(err) + } + if len(encoded) != 4*len(values) { + t.Fatalf("encoded length %d", len(encoded)) + } + got, err := decodeF32(encoded, len(values)) + if err != nil { + t.Fatal(err) + } + for i, want := range values { + if math.IsNaN(want) { + if !math.IsNaN(got[i]) { + t.Fatalf("value %d: want NaN, got %v", i, got[i]) + } + continue + } + if float32(got[i]) != float32(want) { + t.Fatalf("value %d: got %v want %v", i, got[i], want) + } + } + if _, err := decodeF32(encoded, len(values)-1); err == nil { + t.Fatal("length mismatch was accepted") + } + bytes, err := encodeU8([]byte{0, 1, 255}) + if err != nil { + t.Fatal(err) + } + if _, err := decodeU8(bytes, 2); err == nil { + t.Fatal("byte length mismatch was accepted") + } +} + +func TestForecastLongCodecRejectsNonFinite(t *testing.T) { + if _, err := encodeF32([]float64{math.Inf(1)}); err == nil { + t.Fatal("positive infinity was accepted") + } + if _, err := encodeF32([]float64{math.Inf(-1)}); err == nil { + t.Fatal("negative infinity was accepted") + } + if _, err := encodeF32([]float64{math.MaxFloat32 * 2}); err == nil { + t.Fatal("value outside float32 was accepted") + } +} + +func TestForecastLongIrregularGridAndPartialPoint(t *testing.T) { + start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC).UnixMilli() + points := []forecasting.Point{ + {StartMS: start, EndMS: start + 15*60*1000, PredictionStartMS: start + 60*1000}, + {StartMS: start + 30*60*1000, EndMS: start + 90*60*1000}, + } + layout, err := layoutSeries(points) + if err != nil { + t.Fatal(err) + } + if layout.predictionStartMS != start+60*1000 { + t.Fatalf("partial first point = %d", layout.predictionStartMS) + } + if len(layout.starts) != 16 { + t.Fatalf("irregular grid bytes = %d, want 16", len(layout.starts)) + } + starts, ends, err := expandLayout(layout) + if err != nil { + t.Fatal(err) + } + if starts[0] != points[0].StartMS || ends[0] != points[0].EndMS || starts[1] != points[1].StartMS || ends[1] != points[1].EndMS { + t.Fatalf("grid = %v %v", starts, ends) + } + points[1].PredictionStartMS = points[1].StartMS + 1000 + if _, err := layoutSeries(points); err == nil { + t.Fatal("partial point after the first was accepted") + } +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go new file mode 100644 index 000000000..8858186ec --- /dev/null +++ b/go/internal/state/forecast_long_copy.go @@ -0,0 +1,324 @@ +package state + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +const ( + forecastLongIssuePage = 16 + forecastLongObsPage = 128 +) + +// forecastLongAfterIssuePage runs after a committed issue page. Tests use it +// to stop a copy between pages. +var forecastLongAfterIssuePage func(rows int) error + +type forecastLongIssueCursor struct { + IssuedAtMS int64 `json:"issued_at_ms"` + ID string `json:"id"` +} + +type forecastLongObsCursor struct { + StartMS int64 `json:"start_ms"` + EndMS int64 `json:"end_ms"` + Config string `json:"config_version"` +} + +// Sync copies the 30-day issue and observation tables, then at most once a +// day thins, prunes and gives free pages back. Failures stay in the published +// status. A full disk pauses until the next call instead of retrying. +func (a *ForecastLongArchive) Sync(ctx context.Context, src *Store, now time.Time) error { + if a == nil || src == nil { + return errors.New("forecast archive: copy source unavailable") + } + if now.IsZero() { + now = time.Now().UTC() + } + a.mu.Lock() + a.status.LastError = "" + a.mu.Unlock() + if err := src.InitForecastArchive(ctx); err != nil { + a.fail("schema", err) + return err + } + var paused error + if !a.blocked() { + if err := a.copyIssues(ctx, src); err != nil { + if forecastLongPaused(err) { + paused = err + } else { + a.fail("copy-issues", err) + return err + } + } + } + if paused == nil && !a.blocked() { + if err := a.copyObservations(ctx, src); err != nil { + if forecastLongPaused(err) { + paused = err + } else { + a.fail("copy-observations", err) + return err + } + } + } + if paused == nil { + due, err := a.maintenanceDue(ctx, now) + if err != nil { + a.fail("maintenance", err) + return err + } + if due { + if err := a.maintain(ctx, now); err != nil { + if forecastLongPaused(err) { + paused = err + } else { + a.fail("maintenance", err) + return err + } + } else if err := a.setMeta(ctx, "last_maintain_ms", jsonInt(now.UnixMilli())); err != nil { + a.fail("maintenance", err) + return err + } + } + } + if err := a.recompute(ctx, src, now, paused); err != nil { + a.fail("status", err) + return err + } + return nil +} + +func forecastLongPaused(err error) bool { + return errors.Is(err, errForecastArchiveDiskFull) || sqliteFull(err) +} + +func (a *ForecastLongArchive) blocked() bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.status.State == "over_cap" +} + +func (a *ForecastLongArchive) fail(kind string, err error) { + a.warn(kind, err) + a.mu.Lock() + if a.status.State == "" || a.status.State == "starting" || a.status.State == "ok" { + a.status.State = "unavailable" + } + a.mu.Unlock() + a.publish() +} + +func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store) error { + cursor, err := a.issueCursor(ctx) + if err != nil { + return err + } + for { + if err := ctx.Err(); err != nil { + return err + } + pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) + var page []ForecastIssueCopy + err = src.VisitForecastIssuesAfter(pageCtx, cursor.IssuedAtMS, cursor.ID, forecastLongIssuePage, func(item ForecastIssueCopy) error { + page = append(page, item) + return nil + }) + cancel() + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + batch := make([]ForecastArchiveIssue, 0, len(page)) + for _, item := range page { + cursor = forecastLongIssueCursor{IssuedAtMS: item.IssuedAtMS, ID: item.ID} + if item.Err != nil { + a.warn("decode", item.Err) + continue + } + states, err := forecastLongStatesFrom(ctx, src, item.Issue) + if err != nil { + return err + } + batch = append(batch, ForecastArchiveIssue{ + Issue: item.Issue, JSONBytes: item.JSONBytes, GzipBytes: item.GzipBytes, States: states, + }) + } + writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) + err = a.ArchiveIssues(writeCtx, batch) + if err == nil || errors.Is(err, errForecastArchiveImmutable) { + err = a.writeIssueCursor(writeCtx, cursor) + } + cancelWrite() + if err != nil { + return err + } + if forecastLongAfterIssuePage != nil { + if err := forecastLongAfterIssuePage(len(page)); err != nil { + return err + } + } + a.mu.Lock() + a.status.LastCopyMS = time.Now().UnixMilli() + a.mu.Unlock() + if len(page) < forecastLongIssuePage { + return nil + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.Issue) (map[string]ForecastModelBlob, error) { + states := map[string]ForecastModelBlob{} + for _, model := range issue.Models { + if model.StateID == "" { + continue + } + if _, ok := states[model.StateID]; ok { + continue + } + expanded, payload, err := src.LoadForecastModelStateRaw(ctx, model.StateID) + if errors.Is(err, sql.ErrNoRows) { + continue + } + if err != nil { + return nil, err + } + states[model.StateID] = ForecastModelBlob{Expanded: expanded, Gzip: payload} + } + return states, nil +} + +func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store) error { + cursor, err := a.obsCursor(ctx) + if err != nil { + return err + } + for { + if err := ctx.Err(); err != nil { + return err + } + pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) + var page []forecasting.Observation + err = src.VisitForecastObservationsAfter(pageCtx, cursor.StartMS, cursor.EndMS, cursor.Config, forecastLongObsPage, func(row forecasting.Observation) error { + page = append(page, row) + return nil + }) + cancel() + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) + err = a.ArchiveObservations(writeCtx, page) + if err == nil { + last := page[len(page)-1] + cursor = forecastLongObsCursor{StartMS: last.StartMS, EndMS: last.EndMS, Config: last.ConfigVersion} + err = a.writeObsCursor(writeCtx, cursor) + } + cancelWrite() + if err != nil { + return err + } + if len(page) < forecastLongObsPage { + return nil + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) issueCursor(ctx context.Context) (forecastLongIssueCursor, error) { + var cursor forecastLongIssueCursor + raw, err := a.metaValue(ctx, "copy_cursor_issue") + if err != nil || raw == "" { + return cursor, err + } + err = json.Unmarshal([]byte(raw), &cursor) + return cursor, err +} + +func (a *ForecastLongArchive) writeIssueCursor(ctx context.Context, cursor forecastLongIssueCursor) error { + data, err := json.Marshal(cursor) + if err != nil { + return err + } + return a.setMeta(ctx, "copy_cursor_issue", string(data)) +} + +func (a *ForecastLongArchive) obsCursor(ctx context.Context) (forecastLongObsCursor, error) { + var cursor forecastLongObsCursor + raw, err := a.metaValue(ctx, "copy_cursor_obs") + if err != nil || raw == "" { + return cursor, err + } + err = json.Unmarshal([]byte(raw), &cursor) + return cursor, err +} + +func (a *ForecastLongArchive) writeObsCursor(ctx context.Context, cursor forecastLongObsCursor) error { + data, err := json.Marshal(cursor) + if err != nil { + return err + } + return a.setMeta(ctx, "copy_cursor_obs", string(data)) +} + +func (a *ForecastLongArchive) metaValue(ctx context.Context, key string) (string, error) { + var value string + err := a.db.QueryRowContext(ctx, `SELECT value FROM fa_meta WHERE key=?`, key).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return value, err +} + +func (a *ForecastLongArchive) setMeta(ctx context.Context, key, value string) error { + _, err := a.db.ExecContext(ctx, `INSERT INTO fa_meta(key, value) VALUES(?, ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value`, key, value) + return err +} + +func (a *ForecastLongArchive) maintenanceDue(ctx context.Context, now time.Time) (bool, error) { + raw, err := a.metaValue(ctx, "last_maintain_ms") + if err != nil || raw == "" { + return err == nil, err + } + var ms int64 + if err := json.Unmarshal([]byte(raw), &ms); err != nil { + return true, nil + } + return now.Sub(time.UnixMilli(ms)) >= a.maintainEvery, nil +} + +func jsonInt(v int64) string { + data, _ := json.Marshal(v) + return string(data) +} + +func (src *Store) forecastCopyLag(ctx context.Context, cursor forecastLongIssueCursor, now int64) (int64, error) { + var issued sql.NullInt64 + err := src.db.QueryRowContext(ctx, `SELECT MIN(issued_at_ms) FROM forecast_issues + WHERE issued_at_ms > ? OR (issued_at_ms = ? AND id > ?)`, cursor.IssuedAtMS, cursor.IssuedAtMS, cursor.ID).Scan(&issued) + if err != nil || !issued.Valid { + return 0, err + } + lag := now - issued.Int64 + if lag < 0 { + lag = 0 + } + return lag, nil +} diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go new file mode 100644 index 000000000..53657200f --- /dev/null +++ b/go/internal/state/forecast_long_maintain.go @@ -0,0 +1,644 @@ +package state + +import ( + "context" + "database/sql" + "math" + "sort" + "time" +) + +// forecastLongAfterDeleteBatch counts bounded deletes. Tests assert the +// 64-row batches. +var forecastLongAfterDeleteBatch func(table string, n int) + +func (a *ForecastLongArchive) maintain(ctx context.Context, now time.Time) error { + if err := a.thin(ctx, now); err != nil { + return err + } + if err := a.pruneExpired(ctx, now); err != nil { + return err + } + if err := a.pruneToCap(ctx, now); err != nil { + return err + } + if err := a.collectGarbage(ctx); err != nil { + return err + } + return a.incrementalVacuum(ctx) +} + +type forecastLongDay struct { + key string + start int64 + end int64 + ids []string +} + +func (a *ForecastLongArchive) thin(ctx context.Context, now time.Time) error { + cutoff := now.Add(-a.fullDetail).UnixMilli() + for { + if err := ctx.Err(); err != nil { + return err + } + day, ok, err := a.nextUnthinnedDay(ctx, cutoff) + if err != nil || !ok { + return err + } + losers, err := a.thinnedLosers(ctx, day.ids) + if err != nil { + return err + } + if err := a.deleteRunIDs(ctx, losers); err != nil { + return err + } + if err := a.keepDailyState(ctx, day.ids, losers); err != nil { + return err + } + if err := a.markThinned(ctx, day.ids, losers); err != nil { + return err + } + if err := a.collectGarbage(ctx); err != nil { + return err + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) nextUnthinnedDay(ctx context.Context, cutoff int64) (forecastLongDay, bool, error) { + rows, err := a.db.QueryContext(ctx, `SELECT run_id, origin_ms, timezone FROM fa_runs + WHERE thinned=0 AND origin_ms 1` + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + rows, err := a.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var losers []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + losers = append(losers, id) + } + return losers, rows.Err() +} + +func (a *ForecastLongArchive) keepDailyState(ctx context.Context, ids, losers []string) error { + drop := map[string]bool{} + for _, id := range losers { + drop[id] = true + } + var kept []string + for _, id := range ids { + if !drop[id] { + kept = append(kept, id) + } + } + if len(kept) <= 1 { + return nil + } + query := `SELECT run_id FROM fa_runs WHERE run_id IN (` + placeholders(len(kept)) + `) + ORDER BY origin_ms, issued_at_ms, run_id LIMIT 1` + args := make([]any, len(kept)) + for i, id := range kept { + args[i] = id + } + var first string + if err := a.db.QueryRowContext(ctx, query, args...).Scan(&first); err != nil { + return err + } + var extra []string + for _, id := range kept { + if id != first { + extra = append(extra, id) + } + } + return a.deleteWhereIDs(ctx, "fa_run_states", "run_id", extra) +} + +func (a *ForecastLongArchive) markThinned(ctx context.Context, ids, losers []string) error { + drop := map[string]bool{} + for _, id := range losers { + drop[id] = true + } + var kept []string + for _, id := range ids { + if !drop[id] { + kept = append(kept, id) + } + } + if len(kept) == 0 { + return nil + } + _, err := a.db.ExecContext(ctx, `UPDATE fa_runs SET thinned=1 WHERE run_id IN (`+placeholders(len(kept))+`)`, stringsToArgs(kept)...) + return err +} + +func (a *ForecastLongArchive) pruneExpired(ctx context.Context, now time.Time) error { + cutoff := now.Add(-a.retention).UnixMilli() + protect := now.Add(-a.fullDetail).UnixMilli() + if err := a.deleteRunsBefore(ctx, cutoff, protect, "issued_at_ms"); err != nil { + return err + } + return a.deleteObservationsBefore(ctx, cutoff, protect) +} + +func (a *ForecastLongArchive) pruneToCap(ctx context.Context, now time.Time) error { + protect := now.Add(-a.fullDetail).UnixMilli() + for { + if err := ctx.Err(); err != nil { + return err + } + live, err := a.liveByteCount(ctx) + if err != nil { + return err + } + if live <= a.maxBytes { + a.mu.Lock() + if a.status.State == "over_cap" { + a.status.State = "" + } + a.mu.Unlock() + return nil + } + day, ok, err := a.oldestDay(ctx, protect) + if err != nil { + return err + } + if !ok { + a.mu.Lock() + a.status.State = "over_cap" + a.mu.Unlock() + return nil + } + if err := a.deleteRunIDs(ctx, day.ids); err != nil { + return err + } + if err := a.deleteObservationsBetween(ctx, day.start, day.end, protect); err != nil { + return err + } + if err := a.collectGarbage(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) deleteRunsBefore(ctx context.Context, cutoff, protect int64, column string) error { + for { + day, ok, err := a.oldestDay(ctx, protect) + if err != nil || !ok { + return err + } + var ids []string + query := `SELECT run_id FROM fa_runs WHERE run_id IN (` + placeholders(len(day.ids)) + `) AND ` + column + `=? AND origin_ms 0 { + n := forecastPruneBatch + if n > len(ids) { + n = len(ids) + } + chunk := ids[:n] + ids = ids[n:] + if err := a.beforeWrite(); err != nil { + return err + } + tx, err := a.db.BeginTx(ctx, nil) + if err != nil { + return err + } + for _, table := range []string{"fa_series", "fa_run_states", "fa_run_weather", "fa_runs"} { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE run_id IN (`+placeholders(len(chunk))+`)`, stringsToArgs(chunk)...); err != nil { + tx.Rollback() + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch("fa_runs", len(chunk)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } + return nil +} + +func (a *ForecastLongArchive) deleteWhereIDs(ctx context.Context, table, column string, ids []string) error { + for len(ids) > 0 { + n := forecastPruneBatch + if n > len(ids) { + n = len(ids) + } + chunk := ids[:n] + ids = ids[n:] + if _, err := a.db.ExecContext(ctx, `DELETE FROM `+table+` WHERE `+column+` IN (`+placeholders(len(chunk))+`)`, stringsToArgs(chunk)...); err != nil { + return err + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch(table, len(chunk)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } + return nil +} + +func (a *ForecastLongArchive) deleteObservationsBefore(ctx context.Context, cutoff, protect int64) error { + limit := cutoff + if protect < limit { + limit = protect + } + return a.deleteObservationsBetween(ctx, 0, limit, protect) +} + +func (a *ForecastLongArchive) deleteObservationsBetween(ctx context.Context, start, end, protect int64) error { + if end > protect { + end = protect + } + if end <= start && start != 0 { + return nil + } + for { + query := `DELETE FROM fa_observations_15m WHERE (start_ms, end_ms, config_version) IN ( + SELECT start_ms, end_ms, config_version FROM fa_observations_15m WHERE end_ms=? LIMIT ?)` + res, err := a.db.ExecContext(ctx, query, end, start, forecastPruneBatch) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return nil + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch("fa_observations_15m", int(n)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) collectGarbage(ctx context.Context) error { + if err := a.deleteOrphans(ctx, `DELETE FROM fa_weather WHERE (source, available_at_ms, start_ms) IN ( + SELECT source, available_at_ms, start_ms FROM fa_weather w + WHERE NOT EXISTS (SELECT 1 FROM fa_run_weather r WHERE r.available_at_ms=w.available_at_ms) + LIMIT ?)`, "fa_weather"); err != nil { + return err + } + return a.deleteOrphans(ctx, `DELETE FROM fa_model_states WHERE rowid IN ( + SELECT rowid FROM fa_model_states WHERE id NOT IN (SELECT state_id FROM fa_run_states) LIMIT ?)`, "fa_model_states") +} + +func (a *ForecastLongArchive) deleteOrphans(ctx context.Context, query, table string) error { + for { + res, err := a.db.ExecContext(ctx, query, forecastPruneBatch) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return nil + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch(table, int(n)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) incrementalVacuum(ctx context.Context) error { + // WAL holds the freed pages until a checkpoint, and incremental_vacuum + // can only give back pages that are already on the main freelist. + if _, err := a.db.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`); err != nil { + return err + } + deadline := time.Now().Add(forecastLongVacuumBudget) + for { + if err := ctx.Err(); err != nil { + return err + } + var free, page int64 + if err := a.db.QueryRowContext(ctx, `PRAGMA freelist_count`).Scan(&free); err != nil { + return err + } + if err := a.db.QueryRowContext(ctx, `PRAGMA page_size`).Scan(&page); err != nil { + return err + } + if free*page < forecastLongFreeTarget || time.Now().After(deadline) { + return nil + } + if _, err := a.db.ExecContext(ctx, `PRAGMA incremental_vacuum(256)`); err != nil { + return err + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) liveByteCount(ctx context.Context) (int64, error) { + inv, err := sqliteFileInventory(ctx, a.db, a.path) + if err != nil { + return 0, err + } + return inv.LiveBytes, nil +} + +func (a *ForecastLongArchive) recompute(ctx context.Context, src *Store, now time.Time, paused error) error { + var runs int64 + var oldest, newest sql.NullInt64 + if err := a.db.QueryRowContext(ctx, `SELECT COUNT(*), MIN(origin_ms), MAX(origin_ms) FROM fa_runs`).Scan(&runs, &oldest, &newest); err != nil { + return err + } + inv, err := sqliteFileInventory(ctx, a.db, a.path) + if err != nil { + return err + } + cursor, err := a.issueCursor(ctx) + if err != nil { + return err + } + lag, err := src.forecastCopyLag(ctx, cursor, now.UnixMilli()) + if err != nil { + return err + } + p50, p95, gz, err := a.issueSizePercentiles(ctx) + if err != nil { + return err + } + oldestMS := int64(0) + if oldest.Valid { + oldestMS = oldest.Int64 + } + coverage7 := a.hourCoverage(ctx, now, 7*24*time.Hour, oldestMS) + coverage30 := a.hourCoverage(ctx, now, ForecastLongFullDetail, oldestMS) + coverageAll := a.hourCoverage(ctx, now, now.Sub(time.UnixMilli(oldestMS))+time.Hour, oldestMS) + truth := a.truthCoverage(ctx, now, oldestMS) + a.mu.Lock() + lastErr := a.status.LastError + lastCopy := a.status.LastCopyMS + state := "ok" + if a.rebuilt { + state = "rebuilt" + } + if a.status.State == "over_cap" { + state = "over_cap" + } + if paused != nil { + state = "paused_disk_full" + lastErr = paused.Error() + } + a.status = ForecastLongArchiveStatus{ + State: state, LiveBytes: inv.LiveBytes, FileBytes: inv.FileBytes, WALBytes: inv.WALBytes, + CapBytes: a.maxBytes, OldestOriginMS: oldestMS, NewestOriginMS: newest.Int64, Runs: runs, + Coverage7d: coverage7, Coverage30d: coverage30, CoverageAll: coverageAll, + TruthCoverage30d: truth, CopyLagMS: lag, LastCopyMS: lastCopy, LastError: lastErr, + IssueBytesP50: p50, IssueBytesP95: p95, IssueGzipBytesP50: gz, + } + a.mu.Unlock() + a.publish() + return nil +} + +func (a *ForecastLongArchive) hourCoverage(ctx context.Context, now time.Time, window time.Duration, oldest int64) float64 { + if oldest <= 0 { + return 0 + } + start := now.Add(-window).UnixMilli() + if oldest > start { + start = oldest + } + start = time.UnixMilli(start).UTC().Truncate(time.Hour).UnixMilli() + end := now.UTC().Truncate(time.Hour).UnixMilli() + if end < start { + return 0 + } + span := (end-start)/int64(time.Hour/time.Millisecond) + 1 + if span <= 0 { + return 0 + } + var present int64 + hour := int64(time.Hour / time.Millisecond) + if err := a.db.QueryRowContext(ctx, `SELECT COUNT(DISTINCT origin_ms/3600000) FROM fa_runs + WHERE origin_ms>=? AND origin_ms span { + present = span + } + return float64(present) / float64(span) +} + +func (a *ForecastLongArchive) truthCoverage(ctx context.Context, now time.Time, oldest int64) float64 { + if oldest <= 0 { + return 0 + } + end := now.UTC().Truncate(15 * time.Minute).UnixMilli() + start := now.Add(-ForecastLongFullDetail).UnixMilli() + if oldest > start { + start = oldest + } + start = time.UnixMilli(start).UTC().Truncate(15 * time.Minute).UnixMilli() + if end <= start { + return 0 + } + span := (end - start) / forecastLongQuarterMS + if span <= 0 { + return 0 + } + var known int64 + if err := a.db.QueryRowContext(ctx, `SELECT COUNT(DISTINCT start_ms) FROM fa_observations_15m + WHERE load_known=1 AND start_ms>=? AND start_ms span { + known = span + } + return float64(known) / float64(span) +} + +func (a *ForecastLongArchive) issueSizePercentiles(ctx context.Context) (p50, p95, gzipP50 int64, err error) { + rows, err := a.db.QueryContext(ctx, `SELECT json_bytes, gzip_bytes FROM fa_runs WHERE json_bytes>0`) + if err != nil { + return 0, 0, 0, err + } + defer rows.Close() + var jsonBytes, gzipBytes []int64 + for rows.Next() { + var j, g int64 + if err := rows.Scan(&j, &g); err != nil { + return 0, 0, 0, err + } + jsonBytes = append(jsonBytes, j) + gzipBytes = append(gzipBytes, g) + } + if err := rows.Err(); err != nil { + return 0, 0, 0, err + } + sort.Slice(jsonBytes, func(i, j int) bool { return jsonBytes[i] < jsonBytes[j] }) + sort.Slice(gzipBytes, func(i, j int) bool { return gzipBytes[i] < gzipBytes[j] }) + return percentile(jsonBytes, 0.50), percentile(jsonBytes, 0.95), percentile(gzipBytes, 0.50), nil +} + +func percentile(sorted []int64, p float64) int64 { + if len(sorted) == 0 { + return 0 + } + i := int(math.Ceil(p*float64(len(sorted)))) - 1 + if i < 0 { + i = 0 + } + if i >= len(sorted) { + i = len(sorted) - 1 + } + return sorted[i] +} + +func stringsToArgs(ids []string) []any { + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + return args +} + +func errorsIsNoRows(err error) bool { + return err == sql.ErrNoRows +} diff --git a/go/internal/state/snapshot_state_test.go b/go/internal/state/snapshot_state_test.go index 0dc428884..c10af5628 100644 --- a/go/internal/state/snapshot_state_test.go +++ b/go/internal/state/snapshot_state_test.go @@ -172,3 +172,27 @@ func TestMarkerSkipThenBackgroundVerifyHealsNextBoot(t *testing.T) { t.Errorf("precious row not restored: %q err=%v", v, err) } } + +func TestSnapshotOmitsForecastLongArchive(t *testing.T) { + dir := t.TempDir() + st, err := Open(filepath.Join(dir, "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := st.OpenForecastLongArchive(context.Background()); err != nil { + t.Fatal(err) + } + if err := st.SnapshotState(); err != nil { + t.Fatal(err) + } + snap, err := openRaw(filepath.Join(dir, "state.db.snapshot")) + if err != nil { + t.Fatal(err) + } + defer snap.Close() + var n int + if err := snap.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'fa_%'`).Scan(&n); err != nil || n != 0 { + t.Fatalf("quick snapshot contains %d forecast-archive tables, err=%v", n, err) + } +} diff --git a/go/internal/state/storage_inventory.go b/go/internal/state/storage_inventory.go index 54d57e365..da6ebe576 100644 --- a/go/internal/state/storage_inventory.go +++ b/go/internal/state/storage_inventory.go @@ -25,10 +25,13 @@ type SQLiteFileInventory struct { SHMBytes int64 `json:"shm_bytes"` } -// SQLiteInventory reports the two SQLite files that Core owns today. +// SQLiteInventory reports the SQLite files Core has open. ForecastArchive is +// set only after the long-term archive has been opened; the inventory does +// not open it. type SQLiteInventory struct { - State SQLiteFileInventory `json:"state"` - Cache SQLiteFileInventory `json:"cache"` + State SQLiteFileInventory `json:"state"` + Cache SQLiteFileInventory `json:"cache"` + ForecastArchive *SQLiteFileInventory `json:"forecast_archive,omitempty"` } // SQLiteInventory reads page metadata for state.db and cache.db. It does not @@ -50,7 +53,18 @@ func (s *Store) SQLiteInventory(ctx context.Context) (SQLiteInventory, error) { return SQLiteInventory{}, fmt.Errorf("cache inventory: %w", err) } } - return SQLiteInventory{State: stateDB, Cache: cacheDB}, nil + out := SQLiteInventory{State: stateDB, Cache: cacheDB} + s.forecastLongMu.Lock() + arch := s.forecastLong + s.forecastLongMu.Unlock() + if arch != nil { + archiveDB, err := sqliteFileInventory(ctx, arch.db, arch.path) + if err != nil { + return SQLiteInventory{}, fmt.Errorf("forecast archive inventory: %w", err) + } + out.ForecastArchive = &archiveDB + } + return out, nil } func sqliteFileInventory(ctx context.Context, db *sql.DB, path string) (SQLiteFileInventory, error) { diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 5429d2518..75658f906 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -36,12 +36,14 @@ const ( LegacyReleaseMarker = 4 ) -// Store owns three SQLite databases: +// Store owns three SQLite databases opened at startup: // - history: samples, hourly summaries, energy ledger and dashboard history // - db: state.db configuration, devices and learned state // - cache: cache.db prices and forecasts; savings history needs past prices // -// See heal.go for the boot-time integrity gate that populates healEvents. +// forecast-archive.db is a fourth file, opened later by the forecast copier. +// It is not on the boot path. See heal.go for the boot-time integrity gate +// that populates healEvents. type Store struct { aggregateHistory atomic.Bool history *sql.DB @@ -98,6 +100,12 @@ type Store struct { deviceWriteMu sync.Mutex deviceCacheMu sync.RWMutex deviceCache map[string]Device + + // forecastLong is the long-term forecast archive. It stays nil until the + // copier opens it, so a slow or damaged analytics file cannot delay boot. + forecastLongMu sync.Mutex + forecastLong *ForecastLongArchive + forecastLongStatus *ForecastLongArchiveStatus } // Open initializes (or creates) the precious state.db at path plus the @@ -308,6 +316,7 @@ func (s *Store) Close() error { if s.history != nil { err = errors.Join(err, s.history.Close()) } + err = errors.Join(err, s.closeForecastLongArchive()) if s.cache != nil { err = errors.Join(err, s.cache.Close()) } From 4e8a9871c7f913078399ef01e9dcedd7cbaf7781 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 09:53:36 +0100 Subject: [PATCH 2/9] fix(forecast): store each run's exact weather rows in the archive The archive linked a run only to the fetch times it used and, on load, returned every row of those fetches that overlapped the series span. After a partial weather refresh one issue holds rows from two fetches, so another run's row for the same slot came back too and the issue failed validation. Rows outside the span were dropped. fa_run_weather now holds one row per run and fetch, keyed by run, source and fetch time, with the run's slot starts packed as uvarint gaps. LoadIssue reads exactly those rows, all of them, in start order. The fa_runs weather_avail column is gone. The fa-v1 schema changes in place; no box has the file yet. Garbage collection probed fa_run_weather by fetch time with no index, so every candidate row scanned all links. A new index on (source, available_at_ms) serves the weather probe, and the model-state probe uses fa_run_states_ref. A test checks both query plans. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Fredrik Ahlgren --- go/internal/state/forecast_long_archive.go | 168 ++++++------ .../forecast_long_archive_weather_test.go | 241 ++++++++++++++++++ go/internal/state/forecast_long_codec.go | 40 +++ go/internal/state/forecast_long_maintain.go | 23 +- 4 files changed, 393 insertions(+), 79 deletions(-) create mode 100644 go/internal/state/forecast_long_archive_weather_test.go diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go index 2e44c6b7e..4bb1b3060 100644 --- a/go/internal/state/forecast_long_archive.go +++ b/go/internal/state/forecast_long_archive.go @@ -12,7 +12,7 @@ import ( "math" "os" "path/filepath" - "strings" + "sort" "sync" "time" @@ -312,7 +312,6 @@ CREATE TABLE IF NOT EXISTS fa_runs( has_location INTEGER NOT NULL DEFAULT 0, latitude REAL, longitude REAL, models TEXT NOT NULL, occupancy BLOB, occupancy_start_ms INTEGER, occupancy_n INTEGER, - weather_avail TEXT, input_hash TEXT NOT NULL, thinned INTEGER NOT NULL DEFAULT 0, json_bytes INTEGER NOT NULL DEFAULT 0, @@ -338,9 +337,11 @@ CREATE TABLE IF NOT EXISTS fa_weather( PRIMARY KEY(source, available_at_ms, start_ms) ) STRICT, WITHOUT ROWID; CREATE TABLE IF NOT EXISTS fa_run_weather( - run_id TEXT NOT NULL, available_at_ms INTEGER NOT NULL, - PRIMARY KEY(run_id, available_at_ms) + run_id TEXT NOT NULL, source TEXT NOT NULL, available_at_ms INTEGER NOT NULL, + starts BLOB NOT NULL, + PRIMARY KEY(run_id, source, available_at_ms) ) STRICT, WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS fa_run_weather_fetch ON fa_run_weather(source, available_at_ms); CREATE TABLE IF NOT EXISTS fa_observations_15m( start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, config_version TEXT NOT NULL, available_at_ms INTEGER NOT NULL, pv_w REAL NOT NULL, load_w REAL NOT NULL, @@ -536,7 +537,7 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes } } if err := insertForecastWeather(ctx, tx, issue); err != nil { - if !strings.Contains(err.Error(), "is immutable") { + if !errors.Is(err, errForecastWeatherConflict) { return err } a.warn("weather", err) @@ -545,7 +546,6 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes if err != nil { return err } - avail := forecastWeatherAvail(issue.Weather) modelsJSON, err := json.Marshal(issue.Models) if err != nil { return err @@ -566,11 +566,11 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes _, err = tx.ExecContext(ctx, `INSERT INTO fa_runs( run_id, decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, site_id, learning_revision, timezone, has_location, latitude, longitude, - models, occupancy, occupancy_start_ms, occupancy_n, weather_avail, input_hash, - thinned, json_bytes, gzip_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0,?,?)`, + models, occupancy, occupancy_start_ms, occupancy_n, input_hash, + thinned, json_bytes, gzip_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0,?,?)`, issue.ID, issue.DecisionID, issue.OriginMS, issue.IssuedAtMS, issue.LatestInputMS, issue.ConfigVersion, siteID, learning, tz, hasLoc, lat, lon, string(modelsJSON), nullableBlob(occ), nullMS(occStart, occN > 0), nullCount(occN), - nullText(avail), hash, item.JSONBytes, item.GzipBytes) + hash, item.JSONBytes, item.GzipBytes) if err != nil { return err } @@ -596,13 +596,6 @@ func nullCount(n int) any { return n } -func nullText(s string) any { - if s == "" { - return nil - } - return s -} - func nullableBlob(b []byte) any { if len(b) == 0 { return nil @@ -878,35 +871,30 @@ func bytesEqualNil(a, b []byte) bool { return true } -func forecastWeatherAvail(rows []forecasting.Weather) string { - if len(rows) == 0 { - return "" - } - seen := map[int64]bool{} - var ids []int64 - for _, row := range rows { - if seen[row.AvailableAtMS] { - continue - } - seen[row.AvailableAtMS] = true - ids = append(ids, row.AvailableAtMS) - } - data, err := json.Marshal(ids) - if err != nil { - return "" - } - return string(data) +// forecastWeatherFetch is one source's fetch, named by its available-at time. +type forecastWeatherFetch struct { + source string + availMS int64 } +var errForecastWeatherConflict = errors.New("forecast archive weather row is immutable") + +// insertForecastWeather shares weather rows across runs and records, per +// fetch, the exact slots this run used. A partial refresh leaves one issue +// with rows from two fetches, and another run may use other slots of the +// same fetch, so the fetch alone does not say which rows belong to a run. +// A stored row that differs from the issued one is kept and reported as +// errForecastWeatherConflict after every row is linked. func insertForecastWeather(ctx context.Context, tx *sql.Tx, issue forecasting.Issue) error { - seen := map[int64]bool{} + starts := map[forecastWeatherFetch][]int64{} + var order []forecastWeatherFetch + var conflict error for _, row := range issue.Weather { - if !seen[row.AvailableAtMS] { - seen[row.AvailableAtMS] = true - if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_run_weather(run_id, available_at_ms) VALUES(?,?)`, issue.ID, row.AvailableAtMS); err != nil { - return err - } + key := forecastWeatherFetch{source: row.Source, availMS: row.AvailableAtMS} + if _, ok := starts[key]; !ok { + order = append(order, key) } + starts[key] = append(starts[key], row.StartMS) res, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_weather( source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv) VALUES(?,?,?,?,?,?,?,?,?)`, row.Source, row.AvailableAtMS, row.StartMS, row.EndMS, @@ -924,12 +912,22 @@ func insertForecastWeather(ctx context.Context, tx *sql.Tx, issue forecasting.Is WHERE source=? AND available_at_ms=? AND start_ms=?`, row.Source, row.AvailableAtMS, row.StartMS).Scan(&end, &ghi, &cloud, &temp, &direct, &est); err != nil { return err } - if end != row.EndMS || !nullFloatEqual(ghi, row.GHIWm2) || !nullFloatEqual(cloud, row.CloudPct) || !nullFloatEqual(temp, row.TempC) || - !nullFloatEqual(direct, row.DirectPVW) || !nullFloatEqual(est, row.EstimatedPVW) { - return fmt.Errorf("forecast archive weather at %d is immutable", row.AvailableAtMS) + if conflict == nil && (end != row.EndMS || !nullFloatEqual(ghi, row.GHIWm2) || !nullFloatEqual(cloud, row.CloudPct) || + !nullFloatEqual(temp, row.TempC) || !nullFloatEqual(direct, row.DirectPVW) || !nullFloatEqual(est, row.EstimatedPVW)) { + conflict = fmt.Errorf("%w: %s fetch %d slot %d", errForecastWeatherConflict, row.Source, row.AvailableAtMS, row.StartMS) } } - return nil + for _, key := range order { + blob, err := encodeForecastStarts(starts[key]) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO fa_run_weather(run_id, source, available_at_ms, starts) VALUES(?,?,?,?)`, + issue.ID, key.source, key.availMS, blob); err != nil { + return err + } + } + return conflict } func nullFloat(v *float64) any { @@ -950,7 +948,7 @@ func nullFloatEqual(got sql.NullFloat64, want *float64) bool { // get an explicit cold-start band, because only the champion interval was kept. func (a *ForecastLongArchive) LoadIssue(ctx context.Context, runID string) (forecasting.Issue, error) { var issue forecasting.Issue - var siteID, learning, tz, modelsJSON, weatherAvail sql.NullString + var siteID, learning, tz, modelsJSON sql.NullString var lat, lon sql.NullFloat64 var hasLoc int var occ []byte @@ -958,10 +956,10 @@ func (a *ForecastLongArchive) LoadIssue(ctx context.Context, runID string) (fore var occN sql.NullInt64 err := a.db.QueryRowContext(ctx, `SELECT decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, site_id, learning_revision, timezone, has_location, latitude, longitude, models, - occupancy, occupancy_start_ms, occupancy_n, weather_avail + occupancy, occupancy_start_ms, occupancy_n FROM fa_runs WHERE run_id=?`, runID).Scan( &issue.DecisionID, &issue.OriginMS, &issue.IssuedAtMS, &issue.LatestInputMS, &issue.ConfigVersion, - &siteID, &learning, &tz, &hasLoc, &lat, &lon, &modelsJSON, &occ, &occStart, &occN, &weatherAvail) + &siteID, &learning, &tz, &hasLoc, &lat, &lon, &modelsJSON, &occ, &occStart, &occN) if err != nil { return forecasting.Issue{}, err } @@ -997,18 +995,7 @@ func (a *ForecastLongArchive) LoadIssue(ctx context.Context, runID string) (fore if err != nil { return forecasting.Issue{}, err } - var spanStart, spanEnd int64 - for _, series := range issue.Series { - for _, p := range series.Points { - if spanStart == 0 || p.StartMS < spanStart { - spanStart = p.StartMS - } - if p.EndMS > spanEnd { - spanEnd = p.EndMS - } - } - } - if issue.Weather, err = a.loadWeather(ctx, weatherAvail.String, spanStart, spanEnd); err != nil { + if issue.Weather, err = a.loadWeather(ctx, runID); err != nil { return forecasting.Issue{}, err } if err := issue.Validate(); err != nil { @@ -1166,25 +1153,60 @@ func decodeStoredSeries(name, version string, layout seriesLayout, pvB, loadB, f return forecasting.Series{Name: name, ModelVersion: version, Points: points}, nil } -func (a *ForecastLongArchive) loadWeather(ctx context.Context, availJSON string, spanStart, spanEnd int64) ([]forecasting.Weather, error) { - if availJSON == "" || spanEnd <= spanStart { - return nil, nil +// loadWeather returns exactly the weather rows the run was issued with, in +// start order. +func (a *ForecastLongArchive) loadWeather(ctx context.Context, runID string) ([]forecasting.Weather, error) { + rows, err := a.db.QueryContext(ctx, `SELECT source, available_at_ms, starts FROM fa_run_weather WHERE run_id=?`, runID) + if err != nil { + return nil, err + } + type fetch struct { + key forecastWeatherFetch + starts []int64 + } + var fetches []fetch + for rows.Next() { + var f fetch + var blob []byte + if err := rows.Scan(&f.key.source, &f.key.availMS, &blob); err != nil { + rows.Close() + return nil, err + } + if f.starts, err = decodeForecastStarts(blob); err != nil { + rows.Close() + return nil, err + } + fetches = append(fetches, f) } - var ids []int64 - if err := json.Unmarshal([]byte(availJSON), &ids); err != nil { + if err := rows.Close(); err != nil { return nil, err } - if len(ids) == 0 { - return nil, nil + if err := rows.Err(); err != nil { + return nil, err } - query := `SELECT source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv FROM fa_weather - WHERE available_at_ms IN (` + placeholders(len(ids)) + `) AND start_ms < ? AND end_ms > ? ORDER BY start_ms, source` - args := make([]any, 0, len(ids)+2) - for _, id := range ids { - args = append(args, id) + var out []forecasting.Weather + for _, f := range fetches { + got, err := a.fetchWeatherRows(ctx, f.key, f.starts) + if err != nil { + return nil, err + } + if len(got) != len(f.starts) { + return nil, fmt.Errorf("forecast archive run %s is missing %d weather rows", runID, len(f.starts)-len(got)) + } + out = append(out, got...) } - args = append(args, spanEnd, spanStart) - rows, err := a.db.QueryContext(ctx, query, args...) + sort.Slice(out, func(i, j int) bool { return out[i].StartMS < out[j].StartMS }) + return out, nil +} + +func (a *ForecastLongArchive) fetchWeatherRows(ctx context.Context, key forecastWeatherFetch, starts []int64) ([]forecasting.Weather, error) { + args := make([]any, 0, len(starts)+2) + args = append(args, key.source, key.availMS) + for _, start := range starts { + args = append(args, start) + } + rows, err := a.db.QueryContext(ctx, `SELECT source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv FROM fa_weather + WHERE source=? AND available_at_ms=? AND start_ms IN (`+placeholders(len(starts))+`)`, args...) if err != nil { return nil, err } diff --git a/go/internal/state/forecast_long_archive_weather_test.go b/go/internal/state/forecast_long_archive_weather_test.go new file mode 100644 index 000000000..da3c704c3 --- /dev/null +++ b/go/internal/state/forecast_long_archive_weather_test.go @@ -0,0 +1,241 @@ +package state + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func longWeather(source string, start, avail int64) forecasting.Weather { + ghi := float64(start%100000) + float64(avail%1000)/10 + cloud := float64(avail % 100) + temp := -3.5 + return forecasting.Weather{ + StartMS: start, EndMS: start + time.Hour.Milliseconds(), AvailableAtMS: avail, + Source: source, GHIWm2: &ghi, CloudPct: &cloud, TempC: &temp, + } +} + +func countRows(t *testing.T, arch *ForecastLongArchive, query string, args ...any) int { + t.Helper() + var n int + if err := arch.db.QueryRow(query, args...).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// A partial weather refresh leaves one issue with rows from two fetches. +// Another run that used the older fetch for the same slot must not bring it +// back into this one. +func TestForecastLongMixedFetchWeatherRoundTrip(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + fetchA := s - hour + fetchB := s + 10*time.Minute.Milliseconds() + + first := longIssue("first", origin) + first.Weather = []forecasting.Weather{ + longWeather("open_meteo", s, fetchA), longWeather("open_meteo", s+hour, fetchA), longWeather("open_meteo", s+2*hour, fetchA), + } + second := longIssue("second", origin.Add(15*time.Minute)) + second.Weather = []forecasting.Weather{ + longWeather("open_meteo", s, fetchB), longWeather("open_meteo", s+hour, fetchA), longWeather("open_meteo", s+2*hour, fetchA), + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: first}, {Issue: second}}); err != nil { + t.Fatal(err) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_weather`); n != 4 { + t.Fatalf("weather rows = %d, want 4 shared rows", n) + } + for _, want := range []forecasting.Issue{first, second} { + got, err := arch.LoadIssue(ctx, want.ID) + if err != nil { + t.Fatalf("%s: %v", want.ID, err) + } + if err := got.Validate(); err != nil { + t.Fatalf("%s: %v", want.ID, err) + } + if !reflect.DeepEqual(got.Weather, want.Weather) { + t.Fatalf("%s weather = %+v, want %+v", want.ID, got.Weather, want.Weather) + } + } + // The same content again is a no-op; a changed weather row is refused. + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: second}}); err != nil { + t.Fatalf("identical re-archive: %v", err) + } + changed := second + changed.Weather = append([]forecasting.Weather(nil), second.Weather...) + changed.Weather[0] = longWeather("open_meteo", s, fetchA) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: changed}}); !errors.Is(err, errForecastArchiveImmutable) { + t.Fatalf("changed weather err = %v, want immutable", err) + } + got, err := arch.LoadIssue(ctx, "second") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Weather, second.Weather) { + t.Fatalf("stored weather changed to %+v", got.Weather) + } +} + +// Every issued weather row comes back, including rows before the origin and +// past the last series point. +func TestForecastLongWeatherOutsideSeriesSpanRoundTrips(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + issue := longIssue("span", origin) + issue.Weather = []forecasting.Weather{ + longWeather("open_meteo", s-3*hour, s-4*hour), + longWeather("open_meteo", s, s-hour), + longWeather("forecast_solar", s+hour, s-2*hour), + longWeather("open_meteo", s+30*hour, s-hour), + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + got, err := arch.LoadIssue(ctx, "span") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Weather, issue.Weather) { + t.Fatalf("weather = %+v, want %+v", got.Weather, issue.Weather) + } +} + +// The plans are checked, not timed: a scan of a reference table inside the +// correlated subquery is what made garbage collection quadratic. +func TestForecastLongGarbagePlansUseIndexes(t *testing.T) { + _, arch := openLongArchive(t) + for _, tc := range []struct { + query, ref, index string + }{ + {forecastLongWeatherGC, "fa_run_weather", "fa_run_weather_fetch"}, + {forecastLongStateGC, "fa_run_states", "fa_run_states_ref"}, + } { + rows, err := arch.db.Query(`EXPLAIN QUERY PLAN `+tc.query, forecastPruneBatch) + if err != nil { + t.Fatal(err) + } + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + searched := false + for _, detail := range plan { + if strings.HasPrefix(detail, "SCAN "+tc.ref) { + t.Fatalf("%s is scanned: %q", tc.ref, plan) + } + if strings.HasPrefix(detail, "SEARCH "+tc.ref+" ") && strings.Contains(detail, "INDEX "+tc.index+" ") { + searched = true + } + } + if !searched { + t.Fatalf("%s is not searched through %s: %q", tc.ref, tc.index, plan) + } + } +} + +func TestForecastLongGarbageKeepsLinkedWeather(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + fetchA, fetchB, fetchC := s-3*hour, s-2*hour, s-hour + + only := longIssue("only-a", origin) + only.Weather = []forecasting.Weather{longWeather("open_meteo", s, fetchA), longWeather("open_meteo", s+hour, fetchA)} + mixed := longIssue("mixed", origin.Add(15*time.Minute)) + mixed.Weather = []forecasting.Weather{longWeather("open_meteo", s, fetchB), longWeather("open_meteo", s+hour, fetchC)} + shared := longIssue("shared", origin.Add(30*time.Minute)) + shared.Weather = []forecasting.Weather{longWeather("open_meteo", s, fetchB), longWeather("open_meteo", s+2*hour, fetchC)} + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: only}, {Issue: mixed}, {Issue: shared}}); err != nil { + t.Fatal(err) + } + if err := arch.deleteRunIDs(ctx, []string{"only-a", "mixed"}); err != nil { + t.Fatal(err) + } + if err := arch.collectGarbage(ctx); err != nil { + t.Fatal(err) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_weather WHERE available_at_ms=?`, fetchA); n != 0 { + t.Fatalf("orphan fetch rows = %d, want 0", n) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_run_weather`); n != 2 { + t.Fatalf("weather links = %d, want the kept run's two fetches", n) + } + got, err := arch.LoadIssue(ctx, "shared") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Weather, shared.Weather) { + t.Fatalf("kept weather = %+v, want %+v", got.Weather, shared.Weather) + } + if err := arch.deleteRunIDs(ctx, []string{"shared"}); err != nil { + t.Fatal(err) + } + if err := arch.collectGarbage(ctx); err != nil { + t.Fatal(err) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_weather`); n != 0 { + t.Fatalf("weather rows after the last run = %d, want 0", n) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_model_states`); n != 0 { + t.Fatalf("model states after the last run = %d, want 0", n) + } +} + +func TestForecastLongWeatherStartsCodec(t *testing.T) { + hour := time.Hour.Milliseconds() + s := longOrigin().UnixMilli() + starts := []int64{s + 2*hour, s, s + hour, s + 47*hour} + blob, err := encodeForecastStarts(starts) + if err != nil { + t.Fatal(err) + } + got, err := decodeForecastStarts(blob) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []int64{s, s + hour, s + 2*hour, s + 47*hour}) { + t.Fatalf("starts = %v", got) + } + day := make([]int64, 48) + for i := range day { + day[i] = s + int64(i)*hour + } + if blob, err = encodeForecastStarts(day); err != nil || len(blob) > 200 { + t.Fatalf("hourly day = %d bytes err=%v", len(blob), err) + } + for _, bad := range [][]int64{nil, {s, s}, {0}} { + if _, err := encodeForecastStarts(bad); err == nil { + t.Fatalf("%v was accepted", bad) + } + } + for _, bad := range [][]byte{nil, {0x80}, {0x01, 0x00}} { + if _, err := decodeForecastStarts(bad); err == nil { + t.Fatalf("%x was accepted", bad) + } + } +} diff --git a/go/internal/state/forecast_long_codec.go b/go/internal/state/forecast_long_codec.go index ef5fcf7f9..bcb348bfa 100644 --- a/go/internal/state/forecast_long_codec.go +++ b/go/internal/state/forecast_long_codec.go @@ -245,6 +245,46 @@ func decodeBandStats(b []byte, n int) (samples, days [][3]int, err error) { return samples, days, nil } +// encodeForecastStarts packs the weather slot starts one run used from one +// fetch as uvarints: the first start, then each gap to the next. An hourly +// day of 48 slots takes about 200 bytes. +func encodeForecastStarts(starts []int64) ([]byte, error) { + if len(starts) == 0 || len(starts) > forecastLongMaxVector { + return nil, errors.New("forecast archive weather slot list length is invalid") + } + sorted := append([]int64(nil), starts...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + if sorted[0] <= 0 { + return nil, errors.New("forecast archive weather slot start is invalid") + } + out := binary.AppendUvarint(nil, uint64(sorted[0])) + for i := 1; i < len(sorted); i++ { + if sorted[i] == sorted[i-1] { + return nil, errors.New("forecast archive weather slot repeats") + } + out = binary.AppendUvarint(out, uint64(sorted[i]-sorted[i-1])) + } + return out, nil +} + +func decodeForecastStarts(b []byte) ([]int64, error) { + var out []int64 + var last uint64 + for len(b) > 0 { + v, n := binary.Uvarint(b) + if n <= 0 || v == 0 || v > math.MaxInt64-last || len(out) >= forecastLongMaxVector { + return nil, errors.New("forecast archive weather slot list is invalid") + } + last += v + out = append(out, int64(last)) + b = b[n:] + } + if len(out) == 0 { + return nil, errors.New("forecast archive weather slot list is empty") + } + return out, nil +} + // encodeOccupancy stores a bit per quarter when the rows are contiguous, and // the available-at time of each quarter so a restored issue still validates. // Gaps use an explicit record. An empty slice returns a nil blob. diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go index 53657200f..0c520e00b 100644 --- a/go/internal/state/forecast_long_maintain.go +++ b/go/internal/state/forecast_long_maintain.go @@ -413,15 +413,26 @@ func (a *ForecastLongArchive) deleteObservationsBetween(ctx context.Context, sta } } -func (a *ForecastLongArchive) collectGarbage(ctx context.Context) error { - if err := a.deleteOrphans(ctx, `DELETE FROM fa_weather WHERE (source, available_at_ms, start_ms) IN ( +// The orphan deletes probe the reference tables through fa_run_weather_fetch +// and fa_run_states_ref, so each candidate row costs one index lookup rather +// than a scan of every reference. A fetch's weather rows stay while any run +// still links that fetch. +const ( + forecastLongWeatherGC = `DELETE FROM fa_weather WHERE (source, available_at_ms, start_ms) IN ( SELECT source, available_at_ms, start_ms FROM fa_weather w - WHERE NOT EXISTS (SELECT 1 FROM fa_run_weather r WHERE r.available_at_ms=w.available_at_ms) - LIMIT ?)`, "fa_weather"); err != nil { + WHERE NOT EXISTS (SELECT 1 FROM fa_run_weather WHERE fa_run_weather.source=w.source AND fa_run_weather.available_at_ms=w.available_at_ms) + LIMIT ?)` + forecastLongStateGC = `DELETE FROM fa_model_states WHERE rowid IN ( + SELECT rowid FROM fa_model_states m + WHERE NOT EXISTS (SELECT 1 FROM fa_run_states WHERE fa_run_states.state_id=m.id) + LIMIT ?)` +) + +func (a *ForecastLongArchive) collectGarbage(ctx context.Context) error { + if err := a.deleteOrphans(ctx, forecastLongWeatherGC, "fa_weather"); err != nil { return err } - return a.deleteOrphans(ctx, `DELETE FROM fa_model_states WHERE rowid IN ( - SELECT rowid FROM fa_model_states WHERE id NOT IN (SELECT state_id FROM fa_run_states) LIMIT ?)`, "fa_model_states") + return a.deleteOrphans(ctx, forecastLongStateGC, "fa_model_states") } func (a *ForecastLongArchive) deleteOrphans(ctx context.Context, query, table string) error { From c6698f0cc4af6d5a0560bdf01ec3f1e804259bb9 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 10:07:48 +0100 Subject: [PATCH 3/9] fix(forecast): thin whole hours and days, and keep pruning below the floor Each thinning pass split the hour and the local day at the cutoff, so the rest of a split hour kept a second run and a split day kept a second set of model states. In half-hour zones every hour at a day boundary kept two runs. Below the disk floor the deletes refused to run, so nothing was thinned, expired or pruned, and the cap waited for the daily pass. Thinning now works on complete UTC hours. It ranks every run of the hour, keeps a thinned survivor if there is one, and so drops a run copied late into a thinned hour. Model states stay on the first kept run of each complete local day that has them. A partial index finds unthinned runs. Deletes no longer check the floor. Below it, copying stays paused but maintenance runs, and the cap target drops by the missing free space, so the oldest days outside the newest 30 give that space back. Every Sync enforces the cap, and incremental vacuum ends with a checkpoint so the file shrinks at once. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Fredrik Ahlgren --- go/internal/state/forecast_long_archive.go | 21 +- .../state/forecast_long_archive_prune_test.go | 104 +++++++- .../state/forecast_long_archive_thin_test.go | 180 +++++++++++++ go/internal/state/forecast_long_copy.go | 39 +-- go/internal/state/forecast_long_maintain.go | 252 +++++++++--------- 5 files changed, 445 insertions(+), 151 deletions(-) diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go index 4bb1b3060..45fd6937b 100644 --- a/go/internal/state/forecast_long_archive.go +++ b/go/internal/state/forecast_long_archive.go @@ -35,7 +35,9 @@ const ( // around a size estimate, not a measurement from a box. ForecastLongMaxBytes = 512 << 20 // ForecastLongMinFreeBytes is the free-space floor on the data volume. - // Below it the copier pauses instead of filling the card. + // Below it the copier pauses instead of filling the card, and the + // archive deletes its oldest days outside the full-detail window until + // it has given back the shortfall. ForecastLongMinFreeBytes = 1 << 30 forecastLongVacuumPages = 256 @@ -319,6 +321,7 @@ CREATE TABLE IF NOT EXISTS fa_runs( ) STRICT; CREATE INDEX IF NOT EXISTS fa_runs_origin ON fa_runs(origin_ms); CREATE INDEX IF NOT EXISTS fa_runs_issued ON fa_runs(issued_at_ms); +CREATE INDEX IF NOT EXISTS fa_runs_unthinned ON fa_runs(origin_ms) WHERE thinned=0; CREATE TABLE IF NOT EXISTS fa_series( run_id TEXT NOT NULL, series TEXT NOT NULL, model_version TEXT NOT NULL, start_ms INTEGER NOT NULL, slot_ms INTEGER NOT NULL, n INTEGER NOT NULL, @@ -463,16 +466,22 @@ func (a *ForecastLongArchive) beforeWrite() error { return err } } - avail, err := forecastLongDiskAvail(filepath.Dir(a.path)) - if err != nil { - return nil - } - if avail < a.minFree { + if a.freeShortfall() > 0 { return errForecastArchiveDiskFull } return nil } +// freeShortfall is how far the data volume is below the free-space floor. +// A failed probe counts as no shortfall. +func (a *ForecastLongArchive) freeShortfall() int64 { + avail, err := forecastLongDiskAvail(filepath.Dir(a.path)) + if err != nil || avail >= a.minFree { + return 0 + } + return a.minFree - avail +} + var errForecastArchiveDiskFull = errors.New("forecast archive paused: data volume is below the free-space floor") var errForecastArchiveImmutable = errors.New("forecast long archive run is immutable") diff --git a/go/internal/state/forecast_long_archive_prune_test.go b/go/internal/state/forecast_long_archive_prune_test.go index f56041dad..bf99c3912 100644 --- a/go/internal/state/forecast_long_archive_prune_test.go +++ b/go/internal/state/forecast_long_archive_prune_test.go @@ -42,7 +42,7 @@ func TestForecastLongRetentionAndCap(t *testing.T) { if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: bulk}}); err != nil { t.Fatal(err) } - if err := arch.pruneToCap(ctx, now); err != nil { + if _, err := arch.pruneToCap(ctx, now); err != nil { t.Fatal(err) } assertRuns(t, arch, map[string]bool{"recent": true}) @@ -121,3 +121,105 @@ func assertRuns(t *testing.T, arch *ForecastLongArchive, want map[string]bool) { } } } + +func TestForecastLongBelowFloorStillThinsAndGivesSpaceBack(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Hour) + hour := now.Add(-40 * 24 * time.Hour) + later := longIssue("old-later", hour) + later.IssuedAtMS += 1000 + // The pad day is the oldest outside the 30 days. Deleting it gives back + // far more than the 64 KiB shortfall, so the thinned day stays. + pad := ForecastArchiveIssue{ + Issue: longIssue("pad", now.Add(-60*24*time.Hour)), + States: map[string]ForecastModelBlob{"pad-state": {Expanded: 2 << 20, Gzip: make([]byte, 2<<20)}}, + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{ + {Issue: longIssue("expired", now.Add(-800*24*time.Hour))}, pad, + {Issue: longIssue("old-keep", hour)}, {Issue: later}, + }); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastIssue(ctx, longIssue("fresh", now.Add(-time.Hour))); err != nil { + t.Fatal(err) + } + if _, err := arch.db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + t.Fatal(err) + } + before, err := regularFileSize(arch.path) + if err != nil { + t.Fatal(err) + } + restore := ForecastLongDiskAvailForTest(func(string) (int64, error) { return arch.minFree - 64<<10, nil }) + defer restore() + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + assertRuns(t, arch, map[string]bool{"old-keep": true}) + var thinned int + if err := arch.db.QueryRow(`SELECT thinned FROM fa_runs WHERE run_id='old-keep'`).Scan(&thinned); err != nil || thinned != 1 { + t.Fatalf("old-keep thinned = %d err=%v", thinned, err) + } + var pads int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_model_states WHERE id='pad-state'`).Scan(&pads); err != nil || pads != 0 { + t.Fatalf("pad state rows = %d err=%v", pads, err) + } + status, _ := store.ForecastLongArchiveStatus() + if status.State != "paused_disk_full" { + t.Fatalf("state = %q, want paused_disk_full", status.State) + } + var free, page int64 + if err := arch.db.QueryRow(`PRAGMA freelist_count`).Scan(&free); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`PRAGMA page_size`).Scan(&page); err != nil { + t.Fatal(err) + } + after, err := regularFileSize(arch.path) + if err != nil { + t.Fatal(err) + } + if free*page >= forecastLongFreeTarget || after > before-(1<<20) { + t.Fatalf("file %d bytes (was %d) with %d free pages; the pad pages were not given back", after, before, free) + } +} + +func TestForecastLongHourlySyncEnforcesCap(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + restore := ForecastLongDiskAvailForTest(func(string) (int64, error) { return 1 << 40, nil }) + defer restore() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Hour) + // The first copy runs the daily pass; the next is 24 hours away. + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{ + {Issue: longIssue("old", now.Add(-40*24*time.Hour))}, + {Issue: longIssue("recent", now.Add(-time.Hour))}, + }); err != nil { + t.Fatal(err) + } + arch.maxBytes = 1 + if err := arch.Sync(ctx, store, now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + assertRuns(t, arch, map[string]bool{"recent": true}) + if status, _ := store.ForecastLongArchiveStatus(); status.State != "over_cap" { + t.Fatalf("state = %q, want over_cap an hour after the daily pass", status.State) + } + arch.maxBytes = ForecastLongMaxBytes + if err := arch.Sync(ctx, store, now.Add(2*time.Hour)); err != nil { + t.Fatal(err) + } + if status, _ := store.ForecastLongArchiveStatus(); status.State != "ok" { + t.Fatalf("state = %q, want ok once the archive is under the cap", status.State) + } +} diff --git a/go/internal/state/forecast_long_archive_thin_test.go b/go/internal/state/forecast_long_archive_thin_test.go index 2124232ce..a9213dbe7 100644 --- a/go/internal/state/forecast_long_archive_thin_test.go +++ b/go/internal/state/forecast_long_archive_thin_test.go @@ -120,3 +120,183 @@ func TestForecastLongDeletesAreBatched(t *testing.T) { t.Fatalf("batches = %v, want a full batch of %d", batches, forecastPruneBatch) } } + +func TestForecastLongDailyPassesThinWholeHoursAndDays(t *testing.T) { + for _, tc := range []struct { + name, zone string + from, to time.Time + }{ + {"stockholm", "Europe/Stockholm", time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 14, 0, 0, 0, 0, time.UTC)}, + {"kolkata", "Asia/Kolkata", time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 14, 0, 0, 0, 0, time.UTC)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + thinDaily(t, tc.zone, tc.from, tc.to) + }) + } +} + +func TestForecastLongDailyPassesThinDSTDays(t *testing.T) { + for _, tc := range []struct { + name string + from, to time.Time + day time.Time + hours int + }{ + {"spring", time.Date(2026, 3, 28, 0, 0, 0, 0, time.UTC), time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC), time.Date(2026, 3, 29, 0, 0, 0, 0, time.UTC), 23}, + {"autumn", time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC), time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC), time.Date(2026, 10, 25, 0, 0, 0, 0, time.UTC), 25}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + arch := thinDaily(t, "Europe/Stockholm", tc.from, tc.to) + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatal(err) + } + start := time.Date(tc.day.Year(), tc.day.Month(), tc.day.Day(), 0, 0, 0, 0, loc) + end := start.AddDate(0, 0, 1) + var runs, holders int + if err := arch.db.QueryRow(`SELECT COUNT(*), COUNT(DISTINCT s.run_id) FROM fa_runs r + LEFT JOIN fa_run_states s ON s.run_id=r.run_id WHERE r.origin_ms>=? AND r.origin_ms= start && origin < end { + inDay = append(inDay, ids[origin]) + if holders[ids[origin]] { + held = append(held, ids[origin]) + } + } + } + if len(inDay) == 0 { + continue + } + if end <= cut { + if len(held) != 1 || held[0] != inDay[0] { + t.Fatalf("pass %s: complete day %s has state on %v, want only %s", + cutoff.Format(time.RFC3339), day.Format("2006-01-02"), held, inDay[0]) + } + } else if len(held) != len(inDay) { + t.Fatalf("pass %s: open day %s lost state references: %d of %d runs keep them", + cutoff.Format(time.RFC3339), day.Format("2006-01-02"), len(held), len(inDay)) + } + } + } + return arch +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go index 8858186ec..09f8b33bd 100644 --- a/go/internal/state/forecast_long_copy.go +++ b/go/internal/state/forecast_long_copy.go @@ -30,9 +30,10 @@ type forecastLongObsCursor struct { Config string `json:"config_version"` } -// Sync copies the 30-day issue and observation tables, then at most once a -// day thins, prunes and gives free pages back. Failures stay in the published -// status. A full disk pauses until the next call instead of retrying. +// Sync copies the 30-day issue and observation tables and holds the archive +// to its cap. Once a day it also thins, prunes expired runs and gives free +// pages back. Failures stay in the published status. Below the disk floor +// copying pauses until the next call, but the deletes still run. func (a *ForecastLongArchive) Sync(ctx context.Context, src *Store, now time.Time) error { if a == nil || src == nil { return errors.New("forecast archive: copy source unavailable") @@ -68,24 +69,26 @@ func (a *ForecastLongArchive) Sync(ctx context.Context, src *Store, now time.Tim } } } - if paused == nil { - due, err := a.maintenanceDue(ctx, now) - if err != nil { + due, err := a.maintenanceDue(ctx, now) + if err != nil { + a.fail("maintenance", err) + return err + } + if due { + err = a.maintain(ctx, now) + if err == nil { + err = a.setMeta(ctx, "last_maintain_ms", jsonInt(now.UnixMilli())) + } + } else { + err = a.enforceCap(ctx, now) + } + if err != nil { + if !forecastLongPaused(err) { a.fail("maintenance", err) return err } - if due { - if err := a.maintain(ctx, now); err != nil { - if forecastLongPaused(err) { - paused = err - } else { - a.fail("maintenance", err) - return err - } - } else if err := a.setMeta(ctx, "last_maintain_ms", jsonInt(now.UnixMilli())); err != nil { - a.fail("maintenance", err) - return err - } + if paused == nil { + paused = err } } if err := a.recompute(ctx, src, now, paused); err != nil { diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go index 0c520e00b..cd5371b2b 100644 --- a/go/internal/state/forecast_long_maintain.go +++ b/go/internal/state/forecast_long_maintain.go @@ -19,7 +19,7 @@ func (a *ForecastLongArchive) maintain(ctx context.Context, now time.Time) error if err := a.pruneExpired(ctx, now); err != nil { return err } - if err := a.pruneToCap(ctx, now); err != nil { + if _, err := a.pruneToCap(ctx, now); err != nil { return err } if err := a.collectGarbage(ctx); err != nil { @@ -28,6 +28,16 @@ func (a *ForecastLongArchive) maintain(ctx context.Context, now time.Time) error return a.incrementalVacuum(ctx) } +// enforceCap runs on every copy between the daily passes. Under the cap and +// above the disk floor it costs one page count. +func (a *ForecastLongArchive) enforceCap(ctx context.Context, now time.Time) error { + pruned, err := a.pruneToCap(ctx, now) + if err != nil || !pruned { + return err + } + return a.incrementalVacuum(ctx) +} + type forecastLongDay struct { key string start int64 @@ -35,28 +45,42 @@ type forecastLongDay struct { ids []string } +const forecastLongHourMS = int64(time.Hour / time.Millisecond) + +// thin keeps one run per complete UTC hour before the full-detail cutoff and +// model-state references on one run per complete local day. Hours and days +// count as complete when they end at or before the cutoff's whole hour, so a +// day in a half-hour zone waits for its last UTC hour. A day stays unmarked +// until it is complete, and a run copied late into a thinned hour loses to +// the survivor already there. func (a *ForecastLongArchive) thin(ctx context.Context, now time.Time) error { - cutoff := now.Add(-a.fullDetail).UnixMilli() + cut := now.Add(-a.fullDetail).UnixMilli() + cut -= cut % forecastLongHourMS + from := int64(math.MinInt64) for { if err := ctx.Err(); err != nil { return err } - day, ok, err := a.nextUnthinnedDay(ctx, cutoff) + origin, tz, ok, err := a.nextUnthinned(ctx, from, cut) if err != nil || !ok { return err } - losers, err := a.thinnedLosers(ctx, day.ids) + start, end, key := forecastLongLocalDay(origin, tz) + hoursEnd := (end + forecastLongHourMS - 1) / forecastLongHourMS * forecastLongHourMS + if hoursEnd > cut { + hoursEnd = cut + } + losers, err := a.hourLosers(ctx, start-start%forecastLongHourMS, hoursEnd) if err != nil { return err } if err := a.deleteRunIDs(ctx, losers); err != nil { return err } - if err := a.keepDailyState(ctx, day.ids, losers); err != nil { - return err - } - if err := a.markThinned(ctx, day.ids, losers); err != nil { - return err + if end <= cut { + if err := a.thinDay(ctx, start, end, key); err != nil { + return err + } } if err := a.collectGarbage(ctx); err != nil { return err @@ -64,48 +88,22 @@ func (a *ForecastLongArchive) thin(ctx context.Context, now time.Time) error { if err := pauseMaintenance(ctx); err != nil { return err } + from = end } } -func (a *ForecastLongArchive) nextUnthinnedDay(ctx context.Context, cutoff int64) (forecastLongDay, bool, error) { - rows, err := a.db.QueryContext(ctx, `SELECT run_id, origin_ms, timezone FROM fa_runs - WHERE thinned=0 AND origin_ms=? AND origin_ms 1` - args := make([]any, len(ids)) - for i, id := range ids { - args[i] = id - } - rows, err := a.db.QueryContext(ctx, query, args...) + FROM fa_runs WHERE origin_ms>=? AND origin_ms 1`, start, end) if err != nil { return nil, err } @@ -152,54 +145,51 @@ func (a *ForecastLongArchive) thinnedLosers(ctx context.Context, ids []string) ( return losers, rows.Err() } -func (a *ForecastLongArchive) keepDailyState(ctx context.Context, ids, losers []string) error { - drop := map[string]bool{} - for _, id := range losers { - drop[id] = true - } - var kept []string - for _, id := range ids { - if !drop[id] { - kept = append(kept, id) - } - } - if len(kept) <= 1 { - return nil - } - query := `SELECT run_id FROM fa_runs WHERE run_id IN (` + placeholders(len(kept)) + `) - ORDER BY origin_ms, issued_at_ms, run_id LIMIT 1` - args := make([]any, len(kept)) - for i, id := range kept { - args[i] = id - } - var first string - if err := a.db.QueryRowContext(ctx, query, args...).Scan(&first); err != nil { +// thinDay keeps model-state references on the first run of a complete local +// day that has any, drops them from the other runs, and marks the day thinned. +func (a *ForecastLongArchive) thinDay(ctx context.Context, start, end int64, key string) error { + rows, err := a.db.QueryContext(ctx, `SELECT run_id, origin_ms, timezone, thinned, + EXISTS(SELECT 1 FROM fa_run_states s WHERE s.run_id=fa_runs.run_id) + FROM fa_runs WHERE origin_ms>=? AND origin_ms 0 && live-short < target { + target = live - short + } + pruned := false + for live > target { if err := ctx.Err(); err != nil { - return err - } - live, err := a.liveByteCount(ctx) - if err != nil { - return err - } - if live <= a.maxBytes { - a.mu.Lock() - if a.status.State == "over_cap" { - a.status.State = "" - } - a.mu.Unlock() - return nil + return pruned, err } day, ok, err := a.oldestDay(ctx, protect) if err != nil { - return err + return pruned, err } if !ok { - a.mu.Lock() - a.status.State = "over_cap" - a.mu.Unlock() - return nil + break } if err := a.deleteRunIDs(ctx, day.ids); err != nil { - return err + return pruned, err } if err := a.deleteObservationsBetween(ctx, day.start, day.end, protect); err != nil { - return err + return pruned, err } if err := a.collectGarbage(ctx); err != nil { - return err + return pruned, err + } + pruned = true + if live, err = a.liveByteCount(ctx); err != nil { + return pruned, err } } + a.mu.Lock() + if live > a.maxBytes { + a.status.State = "over_cap" + } else if a.status.State == "over_cap" { + a.status.State = "" + } + a.mu.Unlock() + return pruned, nil } func (a *ForecastLongArchive) deleteRunsBefore(ctx context.Context, cutoff, protect int64, column string) error { @@ -323,6 +323,7 @@ func (a *ForecastLongArchive) oldestDay(ctx context.Context, protect int64) (for return day, true, rows.Err() } +// deleteRunIDs does not check the disk floor: deletes give space back. func (a *ForecastLongArchive) deleteRunIDs(ctx context.Context, ids []string) error { for len(ids) > 0 { n := forecastPruneBatch @@ -331,9 +332,6 @@ func (a *ForecastLongArchive) deleteRunIDs(ctx context.Context, ids []string) er } chunk := ids[:n] ids = ids[n:] - if err := a.beforeWrite(); err != nil { - return err - } tx, err := a.db.BeginTx(ctx, nil) if err != nil { return err @@ -473,7 +471,9 @@ func (a *ForecastLongArchive) incrementalVacuum(ctx context.Context) error { return err } if free*page < forecastLongFreeTarget || time.Now().After(deadline) { - return nil + // The file shrinks when the checkpoint copies the truncation. + _, err := a.db.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`) + return err } if _, err := a.db.ExecContext(ctx, `PRAGMA incremental_vacuum(256)`); err != nil { return err From 22ff7c44ceafad6344944445762072f7c0d0d824 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 10:28:51 +0100 Subject: [PATCH 4/9] fix(forecast): open the archive off the health lock and heal only real damage The worker held the store's archive lock while it opened the file. A quick_check of a large file on an SD card made /api/health, the storage inventory, backups and shutdown wait behind it. The open now runs without that lock. A second lock keeps two opens apart, and a store that closes in the meantime gets no handle. Any quick_check error moved a good file aside, so a passing I/O, memory or full-disk error cost the whole archive. Now only a check that finds problems, or a corruption code, does that. Other errors report unavailable and the worker tries again next hour. Each heal removes older quarantined copies, so at most one stays. Backups still leave them out. Damage found while copying waited for a restart. Sync now closes the handle when the archive returns a corruption code, so the next hourly run checks and heals the file. A corruption code read from state.db does not count. The worker gave the open 30 seconds. A full quick_check can take longer on a slow card, and then every hourly try fails the same way. The open is off the boot path and nothing waits on it, so it now runs until it ends or shutdown cancels the worker's context. Signed-off-by: Fredrik Ahlgren Co-Authored-By: Claude Opus 5.5 --- go/cmd/ftw/forecast_long_archive.go | 9 +- .../backup/forecast_archive_backup_test.go | 11 +- go/internal/state/forecast_long_archive.go | 100 ++++++--- .../state/forecast_long_archive_heal_test.go | 23 +- .../state/forecast_long_archive_open_test.go | 210 ++++++++++++++++++ go/internal/state/forecast_long_copy.go | 41 +++- go/internal/state/forecast_long_maintain.go | 2 +- go/internal/state/store.go | 4 + 8 files changed, 358 insertions(+), 42 deletions(-) create mode 100644 go/internal/state/forecast_long_archive_open_test.go diff --git a/go/cmd/ftw/forecast_long_archive.go b/go/cmd/ftw/forecast_long_archive.go index 9e4132bfa..b541003ab 100644 --- a/go/cmd/ftw/forecast_long_archive.go +++ b/go/cmd/ftw/forecast_long_archive.go @@ -83,9 +83,12 @@ func (a *forecastLongArchiver) runOnce(ctx context.Context) { if a == nil || a.store == nil || ctx.Err() != nil { return } - openCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - arch, err := a.store.OpenForecastLongArchive(openCtx) - cancel() + // No deadline beyond the worker's own context. quick_check of a full + // 512 MiB archive can take minutes on a slow SD card; a fixed timeout + // would throw that work away and fail the same way every hour. The open + // is off the boot path, health does not wait on it, and shutdown cancels + // it through ctx. + arch, err := a.store.OpenForecastLongArchive(ctx) if err != nil { if ctx.Err() != nil { return diff --git a/go/internal/backup/forecast_archive_backup_test.go b/go/internal/backup/forecast_archive_backup_test.go index 547b4b04a..195c7511c 100644 --- a/go/internal/backup/forecast_archive_backup_test.go +++ b/go/internal/backup/forecast_archive_backup_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -125,6 +126,12 @@ func TestFullBackupSkipsDamagedForecastArchive(t *testing.T) { if err := os.WriteFile(state.ForecastLongArchivePath(statePath), []byte("not a database"), 0o600); err != nil { t.Fatal(err) } + // Copies an earlier heal moved aside stay out of the backup too. + for _, name := range []string{"forecast-archive.db.corrupt-1", "forecast-archive.db-wal.corrupt-1", "forecast-archive.db-shm.corrupt-1"} { + if err := os.WriteFile(filepath.Join(dataDir, name), []byte("quarantined"), 0o600); err != nil { + t.Fatal(err) + } + } info, err := Create(context.Background(), CreateOptions{ State: st, StatePath: statePath, DataDir: dataDir, OutputDir: filepath.Join(root, "backups"), @@ -151,8 +158,8 @@ func TestFullBackupSkipsDamagedForecastArchive(t *testing.T) { if err != nil { t.Fatal(err) } - if header.Name == "data/forecast-archive.db" || header.Name == "data/forecast-archive.db-wal" { - t.Fatalf("damaged archive was copied: %s", header.Name) + if strings.HasPrefix(header.Name, "data/forecast-archive.db") { + t.Fatalf("damaged or quarantined archive was copied: %s", header.Name) } } } diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go index 45fd6937b..06c180049 100644 --- a/go/internal/state/forecast_long_archive.go +++ b/go/internal/state/forecast_long_archive.go @@ -13,11 +13,11 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" "time" "github.com/srcfl/ftw/go/internal/forecasting" - "modernc.org/sqlite" ) const ( @@ -153,35 +153,52 @@ func (s *Store) ForecastLongArchiveStatus() (ForecastLongArchiveStatus, bool) { // OpenForecastLongArchive opens or creates the archive after startup. A // damaged file is moved aside and replaced with an empty one. Boot does not -// call this. +// call this. The quick_check can take minutes on an SD card, so it runs +// without forecastLongMu: health, inventory, backup and Close never wait +// for it. forecastLongOpenMu keeps two opens apart. func (s *Store) OpenForecastLongArchive(ctx context.Context) (*ForecastLongArchive, error) { if s == nil || s.mainDBPath == "" { return nil, errors.New("forecast archive: state store unavailable") } + s.forecastLongOpenMu.Lock() + defer s.forecastLongOpenMu.Unlock() s.forecastLongMu.Lock() - defer s.forecastLongMu.Unlock() - if s.forecastLong != nil { - return s.forecastLong, nil + cached, closed := s.forecastLong, s.forecastLongClosed + s.forecastLongMu.Unlock() + if closed { + return nil, errForecastArchiveClosed } - path := ForecastLongArchivePath(s.mainDBPath) - arch, ev, err := openForecastLongFile(ctx, path) + if cached != nil { + return cached, nil + } + arch, ev, err := openForecastLongFile(ctx, ForecastLongArchivePath(s.mainDBPath)) if err != nil { return nil, err } arch.store = s + s.forecastLongMu.Lock() + closed = s.forecastLongClosed + if !closed { + s.forecastLong = arch + } + s.forecastLongMu.Unlock() + if closed { + _ = arch.Close() + return nil, errForecastArchiveClosed + } if ev != nil { s.RecordHealEvent(*ev) - arch.rebuilt = true - arch.status.State = "rebuilt" } - s.forecastLong = arch return arch, nil } +var errForecastArchiveClosed = errors.New("forecast archive: state store closed") + func (s *Store) closeForecastLongArchive() error { s.forecastLongMu.Lock() arch := s.forecastLong s.forecastLong = nil + s.forecastLongClosed = true s.forecastLongMu.Unlock() if arch == nil { return nil @@ -189,6 +206,18 @@ func (s *Store) closeForecastLongArchive() error { return arch.Close() } +// dropForecastLongArchive closes a handle that hit damage, so the next +// hourly copy reopens the file, checks it and heals it. Close waits for a +// backup or inventory read that is already running on the handle. +func (s *Store) dropForecastLongArchive(arch *ForecastLongArchive) { + s.forecastLongMu.Lock() + if s.forecastLong == arch { + s.forecastLong = nil + } + s.forecastLongMu.Unlock() + _ = arch.Close() +} + func openForecastLongFile(ctx context.Context, path string) (*ForecastLongArchive, *HealEvent, error) { db, damaged, err := openForecastLongDB(ctx, path) if err != nil && !damaged { @@ -196,13 +225,12 @@ func openForecastLongFile(ctx context.Context, path string) (*ForecastLongArchiv } var ev *HealEvent if damaged { - if db != nil { - _ = db.Close() - } nowMs := time.Now().UnixMilli() + slog.Warn("forecast long archive damaged; moving it aside", "err", err) if err := quarantine(path, nowMs); err != nil { return nil, nil, err } + removeOlderForecastLongQuarantine(path, nowMs) db, _, err = openForecastLongDB(ctx, path) if err != nil { return nil, nil, err @@ -233,6 +261,10 @@ func openForecastLongFile(ctx context.Context, path string) (*ForecastLongArchiv return arch, ev, nil } +// openForecastLongDB reports damage only when quick_check ran and found +// problems or SQLite returned a corruption code. Busy, I/O, full-disk, +// memory and permission errors leave a good file in place; the copier +// reports unavailable and tries again next hour. func openForecastLongDB(ctx context.Context, path string) (*sql.DB, bool, error) { if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { db, err := openRawPragmas(path, forecastLongPragmas) @@ -247,10 +279,7 @@ func openForecastLongDB(ctx context.Context, path string) (*sql.DB, bool, error) } db, err := openRawPragmas(path, forecastLongPragmas) if err != nil { - if sqliteCorrupt(err) { - return nil, true, err - } - return nil, false, err + return nil, sqliteCorrupt(err), err } ok, qerr := quickCheckContext(ctx, db) if qerr == nil && ok { @@ -260,15 +289,37 @@ func openForecastLongDB(ctx context.Context, path string) (*sql.DB, bool, error) } return db, false, nil } + _ = db.Close() + if qerr == nil { + return nil, true, errors.New("forecast archive: quick_check found problems") + } if ctx.Err() != nil { - _ = db.Close() return nil, false, ctx.Err() } - if sqliteBusy(qerr) { - _ = db.Close() - return nil, false, qerr + return nil, sqliteCorrupt(qerr), qerr +} + +// removeOlderForecastLongQuarantine keeps only the copy just moved aside, so +// repeated damage cannot fill the card. Backups never include these files. +func removeOlderForecastLongQuarantine(path string, keepMs int64) { + dir, name := filepath.Split(path) + entries, err := os.ReadDir(dir) + if err != nil { + slog.Warn("forecast long archive: cannot list old quarantined files", "err", err) + return + } + keep := fmt.Sprintf(".corrupt-%d", keepMs) + for _, entry := range entries { + for _, base := range []string{name, name + "-wal", name + "-shm"} { + n := entry.Name() + if !strings.HasPrefix(n, base+".corrupt-") || n == base+keep { + continue + } + if err := os.Remove(filepath.Join(dir, n)); err != nil { + slog.Warn("forecast long archive: cannot remove old quarantined file", "file", n, "err", err) + } + } } - return db, true, qerr } const forecastLongPragmas = sqlitePragmas + "&_pragma=journal_size_limit(4194304)&_pragma=auto_vacuum(INCREMENTAL)" @@ -368,11 +419,6 @@ INSERT OR IGNORE INTO fa_migrations(name) VALUES('fa-v1'); return err } -func sqliteBusy(err error) bool { - var e *sqlite.Error - return errors.As(err, &e) && (e.Code()&0xff == 5 || e.Code()&0xff == 6) -} - func sqliteFull(err error) bool { var code interface{ Code() int } return errors.As(err, &code) && code.Code()&0xff == 13 diff --git a/go/internal/state/forecast_long_archive_heal_test.go b/go/internal/state/forecast_long_archive_heal_test.go index df607d69e..814e99e1a 100644 --- a/go/internal/state/forecast_long_archive_heal_test.go +++ b/go/internal/state/forecast_long_archive_heal_test.go @@ -22,6 +22,14 @@ func TestForecastLongArchiveRebuildsCorruption(t *testing.T) { if err := os.WriteFile(archivePath, []byte("this is not a sqlite database"), 0o600); err != nil { t.Fatal(err) } + // Copies left by earlier damage. Only the newest quarantine stays. + stale := []string{"forecast-archive.db.corrupt-1", "forecast-archive.db-wal.corrupt-1", "forecast-archive.db-shm.corrupt-2"} + others := []string{"cache.db.corrupt-1", "forecast-archive.db.corrupted"} + for _, name := range append(append([]string(nil), stale...), others...) { + if err := os.WriteFile(filepath.Join(dir, name), make([]byte, 4096), 0o600); err != nil { + t.Fatal(err) + } + } reopened, err := Open(path) if err != nil { t.Fatalf("state.Open failed because of the analytics file: %v", err) @@ -46,13 +54,18 @@ func TestForecastLongArchiveRebuildsCorruption(t *testing.T) { if err != nil { t.Fatal(err) } - quarantined := false + var quarantined []string for _, entry := range entries { - if strings.Contains(entry.Name(), "forecast-archive.db.corrupt-") { - quarantined = true + if strings.HasPrefix(entry.Name(), "forecast-archive.db") && strings.Contains(entry.Name(), ".corrupt-") { + quarantined = append(quarantined, entry.Name()) } } - if !quarantined { - t.Fatal("damaged archive was not moved aside") + if len(quarantined) != 1 || quarantined[0] == stale[0] || !strings.HasPrefix(quarantined[0], "forecast-archive.db.corrupt-") { + t.Fatalf("quarantined archive files = %v, want only the new copy", quarantined) + } + for _, name := range others { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Fatalf("unrelated file %s was removed: %v", name, err) + } } } diff --git a/go/internal/state/forecast_long_archive_open_test.go b/go/internal/state/forecast_long_archive_open_test.go new file mode 100644 index 000000000..cc6ada196 --- /dev/null +++ b/go/internal/state/forecast_long_archive_open_test.go @@ -0,0 +1,210 @@ +package state + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// closedLongArchive leaves forecast-archive.db on disk with one run and +// returns a fresh store that has not opened it. +func closedLongArchive(t *testing.T) (*Store, string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + issue := longIssue("kept", time.Now().UTC().Truncate(time.Hour).Add(-time.Hour)) + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reopened.Close() }) + return reopened, ForecastLongArchivePath(path) +} + +func quarantinedFiles(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var out []string + for _, entry := range entries { + if strings.Contains(entry.Name(), ".corrupt-") { + out = append(out, entry.Name()) + } + } + return out +} + +func TestForecastLongOpenDoesNotBlockStatusReaders(t *testing.T) { + store, archivePath := closedLongArchive(t) + // Another connection holds the file, so the open waits in SQLite's busy + // handler as it would behind a long quick_check on an SD card. + lock, err := sql.Open("sqlite", archivePath+"?_pragma=locking_mode(EXCLUSIVE)") + if err != nil { + t.Fatal(err) + } + lock.SetMaxOpenConns(1) + defer lock.Close() + if _, err := lock.Exec(`INSERT OR REPLACE INTO fa_meta(key, value) VALUES('lock', 'held')`); err != nil { + t.Fatal(err) + } + opened := make(chan error, 1) + go func() { + _, err := store.OpenForecastLongArchive(context.Background()) + opened <- err + }() + for store.forecastLongOpenMu.TryLock() { + store.forecastLongOpenMu.Unlock() + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + readers := make(chan error, 1) + go func() { + store.ReportForecastLongArchive(ForecastLongArchiveStatus{State: "starting"}) + if _, ok := store.ForecastLongArchiveStatus(); !ok { + readers <- errors.New("status missing") + return + } + _, err := store.SQLiteInventory(context.Background()) + readers <- err + }() + select { + case err := <-readers: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("health status and storage inventory waited for the archive open") + } + if err := lock.Close(); err != nil { + t.Fatal(err) + } + if err := <-opened; err != nil { + t.Fatalf("open after the lock was released: %v", err) + } + if files := quarantinedFiles(t, filepath.Dir(archivePath)); len(files) != 0 { + t.Fatalf("a locked archive was moved aside: %v", files) + } +} + +func TestForecastLongOpenKeepsFileOnReadError(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + store, archivePath := closedLongArchive(t) + if err := os.Chmod(archivePath, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(archivePath, 0o600) }) + if _, err := store.OpenForecastLongArchive(context.Background()); err == nil { + t.Fatal("unreadable archive opened") + } + if files := quarantinedFiles(t, filepath.Dir(archivePath)); len(files) != 0 { + t.Fatalf("a read error moved a good archive aside: %v", files) + } + if events := store.HealEvents(); len(events) != 0 { + t.Fatalf("heal events = %+v", events) + } + if err := os.Chmod(archivePath, 0o600); err != nil { + t.Fatal(err) + } + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + if _, err := arch.LoadIssue(context.Background(), "kept"); err != nil { + t.Fatalf("the next open lost the stored run: %v", err) + } +} + +func TestForecastLongSyncDropsDamagedArchive(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + issue := longIssue("before", time.Now().UTC().Truncate(time.Hour).Add(-time.Hour)) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + // Each query takes a new connection, which reads the damaged header. + arch.db.SetMaxIdleConns(0) + if _, err := arch.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(arch.path, make([]byte, 8192), 0o600); err != nil { + t.Fatal(err) + } + err := arch.Sync(ctx, store, time.Now().UTC()) + if !sqliteCorrupt(err) { + t.Fatalf("sync err = %v, want a corruption code", err) + } + if err := arch.db.Ping(); err == nil { + t.Fatal("damaged handle is still open") + } + healed, err := store.OpenForecastLongArchive(ctx) + if err != nil { + t.Fatal(err) + } + if healed == arch { + t.Fatal("open returned the dropped handle") + } + events := store.HealEvents() + if len(events) != 1 || events[0].Tier != "forecast_archive" { + t.Fatalf("heal events = %+v", events) + } + if err := healed.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatalf("copy into the rebuilt archive: %v", err) + } + if files := quarantinedFiles(t, filepath.Dir(arch.path)); len(files) == 0 { + t.Fatal("damaged archive was not moved aside") + } + // A corruption code read from state.db is not damage in the archive. + corrupt := sqliteErrorOf(t) + if !forecastLongDamaged(fmt.Errorf("copy: %w", corrupt)) { + t.Fatal("archive corruption was not recognised") + } + if forecastLongDamaged(fmt.Errorf("copy: %w", fromForecastSource(corrupt))) { + t.Fatal("a damaged state.db would drop the archive") + } +} + +// sqliteErrorOf returns a real SQLITE_NOTADB error. +func sqliteErrorOf(t *testing.T) error { + t.Helper() + path := filepath.Join(t.TempDir(), "junk.db") + if err := os.WriteFile(path, []byte("this is not a sqlite database at all, just text"), 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Exec(`SELECT * FROM sqlite_master`) + if !sqliteCorrupt(err) { + t.Fatalf("junk file error = %v", err) + } + return err +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go index 09f8b33bd..7cb8660bc 100644 --- a/go/internal/state/forecast_long_copy.go +++ b/go/internal/state/forecast_long_copy.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "log/slog" "time" "github.com/srcfl/ftw/go/internal/forecasting" @@ -30,11 +31,42 @@ type forecastLongObsCursor struct { Config string `json:"config_version"` } +// forecastLongSourceError marks a failed read of state.db. Damage there is +// healed at boot and must not send the archive to quarantine. +type forecastLongSourceError struct{ err error } + +func (e forecastLongSourceError) Error() string { return "state.db: " + e.err.Error() } +func (e forecastLongSourceError) Unwrap() error { return e.err } + +func fromForecastSource(err error) error { + if err == nil { + return nil + } + return forecastLongSourceError{err} +} + +// forecastLongDamaged reports a corruption code from the archive itself. +func forecastLongDamaged(err error) bool { + var source forecastLongSourceError + return sqliteCorrupt(err) && !errors.As(err, &source) +} + // Sync copies the 30-day issue and observation tables and holds the archive // to its cap. Once a day it also thins, prunes expired runs and gives free // pages back. Failures stay in the published status. Below the disk floor -// copying pauses until the next call, but the deletes still run. +// copying pauses until the next call, but the deletes still run. When the +// archive reports damage, Sync closes it, so the next copy reopens the file, +// runs quick_check and heals it without a restart. func (a *ForecastLongArchive) Sync(ctx context.Context, src *Store, now time.Time) error { + err := a.syncOnce(ctx, src, now) + if forecastLongDamaged(err) && a.store != nil { + slog.Warn("forecast long archive: damage found while copying; the next copy checks the file", "err", err) + a.store.dropForecastLongArchive(a) + } + return err +} + +func (a *ForecastLongArchive) syncOnce(ctx context.Context, src *Store, now time.Time) error { if a == nil || src == nil { return errors.New("forecast archive: copy source unavailable") } @@ -45,6 +77,7 @@ func (a *ForecastLongArchive) Sync(ctx context.Context, src *Store, now time.Tim a.status.LastError = "" a.mu.Unlock() if err := src.InitForecastArchive(ctx); err != nil { + err = fromForecastSource(err) a.fail("schema", err) return err } @@ -135,7 +168,7 @@ func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store) error }) cancel() if err != nil { - return err + return fromForecastSource(err) } if len(page) == 0 { return nil @@ -195,7 +228,7 @@ func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.I continue } if err != nil { - return nil, err + return nil, fromForecastSource(err) } states[model.StateID] = ForecastModelBlob{Expanded: expanded, Gzip: payload} } @@ -219,7 +252,7 @@ func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store) }) cancel() if err != nil { - return err + return fromForecastSource(err) } if len(page) == 0 { return nil diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go index cd5371b2b..3697cf768 100644 --- a/go/internal/state/forecast_long_maintain.go +++ b/go/internal/state/forecast_long_maintain.go @@ -508,7 +508,7 @@ func (a *ForecastLongArchive) recompute(ctx context.Context, src *Store, now tim } lag, err := src.forecastCopyLag(ctx, cursor, now.UnixMilli()) if err != nil { - return err + return fromForecastSource(err) } p50, p95, gz, err := a.issueSizePercentiles(ctx) if err != nil { diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 75658f906..74bcf5188 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -103,8 +103,12 @@ type Store struct { // forecastLong is the long-term forecast archive. It stays nil until the // copier opens it, so a slow or damaged analytics file cannot delay boot. + // forecastLongMu is held only to read or swap these fields, never for + // file work; forecastLongOpenMu keeps two opens apart. + forecastLongOpenMu sync.Mutex forecastLongMu sync.Mutex forecastLong *ForecastLongArchive + forecastLongClosed bool forecastLongStatus *ForecastLongArchiveStatus } From 591c1f57b8fe33b6763a2f08d19b182b4a64f96c Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 10:30:57 +0100 Subject: [PATCH 5/9] fix(forecast): copy late and clock-skewed forecast rows into the archive The copier read strictly after its cursor. A shadow issue saved just after its primary, with the same issued time and an id that sorts first, was skipped for good. Observations were read in start order, so a quarter released after a later one was skipped too. A cursor set while the clock ran ahead stopped all copying until that time came. Observations now follow available_at_ms, the order the tracker writes them in. Each copy reads again from an hour before its cursor and never past the clock plus the five minutes state.db accepts, so a row stamped by a wrong clock cannot pull the cursor ahead. A cursor already past that bound resets, and the copy reads the 30-day tables again. A rescan stays cheap: the copier lists issue keys without payloads and loads only the runs the archive does not hold. Rows read again are ignored on insert. The cursor and last_copy_ms move only when the copy gets past the old cursor or stores a run. The fa-v1 cursor changes in place; an old cursor reads as zero and starts one full rescan. Signed-off-by: Fredrik Ahlgren Co-Authored-By: Claude Opus 5.5 --- go/internal/state/forecast_issues.go | 88 ++++----- .../state/forecast_long_archive_copy_test.go | 172 ++++++++++++++++ go/internal/state/forecast_long_copy.go | 184 ++++++++++++++---- 3 files changed, 352 insertions(+), 92 deletions(-) diff --git a/go/internal/state/forecast_issues.go b/go/internal/state/forecast_issues.go index f674c1efb..c9b68a87c 100644 --- a/go/internal/state/forecast_issues.go +++ b/go/internal/state/forecast_issues.go @@ -704,7 +704,7 @@ func (s *Store) LoadForecastErrors(ctx context.Context, since, until int64, hour return out, nil } -// ForecastIssueCopy is one row from the 30-day issue archive, in issued order. +// ForecastIssueCopy is one row from the 30-day issue archive. // Err is set when the row cannot be decoded; the copier skips it and moves on. type ForecastIssueCopy struct { Issue forecasting.Issue @@ -715,56 +715,40 @@ type ForecastIssueCopy struct { Err error } -// VisitForecastIssuesAfter walks issues strictly after the (issued_at_ms, id) -// cursor, oldest first, at most limit rows. The long-term copier uses it so a -// read snapshot stays short. -func (s *Store) VisitForecastIssuesAfter(ctx context.Context, cursorIssuedMS int64, cursorID string, limit int, visit func(ForecastIssueCopy) error) error { - if visit == nil { - return errors.New("forecast issue visitor is nil") - } - if limit <= 0 || limit > 64 { - limit = 16 - } - rows, err := s.db.QueryContext(ctx, `SELECT issued_at_ms, id, payload FROM forecast_issues - WHERE issued_at_ms > ? OR (issued_at_ms = ? AND id > ?) - ORDER BY issued_at_ms, id LIMIT ?`, cursorIssuedMS, cursorIssuedMS, cursorID, limit) +// forecastIssueKeysAfter lists issues after the (issued_at_ms, id) cursor and +// no later than untilMS, oldest first. It reads no payloads, so the long-term +// copier can skip runs it already holds. +func (s *Store) forecastIssueKeysAfter(ctx context.Context, after forecastLongIssueCursor, untilMS int64, limit int) ([]forecastLongIssueCursor, error) { + rows, err := s.db.QueryContext(ctx, forecastIssueKeysQuery, after.IssuedAtMS, after.ID, untilMS, limit) if err != nil { - return err + return nil, err } defer rows.Close() - type rawIssue struct { - issued int64 - id string - data []byte - } - var page []rawIssue + var keys []forecastLongIssueCursor for rows.Next() { - var row rawIssue - if err = rows.Scan(&row.issued, &row.id, &row.data); err != nil { - return err + var key forecastLongIssueCursor + if err := rows.Scan(&key.IssuedAtMS, &key.ID); err != nil { + return nil, err } - page = append(page, row) - } - if err = rows.Err(); err != nil { - return err + keys = append(keys, key) } - if err = rows.Close(); err != nil { - return err - } - for _, row := range page { - item := ForecastIssueCopy{IssuedAtMS: row.issued, ID: row.id, GzipBytes: len(row.data)} - issue, size, decodeErr := decodeForecastIssue(row.data) - if decodeErr != nil { - item.Err = decodeErr - } else { - item.Issue = issue - item.JSONBytes = size - } - if err = visit(item); err != nil { - return err - } + return keys, rows.Err() +} + +const forecastIssueKeysQuery = `SELECT issued_at_ms, id FROM forecast_issues + WHERE (issued_at_ms, id) > (?, ?) AND issued_at_ms <= ? + ORDER BY issued_at_ms, id LIMIT ?` + +// forecastIssueCopy reads and decodes one issue. It returns sql.ErrNoRows +// when retention removed the row after it was listed. +func (s *Store) forecastIssueCopy(ctx context.Context, key forecastLongIssueCursor) (ForecastIssueCopy, error) { + var data []byte + if err := s.db.QueryRowContext(ctx, `SELECT payload FROM forecast_issues WHERE id=?`, key.ID).Scan(&data); err != nil { + return ForecastIssueCopy{}, err } - return nil + item := ForecastIssueCopy{IssuedAtMS: key.IssuedAtMS, ID: key.ID, GzipBytes: len(data)} + item.Issue, item.JSONBytes, item.Err = decodeForecastIssue(data) + return item, nil } // LoadForecastModelStateRaw returns the gzip bytes stored for a model state. @@ -778,18 +762,18 @@ func (s *Store) LoadForecastModelStateRaw(ctx context.Context, stateID string) ( return expanded, payload, err } -// VisitForecastObservationsAfter walks quarter-hour truth strictly after the -// (start_ms, end_ms, config_version) cursor. -func (s *Store) VisitForecastObservationsAfter(ctx context.Context, startMS, endMS int64, config string, limit int, visit func(forecasting.Observation) error) error { +// visitForecastObservationsAfter walks quarter-hour truth after the cursor in +// (available_at_ms, start_ms, end_ms, config_version) order, no later than +// untilMS. The tracker writes rows in available_at_ms order. +func (s *Store) visitForecastObservationsAfter(ctx context.Context, after forecastLongObsCursor, untilMS int64, limit int, visit func(forecasting.Observation) error) error { if visit == nil { return errors.New("forecast observation visitor is nil") } if limit <= 0 || limit > 512 { limit = 128 } - rows, err := s.db.QueryContext(ctx, `SELECT payload FROM forecast_observations - WHERE start_ms > ? OR (start_ms = ? AND end_ms > ?) OR (start_ms = ? AND end_ms = ? AND config_version > ?) - ORDER BY start_ms, end_ms, config_version LIMIT ?`, startMS, startMS, endMS, startMS, endMS, config, limit) + rows, err := s.db.QueryContext(ctx, forecastObservationsAfterQuery, + after.AvailableMS, after.StartMS, after.EndMS, after.Config, untilMS, limit) if err != nil { return err } @@ -826,6 +810,10 @@ func (s *Store) VisitForecastObservationsAfter(ctx context.Context, startMS, end return nil } +const forecastObservationsAfterQuery = `SELECT payload FROM forecast_observations + WHERE (available_at_ms, start_ms, end_ms, config_version) > (?, ?, ?, ?) AND available_at_ms <= ? + ORDER BY available_at_ms, start_ms, end_ms, config_version LIMIT ?` + // NewestForecastIssueMS is the latest issued_at_ms in the 30-day archive, or // zero when it is empty. The copier uses it only to describe lag. func (s *Store) NewestForecastIssueMS(ctx context.Context) (int64, error) { diff --git a/go/internal/state/forecast_long_archive_copy_test.go b/go/internal/state/forecast_long_archive_copy_test.go index d09c12812..ddeee2258 100644 --- a/go/internal/state/forecast_long_archive_copy_test.go +++ b/go/internal/state/forecast_long_archive_copy_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "strings" "testing" "time" @@ -131,3 +132,174 @@ func forecastLongItoa(n int) string { } return string(b[i:]) } + +func copiedRunIDs(t *testing.T, arch *ForecastLongArchive) map[string]bool { + t.Helper() + rows, err := arch.db.Query(`SELECT run_id FROM fa_runs`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + out := map[string]bool{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + out[id] = true + } + return out +} + +func longObservation(start time.Time, availableAfterEnd time.Duration) forecasting.Observation { + end := start.Add(15 * time.Minute) + return forecasting.Observation{ + StartMS: start.UnixMilli(), EndMS: end.UnixMilli(), AvailableAtMS: end.Add(availableAfterEnd).UnixMilli(), + PVW: 10, LoadW: 20, PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg-v1", + } +} + +func TestForecastLongCopyKeepsLateRows(t *testing.T) { + if forecastLongCopyOverlap < maxForecastFutureSkew { + t.Fatal("the copy overlap must cover the future skew state.db accepts") + } + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Truncate(time.Hour).Add(-2 * time.Hour) + if err := store.SaveForecastIssue(ctx, longIssue("b-primary", origin)); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin, time.Minute)); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + // The shadow has the same issued time and an id that sorts first. The + // earlier quarter was released after the later one. + if err := store.SaveForecastIssue(ctx, longIssue("a-shadow", origin)); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin.Add(-15*time.Minute), 20*time.Minute)); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); !runs["a-shadow"] || !runs["b-primary"] { + t.Fatalf("runs = %v, the late shadow was skipped", runs) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_observations_15m`); n != 2 { + t.Fatalf("observations = %d, the late quarter was skipped", n) + } +} + +func TestForecastLongCopyRecoversFromFutureCursor(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + origin := now.Truncate(time.Hour).Add(-2 * time.Hour) + if err := store.SaveForecastIssue(ctx, longIssue("real", origin)); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin, time.Minute)); err != nil { + t.Fatal(err) + } + // A row stamped by a clock a day ahead. state.db keeps it, since its + // retention counts back from now. + ahead, err := prepareForecastIssue(longIssue("ahead", now.Truncate(time.Hour).Add(24*time.Hour))) + if err != nil { + t.Fatal(err) + } + if _, err := store.db.Exec(`INSERT INTO forecast_issues(id,origin_ms,issued_at_ms,config_version,payload) VALUES(?,?,?,?,?)`, + ahead.issue.ID, ahead.issue.OriginMS, ahead.issue.IssuedAtMS, ahead.issue.ConfigVersion, ahead.payload); err != nil { + t.Fatal(err) + } + // Cursors written while the clock was a year ahead. + yearAhead := now.Add(365 * 24 * time.Hour).UnixMilli() + if err := arch.writeIssueCursor(ctx, forecastLongIssueCursor{IssuedAtMS: yearAhead, ID: "z"}); err != nil { + t.Fatal(err) + } + if err := arch.writeObsCursor(ctx, forecastLongObsCursor{AvailableMS: yearAhead}); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); len(runs) != 1 || !runs["real"] { + t.Fatalf("runs = %v, want only the run issued before now", runs) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_observations_15m`); n != 1 { + t.Fatalf("observations = %d after the cursor reset", n) + } + bound := now.Add(maxForecastFutureSkew).UnixMilli() + issueCursor, err := arch.issueCursor(ctx) + if err != nil || issueCursor.IssuedAtMS > bound { + t.Fatalf("issue cursor = %+v err=%v, ahead of the clock again", issueCursor, err) + } + obsCursor, err := arch.obsCursor(ctx) + if err != nil || obsCursor.AvailableMS > bound { + t.Fatalf("observation cursor = %+v err=%v, ahead of the clock again", obsCursor, err) + } + + // A rescan skips stored runs by id and never reads their payloads. + if _, err := store.db.Exec(`UPDATE forecast_issues SET payload=x'00' WHERE id='real'`); err != nil { + t.Fatal(err) + } + if err := arch.writeIssueCursor(ctx, forecastLongIssueCursor{IssuedAtMS: yearAhead}); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + if status, _ := store.ForecastLongArchiveStatus(); status.LastError != "" { + t.Fatalf("rescan decoded a stored run: %q", status.LastError) + } + + // The row stamped ahead is copied once the clock reaches it. + if err := arch.Sync(ctx, store, now.Add(25*time.Hour)); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); !runs["ahead"] { + t.Fatalf("runs = %v, the row stamped ahead was never copied", runs) + } +} + +func TestForecastLongCopyQueriesUseIndexes(t *testing.T) { + store, _ := openLongArchive(t) + if err := store.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + for _, q := range []struct { + query, index string + args []any + }{ + {forecastIssueKeysQuery, "forecast_issues_time", []any{1, "a", 2, 16}}, + {forecastObservationsAfterQuery, "forecast_observations_time", []any{1, 2, 3, "c", 5, 128}}, + } { + rows, err := store.db.Query("EXPLAIN QUERY PLAN "+q.query, q.args...) + if err != nil { + t.Fatal(err) + } + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + rows.Close() + joined := strings.Join(plan, "; ") + if strings.Contains(joined, "SCAN ") || !strings.Contains(joined, "USING INDEX "+q.index) { + t.Fatalf("plan for the copy query = %s", joined) + } + } +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go index 7cb8660bc..a090073df 100644 --- a/go/internal/state/forecast_long_copy.go +++ b/go/internal/state/forecast_long_copy.go @@ -14,6 +14,13 @@ import ( const ( forecastLongIssuePage = 16 forecastLongObsPage = 128 + // forecastLongCopyOverlap is how far before its cursor each copy reads + // again. A row that commits late with a slightly earlier key, such as a + // shadow issue saved after its primary or a write after a small clock + // step back, is still copied. It must be at least maxForecastFutureSkew: + // a cursor never runs further ahead of the clock than that, so every row + // written after a step back sorts after cursor minus overlap. + forecastLongCopyOverlap = time.Hour ) // forecastLongAfterIssuePage runs after a committed issue page. Tests use it @@ -25,10 +32,31 @@ type forecastLongIssueCursor struct { ID string `json:"id"` } +func (c forecastLongIssueCursor) after(o forecastLongIssueCursor) bool { + return c.IssuedAtMS > o.IssuedAtMS || (c.IssuedAtMS == o.IssuedAtMS && c.ID > o.ID) +} + +// forecastLongObsCursor follows available_at_ms, the time a quarter was +// released. The tracker writes observations in that order, while a quarter +// with an earlier start can be released after a later one. type forecastLongObsCursor struct { - StartMS int64 `json:"start_ms"` - EndMS int64 `json:"end_ms"` - Config string `json:"config_version"` + AvailableMS int64 `json:"available_at_ms"` + StartMS int64 `json:"start_ms"` + EndMS int64 `json:"end_ms"` + Config string `json:"config_version"` +} + +func (c forecastLongObsCursor) after(o forecastLongObsCursor) bool { + if c.AvailableMS != o.AvailableMS { + return c.AvailableMS > o.AvailableMS + } + if c.StartMS != o.StartMS { + return c.StartMS > o.StartMS + } + if c.EndMS != o.EndMS { + return c.EndMS > o.EndMS + } + return c.Config > o.Config } // forecastLongSourceError marks a failed read of state.db. Damage there is @@ -83,7 +111,7 @@ func (a *ForecastLongArchive) syncOnce(ctx context.Context, src *Store, now time } var paused error if !a.blocked() { - if err := a.copyIssues(ctx, src); err != nil { + if err := a.copyIssues(ctx, src, now); err != nil { if forecastLongPaused(err) { paused = err } else { @@ -93,7 +121,7 @@ func (a *ForecastLongArchive) syncOnce(ctx context.Context, src *Store, now time } } if paused == nil && !a.blocked() { - if err := a.copyObservations(ctx, src); err != nil { + if err := a.copyObservations(ctx, src, now); err != nil { if forecastLongPaused(err) { paused = err } else { @@ -151,46 +179,46 @@ func (a *ForecastLongArchive) fail(kind string, err error) { a.publish() } -func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store) error { +// copyIssues reads from an hour before its cursor up to the clock plus +// maxForecastFutureSkew. A run already archived is skipped by id before its +// payload is read, so the overlap and a full rescan stay cheap. A cursor +// further ahead than that bound came from a wrong clock; the copy then +// starts again from the beginning of the 30-day table. +func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store, now time.Time) error { + until := now.UnixMilli() + maxForecastFutureSkew.Milliseconds() cursor, err := a.issueCursor(ctx) if err != nil { return err } + if cursor.IssuedAtMS > until { + slog.Warn("forecast long archive: issue cursor is ahead of the clock; copying the 30-day table again", + "cursor_ms", cursor.IssuedAtMS, "now_ms", now.UnixMilli()) + cursor = forecastLongIssueCursor{} + if err := a.writeIssueCursor(ctx, cursor); err != nil { + return err + } + } + from := forecastLongIssueCursor{IssuedAtMS: cursor.IssuedAtMS - forecastLongCopyOverlap.Milliseconds()} for { if err := ctx.Err(); err != nil { return err } - pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) - var page []ForecastIssueCopy - err = src.VisitForecastIssuesAfter(pageCtx, cursor.IssuedAtMS, cursor.ID, forecastLongIssuePage, func(item ForecastIssueCopy) error { - page = append(page, item) - return nil - }) - cancel() + batch, err := a.readIssuePage(ctx, src, from, until) if err != nil { - return fromForecastSource(err) + return err } - if len(page) == 0 { + if batch.keys == 0 { return nil } - batch := make([]ForecastArchiveIssue, 0, len(page)) - for _, item := range page { - cursor = forecastLongIssueCursor{IssuedAtMS: item.IssuedAtMS, ID: item.ID} - if item.Err != nil { - a.warn("decode", item.Err) - continue - } - states, err := forecastLongStatesFrom(ctx, src, item.Issue) - if err != nil { - return err - } - batch = append(batch, ForecastArchiveIssue{ - Issue: item.Issue, JSONBytes: item.JSONBytes, GzipBytes: item.GzipBytes, States: states, - }) - } + from = batch.last + advanced := from.after(cursor) writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) - err = a.ArchiveIssues(writeCtx, batch) - if err == nil || errors.Is(err, errForecastArchiveImmutable) { + err = a.ArchiveIssues(writeCtx, batch.issues) + if errors.Is(err, errForecastArchiveImmutable) { + err = nil // warned; the stored run stays + } + if err == nil && advanced { + cursor = from err = a.writeIssueCursor(writeCtx, cursor) } cancelWrite() @@ -198,14 +226,16 @@ func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store) error return err } if forecastLongAfterIssuePage != nil { - if err := forecastLongAfterIssuePage(len(page)); err != nil { + if err := forecastLongAfterIssuePage(batch.keys); err != nil { return err } } - a.mu.Lock() - a.status.LastCopyMS = time.Now().UnixMilli() - a.mu.Unlock() - if len(page) < forecastLongIssuePage { + if advanced || len(batch.issues) > 0 { + a.mu.Lock() + a.status.LastCopyMS = time.Now().UnixMilli() + a.mu.Unlock() + } + if batch.keys < forecastLongIssuePage { return nil } if err := pauseMaintenance(ctx); err != nil { @@ -214,6 +244,63 @@ func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store) error } } +type forecastLongIssueBatch struct { + issues []ForecastArchiveIssue + last forecastLongIssueCursor + keys int +} + +// readIssuePage lists one page of keys and loads only the runs the archive +// does not hold. Model states are read outside the page deadline, as +// before, so a slow blob read does not use it up. +func (a *ForecastLongArchive) readIssuePage(ctx context.Context, src *Store, from forecastLongIssueCursor, until int64) (forecastLongIssueBatch, error) { + var batch forecastLongIssueBatch + pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) + defer cancel() + keys, err := src.forecastIssueKeysAfter(pageCtx, from, until, forecastLongIssuePage) + if err != nil || len(keys) == 0 { + return batch, fromForecastSource(err) + } + for _, key := range keys { + known, err := a.hasRun(pageCtx, key.ID) + if err != nil { + return batch, err + } + if known { + continue + } + item, err := src.forecastIssueCopy(pageCtx, key) + if errors.Is(err, sql.ErrNoRows) { + continue // pruned from state.db after the key list + } + if err != nil { + return batch, fromForecastSource(err) + } + if item.Err != nil { + a.warn("decode", item.Err) + continue + } + states, err := forecastLongStatesFrom(ctx, src, item.Issue) + if err != nil { + return batch, err + } + batch.issues = append(batch.issues, ForecastArchiveIssue{ + Issue: item.Issue, JSONBytes: item.JSONBytes, GzipBytes: item.GzipBytes, States: states, + }) + } + batch.last, batch.keys = keys[len(keys)-1], len(keys) + return batch, nil +} + +func (a *ForecastLongArchive) hasRun(ctx context.Context, id string) (bool, error) { + var one int + err := a.db.QueryRowContext(ctx, `SELECT 1 FROM fa_runs WHERE run_id=?`, id).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.Issue) (map[string]ForecastModelBlob, error) { states := map[string]ForecastModelBlob{} for _, model := range issue.Models { @@ -235,18 +322,30 @@ func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.I return states, nil } -func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store) error { +// copyObservations follows the same rules as copyIssues on the +// available_at_ms order. Rows read again are ignored on insert. +func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store, now time.Time) error { + until := now.UnixMilli() + maxForecastFutureSkew.Milliseconds() cursor, err := a.obsCursor(ctx) if err != nil { return err } + if cursor.AvailableMS > until { + slog.Warn("forecast long archive: observation cursor is ahead of the clock; copying the 30-day table again", + "cursor_ms", cursor.AvailableMS, "now_ms", now.UnixMilli()) + cursor = forecastLongObsCursor{} + if err := a.writeObsCursor(ctx, cursor); err != nil { + return err + } + } + from := forecastLongObsCursor{AvailableMS: cursor.AvailableMS - forecastLongCopyOverlap.Milliseconds()} for { if err := ctx.Err(); err != nil { return err } pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) var page []forecasting.Observation - err = src.VisitForecastObservationsAfter(pageCtx, cursor.StartMS, cursor.EndMS, cursor.Config, forecastLongObsPage, func(row forecasting.Observation) error { + err = src.visitForecastObservationsAfter(pageCtx, from, until, forecastLongObsPage, func(row forecasting.Observation) error { page = append(page, row) return nil }) @@ -257,11 +356,12 @@ func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store) if len(page) == 0 { return nil } + last := page[len(page)-1] + from = forecastLongObsCursor{AvailableMS: last.AvailableAtMS, StartMS: last.StartMS, EndMS: last.EndMS, Config: last.ConfigVersion} writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) err = a.ArchiveObservations(writeCtx, page) - if err == nil { - last := page[len(page)-1] - cursor = forecastLongObsCursor{StartMS: last.StartMS, EndMS: last.EndMS, Config: last.ConfigVersion} + if err == nil && from.after(cursor) { + cursor = from err = a.writeObsCursor(writeCtx, cursor) } cancelWrite() From e816d813aacf0c53a9b1feb97a523cde497ffe44 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 10:48:34 +0100 Subject: [PATCH 6/9] fix(forecast): guard thinning with tests and drop dead archive code Three guards in the archive had no test that would fail without them. The daily thinning tests ran 2 minutes past the hour, when the open hour held only its run on the hour, so a pass that thinned a partial hour still passed. They now run at 18:40 UTC: the open hour holds runs a bad pass would delete, and a Kolkata day has just ended inside an open hour. A plan test now checks that the thinning queries read fa_runs through fa_runs_unthinned and fa_runs_origin. Another test deletes a linked weather row and expects LoadIssue to fail. The cleanup removes code nothing used. The delete helpers lose their one-value table and column arguments, forecastLongDay loses its unread key, NewestForecastIssueMS goes, and the observation reader returns a slice with no nil check or limit clamp, as the issue reader does. incrementalVacuum checkpoints only at the end: the vacuum sees pages freed in the WAL, and only the last checkpoint shrinks the file. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Fredrik Ahlgren --- go/internal/state/forecast_issues.go | 42 +++------ .../state/forecast_long_archive_thin_test.go | 19 ++++- .../forecast_long_archive_weather_test.go | 85 ++++++++++++------- go/internal/state/forecast_long_copy.go | 6 +- go/internal/state/forecast_long_maintain.go | 70 ++++++++------- 5 files changed, 120 insertions(+), 102 deletions(-) diff --git a/go/internal/state/forecast_issues.go b/go/internal/state/forecast_issues.go index c9b68a87c..3058681c4 100644 --- a/go/internal/state/forecast_issues.go +++ b/go/internal/state/forecast_issues.go @@ -762,65 +762,47 @@ func (s *Store) LoadForecastModelStateRaw(ctx context.Context, stateID string) ( return expanded, payload, err } -// visitForecastObservationsAfter walks quarter-hour truth after the cursor in +// forecastObservationsAfter lists quarter-hour truth after the cursor in // (available_at_ms, start_ms, end_ms, config_version) order, no later than // untilMS. The tracker writes rows in available_at_ms order. -func (s *Store) visitForecastObservationsAfter(ctx context.Context, after forecastLongObsCursor, untilMS int64, limit int, visit func(forecasting.Observation) error) error { - if visit == nil { - return errors.New("forecast observation visitor is nil") - } - if limit <= 0 || limit > 512 { - limit = 128 - } +func (s *Store) forecastObservationsAfter(ctx context.Context, after forecastLongObsCursor, untilMS int64, limit int) ([]forecasting.Observation, error) { rows, err := s.db.QueryContext(ctx, forecastObservationsAfterQuery, after.AvailableMS, after.StartMS, after.EndMS, after.Config, untilMS, limit) if err != nil { - return err + return nil, err } defer rows.Close() var payloads []string for rows.Next() { var data string if err = rows.Scan(&data); err != nil { - return err + return nil, err } payloads = append(payloads, data) } if err = rows.Err(); err != nil { - return err + return nil, err } if err = rows.Close(); err != nil { - return err + return nil, err } + out := make([]forecasting.Observation, 0, len(payloads)) for _, data := range payloads { if len(data) > maxForecastRecordBytes { - return errors.New("oversized forecast observation") + return nil, errors.New("oversized forecast observation") } var observation forecasting.Observation if err = json.Unmarshal([]byte(data), &observation); err != nil { - return err + return nil, err } if err = observation.Validate(); err != nil { - return err - } - if err = visit(observation); err != nil { - return err + return nil, err } + out = append(out, observation) } - return nil + return out, nil } const forecastObservationsAfterQuery = `SELECT payload FROM forecast_observations WHERE (available_at_ms, start_ms, end_ms, config_version) > (?, ?, ?, ?) AND available_at_ms <= ? ORDER BY available_at_ms, start_ms, end_ms, config_version LIMIT ?` - -// NewestForecastIssueMS is the latest issued_at_ms in the 30-day archive, or -// zero when it is empty. The copier uses it only to describe lag. -func (s *Store) NewestForecastIssueMS(ctx context.Context) (int64, error) { - var newest sql.NullInt64 - err := s.db.QueryRowContext(ctx, `SELECT MAX(issued_at_ms) FROM forecast_issues`).Scan(&newest) - if err != nil || !newest.Valid { - return 0, err - } - return newest.Int64, nil -} diff --git a/go/internal/state/forecast_long_archive_thin_test.go b/go/internal/state/forecast_long_archive_thin_test.go index a9213dbe7..253d050fb 100644 --- a/go/internal/state/forecast_long_archive_thin_test.go +++ b/go/internal/state/forecast_long_archive_thin_test.go @@ -202,10 +202,12 @@ func TestForecastLongLateRunLosesToThinnedSurvivor(t *testing.T) { } // thinDaily archives a run every 15 minutes over [from, to) in zone, then -// thins once a day at 12:02 UTC until every day is past the cutoff. After +// thins once a day at 18:40 UTC until every day is past the cutoff. After // each pass every complete UTC hour holds only its run on the hour and every // complete local day keeps model-state references on its first run. Later -// hours and days are untouched. +// hours and days are untouched. The pass falls 40 minutes into an hour, so +// the open hour already holds runs a pass could wrongly thin, and 10 minutes +// after a Kolkata day ends, so that day must wait for its last UTC hour. func thinDaily(t *testing.T, zone string, from, to time.Time) *ForecastLongArchive { t.Helper() _, arch := openLongArchive(t) @@ -227,7 +229,7 @@ func thinDaily(t *testing.T, zone string, from, to time.Time) *ForecastLongArchi } } hourMS := forecastLongHourMS - for cutoff := from.Add(12*time.Hour + 2*time.Minute); cutoff.Before(to.Add(36 * time.Hour)); cutoff = cutoff.Add(24 * time.Hour) { + for cutoff := from.Add(18*time.Hour + 40*time.Minute); cutoff.Before(to.Add(36 * time.Hour)); cutoff = cutoff.Add(24 * time.Hour) { if err := arch.thin(ctx, cutoff.Add(arch.fullDetail)); err != nil { t.Fatal(err) } @@ -300,3 +302,14 @@ func thinDaily(t *testing.T, zone string, from, to time.Time) *ForecastLongArchi } return arch } + +// Each daily pass looks for the next day left to thin. Through the origin +// index that walks every thinned run first; the partial index holds only +// the runs still to thin. +func TestForecastLongThinPlansUseIndexes(t *testing.T) { + _, arch := openLongArchive(t) + span := []any{int64(0), int64(1)} + assertPlanSearches(t, arch, forecastLongNextUnthinned, span, "fa_runs", "fa_runs_unthinned") + assertPlanSearches(t, arch, forecastLongHourLosers, span, "fa_runs", "fa_runs_origin") + assertPlanSearches(t, arch, forecastLongDayRuns, span, "fa_runs", "fa_runs_origin") +} diff --git a/go/internal/state/forecast_long_archive_weather_test.go b/go/internal/state/forecast_long_archive_weather_test.go index da3c704c3..b987fffc9 100644 --- a/go/internal/state/forecast_long_archive_weather_test.go +++ b/go/internal/state/forecast_long_archive_weather_test.go @@ -118,42 +118,43 @@ func TestForecastLongWeatherOutsideSeriesSpanRoundTrips(t *testing.T) { // correlated subquery is what made garbage collection quadratic. func TestForecastLongGarbagePlansUseIndexes(t *testing.T) { _, arch := openLongArchive(t) - for _, tc := range []struct { - query, ref, index string - }{ - {forecastLongWeatherGC, "fa_run_weather", "fa_run_weather_fetch"}, - {forecastLongStateGC, "fa_run_states", "fa_run_states_ref"}, - } { - rows, err := arch.db.Query(`EXPLAIN QUERY PLAN `+tc.query, forecastPruneBatch) - if err != nil { - t.Fatal(err) - } - var plan []string - for rows.Next() { - var id, parent, unused int - var detail string - if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { - t.Fatal(err) - } - plan = append(plan, detail) - } - if err := rows.Err(); err != nil { + assertPlanSearches(t, arch, forecastLongWeatherGC, []any{forecastPruneBatch}, "fa_run_weather", "fa_run_weather_fetch") + assertPlanSearches(t, arch, forecastLongStateGC, []any{forecastPruneBatch}, "fa_run_states", "fa_run_states_ref") +} + +// assertPlanSearches fails unless the query plan reads table through index +// and never scans it. +func assertPlanSearches(t *testing.T, arch *ForecastLongArchive, query string, args []any, table, index string) { + t.Helper() + rows, err := arch.db.Query(`EXPLAIN QUERY PLAN `+query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { t.Fatal(err) } - rows.Close() - searched := false - for _, detail := range plan { - if strings.HasPrefix(detail, "SCAN "+tc.ref) { - t.Fatalf("%s is scanned: %q", tc.ref, plan) - } - if strings.HasPrefix(detail, "SEARCH "+tc.ref+" ") && strings.Contains(detail, "INDEX "+tc.index+" ") { - searched = true - } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + searched := false + for _, detail := range plan { + if strings.HasPrefix(detail, "SCAN "+table) { + t.Fatalf("%s is scanned: %q", table, plan) } - if !searched { - t.Fatalf("%s is not searched through %s: %q", tc.ref, tc.index, plan) + if strings.HasPrefix(detail, "SEARCH "+table+" ") && strings.Contains(detail, "INDEX "+index+" ") { + searched = true } } + if !searched { + t.Fatalf("%s is not searched through %s: %q", table, index, plan) + } } func TestForecastLongGarbageKeepsLinkedWeather(t *testing.T) { @@ -239,3 +240,25 @@ func TestForecastLongWeatherStartsCodec(t *testing.T) { } } } + +// A run must load with every weather row it was issued with. If one linked +// row is gone, LoadIssue fails rather than return a shorter list. +func TestForecastLongLoadFailsWhenLinkedWeatherIsMissing(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + issue := longIssue("gap", origin) + issue.Weather = []forecasting.Weather{longWeather("open_meteo", s, s-hour), longWeather("open_meteo", s+hour, s-hour)} + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + if _, err := arch.db.ExecContext(ctx, `DELETE FROM fa_weather WHERE start_ms=?`, s+hour); err != nil { + t.Fatal(err) + } + got, err := arch.LoadIssue(ctx, "gap") + if err == nil || !strings.Contains(err.Error(), "missing 1 weather rows") { + t.Fatalf("LoadIssue = %d weather rows, err=%v; want a missing-row error", len(got.Weather), err) + } +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go index a090073df..683a5497a 100644 --- a/go/internal/state/forecast_long_copy.go +++ b/go/internal/state/forecast_long_copy.go @@ -344,11 +344,7 @@ func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store, return err } pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) - var page []forecasting.Observation - err = src.visitForecastObservationsAfter(pageCtx, from, until, forecastLongObsPage, func(row forecasting.Observation) error { - page = append(page, row) - return nil - }) + page, err := src.forecastObservationsAfter(pageCtx, from, until, forecastLongObsPage) cancel() if err != nil { return fromForecastSource(err) diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go index 3697cf768..e70b56b80 100644 --- a/go/internal/state/forecast_long_maintain.go +++ b/go/internal/state/forecast_long_maintain.go @@ -3,6 +3,7 @@ package state import ( "context" "database/sql" + "errors" "math" "sort" "time" @@ -39,7 +40,6 @@ func (a *ForecastLongArchive) enforceCap(ctx context.Context, now time.Time) err } type forecastLongDay struct { - key string start int64 end int64 ids []string @@ -92,12 +92,29 @@ func (a *ForecastLongArchive) thin(ctx context.Context, now time.Time) error { } } +// The thinning queries read fa_runs by origin. nextUnthinned runs once per +// local day, so the partial index fa_runs_unthinned spares it a walk over +// every thinned run before the first one left to thin. +const ( + forecastLongNextUnthinned = `SELECT origin_ms, timezone FROM fa_runs + WHERE thinned=0 AND origin_ms>=? AND origin_ms=? AND origin_ms 1` + forecastLongDayRuns = `SELECT run_id, origin_ms, timezone, thinned, + EXISTS(SELECT 1 FROM fa_run_states s WHERE s.run_id=fa_runs.run_id) + FROM fa_runs WHERE origin_ms>=? AND origin_ms=? AND origin_ms=? AND origin_ms 1`, start, end) + rows, err := a.db.QueryContext(ctx, forecastLongHourLosers, start, end) if err != nil { return nil, err } @@ -148,9 +159,7 @@ func (a *ForecastLongArchive) hourLosers(ctx context.Context, start, end int64) // thinDay keeps model-state references on the first run of a complete local // day that has any, drops them from the other runs, and marks the day thinned. func (a *ForecastLongArchive) thinDay(ctx context.Context, start, end int64, key string) error { - rows, err := a.db.QueryContext(ctx, `SELECT run_id, origin_ms, timezone, thinned, - EXISTS(SELECT 1 FROM fa_run_states s WHERE s.run_id=fa_runs.run_id) - FROM fa_runs WHERE origin_ms>=? AND origin_ms 0 { n := forecastPruneBatch if n > len(ids) { @@ -363,11 +374,11 @@ func (a *ForecastLongArchive) deleteWhereIDs(ctx context.Context, table, column } chunk := ids[:n] ids = ids[n:] - if _, err := a.db.ExecContext(ctx, `DELETE FROM `+table+` WHERE `+column+` IN (`+placeholders(len(chunk))+`)`, stringsToArgs(chunk)...); err != nil { + if _, err := a.db.ExecContext(ctx, `DELETE FROM fa_run_states WHERE run_id IN (`+placeholders(len(chunk))+`)`, stringsToArgs(chunk)...); err != nil { return err } if forecastLongAfterDeleteBatch != nil { - forecastLongAfterDeleteBatch(table, len(chunk)) + forecastLongAfterDeleteBatch("fa_run_states", len(chunk)) } if err := pauseMaintenance(ctx); err != nil { return err @@ -452,12 +463,10 @@ func (a *ForecastLongArchive) deleteOrphans(ctx context.Context, query, table st } } +// incrementalVacuum moves free pages to the end of the file and cuts them +// off. In WAL mode the cut lands in the database file only when a checkpoint +// copies it, so the pass ends with one. func (a *ForecastLongArchive) incrementalVacuum(ctx context.Context) error { - // WAL holds the freed pages until a checkpoint, and incremental_vacuum - // can only give back pages that are already on the main freelist. - if _, err := a.db.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`); err != nil { - return err - } deadline := time.Now().Add(forecastLongVacuumBudget) for { if err := ctx.Err(); err != nil { @@ -471,7 +480,6 @@ func (a *ForecastLongArchive) incrementalVacuum(ctx context.Context) error { return err } if free*page < forecastLongFreeTarget || time.Now().After(deadline) { - // The file shrinks when the checkpoint copies the truncation. _, err := a.db.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`) return err } @@ -649,7 +657,3 @@ func stringsToArgs(ids []string) []any { } return args } - -func errorsIsNoRows(err error) bool { - return err == sql.ErrNoRows -} From c9accf261728a7533198a2a5da990474e7aa5e64 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 10:59:35 +0100 Subject: [PATCH 7/9] fix(forecast): keep the archive copying past bad rows and heal it once - Skip an issue or observation the archive cannot store, with a warning, instead of stopping the copy at that row until state.db prunes it. - After a corruption code, move the file aside on the next open without asking quick_check again, and wait until the last read on the dropped handle has ended, so its close cannot delete the new file's WAL. - Health shows a short reason instead of raw errors with host paths, and an archive heal no longer replaces the boot storage event. - Log model states that state.db pruned before the copy. - Name forecast-archive.db in the operations and backup docs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Fredrik Ahlgren --- .changeset/forecast-long-archive.md | 2 +- docs/backup-and-restore.md | 4 +- docs/operations.md | 8 +- go/cmd/ftw/forecast_long_archive.go | 2 +- go/internal/api/api.go | 3 + go/internal/api/api_health_test.go | 20 ++++ go/internal/state/forecast_issues.go | 46 ++++---- go/internal/state/forecast_long_archive.go | 108 ++++++++++++------ .../state/forecast_long_archive_copy_test.go | 45 ++++++++ .../state/forecast_long_archive_open_test.go | 42 +++++++ go/internal/state/forecast_long_copy.go | 27 +++-- go/internal/state/forecast_long_maintain.go | 2 +- go/internal/state/store.go | 6 +- 13 files changed, 245 insertions(+), 70 deletions(-) diff --git a/.changeset/forecast-long-archive.md b/.changeset/forecast-long-archive.md index 919e5ce67..72cf17e2f 100644 --- a/.changeset/forecast-long-archive.md +++ b/.changeset/forecast-long-archive.md @@ -2,6 +2,6 @@ "ftw": patch --- -Keep issued forecasts and quarter-hour load and PV measurements in a local archive for up to two years, separate from planning. +Keep issued forecasts and quarter-hour load and PV measurements in a local archive, separate from planning. The archive keeps every run for 30 days, then one run per hour, and drops its oldest days to stay under 512 MiB. The archive is informational. A problem in it does not mark the box degraded, and a full backup stores a consistent copy rather than the live file. diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index 6d936d762..c16a09cc1 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -53,7 +53,9 @@ known-good copy. The archive leaves out other backups and old rollback points. It carries a consistent copy of `cache.db`: savings history is costed from its past -prices, and a price provider sends only today and tomorrow. Managed-driver +prices, and a price provider sends only today and tomorrow. It also carries +a consistent copy of `forecast-archive.db`; if that copy fails its check, +the backup goes on without it and Core logs a warning. Managed-driver links that point inside the persistent directory become relative links in the archive, so restore can move the data to another directory or machine. Backup never follows these diff --git a/docs/operations.md b/docs/operations.md index 0668fa010..2e5253f24 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -72,6 +72,9 @@ holds: 90 days, 1 h for 5 years) and the energy ledger; - `cache.db` — prices and forecasts; past prices cost the savings history and come back only from a full backup; +- `forecast-archive.db` — issued forecasts and quarter-hour measurements + kept for later analysis, at most 512 MiB; Core moves a damaged file aside + and starts an empty one; - key files such as `nova.key` — the box's identity; keep them with the data; - `drivers/` — custom and managed drivers; - `backups/` — full backups, unless `state.backup_dir` points elsewhere. @@ -95,8 +98,9 @@ remain on the same disk and cannot recover a failed SD card. Core exposes a read-only storage check at `GET /api/storage/inventory`. It reports allocated, live and free SQLite pages -for `state.db` and `cache.db`, their WAL/SHM sizes, and free space on the volume -that contains the data directory. It does not expose paths, scan a storage +for `state.db` and `cache.db`, and for `forecast-archive.db` once Core has +opened it, their WAL/SHM sizes, and free space on the volume that contains +the data directory. It does not expose paths, scan a storage engine's internal files, change retention, checkpoint WAL, run `VACUUM`, or delete data. `GET /api/backups` lists full backups and their sizes. diff --git a/go/cmd/ftw/forecast_long_archive.go b/go/cmd/ftw/forecast_long_archive.go index b541003ab..932260948 100644 --- a/go/cmd/ftw/forecast_long_archive.go +++ b/go/cmd/ftw/forecast_long_archive.go @@ -95,7 +95,7 @@ func (a *forecastLongArchiver) runOnce(ctx context.Context) { } slog.Warn("forecast long archive unavailable", "err", err) a.store.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ - State: "unavailable", CapBytes: state.ForecastLongMaxBytes, LastError: err.Error(), + State: "unavailable", CapBytes: state.ForecastLongMaxBytes, LastError: state.ForecastLongOpenFailed, }) return } diff --git a/go/internal/api/api.go b/go/internal/api/api.go index a78163bef..66d01bd27 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -791,6 +791,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { if s.deps.State != nil { storage := map[string]any{"state": "ok", "cache": "ok", "history": "ok"} for _, ev := range s.deps.State.HealEvents() { + if ev.Tier == "forecast_archive" { + continue // shown in forecast_archive; it must not replace a boot event + } storage[ev.Tier] = ev.Action // "rebuilt" | "restored" storage["last_event_ms"] = ev.AtMs storage["detail"] = ev.Detail diff --git a/go/internal/api/api_health_test.go b/go/internal/api/api_health_test.go index bb699bf7d..33e32a79d 100644 --- a/go/internal/api/api_health_test.go +++ b/go/internal/api/api_health_test.go @@ -136,3 +136,23 @@ func corruptFileAt(t *testing.T, path string, offset int64) { t.Fatal(err) } } + +func TestHealthArchiveHealKeepsBootStorageEvent(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + st.RecordHealEvent(state.HealEvent{Tier: "state", Action: "restored", AtMs: 100, Detail: "state.db restored"}) + st.RecordHealEvent(state.HealEvent{Tier: "forecast_archive", Action: "rebuilt", AtMs: 200, Detail: "archive rebuilt"}) + storage, ok := getHealth(t, st)["storage"].(map[string]any) + if !ok { + t.Fatal("no storage object in /api/health") + } + if storage["state"] != "restored" || storage["detail"] != "state.db restored" || storage["last_event_ms"] != float64(100) { + t.Fatalf("archive heal replaced the boot event: %v", storage) + } + if _, ok := storage["forecast_archive"]; ok { + t.Fatalf("storage carries the archive tier: %v", storage) + } +} diff --git a/go/internal/state/forecast_issues.go b/go/internal/state/forecast_issues.go index 3058681c4..2ae01aa3d 100644 --- a/go/internal/state/forecast_issues.go +++ b/go/internal/state/forecast_issues.go @@ -762,47 +762,51 @@ func (s *Store) LoadForecastModelStateRaw(ctx context.Context, stateID string) ( return expanded, payload, err } +// forecastObservationCopy is one quarter-hour row for the long-term copier. +// Err is set when the payload cannot be read; the copier skips the row. +type forecastObservationCopy struct { + Key forecastLongObsCursor + Observation forecasting.Observation + Err error +} + // forecastObservationsAfter lists quarter-hour truth after the cursor in // (available_at_ms, start_ms, end_ms, config_version) order, no later than // untilMS. The tracker writes rows in available_at_ms order. -func (s *Store) forecastObservationsAfter(ctx context.Context, after forecastLongObsCursor, untilMS int64, limit int) ([]forecasting.Observation, error) { +func (s *Store) forecastObservationsAfter(ctx context.Context, after forecastLongObsCursor, untilMS int64, limit int) ([]forecastObservationCopy, error) { rows, err := s.db.QueryContext(ctx, forecastObservationsAfterQuery, after.AvailableMS, after.StartMS, after.EndMS, after.Config, untilMS, limit) if err != nil { return nil, err } defer rows.Close() - var payloads []string + var out []forecastObservationCopy for rows.Next() { + var row forecastObservationCopy var data string - if err = rows.Scan(&data); err != nil { + if err = rows.Scan(&row.Key.AvailableMS, &row.Key.StartMS, &row.Key.EndMS, &row.Key.Config, &data); err != nil { return nil, err } - payloads = append(payloads, data) + row.Observation, row.Err = decodeForecastObservation(data) + out = append(out, row) } if err = rows.Err(); err != nil { return nil, err } - if err = rows.Close(); err != nil { - return nil, err + return out, rows.Close() +} + +func decodeForecastObservation(data string) (forecasting.Observation, error) { + var observation forecasting.Observation + if len(data) > maxForecastRecordBytes { + return observation, errors.New("oversized forecast observation") } - out := make([]forecasting.Observation, 0, len(payloads)) - for _, data := range payloads { - if len(data) > maxForecastRecordBytes { - return nil, errors.New("oversized forecast observation") - } - var observation forecasting.Observation - if err = json.Unmarshal([]byte(data), &observation); err != nil { - return nil, err - } - if err = observation.Validate(); err != nil { - return nil, err - } - out = append(out, observation) + if err := json.Unmarshal([]byte(data), &observation); err != nil { + return observation, err } - return out, nil + return observation, observation.Validate() } -const forecastObservationsAfterQuery = `SELECT payload FROM forecast_observations +const forecastObservationsAfterQuery = `SELECT available_at_ms, start_ms, end_ms, config_version, payload FROM forecast_observations WHERE (available_at_ms, start_ms, end_ms, config_version) > (?, ?, ?, ?) AND available_at_ms <= ? ORDER BY available_at_ms, start_ms, end_ms, config_version LIMIT ?` diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go index 06c180049..c6c954934 100644 --- a/go/internal/state/forecast_long_archive.go +++ b/go/internal/state/forecast_long_archive.go @@ -163,7 +163,7 @@ func (s *Store) OpenForecastLongArchive(ctx context.Context) (*ForecastLongArchi s.forecastLongOpenMu.Lock() defer s.forecastLongOpenMu.Unlock() s.forecastLongMu.Lock() - cached, closed := s.forecastLong, s.forecastLongClosed + cached, closed, dropped := s.forecastLong, s.forecastLongClosed, s.forecastLongDropped s.forecastLongMu.Unlock() if closed { return nil, errForecastArchiveClosed @@ -171,7 +171,12 @@ func (s *Store) OpenForecastLongArchive(ctx context.Context) (*ForecastLongArchi if cached != nil { return cached, nil } - arch, ev, err := openForecastLongFile(ctx, ForecastLongArchivePath(s.mainDBPath)) + if dropped != nil && dropped.Stats().OpenConnections > 0 { + // Its last connection would delete the WAL by name, which after a + // rebuild belongs to the new file. Try again next hour. + return nil, errors.New("forecast archive: a read on the damaged file has not ended") + } + arch, ev, err := openForecastLongFile(ctx, ForecastLongArchivePath(s.mainDBPath), dropped != nil) if err != nil { return nil, err } @@ -180,6 +185,7 @@ func (s *Store) OpenForecastLongArchive(ctx context.Context) (*ForecastLongArchi closed = s.forecastLongClosed if !closed { s.forecastLong = arch + s.forecastLongDropped = nil } s.forecastLongMu.Unlock() if closed { @@ -206,20 +212,27 @@ func (s *Store) closeForecastLongArchive() error { return arch.Close() } -// dropForecastLongArchive closes a handle that hit damage, so the next -// hourly copy reopens the file, checks it and heals it. Close waits for a -// backup or inventory read that is already running on the handle. +// dropForecastLongArchive closes a handle that returned a corruption code, +// so the next hourly copy moves the file aside and starts an empty one. +// quick_check can miss what the copy hit, so it is not asked again. Close +// does not wait for a backup or inventory read already running on the +// handle; that connection closes when the read ends. func (s *Store) dropForecastLongArchive(arch *ForecastLongArchive) { s.forecastLongMu.Lock() if s.forecastLong == arch { s.forecastLong = nil + s.forecastLongDropped = arch.db } s.forecastLongMu.Unlock() _ = arch.Close() } -func openForecastLongFile(ctx context.Context, path string) (*ForecastLongArchive, *HealEvent, error) { - db, damaged, err := openForecastLongDB(ctx, path) +func openForecastLongFile(ctx context.Context, path string, damagedBefore bool) (*ForecastLongArchive, *HealEvent, error) { + var db *sql.DB + damaged, err := true, errors.New("forecast archive: the last copy hit a corruption code") + if !damagedBefore { + db, damaged, err = openForecastLongDB(ctx, path) + } if err != nil && !damaged { return nil, nil, err } @@ -438,7 +451,7 @@ func (a *ForecastLongArchive) warn(kind string, err error) { } a.mu.Lock() defer a.mu.Unlock() - a.status.LastError = err.Error() + a.status.LastError = forecastLongReason(kind) now := time.Now() if a.warned == nil { a.warned = map[string]time.Time{} @@ -459,19 +472,23 @@ func (a *ForecastLongArchive) publish() { } } -func (a *ForecastLongArchive) setState(state, lastError string) { - a.mu.Lock() - a.status.State = state - if lastError != "" { - a.status.LastError = lastError - } - if state == "ok" || state == "rebuilt" { - a.status.LastError = lastError - } - a.mu.Unlock() - a.publish() +// forecastLongReason is the short text health shows for a failure. Raw +// errors carry host paths, so they go only to the Core log. +func forecastLongReason(kind string) string { + switch kind { + case "immutable": + return "a forecast differs from its archived copy; the archived copy is kept" + case "weather": + return "a weather row differs from its archived copy; the archived copy is kept" + case "decode", "skip-series", "invalid": + return "skipped a row the archive cannot store" + } + return kind + " failed; see the Core log" } +// ForecastLongOpenFailed is the health text when the archive cannot open. +const ForecastLongOpenFailed = "cannot open the archive; see the Core log" + // ArchiveIssues stores one page of issued forecasts. An identical run is left // untouched. A different payload with the same id is rejected and the stored // run is kept. Baselines are not stored; they can be rebuilt from observations @@ -492,12 +509,16 @@ func (a *ForecastLongArchive) ArchiveIssues(ctx context.Context, batch []Forecas var immutable error for _, item := range batch { if err := a.archiveOne(ctx, tx, codes, item); err != nil { - if errors.Is(err, errForecastArchiveImmutable) { + var invalid forecastLongInvalid + switch { + case errors.Is(err, errForecastArchiveImmutable): a.warn("immutable", err) immutable = err - continue + case errors.As(err, &invalid): + a.warn("invalid", err) // skipped; one bad run must not stop the copy + default: + return err } - return err } } if err := tx.Commit(); err != nil { @@ -531,10 +552,19 @@ func (a *ForecastLongArchive) freeShortfall() int64 { var errForecastArchiveDiskFull = errors.New("forecast archive paused: data volume is below the free-space floor") var errForecastArchiveImmutable = errors.New("forecast long archive run is immutable") +// forecastLongInvalid marks an issue the archive cannot store. archiveOne +// returns it before writing any run rows, so the page can skip the issue. +type forecastLongInvalid struct{ err error } + +func (e forecastLongInvalid) Error() string { + return "forecast archive cannot store issue: " + e.err.Error() +} +func (e forecastLongInvalid) Unwrap() error { return e.err } + func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes map[string]byte, item ForecastArchiveIssue) error { issue := item.Issue if err := issue.Validate(); err != nil { - return err + return forecastLongInvalid{err} } kept := make([]forecasting.Series, 0, 4) for _, series := range issue.Series { @@ -548,18 +578,34 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes } states, models, err := forecastLongModelBlobs(issue, item.States) if err != nil { - return err + return forecastLongInvalid{err} } issue.Models = models hash, err := forecastLongInputHash(issue) if err != nil { - return err + return forecastLongInvalid{err} } + var dictErr error encoded, err := encodeKeptSeries(kept, func(kind, value string) (byte, error) { - return forecastLongDict(ctx, tx, codes, kind, value) + code, err := forecastLongDict(ctx, tx, codes, kind, value) + if err != nil { + dictErr = err + } + return code, err }) + if dictErr != nil { + return dictErr + } if err != nil { - return err + return forecastLongInvalid{err} + } + occ, occStart, occN, err := encodeOccupancy(issue.Occupancy) + if err != nil { + return forecastLongInvalid{err} + } + modelsJSON, err := json.Marshal(issue.Models) + if err != nil { + return forecastLongInvalid{err} } var decision, config, existingHash string var origin, issued, latest int64 @@ -597,14 +643,6 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes } a.warn("weather", err) } - occ, occStart, occN, err := encodeOccupancy(issue.Occupancy) - if err != nil { - return err - } - modelsJSON, err := json.Marshal(issue.Models) - if err != nil { - return err - } var siteID, learning, tz sql.NullString var lat, lon sql.NullFloat64 hasLoc := 0 diff --git a/go/internal/state/forecast_long_archive_copy_test.go b/go/internal/state/forecast_long_archive_copy_test.go index ddeee2258..9454dd0fa 100644 --- a/go/internal/state/forecast_long_archive_copy_test.go +++ b/go/internal/state/forecast_long_archive_copy_test.go @@ -303,3 +303,48 @@ func TestForecastLongCopyQueriesUseIndexes(t *testing.T) { } } } + +// One row the archive cannot store is skipped with a short health reason. +// It must not stop the copy, and health must not carry the raw error. +func TestForecastLongCopySkipsRowsItCannotStore(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Truncate(time.Hour).Add(-2 * time.Hour) + // float32 holds no 1e39, so the series cannot be encoded. + wide := longIssue("a-wide", origin) + wide.Series[0].Points[0].PVW = 1e39 + if err := store.SaveForecastIssue(ctx, wide); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastIssue(ctx, longIssue("b-good", origin)); err != nil { + t.Fatal(err) + } + bad := longObservation(origin, time.Minute) + if _, err := store.db.ExecContext(ctx, `INSERT INTO forecast_observations(start_ms,end_ms,available_at_ms,config_version,payload) + VALUES(?,?,?,?,?)`, bad.StartMS, bad.EndMS, bad.AvailableAtMS, bad.ConfigVersion, `{"start_ms":"not a number"}`); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin.Add(15*time.Minute), time.Minute)); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); runs["a-wide"] || !runs["b-good"] { + t.Fatalf("runs = %v, want only b-good", runs) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_observations_15m`); n != 1 { + t.Fatalf("observations = %d, the copy stopped at the bad row", n) + } + cursor, err := arch.obsCursor(ctx) + if err != nil || cursor.StartMS != origin.Add(15*time.Minute).UnixMilli() { + t.Fatalf("observation cursor = %+v, %v; want it past the bad row", cursor, err) + } + status, _ := store.ForecastLongArchiveStatus() + if status.LastError != forecastLongReason("invalid") { + t.Fatalf("health last_error = %q, want a short reason", status.LastError) + } +} diff --git a/go/internal/state/forecast_long_archive_open_test.go b/go/internal/state/forecast_long_archive_open_test.go index cc6ada196..5e6d679fb 100644 --- a/go/internal/state/forecast_long_archive_open_test.go +++ b/go/internal/state/forecast_long_archive_open_test.go @@ -208,3 +208,45 @@ func sqliteErrorOf(t *testing.T) error { } return err } + +// A corruption code that quick_check would not see still heals the file, and +// the rebuild waits for the last read on the dropped handle. +func TestForecastLongDropRebuildsAfterLastRead(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: longIssue("kept", time.Now().UTC().Truncate(time.Hour))}}); err != nil { + t.Fatal(err) + } + // A backup still reading on the handle when Sync drops it. + reading, err := arch.db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + store.dropForecastLongArchive(arch) + if _, err := store.OpenForecastLongArchive(ctx); err == nil { + t.Fatal("open rebuilt the file while a read on the dropped handle was running") + } + if files := quarantinedFiles(t, filepath.Dir(arch.path)); len(files) != 0 { + t.Fatalf("file moved aside under a running read: %v", files) + } + if err := reading.Close(); err != nil { + t.Fatal(err) + } + healed, err := store.OpenForecastLongArchive(ctx) + if err != nil { + t.Fatal(err) + } + if files := quarantinedFiles(t, filepath.Dir(arch.path)); len(files) == 0 { + t.Fatal("file that passed quick_check was not moved aside after a corruption code") + } + if events := store.HealEvents(); len(events) != 1 || events[0].Tier != "forecast_archive" { + t.Fatalf("heal events = %+v", events) + } + if _, err := healed.LoadIssue(ctx, "kept"); err == nil { + t.Fatal("rebuilt archive still holds the old run") + } + again, err := store.OpenForecastLongArchive(ctx) + if err != nil || again != healed { + t.Fatalf("second open = %v, %v; want the cached handle", again, err) + } +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go index 683a5497a..864c4e800 100644 --- a/go/internal/state/forecast_long_copy.go +++ b/go/internal/state/forecast_long_copy.go @@ -280,10 +280,14 @@ func (a *ForecastLongArchive) readIssuePage(ctx context.Context, src *Store, fro a.warn("decode", item.Err) continue } - states, err := forecastLongStatesFrom(ctx, src, item.Issue) + states, missing, err := forecastLongStatesFrom(ctx, src, item.Issue) if err != nil { return batch, err } + if missing > 0 { + // state.db pruned these blobs first; the run keeps its ids. + slog.Info("forecast long archive: model state no longer in state.db", "run", key.ID, "missing", missing) + } batch.issues = append(batch.issues, ForecastArchiveIssue{ Issue: item.Issue, JSONBytes: item.JSONBytes, GzipBytes: item.GzipBytes, States: states, }) @@ -301,8 +305,9 @@ func (a *ForecastLongArchive) hasRun(ctx context.Context, id string) (bool, erro return err == nil, err } -func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.Issue) (map[string]ForecastModelBlob, error) { +func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.Issue) (map[string]ForecastModelBlob, int, error) { states := map[string]ForecastModelBlob{} + missing := 0 for _, model := range issue.Models { if model.StateID == "" { continue @@ -312,14 +317,15 @@ func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.I } expanded, payload, err := src.LoadForecastModelStateRaw(ctx, model.StateID) if errors.Is(err, sql.ErrNoRows) { + missing++ continue } if err != nil { - return nil, fromForecastSource(err) + return nil, 0, fromForecastSource(err) } states[model.StateID] = ForecastModelBlob{Expanded: expanded, Gzip: payload} } - return states, nil + return states, missing, nil } // copyObservations follows the same rules as copyIssues on the @@ -352,10 +358,17 @@ func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store, if len(page) == 0 { return nil } - last := page[len(page)-1] - from = forecastLongObsCursor{AvailableMS: last.AvailableAtMS, StartMS: last.StartMS, EndMS: last.EndMS, Config: last.ConfigVersion} + from = page[len(page)-1].Key + rows := make([]forecasting.Observation, 0, len(page)) + for _, row := range page { + if row.Err != nil { + a.warn("decode", row.Err) + continue + } + rows = append(rows, row.Observation) + } writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) - err = a.ArchiveObservations(writeCtx, page) + err = a.ArchiveObservations(writeCtx, rows) if err == nil && from.after(cursor) { cursor = from err = a.writeObsCursor(writeCtx, cursor) diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go index e70b56b80..124ca60b3 100644 --- a/go/internal/state/forecast_long_maintain.go +++ b/go/internal/state/forecast_long_maintain.go @@ -542,7 +542,7 @@ func (a *ForecastLongArchive) recompute(ctx context.Context, src *Store, now tim } if paused != nil { state = "paused_disk_full" - lastErr = paused.Error() + lastErr = "copying paused: the data volume is full or below the free-space floor" } a.status = ForecastLongArchiveStatus{ State: state, LiveBytes: inv.LiveBytes, FileBytes: inv.FileBytes, WALBytes: inv.WALBytes, diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 74bcf5188..5e577bdd4 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -109,7 +109,11 @@ type Store struct { forecastLongMu sync.Mutex forecastLong *ForecastLongArchive forecastLongClosed bool - forecastLongStatus *ForecastLongArchiveStatus + // forecastLongDropped is a handle Sync closed after a corruption code. + // The next open moves the file aside without quick_check, once the last + // read on that handle has ended. + forecastLongDropped *sql.DB + forecastLongStatus *ForecastLongArchiveStatus } // Open initializes (or creates) the precious state.db at path plus the From fced718d453c4b3774da93df17fb6195b4c48904 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 11:19:42 +0100 Subject: [PATCH 8/9] fix(forecast): keep model states once per hour and cap the archive at 1 GiB The Energyplan state has a fixed shape (56 days of 72 load buckets and 216 PV cells), about 450 KiB of JSON and 75-100 KiB gzip, and the worker writes a new one about every quarter. Kept on every run, it filled about 250 MiB of the newest 30 days, so 512 MiB held only four to six months. Inside the newest 30 days a run now keeps its model states only when it is on the hour, or when it is the first run copied into an hour that has none. Thinning already keeps the run on the hour, and the states between can be rebuilt from the archived observations. With the cap at 1 GiB, two years fit by the same estimate. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Fredrik Ahlgren --- .changeset/forecast-long-archive.md | 2 +- docs/operations.md | 2 +- go/cmd/ftw/forecast_long_archive.go | 2 +- go/internal/state/forecast_long_archive.go | 30 ++++++-- .../state/forecast_long_archive_thin_test.go | 69 +++++++++++++++++-- 5 files changed, 91 insertions(+), 14 deletions(-) diff --git a/.changeset/forecast-long-archive.md b/.changeset/forecast-long-archive.md index 72cf17e2f..9eb39895e 100644 --- a/.changeset/forecast-long-archive.md +++ b/.changeset/forecast-long-archive.md @@ -2,6 +2,6 @@ "ftw": patch --- -Keep issued forecasts and quarter-hour load and PV measurements in a local archive, separate from planning. The archive keeps every run for 30 days, then one run per hour, and drops its oldest days to stay under 512 MiB. +Keep issued forecasts and quarter-hour load and PV measurements in a local archive for up to two years, separate from planning. The archive keeps every run for 30 days, then one run per hour, and drops its oldest days to stay under 1 GiB. The archive is informational. A problem in it does not mark the box degraded, and a full backup stores a consistent copy rather than the live file. diff --git a/docs/operations.md b/docs/operations.md index 2e5253f24..9071d1e87 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -73,7 +73,7 @@ holds: - `cache.db` — prices and forecasts; past prices cost the savings history and come back only from a full backup; - `forecast-archive.db` — issued forecasts and quarter-hour measurements - kept for later analysis, at most 512 MiB; Core moves a damaged file aside + kept for later analysis, at most 1 GiB; Core moves a damaged file aside and starts an empty one; - key files such as `nova.key` — the box's identity; keep them with the data; - `drivers/` — custom and managed drivers; diff --git a/go/cmd/ftw/forecast_long_archive.go b/go/cmd/ftw/forecast_long_archive.go index 932260948..7ee9359fc 100644 --- a/go/cmd/ftw/forecast_long_archive.go +++ b/go/cmd/ftw/forecast_long_archive.go @@ -84,7 +84,7 @@ func (a *forecastLongArchiver) runOnce(ctx context.Context) { return } // No deadline beyond the worker's own context. quick_check of a full - // 512 MiB archive can take minutes on a slow SD card; a fixed timeout + // 1 GiB archive can take minutes on a slow SD card; a fixed timeout // would throw that work away and fail the same way every hour. The open // is off the boot path, health does not wait on it, and shutdown cancels // it through ctx. diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go index c6c954934..1b2a6e786 100644 --- a/go/internal/state/forecast_long_archive.go +++ b/go/internal/state/forecast_long_archive.go @@ -31,9 +31,11 @@ const ( // not a setting. ForecastLongRetention = 2 * 365 * 24 * time.Hour // ForecastLongMaxBytes is the live-page cap. Past it, the oldest days - // outside the full-detail window are deleted. The figure is a margin - // around a size estimate, not a measurement from a box. - ForecastLongMaxBytes = 512 << 20 + // outside the full-detail window are deleted. Sized for two years: the + // newest 30 days take about 160 MiB and a thinned day about 1.2 MiB, + // estimated with a 32 KiB-expanded Energyplan state in a simulation and + // the real state's fixed shape, not measured on a box. + ForecastLongMaxBytes = 1 << 30 // ForecastLongMinFreeBytes is the free-space floor on the data volume. // Below it the copier pauses instead of filling the card, and the // archive deletes its oldest days outside the full-detail window until @@ -625,8 +627,12 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes if !errors.Is(err, sql.ErrNoRows) { return err } + keepStates, err := forecastLongHourNeedsStates(ctx, tx, issue.OriginMS) + if err != nil { + return err + } for id, blob := range states { - if len(blob.Gzip) == 0 { + if len(blob.Gzip) == 0 || !keepStates { continue } if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_model_states(id, created_at_ms, expanded_bytes, payload) VALUES(?,?,?,?)`, @@ -675,6 +681,22 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes return nil } +// forecastLongHourNeedsStates reports whether a run keeps its model states. +// Inside the newest 30 days one run per UTC hour is enough: the run on the +// hour, or the first run copied into an hour that has no states yet. The +// states between can be rebuilt from the archived observations, and the +// Energyplan state alone is about 90 KiB gzip per run. +func forecastLongHourNeedsStates(ctx context.Context, tx *sql.Tx, originMS int64) (bool, error) { + hour := originMS - originMS%forecastLongHourMS + if originMS == hour { + return true, nil + } + var held bool + err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM fa_runs r JOIN fa_run_states s ON s.run_id=r.run_id + WHERE r.origin_ms>=? AND r.origin_ms= start && origin < end && origin%hourMS == 0 { + onHour = append(onHour, ids[origin]) + } + } + if strings.Join(held, ",") != strings.Join(onHour, ",") { + t.Fatalf("pass %s: open day %s has state on %v, want its runs on the hour %v", + cutoff.Format(time.RFC3339), day.Format("2006-01-02"), held, onHour) + } } } } @@ -313,3 +325,46 @@ func TestForecastLongThinPlansUseIndexes(t *testing.T) { assertPlanSearches(t, arch, forecastLongHourLosers, span, "fa_runs", "fa_runs_origin") assertPlanSearches(t, arch, forecastLongDayRuns, span, "fa_runs", "fa_runs_origin") } + +// Inside the newest 30 days one run per UTC hour keeps its model states: the +// run on the hour, or the first run copied into an hour without states. +func TestForecastLongKeepsStatesOncePerHour(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + hour := time.Now().UTC().Truncate(time.Hour).Add(-3 * time.Hour) + for _, run := range []struct { + id string + origin time.Time + }{ + {"a-00", hour}, + {"a-15", hour.Add(15 * time.Minute)}, + {"b-15", hour.Add(75 * time.Minute)}, // the next hour has no run on the hour yet + {"b-30", hour.Add(90 * time.Minute)}, + {"b-00", hour.Add(time.Hour)}, // copied late; a run on the hour always keeps them + } { + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: longIssue(run.id, run.origin)}}); err != nil { + t.Fatalf("%s: %v", run.id, err) + } + } + rows, err := arch.db.Query(`SELECT DISTINCT run_id FROM fa_run_states ORDER BY run_id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var held []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + held = append(held, id) + } + if got := strings.Join(held, ","); got != "a-00,b-00,b-15" { + t.Fatalf("state holders = %s, want a-00,b-00,b-15", got) + } + for _, id := range []string{"a-15", "b-30"} { + if _, err := arch.LoadIssue(ctx, id); err != nil { + t.Fatalf("%s without states does not load: %v", id, err) + } + } +} From 6e446f007da8c69f9d1e9b446a52c11bc9e9cf1d Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Mon, 16 Nov 2026 11:24:32 +0100 Subject: [PATCH 9/9] fix(forecast): keep the hour's state holder when thinning In an hour with no run on the hour, the first run copied holds the model states. A run issued earlier but copied later used to win the hour, so the holder was deleted and the day could lose its states. Hour thinning now ranks a run with states first. The plan test covers the new hour query and checks fa_run_states is never scanned. The cap comment now gives the measured sizes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Fredrik Ahlgren --- go/internal/state/forecast_long_archive.go | 17 +++-- .../state/forecast_long_archive_thin_test.go | 66 +++++++++++++++++++ go/internal/state/forecast_long_maintain.go | 7 +- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go index 1b2a6e786..e41b95245 100644 --- a/go/internal/state/forecast_long_archive.go +++ b/go/internal/state/forecast_long_archive.go @@ -31,10 +31,10 @@ const ( // not a setting. ForecastLongRetention = 2 * 365 * 24 * time.Hour // ForecastLongMaxBytes is the live-page cap. Past it, the oldest days - // outside the full-detail window are deleted. Sized for two years: the - // newest 30 days take about 160 MiB and a thinned day about 1.2 MiB, - // estimated with a 32 KiB-expanded Energyplan state in a simulation and - // the real state's fixed shape, not measured on a box. + // outside the full-detail window are deleted. Sized for two years: in a + // 45-day simulation with an Energyplan state of the real shape (about + // 464 KiB, 87 KiB gzip) the newest 30 days took 160 MiB and a thinned day + // 0.87 MiB, so two years come to about 770 MiB. Not measured on a box. ForecastLongMaxBytes = 1 << 30 // ForecastLongMinFreeBytes is the free-space floor on the data volume. // Below it the copier pauses instead of filling the card, and the @@ -683,7 +683,8 @@ func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes // forecastLongHourNeedsStates reports whether a run keeps its model states. // Inside the newest 30 days one run per UTC hour is enough: the run on the -// hour, or the first run copied into an hour that has no states yet. The +// hour, or the first run copied into an hour that has no states yet. Hour +// thinning ranks a run with states first, so the hour keeps them. The // states between can be rebuilt from the archived observations, and the // Energyplan state alone is about 90 KiB gzip per run. func forecastLongHourNeedsStates(ctx context.Context, tx *sql.Tx, originMS int64) (bool, error) { @@ -692,11 +693,13 @@ func forecastLongHourNeedsStates(ctx context.Context, tx *sql.Tx, originMS int64 return true, nil } var held bool - err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM fa_runs r JOIN fa_run_states s ON s.run_id=r.run_id - WHERE r.origin_ms>=? AND r.origin_ms=? AND r.origin_ms=? AND origin_ms 1` @@ -137,8 +138,8 @@ func forecastLongLocalDay(origin int64, tz string) (start, end int64, key string } // hourLosers ranks every run of each UTC hour in [start, end), thinned or -// not. A thinned survivor keeps its place; otherwise the run on the hour -// wins, then the earliest issued. +// not. A thinned survivor keeps its place; otherwise a run with model states +// wins, then the run on the hour, then the earliest issued. func (a *ForecastLongArchive) hourLosers(ctx context.Context, start, end int64) ([]string, error) { rows, err := a.db.QueryContext(ctx, forecastLongHourLosers, start, end) if err != nil {