diff --git a/.changeset/forecast-long-archive.md b/.changeset/forecast-long-archive.md new file mode 100644 index 000000000..9eb39895e --- /dev/null +++ b/.changeset/forecast-long-archive.md @@ -0,0 +1,7 @@ +--- +"ftw": patch +--- + +Keep issued forecasts and quarter-hour load and PV measurements in a local archive for up to two years, separate from planning. The archive keeps every run for 30 days, then one run per hour, and drops its oldest days to stay under 1 GiB. + +The archive is informational. A problem in it does not mark the box degraded, and a full backup stores a consistent copy rather than the live file. diff --git a/docs/architecture.md b/docs/architecture.md index 2a5dd2d57..93fe750e6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -149,6 +149,10 @@ SQLite history.db owns bucketed history, dashboard summaries and the energy ledger. A separate state.db owns goals, device identity, learned state and planner diagnostics. Prices and forecasts live in cache.db; full backups keep its past prices, which savings history needs. +Issued forecasts and quarter-hour measurements also accumulate in +forecast-archive.db, opened after startup and kept out of the control path. +A full backup stores a consistent copy of that file. A problem there does +not mark the box degraded. Database access stays in [`go/internal/state`](../go/internal/state). diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index 6d936d762..c16a09cc1 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -53,7 +53,9 @@ known-good copy. The archive leaves out other backups and old rollback points. It carries a consistent copy of `cache.db`: savings history is costed from its past -prices, and a price provider sends only today and tomorrow. Managed-driver +prices, and a price provider sends only today and tomorrow. It also carries +a consistent copy of `forecast-archive.db`; if that copy fails its check, +the backup goes on without it and Core logs a warning. Managed-driver links that point inside the persistent directory become relative links in the archive, so restore can move the data to another directory or machine. Backup never follows these diff --git a/docs/operations.md b/docs/operations.md index 0668fa010..9071d1e87 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -72,6 +72,9 @@ holds: 90 days, 1 h for 5 years) and the energy ledger; - `cache.db` — prices and forecasts; past prices cost the savings history and come back only from a full backup; +- `forecast-archive.db` — issued forecasts and quarter-hour measurements + kept for later analysis, at most 1 GiB; Core moves a damaged file aside + and starts an empty one; - key files such as `nova.key` — the box's identity; keep them with the data; - `drivers/` — custom and managed drivers; - `backups/` — full backups, unless `state.backup_dir` points elsewhere. @@ -95,8 +98,9 @@ remain on the same disk and cannot recover a failed SD card. Core exposes a read-only storage check at `GET /api/storage/inventory`. It reports allocated, live and free SQLite pages -for `state.db` and `cache.db`, their WAL/SHM sizes, and free space on the volume -that contains the data directory. It does not expose paths, scan a storage +for `state.db` and `cache.db`, and for `forecast-archive.db` once Core has +opened it, their WAL/SHM sizes, and free space on the volume that contains +the data directory. It does not expose paths, scan a storage engine's internal files, change retention, checkpoint WAL, run `VACUUM`, or delete data. `GET /api/backups` lists full backups and their sizes. diff --git a/go/cmd/ftw/forecast_long_archive.go b/go/cmd/ftw/forecast_long_archive.go new file mode 100644 index 000000000..7ee9359fc --- /dev/null +++ b/go/cmd/ftw/forecast_long_archive.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/srcfl/ftw/go/internal/state" +) + +// forecastLongArchiver copies the 30-day forecast tables into +// forecast-archive.db. It does not share the tracker's queue or its lock. +type forecastLongArchiver struct { + store *state.Store + cancel context.CancelFunc + wg sync.WaitGroup + startDelay time.Duration + interval time.Duration + now func() time.Time +} + +func startForecastLongArchive(ctx context.Context, st *state.Store) *forecastLongArchiver { + a := &forecastLongArchiver{store: st, startDelay: 2 * time.Minute, interval: time.Hour} + a.Start(ctx) + return a +} + +func (a *forecastLongArchiver) Start(ctx context.Context) { + if a == nil || a.store == nil || a.cancel != nil { + return + } + workerCtx, cancel := context.WithCancel(ctx) + a.cancel = cancel + a.wg.Add(1) + go func() { + defer a.wg.Done() + a.run(workerCtx) + }() +} + +func (a *forecastLongArchiver) Stop() { + if a == nil || a.cancel == nil { + return + } + a.cancel() + a.wg.Wait() +} + +func (a *forecastLongArchiver) run(ctx context.Context) { + a.store.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ + State: "starting", CapBytes: state.ForecastLongMaxBytes, + }) + delay := a.startDelay + if delay < 0 { + delay = 0 + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + a.runOnce(ctx) + interval := a.interval + if interval <= 0 { + interval = time.Hour + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.runOnce(ctx) + } + } +} + +func (a *forecastLongArchiver) runOnce(ctx context.Context) { + if a == nil || a.store == nil || ctx.Err() != nil { + return + } + // No deadline beyond the worker's own context. quick_check of a full + // 1 GiB archive can take minutes on a slow SD card; a fixed timeout + // would throw that work away and fail the same way every hour. The open + // is off the boot path, health does not wait on it, and shutdown cancels + // it through ctx. + arch, err := a.store.OpenForecastLongArchive(ctx) + if err != nil { + if ctx.Err() != nil { + return + } + slog.Warn("forecast long archive unavailable", "err", err) + a.store.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ + State: "unavailable", CapBytes: state.ForecastLongMaxBytes, LastError: state.ForecastLongOpenFailed, + }) + return + } + now := time.Now().UTC() + if a.now != nil { + now = a.now().UTC() + } + if err := arch.Sync(ctx, a.store, now); err != nil && ctx.Err() == nil { + slog.Warn("forecast long archive copy", "err", err) + } +} diff --git a/go/cmd/ftw/forecast_long_archive_test.go b/go/cmd/ftw/forecast_long_archive_test.go new file mode 100644 index 000000000..cb031f5a2 --- /dev/null +++ b/go/cmd/ftw/forecast_long_archive_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +func forecastingIssueForArchive(origin time.Time) forecasting.Issue { + start := origin.UnixMilli() + band := forecasting.Band{Method: forecasting.BandMethodColdStart} + point := forecasting.Point{ + StartMS: start, EndMS: start + int64(time.Hour/time.Millisecond), + PVW: 1, LoadW: 2, PVKnown: true, LoadKnown: true, + PVQuality: "ready", LoadQuality: "ready", + PVBand: band, LoadBand: band, NetBand: band, + } + return forecasting.Issue{ + Schema: forecasting.Schema, ID: "issued", DecisionID: "decision", + OriginMS: start, IssuedAtMS: start, LatestInputMS: start, ConfigVersion: "cfg", + Series: []forecasting.Series{{Name: "champion", ModelVersion: "v1", Points: []forecasting.Point{point}}}, + } +} + +func TestForecastLongArchiverStops(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + before := runtime.NumGoroutine() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + archiver := &forecastLongArchiver{store: st, startDelay: 0, interval: time.Hour} + archiver.Start(ctx) + archiver.Stop() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && runtime.NumGoroutine() > before { + time.Sleep(20 * time.Millisecond) + } + if got := runtime.NumGoroutine(); got > before { + t.Fatalf("goroutines leaked: before %d after %d", before, got) + } +} + +func TestForecastLongArchiverPausesWhenDiskIsFull(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err := st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Add(-time.Hour).Truncate(time.Hour) + issue := forecastingIssueForArchive(origin) + if err := st.SaveForecastIssue(context.Background(), issue); err != nil { + t.Fatal(err) + } + restore := state.ForecastLongDiskAvailForTest(func(string) (int64, error) { return 0, nil }) + defer restore() + archiver := &forecastLongArchiver{store: st, startDelay: time.Hour, interval: time.Hour} + archiver.runOnce(context.Background()) + status, ok := st.ForecastLongArchiveStatus() + if !ok || status.State != "paused_disk_full" || status.Runs != 0 { + t.Fatalf("status = %+v ok=%v", status, ok) + } +} + +func TestForecastRecordDoesNotWaitOnLongArchive(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + origin := time.Now().UTC().Add(-time.Hour).Truncate(time.Hour) + if err := st.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + if err := st.SaveForecastIssue(context.Background(), forecastingIssueForArchive(origin)); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + release := make(chan struct{}) + restore := state.ForecastLongDiskAvailForTest(func(string) (int64, error) { + select { + case <-entered: + default: + close(entered) + } + <-release + return 1 << 40, nil + }) + defer restore() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + archiver := &forecastLongArchiver{store: st, startDelay: 0, interval: time.Hour} + archiver.Start(ctx) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("copier did not reach a write") + } + f := trackerFixture(origin) + f.store = st + in := f.Snapshot(origin, trackerWeather(origin.Add(time.Minute), origin)) + slots := trackerSlots(origin.Add(time.Minute), 1) + started := time.Now() + for i := 0; i < 30; i++ { + in.Record(slots, slots, nil, "decision", origin.UnixMilli()) + } + if time.Since(started) > time.Second { + t.Fatal("planner record waited on the long archive") + } + close(release) + archiver.Stop() +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index e5bef56c7..6144f9273 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1667,6 +1667,10 @@ func main() { } else { defer forecastTrackerSvc.Stop() } + // The long-term archive copies the 30-day tables on its own. It starts + // even when the tracker did not, and it never shares the tracker's queue. + forecastLongArchiveSvc := startForecastLongArchive(ctx, st) + defer forecastLongArchiveSvc.Stop() // A small live-identity poll also protects legacy learners when MPC is off. // Config/schema/script work stays in Configure; this only reads host identity. diff --git a/go/internal/api/api.go b/go/internal/api/api.go index 733cdbc29..66d01bd27 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -791,6 +791,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { if s.deps.State != nil { storage := map[string]any{"state": "ok", "cache": "ok", "history": "ok"} for _, ev := range s.deps.State.HealEvents() { + if ev.Tier == "forecast_archive" { + continue // shown in forecast_archive; it must not replace a boot event + } storage[ev.Tier] = ev.Action // "rebuilt" | "restored" storage["last_event_ms"] = ev.AtMs storage["detail"] = ev.Detail @@ -803,6 +806,11 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { if writer.LastError != "" || writer.MaintenanceError != "" || s.deps.State.HistoryMaintenanceStatus().LastError != "" || (writer.LastRejectMS > 0 && time.Now().UnixMilli()-writer.LastRejectMS < time.Minute.Milliseconds()) { resp["status"] = "degraded" } + // The long-term forecast archive is informational. A full disk or a + // rebuilt file must not mark the box degraded. + if archive, ok := s.deps.State.ForecastLongArchiveStatus(); ok { + resp["forecast_archive"] = archive + } } writeJSON(w, 200, resp) } diff --git a/go/internal/api/api_health_test.go b/go/internal/api/api_health_test.go index fcff17835..33e32a79d 100644 --- a/go/internal/api/api_health_test.go +++ b/go/internal/api/api_health_test.go @@ -83,6 +83,32 @@ func TestHealthStorageReportsCacheRebuild(t *testing.T) { } } +func TestHealthForecastArchiveIsInformational(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + body := getHealth(t, st) + if _, ok := body["forecast_archive"]; ok { + t.Fatal("health published an archive block before the copier started") + } + if body["status"] != "ok" { + t.Fatalf("status = %v", body["status"]) + } + st.ReportForecastLongArchive(state.ForecastLongArchiveStatus{ + State: "unavailable", LastError: "forecast archive rebuilt", CapBytes: state.ForecastLongMaxBytes, + }) + body = getHealth(t, st) + if body["status"] != "ok" { + t.Fatalf("archive failure changed box status to %v", body["status"]) + } + block, ok := body["forecast_archive"].(map[string]any) + if !ok || block["state"] != "unavailable" { + t.Fatalf("forecast_archive = %#v", body["forecast_archive"]) + } +} + func checkpoint(t *testing.T, path string) { t.Helper() db, err := sql.Open("sqlite", path) @@ -110,3 +136,23 @@ func corruptFileAt(t *testing.T, path string, offset int64) { t.Fatal(err) } } + +func TestHealthArchiveHealKeepsBootStorageEvent(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + st.RecordHealEvent(state.HealEvent{Tier: "state", Action: "restored", AtMs: 100, Detail: "state.db restored"}) + st.RecordHealEvent(state.HealEvent{Tier: "forecast_archive", Action: "rebuilt", AtMs: 200, Detail: "archive rebuilt"}) + storage, ok := getHealth(t, st)["storage"].(map[string]any) + if !ok { + t.Fatal("no storage object in /api/health") + } + if storage["state"] != "restored" || storage["detail"] != "state.db restored" || storage["last_event_ms"] != float64(100) { + t.Fatalf("archive heal replaced the boot event: %v", storage) + } + if _, ok := storage["forecast_archive"]; ok { + t.Fatalf("storage carries the archive tier: %v", storage) + } +} diff --git a/go/internal/api/api_storage_test.go b/go/internal/api/api_storage_test.go index 06446f975..d769ae324 100644 --- a/go/internal/api/api_storage_test.go +++ b/go/internal/api/api_storage_test.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -54,6 +55,31 @@ func TestHandleStorageInventoryReportsCoreStateAndDataHeadroom(t *testing.T) { } } +func TestHandleStorageInventoryIncludesForecastArchive(t *testing.T) { + dataDir := t.TempDir() + st, err := state.Open(filepath.Join(dataDir, "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := st.OpenForecastLongArchive(context.Background()); err != nil { + t.Fatal(err) + } + srv := New(&Deps{State: st, DataDir: dataDir}) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/api/storage/inventory", nil)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rr.Code, rr.Body.String()) + } + var got storageInventoryResponse + if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Databases.ForecastArchive == nil || got.Databases.ForecastArchive.AllocatedPages == 0 { + t.Fatalf("forecast archive inventory = %+v", got.Databases.ForecastArchive) + } +} + func TestHandleStorageInventoryRequiresPersistentWiring(t *testing.T) { srv := New(&Deps{}) rr := httptest.NewRecorder() diff --git a/go/internal/backup/archive.go b/go/internal/backup/archive.go index d51752584..1621d5abe 100644 --- a/go/internal/backup/archive.go +++ b/go/internal/backup/archive.go @@ -183,6 +183,9 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { if err != nil { return Info{}, err } + // The live archive is excluded above. The staged consistent copy still + // needs room, so count the file that copy is made from. + extraBytes += forecastArchiveBytes(statePath) if err := ensureBackupDiskSpace(outputDir, state.BackupArchiveScratch(opts.State.BackupSourceBytes(), extraBytes)); err != nil { return Info{}, err } @@ -259,6 +262,21 @@ func Create(ctx context.Context, opts CreateOptions) (Info, error) { } else if found { sources = append(sources, sourceEntry{archivePath: "data/" + path.Join(path.Dir(databaseFile), "cache.db"), sourcePath: stagedCache}) } + // Same treatment as cache.db: a consistent copy, never the live file or + // its WAL. A damaged analytics file must not cost the owner the backup. + stagedArchive := filepath.Join(stageDir, state.ForecastLongArchiveFilename) + if found, err := opts.State.BackupForecastLongArchive(ctx, stagedArchive); err != nil || (found && verifyDatabaseContext(ctx, stagedArchive) != nil) { + if ctx.Err() != nil { + return Info{}, ctx.Err() + } + if err == nil { + err = verifyDatabaseContext(ctx, stagedArchive) + } + _ = os.Remove(stagedArchive) + slog.Warn("full backup continues without forecast-archive.db", "err", err) + } else if found { + sources = append(sources, sourceEntry{archivePath: "data/" + path.Join(path.Dir(databaseFile), state.ForecastLongArchiveFilename), sourcePath: stagedArchive}) + } sources = append(sources, sourceEntry{ archivePath: databaseEntry, sourcePath: databaseGzip, @@ -346,6 +364,19 @@ func backupExtraBytes(sources []sourceEntry) (int64, error) { return total, nil } +func forecastArchiveBytes(statePath string) int64 { + path := state.ForecastLongArchivePath(statePath) + var n int64 + for _, suffix := range []string{"", "-wal", "-shm"} { + info, err := os.Lstat(path + suffix) + if err != nil || !info.Mode().IsRegular() { + continue + } + n += info.Size() + } + return n +} + func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) { stateRel, _ := filepath.Rel(dataDir, statePath) cacheRel, _ := filepath.Rel(dataDir, filepath.Join(filepath.Dir(statePath), "cache.db")) @@ -410,6 +441,13 @@ func collectSources(dataDir, statePath, outputDir string) ([]sourceEntry, error) if strings.HasPrefix(rel, stateRel+".snapshot") || rel == cacheRel || strings.HasPrefix(rel, cacheRel+"-") { return nil } + // The long-term forecast archive is staged as a consistent copy above. + // Copying the live file, its WAL, or a quarantined sibling would be + // inconsistent and can vanish during a backup. + archiveRel, _ := filepath.Rel(dataDir, state.ForecastLongArchivePath(statePath)) + if rel == archiveRel || strings.HasPrefix(rel, archiveRel+"-") || strings.HasPrefix(rel, archiveRel+".") { + return nil + } // FTW Cloud's key never goes into a backup. A box restored elsewhere // must not speak for the old box's home: its grant reads as off until // an owner turns FTW Cloud on again, under a new home id. @@ -704,8 +742,12 @@ func verifyInWorkspaceProgressContext(ctx context.Context, archivePath, workspac var writer io.Writer = h var dbFile *os.File parquetPath := "" + sqliteSide := "" if entry.Path == manifest.DatabaseEntry { dbFile, err = os.OpenFile(dbGzip, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + } else if path.Base(entry.Path) == state.ForecastLongArchiveFilename { + sqliteSide = filepath.Join(tmpDir, state.ForecastLongArchiveFilename) + dbFile, err = os.OpenFile(sqliteSide, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) } else if strings.HasSuffix(strings.ToLower(entry.Path), ".parquet") { // Stage and remove one day at a time, not the entire cold archive. parquetPath = filepath.Join(tmpDir, "parquet.tmp") @@ -739,6 +781,14 @@ func verifyInWorkspaceProgressContext(ctx context.Context, archivePath, workspac return Manifest{}, err } } + if sqliteSide != "" { + if err := verifyDatabaseContext(ctx, sqliteSide); err != nil { + return Manifest{}, fmt.Errorf("backup: forecast archive: %w", err) + } + if err := os.Remove(sqliteSide); err != nil { + return Manifest{}, err + } + } } if len(seen) != len(want) { return Manifest{}, errors.New("backup: archive is missing one or more manifest files") @@ -1222,7 +1272,16 @@ func extractArchive(archivePath, staging string) error { if len(seen) != len(want) { return errors.New("backup: archive is missing one or more manifest files during extraction") } - return verifyDatabase(filepath.Join(staging, filepath.FromSlash(manifest.DatabaseFile))) + if err := verifyDatabase(filepath.Join(staging, filepath.FromSlash(manifest.DatabaseFile))); err != nil { + return err + } + archived := filepath.Join(staging, filepath.Dir(filepath.FromSlash(manifest.DatabaseFile)), state.ForecastLongArchiveFilename) + if _, err := os.Stat(archived); err == nil { + if err := verifyDatabase(archived); err != nil { + return fmt.Errorf("backup: forecast archive: %w", err) + } + } + return nil } func verifyCompressedDatabase(src, workDir string) error { diff --git a/go/internal/backup/forecast_archive_backup_test.go b/go/internal/backup/forecast_archive_backup_test.go new file mode 100644 index 000000000..195c7511c --- /dev/null +++ b/go/internal/backup/forecast_archive_backup_test.go @@ -0,0 +1,197 @@ +package backup + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" + "github.com/srcfl/ftw/go/internal/state" +) + +func backupForecastIssue(origin time.Time) forecasting.Issue { + start := origin.UnixMilli() + band := forecasting.Band{Method: forecasting.BandMethodColdStart} + point := forecasting.Point{ + StartMS: start, EndMS: start + int64(time.Hour/time.Millisecond), + PVW: 12, LoadW: 34, PVKnown: true, LoadKnown: true, + PVQuality: "ready", LoadQuality: "ready", + PVBand: band, LoadBand: band, NetBand: band, + } + return forecasting.Issue{ + Schema: forecasting.Schema, ID: "backed-up", DecisionID: "decision", + OriginMS: start, IssuedAtMS: start, LatestInputMS: start, ConfigVersion: "cfg", + Series: []forecasting.Series{{Name: "champion", ModelVersion: "v1", Points: []forecasting.Point{point}}}, + } +} + +func TestFullBackupIncludesConsistentForecastArchive(t *testing.T) { + for _, offline := range []bool{false, true} { + t.Run(map[bool]string{false: "live", true: "offline"}[offline], func(t *testing.T) { + root := t.TempDir() + dataDir := filepath.Join(root, "data") + if err := os.MkdirAll(dataDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dataDir, "state.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + arch, err := st.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + origin := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) + if err := arch.ArchiveIssues(context.Background(), []state.ForecastArchiveIssue{{Issue: backupForecastIssue(origin)}}); err != nil { + t.Fatal(err) + } + src := st + if offline { + if err := st.Close(); err != nil { + t.Fatal(err) + } + src, err = state.OpenBackupSource(statePath) + if err != nil { + t.Fatal(err) + } + } + defer src.Close() + info, err := Create(context.Background(), CreateOptions{ + State: src, StatePath: statePath, DataDir: dataDir, + OutputDir: filepath.Join(root, "backups"), + }) + if err != nil { + t.Fatal(err) + } + manifest, _, err := Inspect(context.Background(), info.Path) + if err != nil { + t.Fatal(err) + } + found := false + for _, file := range manifest.Files { + if file.Path == "data/forecast-archive.db" { + found = true + } + if file.Path == "data/forecast-archive.db-wal" || file.Path == "data/forecast-archive.db-shm" { + t.Fatalf("backup contains a live sidecar: %s", file.Path) + } + } + if !found { + t.Fatal("backup is missing forecast-archive.db") + } + restoredDir := filepath.Join(root, "restored") + if _, err := restoreFresh(info.Path, restoredDir, time.Time{}); err != nil { + t.Fatal(err) + } + restored, err := state.Open(filepath.Join(restoredDir, "state.db")) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + restoredArch, err := restored.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + loaded, err := restoredArch.LoadIssue(context.Background(), "backed-up") + if err != nil { + t.Fatal(err) + } + if loaded.Series[0].Points[0].LoadW != 34 { + t.Fatalf("restored issue = %+v", loaded.Series[0].Points[0]) + } + }) + } +} + +func TestFullBackupSkipsDamagedForecastArchive(t *testing.T) { + root := t.TempDir() + dataDir := filepath.Join(root, "data") + if err := os.MkdirAll(dataDir, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dataDir, "state.db") + st, err := state.Open(statePath) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err := os.WriteFile(state.ForecastLongArchivePath(statePath), []byte("not a database"), 0o600); err != nil { + t.Fatal(err) + } + // Copies an earlier heal moved aside stay out of the backup too. + for _, name := range []string{"forecast-archive.db.corrupt-1", "forecast-archive.db-wal.corrupt-1", "forecast-archive.db-shm.corrupt-1"} { + if err := os.WriteFile(filepath.Join(dataDir, name), []byte("quarantined"), 0o600); err != nil { + t.Fatal(err) + } + } + info, err := Create(context.Background(), CreateOptions{ + State: st, StatePath: statePath, DataDir: dataDir, + OutputDir: filepath.Join(root, "backups"), + }) + if err != nil { + t.Fatal(err) + } + f, err := os.Open(info.Path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + zr, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer zr.Close() + tr := tar.NewReader(zr) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(header.Name, "data/forecast-archive.db") { + t.Fatalf("damaged or quarantined archive was copied: %s", header.Name) + } + } +} + +func TestBackupPreflightCountsForecastArchive(t *testing.T) { + root := t.TempDir() + st, err := state.Open(filepath.Join(root, "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + archivePath := state.ForecastLongArchivePath(filepath.Join(root, "state.db")) + const archiveBytes = 1 << 20 + if err := os.WriteFile(archivePath, make([]byte, archiveBytes), 0o600); err != nil { + t.Fatal(err) + } + old := ensureBackupDiskSpace + t.Cleanup(func() { ensureBackupDiskSpace = old }) + var needed int64 + ensureBackupDiskSpace = func(dir string, n int64) error { + needed = n + return os.ErrPermission + } + _, err = Create(context.Background(), CreateOptions{ + State: st, StatePath: filepath.Join(root, "state.db"), DataDir: root, + OutputDir: filepath.Join(root, "backups"), + }) + if err == nil { + t.Fatal("preflight did not run") + } + minimum := state.BackupArchiveScratch(st.BackupSourceBytes(), archiveBytes) + if needed < minimum { + t.Fatalf("space check %d omits the %d byte forecast archive (minimum %d)", needed, archiveBytes, minimum) + } +} diff --git a/go/internal/state/backup_state.go b/go/internal/state/backup_state.go index 16dcff2d3..00a11c3bc 100644 --- a/go/internal/state/backup_state.go +++ b/go/internal/state/backup_state.go @@ -99,6 +99,40 @@ func (s *Store) BackupCache(ctx context.Context, dstPath string) (bool, error) { return true, nil } +// BackupForecastLongArchive writes a consistent, compact copy of +// forecast-archive.db. It reports false when the archive has not been +// created. The caller stages that copy and leaves the live file out of the +// archive, including its WAL. +func (s *Store) BackupForecastLongArchive(ctx context.Context, dstPath string) (bool, error) { + if s == nil { + return false, nil + } + s.forecastLongMu.Lock() + arch := s.forecastLong + s.forecastLongMu.Unlock() + var db *sql.DB + if arch != nil { + db = arch.db + } else { + path := ForecastLongArchivePath(s.mainDBPath) + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + ro, err := sql.Open("sqlite", ReadOnlyDatabaseURI(path)) + if err != nil { + return false, err + } + defer ro.Close() + db = ro + } + if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, dstPath); err != nil { + return false, fmt.Errorf("backup forecast archive: %w", err) + } + return true, nil +} + // BackupSourceBytes is the on-disk size of state and history files, including // WAL and SHM. Used to preflight scratch space before a portable export. func (s *Store) BackupSourceBytes() int64 { diff --git a/go/internal/state/forecast_issues.go b/go/internal/state/forecast_issues.go index e8a4af11e..2ae01aa3d 100644 --- a/go/internal/state/forecast_issues.go +++ b/go/internal/state/forecast_issues.go @@ -703,3 +703,110 @@ func (s *Store) LoadForecastErrors(ctx context.Context, since, until int64, hour } return out, nil } + +// ForecastIssueCopy is one row from the 30-day issue archive. +// Err is set when the row cannot be decoded; the copier skips it and moves on. +type ForecastIssueCopy struct { + Issue forecasting.Issue + IssuedAtMS int64 + ID string + JSONBytes int + GzipBytes int + Err error +} + +// forecastIssueKeysAfter lists issues after the (issued_at_ms, id) cursor and +// no later than untilMS, oldest first. It reads no payloads, so the long-term +// copier can skip runs it already holds. +func (s *Store) forecastIssueKeysAfter(ctx context.Context, after forecastLongIssueCursor, untilMS int64, limit int) ([]forecastLongIssueCursor, error) { + rows, err := s.db.QueryContext(ctx, forecastIssueKeysQuery, after.IssuedAtMS, after.ID, untilMS, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var keys []forecastLongIssueCursor + for rows.Next() { + var key forecastLongIssueCursor + if err := rows.Scan(&key.IssuedAtMS, &key.ID); err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, rows.Err() +} + +const forecastIssueKeysQuery = `SELECT issued_at_ms, id FROM forecast_issues + WHERE (issued_at_ms, id) > (?, ?) AND issued_at_ms <= ? + ORDER BY issued_at_ms, id LIMIT ?` + +// forecastIssueCopy reads and decodes one issue. It returns sql.ErrNoRows +// when retention removed the row after it was listed. +func (s *Store) forecastIssueCopy(ctx context.Context, key forecastLongIssueCursor) (ForecastIssueCopy, error) { + var data []byte + if err := s.db.QueryRowContext(ctx, `SELECT payload FROM forecast_issues WHERE id=?`, key.ID).Scan(&data); err != nil { + return ForecastIssueCopy{}, err + } + item := ForecastIssueCopy{IssuedAtMS: key.IssuedAtMS, ID: key.ID, GzipBytes: len(data)} + item.Issue, item.JSONBytes, item.Err = decodeForecastIssue(data) + return item, nil +} + +// LoadForecastModelStateRaw returns the gzip bytes stored for a model state. +// The long-term archive copies them unchanged. +func (s *Store) LoadForecastModelStateRaw(ctx context.Context, stateID string) (expanded int, payload []byte, err error) { + decodedID, err := hex.DecodeString(stateID) + if err != nil || len(decodedID) != sha256.Size { + return 0, nil, errors.New("invalid forecast model state ID") + } + err = s.db.QueryRowContext(ctx, `SELECT expanded_bytes, payload FROM forecast_model_states WHERE id=?`, stateID).Scan(&expanded, &payload) + return expanded, payload, err +} + +// forecastObservationCopy is one quarter-hour row for the long-term copier. +// Err is set when the payload cannot be read; the copier skips the row. +type forecastObservationCopy struct { + Key forecastLongObsCursor + Observation forecasting.Observation + Err error +} + +// forecastObservationsAfter lists quarter-hour truth after the cursor in +// (available_at_ms, start_ms, end_ms, config_version) order, no later than +// untilMS. The tracker writes rows in available_at_ms order. +func (s *Store) forecastObservationsAfter(ctx context.Context, after forecastLongObsCursor, untilMS int64, limit int) ([]forecastObservationCopy, error) { + rows, err := s.db.QueryContext(ctx, forecastObservationsAfterQuery, + after.AvailableMS, after.StartMS, after.EndMS, after.Config, untilMS, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []forecastObservationCopy + for rows.Next() { + var row forecastObservationCopy + var data string + if err = rows.Scan(&row.Key.AvailableMS, &row.Key.StartMS, &row.Key.EndMS, &row.Key.Config, &data); err != nil { + return nil, err + } + row.Observation, row.Err = decodeForecastObservation(data) + out = append(out, row) + } + if err = rows.Err(); err != nil { + return nil, err + } + return out, rows.Close() +} + +func decodeForecastObservation(data string) (forecasting.Observation, error) { + var observation forecasting.Observation + if len(data) > maxForecastRecordBytes { + return observation, errors.New("oversized forecast observation") + } + if err := json.Unmarshal([]byte(data), &observation); err != nil { + return observation, err + } + return observation, observation.Validate() +} + +const forecastObservationsAfterQuery = `SELECT available_at_ms, start_ms, end_ms, config_version, payload FROM forecast_observations + WHERE (available_at_ms, start_ms, end_ms, config_version) > (?, ?, ?, ?) AND available_at_ms <= ? + ORDER BY available_at_ms, start_ms, end_ms, config_version LIMIT ?` diff --git a/go/internal/state/forecast_long_archive.go b/go/internal/state/forecast_long_archive.go new file mode 100644 index 000000000..e41b95245 --- /dev/null +++ b/go/internal/state/forecast_long_archive.go @@ -0,0 +1,1403 @@ +package state + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "math" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +const ( + // ForecastLongArchiveFilename is the long-term forecast archive beside state.db. + ForecastLongArchiveFilename = "forecast-archive.db" + + // ForecastLongFullDetail keeps every issued run. Older runs are thinned + // to one per UTC hour. It matches the 30-day issue archive in state.db. + ForecastLongFullDetail = ForecastIssueRetention + // ForecastLongRetention is two 365-day years. It is a fixed storage policy, + // not a setting. + ForecastLongRetention = 2 * 365 * 24 * time.Hour + // ForecastLongMaxBytes is the live-page cap. Past it, the oldest days + // outside the full-detail window are deleted. Sized for two years: in a + // 45-day simulation with an Energyplan state of the real shape (about + // 464 KiB, 87 KiB gzip) the newest 30 days took 160 MiB and a thinned day + // 0.87 MiB, so two years come to about 770 MiB. Not measured on a box. + ForecastLongMaxBytes = 1 << 30 + // ForecastLongMinFreeBytes is the free-space floor on the data volume. + // Below it the copier pauses instead of filling the card, and the + // archive deletes its oldest days outside the full-detail window until + // it has given back the shortfall. + ForecastLongMinFreeBytes = 1 << 30 + + forecastLongVacuumPages = 256 + forecastLongVacuumBudget = 30 * time.Second + forecastLongFreeTarget = 1 << 20 + forecastLongPageTimeout = 6 * time.Second + forecastLongWarnGap = time.Hour +) + +// ForecastLongArchiveStatus is the cached view published for /api/health. +// Readers never scan the archive. +type ForecastLongArchiveStatus struct { + State string `json:"state"` + LiveBytes int64 `json:"live_bytes"` + FileBytes int64 `json:"file_bytes"` + WALBytes int64 `json:"wal_bytes"` + CapBytes int64 `json:"cap_bytes"` + OldestOriginMS int64 `json:"oldest_origin_ms"` + NewestOriginMS int64 `json:"newest_origin_ms"` + Runs int64 `json:"runs"` + Coverage7d float64 `json:"coverage_7d"` + Coverage30d float64 `json:"coverage_30d"` + CoverageAll float64 `json:"coverage_all"` + TruthCoverage30d float64 `json:"truth_coverage_30d"` + CopyLagMS int64 `json:"copy_lag_ms"` + LastCopyMS int64 `json:"last_copy_ms"` + LastError string `json:"last_error"` + IssueBytesP50 int64 `json:"issue_bytes_p50"` + IssueBytesP95 int64 `json:"issue_bytes_p95"` + IssueGzipBytesP50 int64 `json:"issue_gzip_bytes_p50"` +} + +// ForecastArchiveIssue is one issued forecast plus the gzip model states +// already stored in state.db. Passing those bytes avoids compressing them again. +type ForecastArchiveIssue struct { + Issue forecasting.Issue + JSONBytes int + GzipBytes int + States map[string]ForecastModelBlob +} + +// ForecastModelBlob is the gzip payload stored for one model-state id. +type ForecastModelBlob struct { + Expanded int + Gzip []byte +} + +// ForecastLongArchive is the long-term forecast file. It is opened by the +// background copier, not during boot. +type ForecastLongArchive struct { + db *sql.DB + path string + store *Store + + mu sync.Mutex + status ForecastLongArchiveStatus + rebuilt bool + warned map[string]time.Time + + maxBytes int64 + retention time.Duration + fullDetail time.Duration + minFree int64 + maintainEvery time.Duration +} + +var forecastLongDiskAvail = diskAvail + +// forecastLongBeforeWrite is a test hook. A SQLITE_FULL error pauses copying. +var forecastLongBeforeWrite func() error + +// ForecastLongDiskAvailForTest replaces the free-space probe. The returned +// function restores it. +func ForecastLongDiskAvailForTest(fn func(string) (int64, error)) func() { + prev := forecastLongDiskAvail + if fn == nil { + forecastLongDiskAvail = diskAvail + } else { + forecastLongDiskAvail = fn + } + return func() { forecastLongDiskAvail = prev } +} + +// ForecastLongArchivePath is forecast-archive.db beside a state.db path. +func ForecastLongArchivePath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), ForecastLongArchiveFilename) +} + +// ReportForecastLongArchive publishes the informational health block. It does +// not change control, dispatch or the top-level box status. +func (s *Store) ReportForecastLongArchive(st ForecastLongArchiveStatus) { + if s == nil { + return + } + s.forecastLongMu.Lock() + s.forecastLongStatus = &st + s.forecastLongMu.Unlock() +} + +// ForecastLongArchiveStatus returns the last published block. The second +// result is false until the copier has published one, so health can omit it. +func (s *Store) ForecastLongArchiveStatus() (ForecastLongArchiveStatus, bool) { + if s == nil { + return ForecastLongArchiveStatus{}, false + } + s.forecastLongMu.Lock() + defer s.forecastLongMu.Unlock() + if s.forecastLongStatus == nil { + return ForecastLongArchiveStatus{}, false + } + return *s.forecastLongStatus, true +} + +// OpenForecastLongArchive opens or creates the archive after startup. A +// damaged file is moved aside and replaced with an empty one. Boot does not +// call this. The quick_check can take minutes on an SD card, so it runs +// without forecastLongMu: health, inventory, backup and Close never wait +// for it. forecastLongOpenMu keeps two opens apart. +func (s *Store) OpenForecastLongArchive(ctx context.Context) (*ForecastLongArchive, error) { + if s == nil || s.mainDBPath == "" { + return nil, errors.New("forecast archive: state store unavailable") + } + s.forecastLongOpenMu.Lock() + defer s.forecastLongOpenMu.Unlock() + s.forecastLongMu.Lock() + cached, closed, dropped := s.forecastLong, s.forecastLongClosed, s.forecastLongDropped + s.forecastLongMu.Unlock() + if closed { + return nil, errForecastArchiveClosed + } + if cached != nil { + return cached, nil + } + if dropped != nil && dropped.Stats().OpenConnections > 0 { + // Its last connection would delete the WAL by name, which after a + // rebuild belongs to the new file. Try again next hour. + return nil, errors.New("forecast archive: a read on the damaged file has not ended") + } + arch, ev, err := openForecastLongFile(ctx, ForecastLongArchivePath(s.mainDBPath), dropped != nil) + if err != nil { + return nil, err + } + arch.store = s + s.forecastLongMu.Lock() + closed = s.forecastLongClosed + if !closed { + s.forecastLong = arch + s.forecastLongDropped = nil + } + s.forecastLongMu.Unlock() + if closed { + _ = arch.Close() + return nil, errForecastArchiveClosed + } + if ev != nil { + s.RecordHealEvent(*ev) + } + return arch, nil +} + +var errForecastArchiveClosed = errors.New("forecast archive: state store closed") + +func (s *Store) closeForecastLongArchive() error { + s.forecastLongMu.Lock() + arch := s.forecastLong + s.forecastLong = nil + s.forecastLongClosed = true + s.forecastLongMu.Unlock() + if arch == nil { + return nil + } + return arch.Close() +} + +// dropForecastLongArchive closes a handle that returned a corruption code, +// so the next hourly copy moves the file aside and starts an empty one. +// quick_check can miss what the copy hit, so it is not asked again. Close +// does not wait for a backup or inventory read already running on the +// handle; that connection closes when the read ends. +func (s *Store) dropForecastLongArchive(arch *ForecastLongArchive) { + s.forecastLongMu.Lock() + if s.forecastLong == arch { + s.forecastLong = nil + s.forecastLongDropped = arch.db + } + s.forecastLongMu.Unlock() + _ = arch.Close() +} + +func openForecastLongFile(ctx context.Context, path string, damagedBefore bool) (*ForecastLongArchive, *HealEvent, error) { + var db *sql.DB + damaged, err := true, errors.New("forecast archive: the last copy hit a corruption code") + if !damagedBefore { + db, damaged, err = openForecastLongDB(ctx, path) + } + if err != nil && !damaged { + return nil, nil, err + } + var ev *HealEvent + if damaged { + nowMs := time.Now().UnixMilli() + slog.Warn("forecast long archive damaged; moving it aside", "err", err) + if err := quarantine(path, nowMs); err != nil { + return nil, nil, err + } + removeOlderForecastLongQuarantine(path, nowMs) + db, _, err = openForecastLongDB(ctx, path) + if err != nil { + return nil, nil, err + } + ev = &HealEvent{ + Tier: "forecast_archive", Action: healRebuilt, AtMs: nowMs, + Detail: "forecast-archive.db was damaged — moved aside and started an empty forecast archive", + } + } + if err := initForecastLongSchema(ctx, db); err != nil { + _ = db.Close() + return nil, nil, err + } + arch := &ForecastLongArchive{ + db: db, path: path, + status: ForecastLongArchiveStatus{State: "starting", CapBytes: ForecastLongMaxBytes}, + warned: map[string]time.Time{}, + maxBytes: ForecastLongMaxBytes, + retention: ForecastLongRetention, + fullDetail: ForecastLongFullDetail, + minFree: ForecastLongMinFreeBytes, + maintainEvery: 24 * time.Hour, + } + if ev != nil { + arch.rebuilt = true + arch.status.State = "rebuilt" + } + return arch, ev, nil +} + +// openForecastLongDB reports damage only when quick_check ran and found +// problems or SQLite returned a corruption code. Busy, I/O, full-disk, +// memory and permission errors leave a good file in place; the copier +// reports unavailable and tries again next hour. +func openForecastLongDB(ctx context.Context, path string) (*sql.DB, bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + db, err := openRawPragmas(path, forecastLongPragmas) + if err != nil { + return nil, false, err + } + if err := prepareForecastLongFile(ctx, db); err != nil { + _ = db.Close() + return nil, false, err + } + return db, false, nil + } + db, err := openRawPragmas(path, forecastLongPragmas) + if err != nil { + return nil, sqliteCorrupt(err), err + } + ok, qerr := quickCheckContext(ctx, db) + if qerr == nil && ok { + if err := prepareForecastLongFile(ctx, db); err != nil { + _ = db.Close() + return nil, false, err + } + return db, false, nil + } + _ = db.Close() + if qerr == nil { + return nil, true, errors.New("forecast archive: quick_check found problems") + } + if ctx.Err() != nil { + return nil, false, ctx.Err() + } + return nil, sqliteCorrupt(qerr), qerr +} + +// removeOlderForecastLongQuarantine keeps only the copy just moved aside, so +// repeated damage cannot fill the card. Backups never include these files. +func removeOlderForecastLongQuarantine(path string, keepMs int64) { + dir, name := filepath.Split(path) + entries, err := os.ReadDir(dir) + if err != nil { + slog.Warn("forecast long archive: cannot list old quarantined files", "err", err) + return + } + keep := fmt.Sprintf(".corrupt-%d", keepMs) + for _, entry := range entries { + for _, base := range []string{name, name + "-wal", name + "-shm"} { + n := entry.Name() + if !strings.HasPrefix(n, base+".corrupt-") || n == base+keep { + continue + } + if err := os.Remove(filepath.Join(dir, n)); err != nil { + slog.Warn("forecast long archive: cannot remove old quarantined file", "file", n, "err", err) + } + } + } +} + +const forecastLongPragmas = sqlitePragmas + "&_pragma=journal_size_limit(4194304)&_pragma=auto_vacuum(INCREMENTAL)" + +func prepareForecastLongFile(ctx context.Context, db *sql.DB) error { + if _, err := db.ExecContext(ctx, `PRAGMA journal_size_limit=4194304`); err != nil { + return err + } + var mode, tables int + if err := db.QueryRowContext(ctx, `PRAGMA auto_vacuum`).Scan(&mode); err != nil { + return err + } + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`).Scan(&tables); err != nil { + return err + } + if tables == 0 { + // A connection-string pragma can report incremental vacuum before the + // file has a pointer map. Vacuuming the empty file makes later + // incremental_vacuum calls actually return pages. + if mode != 2 { + if _, err := db.ExecContext(ctx, `PRAGMA auto_vacuum=INCREMENTAL`); err != nil { + return err + } + } + if _, err := db.ExecContext(ctx, `VACUUM`); err != nil { + return err + } + } + return nil +} + +func initForecastLongSchema(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS fa_migrations(name TEXT PRIMARY KEY, completed_at TEXT DEFAULT current_timestamp) STRICT; +CREATE TABLE IF NOT EXISTS fa_meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) STRICT; +CREATE TABLE IF NOT EXISTS fa_dict(code INTEGER PRIMARY KEY, kind TEXT NOT NULL, value TEXT NOT NULL, UNIQUE(kind, value)) STRICT; +CREATE TABLE IF NOT EXISTS fa_runs( + run_id TEXT PRIMARY KEY, + decision_id TEXT NOT NULL, + origin_ms INTEGER NOT NULL, issued_at_ms INTEGER NOT NULL, latest_input_ms INTEGER NOT NULL, + config_version TEXT NOT NULL, + site_id TEXT, learning_revision TEXT, timezone TEXT, + has_location INTEGER NOT NULL DEFAULT 0, latitude REAL, longitude REAL, + models TEXT NOT NULL, + occupancy BLOB, occupancy_start_ms INTEGER, occupancy_n INTEGER, + input_hash TEXT NOT NULL, + thinned INTEGER NOT NULL DEFAULT 0, + json_bytes INTEGER NOT NULL DEFAULT 0, + gzip_bytes INTEGER NOT NULL DEFAULT 0 +) STRICT; +CREATE INDEX IF NOT EXISTS fa_runs_origin ON fa_runs(origin_ms); +CREATE INDEX IF NOT EXISTS fa_runs_issued ON fa_runs(issued_at_ms); +CREATE INDEX IF NOT EXISTS fa_runs_unthinned ON fa_runs(origin_ms) WHERE thinned=0; +CREATE TABLE IF NOT EXISTS fa_series( + run_id TEXT NOT NULL, series TEXT NOT NULL, model_version TEXT NOT NULL, + start_ms INTEGER NOT NULL, slot_ms INTEGER NOT NULL, n INTEGER NOT NULL, + starts BLOB, prediction_start_ms INTEGER, + pv_w BLOB NOT NULL, load_w BLOB NOT NULL, + flags BLOB NOT NULL, + pv_quality BLOB NOT NULL, load_quality BLOB NOT NULL, + pv_source BLOB, load_source BLOB, + pv_lo BLOB, pv_hi BLOB, load_lo BLOB, load_hi BLOB, net_lo BLOB, net_hi BLOB, + band_method BLOB, band_stats BLOB, + PRIMARY KEY(run_id, series) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_weather( + source TEXT NOT NULL, available_at_ms INTEGER NOT NULL, start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, + ghi REAL, cloud REAL, temp REAL, direct_pv REAL, est_pv REAL, + PRIMARY KEY(source, available_at_ms, start_ms) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_run_weather( + run_id TEXT NOT NULL, source TEXT NOT NULL, available_at_ms INTEGER NOT NULL, + starts BLOB NOT NULL, + PRIMARY KEY(run_id, source, available_at_ms) +) STRICT, WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS fa_run_weather_fetch ON fa_run_weather(source, available_at_ms); +CREATE TABLE IF NOT EXISTS fa_observations_15m( + start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, config_version TEXT NOT NULL, + available_at_ms INTEGER NOT NULL, pv_w REAL NOT NULL, load_w REAL NOT NULL, + pv_known INTEGER NOT NULL, load_known INTEGER NOT NULL, quality TEXT NOT NULL, + PRIMARY KEY(start_ms, end_ms, config_version) +) STRICT, WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS fa_model_states( + id TEXT PRIMARY KEY, created_at_ms INTEGER NOT NULL, expanded_bytes INTEGER NOT NULL, payload BLOB NOT NULL +) STRICT; +CREATE TABLE IF NOT EXISTS fa_run_states( + run_id TEXT NOT NULL, state_id TEXT NOT NULL, + PRIMARY KEY(run_id, state_id) +) STRICT, WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS fa_run_states_ref ON fa_run_states(state_id); +INSERT OR IGNORE INTO fa_migrations(name) VALUES('fa-v1'); +`) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, `INSERT OR IGNORE INTO fa_meta(key, value) VALUES('created_at_ms', ?)`, fmt.Sprint(time.Now().UnixMilli())) + return err +} + +func sqliteFull(err error) bool { + var code interface{ Code() int } + return errors.As(err, &code) && code.Code()&0xff == 13 +} + +// Close releases the archive file. Store.Close does this before cache.db. +func (a *ForecastLongArchive) Close() error { + if a == nil || a.db == nil { + return nil + } + return a.db.Close() +} + +func (a *ForecastLongArchive) warn(kind string, err error) { + if err == nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.status.LastError = forecastLongReason(kind) + now := time.Now() + if a.warned == nil { + a.warned = map[string]time.Time{} + } + if last, ok := a.warned[kind]; ok && now.Sub(last) < forecastLongWarnGap { + return + } + a.warned[kind] = now + slog.Warn("forecast long archive", "kind", kind, "err", err) +} + +func (a *ForecastLongArchive) publish() { + a.mu.Lock() + st := a.status + a.mu.Unlock() + if a.store != nil { + a.store.ReportForecastLongArchive(st) + } +} + +// forecastLongReason is the short text health shows for a failure. Raw +// errors carry host paths, so they go only to the Core log. +func forecastLongReason(kind string) string { + switch kind { + case "immutable": + return "a forecast differs from its archived copy; the archived copy is kept" + case "weather": + return "a weather row differs from its archived copy; the archived copy is kept" + case "decode", "skip-series", "invalid": + return "skipped a row the archive cannot store" + } + return kind + " failed; see the Core log" +} + +// ForecastLongOpenFailed is the health text when the archive cannot open. +const ForecastLongOpenFailed = "cannot open the archive; see the Core log" + +// ArchiveIssues stores one page of issued forecasts. An identical run is left +// untouched. A different payload with the same id is rejected and the stored +// run is kept. Baselines are not stored; they can be rebuilt from observations +// and weather. Quarter-hour truth copied from state.db has load and PV only. +func (a *ForecastLongArchive) ArchiveIssues(ctx context.Context, batch []ForecastArchiveIssue) error { + if len(batch) == 0 { + return nil + } + if err := a.beforeWrite(); err != nil { + return err + } + tx, err := a.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + codes := map[string]byte{} + var immutable error + for _, item := range batch { + if err := a.archiveOne(ctx, tx, codes, item); err != nil { + var invalid forecastLongInvalid + switch { + case errors.Is(err, errForecastArchiveImmutable): + a.warn("immutable", err) + immutable = err + case errors.As(err, &invalid): + a.warn("invalid", err) // skipped; one bad run must not stop the copy + default: + return err + } + } + } + if err := tx.Commit(); err != nil { + return err + } + return immutable +} + +func (a *ForecastLongArchive) beforeWrite() error { + if forecastLongBeforeWrite != nil { + if err := forecastLongBeforeWrite(); err != nil { + return err + } + } + if a.freeShortfall() > 0 { + return errForecastArchiveDiskFull + } + return nil +} + +// freeShortfall is how far the data volume is below the free-space floor. +// A failed probe counts as no shortfall. +func (a *ForecastLongArchive) freeShortfall() int64 { + avail, err := forecastLongDiskAvail(filepath.Dir(a.path)) + if err != nil || avail >= a.minFree { + return 0 + } + return a.minFree - avail +} + +var errForecastArchiveDiskFull = errors.New("forecast archive paused: data volume is below the free-space floor") +var errForecastArchiveImmutable = errors.New("forecast long archive run is immutable") + +// forecastLongInvalid marks an issue the archive cannot store. archiveOne +// returns it before writing any run rows, so the page can skip the issue. +type forecastLongInvalid struct{ err error } + +func (e forecastLongInvalid) Error() string { + return "forecast archive cannot store issue: " + e.err.Error() +} +func (e forecastLongInvalid) Unwrap() error { return e.err } + +func (a *ForecastLongArchive) archiveOne(ctx context.Context, tx *sql.Tx, codes map[string]byte, item ForecastArchiveIssue) error { + issue := item.Issue + if err := issue.Validate(); err != nil { + return forecastLongInvalid{err} + } + kept := make([]forecasting.Series, 0, 4) + for _, series := range issue.Series { + if forecastLongKeepSeries(series.Name) { + kept = append(kept, series) + } + } + if len(kept) == 0 { + a.warn("skip-series", errors.New("issue "+issue.ID+" has none of the stored series")) + return nil + } + states, models, err := forecastLongModelBlobs(issue, item.States) + if err != nil { + return forecastLongInvalid{err} + } + issue.Models = models + hash, err := forecastLongInputHash(issue) + if err != nil { + return forecastLongInvalid{err} + } + var dictErr error + encoded, err := encodeKeptSeries(kept, func(kind, value string) (byte, error) { + code, err := forecastLongDict(ctx, tx, codes, kind, value) + if err != nil { + dictErr = err + } + return code, err + }) + if dictErr != nil { + return dictErr + } + if err != nil { + return forecastLongInvalid{err} + } + occ, occStart, occN, err := encodeOccupancy(issue.Occupancy) + if err != nil { + return forecastLongInvalid{err} + } + modelsJSON, err := json.Marshal(issue.Models) + if err != nil { + return forecastLongInvalid{err} + } + var decision, config, existingHash string + var origin, issued, latest int64 + err = tx.QueryRowContext(ctx, `SELECT decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, input_hash + FROM fa_runs WHERE run_id=?`, issue.ID).Scan(&decision, &origin, &issued, &latest, &config, &existingHash) + if err == nil { + same, cmpErr := forecastLongSameSeries(ctx, tx, issue.ID, encoded) + if cmpErr != nil { + return cmpErr + } + if existingHash != hash || decision != issue.DecisionID || origin != issue.OriginMS || issued != issue.IssuedAtMS || + latest != issue.LatestInputMS || config != issue.ConfigVersion || !same { + return fmt.Errorf("%w: %s", errForecastArchiveImmutable, issue.ID) + } + return nil + } + if !errors.Is(err, sql.ErrNoRows) { + return err + } + keepStates, err := forecastLongHourNeedsStates(ctx, tx, issue.OriginMS) + if err != nil { + return err + } + for id, blob := range states { + if len(blob.Gzip) == 0 || !keepStates { + continue + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_model_states(id, created_at_ms, expanded_bytes, payload) VALUES(?,?,?,?)`, + id, issue.IssuedAtMS, blob.Expanded, blob.Gzip); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_run_states(run_id, state_id) VALUES(?,?)`, issue.ID, id); err != nil { + return err + } + } + if err := insertForecastWeather(ctx, tx, issue); err != nil { + if !errors.Is(err, errForecastWeatherConflict) { + return err + } + a.warn("weather", err) + } + var siteID, learning, tz sql.NullString + var lat, lon sql.NullFloat64 + hasLoc := 0 + if issue.Site != nil { + siteID = sql.NullString{String: issue.Site.SiteID, Valid: true} + learning = sql.NullString{String: issue.Site.LearningRevision, Valid: true} + tz = sql.NullString{String: issue.Site.Timezone, Valid: true} + lat = sql.NullFloat64{Float64: issue.Site.Latitude, Valid: true} + lon = sql.NullFloat64{Float64: issue.Site.Longitude, Valid: true} + if issue.Site.HasLocation { + hasLoc = 1 + } + } + _, err = tx.ExecContext(ctx, `INSERT INTO fa_runs( + run_id, decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, + site_id, learning_revision, timezone, has_location, latitude, longitude, + models, occupancy, occupancy_start_ms, occupancy_n, input_hash, + thinned, json_bytes, gzip_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0,?,?)`, + issue.ID, issue.DecisionID, issue.OriginMS, issue.IssuedAtMS, issue.LatestInputMS, issue.ConfigVersion, + siteID, learning, tz, hasLoc, lat, lon, string(modelsJSON), nullableBlob(occ), nullMS(occStart, occN > 0), nullCount(occN), + hash, item.JSONBytes, item.GzipBytes) + if err != nil { + return err + } + for _, series := range encoded { + if err := insertEncodedSeries(ctx, tx, issue.ID, series); err != nil { + return err + } + } + return nil +} + +// forecastLongHourNeedsStates reports whether a run keeps its model states. +// Inside the newest 30 days one run per UTC hour is enough: the run on the +// hour, or the first run copied into an hour that has no states yet. Hour +// thinning ranks a run with states first, so the hour keeps them. The +// states between can be rebuilt from the archived observations, and the +// Energyplan state alone is about 90 KiB gzip per run. +func forecastLongHourNeedsStates(ctx context.Context, tx *sql.Tx, originMS int64) (bool, error) { + hour := originMS - originMS%forecastLongHourMS + if originMS == hour { + return true, nil + } + var held bool + err := tx.QueryRowContext(ctx, forecastLongHourHeld, hour, hour+forecastLongHourMS).Scan(&held) + return !held, err +} + +const forecastLongHourHeld = `SELECT EXISTS(SELECT 1 FROM fa_runs r JOIN fa_run_states s ON s.run_id=r.run_id + WHERE r.origin_ms>=? AND r.origin_ms 255 { + return 0, fmt.Errorf("forecast archive dictionary is full for %s", kind) + } + cache[key] = byte(code) + return byte(code), nil +} + +func insertEncodedSeries(ctx context.Context, tx *sql.Tx, runID string, s encodedSeries) error { + var pred any + if s.layout.predictionStartMS != 0 { + pred = s.layout.predictionStartMS + } + _, err := tx.ExecContext(ctx, `INSERT INTO fa_series( + run_id, series, model_version, start_ms, slot_ms, n, starts, prediction_start_ms, + pv_w, load_w, flags, pv_quality, load_quality, pv_source, load_source, + pv_lo, pv_hi, load_lo, load_hi, net_lo, net_hi, band_method, band_stats) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + runID, s.name, s.version, s.layout.startMS, s.layout.slotMS, s.layout.n, nullableBlob(s.layout.starts), pred, + s.pv, s.load, s.flags, s.pvQuality, s.loadQuality, nullableBlob(s.pvSource), nullableBlob(s.loadSource), + nullableBlob(s.pvLo), nullableBlob(s.pvHi), nullableBlob(s.loadLo), nullableBlob(s.loadHi), + nullableBlob(s.netLo), nullableBlob(s.netHi), nullableBlob(s.bandMethod), nullableBlob(s.bandStats)) + return err +} + +func forecastLongSameSeries(ctx context.Context, tx *sql.Tx, runID string, want []encodedSeries) (bool, error) { + rows, err := tx.QueryContext(ctx, `SELECT series, model_version, start_ms, slot_ms, n, starts, prediction_start_ms, + pv_w, load_w, flags, pv_quality, load_quality, pv_source, load_source, + pv_lo, pv_hi, load_lo, load_hi, net_lo, net_hi, band_method, band_stats + FROM fa_series WHERE run_id=?`, runID) + if err != nil { + return false, err + } + defer rows.Close() + got := map[string]encodedSeries{} + for rows.Next() { + var s encodedSeries + var starts, pvSource, loadSource, pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats []byte + var pred sql.NullInt64 + if err := rows.Scan(&s.name, &s.version, &s.layout.startMS, &s.layout.slotMS, &s.layout.n, &starts, &pred, + &s.pv, &s.load, &s.flags, &s.pvQuality, &s.loadQuality, &pvSource, &loadSource, + &pvLo, &pvHi, &loadLo, &loadHi, &netLo, &netHi, &method, &stats); err != nil { + return false, err + } + s.layout.starts = starts + if pred.Valid { + s.layout.predictionStartMS = pred.Int64 + } + s.pvSource, s.loadSource = pvSource, loadSource + s.pvLo, s.pvHi, s.loadLo, s.loadHi, s.netLo, s.netHi = pvLo, pvHi, loadLo, loadHi, netLo, netHi + s.bandMethod, s.bandStats = method, stats + got[s.name] = s + } + if err := rows.Err(); err != nil { + return false, err + } + if len(got) != len(want) { + return false, nil + } + for _, s := range want { + if !encodedSeriesEqual(got[s.name], s) { + return false, nil + } + } + return true, nil +} + +func encodedSeriesEqual(a, b encodedSeries) bool { + if a.name != b.name || a.version != b.version || a.layout.startMS != b.layout.startMS || a.layout.slotMS != b.layout.slotMS || + a.layout.n != b.layout.n || a.layout.predictionStartMS != b.layout.predictionStartMS { + return false + } + pairs := [][2][]byte{ + {a.layout.starts, b.layout.starts}, {a.pv, b.pv}, {a.load, b.load}, {a.flags, b.flags}, + {a.pvQuality, b.pvQuality}, {a.loadQuality, b.loadQuality}, {a.pvSource, b.pvSource}, {a.loadSource, b.loadSource}, + {a.pvLo, b.pvLo}, {a.pvHi, b.pvHi}, {a.loadLo, b.loadLo}, {a.loadHi, b.loadHi}, + {a.netLo, b.netLo}, {a.netHi, b.netHi}, {a.bandMethod, b.bandMethod}, {a.bandStats, b.bandStats}, + } + for _, pair := range pairs { + if !bytesEqualNil(pair[0], pair[1]) { + return false + } + } + return true +} + +func bytesEqualNil(a, b []byte) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// forecastWeatherFetch is one source's fetch, named by its available-at time. +type forecastWeatherFetch struct { + source string + availMS int64 +} + +var errForecastWeatherConflict = errors.New("forecast archive weather row is immutable") + +// insertForecastWeather shares weather rows across runs and records, per +// fetch, the exact slots this run used. A partial refresh leaves one issue +// with rows from two fetches, and another run may use other slots of the +// same fetch, so the fetch alone does not say which rows belong to a run. +// A stored row that differs from the issued one is kept and reported as +// errForecastWeatherConflict after every row is linked. +func insertForecastWeather(ctx context.Context, tx *sql.Tx, issue forecasting.Issue) error { + starts := map[forecastWeatherFetch][]int64{} + var order []forecastWeatherFetch + var conflict error + for _, row := range issue.Weather { + key := forecastWeatherFetch{source: row.Source, availMS: row.AvailableAtMS} + if _, ok := starts[key]; !ok { + order = append(order, key) + } + starts[key] = append(starts[key], row.StartMS) + res, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_weather( + source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv) VALUES(?,?,?,?,?,?,?,?,?)`, + row.Source, row.AvailableAtMS, row.StartMS, row.EndMS, + nullFloat(row.GHIWm2), nullFloat(row.CloudPct), nullFloat(row.TempC), nullFloat(row.DirectPVW), nullFloat(row.EstimatedPVW)) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n > 0 { + continue + } + var end int64 + var ghi, cloud, temp, direct, est sql.NullFloat64 + if err := tx.QueryRowContext(ctx, `SELECT end_ms, ghi, cloud, temp, direct_pv, est_pv FROM fa_weather + WHERE source=? AND available_at_ms=? AND start_ms=?`, row.Source, row.AvailableAtMS, row.StartMS).Scan(&end, &ghi, &cloud, &temp, &direct, &est); err != nil { + return err + } + if conflict == nil && (end != row.EndMS || !nullFloatEqual(ghi, row.GHIWm2) || !nullFloatEqual(cloud, row.CloudPct) || + !nullFloatEqual(temp, row.TempC) || !nullFloatEqual(direct, row.DirectPVW) || !nullFloatEqual(est, row.EstimatedPVW)) { + conflict = fmt.Errorf("%w: %s fetch %d slot %d", errForecastWeatherConflict, row.Source, row.AvailableAtMS, row.StartMS) + } + } + for _, key := range order { + blob, err := encodeForecastStarts(starts[key]) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO fa_run_weather(run_id, source, available_at_ms, starts) VALUES(?,?,?,?)`, + issue.ID, key.source, key.availMS, blob); err != nil { + return err + } + } + return conflict +} + +func nullFloat(v *float64) any { + if v == nil || math.IsNaN(*v) { + return nil + } + return *v +} + +func nullFloatEqual(got sql.NullFloat64, want *float64) bool { + if want == nil || math.IsNaN(*want) { + return !got.Valid + } + return got.Valid && got.Float64 == *want +} + +// LoadIssue rebuilds one stored run. Champion bands are restored. Other series +// get an explicit cold-start band, because only the champion interval was kept. +func (a *ForecastLongArchive) LoadIssue(ctx context.Context, runID string) (forecasting.Issue, error) { + var issue forecasting.Issue + var siteID, learning, tz, modelsJSON sql.NullString + var lat, lon sql.NullFloat64 + var hasLoc int + var occ []byte + var occStart sql.NullInt64 + var occN sql.NullInt64 + err := a.db.QueryRowContext(ctx, `SELECT decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, + site_id, learning_revision, timezone, has_location, latitude, longitude, models, + occupancy, occupancy_start_ms, occupancy_n + FROM fa_runs WHERE run_id=?`, runID).Scan( + &issue.DecisionID, &issue.OriginMS, &issue.IssuedAtMS, &issue.LatestInputMS, &issue.ConfigVersion, + &siteID, &learning, &tz, &hasLoc, &lat, &lon, &modelsJSON, &occ, &occStart, &occN) + if err != nil { + return forecasting.Issue{}, err + } + issue.Schema = forecasting.Schema + issue.ID = runID + if siteID.Valid && siteID.String != "" { + issue.Site = &forecasting.SiteContext{ + SiteID: siteID.String, LearningRevision: learning.String, Timezone: tz.String, + HasLocation: hasLoc != 0, Latitude: lat.Float64, Longitude: lon.Float64, + } + } + if modelsJSON.Valid && modelsJSON.String != "" { + if err := json.Unmarshal([]byte(modelsJSON.String), &issue.Models); err != nil { + return forecasting.Issue{}, err + } + } + nOcc := 0 + if occN.Valid { + nOcc = int(occN.Int64) + } + startOcc := int64(0) + if occStart.Valid { + startOcc = occStart.Int64 + } + if issue.Occupancy, err = decodeOccupancy(occ, startOcc, nOcc); err != nil { + return forecasting.Issue{}, err + } + dict, err := a.loadDict(ctx) + if err != nil { + return forecasting.Issue{}, err + } + issue.Series, err = a.loadSeries(ctx, runID, dict) + if err != nil { + return forecasting.Issue{}, err + } + if issue.Weather, err = a.loadWeather(ctx, runID); err != nil { + return forecasting.Issue{}, err + } + if err := issue.Validate(); err != nil { + return forecasting.Issue{}, err + } + return issue, nil +} + +func (a *ForecastLongArchive) loadDict(ctx context.Context) (map[byte]string, error) { + rows, err := a.db.QueryContext(ctx, `SELECT code, value FROM fa_dict`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[byte]string{} + for rows.Next() { + var code int + var value string + if err := rows.Scan(&code, &value); err != nil { + return nil, err + } + if code >= 0 && code <= 255 { + out[byte(code)] = value + } + } + return out, rows.Err() +} + +func (a *ForecastLongArchive) loadSeries(ctx context.Context, runID string, dict map[byte]string) ([]forecasting.Series, error) { + rows, err := a.db.QueryContext(ctx, `SELECT series, model_version, start_ms, slot_ms, n, starts, prediction_start_ms, + pv_w, load_w, flags, pv_quality, load_quality, pv_source, load_source, + pv_lo, pv_hi, load_lo, load_hi, net_lo, net_hi, band_method, band_stats + FROM fa_series WHERE run_id=?`, runID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []forecasting.Series + for rows.Next() { + var name, version string + var layout seriesLayout + var starts, pv, load, flags, pvQ, loadQ, pvS, loadS []byte + var pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats []byte + var pred sql.NullInt64 + if err := rows.Scan(&name, &version, &layout.startMS, &layout.slotMS, &layout.n, &starts, &pred, + &pv, &load, &flags, &pvQ, &loadQ, &pvS, &loadS, &pvLo, &pvHi, &loadLo, &loadHi, &netLo, &netHi, &method, &stats); err != nil { + return nil, err + } + layout.starts = starts + if pred.Valid { + layout.predictionStartMS = pred.Int64 + } + series, err := decodeStoredSeries(name, version, layout, pv, load, flags, pvQ, loadQ, pvS, loadS, pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats, dict) + if err != nil { + return nil, err + } + out = append(out, series) + } + return out, rows.Err() +} + +func decodeStoredSeries(name, version string, layout seriesLayout, pvB, loadB, flags, pvQ, loadQ, pvS, loadS, pvLo, pvHi, loadLo, loadHi, netLo, netHi, method, stats []byte, dict map[byte]string) (forecasting.Series, error) { + starts, ends, err := expandLayout(layout) + if err != nil { + return forecasting.Series{}, err + } + pv, err := decodeF32(pvB, layout.n) + if err != nil { + return forecasting.Series{}, err + } + load, err := decodeF32(loadB, layout.n) + if err != nil { + return forecasting.Series{}, err + } + flagB, err := decodeU8(flags, layout.n) + if err != nil { + return forecasting.Series{}, err + } + pvQB, err := decodeU8(pvQ, layout.n) + if err != nil { + return forecasting.Series{}, err + } + loadQB, err := decodeU8(loadQ, layout.n) + if err != nil { + return forecasting.Series{}, err + } + var pvSB, loadSB, methodB []byte + if len(pvS) > 0 { + if pvSB, err = decodeU8(pvS, layout.n); err != nil { + return forecasting.Series{}, err + } + } + if len(loadS) > 0 { + if loadSB, err = decodeU8(loadS, layout.n); err != nil { + return forecasting.Series{}, err + } + } + var loPV, hiPV, loLoad, hiLoad, loNet, hiNet []float64 + var samples, days [][3]int + champion := name == forecastLongSeriesChampion + if champion { + if methodB, err = decodeU8(method, layout.n); err != nil { + return forecasting.Series{}, err + } + for _, pair := range []struct { + b []byte + dst *[]float64 + }{{pvLo, &loPV}, {pvHi, &hiPV}, {loadLo, &loLoad}, {loadHi, &hiLoad}, {netLo, &loNet}, {netHi, &hiNet}} { + vals, err := decodeF32(pair.b, layout.n) + if err != nil { + return forecasting.Series{}, err + } + *pair.dst = vals + } + if samples, days, err = decodeBandStats(stats, layout.n); err != nil { + return forecasting.Series{}, err + } + } + points := make([]forecasting.Point, layout.n) + for i := 0; i < layout.n; i++ { + p := forecasting.Point{ + StartMS: starts[i], EndMS: ends[i], PVW: pv[i], LoadW: load[i], + PVKnown: flagB[i]&1 != 0, LoadKnown: flagB[i]&2 != 0, + PVQuality: dict[pvQB[i]], LoadQuality: dict[loadQB[i]], + } + if i == 0 { + p.PredictionStartMS = layout.predictionStartMS + } + if len(pvSB) > 0 { + p.PVSource = dict[pvSB[i]] + } + if len(loadSB) > 0 { + p.LoadSource = dict[loadSB[i]] + } + if champion { + pvM, loadM, netM := unpackBandMethod(methodB[i]) + p.PVBand = forecasting.Band{LowW: loPV[i], HighW: hiPV[i], Method: pvM, Samples: samples[i][0], Days: days[i][0]} + p.LoadBand = forecasting.Band{LowW: loLoad[i], HighW: hiLoad[i], Method: loadM, Samples: samples[i][1], Days: days[i][1]} + p.NetBand = forecasting.Band{LowW: loNet[i], HighW: hiNet[i], Method: netM, Samples: samples[i][2], Days: days[i][2]} + if p.PVBand.LowW > p.PVBand.HighW { + p.PVBand.HighW = p.PVBand.LowW + } + if p.LoadBand.LowW > p.LoadBand.HighW { + p.LoadBand.HighW = p.LoadBand.LowW + } + if p.NetBand.LowW > p.NetBand.HighW { + p.NetBand.HighW = p.NetBand.LowW + } + } else { + cold := forecasting.Band{Method: forecasting.BandMethodColdStart} + p.PVBand, p.LoadBand, p.NetBand = cold, cold, cold + } + points[i] = p + } + return forecasting.Series{Name: name, ModelVersion: version, Points: points}, nil +} + +// loadWeather returns exactly the weather rows the run was issued with, in +// start order. +func (a *ForecastLongArchive) loadWeather(ctx context.Context, runID string) ([]forecasting.Weather, error) { + rows, err := a.db.QueryContext(ctx, `SELECT source, available_at_ms, starts FROM fa_run_weather WHERE run_id=?`, runID) + if err != nil { + return nil, err + } + type fetch struct { + key forecastWeatherFetch + starts []int64 + } + var fetches []fetch + for rows.Next() { + var f fetch + var blob []byte + if err := rows.Scan(&f.key.source, &f.key.availMS, &blob); err != nil { + rows.Close() + return nil, err + } + if f.starts, err = decodeForecastStarts(blob); err != nil { + rows.Close() + return nil, err + } + fetches = append(fetches, f) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + var out []forecasting.Weather + for _, f := range fetches { + got, err := a.fetchWeatherRows(ctx, f.key, f.starts) + if err != nil { + return nil, err + } + if len(got) != len(f.starts) { + return nil, fmt.Errorf("forecast archive run %s is missing %d weather rows", runID, len(f.starts)-len(got)) + } + out = append(out, got...) + } + sort.Slice(out, func(i, j int) bool { return out[i].StartMS < out[j].StartMS }) + return out, nil +} + +func (a *ForecastLongArchive) fetchWeatherRows(ctx context.Context, key forecastWeatherFetch, starts []int64) ([]forecasting.Weather, error) { + args := make([]any, 0, len(starts)+2) + args = append(args, key.source, key.availMS) + for _, start := range starts { + args = append(args, start) + } + rows, err := a.db.QueryContext(ctx, `SELECT source, available_at_ms, start_ms, end_ms, ghi, cloud, temp, direct_pv, est_pv FROM fa_weather + WHERE source=? AND available_at_ms=? AND start_ms IN (`+placeholders(len(starts))+`)`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []forecasting.Weather + for rows.Next() { + var row forecasting.Weather + var ghi, cloud, temp, direct, est sql.NullFloat64 + if err := rows.Scan(&row.Source, &row.AvailableAtMS, &row.StartMS, &row.EndMS, &ghi, &cloud, &temp, &direct, &est); err != nil { + return nil, err + } + row.GHIWm2 = floatPtr(ghi) + row.CloudPct = floatPtr(cloud) + row.TempC = floatPtr(temp) + row.DirectPVW = floatPtr(direct) + row.EstimatedPVW = floatPtr(est) + out = append(out, row) + } + return out, rows.Err() +} + +func placeholders(n int) string { + if n <= 0 { + return "" + } + b := make([]byte, 0, n*2) + for i := 0; i < n; i++ { + if i > 0 { + b = append(b, ',') + } + b = append(b, '?') + } + return string(b) +} + +func floatPtr(v sql.NullFloat64) *float64 { + if !v.Valid || math.IsNaN(v.Float64) { + return nil + } + f := v.Float64 + return &f +} + +// ArchiveObservations stores quarter-hour truth. The 30-day source records +// household load and PV, not grid power, so grid is not invented here. +func (a *ForecastLongArchive) ArchiveObservations(ctx context.Context, rows []forecasting.Observation) error { + if len(rows) == 0 { + return nil + } + if err := a.beforeWrite(); err != nil { + return err + } + tx, err := a.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, row := range rows { + if err := row.Validate(); err != nil { + return err + } + pv, load := 0, 0 + if row.PVKnown { + pv = 1 + } + if row.LoadKnown { + load = 1 + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO fa_observations_15m( + start_ms, end_ms, config_version, available_at_ms, pv_w, load_w, pv_known, load_known, quality) + VALUES(?,?,?,?,?,?,?,?,?)`, row.StartMS, row.EndMS, row.ConfigVersion, row.AvailableAtMS, row.PVW, row.LoadW, pv, load, row.Quality); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/go/internal/state/forecast_long_archive_bench_test.go b/go/internal/state/forecast_long_archive_bench_test.go new file mode 100644 index 000000000..ea56da3ba --- /dev/null +++ b/go/internal/state/forecast_long_archive_bench_test.go @@ -0,0 +1,28 @@ +package state + +import ( + "context" + "testing" +) + +func BenchmarkForecastLongArchiveCopy(b *testing.B) { + store, err := Open(b.TempDir() + "/state.db") + if err != nil { + b.Fatal(err) + } + defer store.Close() + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + b.Fatal(err) + } + base := longIssue("bench", longOrigin()) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + item := base + item.ID = "bench-" + forecastLongItoa(i) + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: item, JSONBytes: 8000, GzipBytes: 2000}}); err != nil { + b.Fatal(err) + } + } +} diff --git a/go/internal/state/forecast_long_archive_copy_test.go b/go/internal/state/forecast_long_archive_copy_test.go new file mode 100644 index 000000000..9454dd0fa --- /dev/null +++ b/go/internal/state/forecast_long_archive_copy_test.go @@ -0,0 +1,350 @@ +package state + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongCopyPagesAndResumes(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Add(-6 * time.Hour).Truncate(time.Hour) + const n = 20 + for i := 0; i < n; i++ { + issue := longIssue("run-"+forecastLongItoa(i), origin.Add(time.Duration(i)*15*time.Minute)) + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + } + obs := forecasting.Observation{ + StartMS: origin.UnixMilli(), EndMS: origin.Add(15 * time.Minute).UnixMilli(), + AvailableAtMS: origin.Add(15 * time.Minute).UnixMilli(), PVW: 10, LoadW: 20, + PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg-v1", + } + if err := store.SaveForecastObservation(ctx, obs); err != nil { + t.Fatal(err) + } + pages := 0 + forecastLongAfterIssuePage = func(rows int) error { + pages++ + if pages == 1 { + return errors.New("stop after the first page") + } + return nil + } + t.Cleanup(func() { forecastLongAfterIssuePage = nil }) + if err := arch.Sync(ctx, store, time.Now().UTC()); err == nil { + t.Fatal("interrupted copy reported success") + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil { + t.Fatal(err) + } + if runs != forecastLongIssuePage { + t.Fatalf("first page stored %d runs, want %d", runs, forecastLongIssuePage) + } + forecastLongAfterIssuePage = nil + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != n { + t.Fatalf("resumed runs = %d err=%v, want %d", runs, err, n) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != n { + t.Fatalf("second copy duplicated runs: %d err=%v", runs, err) + } + var observations int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_observations_15m`).Scan(&observations); err != nil || observations != 1 { + t.Fatalf("observations = %d err=%v", observations, err) + } + var srcGzip, dstGzip []byte + if err := store.db.QueryRow(`SELECT payload FROM forecast_model_states LIMIT 1`).Scan(&srcGzip); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT payload FROM fa_model_states LIMIT 1`).Scan(&dstGzip); err != nil { + t.Fatal(err) + } + if !bytes.Equal(srcGzip, dstGzip) { + t.Fatal("model state was recompressed") + } + var states int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_model_states`).Scan(&states); err != nil || states != 1 { + t.Fatalf("model states = %d err=%v, want the one shared snapshot", states, err) + } +} + +func TestForecastLongCopySkipsPrunedSource(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + issue := longIssue("kept", time.Now().UTC().Add(-time.Hour).Truncate(time.Hour)) + if err := store.SaveForecastIssue(ctx, issue); err != nil { + t.Fatal(err) + } + if _, err := store.db.Exec(`DELETE FROM forecast_model_states`); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + var runs, states int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("runs = %d err=%v", runs, err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_model_states`).Scan(&states); err != nil || states != 0 { + t.Fatalf("pruned model state was invented: %d err=%v", states, err) + } + if _, err := store.db.Exec(`DELETE FROM forecast_issues`); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("archive dropped a copied run after the source was pruned: %d err=%v", runs, err) + } +} + +func forecastLongItoa(n int) string { + if n == 0 { + return "0" + } + var b [8]byte + i := len(b) + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + return string(b[i:]) +} + +func copiedRunIDs(t *testing.T, arch *ForecastLongArchive) map[string]bool { + t.Helper() + rows, err := arch.db.Query(`SELECT run_id FROM fa_runs`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + out := map[string]bool{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + out[id] = true + } + return out +} + +func longObservation(start time.Time, availableAfterEnd time.Duration) forecasting.Observation { + end := start.Add(15 * time.Minute) + return forecasting.Observation{ + StartMS: start.UnixMilli(), EndMS: end.UnixMilli(), AvailableAtMS: end.Add(availableAfterEnd).UnixMilli(), + PVW: 10, LoadW: 20, PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg-v1", + } +} + +func TestForecastLongCopyKeepsLateRows(t *testing.T) { + if forecastLongCopyOverlap < maxForecastFutureSkew { + t.Fatal("the copy overlap must cover the future skew state.db accepts") + } + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Truncate(time.Hour).Add(-2 * time.Hour) + if err := store.SaveForecastIssue(ctx, longIssue("b-primary", origin)); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin, time.Minute)); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + // The shadow has the same issued time and an id that sorts first. The + // earlier quarter was released after the later one. + if err := store.SaveForecastIssue(ctx, longIssue("a-shadow", origin)); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin.Add(-15*time.Minute), 20*time.Minute)); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); !runs["a-shadow"] || !runs["b-primary"] { + t.Fatalf("runs = %v, the late shadow was skipped", runs) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_observations_15m`); n != 2 { + t.Fatalf("observations = %d, the late quarter was skipped", n) + } +} + +func TestForecastLongCopyRecoversFromFutureCursor(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + origin := now.Truncate(time.Hour).Add(-2 * time.Hour) + if err := store.SaveForecastIssue(ctx, longIssue("real", origin)); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin, time.Minute)); err != nil { + t.Fatal(err) + } + // A row stamped by a clock a day ahead. state.db keeps it, since its + // retention counts back from now. + ahead, err := prepareForecastIssue(longIssue("ahead", now.Truncate(time.Hour).Add(24*time.Hour))) + if err != nil { + t.Fatal(err) + } + if _, err := store.db.Exec(`INSERT INTO forecast_issues(id,origin_ms,issued_at_ms,config_version,payload) VALUES(?,?,?,?,?)`, + ahead.issue.ID, ahead.issue.OriginMS, ahead.issue.IssuedAtMS, ahead.issue.ConfigVersion, ahead.payload); err != nil { + t.Fatal(err) + } + // Cursors written while the clock was a year ahead. + yearAhead := now.Add(365 * 24 * time.Hour).UnixMilli() + if err := arch.writeIssueCursor(ctx, forecastLongIssueCursor{IssuedAtMS: yearAhead, ID: "z"}); err != nil { + t.Fatal(err) + } + if err := arch.writeObsCursor(ctx, forecastLongObsCursor{AvailableMS: yearAhead}); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); len(runs) != 1 || !runs["real"] { + t.Fatalf("runs = %v, want only the run issued before now", runs) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_observations_15m`); n != 1 { + t.Fatalf("observations = %d after the cursor reset", n) + } + bound := now.Add(maxForecastFutureSkew).UnixMilli() + issueCursor, err := arch.issueCursor(ctx) + if err != nil || issueCursor.IssuedAtMS > bound { + t.Fatalf("issue cursor = %+v err=%v, ahead of the clock again", issueCursor, err) + } + obsCursor, err := arch.obsCursor(ctx) + if err != nil || obsCursor.AvailableMS > bound { + t.Fatalf("observation cursor = %+v err=%v, ahead of the clock again", obsCursor, err) + } + + // A rescan skips stored runs by id and never reads their payloads. + if _, err := store.db.Exec(`UPDATE forecast_issues SET payload=x'00' WHERE id='real'`); err != nil { + t.Fatal(err) + } + if err := arch.writeIssueCursor(ctx, forecastLongIssueCursor{IssuedAtMS: yearAhead}); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + if status, _ := store.ForecastLongArchiveStatus(); status.LastError != "" { + t.Fatalf("rescan decoded a stored run: %q", status.LastError) + } + + // The row stamped ahead is copied once the clock reaches it. + if err := arch.Sync(ctx, store, now.Add(25*time.Hour)); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); !runs["ahead"] { + t.Fatalf("runs = %v, the row stamped ahead was never copied", runs) + } +} + +func TestForecastLongCopyQueriesUseIndexes(t *testing.T) { + store, _ := openLongArchive(t) + if err := store.InitForecastArchive(context.Background()); err != nil { + t.Fatal(err) + } + for _, q := range []struct { + query, index string + args []any + }{ + {forecastIssueKeysQuery, "forecast_issues_time", []any{1, "a", 2, 16}}, + {forecastObservationsAfterQuery, "forecast_observations_time", []any{1, 2, 3, "c", 5, 128}}, + } { + rows, err := store.db.Query("EXPLAIN QUERY PLAN "+q.query, q.args...) + if err != nil { + t.Fatal(err) + } + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + rows.Close() + joined := strings.Join(plan, "; ") + if strings.Contains(joined, "SCAN ") || !strings.Contains(joined, "USING INDEX "+q.index) { + t.Fatalf("plan for the copy query = %s", joined) + } + } +} + +// One row the archive cannot store is skipped with a short health reason. +// It must not stop the copy, and health must not carry the raw error. +func TestForecastLongCopySkipsRowsItCannotStore(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + origin := time.Now().UTC().Truncate(time.Hour).Add(-2 * time.Hour) + // float32 holds no 1e39, so the series cannot be encoded. + wide := longIssue("a-wide", origin) + wide.Series[0].Points[0].PVW = 1e39 + if err := store.SaveForecastIssue(ctx, wide); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastIssue(ctx, longIssue("b-good", origin)); err != nil { + t.Fatal(err) + } + bad := longObservation(origin, time.Minute) + if _, err := store.db.ExecContext(ctx, `INSERT INTO forecast_observations(start_ms,end_ms,available_at_ms,config_version,payload) + VALUES(?,?,?,?,?)`, bad.StartMS, bad.EndMS, bad.AvailableAtMS, bad.ConfigVersion, `{"start_ms":"not a number"}`); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastObservation(ctx, longObservation(origin.Add(15*time.Minute), time.Minute)); err != nil { + t.Fatal(err) + } + if err := arch.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatal(err) + } + if runs := copiedRunIDs(t, arch); runs["a-wide"] || !runs["b-good"] { + t.Fatalf("runs = %v, want only b-good", runs) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_observations_15m`); n != 1 { + t.Fatalf("observations = %d, the copy stopped at the bad row", n) + } + cursor, err := arch.obsCursor(ctx) + if err != nil || cursor.StartMS != origin.Add(15*time.Minute).UnixMilli() { + t.Fatalf("observation cursor = %+v, %v; want it past the bad row", cursor, err) + } + status, _ := store.ForecastLongArchiveStatus() + if status.LastError != forecastLongReason("invalid") { + t.Fatalf("health last_error = %q, want a short reason", status.LastError) + } +} diff --git a/go/internal/state/forecast_long_archive_coverage_test.go b/go/internal/state/forecast_long_archive_coverage_test.go new file mode 100644 index 000000000..f9824176a --- /dev/null +++ b/go/internal/state/forecast_long_archive_coverage_test.go @@ -0,0 +1,51 @@ +package state + +import ( + "context" + "testing" + "time" +) + +func TestForecastLongCoverageClipsToArchiveStart(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + start := now.Add(-4 * time.Hour) + for _, hour := range []int{0, 1, 3} { + at := start.Add(time.Duration(hour) * time.Hour).UnixMilli() + if _, err := arch.db.ExecContext(ctx, `INSERT INTO fa_runs( + run_id, decision_id, origin_ms, issued_at_ms, latest_input_ms, config_version, models, input_hash) + VALUES(?, '', ?, ?, 0, 'cfg', '[]', 'hash')`, "h"+string(rune('0'+hour)), at, at); err != nil { + t.Fatal(err) + } + } + quarter := int64(15 * 60 * 1000) + // Four known quarters and four unknown ones inside the clipped window. + for i := 0; i < 8; i++ { + known := 0 + if i < 4 { + known = 1 + } + at := start.UnixMilli() + int64(i)*quarter + if _, err := arch.db.ExecContext(ctx, `INSERT INTO fa_observations_15m( + start_ms, end_ms, config_version, available_at_ms, pv_w, load_w, pv_known, load_known, quality) + VALUES(?, ?, 'cfg', ?, 1, 1, 1, ?, 'complete')`, at, at+quarter, at+quarter, known); err != nil { + t.Fatal(err) + } + } + got := arch.hourCoverage(ctx, now, 7*24*time.Hour, start.UnixMilli()) + // Hours 08:00 through 12:00 inclusive once clipped to 08:00, with 08, 09 + // and 11 present. 12:00 itself has no run. + if got != 0.6 { + t.Fatalf("coverage = %v, want 0.6", got) + } + truth := arch.truthCoverage(ctx, now, start.UnixMilli()) + span := (now.Truncate(15*time.Minute).UnixMilli() - start.UnixMilli()) / quarter + if truth != float64(4)/float64(span) { + t.Fatalf("truth coverage = %v, want %v", truth, float64(4)/float64(span)) + } + at := start.Add(3 * time.Hour) + if all := arch.hourCoverage(ctx, at, time.Hour, at.UnixMilli()); all != 1 { + t.Fatalf("coverage clipped to a later start = %v, want 1", all) + } +} diff --git a/go/internal/state/forecast_long_archive_heal_test.go b/go/internal/state/forecast_long_archive_heal_test.go new file mode 100644 index 000000000..814e99e1a --- /dev/null +++ b/go/internal/state/forecast_long_archive_heal_test.go @@ -0,0 +1,71 @@ +package state + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestForecastLongArchiveRebuildsCorruption(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + archivePath := ForecastLongArchivePath(path) + if err := os.WriteFile(archivePath, []byte("this is not a sqlite database"), 0o600); err != nil { + t.Fatal(err) + } + // Copies left by earlier damage. Only the newest quarantine stays. + stale := []string{"forecast-archive.db.corrupt-1", "forecast-archive.db-wal.corrupt-1", "forecast-archive.db-shm.corrupt-2"} + others := []string{"cache.db.corrupt-1", "forecast-archive.db.corrupted"} + for _, name := range append(append([]string(nil), stale...), others...) { + if err := os.WriteFile(filepath.Join(dir, name), make([]byte, 4096), 0o600); err != nil { + t.Fatal(err) + } + } + reopened, err := Open(path) + if err != nil { + t.Fatalf("state.Open failed because of the analytics file: %v", err) + } + defer reopened.Close() + if events := reopened.HealEvents(); len(events) != 0 { + t.Fatalf("opening state healed the archive on the boot path: %+v", events) + } + arch, err := reopened.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + events := reopened.HealEvents() + if len(events) != 1 || events[0].Tier != "forecast_archive" || events[0].Action != healRebuilt { + t.Fatalf("heal events = %+v", events) + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 0 { + t.Fatalf("rebuilt archive runs = %d err=%v", runs, err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var quarantined []string + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "forecast-archive.db") && strings.Contains(entry.Name(), ".corrupt-") { + quarantined = append(quarantined, entry.Name()) + } + } + if len(quarantined) != 1 || quarantined[0] == stale[0] || !strings.HasPrefix(quarantined[0], "forecast-archive.db.corrupt-") { + t.Fatalf("quarantined archive files = %v, want only the new copy", quarantined) + } + for _, name := range others { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Fatalf("unrelated file %s was removed: %v", name, err) + } + } +} diff --git a/go/internal/state/forecast_long_archive_open_test.go b/go/internal/state/forecast_long_archive_open_test.go new file mode 100644 index 000000000..5e6d679fb --- /dev/null +++ b/go/internal/state/forecast_long_archive_open_test.go @@ -0,0 +1,252 @@ +package state + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// closedLongArchive leaves forecast-archive.db on disk with one run and +// returns a fresh store that has not opened it. +func closedLongArchive(t *testing.T) (*Store, string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "state.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + issue := longIssue("kept", time.Now().UTC().Truncate(time.Hour).Add(-time.Hour)) + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reopened.Close() }) + return reopened, ForecastLongArchivePath(path) +} + +func quarantinedFiles(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var out []string + for _, entry := range entries { + if strings.Contains(entry.Name(), ".corrupt-") { + out = append(out, entry.Name()) + } + } + return out +} + +func TestForecastLongOpenDoesNotBlockStatusReaders(t *testing.T) { + store, archivePath := closedLongArchive(t) + // Another connection holds the file, so the open waits in SQLite's busy + // handler as it would behind a long quick_check on an SD card. + lock, err := sql.Open("sqlite", archivePath+"?_pragma=locking_mode(EXCLUSIVE)") + if err != nil { + t.Fatal(err) + } + lock.SetMaxOpenConns(1) + defer lock.Close() + if _, err := lock.Exec(`INSERT OR REPLACE INTO fa_meta(key, value) VALUES('lock', 'held')`); err != nil { + t.Fatal(err) + } + opened := make(chan error, 1) + go func() { + _, err := store.OpenForecastLongArchive(context.Background()) + opened <- err + }() + for store.forecastLongOpenMu.TryLock() { + store.forecastLongOpenMu.Unlock() + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + readers := make(chan error, 1) + go func() { + store.ReportForecastLongArchive(ForecastLongArchiveStatus{State: "starting"}) + if _, ok := store.ForecastLongArchiveStatus(); !ok { + readers <- errors.New("status missing") + return + } + _, err := store.SQLiteInventory(context.Background()) + readers <- err + }() + select { + case err := <-readers: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("health status and storage inventory waited for the archive open") + } + if err := lock.Close(); err != nil { + t.Fatal(err) + } + if err := <-opened; err != nil { + t.Fatalf("open after the lock was released: %v", err) + } + if files := quarantinedFiles(t, filepath.Dir(archivePath)); len(files) != 0 { + t.Fatalf("a locked archive was moved aside: %v", files) + } +} + +func TestForecastLongOpenKeepsFileOnReadError(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + store, archivePath := closedLongArchive(t) + if err := os.Chmod(archivePath, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(archivePath, 0o600) }) + if _, err := store.OpenForecastLongArchive(context.Background()); err == nil { + t.Fatal("unreadable archive opened") + } + if files := quarantinedFiles(t, filepath.Dir(archivePath)); len(files) != 0 { + t.Fatalf("a read error moved a good archive aside: %v", files) + } + if events := store.HealEvents(); len(events) != 0 { + t.Fatalf("heal events = %+v", events) + } + if err := os.Chmod(archivePath, 0o600); err != nil { + t.Fatal(err) + } + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + if _, err := arch.LoadIssue(context.Background(), "kept"); err != nil { + t.Fatalf("the next open lost the stored run: %v", err) + } +} + +func TestForecastLongSyncDropsDamagedArchive(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + issue := longIssue("before", time.Now().UTC().Truncate(time.Hour).Add(-time.Hour)) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + // Each query takes a new connection, which reads the damaged header. + arch.db.SetMaxIdleConns(0) + if _, err := arch.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(arch.path, make([]byte, 8192), 0o600); err != nil { + t.Fatal(err) + } + err := arch.Sync(ctx, store, time.Now().UTC()) + if !sqliteCorrupt(err) { + t.Fatalf("sync err = %v, want a corruption code", err) + } + if err := arch.db.Ping(); err == nil { + t.Fatal("damaged handle is still open") + } + healed, err := store.OpenForecastLongArchive(ctx) + if err != nil { + t.Fatal(err) + } + if healed == arch { + t.Fatal("open returned the dropped handle") + } + events := store.HealEvents() + if len(events) != 1 || events[0].Tier != "forecast_archive" { + t.Fatalf("heal events = %+v", events) + } + if err := healed.Sync(ctx, store, time.Now().UTC()); err != nil { + t.Fatalf("copy into the rebuilt archive: %v", err) + } + if files := quarantinedFiles(t, filepath.Dir(arch.path)); len(files) == 0 { + t.Fatal("damaged archive was not moved aside") + } + // A corruption code read from state.db is not damage in the archive. + corrupt := sqliteErrorOf(t) + if !forecastLongDamaged(fmt.Errorf("copy: %w", corrupt)) { + t.Fatal("archive corruption was not recognised") + } + if forecastLongDamaged(fmt.Errorf("copy: %w", fromForecastSource(corrupt))) { + t.Fatal("a damaged state.db would drop the archive") + } +} + +// sqliteErrorOf returns a real SQLITE_NOTADB error. +func sqliteErrorOf(t *testing.T) error { + t.Helper() + path := filepath.Join(t.TempDir(), "junk.db") + if err := os.WriteFile(path, []byte("this is not a sqlite database at all, just text"), 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Exec(`SELECT * FROM sqlite_master`) + if !sqliteCorrupt(err) { + t.Fatalf("junk file error = %v", err) + } + return err +} + +// A corruption code that quick_check would not see still heals the file, and +// the rebuild waits for the last read on the dropped handle. +func TestForecastLongDropRebuildsAfterLastRead(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: longIssue("kept", time.Now().UTC().Truncate(time.Hour))}}); err != nil { + t.Fatal(err) + } + // A backup still reading on the handle when Sync drops it. + reading, err := arch.db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + store.dropForecastLongArchive(arch) + if _, err := store.OpenForecastLongArchive(ctx); err == nil { + t.Fatal("open rebuilt the file while a read on the dropped handle was running") + } + if files := quarantinedFiles(t, filepath.Dir(arch.path)); len(files) != 0 { + t.Fatalf("file moved aside under a running read: %v", files) + } + if err := reading.Close(); err != nil { + t.Fatal(err) + } + healed, err := store.OpenForecastLongArchive(ctx) + if err != nil { + t.Fatal(err) + } + if files := quarantinedFiles(t, filepath.Dir(arch.path)); len(files) == 0 { + t.Fatal("file that passed quick_check was not moved aside after a corruption code") + } + if events := store.HealEvents(); len(events) != 1 || events[0].Tier != "forecast_archive" { + t.Fatalf("heal events = %+v", events) + } + if _, err := healed.LoadIssue(ctx, "kept"); err == nil { + t.Fatal("rebuilt archive still holds the old run") + } + again, err := store.OpenForecastLongArchive(ctx) + if err != nil || again != healed { + t.Fatalf("second open = %v, %v; want the cached handle", again, err) + } +} diff --git a/go/internal/state/forecast_long_archive_prune_test.go b/go/internal/state/forecast_long_archive_prune_test.go new file mode 100644 index 000000000..bf99c3912 --- /dev/null +++ b/go/internal/state/forecast_long_archive_prune_test.go @@ -0,0 +1,225 @@ +package state + +import ( + "context" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongRetentionAndCap(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + arch.retention = 48 * time.Hour + arch.fullDetail = 24 * time.Hour + old := longIssue("old", now.Add(-72*time.Hour)) + recent := longIssue("recent", now.Add(-time.Hour)) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: old}, {Issue: recent}}); err != nil { + t.Fatal(err) + } + obs := func(at time.Time) forecasting.Observation { + return forecasting.Observation{ + StartMS: at.UnixMilli(), EndMS: at.Add(15 * time.Minute).UnixMilli(), + AvailableAtMS: at.Add(15 * time.Minute).UnixMilli(), PVW: 1, LoadW: 2, + PVKnown: true, LoadKnown: true, Quality: "complete", ConfigVersion: "cfg-v1", + } + } + if err := arch.ArchiveObservations(ctx, []forecasting.Observation{obs(now.Add(-72 * time.Hour)), obs(now.Add(-time.Hour))}); err != nil { + t.Fatal(err) + } + if err := arch.pruneExpired(ctx, now); err != nil { + t.Fatal(err) + } + assertRuns(t, arch, map[string]bool{"recent": true}) + var oldObs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_observations_15m WHERE end_ms= infoBefore && free > 0 { + t.Fatalf("vacuum left the file at %d (was %d) with %d free pages; live before delete %d", after, infoBefore, free, before) + } +} + +func assertRuns(t *testing.T, arch *ForecastLongArchive, want map[string]bool) { + t.Helper() + rows, err := arch.db.Query(`SELECT run_id FROM fa_runs`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + got := map[string]bool{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + got[id] = true + } + if len(got) != len(want) { + t.Fatalf("runs = %v, want %v", got, want) + } + for id := range want { + if !got[id] { + t.Fatalf("missing %s in %v", id, got) + } + } +} + +func TestForecastLongBelowFloorStillThinsAndGivesSpaceBack(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Hour) + hour := now.Add(-40 * 24 * time.Hour) + later := longIssue("old-later", hour) + later.IssuedAtMS += 1000 + // The pad day is the oldest outside the 30 days. Deleting it gives back + // far more than the 64 KiB shortfall, so the thinned day stays. + pad := ForecastArchiveIssue{ + Issue: longIssue("pad", now.Add(-60*24*time.Hour)), + States: map[string]ForecastModelBlob{"pad-state": {Expanded: 2 << 20, Gzip: make([]byte, 2<<20)}}, + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{ + {Issue: longIssue("expired", now.Add(-800*24*time.Hour))}, pad, + {Issue: longIssue("old-keep", hour)}, {Issue: later}, + }); err != nil { + t.Fatal(err) + } + if err := store.SaveForecastIssue(ctx, longIssue("fresh", now.Add(-time.Hour))); err != nil { + t.Fatal(err) + } + if _, err := arch.db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + t.Fatal(err) + } + before, err := regularFileSize(arch.path) + if err != nil { + t.Fatal(err) + } + restore := ForecastLongDiskAvailForTest(func(string) (int64, error) { return arch.minFree - 64<<10, nil }) + defer restore() + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + assertRuns(t, arch, map[string]bool{"old-keep": true}) + var thinned int + if err := arch.db.QueryRow(`SELECT thinned FROM fa_runs WHERE run_id='old-keep'`).Scan(&thinned); err != nil || thinned != 1 { + t.Fatalf("old-keep thinned = %d err=%v", thinned, err) + } + var pads int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_model_states WHERE id='pad-state'`).Scan(&pads); err != nil || pads != 0 { + t.Fatalf("pad state rows = %d err=%v", pads, err) + } + status, _ := store.ForecastLongArchiveStatus() + if status.State != "paused_disk_full" { + t.Fatalf("state = %q, want paused_disk_full", status.State) + } + var free, page int64 + if err := arch.db.QueryRow(`PRAGMA freelist_count`).Scan(&free); err != nil { + t.Fatal(err) + } + if err := arch.db.QueryRow(`PRAGMA page_size`).Scan(&page); err != nil { + t.Fatal(err) + } + after, err := regularFileSize(arch.path) + if err != nil { + t.Fatal(err) + } + if free*page >= forecastLongFreeTarget || after > before-(1<<20) { + t.Fatalf("file %d bytes (was %d) with %d free pages; the pad pages were not given back", after, before, free) + } +} + +func TestForecastLongHourlySyncEnforcesCap(t *testing.T) { + store, arch := openLongArchive(t) + ctx := context.Background() + restore := ForecastLongDiskAvailForTest(func(string) (int64, error) { return 1 << 40, nil }) + defer restore() + if err := store.InitForecastArchive(ctx); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Hour) + // The first copy runs the daily pass; the next is 24 hours away. + if err := arch.Sync(ctx, store, now); err != nil { + t.Fatal(err) + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{ + {Issue: longIssue("old", now.Add(-40*24*time.Hour))}, + {Issue: longIssue("recent", now.Add(-time.Hour))}, + }); err != nil { + t.Fatal(err) + } + arch.maxBytes = 1 + if err := arch.Sync(ctx, store, now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + assertRuns(t, arch, map[string]bool{"recent": true}) + if status, _ := store.ForecastLongArchiveStatus(); status.State != "over_cap" { + t.Fatalf("state = %q, want over_cap an hour after the daily pass", status.State) + } + arch.maxBytes = ForecastLongMaxBytes + if err := arch.Sync(ctx, store, now.Add(2*time.Hour)); err != nil { + t.Fatal(err) + } + if status, _ := store.ForecastLongArchiveStatus(); status.State != "ok" { + t.Fatalf("state = %q, want ok once the archive is under the cap", status.State) + } +} diff --git a/go/internal/state/forecast_long_archive_test.go b/go/internal/state/forecast_long_archive_test.go new file mode 100644 index 000000000..d1ddb7309 --- /dev/null +++ b/go/internal/state/forecast_long_archive_test.go @@ -0,0 +1,312 @@ +package state + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "math" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func openLongArchive(t *testing.T) (*Store, *ForecastLongArchive) { + t.Helper() + store, err := Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + arch, err := store.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + return store, arch +} + +func longOrigin() time.Time { + return time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC) +} + +func longBand(low, high float64, method string) forecasting.Band { + b := forecasting.Band{LowW: low, HighW: high, Method: method} + if method == forecasting.BandMethodEmpirical { + b.Samples, b.Days = 48, 7 + } + return b +} + +func longPoint(start int64, pv, load float64, partial bool) forecasting.Point { + p := forecasting.Point{ + StartMS: start, EndMS: start + 15*60*1000, PVW: pv, LoadW: load, + PVKnown: true, LoadKnown: load >= 0, PVQuality: "ready", LoadQuality: "learning", + PVSource: "energyplan", LoadSource: "legacy", + PVBand: longBand(0, pv+10, forecasting.BandMethodEmpirical), + LoadBand: longBand(0, 0, forecasting.BandMethodColdStart), + NetBand: longBand(0, 0, forecasting.BandMethodColdStart), + } + if partial { + p.PredictionStartMS = start + 60*1000 + } + return p +} + +func longIssue(id string, origin time.Time) forecasting.Issue { + start := origin.UnixMilli() + ghi := 120.5 + temp := 4.25 + quarter := int64(15 * 60 * 1000) + return forecasting.Issue{ + Schema: forecasting.Schema, ID: id, DecisionID: "decision-" + id, + OriginMS: start, IssuedAtMS: start + 1000, LatestInputMS: start, + ConfigVersion: "cfg-v1", + Site: &forecasting.SiteContext{ + SiteID: "site-1", LearningRevision: "learn-1", Timezone: "UTC", + HasLocation: true, Latitude: 59.3293, Longitude: 18.0686, + }, + Models: []forecasting.ModelState{{ + Name: "load", Version: "load-v3", Quality: forecasting.ModelQualityWarm, + UpdatedAtMS: start, State: []byte(`{"weights":[1,2,3]}`), + }}, + Weather: []forecasting.Weather{{ + StartMS: start, EndMS: start + quarter, AvailableAtMS: start, + Source: "open_meteo", GHIWm2: &ghi, TempC: &temp, + }}, + Occupancy: []forecasting.Occupancy{ + {StartMS: start - 2*quarter, EndMS: start - quarter, AvailableAtMS: start - quarter, Home: true}, + {StartMS: start - quarter, EndMS: start, AvailableAtMS: start, Home: false}, + }, + Series: []forecasting.Series{ + {Name: "champion", ModelVersion: "champ-1", Points: []forecasting.Point{longPoint(start, 1500.5, 800.25, true)}}, + {Name: "planning", ModelVersion: "plan-1", Points: []forecasting.Point{longPoint(start, 1400, 900, false)}}, + {Name: "legacy_shadow", ModelVersion: "legacy-1", Points: []forecasting.Point{longPoint(start, 1000, 700, false)}}, + {Name: "energyplan", ModelVersion: "ep-1", Points: []forecasting.Point{ + longPoint(start, 1500.5, 800.25, false), + func() forecasting.Point { + p := longPoint(start+30*60*1000, 400, 650, false) + p.EndMS = start + 90*60*1000 + return p + }(), + }}, + {Name: "previous_day", ModelVersion: "baseline", Points: []forecasting.Point{longPoint(start, 1, 1, false)}}, + }, + } +} + +func TestForecastLongArchiveOpensAndReopens(t *testing.T) { + store, arch := openLongArchive(t) + var marker string + if err := arch.db.QueryRow(`SELECT name FROM fa_migrations`).Scan(&marker); err != nil || marker != "fa-v1" { + t.Fatalf("migration = %q err=%v", marker, err) + } + var mode int + if err := arch.db.QueryRow(`PRAGMA auto_vacuum`).Scan(&mode); err != nil || mode != 2 { + t.Fatalf("auto_vacuum = %d err=%v, want incremental", mode, err) + } + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: longIssue("one", longOrigin())}}); err != nil { + t.Fatal(err) + } + store.Close() + again, err := Open(store.mainDBPath) + if err != nil { + t.Fatal(err) + } + defer again.Close() + arch, err = again.OpenForecastLongArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + var n int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_migrations`).Scan(&n); err != nil || n != 1 { + t.Fatalf("migrations = %d err=%v", n, err) + } + if _, err := arch.LoadIssue(context.Background(), "one"); err != nil { + t.Fatal(err) + } +} + +func TestForecastLongArchiveRoundTrip(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + issue := longIssue("round", longOrigin()) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue, JSONBytes: 1200, GzipBytes: 400}}); err != nil { + t.Fatal(err) + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue, JSONBytes: 1200, GzipBytes: 400}}); err != nil { + t.Fatal(err) + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("runs = %d err=%v", runs, err) + } + loaded, err := arch.LoadIssue(ctx, "round") + if err != nil { + t.Fatal(err) + } + if err := loaded.Validate(); err != nil { + t.Fatal(err) + } + byName := map[string]forecasting.Series{} + for _, series := range loaded.Series { + byName[series.Name] = series + } + if len(byName) != 4 { + t.Fatalf("stored series = %d, baselines should be omitted", len(byName)) + } + if _, ok := byName["previous_day"]; ok { + t.Fatal("baseline series was stored") + } + want := map[string]forecasting.Series{} + for _, series := range issue.Series { + if forecastLongKeepSeries(series.Name) { + want[series.Name] = series + } + } + for name, series := range want { + got := byName[name] + if got.ModelVersion != series.ModelVersion || len(got.Points) != len(series.Points) { + t.Fatalf("%s identity = %+v", name, got) + } + for i, point := range series.Points { + gp := got.Points[i] + if float32(gp.PVW) != float32(point.PVW) || float32(gp.LoadW) != float32(point.LoadW) || + gp.PVKnown != point.PVKnown || gp.LoadKnown != point.LoadKnown || + gp.PVQuality != point.PVQuality || gp.LoadQuality != point.LoadQuality || + gp.PVSource != point.PVSource || gp.LoadSource != point.LoadSource || + gp.StartMS != point.StartMS || gp.EndMS != point.EndMS { + t.Fatalf("%s point %d = %+v want %+v", name, i, gp, point) + } + } + } + champ := byName["champion"].Points[0] + if champ.PredictionStartMS != issue.Series[0].Points[0].PredictionStartMS || + champ.PVBand.Method != forecasting.BandMethodEmpirical || champ.PVBand.Samples != 48 || champ.PVBand.Days != 7 || + float32(champ.PVBand.HighW) != float32(issue.Series[0].Points[0].PVBand.HighW) { + t.Fatalf("champion band = %+v", champ) + } + if byName["planning"].Points[0].PVBand.Method != forecasting.BandMethodColdStart { + t.Fatal("non-champion band was stored") + } + if loaded.Site == nil || loaded.Site.SiteID != "site-1" || loaded.Site.Latitude != issue.Site.Latitude { + t.Fatalf("site = %+v", loaded.Site) + } + if len(loaded.Occupancy) != 2 || loaded.Occupancy[0].Home != true || loaded.Occupancy[1].Home != false || + loaded.Occupancy[0].AvailableAtMS != issue.Occupancy[0].AvailableAtMS { + t.Fatalf("occupancy = %+v", loaded.Occupancy) + } + if len(loaded.Weather) != 1 || loaded.Weather[0].GHIWm2 == nil || *loaded.Weather[0].GHIWm2 != *issue.Weather[0].GHIWm2 || + loaded.Weather[0].CloudPct != nil { + t.Fatalf("weather = %+v", loaded.Weather) + } + sum := sha256.Sum256(issue.Models[0].State) + if len(loaded.Models) != 1 || loaded.Models[0].StateID != hex.EncodeToString(sum[:]) || len(loaded.Models[0].State) != 0 || + loaded.Models[0].Version != "load-v3" { + t.Fatalf("models = %+v", loaded.Models) + } + var jsonBytes, gzipBytes int + if err := arch.db.QueryRow(`SELECT json_bytes, gzip_bytes FROM fa_runs WHERE run_id='round'`).Scan(&jsonBytes, &gzipBytes); err != nil || jsonBytes != 1200 || gzipBytes != 400 { + t.Fatalf("measured sizes %d %d err=%v", jsonBytes, gzipBytes, err) + } +} + +func TestForecastLongArchiveRejectsChangedIdentity(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + issue := longIssue("same", longOrigin()) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + changed := issue + changed.Series = append([]forecasting.Series(nil), issue.Series...) + changed.Series[0].Points = append([]forecasting.Point(nil), issue.Series[0].Points...) + changed.Series[0].Points[0].LoadW = 42 + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: changed}}); err == nil { + t.Fatal("changed payload was accepted") + } + loaded, err := arch.LoadIssue(ctx, "same") + if err != nil { + t.Fatal(err) + } + var champ forecasting.Point + for _, series := range loaded.Series { + if series.Name == "champion" { + champ = series.Points[0] + } + } + if float32(champ.LoadW) != float32(issue.Series[0].Points[0].LoadW) { + t.Fatalf("stored load changed to %v", champ.LoadW) + } +} + +func TestForecastLongArchiveDeduplicatesWeather(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + first := longIssue("a", longOrigin()) + second := longIssue("b", longOrigin().Add(time.Hour)) + second.Weather = first.Weather + second.Weather[0].AvailableAtMS = first.OriginMS + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: first}, {Issue: second}}); err != nil { + t.Fatal(err) + } + var n int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_weather`).Scan(&n); err != nil || n != 1 { + t.Fatalf("weather rows = %d err=%v", n, err) + } +} + +func TestForecastLongEncodedHorizonSize(t *testing.T) { + _, arch := openLongArchive(t) + origin := longOrigin() + issue := longIssue("horizon", origin) + quarter := int64(15 * 60 * 1000) + points := make([]forecasting.Point, forecasting.MaxPoints) + for i := range points { + points[i] = longPoint(origin.UnixMilli()+int64(i)*quarter, float64(100+i), float64(500+i), i == 0) + } + for i := range issue.Series { + if forecastLongKeepSeries(issue.Series[i].Name) { + issue.Series[i].Points = points + } + } + // The last baseline series is not stored. Drop the irregular second point + // by replacing every kept series with the regular horizon. + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + var blobs int + if err := arch.db.QueryRow(`SELECT COALESCE(SUM(length(pv_w)+length(load_w)+COALESCE(length(pv_lo),0)+COALESCE(length(pv_hi),0)),0) FROM fa_series`).Scan(&blobs); err != nil { + t.Fatal(err) + } + // Four series of 193 float32 pairs, plus champion bands. This is a fixture + // measurement, not a box measurement. + t.Logf("encoded power and champion band bytes=%d", blobs) + if blobs < 4000 || blobs > 20000 { + t.Fatalf("encoded horizon = %d bytes, outside the expected fixture range", blobs) + } + if math.IsNaN(float64(blobs)) { + t.Fatal("size was not a number") + } +} + +func TestForecastLongGappedOccupancy(t *testing.T) { + _, arch := openLongArchive(t) + issue := longIssue("gap", longOrigin()) + quarter := int64(15 * 60 * 1000) + start := issue.OriginMS + issue.Occupancy = []forecasting.Occupancy{ + {StartMS: start - 4*quarter, EndMS: start - 3*quarter, AvailableAtMS: start - 3*quarter, Home: true}, + {StartMS: start - quarter, EndMS: start, AvailableAtMS: start, Home: false}, + } + if err := arch.ArchiveIssues(context.Background(), []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + loaded, err := arch.LoadIssue(context.Background(), "gap") + if err != nil { + t.Fatal(err) + } + if len(loaded.Occupancy) != 2 || loaded.Occupancy[0].StartMS != issue.Occupancy[0].StartMS || loaded.Occupancy[1].Home { + t.Fatalf("gapped occupancy = %+v", loaded.Occupancy) + } +} diff --git a/go/internal/state/forecast_long_archive_thin_test.go b/go/internal/state/forecast_long_archive_thin_test.go new file mode 100644 index 000000000..fed72a27f --- /dev/null +++ b/go/internal/state/forecast_long_archive_thin_test.go @@ -0,0 +1,436 @@ +package state + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongThinningKeepsOneRunPerHour(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + old := now.Add(-40 * 24 * time.Hour).Truncate(time.Hour) + young := now.Add(-24 * time.Hour).Truncate(time.Hour) + issues := []struct { + id string + origin time.Time + issued time.Time + }{ + {"hour-keep", old, old.Add(time.Second)}, + {"hour-later", old, old.Add(2 * time.Second)}, + {"hour-quarter", old.Add(15 * time.Minute), old}, + {"next-hour", old.Add(time.Hour), old.Add(time.Hour)}, + {"young-a", young, young}, + {"young-b", young.Add(15 * time.Minute), young}, + } + for _, item := range issues { + issue := longIssue(item.id, item.origin) + issue.IssuedAtMS = item.issued.UnixMilli() + if issue.IssuedAtMS < issue.OriginMS { + issue.IssuedAtMS = issue.OriginMS + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatalf("%s: %v", item.id, err) + } + } + // The quarter-hour run is issued at the same instant as its origin above + // when that would otherwise precede the origin. Give it a later issue time + // that is still earlier than the :00 run's issue time only if the :00 + // preference is what keeps hour-keep. + if err := arch.thin(ctx, now); err != nil { + t.Fatal(err) + } + kept := map[string]bool{} + rows, err := arch.db.Query(`SELECT run_id, thinned FROM fa_runs`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + thinned := map[string]int{} + for rows.Next() { + var id string + var flag int + if err := rows.Scan(&id, &flag); err != nil { + t.Fatal(err) + } + kept[id] = true + thinned[id] = flag + } + if kept["hour-later"] || kept["hour-quarter"] || !kept["hour-keep"] || !kept["next-hour"] { + t.Fatalf("old hours = %v", kept) + } + if !kept["young-a"] || !kept["young-b"] || thinned["young-a"] != 0 || thinned["young-b"] != 0 { + t.Fatalf("young runs were thinned: %v flags %v", kept, thinned) + } + if thinned["hour-keep"] != 1 || thinned["next-hour"] != 1 { + t.Fatalf("kept old runs not marked thinned: %v", thinned) + } + var states int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_run_states`).Scan(&states); err != nil { + t.Fatal(err) + } + // One reference for the first kept run of each local day. Both old hours + // are the same local day, so one reference remains. Young runs are not + // thinned, and only young-a, on the hour, holds states. + if states != 2 { + t.Fatalf("state references = %d, want the daily old reference plus young-a", states) + } + var weather int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_weather`).Scan(&weather); err != nil || weather != 4 { + t.Fatalf("weather rows = %d err=%v, want one per kept run", weather, err) + } +} + +func TestForecastLongDeletesAreBatched(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + old := now.Add(-40 * 24 * time.Hour).Truncate(time.Hour) + var batches []int + forecastLongAfterDeleteBatch = func(table string, n int) { + if table == "fa_runs" { + batches = append(batches, n) + } + } + t.Cleanup(func() { forecastLongAfterDeleteBatch = nil }) + for i := 0; i < 70; i++ { + // Same UTC hour. The smallest id is kept; the other 69 are one full delete batch plus a remainder. + issue := longIssue("b-"+forecastLongItoa(i), old) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatalf("issue %d: %v", i, err) + } + } + if err := arch.thin(ctx, now); err != nil { + t.Fatal(err) + } + var runs int + if err := arch.db.QueryRow(`SELECT COUNT(*) FROM fa_runs`).Scan(&runs); err != nil || runs != 1 { + t.Fatalf("runs after thinning = %d err=%v", runs, err) + } + sawFull := false + for _, n := range batches { + if n > forecastPruneBatch { + t.Fatalf("delete batch %d exceeds %d", n, forecastPruneBatch) + } + if n == forecastPruneBatch { + sawFull = true + } + } + if !sawFull { + t.Fatalf("batches = %v, want a full batch of %d", batches, forecastPruneBatch) + } +} + +func TestForecastLongDailyPassesThinWholeHoursAndDays(t *testing.T) { + for _, tc := range []struct { + name, zone string + from, to time.Time + }{ + {"stockholm", "Europe/Stockholm", time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 14, 0, 0, 0, 0, time.UTC)}, + {"kolkata", "Asia/Kolkata", time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 14, 0, 0, 0, 0, time.UTC)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + thinDaily(t, tc.zone, tc.from, tc.to) + }) + } +} + +func TestForecastLongDailyPassesThinDSTDays(t *testing.T) { + for _, tc := range []struct { + name string + from, to time.Time + day time.Time + hours int + }{ + {"spring", time.Date(2026, 3, 28, 0, 0, 0, 0, time.UTC), time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC), time.Date(2026, 3, 29, 0, 0, 0, 0, time.UTC), 23}, + {"autumn", time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC), time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC), time.Date(2026, 10, 25, 0, 0, 0, 0, time.UTC), 25}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + arch := thinDaily(t, "Europe/Stockholm", tc.from, tc.to) + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatal(err) + } + start := time.Date(tc.day.Year(), tc.day.Month(), tc.day.Day(), 0, 0, 0, 0, loc) + end := start.AddDate(0, 0, 1) + var runs, holders int + if err := arch.db.QueryRow(`SELECT COUNT(*), COUNT(DISTINCT s.run_id) FROM fa_runs r + LEFT JOIN fa_run_states s ON s.run_id=r.run_id WHERE r.origin_ms>=? AND r.origin_ms= start && origin < end { + inDay = append(inDay, ids[origin]) + if holders[ids[origin]] { + held = append(held, ids[origin]) + } + } + } + if len(inDay) == 0 { + continue + } + if end <= cut { + if len(held) != 1 || held[0] != inDay[0] { + t.Fatalf("pass %s: complete day %s has state on %v, want only %s", + cutoff.Format(time.RFC3339), day.Format("2006-01-02"), held, inDay[0]) + } + } else { + // An open day keeps states on its runs on the hour only. + var onHour []string + for _, origin := range order { + if origin >= start && origin < end && origin%hourMS == 0 { + onHour = append(onHour, ids[origin]) + } + } + if strings.Join(held, ",") != strings.Join(onHour, ",") { + t.Fatalf("pass %s: open day %s has state on %v, want its runs on the hour %v", + cutoff.Format(time.RFC3339), day.Format("2006-01-02"), held, onHour) + } + } + } + } + return arch +} + +// Each daily pass looks for the next day left to thin. Through the origin +// index that walks every thinned run first; the partial index holds only +// the runs still to thin. +func TestForecastLongThinPlansUseIndexes(t *testing.T) { + _, arch := openLongArchive(t) + span := []any{int64(0), int64(1)} + assertPlanSearches(t, arch, forecastLongNextUnthinned, span, "fa_runs", "fa_runs_unthinned") + assertPlanSearches(t, arch, forecastLongHourLosers, span, "fa_runs", "fa_runs_origin") + assertPlanSearches(t, arch, forecastLongDayRuns, span, "fa_runs", "fa_runs_origin") + assertPlanSearches(t, arch, forecastLongHourHeld, span, "r", "fa_runs_origin") + for _, query := range []string{forecastLongHourLosers, forecastLongDayRuns, forecastLongHourHeld} { + assertPlanNoScan(t, arch, query, span, "s") + } +} + +// assertPlanNoScan fails when the plan reads table (or alias) row by row. +func assertPlanNoScan(t *testing.T, arch *ForecastLongArchive, query string, args []any, table string) { + t.Helper() + rows, err := arch.db.Query(`EXPLAIN QUERY PLAN `+query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if detail == "SCAN "+table || strings.HasPrefix(detail, "SCAN "+table+" ") { + t.Fatalf("%s is scanned: %q", table, detail) + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } +} + +// In an hour with no run on the hour, the run that holds the states may be +// copied before a run issued earlier. Thinning must keep the holder, or the +// whole day loses its model states. +func TestForecastLongThinningKeepsTheStateHolder(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + now := time.Date(2026, 6, 15, 12, 2, 0, 0, time.UTC) + day := now.Add(-40 * 24 * time.Hour).Truncate(24 * time.Hour) + for h := 0; h < 24; h++ { + hour := day.Add(time.Duration(h) * time.Hour) + late := longIssue("x-"+forecastLongItoa(h), hour.Add(30*time.Minute)) + late.IssuedAtMS = hour.Add(31 * time.Minute).UnixMilli() + early := longIssue("y-"+forecastLongItoa(h), hour.Add(15*time.Minute)) + early.IssuedAtMS = hour.Add(16 * time.Minute).UnixMilli() + for _, issue := range []forecasting.Issue{late, early} { // copied out of issue order + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + } + } + if err := arch.thin(ctx, now); err != nil { + t.Fatal(err) + } + var runs, holders int + if err := arch.db.QueryRow(`SELECT COUNT(*), COUNT(DISTINCT s.run_id) FROM fa_runs r + LEFT JOIN fa_run_states s ON s.run_id=r.run_id`).Scan(&runs, &holders); err != nil { + t.Fatal(err) + } + if runs != 24 || holders != 1 { + t.Fatalf("kept %d runs and %d state holders, want 24 and 1", runs, holders) + } + var holder string + if err := arch.db.QueryRow(`SELECT run_id FROM fa_run_states LIMIT 1`).Scan(&holder); err != nil || holder != "x-0" { + t.Fatalf("state holder = %q err=%v, want x-0", holder, err) + } +} + +// Inside the newest 30 days one run per UTC hour keeps its model states: the +// run on the hour, or the first run copied into an hour without states. +func TestForecastLongKeepsStatesOncePerHour(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + hour := time.Now().UTC().Truncate(time.Hour).Add(-3 * time.Hour) + for _, run := range []struct { + id string + origin time.Time + }{ + {"a-00", hour}, + {"a-15", hour.Add(15 * time.Minute)}, + {"b-15", hour.Add(75 * time.Minute)}, // the next hour has no run on the hour yet + {"b-30", hour.Add(90 * time.Minute)}, + {"b-00", hour.Add(time.Hour)}, // copied late; a run on the hour always keeps them + } { + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: longIssue(run.id, run.origin)}}); err != nil { + t.Fatalf("%s: %v", run.id, err) + } + } + rows, err := arch.db.Query(`SELECT DISTINCT run_id FROM fa_run_states ORDER BY run_id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var held []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + held = append(held, id) + } + if got := strings.Join(held, ","); got != "a-00,b-00,b-15" { + t.Fatalf("state holders = %s, want a-00,b-00,b-15", got) + } + for _, id := range []string{"a-15", "b-30"} { + if _, err := arch.LoadIssue(ctx, id); err != nil { + t.Fatalf("%s without states does not load: %v", id, err) + } + } +} diff --git a/go/internal/state/forecast_long_archive_weather_test.go b/go/internal/state/forecast_long_archive_weather_test.go new file mode 100644 index 000000000..b987fffc9 --- /dev/null +++ b/go/internal/state/forecast_long_archive_weather_test.go @@ -0,0 +1,264 @@ +package state + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func longWeather(source string, start, avail int64) forecasting.Weather { + ghi := float64(start%100000) + float64(avail%1000)/10 + cloud := float64(avail % 100) + temp := -3.5 + return forecasting.Weather{ + StartMS: start, EndMS: start + time.Hour.Milliseconds(), AvailableAtMS: avail, + Source: source, GHIWm2: &ghi, CloudPct: &cloud, TempC: &temp, + } +} + +func countRows(t *testing.T, arch *ForecastLongArchive, query string, args ...any) int { + t.Helper() + var n int + if err := arch.db.QueryRow(query, args...).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// A partial weather refresh leaves one issue with rows from two fetches. +// Another run that used the older fetch for the same slot must not bring it +// back into this one. +func TestForecastLongMixedFetchWeatherRoundTrip(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + fetchA := s - hour + fetchB := s + 10*time.Minute.Milliseconds() + + first := longIssue("first", origin) + first.Weather = []forecasting.Weather{ + longWeather("open_meteo", s, fetchA), longWeather("open_meteo", s+hour, fetchA), longWeather("open_meteo", s+2*hour, fetchA), + } + second := longIssue("second", origin.Add(15*time.Minute)) + second.Weather = []forecasting.Weather{ + longWeather("open_meteo", s, fetchB), longWeather("open_meteo", s+hour, fetchA), longWeather("open_meteo", s+2*hour, fetchA), + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: first}, {Issue: second}}); err != nil { + t.Fatal(err) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_weather`); n != 4 { + t.Fatalf("weather rows = %d, want 4 shared rows", n) + } + for _, want := range []forecasting.Issue{first, second} { + got, err := arch.LoadIssue(ctx, want.ID) + if err != nil { + t.Fatalf("%s: %v", want.ID, err) + } + if err := got.Validate(); err != nil { + t.Fatalf("%s: %v", want.ID, err) + } + if !reflect.DeepEqual(got.Weather, want.Weather) { + t.Fatalf("%s weather = %+v, want %+v", want.ID, got.Weather, want.Weather) + } + } + // The same content again is a no-op; a changed weather row is refused. + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: second}}); err != nil { + t.Fatalf("identical re-archive: %v", err) + } + changed := second + changed.Weather = append([]forecasting.Weather(nil), second.Weather...) + changed.Weather[0] = longWeather("open_meteo", s, fetchA) + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: changed}}); !errors.Is(err, errForecastArchiveImmutable) { + t.Fatalf("changed weather err = %v, want immutable", err) + } + got, err := arch.LoadIssue(ctx, "second") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Weather, second.Weather) { + t.Fatalf("stored weather changed to %+v", got.Weather) + } +} + +// Every issued weather row comes back, including rows before the origin and +// past the last series point. +func TestForecastLongWeatherOutsideSeriesSpanRoundTrips(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + issue := longIssue("span", origin) + issue.Weather = []forecasting.Weather{ + longWeather("open_meteo", s-3*hour, s-4*hour), + longWeather("open_meteo", s, s-hour), + longWeather("forecast_solar", s+hour, s-2*hour), + longWeather("open_meteo", s+30*hour, s-hour), + } + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + got, err := arch.LoadIssue(ctx, "span") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Weather, issue.Weather) { + t.Fatalf("weather = %+v, want %+v", got.Weather, issue.Weather) + } +} + +// The plans are checked, not timed: a scan of a reference table inside the +// correlated subquery is what made garbage collection quadratic. +func TestForecastLongGarbagePlansUseIndexes(t *testing.T) { + _, arch := openLongArchive(t) + assertPlanSearches(t, arch, forecastLongWeatherGC, []any{forecastPruneBatch}, "fa_run_weather", "fa_run_weather_fetch") + assertPlanSearches(t, arch, forecastLongStateGC, []any{forecastPruneBatch}, "fa_run_states", "fa_run_states_ref") +} + +// assertPlanSearches fails unless the query plan reads table through index +// and never scans it. +func assertPlanSearches(t *testing.T, arch *ForecastLongArchive, query string, args []any, table, index string) { + t.Helper() + rows, err := arch.db.Query(`EXPLAIN QUERY PLAN `+query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + searched := false + for _, detail := range plan { + if strings.HasPrefix(detail, "SCAN "+table) { + t.Fatalf("%s is scanned: %q", table, plan) + } + if strings.HasPrefix(detail, "SEARCH "+table+" ") && strings.Contains(detail, "INDEX "+index+" ") { + searched = true + } + } + if !searched { + t.Fatalf("%s is not searched through %s: %q", table, index, plan) + } +} + +func TestForecastLongGarbageKeepsLinkedWeather(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + fetchA, fetchB, fetchC := s-3*hour, s-2*hour, s-hour + + only := longIssue("only-a", origin) + only.Weather = []forecasting.Weather{longWeather("open_meteo", s, fetchA), longWeather("open_meteo", s+hour, fetchA)} + mixed := longIssue("mixed", origin.Add(15*time.Minute)) + mixed.Weather = []forecasting.Weather{longWeather("open_meteo", s, fetchB), longWeather("open_meteo", s+hour, fetchC)} + shared := longIssue("shared", origin.Add(30*time.Minute)) + shared.Weather = []forecasting.Weather{longWeather("open_meteo", s, fetchB), longWeather("open_meteo", s+2*hour, fetchC)} + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: only}, {Issue: mixed}, {Issue: shared}}); err != nil { + t.Fatal(err) + } + if err := arch.deleteRunIDs(ctx, []string{"only-a", "mixed"}); err != nil { + t.Fatal(err) + } + if err := arch.collectGarbage(ctx); err != nil { + t.Fatal(err) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_weather WHERE available_at_ms=?`, fetchA); n != 0 { + t.Fatalf("orphan fetch rows = %d, want 0", n) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_run_weather`); n != 2 { + t.Fatalf("weather links = %d, want the kept run's two fetches", n) + } + got, err := arch.LoadIssue(ctx, "shared") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Weather, shared.Weather) { + t.Fatalf("kept weather = %+v, want %+v", got.Weather, shared.Weather) + } + if err := arch.deleteRunIDs(ctx, []string{"shared"}); err != nil { + t.Fatal(err) + } + if err := arch.collectGarbage(ctx); err != nil { + t.Fatal(err) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_weather`); n != 0 { + t.Fatalf("weather rows after the last run = %d, want 0", n) + } + if n := countRows(t, arch, `SELECT COUNT(*) FROM fa_model_states`); n != 0 { + t.Fatalf("model states after the last run = %d, want 0", n) + } +} + +func TestForecastLongWeatherStartsCodec(t *testing.T) { + hour := time.Hour.Milliseconds() + s := longOrigin().UnixMilli() + starts := []int64{s + 2*hour, s, s + hour, s + 47*hour} + blob, err := encodeForecastStarts(starts) + if err != nil { + t.Fatal(err) + } + got, err := decodeForecastStarts(blob) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []int64{s, s + hour, s + 2*hour, s + 47*hour}) { + t.Fatalf("starts = %v", got) + } + day := make([]int64, 48) + for i := range day { + day[i] = s + int64(i)*hour + } + if blob, err = encodeForecastStarts(day); err != nil || len(blob) > 200 { + t.Fatalf("hourly day = %d bytes err=%v", len(blob), err) + } + for _, bad := range [][]int64{nil, {s, s}, {0}} { + if _, err := encodeForecastStarts(bad); err == nil { + t.Fatalf("%v was accepted", bad) + } + } + for _, bad := range [][]byte{nil, {0x80}, {0x01, 0x00}} { + if _, err := decodeForecastStarts(bad); err == nil { + t.Fatalf("%x was accepted", bad) + } + } +} + +// A run must load with every weather row it was issued with. If one linked +// row is gone, LoadIssue fails rather than return a shorter list. +func TestForecastLongLoadFailsWhenLinkedWeatherIsMissing(t *testing.T) { + _, arch := openLongArchive(t) + ctx := context.Background() + origin := longOrigin() + s := origin.UnixMilli() + hour := time.Hour.Milliseconds() + issue := longIssue("gap", origin) + issue.Weather = []forecasting.Weather{longWeather("open_meteo", s, s-hour), longWeather("open_meteo", s+hour, s-hour)} + if err := arch.ArchiveIssues(ctx, []ForecastArchiveIssue{{Issue: issue}}); err != nil { + t.Fatal(err) + } + if _, err := arch.db.ExecContext(ctx, `DELETE FROM fa_weather WHERE start_ms=?`, s+hour); err != nil { + t.Fatal(err) + } + got, err := arch.LoadIssue(ctx, "gap") + if err == nil || !strings.Contains(err.Error(), "missing 1 weather rows") { + t.Fatalf("LoadIssue = %d weather rows, err=%v; want a missing-row error", len(got.Weather), err) + } +} diff --git a/go/internal/state/forecast_long_codec.go b/go/internal/state/forecast_long_codec.go new file mode 100644 index 000000000..bcb348bfa --- /dev/null +++ b/go/internal/state/forecast_long_codec.go @@ -0,0 +1,414 @@ +package state + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "sort" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +const ( + forecastLongSeriesChampion = "champion" + forecastLongSeriesPlanning = "planning" + forecastLongSeriesLegacy = "legacy_shadow" + forecastLongSeriesEnergyplan = "energyplan" + forecastLongQuarterMS = int64(15 * 60 * 1000) + forecastLongMaxVector = forecasting.MaxPoints + forecastLongBandCold byte = 0 + forecastLongBandEmpirical byte = 1 +) + +func forecastLongKeepSeries(name string) bool { + switch name { + case forecastLongSeriesChampion, forecastLongSeriesPlanning, forecastLongSeriesLegacy, forecastLongSeriesEnergyplan: + return true + default: + return false + } +} + +// encodeF32 writes little-endian float32 values. NaN is kept for an unknown +// weather or band value. Infinity and magnitudes float32 cannot hold are +// rejected; contract points are already finite. +func encodeF32(vals []float64) ([]byte, error) { + if len(vals) > forecastLongMaxVector { + return nil, errors.New("forecast archive vector is too long") + } + out := make([]byte, 4*len(vals)) + for i, v := range vals { + if math.IsNaN(v) { + binary.LittleEndian.PutUint32(out[i*4:], math.Float32bits(float32(math.NaN()))) + continue + } + if math.IsInf(v, 0) || v > math.MaxFloat32 || v < -math.MaxFloat32 { + return nil, fmt.Errorf("forecast archive value %v is outside float32", v) + } + binary.LittleEndian.PutUint32(out[i*4:], math.Float32bits(float32(v))) + } + return out, nil +} + +func decodeF32(b []byte, n int) ([]float64, error) { + if n < 0 || n > forecastLongMaxVector || len(b) != 4*n { + return nil, errors.New("forecast archive float32 length mismatch") + } + out := make([]float64, n) + for i := 0; i < n; i++ { + out[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(b[i*4:]))) + } + return out, nil +} + +func encodeU8(vals []byte) ([]byte, error) { + if len(vals) > forecastLongMaxVector { + return nil, errors.New("forecast archive vector is too long") + } + return append([]byte(nil), vals...), nil +} + +func decodeU8(b []byte, n int) ([]byte, error) { + if n < 0 || n > forecastLongMaxVector || len(b) != n { + return nil, errors.New("forecast archive byte vector length mismatch") + } + return append([]byte(nil), b...), nil +} + +// seriesLayout is the time grid of one stored series. A regular grid leaves +// starts empty. An irregular grid stores little-endian int32 offsets, in +// seconds, from startMS. When durations differ, each point is an offset and +// a duration (2n values) so gaps still round-trip. +type seriesLayout struct { + startMS int64 + slotMS int64 + n int + starts []byte + predictionStartMS int64 +} + +func layoutSeries(points []forecasting.Point) (seriesLayout, error) { + if len(points) == 0 || len(points) > forecastLongMaxVector { + return seriesLayout{}, errors.New("forecast archive series length is invalid") + } + for i, p := range points { + if i > 0 && p.PredictionStartMS != 0 { + return seriesLayout{}, errors.New("forecast archive partial point is only stored on the first interval") + } + } + start := points[0].StartMS + slot := points[0].EndMS - points[0].StartMS + regular := true + sameSlot := true + for i, p := range points { + d := p.EndMS - p.StartMS + if d <= 0 || d > int64(60*60*1000) { + return seriesLayout{}, errors.New("forecast archive interval is invalid") + } + if d != slot { + sameSlot = false + regular = false + } + if p.StartMS != start+int64(i)*slot { + regular = false + } + if (p.StartMS-start)%1000 != 0 || d%1000 != 0 { + return seriesLayout{}, errors.New("forecast archive interval is not a whole number of seconds") + } + } + out := seriesLayout{startMS: start, n: len(points), predictionStartMS: points[0].PredictionStartMS} + if regular { + out.slotMS = slot + return out, nil + } + if sameSlot { + out.slotMS = slot + out.starts = make([]byte, 4*len(points)) + for i, p := range points { + off := (p.StartMS - start) / 1000 + if off > math.MaxInt32 || off < math.MinInt32 { + return seriesLayout{}, errors.New("forecast archive time offset overflows") + } + binary.LittleEndian.PutUint32(out.starts[i*4:], uint32(int32(off))) + } + return out, nil + } + out.starts = make([]byte, 8*len(points)) + for i, p := range points { + off := (p.StartMS - start) / 1000 + dur := (p.EndMS - p.StartMS) / 1000 + if off > math.MaxInt32 || off < math.MinInt32 || dur > math.MaxInt32 || dur <= 0 { + return seriesLayout{}, errors.New("forecast archive time offset overflows") + } + binary.LittleEndian.PutUint32(out.starts[i*8:], uint32(int32(off))) + binary.LittleEndian.PutUint32(out.starts[i*8+4:], uint32(int32(dur))) + } + return out, nil +} + +func expandLayout(l seriesLayout) (starts, ends []int64, err error) { + if l.n <= 0 || l.startMS <= 0 { + return nil, nil, errors.New("forecast archive series grid is empty") + } + starts = make([]int64, l.n) + ends = make([]int64, l.n) + switch { + case len(l.starts) == 0: + if l.slotMS <= 0 { + return nil, nil, errors.New("forecast archive regular grid has no slot") + } + for i := 0; i < l.n; i++ { + starts[i] = l.startMS + int64(i)*l.slotMS + ends[i] = starts[i] + l.slotMS + } + case len(l.starts) == 4*l.n: + if l.slotMS <= 0 { + return nil, nil, errors.New("forecast archive irregular grid has no slot") + } + for i := 0; i < l.n; i++ { + off := int32(binary.LittleEndian.Uint32(l.starts[i*4:])) + starts[i] = l.startMS + int64(off)*1000 + ends[i] = starts[i] + l.slotMS + } + case len(l.starts) == 8*l.n: + for i := 0; i < l.n; i++ { + off := int32(binary.LittleEndian.Uint32(l.starts[i*8:])) + dur := int32(binary.LittleEndian.Uint32(l.starts[i*8+4:])) + if dur <= 0 { + return nil, nil, errors.New("forecast archive irregular duration is empty") + } + starts[i] = l.startMS + int64(off)*1000 + ends[i] = starts[i] + int64(dur)*1000 + } + default: + return nil, nil, errors.New("forecast archive time grid length mismatch") + } + return starts, ends, nil +} + +func packBandMethod(p forecasting.Point) byte { + code := func(method string) byte { + if method == forecasting.BandMethodEmpirical { + return forecastLongBandEmpirical + } + return forecastLongBandCold + } + return code(p.PVBand.Method) | code(p.LoadBand.Method)<<2 | code(p.NetBand.Method)<<4 +} + +func unpackBandMethod(v byte) (pv, load, net string) { + name := func(code byte) string { + if code&0x3 == forecastLongBandEmpirical { + return forecasting.BandMethodEmpirical + } + return forecasting.BandMethodColdStart + } + return name(v), name(v >> 2), name(v >> 4) +} + +func encodeBandStats(points []forecasting.Point) ([]byte, error) { + out := make([]byte, len(points)*6*4) + put := func(i, field, v int) error { + if v < 0 || v > math.MaxUint32 { + return errors.New("forecast archive band statistic overflows") + } + binary.LittleEndian.PutUint32(out[(i*6+field)*4:], uint32(v)) + return nil + } + for i, p := range points { + stats := []int{p.PVBand.Samples, p.PVBand.Days, p.LoadBand.Samples, p.LoadBand.Days, p.NetBand.Samples, p.NetBand.Days} + for field, v := range stats { + if err := put(i, field, v); err != nil { + return nil, err + } + } + } + return out, nil +} + +func decodeBandStats(b []byte, n int) (samples, days [][3]int, err error) { + if len(b) != n*6*4 { + return nil, nil, errors.New("forecast archive band statistic length mismatch") + } + samples = make([][3]int, n) + days = make([][3]int, n) + for i := 0; i < n; i++ { + for field := 0; field < 3; field++ { + samples[i][field] = int(binary.LittleEndian.Uint32(b[(i*6+field*2)*4:])) + days[i][field] = int(binary.LittleEndian.Uint32(b[(i*6+field*2+1)*4:])) + } + } + return samples, days, nil +} + +// encodeForecastStarts packs the weather slot starts one run used from one +// fetch as uvarints: the first start, then each gap to the next. An hourly +// day of 48 slots takes about 200 bytes. +func encodeForecastStarts(starts []int64) ([]byte, error) { + if len(starts) == 0 || len(starts) > forecastLongMaxVector { + return nil, errors.New("forecast archive weather slot list length is invalid") + } + sorted := append([]int64(nil), starts...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + if sorted[0] <= 0 { + return nil, errors.New("forecast archive weather slot start is invalid") + } + out := binary.AppendUvarint(nil, uint64(sorted[0])) + for i := 1; i < len(sorted); i++ { + if sorted[i] == sorted[i-1] { + return nil, errors.New("forecast archive weather slot repeats") + } + out = binary.AppendUvarint(out, uint64(sorted[i]-sorted[i-1])) + } + return out, nil +} + +func decodeForecastStarts(b []byte) ([]int64, error) { + var out []int64 + var last uint64 + for len(b) > 0 { + v, n := binary.Uvarint(b) + if n <= 0 || v == 0 || v > math.MaxInt64-last || len(out) >= forecastLongMaxVector { + return nil, errors.New("forecast archive weather slot list is invalid") + } + last += v + out = append(out, int64(last)) + b = b[n:] + } + if len(out) == 0 { + return nil, errors.New("forecast archive weather slot list is empty") + } + return out, nil +} + +// encodeOccupancy stores a bit per quarter when the rows are contiguous, and +// the available-at time of each quarter so a restored issue still validates. +// Gaps use an explicit record. An empty slice returns a nil blob. +func encodeOccupancy(rows []forecasting.Occupancy) (blob []byte, startMS int64, n int, err error) { + if len(rows) == 0 { + return nil, 0, 0, nil + } + if len(rows) > 512 { + return nil, 0, 0, errors.New("forecast archive occupancy is too long") + } + n = len(rows) + startMS = rows[0].StartMS + contiguous := true + for i, row := range rows { + if row.EndMS-row.StartMS != forecastLongQuarterMS { + contiguous = false + break + } + if i > 0 && row.StartMS != rows[i-1].EndMS { + contiguous = false + break + } + } + if contiguous { + bits := (n + 7) / 8 + blob = make([]byte, 1+bits+n*8) + blob[0] = 0 + for i, row := range rows { + if row.Home { + blob[1+i/8] |= 1 << (uint(i) % 8) + } + binary.LittleEndian.PutUint64(blob[1+bits+i*8:], uint64(row.AvailableAtMS)) + } + return blob, startMS, n, nil + } + blob = make([]byte, 1+n*25) + blob[0] = 1 + off := 1 + for _, row := range rows { + binary.LittleEndian.PutUint64(blob[off:], uint64(row.StartMS)) + binary.LittleEndian.PutUint64(blob[off+8:], uint64(row.EndMS)) + binary.LittleEndian.PutUint64(blob[off+16:], uint64(row.AvailableAtMS)) + if row.Home { + blob[off+24] = 1 + } + off += 25 + } + return blob, startMS, n, nil +} + +func decodeOccupancy(blob []byte, startMS int64, n int) ([]forecasting.Occupancy, error) { + if n == 0 { + return nil, nil + } + if n < 0 || n > 512 || len(blob) == 0 { + return nil, errors.New("forecast archive occupancy is invalid") + } + switch blob[0] { + case 0: + bits := (n + 7) / 8 + if len(blob) != 1+bits+n*8 || startMS <= 0 { + return nil, errors.New("forecast archive occupancy bitset length mismatch") + } + out := make([]forecasting.Occupancy, n) + for i := 0; i < n; i++ { + out[i] = forecasting.Occupancy{ + StartMS: startMS + int64(i)*forecastLongQuarterMS, + EndMS: startMS + int64(i+1)*forecastLongQuarterMS, + AvailableAtMS: int64(binary.LittleEndian.Uint64(blob[1+bits+i*8:])), + Home: blob[1+i/8]&(1<<(uint(i)%8)) != 0, + } + } + return out, nil + case 1: + if len(blob) != 1+n*25 { + return nil, errors.New("forecast archive occupancy record length mismatch") + } + out := make([]forecasting.Occupancy, n) + off := 1 + for i := 0; i < n; i++ { + out[i] = forecasting.Occupancy{ + StartMS: int64(binary.LittleEndian.Uint64(blob[off:])), + EndMS: int64(binary.LittleEndian.Uint64(blob[off+8:])), + AvailableAtMS: int64(binary.LittleEndian.Uint64(blob[off+16:])), + Home: blob[off+24] != 0, + } + off += 25 + } + return out, nil + default: + return nil, errors.New("forecast archive occupancy version is unknown") + } +} + +// forecastLongInputHash is the SHA-256 of the frozen site, occupancy, weather +// and model references. Series watts are stored beside it and compared on +// their own, so a changed forecast with the same inputs is still rejected. +func forecastLongInputHash(issue forecasting.Issue) (string, error) { + models := append([]forecasting.ModelState(nil), issue.Models...) + sort.Slice(models, func(i, j int) bool { return models[i].Name < models[j].Name }) + for i := range models { + models[i].State = nil + } + weather := append([]forecasting.Weather(nil), issue.Weather...) + sort.Slice(weather, func(i, j int) bool { + if weather[i].AvailableAtMS != weather[j].AvailableAtMS { + return weather[i].AvailableAtMS < weather[j].AvailableAtMS + } + if weather[i].Source != weather[j].Source { + return weather[i].Source < weather[j].Source + } + return weather[i].StartMS < weather[j].StartMS + }) + occupancy := append([]forecasting.Occupancy(nil), issue.Occupancy...) + sort.Slice(occupancy, func(i, j int) bool { return occupancy[i].StartMS < occupancy[j].StartMS }) + payload, err := json.Marshal(struct { + Site *forecasting.SiteContext `json:"site,omitempty"` + Occupancy []forecasting.Occupancy `json:"occupancy,omitempty"` + Weather []forecasting.Weather `json:"weather,omitempty"` + Models []forecasting.ModelState `json:"models,omitempty"` + }{issue.Site, occupancy, weather, models}) + if err != nil { + return "", err + } + sum := sha256.Sum256(payload) + return hex.EncodeToString(sum[:]), nil +} diff --git a/go/internal/state/forecast_long_codec_test.go b/go/internal/state/forecast_long_codec_test.go new file mode 100644 index 000000000..8460c69e1 --- /dev/null +++ b/go/internal/state/forecast_long_codec_test.go @@ -0,0 +1,86 @@ +package state + +import ( + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +func TestForecastLongCodecRoundTrip(t *testing.T) { + values := []float64{0, 1, 1500.5, math.MaxFloat32, -math.MaxFloat32, math.NaN()} + encoded, err := encodeF32(values) + if err != nil { + t.Fatal(err) + } + if len(encoded) != 4*len(values) { + t.Fatalf("encoded length %d", len(encoded)) + } + got, err := decodeF32(encoded, len(values)) + if err != nil { + t.Fatal(err) + } + for i, want := range values { + if math.IsNaN(want) { + if !math.IsNaN(got[i]) { + t.Fatalf("value %d: want NaN, got %v", i, got[i]) + } + continue + } + if float32(got[i]) != float32(want) { + t.Fatalf("value %d: got %v want %v", i, got[i], want) + } + } + if _, err := decodeF32(encoded, len(values)-1); err == nil { + t.Fatal("length mismatch was accepted") + } + bytes, err := encodeU8([]byte{0, 1, 255}) + if err != nil { + t.Fatal(err) + } + if _, err := decodeU8(bytes, 2); err == nil { + t.Fatal("byte length mismatch was accepted") + } +} + +func TestForecastLongCodecRejectsNonFinite(t *testing.T) { + if _, err := encodeF32([]float64{math.Inf(1)}); err == nil { + t.Fatal("positive infinity was accepted") + } + if _, err := encodeF32([]float64{math.Inf(-1)}); err == nil { + t.Fatal("negative infinity was accepted") + } + if _, err := encodeF32([]float64{math.MaxFloat32 * 2}); err == nil { + t.Fatal("value outside float32 was accepted") + } +} + +func TestForecastLongIrregularGridAndPartialPoint(t *testing.T) { + start := time.Date(2026, 1, 5, 12, 0, 0, 0, time.UTC).UnixMilli() + points := []forecasting.Point{ + {StartMS: start, EndMS: start + 15*60*1000, PredictionStartMS: start + 60*1000}, + {StartMS: start + 30*60*1000, EndMS: start + 90*60*1000}, + } + layout, err := layoutSeries(points) + if err != nil { + t.Fatal(err) + } + if layout.predictionStartMS != start+60*1000 { + t.Fatalf("partial first point = %d", layout.predictionStartMS) + } + if len(layout.starts) != 16 { + t.Fatalf("irregular grid bytes = %d, want 16", len(layout.starts)) + } + starts, ends, err := expandLayout(layout) + if err != nil { + t.Fatal(err) + } + if starts[0] != points[0].StartMS || ends[0] != points[0].EndMS || starts[1] != points[1].StartMS || ends[1] != points[1].EndMS { + t.Fatalf("grid = %v %v", starts, ends) + } + points[1].PredictionStartMS = points[1].StartMS + 1000 + if _, err := layoutSeries(points); err == nil { + t.Fatal("partial point after the first was accepted") + } +} diff --git a/go/internal/state/forecast_long_copy.go b/go/internal/state/forecast_long_copy.go new file mode 100644 index 000000000..864c4e800 --- /dev/null +++ b/go/internal/state/forecast_long_copy.go @@ -0,0 +1,469 @@ +package state + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "log/slog" + "time" + + "github.com/srcfl/ftw/go/internal/forecasting" +) + +const ( + forecastLongIssuePage = 16 + forecastLongObsPage = 128 + // forecastLongCopyOverlap is how far before its cursor each copy reads + // again. A row that commits late with a slightly earlier key, such as a + // shadow issue saved after its primary or a write after a small clock + // step back, is still copied. It must be at least maxForecastFutureSkew: + // a cursor never runs further ahead of the clock than that, so every row + // written after a step back sorts after cursor minus overlap. + forecastLongCopyOverlap = time.Hour +) + +// forecastLongAfterIssuePage runs after a committed issue page. Tests use it +// to stop a copy between pages. +var forecastLongAfterIssuePage func(rows int) error + +type forecastLongIssueCursor struct { + IssuedAtMS int64 `json:"issued_at_ms"` + ID string `json:"id"` +} + +func (c forecastLongIssueCursor) after(o forecastLongIssueCursor) bool { + return c.IssuedAtMS > o.IssuedAtMS || (c.IssuedAtMS == o.IssuedAtMS && c.ID > o.ID) +} + +// forecastLongObsCursor follows available_at_ms, the time a quarter was +// released. The tracker writes observations in that order, while a quarter +// with an earlier start can be released after a later one. +type forecastLongObsCursor struct { + AvailableMS int64 `json:"available_at_ms"` + StartMS int64 `json:"start_ms"` + EndMS int64 `json:"end_ms"` + Config string `json:"config_version"` +} + +func (c forecastLongObsCursor) after(o forecastLongObsCursor) bool { + if c.AvailableMS != o.AvailableMS { + return c.AvailableMS > o.AvailableMS + } + if c.StartMS != o.StartMS { + return c.StartMS > o.StartMS + } + if c.EndMS != o.EndMS { + return c.EndMS > o.EndMS + } + return c.Config > o.Config +} + +// forecastLongSourceError marks a failed read of state.db. Damage there is +// healed at boot and must not send the archive to quarantine. +type forecastLongSourceError struct{ err error } + +func (e forecastLongSourceError) Error() string { return "state.db: " + e.err.Error() } +func (e forecastLongSourceError) Unwrap() error { return e.err } + +func fromForecastSource(err error) error { + if err == nil { + return nil + } + return forecastLongSourceError{err} +} + +// forecastLongDamaged reports a corruption code from the archive itself. +func forecastLongDamaged(err error) bool { + var source forecastLongSourceError + return sqliteCorrupt(err) && !errors.As(err, &source) +} + +// Sync copies the 30-day issue and observation tables and holds the archive +// to its cap. Once a day it also thins, prunes expired runs and gives free +// pages back. Failures stay in the published status. Below the disk floor +// copying pauses until the next call, but the deletes still run. When the +// archive reports damage, Sync closes it, so the next copy reopens the file, +// runs quick_check and heals it without a restart. +func (a *ForecastLongArchive) Sync(ctx context.Context, src *Store, now time.Time) error { + err := a.syncOnce(ctx, src, now) + if forecastLongDamaged(err) && a.store != nil { + slog.Warn("forecast long archive: damage found while copying; the next copy checks the file", "err", err) + a.store.dropForecastLongArchive(a) + } + return err +} + +func (a *ForecastLongArchive) syncOnce(ctx context.Context, src *Store, now time.Time) error { + if a == nil || src == nil { + return errors.New("forecast archive: copy source unavailable") + } + if now.IsZero() { + now = time.Now().UTC() + } + a.mu.Lock() + a.status.LastError = "" + a.mu.Unlock() + if err := src.InitForecastArchive(ctx); err != nil { + err = fromForecastSource(err) + a.fail("schema", err) + return err + } + var paused error + if !a.blocked() { + if err := a.copyIssues(ctx, src, now); err != nil { + if forecastLongPaused(err) { + paused = err + } else { + a.fail("copy-issues", err) + return err + } + } + } + if paused == nil && !a.blocked() { + if err := a.copyObservations(ctx, src, now); err != nil { + if forecastLongPaused(err) { + paused = err + } else { + a.fail("copy-observations", err) + return err + } + } + } + due, err := a.maintenanceDue(ctx, now) + if err != nil { + a.fail("maintenance", err) + return err + } + if due { + err = a.maintain(ctx, now) + if err == nil { + err = a.setMeta(ctx, "last_maintain_ms", jsonInt(now.UnixMilli())) + } + } else { + err = a.enforceCap(ctx, now) + } + if err != nil { + if !forecastLongPaused(err) { + a.fail("maintenance", err) + return err + } + if paused == nil { + paused = err + } + } + if err := a.recompute(ctx, src, now, paused); err != nil { + a.fail("status", err) + return err + } + return nil +} + +func forecastLongPaused(err error) bool { + return errors.Is(err, errForecastArchiveDiskFull) || sqliteFull(err) +} + +func (a *ForecastLongArchive) blocked() bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.status.State == "over_cap" +} + +func (a *ForecastLongArchive) fail(kind string, err error) { + a.warn(kind, err) + a.mu.Lock() + if a.status.State == "" || a.status.State == "starting" || a.status.State == "ok" { + a.status.State = "unavailable" + } + a.mu.Unlock() + a.publish() +} + +// copyIssues reads from an hour before its cursor up to the clock plus +// maxForecastFutureSkew. A run already archived is skipped by id before its +// payload is read, so the overlap and a full rescan stay cheap. A cursor +// further ahead than that bound came from a wrong clock; the copy then +// starts again from the beginning of the 30-day table. +func (a *ForecastLongArchive) copyIssues(ctx context.Context, src *Store, now time.Time) error { + until := now.UnixMilli() + maxForecastFutureSkew.Milliseconds() + cursor, err := a.issueCursor(ctx) + if err != nil { + return err + } + if cursor.IssuedAtMS > until { + slog.Warn("forecast long archive: issue cursor is ahead of the clock; copying the 30-day table again", + "cursor_ms", cursor.IssuedAtMS, "now_ms", now.UnixMilli()) + cursor = forecastLongIssueCursor{} + if err := a.writeIssueCursor(ctx, cursor); err != nil { + return err + } + } + from := forecastLongIssueCursor{IssuedAtMS: cursor.IssuedAtMS - forecastLongCopyOverlap.Milliseconds()} + for { + if err := ctx.Err(); err != nil { + return err + } + batch, err := a.readIssuePage(ctx, src, from, until) + if err != nil { + return err + } + if batch.keys == 0 { + return nil + } + from = batch.last + advanced := from.after(cursor) + writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) + err = a.ArchiveIssues(writeCtx, batch.issues) + if errors.Is(err, errForecastArchiveImmutable) { + err = nil // warned; the stored run stays + } + if err == nil && advanced { + cursor = from + err = a.writeIssueCursor(writeCtx, cursor) + } + cancelWrite() + if err != nil { + return err + } + if forecastLongAfterIssuePage != nil { + if err := forecastLongAfterIssuePage(batch.keys); err != nil { + return err + } + } + if advanced || len(batch.issues) > 0 { + a.mu.Lock() + a.status.LastCopyMS = time.Now().UnixMilli() + a.mu.Unlock() + } + if batch.keys < forecastLongIssuePage { + return nil + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +type forecastLongIssueBatch struct { + issues []ForecastArchiveIssue + last forecastLongIssueCursor + keys int +} + +// readIssuePage lists one page of keys and loads only the runs the archive +// does not hold. Model states are read outside the page deadline, as +// before, so a slow blob read does not use it up. +func (a *ForecastLongArchive) readIssuePage(ctx context.Context, src *Store, from forecastLongIssueCursor, until int64) (forecastLongIssueBatch, error) { + var batch forecastLongIssueBatch + pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) + defer cancel() + keys, err := src.forecastIssueKeysAfter(pageCtx, from, until, forecastLongIssuePage) + if err != nil || len(keys) == 0 { + return batch, fromForecastSource(err) + } + for _, key := range keys { + known, err := a.hasRun(pageCtx, key.ID) + if err != nil { + return batch, err + } + if known { + continue + } + item, err := src.forecastIssueCopy(pageCtx, key) + if errors.Is(err, sql.ErrNoRows) { + continue // pruned from state.db after the key list + } + if err != nil { + return batch, fromForecastSource(err) + } + if item.Err != nil { + a.warn("decode", item.Err) + continue + } + states, missing, err := forecastLongStatesFrom(ctx, src, item.Issue) + if err != nil { + return batch, err + } + if missing > 0 { + // state.db pruned these blobs first; the run keeps its ids. + slog.Info("forecast long archive: model state no longer in state.db", "run", key.ID, "missing", missing) + } + batch.issues = append(batch.issues, ForecastArchiveIssue{ + Issue: item.Issue, JSONBytes: item.JSONBytes, GzipBytes: item.GzipBytes, States: states, + }) + } + batch.last, batch.keys = keys[len(keys)-1], len(keys) + return batch, nil +} + +func (a *ForecastLongArchive) hasRun(ctx context.Context, id string) (bool, error) { + var one int + err := a.db.QueryRowContext(ctx, `SELECT 1 FROM fa_runs WHERE run_id=?`, id).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return err == nil, err +} + +func forecastLongStatesFrom(ctx context.Context, src *Store, issue forecasting.Issue) (map[string]ForecastModelBlob, int, error) { + states := map[string]ForecastModelBlob{} + missing := 0 + for _, model := range issue.Models { + if model.StateID == "" { + continue + } + if _, ok := states[model.StateID]; ok { + continue + } + expanded, payload, err := src.LoadForecastModelStateRaw(ctx, model.StateID) + if errors.Is(err, sql.ErrNoRows) { + missing++ + continue + } + if err != nil { + return nil, 0, fromForecastSource(err) + } + states[model.StateID] = ForecastModelBlob{Expanded: expanded, Gzip: payload} + } + return states, missing, nil +} + +// copyObservations follows the same rules as copyIssues on the +// available_at_ms order. Rows read again are ignored on insert. +func (a *ForecastLongArchive) copyObservations(ctx context.Context, src *Store, now time.Time) error { + until := now.UnixMilli() + maxForecastFutureSkew.Milliseconds() + cursor, err := a.obsCursor(ctx) + if err != nil { + return err + } + if cursor.AvailableMS > until { + slog.Warn("forecast long archive: observation cursor is ahead of the clock; copying the 30-day table again", + "cursor_ms", cursor.AvailableMS, "now_ms", now.UnixMilli()) + cursor = forecastLongObsCursor{} + if err := a.writeObsCursor(ctx, cursor); err != nil { + return err + } + } + from := forecastLongObsCursor{AvailableMS: cursor.AvailableMS - forecastLongCopyOverlap.Milliseconds()} + for { + if err := ctx.Err(); err != nil { + return err + } + pageCtx, cancel := context.WithTimeout(ctx, forecastLongPageTimeout) + page, err := src.forecastObservationsAfter(pageCtx, from, until, forecastLongObsPage) + cancel() + if err != nil { + return fromForecastSource(err) + } + if len(page) == 0 { + return nil + } + from = page[len(page)-1].Key + rows := make([]forecasting.Observation, 0, len(page)) + for _, row := range page { + if row.Err != nil { + a.warn("decode", row.Err) + continue + } + rows = append(rows, row.Observation) + } + writeCtx, cancelWrite := context.WithTimeout(ctx, forecastLongPageTimeout) + err = a.ArchiveObservations(writeCtx, rows) + if err == nil && from.after(cursor) { + cursor = from + err = a.writeObsCursor(writeCtx, cursor) + } + cancelWrite() + if err != nil { + return err + } + if len(page) < forecastLongObsPage { + return nil + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) issueCursor(ctx context.Context) (forecastLongIssueCursor, error) { + var cursor forecastLongIssueCursor + raw, err := a.metaValue(ctx, "copy_cursor_issue") + if err != nil || raw == "" { + return cursor, err + } + err = json.Unmarshal([]byte(raw), &cursor) + return cursor, err +} + +func (a *ForecastLongArchive) writeIssueCursor(ctx context.Context, cursor forecastLongIssueCursor) error { + data, err := json.Marshal(cursor) + if err != nil { + return err + } + return a.setMeta(ctx, "copy_cursor_issue", string(data)) +} + +func (a *ForecastLongArchive) obsCursor(ctx context.Context) (forecastLongObsCursor, error) { + var cursor forecastLongObsCursor + raw, err := a.metaValue(ctx, "copy_cursor_obs") + if err != nil || raw == "" { + return cursor, err + } + err = json.Unmarshal([]byte(raw), &cursor) + return cursor, err +} + +func (a *ForecastLongArchive) writeObsCursor(ctx context.Context, cursor forecastLongObsCursor) error { + data, err := json.Marshal(cursor) + if err != nil { + return err + } + return a.setMeta(ctx, "copy_cursor_obs", string(data)) +} + +func (a *ForecastLongArchive) metaValue(ctx context.Context, key string) (string, error) { + var value string + err := a.db.QueryRowContext(ctx, `SELECT value FROM fa_meta WHERE key=?`, key).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return value, err +} + +func (a *ForecastLongArchive) setMeta(ctx context.Context, key, value string) error { + _, err := a.db.ExecContext(ctx, `INSERT INTO fa_meta(key, value) VALUES(?, ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value`, key, value) + return err +} + +func (a *ForecastLongArchive) maintenanceDue(ctx context.Context, now time.Time) (bool, error) { + raw, err := a.metaValue(ctx, "last_maintain_ms") + if err != nil || raw == "" { + return err == nil, err + } + var ms int64 + if err := json.Unmarshal([]byte(raw), &ms); err != nil { + return true, nil + } + return now.Sub(time.UnixMilli(ms)) >= a.maintainEvery, nil +} + +func jsonInt(v int64) string { + data, _ := json.Marshal(v) + return string(data) +} + +func (src *Store) forecastCopyLag(ctx context.Context, cursor forecastLongIssueCursor, now int64) (int64, error) { + var issued sql.NullInt64 + err := src.db.QueryRowContext(ctx, `SELECT MIN(issued_at_ms) FROM forecast_issues + WHERE issued_at_ms > ? OR (issued_at_ms = ? AND id > ?)`, cursor.IssuedAtMS, cursor.IssuedAtMS, cursor.ID).Scan(&issued) + if err != nil || !issued.Valid { + return 0, err + } + lag := now - issued.Int64 + if lag < 0 { + lag = 0 + } + return lag, nil +} diff --git a/go/internal/state/forecast_long_maintain.go b/go/internal/state/forecast_long_maintain.go new file mode 100644 index 000000000..34039b939 --- /dev/null +++ b/go/internal/state/forecast_long_maintain.go @@ -0,0 +1,660 @@ +package state + +import ( + "context" + "database/sql" + "errors" + "math" + "sort" + "time" +) + +// forecastLongAfterDeleteBatch counts bounded deletes. Tests assert the +// 64-row batches. +var forecastLongAfterDeleteBatch func(table string, n int) + +func (a *ForecastLongArchive) maintain(ctx context.Context, now time.Time) error { + if err := a.thin(ctx, now); err != nil { + return err + } + if err := a.pruneExpired(ctx, now); err != nil { + return err + } + if _, err := a.pruneToCap(ctx, now); err != nil { + return err + } + if err := a.collectGarbage(ctx); err != nil { + return err + } + return a.incrementalVacuum(ctx) +} + +// enforceCap runs on every copy between the daily passes. Under the cap and +// above the disk floor it costs one page count. +func (a *ForecastLongArchive) enforceCap(ctx context.Context, now time.Time) error { + pruned, err := a.pruneToCap(ctx, now) + if err != nil || !pruned { + return err + } + return a.incrementalVacuum(ctx) +} + +type forecastLongDay struct { + start int64 + end int64 + ids []string +} + +const forecastLongHourMS = int64(time.Hour / time.Millisecond) + +// thin keeps one run per complete UTC hour before the full-detail cutoff and +// model-state references on one run per complete local day. Hours and days +// count as complete when they end at or before the cutoff's whole hour, so a +// day in a half-hour zone waits for its last UTC hour. A day stays unmarked +// until it is complete, and a run copied late into a thinned hour loses to +// the survivor already there. +func (a *ForecastLongArchive) thin(ctx context.Context, now time.Time) error { + cut := now.Add(-a.fullDetail).UnixMilli() + cut -= cut % forecastLongHourMS + from := int64(math.MinInt64) + for { + if err := ctx.Err(); err != nil { + return err + } + origin, tz, ok, err := a.nextUnthinned(ctx, from, cut) + if err != nil || !ok { + return err + } + start, end, key := forecastLongLocalDay(origin, tz) + hoursEnd := (end + forecastLongHourMS - 1) / forecastLongHourMS * forecastLongHourMS + if hoursEnd > cut { + hoursEnd = cut + } + losers, err := a.hourLosers(ctx, start-start%forecastLongHourMS, hoursEnd) + if err != nil { + return err + } + if err := a.deleteRunIDs(ctx, losers); err != nil { + return err + } + if end <= cut { + if err := a.thinDay(ctx, start, end, key); err != nil { + return err + } + } + if err := a.collectGarbage(ctx); err != nil { + return err + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + from = end + } +} + +// The thinning queries read fa_runs by origin. nextUnthinned runs once per +// local day, so the partial index fa_runs_unthinned spares it a walk over +// every thinned run before the first one left to thin. +const ( + forecastLongNextUnthinned = `SELECT origin_ms, timezone FROM fa_runs + WHERE thinned=0 AND origin_ms>=? AND origin_ms=? AND origin_ms 1` + forecastLongDayRuns = `SELECT run_id, origin_ms, timezone, thinned, + EXISTS(SELECT 1 FROM fa_run_states s WHERE s.run_id=fa_runs.run_id) + FROM fa_runs WHERE origin_ms>=? AND origin_ms 0 && live-short < target { + target = live - short + } + pruned := false + for live > target { + if err := ctx.Err(); err != nil { + return pruned, err + } + day, ok, err := a.oldestDay(ctx, protect) + if err != nil { + return pruned, err + } + if !ok { + break + } + if err := a.deleteRunIDs(ctx, day.ids); err != nil { + return pruned, err + } + if err := a.deleteObservationsBetween(ctx, day.start, day.end, protect); err != nil { + return pruned, err + } + if err := a.collectGarbage(ctx); err != nil { + return pruned, err + } + pruned = true + if live, err = a.liveByteCount(ctx); err != nil { + return pruned, err + } + } + a.mu.Lock() + if live > a.maxBytes { + a.status.State = "over_cap" + } else if a.status.State == "over_cap" { + a.status.State = "" + } + a.mu.Unlock() + return pruned, nil +} + +func (a *ForecastLongArchive) deleteRunsIssuedBefore(ctx context.Context, cutoff, protect int64) error { + for { + day, ok, err := a.oldestDay(ctx, protect) + if err != nil || !ok { + return err + } + var ids []string + query := `SELECT run_id FROM fa_runs WHERE run_id IN (` + placeholders(len(day.ids)) + `) AND issued_at_ms=? AND origin_ms 0 { + n := forecastPruneBatch + if n > len(ids) { + n = len(ids) + } + chunk := ids[:n] + ids = ids[n:] + tx, err := a.db.BeginTx(ctx, nil) + if err != nil { + return err + } + for _, table := range []string{"fa_series", "fa_run_states", "fa_run_weather", "fa_runs"} { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE run_id IN (`+placeholders(len(chunk))+`)`, stringsToArgs(chunk)...); err != nil { + tx.Rollback() + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch("fa_runs", len(chunk)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } + return nil +} + +// deleteRunStates drops the model-state references of runs that keep their +// series. Garbage collection removes the states nothing references. +func (a *ForecastLongArchive) deleteRunStates(ctx context.Context, ids []string) error { + for len(ids) > 0 { + n := forecastPruneBatch + if n > len(ids) { + n = len(ids) + } + chunk := ids[:n] + ids = ids[n:] + if _, err := a.db.ExecContext(ctx, `DELETE FROM fa_run_states WHERE run_id IN (`+placeholders(len(chunk))+`)`, stringsToArgs(chunk)...); err != nil { + return err + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch("fa_run_states", len(chunk)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } + return nil +} + +func (a *ForecastLongArchive) deleteObservationsBefore(ctx context.Context, cutoff, protect int64) error { + limit := cutoff + if protect < limit { + limit = protect + } + return a.deleteObservationsBetween(ctx, 0, limit, protect) +} + +func (a *ForecastLongArchive) deleteObservationsBetween(ctx context.Context, start, end, protect int64) error { + if end > protect { + end = protect + } + if end <= start && start != 0 { + return nil + } + for { + query := `DELETE FROM fa_observations_15m WHERE (start_ms, end_ms, config_version) IN ( + SELECT start_ms, end_ms, config_version FROM fa_observations_15m WHERE end_ms=? LIMIT ?)` + res, err := a.db.ExecContext(ctx, query, end, start, forecastPruneBatch) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return nil + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch("fa_observations_15m", int(n)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +// The orphan deletes probe the reference tables through fa_run_weather_fetch +// and fa_run_states_ref, so each candidate row costs one index lookup rather +// than a scan of every reference. A fetch's weather rows stay while any run +// still links that fetch. +const ( + forecastLongWeatherGC = `DELETE FROM fa_weather WHERE (source, available_at_ms, start_ms) IN ( + SELECT source, available_at_ms, start_ms FROM fa_weather w + WHERE NOT EXISTS (SELECT 1 FROM fa_run_weather WHERE fa_run_weather.source=w.source AND fa_run_weather.available_at_ms=w.available_at_ms) + LIMIT ?)` + forecastLongStateGC = `DELETE FROM fa_model_states WHERE rowid IN ( + SELECT rowid FROM fa_model_states m + WHERE NOT EXISTS (SELECT 1 FROM fa_run_states WHERE fa_run_states.state_id=m.id) + LIMIT ?)` +) + +func (a *ForecastLongArchive) collectGarbage(ctx context.Context) error { + if err := a.deleteOrphans(ctx, forecastLongWeatherGC, "fa_weather"); err != nil { + return err + } + return a.deleteOrphans(ctx, forecastLongStateGC, "fa_model_states") +} + +func (a *ForecastLongArchive) deleteOrphans(ctx context.Context, query, table string) error { + for { + res, err := a.db.ExecContext(ctx, query, forecastPruneBatch) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return nil + } + if forecastLongAfterDeleteBatch != nil { + forecastLongAfterDeleteBatch(table, int(n)) + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +// incrementalVacuum moves free pages to the end of the file and cuts them +// off. In WAL mode the cut lands in the database file only when a checkpoint +// copies it, so the pass ends with one. +func (a *ForecastLongArchive) incrementalVacuum(ctx context.Context) error { + deadline := time.Now().Add(forecastLongVacuumBudget) + for { + if err := ctx.Err(); err != nil { + return err + } + var free, page int64 + if err := a.db.QueryRowContext(ctx, `PRAGMA freelist_count`).Scan(&free); err != nil { + return err + } + if err := a.db.QueryRowContext(ctx, `PRAGMA page_size`).Scan(&page); err != nil { + return err + } + if free*page < forecastLongFreeTarget || time.Now().After(deadline) { + _, err := a.db.ExecContext(ctx, `PRAGMA wal_checkpoint(PASSIVE)`) + return err + } + if _, err := a.db.ExecContext(ctx, `PRAGMA incremental_vacuum(256)`); err != nil { + return err + } + if err := pauseMaintenance(ctx); err != nil { + return err + } + } +} + +func (a *ForecastLongArchive) liveByteCount(ctx context.Context) (int64, error) { + inv, err := sqliteFileInventory(ctx, a.db, a.path) + if err != nil { + return 0, err + } + return inv.LiveBytes, nil +} + +func (a *ForecastLongArchive) recompute(ctx context.Context, src *Store, now time.Time, paused error) error { + var runs int64 + var oldest, newest sql.NullInt64 + if err := a.db.QueryRowContext(ctx, `SELECT COUNT(*), MIN(origin_ms), MAX(origin_ms) FROM fa_runs`).Scan(&runs, &oldest, &newest); err != nil { + return err + } + inv, err := sqliteFileInventory(ctx, a.db, a.path) + if err != nil { + return err + } + cursor, err := a.issueCursor(ctx) + if err != nil { + return err + } + lag, err := src.forecastCopyLag(ctx, cursor, now.UnixMilli()) + if err != nil { + return fromForecastSource(err) + } + p50, p95, gz, err := a.issueSizePercentiles(ctx) + if err != nil { + return err + } + oldestMS := int64(0) + if oldest.Valid { + oldestMS = oldest.Int64 + } + coverage7 := a.hourCoverage(ctx, now, 7*24*time.Hour, oldestMS) + coverage30 := a.hourCoverage(ctx, now, ForecastLongFullDetail, oldestMS) + coverageAll := a.hourCoverage(ctx, now, now.Sub(time.UnixMilli(oldestMS))+time.Hour, oldestMS) + truth := a.truthCoverage(ctx, now, oldestMS) + a.mu.Lock() + lastErr := a.status.LastError + lastCopy := a.status.LastCopyMS + state := "ok" + if a.rebuilt { + state = "rebuilt" + } + if a.status.State == "over_cap" { + state = "over_cap" + } + if paused != nil { + state = "paused_disk_full" + lastErr = "copying paused: the data volume is full or below the free-space floor" + } + a.status = ForecastLongArchiveStatus{ + State: state, LiveBytes: inv.LiveBytes, FileBytes: inv.FileBytes, WALBytes: inv.WALBytes, + CapBytes: a.maxBytes, OldestOriginMS: oldestMS, NewestOriginMS: newest.Int64, Runs: runs, + Coverage7d: coverage7, Coverage30d: coverage30, CoverageAll: coverageAll, + TruthCoverage30d: truth, CopyLagMS: lag, LastCopyMS: lastCopy, LastError: lastErr, + IssueBytesP50: p50, IssueBytesP95: p95, IssueGzipBytesP50: gz, + } + a.mu.Unlock() + a.publish() + return nil +} + +func (a *ForecastLongArchive) hourCoverage(ctx context.Context, now time.Time, window time.Duration, oldest int64) float64 { + if oldest <= 0 { + return 0 + } + start := now.Add(-window).UnixMilli() + if oldest > start { + start = oldest + } + start = time.UnixMilli(start).UTC().Truncate(time.Hour).UnixMilli() + end := now.UTC().Truncate(time.Hour).UnixMilli() + if end < start { + return 0 + } + span := (end-start)/int64(time.Hour/time.Millisecond) + 1 + if span <= 0 { + return 0 + } + var present int64 + hour := int64(time.Hour / time.Millisecond) + if err := a.db.QueryRowContext(ctx, `SELECT COUNT(DISTINCT origin_ms/3600000) FROM fa_runs + WHERE origin_ms>=? AND origin_ms span { + present = span + } + return float64(present) / float64(span) +} + +func (a *ForecastLongArchive) truthCoverage(ctx context.Context, now time.Time, oldest int64) float64 { + if oldest <= 0 { + return 0 + } + end := now.UTC().Truncate(15 * time.Minute).UnixMilli() + start := now.Add(-ForecastLongFullDetail).UnixMilli() + if oldest > start { + start = oldest + } + start = time.UnixMilli(start).UTC().Truncate(15 * time.Minute).UnixMilli() + if end <= start { + return 0 + } + span := (end - start) / forecastLongQuarterMS + if span <= 0 { + return 0 + } + var known int64 + if err := a.db.QueryRowContext(ctx, `SELECT COUNT(DISTINCT start_ms) FROM fa_observations_15m + WHERE load_known=1 AND start_ms>=? AND start_ms span { + known = span + } + return float64(known) / float64(span) +} + +func (a *ForecastLongArchive) issueSizePercentiles(ctx context.Context) (p50, p95, gzipP50 int64, err error) { + rows, err := a.db.QueryContext(ctx, `SELECT json_bytes, gzip_bytes FROM fa_runs WHERE json_bytes>0`) + if err != nil { + return 0, 0, 0, err + } + defer rows.Close() + var jsonBytes, gzipBytes []int64 + for rows.Next() { + var j, g int64 + if err := rows.Scan(&j, &g); err != nil { + return 0, 0, 0, err + } + jsonBytes = append(jsonBytes, j) + gzipBytes = append(gzipBytes, g) + } + if err := rows.Err(); err != nil { + return 0, 0, 0, err + } + sort.Slice(jsonBytes, func(i, j int) bool { return jsonBytes[i] < jsonBytes[j] }) + sort.Slice(gzipBytes, func(i, j int) bool { return gzipBytes[i] < gzipBytes[j] }) + return percentile(jsonBytes, 0.50), percentile(jsonBytes, 0.95), percentile(gzipBytes, 0.50), nil +} + +func percentile(sorted []int64, p float64) int64 { + if len(sorted) == 0 { + return 0 + } + i := int(math.Ceil(p*float64(len(sorted)))) - 1 + if i < 0 { + i = 0 + } + if i >= len(sorted) { + i = len(sorted) - 1 + } + return sorted[i] +} + +func stringsToArgs(ids []string) []any { + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + return args +} diff --git a/go/internal/state/snapshot_state_test.go b/go/internal/state/snapshot_state_test.go index 0dc428884..c10af5628 100644 --- a/go/internal/state/snapshot_state_test.go +++ b/go/internal/state/snapshot_state_test.go @@ -172,3 +172,27 @@ func TestMarkerSkipThenBackgroundVerifyHealsNextBoot(t *testing.T) { t.Errorf("precious row not restored: %q err=%v", v, err) } } + +func TestSnapshotOmitsForecastLongArchive(t *testing.T) { + dir := t.TempDir() + st, err := Open(filepath.Join(dir, "state.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := st.OpenForecastLongArchive(context.Background()); err != nil { + t.Fatal(err) + } + if err := st.SnapshotState(); err != nil { + t.Fatal(err) + } + snap, err := openRaw(filepath.Join(dir, "state.db.snapshot")) + if err != nil { + t.Fatal(err) + } + defer snap.Close() + var n int + if err := snap.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'fa_%'`).Scan(&n); err != nil || n != 0 { + t.Fatalf("quick snapshot contains %d forecast-archive tables, err=%v", n, err) + } +} diff --git a/go/internal/state/storage_inventory.go b/go/internal/state/storage_inventory.go index 54d57e365..da6ebe576 100644 --- a/go/internal/state/storage_inventory.go +++ b/go/internal/state/storage_inventory.go @@ -25,10 +25,13 @@ type SQLiteFileInventory struct { SHMBytes int64 `json:"shm_bytes"` } -// SQLiteInventory reports the two SQLite files that Core owns today. +// SQLiteInventory reports the SQLite files Core has open. ForecastArchive is +// set only after the long-term archive has been opened; the inventory does +// not open it. type SQLiteInventory struct { - State SQLiteFileInventory `json:"state"` - Cache SQLiteFileInventory `json:"cache"` + State SQLiteFileInventory `json:"state"` + Cache SQLiteFileInventory `json:"cache"` + ForecastArchive *SQLiteFileInventory `json:"forecast_archive,omitempty"` } // SQLiteInventory reads page metadata for state.db and cache.db. It does not @@ -50,7 +53,18 @@ func (s *Store) SQLiteInventory(ctx context.Context) (SQLiteInventory, error) { return SQLiteInventory{}, fmt.Errorf("cache inventory: %w", err) } } - return SQLiteInventory{State: stateDB, Cache: cacheDB}, nil + out := SQLiteInventory{State: stateDB, Cache: cacheDB} + s.forecastLongMu.Lock() + arch := s.forecastLong + s.forecastLongMu.Unlock() + if arch != nil { + archiveDB, err := sqliteFileInventory(ctx, arch.db, arch.path) + if err != nil { + return SQLiteInventory{}, fmt.Errorf("forecast archive inventory: %w", err) + } + out.ForecastArchive = &archiveDB + } + return out, nil } func sqliteFileInventory(ctx context.Context, db *sql.DB, path string) (SQLiteFileInventory, error) { diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 5429d2518..5e577bdd4 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -36,12 +36,14 @@ const ( LegacyReleaseMarker = 4 ) -// Store owns three SQLite databases: +// Store owns three SQLite databases opened at startup: // - history: samples, hourly summaries, energy ledger and dashboard history // - db: state.db configuration, devices and learned state // - cache: cache.db prices and forecasts; savings history needs past prices // -// See heal.go for the boot-time integrity gate that populates healEvents. +// forecast-archive.db is a fourth file, opened later by the forecast copier. +// It is not on the boot path. See heal.go for the boot-time integrity gate +// that populates healEvents. type Store struct { aggregateHistory atomic.Bool history *sql.DB @@ -98,6 +100,20 @@ type Store struct { deviceWriteMu sync.Mutex deviceCacheMu sync.RWMutex deviceCache map[string]Device + + // forecastLong is the long-term forecast archive. It stays nil until the + // copier opens it, so a slow or damaged analytics file cannot delay boot. + // forecastLongMu is held only to read or swap these fields, never for + // file work; forecastLongOpenMu keeps two opens apart. + forecastLongOpenMu sync.Mutex + forecastLongMu sync.Mutex + forecastLong *ForecastLongArchive + forecastLongClosed bool + // forecastLongDropped is a handle Sync closed after a corruption code. + // The next open moves the file aside without quick_check, once the last + // read on that handle has ended. + forecastLongDropped *sql.DB + forecastLongStatus *ForecastLongArchiveStatus } // Open initializes (or creates) the precious state.db at path plus the @@ -308,6 +324,7 @@ func (s *Store) Close() error { if s.history != nil { err = errors.Join(err, s.history.Close()) } + err = errors.Join(err, s.closeForecastLongArchive()) if s.cache != nil { err = errors.Join(err, s.cache.Close()) }