diff --git a/.gitignore b/.gitignore index 4c9e204..7717537 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /essence /result +*.db diff --git a/go.mod b/go.mod index 8b2175c..0b4c827 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,17 @@ module github.com/polen/essence -go 1.26.1 +go 1.25.0 + +require modernc.org/sqlite v1.48.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.70.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8411195 --- /dev/null +++ b/go.sum @@ -0,0 +1,51 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= +modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= +modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.48.0 h1:ElZyLop3Q2mHYk5IFPPXADejZrlHu7APbpB0sF78bq4= +modernc.org/sqlite v1.48.0/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/main.go b/main.go index 5e93387..5e09cbf 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "compress/gzip" + "database/sql" "embed" "encoding/json" "fmt" @@ -10,18 +11,21 @@ import ( "log" "net/http" "os" + "strconv" "strings" "sync" "time" + + _ "modernc.org/sqlite" ) //go:embed static/* var staticFiles embed.FS const ( - geojsonURL = "https://regieessencequebec.ca/stations.geojson.gz" - defaultPort = "8080" - cacheTTL = 5 * time.Minute + geojsonURL = "https://regieessencequebec.ca/stations.geojson.gz" + defaultPort = "8080" + pollInterval = 5 * time.Minute ) // GeoJSON structures matching the upstream format. @@ -57,35 +61,195 @@ type StationsResponse struct { Stations []Station `json:"stations"` } +// Snapshot holds the aggregated statistics for one fetch. +type Snapshot struct { + GeneratedAt string `json:"generatedAt"` + FetchedAt string `json:"fetchedAt"` + RegularAvg float64 `json:"regularAvg"` + RegularMin float64 `json:"regularMin"` + RegularMax float64 `json:"regularMax"` + SuperAvg float64 `json:"superAvg"` + DieselAvg float64 `json:"dieselAvg"` + StationCount int `json:"stationCount"` +} + +type StatsResponse struct { + Snapshots []Snapshot `json:"snapshots"` +} + // In-memory cache var ( - cacheMu sync.RWMutex - cachedResp *StationsResponse - cacheExpiry time.Time + cacheMu sync.RWMutex + cachedResp *StationsResponse ) +var db *sql.DB + func main() { port := os.Getenv("PORT") if port == "" { port = defaultPort } + dbPath := os.Getenv("ESSENCE_DB") + if dbPath == "" { + dbPath = "./essence.db" + } + + var err error + db, err = initDB(dbPath) + if err != nil { + log.Fatalf("db init: %v", err) + } + defer db.Close() + + // Initial fetch, then background poll every 5 minutes. + go poller() + staticSub, err := fs.Sub(staticFiles, "static") if err != nil { log.Fatalf("static files: %v", err) } http.HandleFunc("/api/stations", handleStations) + http.HandleFunc("/api/stats", handleStats) http.Handle("/", http.FileServer(http.FS(staticSub))) log.Printf("Listening on http://localhost:%s", port) log.Fatal(http.ListenAndServe(":"+port, nil)) } -func handleStations(w http.ResponseWriter, r *http.Request) { - resp, err := getStations() +// initDB opens (or creates) the SQLite database and ensures the schema exists. +func initDB(path string) (*sql.DB, error) { + d, err := sql.Open("sqlite", path) if err != nil { - http.Error(w, fmt.Sprintf("error: %v", err), http.StatusInternalServerError) + return nil, err + } + + _, err = d.Exec(` + CREATE TABLE IF NOT EXISTS snapshots ( + generated_at TEXT PRIMARY KEY, + fetched_at TEXT NOT NULL, + regular_avg REAL, + regular_min REAL, + regular_max REAL, + super_avg REAL, + diesel_avg REAL, + station_count INTEGER + ) + `) + if err != nil { + return nil, fmt.Errorf("create table: %w", err) + } + return d, nil +} + +// poller fetches immediately then every pollInterval. +func poller() { + fetchAndStore() + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + for range ticker.C { + fetchAndStore() + } +} + +// fetchAndStore fetches upstream data, updates the in-memory cache, and +// persists a snapshot to SQLite if the data has a new generated_at value. +func fetchAndStore() { + resp, err := fetchAndParse() + if err != nil { + log.Printf("fetch error: %v", err) + return + } + + cacheMu.Lock() + cachedResp = resp + cacheMu.Unlock() + + if resp.LastUpdated == "" { + return + } + + // Compute aggregates. + snap := computeSnapshot(resp) + + // Only insert if this generated_at is new. + _, err = db.Exec(` + INSERT OR IGNORE INTO snapshots + (generated_at, fetched_at, regular_avg, regular_min, regular_max, super_avg, diesel_avg, station_count) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + snap.GeneratedAt, + snap.FetchedAt, + snap.RegularAvg, + snap.RegularMin, + snap.RegularMax, + snap.SuperAvg, + snap.DieselAvg, + snap.StationCount, + ) + if err != nil { + log.Printf("db insert error: %v", err) + } +} + +// computeSnapshot derives aggregate statistics from a StationsResponse. +func computeSnapshot(resp *StationsResponse) Snapshot { + now := time.Now().UTC().Format(time.RFC3339) + snap := Snapshot{ + GeneratedAt: resp.LastUpdated, + FetchedAt: now, + } + + var regularSum, superSum, dieselSum float64 + var regularCount, superCount, dieselCount int + snap.RegularMin = 1<<53 - 1 + snap.RegularMax = 0 + + for _, s := range resp.Stations { + if s.Regular > 0 { + regularSum += s.Regular + regularCount++ + if s.Regular < snap.RegularMin { + snap.RegularMin = s.Regular + } + if s.Regular > snap.RegularMax { + snap.RegularMax = s.Regular + } + } + if s.Super > 0 { + superSum += s.Super + superCount++ + } + if s.Diesel > 0 { + dieselSum += s.Diesel + dieselCount++ + } + } + + snap.StationCount = len(resp.Stations) + if regularCount > 0 { + snap.RegularAvg = regularSum / float64(regularCount) + } else { + snap.RegularMin = 0 + } + if superCount > 0 { + snap.SuperAvg = superSum / float64(superCount) + } + if dieselCount > 0 { + snap.DieselAvg = dieselSum / float64(dieselCount) + } + + return snap +} + +func handleStations(w http.ResponseWriter, r *http.Request) { + cacheMu.RLock() + resp := cachedResp + cacheMu.RUnlock() + + if resp == nil { + http.Error(w, "data not yet available, please retry shortly", http.StatusServiceUnavailable) return } @@ -94,25 +258,54 @@ func handleStations(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(resp) } -func getStations() (*StationsResponse, error) { - cacheMu.RLock() - if cachedResp != nil && time.Now().Before(cacheExpiry) { - defer cacheMu.RUnlock() - return cachedResp, nil +func handleStats(w http.ResponseWriter, r *http.Request) { + daysStr := r.URL.Query().Get("days") + days := 7 + if daysStr != "" { + if d, err := strconv.Atoi(daysStr); err == nil && d > 0 { + days = d + } } - cacheMu.RUnlock() - resp, err := fetchAndParse() + // days=0 means all data + var rows *sql.Rows + var err error + if daysStr == "0" { + rows, err = db.Query(` + SELECT generated_at, fetched_at, regular_avg, regular_min, regular_max, + super_avg, diesel_avg, station_count + FROM snapshots + ORDER BY generated_at ASC + `) + } else { + since := time.Now().UTC().Add(-time.Duration(days) * 24 * time.Hour).Format(time.RFC3339) + rows, err = db.Query(` + SELECT generated_at, fetched_at, regular_avg, regular_min, regular_max, + super_avg, diesel_avg, station_count + FROM snapshots + WHERE generated_at >= ? + ORDER BY generated_at ASC + `, since) + } if err != nil { - return nil, err + http.Error(w, fmt.Sprintf("db query: %v", err), http.StatusInternalServerError) + return + } + defer rows.Close() + + snapshots := []Snapshot{} + for rows.Next() { + var s Snapshot + if err := rows.Scan(&s.GeneratedAt, &s.FetchedAt, &s.RegularAvg, &s.RegularMin, &s.RegularMax, + &s.SuperAvg, &s.DieselAvg, &s.StationCount); err != nil { + http.Error(w, fmt.Sprintf("db scan: %v", err), http.StatusInternalServerError) + return + } + snapshots = append(snapshots, s) } - cacheMu.Lock() - cachedResp = resp - cacheExpiry = time.Now().Add(cacheTTL) - cacheMu.Unlock() - - return resp, nil + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(StatsResponse{Snapshots: snapshots}) } func fetchAndParse() (*StationsResponse, error) { diff --git a/nixos-module.nix b/nixos-module.nix index 0bdb3c4..a2367c5 100644 --- a/nixos-module.nix +++ b/nixos-module.nix @@ -14,6 +14,12 @@ in description = "Port the Essence web server listens on."; }; + dataDir = lib.mkOption { + type = lib.types.str; + default = "/var/lib/essence"; + description = "Directory where the SQLite database is stored."; + }; + openFirewall = lib.mkOption { type = lib.types.bool; default = false; @@ -36,7 +42,8 @@ in wants = [ "network-online.target" ]; environment = { - PORT = toString cfg.port; + PORT = toString cfg.port; + ESSENCE_DB = "${cfg.dataDir}/essence.db"; }; serviceConfig = { @@ -45,6 +52,8 @@ in RestartSec = 5; DynamicUser = true; + StateDirectory = "essence"; + StateDirectoryMode = "0750"; # Hardening NoNewPrivileges = true; diff --git a/static/index.html b/static/index.html index 979b162..8133530 100644 --- a/static/index.html +++ b/static/index.html @@ -3,144 +3,455 @@ - Essence Québec - Carte des prix + Essence Québec + -
Chargement des données...
-
-
- - + +
+
+ Chargement des données...
-
- - -
- - - - -
-
-
+
-
+ +
+ Essence QC + +
-
-

Prix régulier (¢/L)

-
-
- - - - -
-
-
+ +
+ + +
+
+ +
+
+ + + +
+ + + +
+ - + - +
+
+
+ +
+ +
+

Prix régulier (¢/L)

+
+
+ - + - +
+
+
+
+ + +
+

Statistiques des prix

+ + +
+
+

Régulier — Moyenne

+

—

+

¢/L

+
+
+

Régulier — Min

+

—

+

¢/L

+
+
+

Régulier — Max

+

—

+

¢/L

+
+
+

Super — Moyenne

+

—

+

¢/L

+
+
+

Diesel — Moyenne

+

—

+

¢/L

+
+
+

Stations

+

—

+

dernière mise à jour

+
+
+ + +
+

Évolution des prix

+

Prix moyen par carburant (¢/L)

+
+ + + +
+ + +
+ + +
+

Historique récent

+

10 dernières mises à jour

+
+ + + + + + + + + + + + + + + +
DateRégulier moy.Régulier minRégulier maxSuper moy.Diesel moy.Stations
Chargement...
+
+
+
+ +
+
- + +