diff --git a/.gitignore b/.gitignore index a78349e..9c1b9fa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env config.json *.xmltv +!testdata/xmlguide_golden.xmltv tmdb_cache.json tvlogo_cache.json guide_cache.json diff --git a/CLAUDE.md b/CLAUDE.md index 9dc0268..69f072a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,13 +27,13 @@ Run `go test ./...` for the setup, configuration, and provider-client tests. ## Architecture -The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. Runtime orchestration lives in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`. +The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. The grid download loop and guide assembly live in `scrape/` (`scrape.Fetch`), which is importable by other programs and does no enrichment or env reads (the default `web.Client` it uses does keep `grid_cache/` on disk). Enrichment, persistence, and runtime orchestration live in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`. **Data flow:** 1. `/setup` uses `web.ProviderClient` to discover Gracenote lineups by country and postal code. `appconfig.Store` persists the selected non-secret source in `config.json`; complete legacy `GN_*` settings can bootstrap it. -2. `web.Client.GetDataByTime` fetches 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Raw JSON types live in `web/web.go`. -3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`. +2. `scrape.Fetch` drives `web.Client.GetDataByTimeContext` over 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Failed slots are skipped; if every slot fails the scrape returns `scrape.ErrNoData` and the previous guide stays live. Raw JSON types live in `web/web.go`. +3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). `TVGuide.Channels` is deduplicated by station for XMLTV; `TVGuide.Lineup` retains every provider position (number plus station) and is served by `/api/lineup.json`. `TVGuide.Source` records the lineup the guide came from. The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`. 4. `tmdb.Client.Lookup` enriches programs (poster images, ratings, overview, year) via TMDB search API. Deduplicates by `(title, isMovie)` before hitting the API. Rate-limited to ~4 req/sec. 5. `tvlogo.Client.Resolve` replaces Gracenote channel icons with verified PNGs from `github.com/tv-logo/tv-logos`. Generates candidate URL slugs from callsign/affiliate name and HEAD-checks each (rate-limited to ~5 req/sec). 6. `fixDeadImageURLs` rewrites `zap2it.tmsimg.com` → `tmsimg.com` for broken Gracenote image URLs. @@ -43,7 +43,7 @@ The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV l | Cache | File | TTL | |---|---|---| -| Guide (in-memory + disk) | `guide_cache.json` | 4h (startup skip) / 24h (rescrape) | +| Guide (in-memory + disk) | `guide_cache.json` | 4h (startup skip) / 24h (rescrape); carries a schema version, older caches are rebuilt once | | TMDB lookups | `tmdb_cache.json` | 7 days | | TV logo HEAD checks | `tvlogo_cache.json` | persisted, no expiry | | Image proxy | `image_cache/` dir | indefinite (per-URL SHA256 key) | diff --git a/README.md b/README.md index ae97626..cec7897 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ A saved `CONFIG_PATH` selection takes precedence over legacy `GN_*` settings. De | `POST /api/setup/provider` | Save the selected provider and queue a fresh guide | | `GET /xmlguide.xmltv` | XMLTV guide data (point your DVR here) | | `GET /api/guide.json` | Guide data as JSON | +| `GET /api/lineup.json` | Every channel position in the active provider lineup as JSON (see below) | | `GET /` | The Grid — built-in web UI | | `GET /img?url=...` | Image proxy with local cache | | `GET /api/livetv/config` | Returns `{"enabled":true/false}` — whether Jellyfin live TV is configured | @@ -125,17 +126,80 @@ A saved `CONFIG_PATH` selection takes precedence over legacy `GN_*` settings. De | `GET /api/livetv/tune?id=` | Starts a live stream for the given channel and returns an HLS URL | | `POST /api/livetv/stop` | Forwards a playback-stop notification to Jellyfin to end a live stream | +### Lineup JSON + +`/api/lineup.json` describes the provider lineup itself rather than the schedule: one entry per channel number, never collapsed, so a station carried at two numbers appears twice. It returns `503` with a `Retry-After` header until the first guide has been built. + +```json +{ + "generated": "2026-09-13T04:10:22Z", + "source": { + "providerName": "Local Over the Air Broadcast", + "providerType": "OTA", + "location": "", + "lineupId": "USA-lineupId-DEFAULT", + "headendId": "lineupId", + "postalCode": "13490", + "country": "USA", + "device": "-", + "language": "en-us" + }, + "positions": [ + { + "number": "2.1", + "stationId": "53158", + "placementId": "531580", + "callSign": "WKTVDT", + "affiliate": "NATIONAL BROADCASTING COMPANY", + "affiliateCallSign": "", + "filters": ["sports", "news"], + "logoUrl": "https://raw.githubusercontent.com/tv-logo/tv-logos/main/countries/united-states/nbc-us.png" + } + ] +} +``` + +`filters` are Gracenote's own station tags with the `filter-` prefix removed. `placementId` is Gracenote's row identifier and is not stable across scrapes; use `number` plus `stationId` to identify a position. `providerName`, `providerType`, and `location` are filled from the saved setup when it matches the lineup the guide was built from. + ## The Grid The server includes a built-in retro-styled TV guide web UI at the root URL. If no provider is configured, `/` redirects to `/setup`. Once configured, the guide auto-scrolls through your channel lineup and shows program details, posters, and metadata. Handy for a quick glance at what's on without opening your DVR app. ![The Grid](https://gist.githubusercontent.com/daniel-widrick/2c52c4d023ffe75d163b4eff58263c77/raw/demo.gif) +## Using as a Library + +The grid download and conversion live in the `scrape` package, so another Go program can fetch a lineup's listings without running the server: + +```go +import ( + "context" + "fmt" + + "github.com/daniel-widrick/GraceNoteScraper/scrape" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +func main() { + prefs := web.Preferences{Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us"} + g, err := scrape.Fetch(context.Background(), prefs, scrape.Options{Days: 1}) + if err != nil { + panic(err) + } + fmt.Printf("%d stations, %d lineup positions, %d programs\n", len(g.Channels), len(g.Lineup), len(g.Programs)) +} +``` + +`Fetch` returns a `guide.TVGuide`: `Channels` deduplicated by station, `Lineup` with every position, `Programs`, and `Source`. It pauses five seconds between grid requests by default and skips slots that fail; it returns `scrape.ErrNoData` only when every slot failed. Logo and TMDB enrichment are not part of the package. Lineup discovery by postal code is available through `web.NewProviderClient().FindProviders`. + +The default grid client is `web.NewClient`, which retries failed requests and keeps a raw-response cache in a `grid_cache/` directory under the working directory. Pass your own `Options.Fetcher` to change that. + ## Project Structure ``` appconfig/ Persisted non-secret provider configuration -main.go Entry point, HTTP server, scraper, image proxy +main.go Entry point, HTTP server, enrichment, image proxy +scrape/ Grid download loop and guide assembly (importable) guide/ GraceNote data types and XMLTV conversion web/ HTTP client for GraceNote API tmdb/ TMDB client and cache diff --git a/guide/guide.go b/guide/guide.go index 821f5f7..b502c4e 100644 --- a/guide/guide.go +++ b/guide/guide.go @@ -2,25 +2,124 @@ package guide import ( "fmt" + "sort" "strconv" "strings" + "time" "unicode" "github.com/daniel-widrick/GraceNoteScraper/web" ) type TVGuide struct { + // Channels is the XMLTV view: one entry per Gracenote station. Channels []Channel Programs []Program + // Lineup retains every provider position, so a station carried at two + // channel numbers appears twice. It is never collapsed. + Lineup []LineupPosition + // Source records which provider lineup produced this guide. + Source Source +} + +// Source identifies the Gracenote lineup a guide was built from. +type Source struct { + Country string + PostalCode string + HeadendID string + LineupID string + Device string + Language string + GeneratedAt time.Time +} + +// SourceFromPreferences copies the request preferences into a Source. +func SourceFromPreferences(p web.Preferences, generatedAt time.Time) Source { + return Source{ + Country: p.Country, + PostalCode: p.ZipCode, + HeadendID: p.Headend, + LineupID: p.LineupId, + Device: p.Device, + Language: p.Language, + GeneratedAt: generatedAt, + } +} + +// LineupPosition is one channel number in a provider lineup. +type LineupPosition struct { + ChannelNo string + StationID string + PlacementID string // Gracenote row id; carried for fidelity, not a stable key + CallSign string + Affiliate string + AffiliateCallSign string + Filters []string + LogoURL string +} + +// Key identifies a position across grid slices: the same station at the same +// number is one position no matter how many responses it appears in. +func (p LineupPosition) Key() string { + return p.ChannelNo + "|" + p.StationID +} + +// ConvertLineupPosition converts a JSON channel row to a lineup position. +func ConvertLineupPosition(ch web.JSONChannel) LineupPosition { + return LineupPosition{ + ChannelNo: ch.ChannelNo, + StationID: ch.ChannelID, + PlacementID: ch.ID, + CallSign: ch.CallSign, + Affiliate: ch.AffiliateName, + AffiliateCallSign: normalizeNull(ch.AffiliateCallSign), + Filters: stripFilterPrefixes(ch.StationFilters), + LogoURL: gracenoteIconURL(ch.Thumbnail), + } +} + +// ChannelNumberLess orders channel numbers numerically where both parse +// (so "2.1" < "10" < "100"), places numeric numbers before non-numeric ones, +// and falls back to string order. Equal numbers compare as strings so the +// ordering is strict. +func ChannelNumberLess(a, b string) bool { + af, errA := strconv.ParseFloat(strings.TrimSpace(a), 64) + bf, errB := strconv.ParseFloat(strings.TrimSpace(b), 64) + switch { + case errA == nil && errB == nil: + if af != bf { + return af < bf + } + return a < b + case errA == nil: + return true + case errB == nil: + return false + default: + return a < b + } +} + +// SortLineup orders positions by channel number, then station ID, in place. +func SortLineup(positions []LineupPosition) { + sort.SliceStable(positions, func(i, j int) bool { + if positions[i].ChannelNo != positions[j].ChannelNo { + return ChannelNumberLess(positions[i].ChannelNo, positions[j].ChannelNo) + } + return positions[i].StationID < positions[j].StationID + }) } type Channel struct { - ID string - DisplayNames []DisplayName - IconURL string - CallSign string // internal, not in template - Affiliate string // internal, not in template - ChannelNo string // internal, not in template + ID string + DisplayNames []DisplayName + IconURL string + CallSign string // internal, not in template + Affiliate string // internal, not in template + ChannelNo string // internal, not in template + PlacementID string // internal, not in template; Gracenote row id, not a stable key + AffiliateCallSign string // internal, not in template + Filters []string // internal, not in template; Gracenote station filters, prefix stripped } type DisplayName struct { @@ -45,6 +144,10 @@ type Program struct { Country string EpisodeNumbers []EpisodeNumber Categories []Category + Filters []string // internal, not in template; raw Gracenote event filters, prefix stripped + TMSID string // internal, not in template + ReleaseYear string // internal, not in template + Generic bool // internal, not in template New bool Premiere bool PreviouslyShown bool @@ -95,21 +198,26 @@ func formatXMLTVTime(iso string) string { return s } +// gracenoteIconURL builds an absolute icon URL from a Gracenote thumbnail +// path: strip leading slashes, strip query params, prepend http:// +func gracenoteIconURL(thumbnail string) string { + if thumbnail == "" { + return "" + } + raw := thumbnail + if idx := strings.Index(raw, "?"); idx >= 0 { + raw = raw[:idx] + } + raw = strings.TrimLeft(raw, "/") + if raw == "" { + return "" + } + return "http://" + raw +} + // converts a JSON channel to a template Channel struct. func ConvertChannel(ch web.JSONChannel) Channel { - // Build icon URL: strip leading slashes, strip query params, prepend http:// - iconURL := "" - if ch.Thumbnail != "" { - raw := ch.Thumbnail - // Strip query string - if idx := strings.Index(raw, "?"); idx >= 0 { - raw = raw[:idx] - } - raw = strings.TrimLeft(raw, "/") - if raw != "" { - iconURL = "http://" + raw - } - } + iconURL := gracenoteIconURL(ch.Thumbnail) return Channel{ ID: ch.ChannelID, @@ -119,13 +227,37 @@ func ConvertChannel(ch web.JSONChannel) Channel { {Name: xmlEscape(ch.CallSign)}, {Name: xmlEscape(titleCase(ch.AffiliateName))}, }, - IconURL: iconURL, - CallSign: ch.CallSign, - Affiliate: ch.AffiliateName, - ChannelNo: ch.ChannelNo, + IconURL: iconURL, + CallSign: ch.CallSign, + Affiliate: ch.AffiliateName, + ChannelNo: ch.ChannelNo, + PlacementID: ch.ID, + AffiliateCallSign: normalizeNull(ch.AffiliateCallSign), + Filters: stripFilterPrefixes(ch.StationFilters), } } +// normalizeNull maps Gracenote's literal "null" string to an empty value. +func normalizeNull(s string) string { + if strings.EqualFold(strings.TrimSpace(s), "null") { + return "" + } + return s +} + +// stripFilterPrefixes turns Gracenote filter tags such as "filter-sports" into +// "sports". A nil input stays nil so callers can distinguish absent from empty. +func stripFilterPrefixes(filters []string) []string { + if filters == nil { + return nil + } + out := make([]string, 0, len(filters)) + for _, f := range filters { + out = append(out, strings.TrimPrefix(f, "filter-")) + } + return out +} + // converts a JSON event to a template Program struct. func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { season := 0 @@ -163,10 +295,13 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { // URL programURL := "https://tvlistings.gracenote.com//overview.html?programSeriesId=" + ev.SeriesID + "&tmsId=" + ev.Program.ID + // Raw Gracenote filters, kept separately so consumers can tell them apart + // from the Series and Finale labels added below. + filters := stripFilterPrefixes(ev.Filter) + // Categories from filter array (strip "filter-" prefix) var categories []Category - for _, f := range ev.Filter { - name := strings.TrimPrefix(f, "filter-") + for _, name := range filters { categories = append(categories, Category{Name: name, Lang: lang}) } @@ -266,6 +401,10 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { Country: country, EpisodeNumbers: episodeNumbers, Categories: categories, + Filters: filters, + TMSID: ev.Program.TmsID, + ReleaseYear: string(ev.Program.ReleaseYear), + Generic: bool(ev.Program.IsGeneric), New: isNew, Premiere: isPremiere, PreviouslyShown: !isNew, diff --git a/guide/guide_test.go b/guide/guide_test.go new file mode 100644 index 0000000..4274f9d --- /dev/null +++ b/guide/guide_test.go @@ -0,0 +1,197 @@ +package guide + +import ( + "reflect" + "testing" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +func sampleChannel() web.JSONChannel { + return web.JSONChannel{ + ChannelID: "53158", + ID: "531580", + ChannelNo: "2.1", + CallSign: "WKTVDT", + AffiliateName: "NATIONAL BROADCASTING COMPANY", + AffiliateCallSign: "null", + StationFilters: []string{"filter-sports", "filter-news"}, + Thumbnail: "//images.example.invalid/station/53158.png?w=55", + } +} + +func TestConvertChannelCarriesRawFields(t *testing.T) { + ch := ConvertChannel(sampleChannel()) + + if ch.ID != "53158" || ch.ChannelNo != "2.1" || ch.CallSign != "WKTVDT" { + t.Fatalf("basic fields wrong: %+v", ch) + } + if ch.PlacementID != "531580" { + t.Errorf("PlacementID = %q", ch.PlacementID) + } + if ch.AffiliateCallSign != "" { + t.Errorf(`"null" affiliate callsign should normalize to empty, got %q`, ch.AffiliateCallSign) + } + if !reflect.DeepEqual(ch.Filters, []string{"sports", "news"}) { + t.Errorf("Filters = %v", ch.Filters) + } + if ch.IconURL != "http://images.example.invalid/station/53158.png" { + t.Errorf("IconURL = %q", ch.IconURL) + } + // Existing XMLTV-facing fields must be untouched by the additions. + wantNames := []DisplayName{{"2.1 WKTVDT"}, {"2.1"}, {"WKTVDT"}, {"NATIONAL BROADCASTING COMPANY"}} + if !reflect.DeepEqual(ch.DisplayNames, wantNames) { + t.Errorf("DisplayNames = %v", ch.DisplayNames) + } +} + +func TestConvertChannelKeepsRealAffiliateCallSign(t *testing.T) { + in := sampleChannel() + in.AffiliateCallSign = "NBC" + in.StationFilters = nil + ch := ConvertChannel(in) + if ch.AffiliateCallSign != "NBC" { + t.Errorf("AffiliateCallSign = %q", ch.AffiliateCallSign) + } + if ch.Filters != nil { + t.Errorf("nil station filters should stay nil, got %v", ch.Filters) + } +} + +func sampleEvent() web.JSONEvent { + season, episode, title := "3", "7", "The One" + return web.JSONEvent{ + StartTime: "2026-07-25T07:00:00Z", + EndTime: "2026-07-25T08:00:00Z", + Duration: "60", + SeriesID: "SH06270099", + Flag: []string{"New", "Finale"}, + Filter: []string{"filter-sports"}, + Program: web.JSONProgram{ + ID: "EP062700990343", + TmsID: "EP062700990343", + Title: "Morning Business", + EpisodeTitle: &title, + Season: &season, + Episode: &episode, + ReleaseYear: "2019", + IsGeneric: true, + }, + } +} + +func TestConvertEventSeparatesRawFiltersFromCategories(t *testing.T) { + p := ConvertEvent(sampleEvent(), "53158", "en-us", "USA") + + if !reflect.DeepEqual(p.Filters, []string{"sports"}) { + t.Errorf("Filters = %v", p.Filters) + } + // Categories keep the historical shape: raw filters, then Series (from an + // episode number), then Finale (from the flag). + want := []Category{{"sports", "en-us"}, {"Series", "en-us"}, {"Finale", "en-us"}} + if !reflect.DeepEqual(p.Categories, want) { + t.Errorf("Categories = %v, want %v", p.Categories, want) + } + if p.TMSID != "EP062700990343" || p.ReleaseYear != "2019" || !p.Generic { + t.Errorf("program metadata = tms %q year %q generic %v", p.TMSID, p.ReleaseYear, p.Generic) + } + if !p.New || p.PreviouslyShown { + t.Errorf("flag handling changed: new=%v previouslyShown=%v", p.New, p.PreviouslyShown) + } +} + +func TestConvertEventWithoutFilters(t *testing.T) { + ev := sampleEvent() + ev.Filter = nil + ev.Flag = nil + ev.Program.Season, ev.Program.Episode = nil, nil + p := ConvertEvent(ev, "53158", "en", "USA") + if p.Filters != nil { + t.Errorf("Filters should be nil, got %v", p.Filters) + } + if len(p.Categories) != 0 { + t.Errorf("Categories should be empty, got %v", p.Categories) + } +} + +func TestConvertLineupPosition(t *testing.T) { + in := sampleChannel() + p := ConvertLineupPosition(in) + want := LineupPosition{ + ChannelNo: "2.1", + StationID: "53158", + PlacementID: "531580", + CallSign: "WKTVDT", + Affiliate: "NATIONAL BROADCASTING COMPANY", + AffiliateCallSign: "", + Filters: []string{"sports", "news"}, + LogoURL: "http://images.example.invalid/station/53158.png", + } + if !reflect.DeepEqual(p, want) { + t.Fatalf("got %+v\nwant %+v", p, want) + } + if p.Key() != "2.1|53158" { + t.Errorf("Key = %q", p.Key()) + } + other := in + other.ChannelNo = "1002" + other.ID = "5315899" + if ConvertLineupPosition(other).Key() == p.Key() { + t.Error("same station at a different number must be a distinct position") + } +} + +func TestChannelNumberLess(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.1", "10", true}, + {"10", "2.1", false}, + {"10", "100", true}, + {"9", "10", true}, + {"2.1", "2.10", true}, // equal numerically, string order breaks the tie + {"2.10", "2.1", false}, + {"5", "5", false}, + {"12", "ABC", true}, // numeric before non-numeric + {"ABC", "12", false}, + {"ABC", "ABD", true}, + } + for _, c := range cases { + if got := ChannelNumberLess(c.a, c.b); got != c.want { + t.Errorf("ChannelNumberLess(%q, %q) = %v, want %v", c.a, c.b, got, c.want) + } + } +} + +func TestSortLineup(t *testing.T) { + positions := []LineupPosition{ + {ChannelNo: "100", StationID: "b"}, + {ChannelNo: "ZZ", StationID: "z"}, + {ChannelNo: "2.1", StationID: "a"}, + {ChannelNo: "10", StationID: "c"}, + {ChannelNo: "10", StationID: "a"}, + {ChannelNo: "9", StationID: "d"}, + } + SortLineup(positions) + var got []string + for _, p := range positions { + got = append(got, p.ChannelNo+"/"+p.StationID) + } + want := []string{"2.1/a", "9/d", "10/a", "10/c", "100/b", "ZZ/z"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("order = %v, want %v", got, want) + } +} + +func TestSourceFromPreferences(t *testing.T) { + at := time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC) + src := SourceFromPreferences(web.Preferences{ + Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", + }, at) + want := Source{Country: "USA", PostalCode: "13490", HeadendID: "lineupId", LineupID: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", GeneratedAt: at} + if src != want { + t.Fatalf("got %+v want %+v", src, want) + } +} diff --git a/lineup_handler_test.go b/lineup_handler_test.go new file mode 100644 index 0000000..5e4b17c --- /dev/null +++ b/lineup_handler_test.go @@ -0,0 +1,195 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "sort" + "testing" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/appconfig" + "github.com/daniel-widrick/GraceNoteScraper/guide" +) + +func lineupTestConfig() appconfig.Config { + return appconfig.Config{ + Version: appconfig.CurrentVersion, + Gracenote: appconfig.GracenoteConfig{ + Country: "USA", PostalCode: "13490", Language: "en-us", ProviderType: "OTA", Device: "-", + LineupID: "USA-lineupId-DEFAULT", ProviderName: "Local Over the Air Broadcast", Location: "Utica", HeadendID: "lineupId", + }, + } +} + +func lineupTestGuide() *guide.TVGuide { + return &guide.TVGuide{ + Lineup: []guide.LineupPosition{ + {ChannelNo: "10", StationID: "s10", PlacementID: "s100", CallSign: "TEN", Affiliate: "Ten Net", Filters: []string{"news"}, LogoURL: "http://logo/ten.png"}, + {ChannelNo: "2.1", StationID: "s2", PlacementID: "s20", CallSign: "TWO", Affiliate: "Two Net", AffiliateCallSign: "TW"}, + {ChannelNo: "1002", StationID: "s2", PlacementID: "s299", CallSign: "TWO", Affiliate: "Two Net", AffiliateCallSign: "TW"}, + }, + Source: guide.Source{ + Country: "USA", PostalCode: "13490", HeadendID: "lineupId", LineupID: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", + GeneratedAt: time.Date(2026, 9, 13, 4, 10, 22, 0, time.UTC), + }, + } +} + +func newLineupServer(t *testing.T, g *guide.TVGuide, save bool) http.Handler { + t.Helper() + store, err := appconfig.LoadStore(filepath.Join(t.TempDir(), "config.json")) + if err != nil { + t.Fatalf("LoadStore: %v", err) + } + if save { + if err := store.Save(lineupTestConfig()); err != nil { + t.Fatalf("Save: %v", err) + } + } + state := &GuideState{} + state.Update(g) + return handleLineupJSON(state, store) +} + +func TestLineupJSONUnavailableBeforeFirstGuide(t *testing.T) { + rec := httptest.NewRecorder() + newLineupServer(t, nil, true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d", rec.Code) + } + if rec.Header().Get("Retry-After") != "30" { + t.Fatalf("Retry-After = %q", rec.Header().Get("Retry-After")) + } +} + +func TestLineupJSONResponse(t *testing.T) { + rec := httptest.NewRecorder() + newLineupServer(t, lineupTestGuide(), true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q", ct) + } + if rec.Header().Get("Access-Control-Allow-Origin") != "*" { + t.Error("CORS header missing") + } + + var got APILineup + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Generated != "2026-09-13T04:10:22Z" { + t.Errorf("generated = %q", got.Generated) + } + wantSource := APILineupSource{ + ProviderName: "Local Over the Air Broadcast", ProviderType: "OTA", Location: "Utica", + LineupID: "USA-lineupId-DEFAULT", HeadendID: "lineupId", PostalCode: "13490", Country: "USA", Device: "-", Language: "en-us", + } + if got.Source != wantSource { + t.Errorf("source = %+v\nwant %+v", got.Source, wantSource) + } + + var numbers []string + for _, p := range got.Positions { + numbers = append(numbers, p.Number) + } + if !reflect.DeepEqual(numbers, []string{"2.1", "10", "1002"}) { + t.Errorf("positions not sorted by number: %v", numbers) + } + ten := got.Positions[1] + want := APILineupPosition{Number: "10", StationID: "s10", PlacementID: "s100", CallSign: "TEN", Affiliate: "Ten Net", Filters: []string{"news"}, LogoURL: "http://logo/ten.png"} + if !reflect.DeepEqual(ten, want) { + t.Errorf("position = %+v\nwant %+v", ten, want) + } + if got.Positions[0].StationID != "s2" || got.Positions[2].StationID != "s2" { + t.Error("the same station at two numbers must appear twice") + } +} + +func TestLineupJSONKeySetIsStable(t *testing.T) { + rec := httptest.NewRecorder() + newLineupServer(t, lineupTestGuide(), true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + + var raw map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil { + t.Fatal(err) + } + if keys := sortedKeys(raw); !reflect.DeepEqual(keys, []string{"generated", "positions", "source"}) { + t.Errorf("top-level keys = %v", keys) + } + var source map[string]json.RawMessage + if err := json.Unmarshal(raw["source"], &source); err != nil { + t.Fatal(err) + } + if keys := sortedKeys(source); !reflect.DeepEqual(keys, []string{"country", "device", "headendId", "language", "lineupId", "location", "postalCode", "providerName", "providerType"}) { + t.Errorf("source keys = %v", keys) + } + var positions []map[string]json.RawMessage + if err := json.Unmarshal(raw["positions"], &positions); err != nil { + t.Fatal(err) + } + // Position with filters carries the filters key; one without omits it. + withFilters := sortedKeys(positions[1]) + if !reflect.DeepEqual(withFilters, []string{"affiliate", "affiliateCallSign", "callSign", "filters", "logoUrl", "number", "placementId", "stationId"}) { + t.Errorf("position keys = %v", withFilters) + } + if _, ok := positions[0]["filters"]; ok { + t.Error("filters should be omitted when empty") + } + if _, ok := positions[0]["logoUrl"]; !ok { + t.Error("logoUrl must always be present, even when empty") + } +} + +func TestLineupJSONSourceNamingRequiresMatchingConfig(t *testing.T) { + // No saved configuration at all. + rec := httptest.NewRecorder() + newLineupServer(t, lineupTestGuide(), false).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + var got APILineup + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Source.ProviderName != "" || got.Source.ProviderType != "" || got.Source.Location != "" { + t.Errorf("provider naming should be blank without config: %+v", got.Source) + } + if got.Source.LineupID != "USA-lineupId-DEFAULT" { + t.Errorf("guide source fields must still be present: %+v", got.Source) + } + + // Saved configuration describes a different lineup than the guide. + g := lineupTestGuide() + g.Source.LineupID = "USA-OTHER-DEFAULT" + rec = httptest.NewRecorder() + newLineupServer(t, g, true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Source.ProviderName != "" { + t.Errorf("provider name leaked across lineups: %+v", got.Source) + } + if got.Source.LineupID != "USA-OTHER-DEFAULT" { + t.Errorf("lineup id should come from the guide: %+v", got.Source) + } +} + +func TestLineupJSONDoesNotMutateGuideOrder(t *testing.T) { + g := lineupTestGuide() + before := append([]guide.LineupPosition(nil), g.Lineup...) + lineupToJSON(g, lineupTestConfig(), true) + if !reflect.DeepEqual(before, g.Lineup) { + t.Fatal("lineupToJSON must sort a copy, not the live guide") + } +} + +func sortedKeys(m map[string]json.RawMessage) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/lineup_test.go b/lineup_test.go new file mode 100644 index 0000000..2a22296 --- /dev/null +++ b/lineup_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "net/url" + "reflect" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" +) + +func sampleLineupGuide() *guide.TVGuide { + return &guide.TVGuide{ + Channels: []guide.Channel{ + {ID: "s1", ChannelNo: "2", IconURL: "http://logos.example/s1.png", DisplayNames: []guide.DisplayName{{Name: "2 AAA"}, {Name: "2"}, {Name: "AAA"}}}, + {ID: "s2", ChannelNo: "5", IconURL: "", DisplayNames: []guide.DisplayName{{Name: "5 BBB"}, {Name: "5"}, {Name: "BBB"}}}, + }, + Programs: []guide.Program{ + {Channel: "s1", Title: "one", IconSrc: "http://img.example/a.jpg", Images: []guide.Image{{URL: "http://img.example/b.jpg"}}}, + {Channel: "s2", Title: "two"}, + }, + Lineup: []guide.LineupPosition{ + {ChannelNo: "2", StationID: "s1"}, + {ChannelNo: "5", StationID: "s2"}, + {ChannelNo: "1002", StationID: "s1"}, // same station, second position + }, + Source: guide.Source{LineupID: "L"}, + } +} + +func TestPropagateChannelLogosCopiesByStation(t *testing.T) { + g := sampleLineupGuide() + propagateChannelLogos(g.Channels, g.Lineup) + + if g.Lineup[0].LogoURL != "http://logos.example/s1.png" || g.Lineup[2].LogoURL != "http://logos.example/s1.png" { + t.Fatalf("both positions of s1 should carry its logo: %+v", g.Lineup) + } + if g.Lineup[1].LogoURL != "" { + t.Fatalf("s2 has no logo, got %q", g.Lineup[1].LogoURL) + } +} + +func TestRewriteImageURLsCoversLineup(t *testing.T) { + g := sampleLineupGuide() + propagateChannelLogos(g.Channels, g.Lineup) + rewriteImageURLs("http://host:8080/", g.Channels, g.Lineup, g.Programs) + + want := "http://host:8080/img?url=" + url.QueryEscape("http://logos.example/s1.png") + if g.Channels[0].IconURL != want { + t.Errorf("channel icon = %q, want %q", g.Channels[0].IconURL, want) + } + if g.Lineup[0].LogoURL != want || g.Lineup[2].LogoURL != want { + t.Errorf("lineup logos = %q / %q, want %q", g.Lineup[0].LogoURL, g.Lineup[2].LogoURL, want) + } + if g.Lineup[1].LogoURL != "" || g.Channels[1].IconURL != "" { + t.Error("empty URLs must stay empty") + } + if g.Programs[0].IconSrc != "http://host:8080/img?url="+url.QueryEscape("http://img.example/a.jpg") { + t.Errorf("program icon = %q", g.Programs[0].IconSrc) + } + if g.Programs[0].Images[0].URL != "http://host:8080/img?url="+url.QueryEscape("http://img.example/b.jpg") { + t.Errorf("program image = %q", g.Programs[0].Images[0].URL) + } +} + +func TestFilterGuideChannelsFiltersLineup(t *testing.T) { + g := sampleLineupGuide() + filtered := filterGuideChannels(g, map[string]bool{"2": true, "1002": true}) + + if len(filtered.Channels) != 1 || filtered.Channels[0].ID != "s1" { + t.Fatalf("channels = %+v", filtered.Channels) + } + if len(filtered.Programs) != 1 || filtered.Programs[0].Title != "one" { + t.Fatalf("programs = %+v", filtered.Programs) + } + var numbers []string + for _, p := range filtered.Lineup { + numbers = append(numbers, p.ChannelNo) + } + if !reflect.DeepEqual(numbers, []string{"2", "1002"}) { + t.Fatalf("lineup numbers = %v", numbers) + } + if filtered.Source != g.Source { + t.Fatal("source not carried through filter") + } +} diff --git a/main.go b/main.go index 69dc22e..d0643ad 100644 --- a/main.go +++ b/main.go @@ -21,12 +21,14 @@ import ( "sort" "strings" "sync" + "sync/atomic" "syscall" "text/template" "time" "github.com/daniel-widrick/GraceNoteScraper/appconfig" "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/scrape" "github.com/daniel-widrick/GraceNoteScraper/tmdb" "github.com/daniel-widrick/GraceNoteScraper/tvlogo" "github.com/daniel-widrick/GraceNoteScraper/util" @@ -87,6 +89,37 @@ type APIProgram struct { Description string `json:"description,omitempty"` } +// APILineup is the response shape of /api/lineup.json: every provider +// position for the active lineup, plus the source it came from. +type APILineup struct { + Generated string `json:"generated"` + Source APILineupSource `json:"source"` + Positions []APILineupPosition `json:"positions"` +} + +type APILineupSource struct { + ProviderName string `json:"providerName"` + ProviderType string `json:"providerType"` + Location string `json:"location"` + LineupID string `json:"lineupId"` + HeadendID string `json:"headendId"` + PostalCode string `json:"postalCode"` + Country string `json:"country"` + Device string `json:"device"` + Language string `json:"language"` +} + +type APILineupPosition struct { + Number string `json:"number"` + StationID string `json:"stationId"` + PlacementID string `json:"placementId"` + CallSign string `json:"callSign"` + Affiliate string `json:"affiliate"` + AffiliateCallSign string `json:"affiliateCallSign"` + Filters []string `json:"filters,omitempty"` + LogoURL string `json:"logoUrl"` +} + // ---------- Conversion ---------- // guideToJSON converts a TVGuide into the simplified JSON API format. @@ -145,7 +178,7 @@ func guideToJSON(g *guide.TVGuide) APIGuide { // Sort channels by number (numeric sort) sort.Slice(channels, func(i, j int) bool { - return channelNumberLess(channels[i].Number, channels[j].Number) + return guide.ChannelNumberLess(channels[i].Number, channels[j].Number) }) return APIGuide{ @@ -154,16 +187,54 @@ func guideToJSON(g *guide.TVGuide) APIGuide { } } -// channelNumberLess compares channel numbers numerically where possible. -func channelNumberLess(a, b string) bool { - // Try to parse as float for numeric comparison (handles "5.1", "12", etc.) - var ai, bi float64 - _, errA := fmt.Sscanf(a, "%f", &ai) - _, errB := fmt.Sscanf(b, "%f", &bi) - if errA == nil && errB == nil { - return ai < bi +// lineupToJSON converts a guide's lineup into the API shape. Provider naming +// comes from the saved configuration only when it describes the same source +// the guide was built from. +func lineupToJSON(g *guide.TVGuide, config appconfig.Config, configured bool) APILineup { + positions := make([]guide.LineupPosition, len(g.Lineup)) + copy(positions, g.Lineup) + guide.SortLineup(positions) + + out := APILineup{ + Generated: g.Source.GeneratedAt.UTC().Format(time.RFC3339), + Source: APILineupSource{ + LineupID: g.Source.LineupID, + HeadendID: g.Source.HeadendID, + PostalCode: g.Source.PostalCode, + Country: g.Source.Country, + Device: g.Source.Device, + Language: g.Source.Language, + }, + Positions: make([]APILineupPosition, 0, len(positions)), + } + if g.Source.GeneratedAt.IsZero() { + out.Generated = time.Now().UTC().Format(time.RFC3339) + } + if configured && sourceMatchesConfig(g.Source, config) { + out.Source.ProviderName = config.Gracenote.ProviderName + out.Source.ProviderType = config.Gracenote.ProviderType + out.Source.Location = config.Gracenote.Location + } + for _, p := range positions { + out.Positions = append(out.Positions, APILineupPosition{ + Number: p.ChannelNo, + StationID: p.StationID, + PlacementID: p.PlacementID, + CallSign: p.CallSign, + Affiliate: p.Affiliate, + AffiliateCallSign: p.AffiliateCallSign, + Filters: p.Filters, + LogoURL: p.LogoURL, + }) } - return a < b + return out +} + +// sourceMatchesConfig reports whether a guide was built from the configured lineup. +func sourceMatchesConfig(src guide.Source, config appconfig.Config) bool { + p := config.Preferences() + return src.LineupID == p.LineupId && src.HeadendID == p.Headend && src.PostalCode == p.ZipCode && + src.Country == p.Country && src.Device == p.Device && src.Language == p.Language } // xmltvTimeToISO converts "20250225200000 +0000" → "2025-02-25T20:00:00Z" @@ -207,60 +278,32 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch } } } - client := web.NewClient(pref) - - now := time.Now().UTC() - midnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) - endTime := midnight.Add(14 * 24 * time.Hour) - - channelMap := make(map[string]guide.Channel) - eventMap := make(map[string]bool) - var programs []guide.Program - totalSlots := int(endTime.Sub(midnight) / (6 * time.Hour)) - slot := 0 - for t := midnight; t.Before(endTime); t = t.Add(6 * time.Hour) { - if sourceCurrent != nil && !sourceCurrent() { - return nil, errScrapeSourceChanged - } - slot++ - ts := t.Unix() - report(scrapeProgressUpdate{Stage: "gracenote", Message: fmt.Sprintf("Downloading guide data (%d of %d)", slot, totalSlots), Completed: slot - 1, Total: totalSlots, Channels: len(channelMap), Programs: len(programs)}) - log.Printf("Fetching grid %d/%d for time=%d (%s)", slot, totalSlots, ts, t.Format(time.RFC3339)) - - grid, err := client.GetDataByTime(ts) - if err != nil { - log.Printf("Error fetching grid at %d: %v", ts, err) - continue - } - - for _, ch := range grid.Channels { - if _, exists := channelMap[ch.ChannelID]; !exists { - channelMap[ch.ChannelID] = guide.ConvertChannel(ch) - } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sourceChanged := watchSourceChange(ctx, cancel, sourceCurrent) - for _, ev := range ch.Events { - dedupKey := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime - if eventMap[dedupKey] { - continue - } - eventMap[dedupKey] = true - programs = append(programs, guide.ConvertEvent(ev, ch.ChannelID, pref.Language, pref.Country)) - } + opts := newScrapeOptions(pref) + opts.Progress = func(p scrape.Progress) { + message := fmt.Sprintf("Downloading guide data (%d of %d)", p.Slot, p.TotalSlots) + if p.Phase == scrape.PhaseFetched { + message = fmt.Sprintf("Downloaded guide data (%d of %d)", p.Slot, p.TotalSlots) } - - log.Printf("Channels so far: %d, Events so far: %d", len(channelMap), len(programs)) - report(scrapeProgressUpdate{Stage: "gracenote", Message: fmt.Sprintf("Downloaded guide data (%d of %d)", slot, totalSlots), Completed: slot, Total: totalSlots, Channels: len(channelMap), Programs: len(programs)}) - - if t.Add(6 * time.Hour).Before(endTime) { - time.Sleep(5 * time.Second) + report(scrapeProgressUpdate{Stage: "gracenote", Message: message, Completed: p.Completed, Total: p.TotalSlots, Channels: p.Channels, Programs: p.Programs}) + } + fetched, err := scrape.Fetch(ctx, pref, opts) + if err != nil { + if sourceChanged() { + return nil, errScrapeSourceChanged } + return nil, err } - - var channels []guide.Channel - for _, ch := range channelMap { - channels = append(channels, ch) + if sourceChanged() { + return nil, errScrapeSourceChanged } + channels := fetched.Channels + lineup := fetched.Lineup + programs := fetched.Programs logoClient := tvlogo.NewClient(pref.Country, "tvlogo_cache.json") if logoClient != nil { @@ -268,6 +311,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch } report(scrapeProgressUpdate{Stage: "logos", Message: "Matching channel logos", Channels: len(channels), Programs: len(programs)}) enrichChannelIcons(logoClient, channels) + propagateChannelLogos(channels, lineup) enrichProgramThumbnails(tmdbClient, programs, func(completed, total int) { report(scrapeProgressUpdate{Stage: "tmdb", Message: fmt.Sprintf("Enriching program titles (%d of %d)", completed, total), Completed: completed, Total: total, Channels: len(channels), Programs: len(programs)}) }) @@ -275,28 +319,15 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch // Rewrite image URLs to go through the local proxy if baseURL != "" { - proxy := strings.TrimRight(baseURL, "/") + "/img?url=" - for i := range channels { - if channels[i].IconURL != "" { - channels[i].IconURL = proxy + neturl.QueryEscape(channels[i].IconURL) - } - } - for i := range programs { - if programs[i].IconSrc != "" { - programs[i].IconSrc = proxy + neturl.QueryEscape(programs[i].IconSrc) - } - for j := range programs[i].Images { - if programs[i].Images[j].URL != "" { - programs[i].Images[j].URL = proxy + neturl.QueryEscape(programs[i].Images[j].URL) - } - } - } + rewriteImageURLs(baseURL, channels, lineup, programs) log.Printf("Rewrote image URLs with base %s", baseURL) } tvGuide := &guide.TVGuide{ Channels: channels, Programs: programs, + Lineup: lineup, + Source: fetched.Source, } if channelFilter != nil { @@ -320,14 +351,66 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch return tvGuide, nil } -func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { - log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) - - // Parse embedded template +// renderXMLTV writes the guide as XMLTV using the embedded template. +func renderXMLTV(w io.Writer, tvGuide *guide.TVGuide) error { tmpl, err := template.ParseFS(guideTmplFS, "guide.tmpl") if err != nil { return fmt.Errorf("failed to parse template: %w", err) } + if err := tmpl.Execute(w, tvGuide); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + return nil +} + +// newScrapeOptions builds the production scrape configuration. Tests replace +// it to inject a fake grid fetcher and remove the inter-slot delay. +var newScrapeOptions = func(pref web.Preferences) scrape.Options { + return scrape.Options{Fetcher: web.NewClient(pref)} +} + +// watchSourceChange cancels ctx as soon as sourceCurrent reports false, so a +// lineup change during the grid download stops the scrape promptly. The +// returned func reports whether a change has been observed, re-checking live +// so a change between the last tick and the call is not missed. +func watchSourceChange(ctx context.Context, cancel context.CancelFunc, sourceCurrent func() bool) func() bool { + if sourceCurrent == nil { + return func() bool { return false } + } + var changed atomic.Bool + check := func() bool { + if changed.Load() { + return true + } + if !sourceCurrent() { + changed.Store(true) + cancel() + return true + } + return false + } + if check() { + return check + } + go func() { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if check() { + return + } + } + } + }() + return check +} + +func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { + log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) // Atomic write: write to temp file, then rename tmpFile, err := os.CreateTemp(".", "xmlguide-*.tmp") @@ -336,10 +419,10 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { } tmpName := tmpFile.Name() - if err := tmpl.Execute(tmpFile, tvGuide); err != nil { + if err := renderXMLTV(tmpFile, tvGuide); err != nil { tmpFile.Close() os.Remove(tmpName) - return fmt.Errorf("failed to execute template: %w", err) + return err } if err := tmpFile.Close(); err != nil { os.Remove(tmpName) @@ -363,7 +446,13 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { const guideCachePath = "guide_cache.json" +// guideCacheVersion is bumped whenever the persisted guide shape gains fields +// that a scrape must populate. An older cache is rebuilt rather than served +// with empty fields. +const guideCacheVersion = 2 + type guideCache struct { + Version int `json:"version"` SavedAt time.Time `json:"saved_at"` SourceFingerprint string `json:"source_fingerprint"` Guide guide.TVGuide `json:"guide"` @@ -371,7 +460,7 @@ type guideCache struct { // saveGuideCache persists the TVGuide to a JSON file. func saveGuideCache(g *guide.TVGuide, sourceFingerprint string) { - data, err := json.Marshal(guideCache{SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g}) + data, err := json.Marshal(guideCache{Version: guideCacheVersion, SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g}) if err != nil { log.Printf("guide cache: failed to marshal: %v", err) return @@ -395,6 +484,10 @@ func loadGuideCache(maxAge time.Duration, sourceFingerprint string) (*guide.TVGu log.Printf("guide cache: corrupt, ignoring: %v", err) return nil, 0, false } + if c.Version != guideCacheVersion { + log.Printf("guide cache: schema version %d, want %d; rebuilding", c.Version, guideCacheVersion) + return nil, 0, false + } if c.SourceFingerprint != sourceFingerprint { log.Println("guide cache: source changed, ignoring cached guide") return nil, 0, false @@ -605,6 +698,24 @@ func handleGuideJSON(state *GuideState) http.HandlerFunc { } } +func handleLineupJSON(state *GuideState, store *appconfig.Store) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + g := state.Get() + if g == nil { + w.Header().Set("Retry-After", "30") + http.Error(w, "Guide is being generated", http.StatusServiceUnavailable) + return + } + + config, configured, _ := store.Get() + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Access-Control-Allow-Origin", "*") + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + enc.Encode(lineupToJSON(g, config, configured)) + } +} + // ---------- Image proxy ---------- const imageCacheDir = "image_cache" @@ -958,9 +1069,18 @@ func filterGuideChannels(g *guide.TVGuide, allowed map[string]bool) *guide.TVGui } } + var lineup []guide.LineupPosition + for _, position := range g.Lineup { + if allowed[position.ChannelNo] { + lineup = append(lineup, position) + } + } + return &guide.TVGuide{ Channels: channels, Programs: programs, + Lineup: lineup, + Source: g.Source, } } @@ -1104,6 +1224,7 @@ func main() { mux.HandleFunc("/api/setup/status", setupHandlers.handleScrapeStatus) mux.HandleFunc("/xmlguide.xmltv", handleXMLTV(state)) mux.HandleFunc("/api/guide.json", handleGuideJSON(state)) + mux.HandleFunc("/api/lineup.json", handleLineupJSON(state, configStore)) mux.HandleFunc("/img", handleImage) mux.HandleFunc("/api/livetv/config", handleLiveTVConfig(jellyfinURL, jellyfinAPIKey)) if jellyfinURL != "" && jellyfinAPIKey != "" { @@ -1273,6 +1394,43 @@ func fixDeadImageURLs(programs []guide.Program) { // resolves channel logos from the tv-logo/tv-logos repo, // replacing dead Gracenote icon URLs with verified GitHub-hosted PNGs. +// rewriteImageURLs routes every image URL through the local /img proxy. +func rewriteImageURLs(baseURL string, channels []guide.Channel, lineup []guide.LineupPosition, programs []guide.Program) { + proxy := strings.TrimRight(baseURL, "/") + "/img?url=" + rewrite := func(u string) string { + if u == "" { + return "" + } + return proxy + neturl.QueryEscape(u) + } + for i := range channels { + channels[i].IconURL = rewrite(channels[i].IconURL) + } + for i := range lineup { + lineup[i].LogoURL = rewrite(lineup[i].LogoURL) + } + for i := range programs { + programs[i].IconSrc = rewrite(programs[i].IconSrc) + for j := range programs[i].Images { + programs[i].Images[j].URL = rewrite(programs[i].Images[j].URL) + } + } +} + +// propagateChannelLogos copies each station's resolved icon onto every lineup +// position carrying that station, so logo resolution stays keyed by station. +func propagateChannelLogos(channels []guide.Channel, lineup []guide.LineupPosition) { + byStation := make(map[string]string, len(channels)) + for _, ch := range channels { + byStation[ch.ID] = ch.IconURL + } + for i := range lineup { + if logo, ok := byStation[lineup[i].StationID]; ok { + lineup[i].LogoURL = logo + } + } +} + func enrichChannelIcons(client *tvlogo.Client, channels []guide.Channel) { if client == nil { return diff --git a/run_scrape_test.go b/run_scrape_test.go new file mode 100644 index 0000000..75c1a8e --- /dev/null +++ b/run_scrape_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "errors" + "io" + "log" + "os" + "sync/atomic" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/scrape" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +type countingFetcher struct { + calls atomic.Int32 + grid *web.GridResponse + err error +} + +func (f *countingFetcher) GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) { + f.calls.Add(1) + if err := ctx.Err(); err != nil { + return nil, err + } + if f.err != nil { + return nil, f.err + } + return f.grid, nil +} + +// useFakeScrape swaps the production scrape options for a fake fetcher with +// one day of slots and no delay. The preferences use a country tvlogo does not +// support so no logo lookups reach the network. +func useFakeScrape(t *testing.T, f *countingFetcher) web.Preferences { + t.Helper() + t.Chdir(t.TempDir()) + original := newScrapeOptions + newScrapeOptions = func(web.Preferences) scrape.Options { + return scrape.Options{Fetcher: f, Days: 1, SlotDelay: scrape.NoDelay, Logger: log.New(io.Discard, "", 0)} + } + t.Cleanup(func() { newScrapeOptions = original }) + return web.Preferences{Country: "ZZZ", ZipCode: "00000", Headend: "h", LineupId: "L", Device: "-", Language: "en-us"} +} + +func TestRunScrapeBuildsLineupAndPersists(t *testing.T) { + f := &countingFetcher{grid: &web.GridResponse{Channels: []web.JSONChannel{ + {ChannelID: "s1", ID: "s10", ChannelNo: "2.1", CallSign: "AAA", AffiliateName: "A Net", StationFilters: []string{"filter-news"}, + Events: []web.JSONEvent{{StartTime: "2026-09-13T00:00:00Z", EndTime: "2026-09-13T01:00:00Z", Duration: "60", SeriesID: "SH1", Program: web.JSONProgram{ID: "EP1", Title: "one"}}}}, + {ChannelID: "s1", ID: "s199", ChannelNo: "1002", CallSign: "AAA", AffiliateName: "A Net"}, + }}} + pref := useFakeScrape(t, f) + + var persisted *guide.TVGuide + persister := func(g *guide.TVGuide) (bool, error) { + persisted = g + return true, persistGuideFiles(g, "fp") + } + var updates []scrapeProgressUpdate + got, err := runScrape(pref, nil, "http://base:8080", nil, "fp", func() bool { return true }, persister, func(u scrapeProgressUpdate) { updates = append(updates, u) }) + if err != nil { + t.Fatalf("runScrape: %v", err) + } + if f.calls.Load() != 4 { + t.Errorf("grid fetches = %d, want 4 for one day", f.calls.Load()) + } + if persisted != got { + t.Error("persister should receive the returned guide") + } + if len(got.Channels) != 1 || len(got.Lineup) != 2 || len(got.Programs) != 1 { + t.Fatalf("channels/lineup/programs = %d/%d/%d", len(got.Channels), len(got.Lineup), len(got.Programs)) + } + if got.Lineup[0].LogoURL != "" { + t.Errorf("no logo source available, got %q", got.Lineup[0].LogoURL) + } + if got.Source.LineupID != "L" { + t.Errorf("source = %+v", got.Source) + } + if _, err := os.Stat("xmlguide.xmltv"); err != nil { + t.Errorf("xmlguide.xmltv not written: %v", err) + } + stages := map[string]bool{} + for _, u := range updates { + stages[u.Stage] = true + } + for _, want := range []string{"gracenote", "logos", "saving"} { + if !stages[want] { + t.Errorf("missing progress stage %q (got %v)", want, stages) + } + } + if stages["tmdb"] { + t.Error("no TMDB client configured, so no tmdb stage should be reported") + } + var gracenote []scrapeProgressUpdate + for _, u := range updates { + if u.Stage == "gracenote" { + gracenote = append(gracenote, u) + } + } + if len(gracenote) != 8 || gracenote[0].Total != 4 || gracenote[7].Completed != 4 { + t.Errorf("gracenote progress = %d updates, first %+v, last %+v", len(gracenote), gracenote[0], gracenote[len(gracenote)-1]) + } +} + +func TestRunScrapeReportsSourceChange(t *testing.T) { + f := &countingFetcher{grid: &web.GridResponse{Channels: []web.JSONChannel{{ChannelID: "s1", ChannelNo: "2"}}}} + pref := useFakeScrape(t, f) + + var seen atomic.Int32 + sourceCurrent := func() bool { + // Current for the initial check, changed afterwards. + return seen.Add(1) == 1 + } + persisterCalled := false + _, err := runScrape(pref, nil, "", nil, "fp", sourceCurrent, func(*guide.TVGuide) (bool, error) { + persisterCalled = true + return true, nil + }) + if !errors.Is(err, errScrapeSourceChanged) { + t.Fatalf("err = %v, want errScrapeSourceChanged", err) + } + if persisterCalled { + t.Error("persister must not run after a source change") + } +} + +func TestRunScrapeSurfacesTotalFailure(t *testing.T) { + f := &countingFetcher{err: errors.New("gracenote down")} + pref := useFakeScrape(t, f) + _, err := runScrape(pref, nil, "", nil, "fp", nil, func(*guide.TVGuide) (bool, error) { + t.Fatal("persister must not run with no data") + return false, nil + }) + if !errors.Is(err, scrape.ErrNoData) { + t.Fatalf("err = %v, want ErrNoData", err) + } +} diff --git a/scrape/scrape.go b/scrape/scrape.go new file mode 100644 index 0000000..cf863d5 --- /dev/null +++ b/scrape/scrape.go @@ -0,0 +1,189 @@ +// Package scrape fetches a Gracenote lineup's listings and converts them into +// a guide.TVGuide. It performs no enrichment or environment reads, so it can +// be imported by other programs. The default fetcher, web.NewClient, keeps its +// own raw-response cache in ./grid_cache; supply Options.Fetcher to avoid it. +package scrape + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +const ( + // SlotDuration is the window Gracenote serves per grid request. + SlotDuration = 6 * time.Hour + // DefaultDays is how far ahead a scrape reaches when Options.Days is zero. + DefaultDays = 14 + // DefaultSlotDelay is the pause between grid requests when + // Options.SlotDelay is zero. Gracenote is an unofficial API; be polite. + DefaultSlotDelay = 5 * time.Second +) + +// ErrNoData is returned when every grid slot failed, so there is nothing to +// build a guide from. A partial failure is not an error; failed slots are +// skipped and reported through Options.Progress. +var ErrNoData = errors.New("scrape: no grid slot returned data") + +// GridFetcher retrieves one six-hour grid. *web.Client satisfies it. +type GridFetcher interface { + GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) +} + +// Phase says whether a Progress report precedes or follows a slot fetch. +type Phase int + +const ( + // PhaseFetching is reported just before a slot is requested. + PhaseFetching Phase = iota + // PhaseFetched is reported after a slot succeeded or failed. + PhaseFetched +) + +// Progress describes one step of a scrape. +type Progress struct { + Phase Phase + Slot int // 1-based index of the slot being reported + TotalSlots int // total slots for this scrape + SlotTime time.Time // start of the six-hour window + Completed int // slots finished so far, successful or not + Channels int // distinct stations collected so far + Programs int // programs collected so far + Err error // non-nil on PhaseFetched when the slot failed and was skipped +} + +// Options tunes a Fetch. The zero value is the production configuration. +type Options struct { + Days int // days of listings to fetch; default DefaultDays + SlotDelay time.Duration // pause between slots; default DefaultSlotDelay (tests use a negative value for none) + Fetcher GridFetcher // grid source; default web.NewClient(prefs) + Now func() time.Time // clock; default time.Now + Progress func(Progress) // optional per-slot callback + Logger *log.Logger // default log.Default() +} + +// NoDelay is a SlotDelay value that disables the inter-slot pause. +const NoDelay = -1 + +func (o Options) withDefaults(prefs web.Preferences) Options { + if o.Days <= 0 { + o.Days = DefaultDays + } + if o.SlotDelay == 0 { + o.SlotDelay = DefaultSlotDelay + } + if o.Fetcher == nil { + o.Fetcher = web.NewClient(prefs) + } + if o.Now == nil { + o.Now = time.Now + } + if o.Logger == nil { + o.Logger = log.Default() + } + return o +} + +// Slots returns the six-hour window start times a Fetch will request for the +// given clock and day count, beginning at UTC midnight of the current day. +func Slots(now time.Time, days int) []time.Time { + if days <= 0 { + days = DefaultDays + } + now = now.UTC() + start := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + end := start.Add(time.Duration(days) * 24 * time.Hour) + slots := make([]time.Time, 0, days*int(24*time.Hour/SlotDuration)) + for t := start; t.Before(end); t = t.Add(SlotDuration) { + slots = append(slots, t) + } + return slots +} + +// Fetch downloads every grid slot for prefs and returns the assembled guide. +// Channels are deduplicated by station in first-seen order, Lineup keeps every +// position, and Programs are deduplicated by station, start, and end. +func Fetch(ctx context.Context, prefs web.Preferences, opts Options) (*guide.TVGuide, error) { + opts = opts.withDefaults(prefs) + slots := Slots(opts.Now(), opts.Days) + + channelIndex := make(map[string]int) + lineupIndex := make(map[string]struct{}) + eventIndex := make(map[string]struct{}) + var channels []guide.Channel + var lineup []guide.LineupPosition + var programs []guide.Program + succeeded := 0 + + report := func(p Progress) { + if opts.Progress != nil { + opts.Progress(p) + } + } + + for i, slotTime := range slots { + if err := ctx.Err(); err != nil { + return nil, err + } + slot := i + 1 + report(Progress{Phase: PhaseFetching, Slot: slot, TotalSlots: len(slots), SlotTime: slotTime, Completed: i, Channels: len(channels), Programs: len(programs)}) + opts.Logger.Printf("Fetching grid %d/%d for time=%d (%s)", slot, len(slots), slotTime.Unix(), slotTime.Format(time.RFC3339)) + + grid, err := opts.Fetcher.GetDataByTimeContext(ctx, slotTime.Unix()) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + opts.Logger.Printf("Error fetching grid at %d: %v", slotTime.Unix(), err) + report(Progress{Phase: PhaseFetched, Slot: slot, TotalSlots: len(slots), SlotTime: slotTime, Completed: slot, Channels: len(channels), Programs: len(programs), Err: err}) + } else { + succeeded++ + for _, ch := range grid.Channels { + if _, seen := channelIndex[ch.ChannelID]; !seen { + channelIndex[ch.ChannelID] = len(channels) + channels = append(channels, guide.ConvertChannel(ch)) + } + position := guide.ConvertLineupPosition(ch) + if _, seen := lineupIndex[position.Key()]; !seen { + lineupIndex[position.Key()] = struct{}{} + lineup = append(lineup, position) + } + for _, ev := range ch.Events { + key := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime + if _, seen := eventIndex[key]; seen { + continue + } + eventIndex[key] = struct{}{} + programs = append(programs, guide.ConvertEvent(ev, ch.ChannelID, prefs.Language, prefs.Country)) + } + } + opts.Logger.Printf("Channels so far: %d, Events so far: %d", len(channels), len(programs)) + report(Progress{Phase: PhaseFetched, Slot: slot, TotalSlots: len(slots), SlotTime: slotTime, Completed: slot, Channels: len(channels), Programs: len(programs)}) + } + + if slot < len(slots) && opts.SlotDelay > 0 { + select { + case <-time.After(opts.SlotDelay): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } + + if succeeded == 0 { + return nil, fmt.Errorf("%w (%d slots attempted)", ErrNoData, len(slots)) + } + + guide.SortLineup(lineup) + return &guide.TVGuide{ + Channels: channels, + Programs: programs, + Lineup: lineup, + Source: guide.SourceFromPreferences(prefs, opts.Now().UTC()), + }, nil +} diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go new file mode 100644 index 0000000..ab8c207 --- /dev/null +++ b/scrape/scrape_test.go @@ -0,0 +1,271 @@ +package scrape + +import ( + "context" + "errors" + "io" + "log" + "sync" + "testing" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +var fixedNow = time.Date(2026, 9, 13, 15, 42, 0, 0, time.UTC) + +func testPrefs() web.Preferences { + return web.Preferences{Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us"} +} + +// fakeFetcher serves canned grids keyed by slot time and records every call. +type fakeFetcher struct { + mu sync.Mutex + grids map[int64]*web.GridResponse + errs map[int64]error + always *web.GridResponse + calls []int64 + block chan struct{} // when set, the first call blocks until closed +} + +func (f *fakeFetcher) GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) { + f.mu.Lock() + f.calls = append(f.calls, t) + block := f.block + f.block = nil + f.mu.Unlock() + if block != nil { + select { + case <-block: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if err, ok := f.errs[t]; ok { + return nil, err + } + if g, ok := f.grids[t]; ok { + return g, nil + } + if f.always != nil { + return f.always, nil + } + return &web.GridResponse{}, nil +} + +func channel(station, number string, events ...web.JSONEvent) web.JSONChannel { + return web.JSONChannel{ChannelID: station, ID: station + "0", ChannelNo: number, CallSign: "C" + station, AffiliateName: "Net " + station, Events: events} +} + +func event(start, end, title string) web.JSONEvent { + return web.JSONEvent{StartTime: start, EndTime: end, Duration: "60", SeriesID: "SH1", Program: web.JSONProgram{ID: "EP1", Title: title}} +} + +func quietOptions(f *fakeFetcher) Options { + return Options{Days: 1, SlotDelay: NoDelay, Fetcher: f, Now: func() time.Time { return fixedNow }, Logger: log.New(io.Discard, "", 0)} +} + +func TestSlotsAreMidnightAlignedSixHourWindows(t *testing.T) { + slots := Slots(fixedNow, 2) + if len(slots) != 8 { + t.Fatalf("slots = %d, want 8", len(slots)) + } + if !slots[0].Equal(time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC)) { + t.Errorf("first slot = %s", slots[0]) + } + for i := 1; i < len(slots); i++ { + if slots[i].Sub(slots[i-1]) != SlotDuration { + t.Errorf("slot %d not 6h after previous", i) + } + } + if len(Slots(fixedNow, 0)) != DefaultDays*4 { + t.Errorf("zero days should default to %d days", DefaultDays) + } +} + +func TestFetchRequestsEverySlotAndAssemblesGuide(t *testing.T) { + f := &fakeFetcher{always: &web.GridResponse{Channels: []web.JSONChannel{ + channel("s1", "2.1", event("2026-09-13T00:00:00Z", "2026-09-13T01:00:00Z", "a")), + channel("s2", "5", event("2026-09-13T00:00:00Z", "2026-09-13T02:00:00Z", "b")), + channel("s1", "1002"), // same station at a second number + }}} + opts := quietOptions(f) + opts.Days = 2 + + g, err := Fetch(context.Background(), testPrefs(), opts) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if len(f.calls) != 8 { + t.Fatalf("calls = %d, want 8", len(f.calls)) + } + if f.calls[0] != time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC).Unix() { + t.Errorf("first request time = %d", f.calls[0]) + } + if len(g.Channels) != 2 { + t.Errorf("channels = %d, want 2 (deduplicated by station)", len(g.Channels)) + } + if g.Channels[0].ID != "s1" || g.Channels[1].ID != "s2" { + t.Errorf("channels should keep first-seen order: %v %v", g.Channels[0].ID, g.Channels[1].ID) + } + if len(g.Lineup) != 3 { + t.Fatalf("lineup = %d, want 3 (never collapsed)", len(g.Lineup)) + } + if g.Lineup[0].ChannelNo != "2.1" || g.Lineup[1].ChannelNo != "5" || g.Lineup[2].ChannelNo != "1002" { + t.Errorf("lineup order = %s %s %s", g.Lineup[0].ChannelNo, g.Lineup[1].ChannelNo, g.Lineup[2].ChannelNo) + } + if len(g.Programs) != 2 { + t.Errorf("programs = %d, want 2 (same events across 8 slots deduplicated)", len(g.Programs)) + } + if g.Source.LineupID != "USA-lineupId-DEFAULT" || g.Source.PostalCode != "13490" || !g.Source.GeneratedAt.Equal(fixedNow) { + t.Errorf("source = %+v", g.Source) + } +} + +func TestFetchKeepsDistinctEventsAcrossSlots(t *testing.T) { + slots := Slots(fixedNow, 1) + f := &fakeFetcher{grids: map[int64]*web.GridResponse{ + slots[0].Unix(): {Channels: []web.JSONChannel{channel("s1", "2", event("2026-09-13T00:00:00Z", "2026-09-13T06:00:00Z", "morning"))}}, + slots[1].Unix(): {Channels: []web.JSONChannel{channel("s1", "2", event("2026-09-13T06:00:00Z", "2026-09-13T12:00:00Z", "midday"))}}, + }} + g, err := Fetch(context.Background(), testPrefs(), quietOptions(f)) + if err != nil { + t.Fatal(err) + } + if len(g.Programs) != 2 { + t.Fatalf("programs = %d, want 2", len(g.Programs)) + } +} + +func TestFetchSkipsFailedSlotAndReportsIt(t *testing.T) { + slots := Slots(fixedNow, 1) + boom := errors.New("upstream timeout") + f := &fakeFetcher{ + always: &web.GridResponse{Channels: []web.JSONChannel{channel("s1", "2")}}, + errs: map[int64]error{slots[1].Unix(): boom}, + } + var reports []Progress + opts := quietOptions(f) + opts.Progress = func(p Progress) { reports = append(reports, p) } + + g, err := Fetch(context.Background(), testPrefs(), opts) + if err != nil { + t.Fatalf("partial failure must not be an error: %v", err) + } + if len(g.Channels) != 1 { + t.Fatalf("channels = %d", len(g.Channels)) + } + if len(reports) != 8 { + t.Fatalf("progress reports = %d, want 2 per slot", len(reports)) + } + failed := reports[3] // slot 2, PhaseFetched + if failed.Phase != PhaseFetched || failed.Slot != 2 || !errors.Is(failed.Err, boom) { + t.Errorf("failed slot report = %+v", failed) + } + for i, r := range reports { + if r.TotalSlots != 4 { + t.Errorf("report %d TotalSlots = %d", i, r.TotalSlots) + } + if i > 0 && r.Completed < reports[i-1].Completed { + t.Errorf("Completed went backwards at report %d", i) + } + } + last := reports[len(reports)-1] + if last.Completed != 4 || last.Slot != 4 || last.Phase != PhaseFetched { + t.Errorf("last report = %+v", last) + } +} + +func TestFetchReturnsErrNoDataWhenEverySlotFails(t *testing.T) { + f := &fakeFetcher{errs: map[int64]error{}} + for _, s := range Slots(fixedNow, 1) { + f.errs[s.Unix()] = errors.New("down") + } + _, err := Fetch(context.Background(), testPrefs(), quietOptions(f)) + if !errors.Is(err, ErrNoData) { + t.Fatalf("err = %v, want ErrNoData", err) + } +} + +func TestFetchStopsPromptlyWhenContextCancelledDuringDelay(t *testing.T) { + f := &fakeFetcher{always: &web.GridResponse{Channels: []web.JSONChannel{channel("s1", "2")}}} + opts := quietOptions(f) + opts.SlotDelay = time.Minute + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { + _, err := Fetch(ctx, testPrefs(), opts) + done <- err + }() + // Let the first slot complete and the delay begin, then cancel. + deadline := time.After(2 * time.Second) + for { + f.mu.Lock() + n := len(f.calls) + f.mu.Unlock() + if n >= 1 { + break + } + select { + case <-deadline: + t.Fatal("first slot never requested") + default: + time.Sleep(5 * time.Millisecond) + } + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Fetch did not return after cancellation") + } + if len(f.calls) != 1 { + t.Errorf("calls after cancel = %d, want 1", len(f.calls)) + } +} + +func TestFetchStopsWhenContextCancelledDuringRequest(t *testing.T) { + f := &fakeFetcher{always: &web.GridResponse{}, block: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := Fetch(ctx, testPrefs(), quietOptions(f)) + done <- err + }() + time.Sleep(20 * time.Millisecond) + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Fetch did not return") + } +} + +func TestFetchHonoursAlreadyCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + f := &fakeFetcher{} + if _, err := Fetch(ctx, testPrefs(), quietOptions(f)); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + if len(f.calls) != 0 { + t.Errorf("no requests should be made, got %d", len(f.calls)) + } +} + +func TestOptionsDefaults(t *testing.T) { + o := Options{}.withDefaults(testPrefs()) + if o.Days != DefaultDays || o.SlotDelay != DefaultSlotDelay || o.Fetcher == nil || o.Now == nil || o.Logger == nil { + t.Fatalf("defaults not applied: %+v", o) + } + if _, ok := o.Fetcher.(*web.Client); !ok { + t.Fatalf("default fetcher should be *web.Client, got %T", o.Fetcher) + } +} diff --git a/setup_test.go b/setup_test.go index 7651721..ba03181 100644 --- a/setup_test.go +++ b/setup_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" @@ -226,3 +227,21 @@ func TestGuideCacheRequiresMatchingSource(t *testing.T) { t.Fatal("cache did not load for matching source") } } + +func TestGuideCacheRequiresCurrentSchemaVersion(t *testing.T) { + t.Chdir(t.TempDir()) + + // A cache written by an older build has no version field. + legacy := []byte(`{"saved_at":"` + time.Now().UTC().Format(time.RFC3339) + `","source_fingerprint":"src","guide":{"Channels":[],"Programs":[]}}`) + if err := os.WriteFile(guideCachePath, legacy, 0644); err != nil { + t.Fatal(err) + } + if _, _, ok := loadGuideCache(time.Hour, "src"); ok { + t.Fatal("legacy cache without a version loaded") + } + + saveGuideCache(&guide.TVGuide{}, "src") + if _, _, ok := loadGuideCache(time.Hour, "src"); !ok { + t.Fatal("current-version cache did not load") + } +} diff --git a/testdata/xmlguide_golden.xmltv b/testdata/xmlguide_golden.xmltv new file mode 100644 index 0000000..8944ec8 --- /dev/null +++ b/testdata/xmlguide_golden.xmltv @@ -0,0 +1,167 @@ + + + + + 2.1 WKTVDT + 2.1 + WKTVDT + NATIONAL BROADCASTING COMPANY + + + + 9.3 WSYRDT3 + 9.3 + WSYRDT3 + BOUNCE TV + + + + 24.4 WCNYDT4 + 24.4 + WCNYDT4 + PBS KIDS HD + + + + College Football + Iowa State at Iowa + The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at + 210 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH06270099&tmsId=EP062700990343 + USA + EP06270099.0343 + sports + + + + + NewsChannel 2 at 11 + Stay informed with the latest breaking news and headlines. + 29 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00793300&tmsId=SH007933000000 + USA + SH00793300.0000 + news + + + + + Saturday Night Live + Ryan Gosling; Gorillaz + Host Ryan Gosling; Gorillaz perform. + 94 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00003710&tmsId=EP000037101609 + USA + S51E14 + 50.13 + EP00003710.1609 + Series + + + TV-14 + + + The Protégé + Rescued as a child by the legendary assassin Moody, Anna is + 150 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV01539920&tmsId=MV015399200000 + USA + MV01539920.0000 + movie + + + TV-14 + + + Ninja Assassin + A rogue (Rain) enlists the help of the Europol agent (Naomie + 120 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV00227303&tmsId=MV002273030000 + USA + MV00227303.0000 + movie + + + TV-14 + + + Bullet to the Head + After his partner is killed, a veteran hit man (Sylvester St + 120 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV00375600&tmsId=MV003756000000 + USA + MV00375600.0000 + movie + + + TV-14 + + + Carl the Collector + The Square Dance; The Process + Dylan realizes he doesn't love square-dancing; Carl teaches + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH05284464&tmsId=EP052844640100 + USA + S01E30 + 0.29 + EP05284464.0100 + family + Series + Finale + + + TV-Y + + + Carl the Collector + The Bark Banquet; Carl's Collection of To-Dos + Carl wants to start his own family tradition after Sheldon i + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH05284464&tmsId=EP052844640101 + USA + S01E31 + 0.30 + EP05284464.0101 + family + Series + + + + TV-Y + + + Cyberchase + Buzz and the Tree + It's Arbor Day in Cyberspace, but Buzz is more interested in + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00361880&tmsId=EP003618800155 + USA + S13E06 + 12.5 + EP00361880.0155 + family + Series + + + TV-Y + + diff --git a/web/flex.go b/web/flex.go new file mode 100644 index 0000000..ad88a65 --- /dev/null +++ b/web/flex.go @@ -0,0 +1,76 @@ +package web + +import ( + "bytes" + "encoding/json" + "strconv" + "strings" +) + +// FlexString decodes a JSON string, number, or null into a string. Gracenote +// is inconsistent about scalar encodings, and a decode error on one optional +// field would otherwise discard an entire six-hour grid. +type FlexString string + +func (f *FlexString) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = "" + return nil + } + if data[0] == '"' { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + *f = FlexString(s) + return nil + } + var n json.Number + if err := json.Unmarshal(data, &n); err == nil { + *f = FlexString(n.String()) + return nil + } + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *f = FlexString(strconv.FormatBool(b)) + return nil + } + *f = "" + return nil +} + +// FlexBool decodes a JSON bool, a 0/1 number, a "0"/"1"/"true"/"false" string, +// or null into a bool. Anything unrecognized decodes as false rather than +// failing the surrounding document. +type FlexBool bool + +func (f *FlexBool) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = false + return nil + } + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *f = FlexBool(b) + return nil + } + var n json.Number + if err := json.Unmarshal(data, &n); err == nil { + *f = FlexBool(n.String() != "0" && n.String() != "0.0") + return nil + } + var s string + if err := json.Unmarshal(data, &s); err == nil { + switch strings.ToLower(strings.TrimSpace(s)) { + case "1", "true", "yes": + *f = true + default: + *f = false + } + return nil + } + *f = false + return nil +} diff --git a/web/testdata/grid_sample.json b/web/testdata/grid_sample.json new file mode 100644 index 0000000..bae4ef2 --- /dev/null +++ b/web/testdata/grid_sample.json @@ -0,0 +1,380 @@ +{ + "channels": [ + { + "affiliateCallSign": "null", + "affiliateName": "NATIONAL BROADCASTING COMPANY", + "callSign": "WKTVDT", + "channelId": "53158", + "channelNo": "2.1", + "events": [ + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "210", + "endTime": "2026-09-13T03:00:00Z", + "filter": [ + "filter-sports" + ], + "flag": [ + "Live", + "New" + ], + "program": { + "episode": null, + "episodeTitle": "Iowa State at Iowa", + "id": "EP062700990343", + "isGeneric": "0", + "releaseYear": 2019, + "season": null, + "seriesId": "SH06270099", + "shortDesc": "The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at ", + "title": "College Football", + "tmsId": "EP062700990343" + }, + "rating": null, + "seriesId": "SH06270099", + "startTime": "2026-09-12T23:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + }, + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "29", + "endTime": "2026-09-13T03:29:00Z", + "filter": [ + "filter-news" + ], + "flag": [ + "New" + ], + "program": { + "episode": null, + "episodeTitle": null, + "id": "SH007933000000", + "isGeneric": "0", + "releaseYear": "2021", + "season": null, + "seriesId": "SH00793300", + "shortDesc": "Stay informed with the latest breaking news and headlines.", + "title": "NewsChannel 2 at 11", + "tmsId": "SH007933000000" + }, + "rating": null, + "seriesId": "SH00793300", + "startTime": "2026-09-13T03:00:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + }, + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "94", + "endTime": "2026-09-13T05:03:00Z", + "filter": [], + "flag": [], + "program": { + "episode": "14", + "episodeTitle": "Ryan Gosling; Gorillaz", + "id": "EP000037101609", + "isGeneric": "0", + "releaseYear": null, + "season": "51", + "seriesId": "SH00003710", + "shortDesc": "Host Ryan Gosling; Gorillaz perform.", + "title": "Saturday Night Live", + "tmsId": "EP000037101609" + }, + "rating": "TV-14", + "seriesId": "SH00003710", + "startTime": "2026-09-13T03:29:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p53158_placeholder" + } + ], + "id": "531580", + "stationFilters": [ + "filter-sports", + "filter-news" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/53158.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "BOUNCE TV", + "callSign": "WSYRDT3", + "channelId": "84540", + "channelNo": "9.3", + "events": [ + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "150", + "endTime": "2026-09-13T01:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV015399200000", + "isGeneric": "1", + "releaseYear": "2021", + "season": null, + "seriesId": "MV01539920", + "shortDesc": "Rescued as a child by the legendary assassin Moody, Anna is ", + "title": "The Prot\u00e9g\u00e9", + "tmsId": "MV015399200000" + }, + "rating": "TV-14", + "seriesId": "MV01539920", + "startTime": "2026-09-12T23:00:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + }, + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "120", + "endTime": "2026-09-13T03:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV002273030000", + "isGeneric": 0, + "releaseYear": "2009", + "season": null, + "seriesId": "MV00227303", + "shortDesc": "A rogue (Rain) enlists the help of the Europol agent (Naomie", + "title": "Ninja Assassin", + "tmsId": "MV002273030000" + }, + "rating": "TV-14", + "seriesId": "MV00227303", + "startTime": "2026-09-13T01:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + }, + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "120", + "endTime": "2026-09-13T05:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV003756000000", + "isGeneric": "0", + "releaseYear": "2012", + "season": null, + "seriesId": "MV00375600", + "shortDesc": "After his partner is killed, a veteran hit man (Sylvester St", + "title": "Bullet to the Head", + "tmsId": "MV003756000000" + }, + "rating": "TV-14", + "seriesId": "MV00375600", + "startTime": "2026-09-13T03:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + } + ], + "id": "8454012", + "stationFilters": [ + "filter-movie" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/84540.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "PBS KIDS HD", + "callSign": "WCNYDT4", + "channelId": "49216", + "channelNo": "24.4", + "events": [ + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T00:30:00Z", + "filter": [ + "filter-family" + ], + "flag": [ + "New", + "Finale" + ], + "program": { + "episode": "30", + "episodeTitle": "The Square Dance; The Process", + "id": "EP052844640100", + "isGeneric": "0", + "releaseYear": null, + "season": "1", + "seriesId": "SH05284464", + "shortDesc": "Dylan realizes he doesn't love square-dancing; Carl teaches ", + "title": "Carl the Collector", + "tmsId": "EP052844640100" + }, + "rating": "TV-Y", + "seriesId": "SH05284464", + "startTime": "2026-09-13T00:00:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p49216_placeholder" + }, + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T01:00:00Z", + "filter": [ + "filter-family" + ], + "flag": [ + "Premiere" + ], + "program": { + "episode": "31", + "episodeTitle": "The Bark Banquet; Carl's Collection of To-Dos", + "id": "EP052844640101", + "isGeneric": "0", + "releaseYear": null, + "season": "1", + "seriesId": "SH05284464", + "shortDesc": "Carl wants to start his own family tradition after Sheldon i", + "title": "Carl the Collector", + "tmsId": "EP052844640101" + }, + "rating": "TV-Y", + "seriesId": "SH05284464", + "startTime": "2026-09-13T00:30:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p49216_placeholder" + }, + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T01:30:00Z", + "filter": [ + "filter-family" + ], + "flag": [], + "program": { + "episode": "6", + "episodeTitle": "Buzz and the Tree", + "id": "EP003618800155", + "isGeneric": "0", + "releaseYear": null, + "season": "13", + "seriesId": "SH00361880", + "shortDesc": "It's Arbor Day in Cyberspace, but Buzz is more interested in", + "title": "Cyberchase", + "tmsId": "EP003618800155" + }, + "rating": "TV-Y", + "seriesId": "SH00361880", + "startTime": "2026-09-13T01:00:00Z", + "tags": [ + "Stereo", + "CC", + "DVS" + ], + "thumbnail": "p49216_placeholder" + } + ], + "id": "4921632", + "stationFilters": [ + "filter-family" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/49216.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "NATIONAL BROADCASTING COMPANY", + "callSign": "WKTVDT", + "channelId": "53158", + "channelNo": "1002", + "events": [ + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "210", + "endTime": "2026-09-13T03:00:00Z", + "filter": [ + "filter-sports" + ], + "flag": [ + "Live", + "New" + ], + "program": { + "episode": null, + "episodeTitle": "Iowa State at Iowa", + "id": "EP062700990343", + "isGeneric": "0", + "releaseYear": null, + "season": null, + "seriesId": "SH06270099", + "shortDesc": "The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at ", + "title": "College Football", + "tmsId": "EP062700990343" + }, + "rating": null, + "seriesId": "SH06270099", + "startTime": "2026-09-12T23:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + } + ], + "id": "5315899", + "stationFilters": [ + "filter-sports", + "filter-news" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/53158.png?w=55" + } + ] +} \ No newline at end of file diff --git a/web/web.go b/web/web.go index c4160e3..1d7e1d6 100644 --- a/web/web.go +++ b/web/web.go @@ -32,12 +32,17 @@ type GridResponse struct { } type JSONChannel struct { - ChannelID string `json:"channelId"` - ChannelNo string `json:"channelNo"` - CallSign string `json:"callSign"` - AffiliateName string `json:"affiliateName"` - Thumbnail string `json:"thumbnail"` - Events []JSONEvent `json:"events"` + ChannelID string `json:"channelId"` + // ID is the station ID with the row index appended. It identifies a row + // within one response, not a stable lineup placement. + ID string `json:"id"` + ChannelNo string `json:"channelNo"` + CallSign string `json:"callSign"` + AffiliateName string `json:"affiliateName"` + AffiliateCallSign string `json:"affiliateCallSign"` + StationFilters []string `json:"stationFilters"` + Thumbnail string `json:"thumbnail"` + Events []JSONEvent `json:"events"` } type JSONEvent struct { @@ -54,12 +59,15 @@ type JSONEvent struct { } type JSONProgram struct { - ID string `json:"id"` - Title string `json:"title"` - EpisodeTitle *string `json:"episodeTitle"` - ShortDesc *string `json:"shortDesc"` - Season *string `json:"season"` - Episode *string `json:"episode"` + ID string `json:"id"` + TmsID string `json:"tmsId"` + Title string `json:"title"` + EpisodeTitle *string `json:"episodeTitle"` + ShortDesc *string `json:"shortDesc"` + Season *string `json:"season"` + Episode *string `json:"episode"` + ReleaseYear FlexString `json:"releaseYear"` + IsGeneric FlexBool `json:"isGeneric"` } type Preferences struct { diff --git a/web/web_decode_test.go b/web/web_decode_test.go new file mode 100644 index 0000000..e1af3b7 --- /dev/null +++ b/web/web_decode_test.go @@ -0,0 +1,125 @@ +package web + +import ( + "encoding/json" + "os" + "testing" +) + +func TestGridFixtureDecodesRawFields(t *testing.T) { + data, err := os.ReadFile("testdata/grid_sample.json") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var grid GridResponse + if err := json.Unmarshal(data, &grid); err != nil { + t.Fatalf("decode fixture: %v", err) + } + if len(grid.Channels) != 4 { + t.Fatalf("channels = %d, want 4", len(grid.Channels)) + } + + first := grid.Channels[0] + if first.ID != "531580" || first.ChannelID != "53158" { + t.Fatalf("row id/station id = %q/%q", first.ID, first.ChannelID) + } + if first.AffiliateCallSign != "null" { + t.Fatalf("affiliateCallSign should decode verbatim, got %q", first.AffiliateCallSign) + } + if len(first.StationFilters) != 2 || first.StationFilters[0] != "filter-sports" { + t.Fatalf("stationFilters = %v", first.StationFilters) + } + + // Same station at a second position decodes as its own row. + last := grid.Channels[3] + if last.ChannelID != first.ChannelID || last.ChannelNo == first.ChannelNo || last.ID == first.ID { + t.Fatalf("duplicate position not preserved: %+v", last) + } + + ev := first.Events[0].Program + if ev.TmsID == "" { + t.Fatal("tmsId missing") + } + if ev.ReleaseYear != "2019" { + t.Fatalf("numeric releaseYear = %q, want 2019", ev.ReleaseYear) + } + if first.Events[1].Program.ReleaseYear != "2021" { + t.Fatalf("string releaseYear = %q, want 2021", first.Events[1].Program.ReleaseYear) + } + if first.Events[2].Program.ReleaseYear != "" { + t.Fatalf("null releaseYear = %q, want empty", first.Events[2].Program.ReleaseYear) + } + second := grid.Channels[1] + if !bool(second.Events[0].Program.IsGeneric) { + t.Fatal(`isGeneric "1" should decode true`) + } + if bool(second.Events[1].Program.IsGeneric) { + t.Fatal("isGeneric 0 should decode false") + } +} + +func TestFlexString(t *testing.T) { + cases := []struct { + in string + want string + }{ + {`"2019"`, "2019"}, + {`2019`, "2019"}, + {`2019.0`, "2019.0"}, + {`null`, ""}, + {`true`, "true"}, + {`""`, ""}, + {`[1]`, ""}, + {`{"a":1}`, ""}, + } + for _, c := range cases { + var doc struct { + V FlexString `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":`+c.in+`}`), &doc); err != nil { + t.Fatalf("%s: unexpected error %v", c.in, err) + } + if string(doc.V) != c.want { + t.Errorf("%s: got %q want %q", c.in, doc.V, c.want) + } + } + var doc struct { + V FlexString `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":"unterminated}`), &doc); err == nil { + t.Fatal("malformed JSON should still error") + } +} + +func TestFlexBool(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {`true`, true}, + {`false`, false}, + {`1`, true}, + {`0`, false}, + {`2`, true}, + {`"1"`, true}, + {`"0"`, false}, + {`"true"`, true}, + {`"TRUE"`, true}, + {`"false"`, false}, + {`"yes"`, true}, + {`"maybe"`, false}, + {`null`, false}, + {`[true]`, false}, + } + for _, c := range cases { + var doc struct { + V FlexBool `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":`+c.in+`}`), &doc); err != nil { + t.Fatalf("%s: unexpected error %v", c.in, err) + } + if bool(doc.V) != c.want { + t.Errorf("%s: got %v want %v", c.in, doc.V, c.want) + } + } +} diff --git a/xmltv_golden_test.go b/xmltv_golden_test.go new file mode 100644 index 0000000..d4f67e1 --- /dev/null +++ b/xmltv_golden_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "path/filepath" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +var updateGolden = flag.Bool("update", false, "rewrite testdata/xmlguide_golden.xmltv from the current conversion path") + +const ( + gridFixturePath = "web/testdata/grid_sample.json" + xmltvGoldenPath = "testdata/xmlguide_golden.xmltv" + fixtureLanguage = "en-us" + fixtureCountry = "USA" +) + +// loadGridFixture decodes the shared grid sample used by the golden and +// conversion tests. +func loadGridFixture(t *testing.T) web.GridResponse { + t.Helper() + data, err := os.ReadFile(gridFixturePath) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var grid web.GridResponse + if err := json.Unmarshal(data, &grid); err != nil { + t.Fatalf("decode fixture: %v", err) + } + return grid +} + +// guideFromGrid mirrors the scrape loop's conversion: channels deduplicated by +// station ID in first-seen order, events deduplicated by station, start, and end. +func guideFromGrid(grid web.GridResponse) *guide.TVGuide { + seenChannel := make(map[string]bool) + seenEvent := make(map[string]bool) + g := &guide.TVGuide{} + for _, ch := range grid.Channels { + if !seenChannel[ch.ChannelID] { + seenChannel[ch.ChannelID] = true + g.Channels = append(g.Channels, guide.ConvertChannel(ch)) + } + for _, ev := range ch.Events { + key := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime + if seenEvent[key] { + continue + } + seenEvent[key] = true + g.Programs = append(g.Programs, guide.ConvertEvent(ev, ch.ChannelID, fixtureLanguage, fixtureCountry)) + } + } + return g +} + +// TestXMLTVGolden guards the promise that model changes never alter XMLTV +// output. Regenerate deliberately with: go test -run TestXMLTVGolden -update +func TestXMLTVGolden(t *testing.T) { + g := guideFromGrid(loadGridFixture(t)) + + var rendered bytes.Buffer + if err := renderXMLTV(&rendered, g); err != nil { + t.Fatalf("render: %v", err) + } + + if *updateGolden { + if err := os.MkdirAll(filepath.Dir(xmltvGoldenPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(xmltvGoldenPath, rendered.Bytes(), 0644); err != nil { + t.Fatal(err) + } + t.Logf("wrote %s (%d bytes)", xmltvGoldenPath, rendered.Len()) + return + } + + want, err := os.ReadFile(xmltvGoldenPath) + if err != nil { + t.Fatalf("read golden (run with -update to create it): %v", err) + } + if !bytes.Equal(want, rendered.Bytes()) { + t.Fatalf("XMLTV output changed from golden.\nIf this is intended, run: go test -run TestXMLTVGolden -update\n--- got ---\n%s", rendered.String()) + } +}