diff --git a/backend/internal/db/db.go b/backend/internal/db/db.go index 73ba171..cb2ced1 100644 --- a/backend/internal/db/db.go +++ b/backend/internal/db/db.go @@ -155,20 +155,30 @@ func Init(cfg *config.Config) error { } // Create composite indexes for optimal query performance - // These support the common query patterns: WHERE state='live' AND deadline >= NOW() ORDER BY ... - // Including deadline in the index allows PostgreSQL to filter efficiently before sorting + // All queries filter WHERE state='live' AND deadline >= NOW(), so deadline comes first + // to exclude expired rows early and avoid table scans as expired data grows + // + // Note: PR #38 created indexes without deadline predicate. We drop and recreate + // to ensure upgraded databases get the improved definitions. + // Using partial indexes with WHERE state='live' to reduce index size. if err := DB.Exec(` + -- Drop old indexes from PR #38 (missing deadline predicate) + DROP INDEX IF EXISTS idx_campaigns_trending; + DROP INDEX IF EXISTS idx_campaigns_newest; + DROP INDEX IF EXISTS idx_campaigns_ending; + DROP INDEX IF EXISTS idx_campaigns_category_trending; + -- Trending/Hot queries (with and without category filter) - CREATE INDEX IF NOT EXISTS idx_campaigns_trending + CREATE INDEX idx_campaigns_trending ON campaigns(state, deadline, velocity_24h DESC, percent_funded DESC) WHERE state = 'live'; - CREATE INDEX IF NOT EXISTS idx_campaigns_category_trending + CREATE INDEX idx_campaigns_category_trending ON campaigns(state, deadline, category_id, velocity_24h DESC, percent_funded DESC) WHERE state = 'live'; -- Newest queries (with and without category filter) - CREATE INDEX IF NOT EXISTS idx_campaigns_newest + CREATE INDEX idx_campaigns_newest ON campaigns(state, deadline, first_seen_at DESC) WHERE state = 'live'; @@ -177,7 +187,7 @@ func Init(cfg *config.Config) error { WHERE state = 'live'; -- Ending queries (with and without category filter) - CREATE INDEX IF NOT EXISTS idx_campaigns_ending + CREATE INDEX idx_campaigns_ending ON campaigns(state, deadline ASC) WHERE state = 'live'; diff --git a/backend/internal/handler/campaigns.go b/backend/internal/handler/campaigns.go index a6a367f..93f559b 100644 --- a/backend/internal/handler/campaigns.go +++ b/backend/internal/handler/campaigns.go @@ -2,8 +2,10 @@ package handler import ( "encoding/base64" + "log" "net/http" "strconv" + "strings" "time" "github.com/gin-gonic/gin" @@ -14,6 +16,7 @@ import ( var sortMap = map[string]string{ "trending": "MAGIC", + "hot": "MAGIC", // Fallback uses MAGIC for hot sort (close approximation) "newest": "NEWEST", "ending": "END_DATE", } @@ -35,10 +38,22 @@ func ListCampaigns(client *service.KickstarterScrapingService) gin.HandlerFunc { // - hot: velocity_24h (our metric) // - ending: deadline (exact from Kickstarter) if db.IsEnabled() { + // Detect cursor source: ScrapingBee uses "page:N", DB uses base64 offsets + // If cursor is from ScrapingBee, fall through to ScrapingBee to maintain format + if cursor != "" && strings.HasPrefix(cursor, "page:") { + // ScrapingBee cursor format detected - fall through to ScrapingBee path + // to avoid format mismatch (cannot mix base64 offsets with page numbers) + goto useScrapingBee + } + offset := 0 if cursor != "" { if decoded, err := base64.StdEncoding.DecodeString(cursor); err == nil { offset, _ = strconv.Atoi(string(decoded)) + } else { + // Invalid cursor format - treat as offset 0 but log warning + // This shouldn't happen if client respects cursor format + log.Printf("ListCampaigns: invalid cursor format %q, treating as offset 0", cursor) } } @@ -64,28 +79,35 @@ func ListCampaigns(client *service.KickstarterScrapingService) gin.HandlerFunc { q = q.Where("category_id = ?", categoryID) } - // Only return DB results if we have data; otherwise fall through to ScrapingBee - if err := q.Find(&campaigns).Error; err == nil && len(campaigns) > 0 { - hasMore := len(campaigns) > limit - if hasMore { - campaigns = campaigns[:limit] + // Return DB results if we have data + // Note: Once using DB cursors, always use DB to maintain cursor format consistency + if err := q.Find(&campaigns).Error; err == nil { + // Return DB results if we have data, OR if we're paginating with a DB cursor + // (cursor != "" and not a ScrapingBee cursor means it's a DB cursor) + if len(campaigns) > 0 || cursor != "" { + hasMore := len(campaigns) > limit + if hasMore { + campaigns = campaigns[:limit] + } + + var nextCursor interface{} + if hasMore { + nextOffset := offset + limit + nextCursor = base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(nextOffset))) + } + + // Don't include total for DB queries (we don't track global count) + c.JSON(http.StatusOK, gin.H{"campaigns": campaigns, "next_cursor": nextCursor}) + return } - - var nextCursor interface{} - if hasMore { - nextOffset := offset + limit - nextCursor = base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(nextOffset))) - } - - // Don't include total for DB queries (we don't track global count) - c.JSON(http.StatusOK, gin.H{"campaigns": campaigns, "next_cursor": nextCursor}) - return } - // If DB query fails or returns empty, fall through to ScrapingBee fallback + // Only fall through to ScrapingBee on first load (cursor == "") and DB empty/failed } - // ScrapingBee fallback only for: - // - DB unavailable or empty (cold start, failed query) + useScrapingBee: + // ScrapingBee fallback for: + // - First load (cursor == "") when DB is unavailable or empty (cold start) + // - ScrapingBee pagination (cursor starts with "page:") // - SearchCampaigns endpoint (user search with query text) gqlSort, ok := sortMap[sort] if !ok {