fix(#40): Resolve cursor pagination and index coverage issues

backend/internal/db/db.go

@@ -155,20 +155,30 @@ func Init(cfg *config.Config) error {
 	}
 
 	// Create composite indexes for optimal query performance
-	// These support the common query patterns: WHERE state='live' AND deadline >= NOW() ORDER BY ...
-	// Including deadline in the index allows PostgreSQL to filter efficiently before sorting
+	// All queries filter WHERE state='live' AND deadline >= NOW(), so deadline comes first
+	// to exclude expired rows early and avoid table scans as expired data grows
+	//
+	// Note: PR #38 created indexes without deadline predicate. We drop and recreate
+	// to ensure upgraded databases get the improved definitions.
+	// Using partial indexes with WHERE state='live' to reduce index size.
 	if err := DB.Exec(`
+		-- Drop old indexes from PR #38 (missing deadline predicate)
+		DROP INDEX IF EXISTS idx_campaigns_trending;
+		DROP INDEX IF EXISTS idx_campaigns_newest;
+		DROP INDEX IF EXISTS idx_campaigns_ending;
+		DROP INDEX IF EXISTS idx_campaigns_category_trending;
+
 		-- Trending/Hot queries (with and without category filter)
-		CREATE INDEX IF NOT EXISTS idx_campaigns_trending 
+		CREATE INDEX idx_campaigns_trending 
 		ON campaigns(state, deadline, velocity_24h DESC, percent_funded DESC) 
 		WHERE state = 'live';
 
-		CREATE INDEX IF NOT EXISTS idx_campaigns_category_trending 
+		CREATE INDEX idx_campaigns_category_trending 
 		ON campaigns(state, deadline, category_id, velocity_24h DESC, percent_funded DESC) 
 		WHERE state = 'live';
 
 		-- Newest queries (with and without category filter)
-		CREATE INDEX IF NOT EXISTS idx_campaigns_newest 
+		CREATE INDEX idx_campaigns_newest 
 		ON campaigns(state, deadline, first_seen_at DESC) 
 		WHERE state = 'live';
 
@@ -177,7 +187,7 @@ func Init(cfg *config.Config) error {
 		WHERE state = 'live';
 
 		-- Ending queries (with and without category filter)
-		CREATE INDEX IF NOT EXISTS idx_campaigns_ending 
+		CREATE INDEX idx_campaigns_ending 
 		ON campaigns(state, deadline ASC) 
 		WHERE state = 'live';
 

backend/internal/handler/campaigns.go

@@ -2,8 +2,10 @@ package handler
 
 import (
 	"encoding/base64"
+	"log"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -14,6 +16,7 @@ import (
 
 var sortMap = map[string]string{
 	"trending": "MAGIC",
+	"hot":      "MAGIC", // Fallback uses MAGIC for hot sort (close approximation)
 	"newest":   "NEWEST",
 	"ending":   "END_DATE",
 }
@@ -35,10 +38,22 @@ func ListCampaigns(client *service.KickstarterScrapingService) gin.HandlerFunc {
 		// - hot: velocity_24h (our metric)
 		// - ending: deadline (exact from Kickstarter)
 		if db.IsEnabled() {
+			// Detect cursor source: ScrapingBee uses "page:N", DB uses base64 offsets
+			// If cursor is from ScrapingBee, fall through to ScrapingBee to maintain format
+			if cursor != "" && strings.HasPrefix(cursor, "page:") {
+				// ScrapingBee cursor format detected - fall through to ScrapingBee path
+				// to avoid format mismatch (cannot mix base64 offsets with page numbers)
+				goto useScrapingBee
+			}
+
 			offset := 0
 			if cursor != "" {
 				if decoded, err := base64.StdEncoding.DecodeString(cursor); err == nil {
 					offset, _ = strconv.Atoi(string(decoded))
+				} else {
+					// Invalid cursor format - treat as offset 0 but log warning
+					// This shouldn't happen if client respects cursor format
+					log.Printf("ListCampaigns: invalid cursor format %q, treating as offset 0", cursor)
 				}
 			}
 
@@ -64,28 +79,35 @@ func ListCampaigns(client *service.KickstarterScrapingService) gin.HandlerFunc {
 				q = q.Where("category_id = ?", categoryID)
 			}
 
-			// Only return DB results if we have data; otherwise fall through to ScrapingBee
-			if err := q.Find(&campaigns).Error; err == nil && len(campaigns) > 0 {
-				hasMore := len(campaigns) > limit
-				if hasMore {
-					campaigns = campaigns[:limit]
+			// Return DB results if we have data
+			// Note: Once using DB cursors, always use DB to maintain cursor format consistency
+			if err := q.Find(&campaigns).Error; err == nil {
+				// Return DB results if we have data, OR if we're paginating with a DB cursor
+				// (cursor != "" and not a ScrapingBee cursor means it's a DB cursor)
+				if len(campaigns) > 0 || cursor != "" {
+					hasMore := len(campaigns) > limit
+					if hasMore {
+						campaigns = campaigns[:limit]
+					}
+
+					var nextCursor interface{}
+					if hasMore {
+						nextOffset := offset + limit
+						nextCursor = base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(nextOffset)))
+					}
+
+					// Don't include total for DB queries (we don't track global count)
+					c.JSON(http.StatusOK, gin.H{"campaigns": campaigns, "next_cursor": nextCursor})
+					return
 				}
-
-				var nextCursor interface{}
-				if hasMore {
-					nextOffset := offset + limit
-					nextCursor = base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(nextOffset)))
-				}
-
-				// Don't include total for DB queries (we don't track global count)
-				c.JSON(http.StatusOK, gin.H{"campaigns": campaigns, "next_cursor": nextCursor})
-				return
 			}
-			// If DB query fails or returns empty, fall through to ScrapingBee fallback
+			// Only fall through to ScrapingBee on first load (cursor == "") and DB empty/failed
 		}
 
-		// ScrapingBee fallback only for:
-		// - DB unavailable or empty (cold start, failed query)
+	useScrapingBee:
+		// ScrapingBee fallback for:
+		// - First load (cursor == "") when DB is unavailable or empty (cold start)
+		// - ScrapingBee pagination (cursor starts with "page:")
 		// - SearchCampaigns endpoint (user search with query text)
 		gqlSort, ok := sortMap[sort]
 		if !ok {