From 46b355ae79737c78c4d4619e2b0a8dbc86e3e883 Mon Sep 17 00:00:00 2001 From: Joshua Higgins Date: Tue, 17 Nov 2026 10:44:19 -0500 Subject: [PATCH] fix(convex): keep large libraries under Convex array and read limits Whole-library and favorites queries return lists split into chunks of at most 8,192 values, and search keeps the top 500 per section. Favorite writes and song detail lookups read whole ranges instead of one id at a time, and large unfavorites finish in scheduled batches, so they stay under Convex's 4,096 reads per function. Co-Authored-By: Claude Opus 5.5 --- .../(app)/(tabs)/home/playlist/favorites.tsx | 6 +- .../components/app-intents/catalog-sync.ts | 9 +- .../mobile/components/app-intents/dispatch.ts | 12 +- apps/mobile/contexts/add-to-playlist.tsx | 2 +- apps/mobile/contexts/favorites.tsx | 6 +- apps/mobile/contexts/playback.tsx | 16 +- apps/mobile/hooks/use-media-menu-actions.ts | 2 +- apps/web/app/(app)/(home)/page.tsx | 2 +- apps/web/app/(app)/library/albums/page.tsx | 2 +- apps/web/app/(app)/recently-added/page.tsx | 2 +- .../components/add-to-playlist-provider.tsx | 2 +- apps/web/components/artists-page.tsx | 4 +- apps/web/components/favorites-provider.tsx | 6 +- apps/web/components/playback-provider.tsx | 16 +- apps/web/components/playlist-page.tsx | 4 +- apps/web/hooks/use-media-actions.ts | 2 +- packages/convex/client/favorites.ts | 71 ++++----- .../convex/app-intent-playlists.test.cjs | 2 +- packages/convex/convex/favoriteStore.ts | 144 ++++++++++++++---- packages/convex/convex/favorites.ts | 66 ++++---- packages/convex/convex/library.ts | 89 ++++++++--- packages/convex/convex/limits.ts | 17 +++ packages/convex/convex/playlists.ts | 5 +- packages/convex/convex/types.ts | 6 + 24 files changed, 324 insertions(+), 169 deletions(-) diff --git a/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx b/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx index 68e5e64..a19835c 100644 --- a/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx +++ b/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx @@ -36,7 +36,7 @@ export default function FavoritesScreen() { const colors = useThemeColors() const { config } = useServerConfig() const favorites = useCachedQuery(api.favorites.browseFavoritesPlaylistDetail) - const songs = favorites?.songs ?? [] + const songs = useMemo(() => favorites?.songs.flat() ?? [], [favorites]) const { playInOrder, playShuffled, playFromSong } = usePlaybackActions() const { isSongFavorite } = useFavorites() const { handleSongAction } = useMediaMenuActions() @@ -45,8 +45,8 @@ export default function FavoritesScreen() { usePlaylistSearch(songs) useEffect(() => { - if (favorites) rememberSongs(favorites.songs) - }, [favorites]) + rememberSongs(songs) + }, [songs]) const handlePlaySong = useCallback( (song: BrowseSong) => void playFromSong(song.id, songIds), diff --git a/apps/mobile/components/app-intents/catalog-sync.ts b/apps/mobile/components/app-intents/catalog-sync.ts index 4d4c4fe..91714ce 100644 --- a/apps/mobile/components/app-intents/catalog-sync.ts +++ b/apps/mobile/components/app-intents/catalog-sync.ts @@ -23,19 +23,16 @@ type LibraryKind = (typeof LIBRARY_KINDS)[number] const PAGE_SIZE = 500 type CatalogPage = FunctionReturnType -type Favorites = { - favoriteSongs: string[] - favoriteAlbums: string[] - favoriteArtists: string[] -} +type Favorites = FunctionReturnType function favoriteIds(favorites: Favorites, kind: LibraryKind) { return new Set( - kind === "song" + (kind === "song" ? favorites.favoriteSongs : kind === "album" ? favorites.favoriteAlbums : favorites.favoriteArtists + ).flat() ) } diff --git a/apps/mobile/components/app-intents/dispatch.ts b/apps/mobile/components/app-intents/dispatch.ts index 67fbe2f..4424809 100644 --- a/apps/mobile/components/app-intents/dispatch.ts +++ b/apps/mobile/components/app-intents/dispatch.ts @@ -214,12 +214,13 @@ export function useIntentDispatch() { ) } if (kind === "artist") { - return fetchCachedQuery( + const chunks = await fetchCachedQuery( convex, cacheScope, api.playlists.getOrderedArtistSongIds, { artistId: id as Id<"artists"> } ) + return chunks.flat() } if (kind === "playlist") { if (id === FAVORITES_PLAYLIST_ID) { @@ -229,7 +230,7 @@ export function useIntentDispatch() { api.favorites.getUserFavorites, {} ) - return favorites.favoriteSongs + return favorites.favoriteSongs.flat() } return fetchCachedQuery( convex, @@ -391,17 +392,18 @@ export function useIntentDispatch() { api.favorites.getUserFavorites, {} ) - if (favorites.favoriteSongs.length === 0) { + const favoriteSongs = favorites.favoriteSongs.flat() + if (favoriteSongs.length === 0) { throw new IntentRejected( "Nothing is queued, and your favorites are empty." ) } if (shuffled) { ensureActive(invocation) - await actions.playShuffled(favorites.favoriteSongs) + await actions.playShuffled(favoriteSongs) } else { ensureActive(invocation) - await actions.playInOrder(favorites.favoriteSongs) + await actions.playInOrder(favoriteSongs) } return } diff --git a/apps/mobile/contexts/add-to-playlist.tsx b/apps/mobile/contexts/add-to-playlist.tsx index 817717f..5abdb6b 100644 --- a/apps/mobile/contexts/add-to-playlist.tsx +++ b/apps/mobile/contexts/add-to-playlist.tsx @@ -150,7 +150,7 @@ export function AddToPlaylistProvider({ children }: { children: ReactNode }) { { artistId } ) if (selectionRequestIdRef.current !== requestId) return - setPendingSongIds(songIds) + setPendingSongIds(songIds.flat()) setOpen(true) } catch (error) { if (selectionRequestIdRef.current !== requestId) return diff --git a/apps/mobile/contexts/favorites.tsx b/apps/mobile/contexts/favorites.tsx index 4befb5b..f1213df 100644 --- a/apps/mobile/contexts/favorites.tsx +++ b/apps/mobile/contexts/favorites.tsx @@ -100,15 +100,15 @@ export function FavoritesProvider({ children }: { children: ReactNode }) { ) const songIds = useMemo( - () => new Set(favorites?.favoriteSongs.map(String) ?? []), + () => new Set(favorites?.favoriteSongs.flat()), [favorites?.favoriteSongs] ) const albumIds = useMemo( - () => new Set(favorites?.favoriteAlbums.map(String) ?? []), + () => new Set(favorites?.favoriteAlbums.flat()), [favorites?.favoriteAlbums] ) const artistIds = useMemo( - () => new Set(favorites?.favoriteArtists.map(String) ?? []), + () => new Set(favorites?.favoriteArtists.flat()), [favorites?.favoriteArtists] ) const idsRef = useRef({ songIds, albumIds, artistIds }) diff --git a/apps/mobile/contexts/playback.tsx b/apps/mobile/contexts/playback.tsx index 43aae8d..40ee2c5 100644 --- a/apps/mobile/contexts/playback.tsx +++ b/apps/mobile/contexts/playback.tsx @@ -401,7 +401,7 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { cacheScope, api.playlists.getOrderedArtistSongIds, { artistId } - ), + ).then((chunks) => chunks.flat()), recorded: { entity: { kind: "artist", id: artistId } }, })) const playlist = collection( @@ -482,12 +482,14 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { let songIds: Id<"songs">[] try { - songIds = await fetchCachedQuery( - convex, - cacheScope, - api.library.browseSortedSongIds, - sort - ) + songIds = ( + await fetchCachedQuery( + convex, + cacheScope, + api.library.browseSortedSongIds, + sort + ) + ).flat() } catch { return } diff --git a/apps/mobile/hooks/use-media-menu-actions.ts b/apps/mobile/hooks/use-media-menu-actions.ts index b367040..5d26f56 100644 --- a/apps/mobile/hooks/use-media-menu-actions.ts +++ b/apps/mobile/hooks/use-media-menu-actions.ts @@ -173,7 +173,7 @@ export function useMediaMenuActions() { api.playlists.getOrderedArtistSongIds, { artistId: artist.id as Id<"artists"> } ) - await runQueueAction(action, songIds) + await runQueueAction(action, songIds.flat()) break } default: diff --git a/apps/web/app/(app)/(home)/page.tsx b/apps/web/app/(app)/(home)/page.tsx index fa3adf3..6af2750 100644 --- a/apps/web/app/(app)/(home)/page.tsx +++ b/apps/web/app/(app)/(home)/page.tsx @@ -32,7 +32,7 @@ export default function Page() { diff --git a/apps/web/app/(app)/library/albums/page.tsx b/apps/web/app/(app)/library/albums/page.tsx index f4d361f..37759e2 100644 --- a/apps/web/app/(app)/library/albums/page.tsx +++ b/apps/web/app/(app)/library/albums/page.tsx @@ -18,7 +18,7 @@ export default function Page() { ) : (
- {albums.map((album) => ( + {albums.flat().map((album) => ( ))}
diff --git a/apps/web/app/(app)/recently-added/page.tsx b/apps/web/app/(app)/recently-added/page.tsx index 919d85d..cd20d80 100644 --- a/apps/web/app/(app)/recently-added/page.tsx +++ b/apps/web/app/(app)/recently-added/page.tsx @@ -18,7 +18,7 @@ export default function Page() { ) : (
- {albums.map((album) => ( + {albums.flat().map((album) => ( ))}
diff --git a/apps/web/components/add-to-playlist-provider.tsx b/apps/web/components/add-to-playlist-provider.tsx index c215e54..5cc1ad8 100644 --- a/apps/web/components/add-to-playlist-provider.tsx +++ b/apps/web/components/add-to-playlist-provider.tsx @@ -184,7 +184,7 @@ export function AddToPlaylistProvider({ } ) if (requestId === selectionRequestRef.current) { - showAddToPlaylist(songIds) + showAddToPlaylist(songIds.flat()) } } catch (error) { if (requestId === selectionRequestRef.current) { diff --git a/apps/web/components/artists-page.tsx b/apps/web/components/artists-page.tsx index 645a54a..744ab95 100644 --- a/apps/web/components/artists-page.tsx +++ b/apps/web/components/artists-page.tsx @@ -32,8 +32,8 @@ export function ArtistsPage() { ) : ( diff --git a/apps/web/components/favorites-provider.tsx b/apps/web/components/favorites-provider.tsx index cbd60e8..686d9a9 100644 --- a/apps/web/components/favorites-provider.tsx +++ b/apps/web/components/favorites-provider.tsx @@ -86,15 +86,15 @@ export function FavoritesProvider({ children }: { children: React.ReactNode }) { ) const songIds = React.useMemo( - () => new Set(favorites?.favoriteSongs.map(String) ?? []), + () => new Set(favorites?.favoriteSongs.flat()), [favorites?.favoriteSongs] ) const albumIds = React.useMemo( - () => new Set(favorites?.favoriteAlbums.map(String) ?? []), + () => new Set(favorites?.favoriteAlbums.flat()), [favorites?.favoriteAlbums] ) const artistIds = React.useMemo( - () => new Set(favorites?.favoriteArtists.map(String) ?? []), + () => new Set(favorites?.favoriteArtists.flat()), [favorites?.favoriteArtists] ) diff --git a/apps/web/components/playback-provider.tsx b/apps/web/components/playback-provider.tsx index e6582f6..e4e9cf2 100644 --- a/apps/web/components/playback-provider.tsx +++ b/apps/web/components/playback-provider.tsx @@ -1233,12 +1233,14 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { let songIds: Id<"songs">[] try { - songIds = await fetchCachedQuery( - convex, - cacheScope, - api.library.browseSortedSongIds, - sort - ) + songIds = ( + await fetchCachedQuery( + convex, + cacheScope, + api.library.browseSortedSongIds, + sort + ) + ).flat() } catch { return } @@ -1268,7 +1270,7 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { cacheScope, api.playlists.getOrderedArtistSongIds, { artistId } - ), + ).then((chunks) => chunks.flat()), [convex, cacheScope] ) diff --git a/apps/web/components/playlist-page.tsx b/apps/web/components/playlist-page.tsx index 07398b4..8e26494 100644 --- a/apps/web/components/playlist-page.tsx +++ b/apps/web/components/playlist-page.tsx @@ -34,5 +34,7 @@ export function FavoritesPlaylistPage() { return Sign in to view your favorites. } - return + return ( + + ) } diff --git a/apps/web/hooks/use-media-actions.ts b/apps/web/hooks/use-media-actions.ts index 9439842..07f8bcc 100644 --- a/apps/web/hooks/use-media-actions.ts +++ b/apps/web/hooks/use-media-actions.ts @@ -64,7 +64,7 @@ export function useMediaActions() { api.playlists.getOrderedArtistSongIds, { artistId: artistId as Id<"artists"> } ) - void runQueueAction(action, songIds) + void runQueueAction(action, songIds.flat()) break } } diff --git a/packages/convex/client/favorites.ts b/packages/convex/client/favorites.ts index 69d20eb..a905252 100644 --- a/packages/convex/client/favorites.ts +++ b/packages/convex/client/favorites.ts @@ -1,63 +1,54 @@ import type { OptimisticLocalStore, OptimisticUpdate } from "convex/browser" -import type { FunctionArgs, FunctionReference } from "convex/server" +import type { + FunctionArgs, + FunctionReference, + FunctionReturnType, +} from "convex/server" import { api } from "../convex/_generated/api" -import type { Id } from "../convex/_generated/dataModel" +import { chunk } from "../convex/limits" -type FavoriteIds = { - favoriteSongs: Id<"songs">[] - favoriteAlbums: Id<"albums">[] - favoriteArtists: Id<"artists">[] -} +type UserFavorites = FunctionReturnType +type FavoriteList = keyof UserFavorites -type FavoriteList = keyof FavoriteIds - -function patchFavorites( +/** Rewrites one list of the cached `getUserFavorites` result, if loaded. */ +function patchFavorites( store: OptimisticLocalStore, - list: List, - update: (ids: FavoriteIds[List]) => FavoriteIds[List] + list: FavoriteList, + update: (ids: string[]) => string[] ) { const current = store.getQuery(api.favorites.getUserFavorites, {}) if (!current) return - const next = update(current[list]) - if (next !== current[list]) { - store.setQuery( - api.favorites.getUserFavorites, - {}, - { - ...current, - [list]: next, - } - ) + const ids: string[] = current[list].flat() + const next = update(ids) + if (next !== ids) { + // `next` only holds ids of this list's kind, which TS cannot follow here. + store.setQuery(api.favorites.getUserFavorites, {}, { + ...current, + [list]: chunk(next), + } as UserFavorites) } } -function addFavorites( +function addFavorites( store: OptimisticLocalStore, - list: List, - ids: FavoriteIds[List] + list: FavoriteList, + ids: string[] ) { patchFavorites(store, list, (current) => { - const added = ids.filter((id) => !(current as string[]).includes(id)) - return added.length === 0 - ? current - : ([...new Set([...current, ...added])] as FavoriteIds[List]) + const added = ids.filter((id) => !current.includes(id)) + return added.length === 0 ? current : [...new Set([...current, ...added])] }) } -function removeFavorites( +function removeFavorites( store: OptimisticLocalStore, - list: List, - ids: FavoriteIds[List] + list: FavoriteList, + ids: string[] ) { - const removed = new Set(ids) - patchFavorites( - store, - list, - (current) => - (current as string[]).filter( - (id) => !removed.has(id) - ) as FavoriteIds[List] + const removed = new Set(ids) + patchFavorites(store, list, (current) => + current.filter((id) => !removed.has(id)) ) } diff --git a/packages/convex/convex/app-intent-playlists.test.cjs b/packages/convex/convex/app-intent-playlists.test.cjs index 701d969..3229c5a 100644 --- a/packages/convex/convex/app-intent-playlists.test.cjs +++ b/packages/convex/convex/app-intent-playlists.test.cjs @@ -29,7 +29,7 @@ function playlistMutation() { mutation: (definition) => definition, query: (definition) => definition, } - if (name === "./library") return {} + if (name === "./library" || name === "./limits") return {} throw new Error(`Unexpected import ${name}`) }, }) diff --git a/packages/convex/convex/favoriteStore.ts b/packages/convex/convex/favoriteStore.ts index a75d1a2..749937a 100644 --- a/packages/convex/convex/favoriteStore.ts +++ b/packages/convex/convex/favoriteStore.ts @@ -1,7 +1,10 @@ import type { WithoutSystemFields } from "convex/server" -import { ConvexError } from "convex/values" +import { ConvexError, v } from "convex/values" +import { internal } from "./_generated/api" import type { Doc, Id } from "./_generated/dataModel" import type { MutationCtx, QueryCtx } from "./_generated/server" +import { internalMutation } from "./_generated/server" +import { chunk, type Chunked } from "./limits" export type FavoriteKind = Doc<"favorites">["kind"] @@ -9,13 +12,18 @@ type FavoriteEntityId = { [Kind in FavoriteKind]: Extract, { kind: Kind }>["entityId"] } -/** The `favorites.getUserFavorites` shape, kept from when these lived on `users`. */ +/** A user's favorites, each list oldest first. */ export type FavoriteIds = { favoriteSongs: Id<"songs">[] favoriteAlbums: Id<"albums">[] favoriteArtists: Id<"artists">[] } +/** `FavoriteIds` as `favorites.getUserFavorites` returns it. */ +export type FavoriteIdChunks = { + [List in keyof FavoriteIds]: Chunked +} + function legacyFavorites(user: Doc<"users"> | null): FavoriteIds { return { favoriteSongs: user?.favoriteSongs ?? [], @@ -69,25 +77,59 @@ export async function readFavorites( return legacyFavorites(await ctx.db.get(userId)) } -function favoriteRow( +/** + * Above this many ids, one scan of the user's list beats a lookup per id, and + * keeps bulk writes under Convex's 4,096 reads per function. + */ +const MAX_POINT_LOOKUPS = 64 + +/** Maps each of `ids` the user has favorited to its row. */ +async function favoriteRows( ctx: QueryCtx, userId: Id<"users">, - entityId: FavoriteEntityId[FavoriteKind] -) { - return ctx.db - .query("favorites") - .withIndex("by_user_entity", (q) => - q.eq("userId", userId).eq("entityId", entityId) + kind: Kind, + ids: ReadonlySet +): Promise>> { + const rows = + ids.size <= MAX_POINT_LOOKUPS + ? await Promise.all( + [...ids].map((entityId) => + ctx.db + .query("favorites") + .withIndex("by_user_entity", (q) => + q.eq("userId", userId).eq("entityId", entityId) + ) + .unique() + ) + ) + : await ctx.db + .query("favorites") + .withIndex("by_user_kind", (q) => + q.eq("userId", userId).eq("kind", kind) + ) + .collect() + return new Map( + rows.flatMap((row) => + row && ids.has(row.entityId as FavoriteEntityId[Kind]) + ? [[row.entityId, row._id] as const] + : [] ) - .unique() + ) } -export async function isFavorite( +/** True when `ids` is non-empty and the user has favorited every one. */ +export async function allFavorited( ctx: QueryCtx, userId: Id<"users">, - entityId: FavoriteEntityId[FavoriteKind] + kind: Kind, + ids: FavoriteEntityId[Kind][] ) { - return (await favoriteRow(ctx, userId, entityId)) !== null + const unique = new Set(ids) + if (unique.size === 0) { + return false + } + const rows = await favoriteRows(ctx, userId, kind, unique) + return rows.size === unique.size } /** Appends ids that are not favorited yet, keeping existing ones in place. */ @@ -97,8 +139,10 @@ export async function addFavorites( kind: Kind, ids: FavoriteEntityId[Kind][] ) { - for (const entityId of new Set(ids)) { - if (!(await isFavorite(ctx, userId, entityId))) { + const unique = new Set(ids) + const existing = await favoriteRows(ctx, userId, kind, unique) + for (const entityId of unique) { + if (!existing.has(entityId)) { // TS cannot tie a generic `kind` to its `entityId` union member. await ctx.db.insert("favorites", { userId, @@ -109,19 +153,52 @@ export async function addFavorites( } } -export async function removeFavorites( +export async function removeFavorites( ctx: MutationCtx, userId: Id<"users">, - ids: Iterable + kind: Kind, + ids: FavoriteEntityId[Kind][] ) { - for (const entityId of new Set(ids)) { - const row = await favoriteRow(ctx, userId, entityId) - if (row) { - await ctx.db.delete(row._id) - } + const rows = await favoriteRows(ctx, userId, kind, new Set(ids)) + await deleteFavoriteRows(ctx, [...rows.values()]) +} + +/** + * Rows one call deletes before scheduling the rest. Convex counts each delete + * as a read, so unfavoriting a huge artist cannot happen in one mutation. + */ +const MAX_INLINE_DELETES = 1_000 + +async function deleteFavoriteRows(ctx: MutationCtx, rowIds: Id<"favorites">[]) { + for (const rowId of rowIds.slice(0, MAX_INLINE_DELETES)) { + await ctx.db.delete(rowId) + } + for (const rest of chunk(rowIds.slice(MAX_INLINE_DELETES))) { + await ctx.scheduler.runAfter(0, internal.favoriteStore.deleteRows, { + rowIds: rest, + }) } } +/** Finishes deletes `deleteFavoriteRows` could not fit in one mutation. */ +export const deleteRows = internalMutation({ + args: { rowIds: v.array(v.id("favorites")) }, + handler: async (ctx, { rowIds }) => { + // Rows can disappear in between, e.g. when the scanner purges a song. + for (const rowId of rowIds.slice(0, MAX_INLINE_DELETES)) { + if (await ctx.db.get(rowId)) { + await ctx.db.delete(rowId) + } + } + const rest = rowIds.slice(MAX_INLINE_DELETES) + if (rest.length > 0) { + await ctx.scheduler.runAfter(0, internal.favoriteStore.deleteRows, { + rowIds: rest, + }) + } + }, +}) + /** * Moves a user's legacy favorite arrays into `favorites` rows, keeping their * order, then clears them. Safe to call repeatedly. @@ -180,13 +257,18 @@ export async function deleteUserFavorites( ctx: MutationCtx, userId: Id<"users"> ) { - for (const kind of ["song", "album", "artist"] as const) { - const rows = await ctx.db - .query("favorites") - .withIndex("by_user_kind", (q) => q.eq("userId", userId).eq("kind", kind)) - .collect() - for (const row of rows) { - await ctx.db.delete(row._id) - } - } + const rows = await Promise.all( + (["song", "album", "artist"] as const).map((kind) => + ctx.db + .query("favorites") + .withIndex("by_user_kind", (q) => + q.eq("userId", userId).eq("kind", kind) + ) + .collect() + ) + ) + await deleteFavoriteRows( + ctx, + rows.flat().map((row) => row._id) + ) } diff --git a/packages/convex/convex/favorites.ts b/packages/convex/convex/favorites.ts index 960407f..b1ba701 100644 --- a/packages/convex/convex/favorites.ts +++ b/packages/convex/convex/favorites.ts @@ -5,14 +5,16 @@ import type { MutationCtx, QueryCtx } from "./_generated/server" import { mutation, query } from "./_generated/server" import { addFavorites, - isFavorite, + allFavorited, readFavorites, removeFavorites, requireMigratedFavorites, - type FavoriteIds, + type FavoriteIdChunks, } from "./favoriteStore" -import type { BrowsePlaylistDetail, BrowsePlaylistSummary } from "./library" +import type { BrowsePlaylistSummary } from "./library" import { browseSongsFromIds } from "./library" +import { chunk } from "./limits" +import type { BrowseFavoritesDetail } from "./types" /** Resolves the signed-in user and moves any legacy favorites into rows. */ async function requireFavoritesUser(ctx: MutationCtx) { @@ -48,30 +50,13 @@ async function getSongIdsForArtist(ctx: QueryCtx, artistId: Id<"artists">) { return songs.map((song) => song._id) } -/** True when `ids` is non-empty and every one of them is favorited. */ -async function allFavorited( - ctx: QueryCtx, - userId: Id<"users">, - ids: (Id<"songs"> | Id<"albums">)[] -) { - if (ids.length === 0) { - return false - } - for (const id of ids) { - if (!(await isFavorite(ctx, userId, id))) { - return false - } - } - return true -} - async function promoteArtistIfAllAlbumsFavorited( ctx: MutationCtx, userId: Id<"users">, artistId: Id<"artists"> ) { const albumIds = await getAlbumIdsForArtist(ctx, artistId) - if (await allFavorited(ctx, userId, albumIds)) { + if (await allFavorited(ctx, userId, "album", albumIds)) { await addFavorites(ctx, userId, "artist", [artistId]) } } @@ -84,7 +69,7 @@ async function cascadeUpFromSongs( ) { for (const albumId of new Set(songs.map((song) => song.album))) { const songIds = await getSongIdsForAlbum(ctx, albumId) - if (await allFavorited(ctx, userId, songIds)) { + if (await allFavorited(ctx, userId, "song", songIds)) { await addFavorites(ctx, userId, "album", [albumId]) } } @@ -95,12 +80,17 @@ async function cascadeUpFromSongs( export const getUserFavorites = query({ args: {}, - handler: async (ctx): Promise => { + handler: async (ctx): Promise => { const userId = await getAuthUserId(ctx) if (userId === null) { return { favoriteSongs: [], favoriteAlbums: [], favoriteArtists: [] } } - return readFavorites(ctx, userId) + const favorites = await readFavorites(ctx, userId) + return { + favoriteSongs: chunk(favorites.favoriteSongs), + favoriteAlbums: chunk(favorites.favoriteAlbums), + favoriteArtists: chunk(favorites.favoriteArtists), + } }, }) @@ -135,7 +125,7 @@ async function browseFavoritesSummaryHandler( export const browseFavoritesPlaylistDetail = query({ args: {}, - handler: async (ctx): Promise => { + handler: async (ctx): Promise => { const summary = await browseFavoritesSummaryHandler(ctx) if (!summary) { return null @@ -149,7 +139,7 @@ export const browseFavoritesPlaylistDetail = query({ title: summary.title, owner: summary.owner, cover: summary.cover, - songs, + songs: chunk(songs), isOwner: summary.isOwner, isSubscribed: summary.isSubscribed, } @@ -218,15 +208,17 @@ async function unfavoriteSongsHandler( (song): song is Doc<"songs"> => song !== null ) - await removeFavorites(ctx, userId, songIds) + await removeFavorites(ctx, userId, "song", songIds) await removeFavorites( ctx, userId, + "album", songs.map((song) => song.album) ) await removeFavorites( ctx, userId, + "artist", songs.map((song) => song.albumArtist) ) } @@ -261,8 +253,14 @@ export const unfavoriteAlbum = mutation({ return } - await removeFavorites(ctx, userId, [albumId, album.artist]) - await removeFavorites(ctx, userId, await getSongIdsForAlbum(ctx, albumId)) + await removeFavorites(ctx, userId, "album", [albumId]) + await removeFavorites(ctx, userId, "artist", [album.artist]) + await removeFavorites( + ctx, + userId, + "song", + await getSongIdsForAlbum(ctx, albumId) + ) }, }) @@ -297,12 +295,18 @@ export const unfavoriteArtist = mutation({ args: { artistId: v.id("artists") }, handler: async (ctx, { artistId }) => { const userId = await requireFavoritesUser(ctx) - await removeFavorites(ctx, userId, [artistId]) + await removeFavorites(ctx, userId, "artist", [artistId]) await removeFavorites( ctx, userId, + "album", await getAlbumIdsForArtist(ctx, artistId) ) - await removeFavorites(ctx, userId, await getSongIdsForArtist(ctx, artistId)) + await removeFavorites( + ctx, + userId, + "song", + await getSongIdsForArtist(ctx, artistId) + ) }, }) diff --git a/packages/convex/convex/library.ts b/packages/convex/convex/library.ts index 4ee92ad..6477067 100644 --- a/packages/convex/convex/library.ts +++ b/packages/convex/convex/library.ts @@ -22,6 +22,7 @@ import type { SidebarPlaylists, } from "./types" import { purgeFavoriteEntities } from "./favoriteStore" +import { chunk, type Chunked } from "./limits" import { recordIntentCatalogChanges } from "./intentCatalogChanges" export type { @@ -155,10 +156,19 @@ function toBrowseAlbum( } } +/** + * Past this many songs, scanning the library takes fewer reads than a get per + * song, album and artist, which would hit Convex's 4,096 reads per function. + */ +const MAX_SONG_GETS = 1_000 + export async function browseSongsFromIds( ctx: QueryCtx, songIds: Id<"songs">[] ): Promise { + if (songIds.length > MAX_SONG_GETS) { + return browseSongsByScan(ctx, songIds) + } const songs = await Promise.all( songIds.map((songId) => ctx.db.get("songs", songId)) ) @@ -168,6 +178,27 @@ export async function browseSongsFromIds( ) } +async function browseSongsByScan( + ctx: QueryCtx, + songIds: Id<"songs">[] +): Promise { + const [songs, albums, artistMap] = await Promise.all([ + ctx.db.query("songs").collect(), + ctx.db.query("albums").collect(), + loadArtistMap(ctx), + ]) + const songMap = new Map(songs.map((song) => [song._id, song])) + const albumMap = new Map(albums.map((album) => [album._id, album])) + return songIds.flatMap((songId) => { + const song = songMap.get(songId) + const album = song && albumMap.get(song.album) + const albumArtist = song && artistMap.get(song.albumArtist) + return song && album && albumArtist + ? [toBrowseSong(song, album, albumArtist)] + : [] + }) +} + async function browseSongDocs( ctx: QueryCtx, songs: Doc<"songs">[] @@ -215,32 +246,34 @@ function toBrowseSong( export const browseAlbums = query({ args: {}, - handler: async (ctx) => { + handler: async (ctx): Promise> => { if (!(await isSignedIn(ctx))) { return [] } const artistMap = await loadArtistMap(ctx) const albums = await ctx.db.query("albums").collect() - return albums - .flatMap((album) => { - const artist = artistMap.get(album.artist) - return artist ? [toBrowseAlbum(album, artist)] : [] - }) - .sort((a, b) => a.title.localeCompare(b.title)) + return chunk( + albums + .flatMap((album) => { + const artist = artistMap.get(album.artist) + return artist ? [toBrowseAlbum(album, artist)] : [] + }) + .sort((a, b) => a.title.localeCompare(b.title)) + ) }, }) export const browseArtists = query({ args: {}, - handler: async (ctx) => { + handler: async (ctx): Promise> => { if (!(await isSignedIn(ctx))) { return [] } const artists = await ctx.db.query("artists").collect() - return artists - .map(toBrowseArtist) - .sort((a, b) => a.name.localeCompare(b.name)) + return chunk( + artists.map(toBrowseArtist).sort((a, b) => a.name.localeCompare(b.name)) + ) }, }) @@ -509,7 +542,9 @@ export const browseSortedSongIds = query({ }, handler: async (ctx, args) => { await requireSignedIn(ctx) - return getSortedLibrarySongIds(ctx, args.sortKey, args.sortDirection) + return chunk( + await getSortedLibrarySongIds(ctx, args.sortKey, args.sortDirection) + ) }, }) @@ -606,6 +641,12 @@ function matchScoreSongArtists( ) } +/** + * Results kept per search section. Broad queries like "a" match most of the + * library, which would overflow Convex's array limit and is too long to scroll. + */ +const MAX_SEARCH_RESULTS = 500 + export const searchLibrary = query({ args: { query: v.string() }, handler: async (ctx, args) => { @@ -701,9 +742,15 @@ export const searchLibrary = query({ const result: LibrarySearchResults = { topResults, - artists: artistResults.map((result) => result.item), - albums: albumResults.map((result) => result.item), - songs: songResults.map((result) => result.item), + artists: artistResults + .slice(0, MAX_SEARCH_RESULTS) + .map((result) => result.item), + albums: albumResults + .slice(0, MAX_SEARCH_RESULTS) + .map((result) => result.item), + songs: songResults + .slice(0, MAX_SEARCH_RESULTS) + .map((result) => result.item), } return result }, @@ -711,7 +758,7 @@ export const searchLibrary = query({ export const browseRecentlyAdded = query({ args: { limit: v.optional(v.number()) }, - handler: async (ctx, args) => { + handler: async (ctx, args): Promise> => { if (!(await isSignedIn(ctx))) { return [] } @@ -724,10 +771,12 @@ export const browseRecentlyAdded = query({ albums.map((album) => album.artist) ) - return albums.flatMap((album) => { - const artist = artistMap.get(album.artist) - return artist ? [toBrowseAlbum(album, artist)] : [] - }) + return chunk( + albums.flatMap((album) => { + const artist = artistMap.get(album.artist) + return artist ? [toBrowseAlbum(album, artist)] : [] + }) + ) }, }) diff --git a/packages/convex/convex/limits.ts b/packages/convex/convex/limits.ts index 8adb174..d4ae255 100644 --- a/packages/convex/convex/limits.ts +++ b/packages/convex/convex/limits.ts @@ -8,3 +8,20 @@ export const MAX_QUEUE_LENGTH = 5_000 /** Largest profile photo accepted, in bytes after base64 decoding. */ export const MAX_PROFILE_IMAGE_BYTES = 2 * 1024 * 1024 + +/** Most values Convex accepts in a single array. */ +export const MAX_ARRAY_LENGTH = 8_192 + +/** + * A list split into arrays of at most `MAX_ARRAY_LENGTH` values, so a query can + * return more than one array allows. Read it back with `.flat()`. + */ +export type Chunked = T[][] + +export function chunk(items: readonly T[]): Chunked { + const chunks: Chunked = [] + for (let start = 0; start < items.length; start += MAX_ARRAY_LENGTH) { + chunks.push(items.slice(start, start + MAX_ARRAY_LENGTH)) + } + return chunks +} diff --git a/packages/convex/convex/playlists.ts b/packages/convex/convex/playlists.ts index dd34f42..ab18f2c 100644 --- a/packages/convex/convex/playlists.ts +++ b/packages/convex/convex/playlists.ts @@ -3,6 +3,7 @@ import { ConvexError, v } from "convex/values" import type { Doc, Id } from "./_generated/dataModel" import type { MutationCtx, QueryCtx } from "./_generated/server" import { mutation, query } from "./_generated/server" +import { chunk } from "./limits" import type { BrowsePlaylistDetail, BrowsePlaylistSummary, @@ -290,7 +291,7 @@ export const getArtistSongIds = query({ .query("songs") .withIndex("by_artist", (q) => q.eq("albumArtist", args.artistId)) .collect() - return songs.map((song) => song._id) + return chunk(songs.map((song) => song._id)) }, }) @@ -321,7 +322,7 @@ export const getOrderedArtistSongIds = query({ return a.name.localeCompare(b.name) }) - return songs.map((song) => song._id) + return chunk(songs.map((song) => song._id)) }, }) diff --git a/packages/convex/convex/types.ts b/packages/convex/convex/types.ts index 6cd39de..160e8f5 100644 --- a/packages/convex/convex/types.ts +++ b/packages/convex/convex/types.ts @@ -1,4 +1,5 @@ import type { Id } from "./_generated/dataModel" +import type { Chunked } from "./limits" export type BrowseArtist = { id: Id<"artists"> @@ -75,6 +76,11 @@ export type BrowsePlaylistDetail = { isSubscribed: boolean } +/** Favorites can outgrow one array, so its songs come back chunked. */ +export type BrowseFavoritesDetail = Omit & { + songs: Chunked +} + export type BrowsePlaylistSummary = Omit & { songIds: Id<"songs">[] }