diff --git a/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx b/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx index 68e5e64..a19835c 100644 --- a/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx +++ b/apps/mobile/app/(app)/(tabs)/home/playlist/favorites.tsx @@ -36,7 +36,7 @@ export default function FavoritesScreen() { const colors = useThemeColors() const { config } = useServerConfig() const favorites = useCachedQuery(api.favorites.browseFavoritesPlaylistDetail) - const songs = favorites?.songs ?? [] + const songs = useMemo(() => favorites?.songs.flat() ?? [], [favorites]) const { playInOrder, playShuffled, playFromSong } = usePlaybackActions() const { isSongFavorite } = useFavorites() const { handleSongAction } = useMediaMenuActions() @@ -45,8 +45,8 @@ export default function FavoritesScreen() { usePlaylistSearch(songs) useEffect(() => { - if (favorites) rememberSongs(favorites.songs) - }, [favorites]) + rememberSongs(songs) + }, [songs]) const handlePlaySong = useCallback( (song: BrowseSong) => void playFromSong(song.id, songIds), diff --git a/apps/mobile/components/app-intents/catalog-sync.ts b/apps/mobile/components/app-intents/catalog-sync.ts index 4d4c4fe..91714ce 100644 --- a/apps/mobile/components/app-intents/catalog-sync.ts +++ b/apps/mobile/components/app-intents/catalog-sync.ts @@ -23,19 +23,16 @@ type LibraryKind = (typeof LIBRARY_KINDS)[number] const PAGE_SIZE = 500 type CatalogPage = FunctionReturnType -type Favorites = { - favoriteSongs: string[] - favoriteAlbums: string[] - favoriteArtists: string[] -} +type Favorites = FunctionReturnType function favoriteIds(favorites: Favorites, kind: LibraryKind) { return new Set( - kind === "song" + (kind === "song" ? favorites.favoriteSongs : kind === "album" ? favorites.favoriteAlbums : favorites.favoriteArtists + ).flat() ) } diff --git a/apps/mobile/components/app-intents/dispatch.ts b/apps/mobile/components/app-intents/dispatch.ts index 67fbe2f..4424809 100644 --- a/apps/mobile/components/app-intents/dispatch.ts +++ b/apps/mobile/components/app-intents/dispatch.ts @@ -214,12 +214,13 @@ export function useIntentDispatch() { ) } if (kind === "artist") { - return fetchCachedQuery( + const chunks = await fetchCachedQuery( convex, cacheScope, api.playlists.getOrderedArtistSongIds, { artistId: id as Id<"artists"> } ) + return chunks.flat() } if (kind === "playlist") { if (id === FAVORITES_PLAYLIST_ID) { @@ -229,7 +230,7 @@ export function useIntentDispatch() { api.favorites.getUserFavorites, {} ) - return favorites.favoriteSongs + return favorites.favoriteSongs.flat() } return fetchCachedQuery( convex, @@ -391,17 +392,18 @@ export function useIntentDispatch() { api.favorites.getUserFavorites, {} ) - if (favorites.favoriteSongs.length === 0) { + const favoriteSongs = favorites.favoriteSongs.flat() + if (favoriteSongs.length === 0) { throw new IntentRejected( "Nothing is queued, and your favorites are empty." ) } if (shuffled) { ensureActive(invocation) - await actions.playShuffled(favorites.favoriteSongs) + await actions.playShuffled(favoriteSongs) } else { ensureActive(invocation) - await actions.playInOrder(favorites.favoriteSongs) + await actions.playInOrder(favoriteSongs) } return } diff --git a/apps/mobile/contexts/add-to-playlist.tsx b/apps/mobile/contexts/add-to-playlist.tsx index 817717f..5abdb6b 100644 --- a/apps/mobile/contexts/add-to-playlist.tsx +++ b/apps/mobile/contexts/add-to-playlist.tsx @@ -150,7 +150,7 @@ export function AddToPlaylistProvider({ children }: { children: ReactNode }) { { artistId } ) if (selectionRequestIdRef.current !== requestId) return - setPendingSongIds(songIds) + setPendingSongIds(songIds.flat()) setOpen(true) } catch (error) { if (selectionRequestIdRef.current !== requestId) return diff --git a/apps/mobile/contexts/favorites.tsx b/apps/mobile/contexts/favorites.tsx index 4befb5b..f1213df 100644 --- a/apps/mobile/contexts/favorites.tsx +++ b/apps/mobile/contexts/favorites.tsx @@ -100,15 +100,15 @@ export function FavoritesProvider({ children }: { children: ReactNode }) { ) const songIds = useMemo( - () => new Set(favorites?.favoriteSongs.map(String) ?? []), + () => new Set(favorites?.favoriteSongs.flat()), [favorites?.favoriteSongs] ) const albumIds = useMemo( - () => new Set(favorites?.favoriteAlbums.map(String) ?? []), + () => new Set(favorites?.favoriteAlbums.flat()), [favorites?.favoriteAlbums] ) const artistIds = useMemo( - () => new Set(favorites?.favoriteArtists.map(String) ?? []), + () => new Set(favorites?.favoriteArtists.flat()), [favorites?.favoriteArtists] ) const idsRef = useRef({ songIds, albumIds, artistIds }) diff --git a/apps/mobile/contexts/playback.tsx b/apps/mobile/contexts/playback.tsx index 43aae8d..40ee2c5 100644 --- a/apps/mobile/contexts/playback.tsx +++ b/apps/mobile/contexts/playback.tsx @@ -401,7 +401,7 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { cacheScope, api.playlists.getOrderedArtistSongIds, { artistId } - ), + ).then((chunks) => chunks.flat()), recorded: { entity: { kind: "artist", id: artistId } }, })) const playlist = collection( @@ -482,12 +482,14 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { let songIds: Id<"songs">[] try { - songIds = await fetchCachedQuery( - convex, - cacheScope, - api.library.browseSortedSongIds, - sort - ) + songIds = ( + await fetchCachedQuery( + convex, + cacheScope, + api.library.browseSortedSongIds, + sort + ) + ).flat() } catch { return } diff --git a/apps/mobile/hooks/use-media-menu-actions.ts b/apps/mobile/hooks/use-media-menu-actions.ts index b367040..5d26f56 100644 --- a/apps/mobile/hooks/use-media-menu-actions.ts +++ b/apps/mobile/hooks/use-media-menu-actions.ts @@ -173,7 +173,7 @@ export function useMediaMenuActions() { api.playlists.getOrderedArtistSongIds, { artistId: artist.id as Id<"artists"> } ) - await runQueueAction(action, songIds) + await runQueueAction(action, songIds.flat()) break } default: diff --git a/apps/web/app/(app)/(home)/page.tsx b/apps/web/app/(app)/(home)/page.tsx index fa3adf3..6af2750 100644 --- a/apps/web/app/(app)/(home)/page.tsx +++ b/apps/web/app/(app)/(home)/page.tsx @@ -32,7 +32,7 @@ export default function Page() { diff --git a/apps/web/app/(app)/library/albums/page.tsx b/apps/web/app/(app)/library/albums/page.tsx index f4d361f..37759e2 100644 --- a/apps/web/app/(app)/library/albums/page.tsx +++ b/apps/web/app/(app)/library/albums/page.tsx @@ -18,7 +18,7 @@ export default function Page() { ) : (
- {albums.map((album) => ( + {albums.flat().map((album) => ( ))}
diff --git a/apps/web/app/(app)/recently-added/page.tsx b/apps/web/app/(app)/recently-added/page.tsx index 919d85d..cd20d80 100644 --- a/apps/web/app/(app)/recently-added/page.tsx +++ b/apps/web/app/(app)/recently-added/page.tsx @@ -18,7 +18,7 @@ export default function Page() { ) : (
- {albums.map((album) => ( + {albums.flat().map((album) => ( ))}
diff --git a/apps/web/components/add-to-playlist-provider.tsx b/apps/web/components/add-to-playlist-provider.tsx index c215e54..5cc1ad8 100644 --- a/apps/web/components/add-to-playlist-provider.tsx +++ b/apps/web/components/add-to-playlist-provider.tsx @@ -184,7 +184,7 @@ export function AddToPlaylistProvider({ } ) if (requestId === selectionRequestRef.current) { - showAddToPlaylist(songIds) + showAddToPlaylist(songIds.flat()) } } catch (error) { if (requestId === selectionRequestRef.current) { diff --git a/apps/web/components/artists-page.tsx b/apps/web/components/artists-page.tsx index 645a54a..744ab95 100644 --- a/apps/web/components/artists-page.tsx +++ b/apps/web/components/artists-page.tsx @@ -32,8 +32,8 @@ export function ArtistsPage() { ) : ( diff --git a/apps/web/components/favorites-provider.tsx b/apps/web/components/favorites-provider.tsx index cbd60e8..686d9a9 100644 --- a/apps/web/components/favorites-provider.tsx +++ b/apps/web/components/favorites-provider.tsx @@ -86,15 +86,15 @@ export function FavoritesProvider({ children }: { children: React.ReactNode }) { ) const songIds = React.useMemo( - () => new Set(favorites?.favoriteSongs.map(String) ?? []), + () => new Set(favorites?.favoriteSongs.flat()), [favorites?.favoriteSongs] ) const albumIds = React.useMemo( - () => new Set(favorites?.favoriteAlbums.map(String) ?? []), + () => new Set(favorites?.favoriteAlbums.flat()), [favorites?.favoriteAlbums] ) const artistIds = React.useMemo( - () => new Set(favorites?.favoriteArtists.map(String) ?? []), + () => new Set(favorites?.favoriteArtists.flat()), [favorites?.favoriteArtists] ) diff --git a/apps/web/components/playback-provider.tsx b/apps/web/components/playback-provider.tsx index e6582f6..e4e9cf2 100644 --- a/apps/web/components/playback-provider.tsx +++ b/apps/web/components/playback-provider.tsx @@ -1233,12 +1233,14 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { let songIds: Id<"songs">[] try { - songIds = await fetchCachedQuery( - convex, - cacheScope, - api.library.browseSortedSongIds, - sort - ) + songIds = ( + await fetchCachedQuery( + convex, + cacheScope, + api.library.browseSortedSongIds, + sort + ) + ).flat() } catch { return } @@ -1268,7 +1270,7 @@ export function PlaybackProvider({ children }: { children: ReactNode }) { cacheScope, api.playlists.getOrderedArtistSongIds, { artistId } - ), + ).then((chunks) => chunks.flat()), [convex, cacheScope] ) diff --git a/apps/web/components/playlist-page.tsx b/apps/web/components/playlist-page.tsx index 07398b4..8e26494 100644 --- a/apps/web/components/playlist-page.tsx +++ b/apps/web/components/playlist-page.tsx @@ -34,5 +34,7 @@ export function FavoritesPlaylistPage() { return Sign in to view your favorites. } - return + return ( + + ) } diff --git a/apps/web/hooks/use-media-actions.ts b/apps/web/hooks/use-media-actions.ts index 9439842..07f8bcc 100644 --- a/apps/web/hooks/use-media-actions.ts +++ b/apps/web/hooks/use-media-actions.ts @@ -64,7 +64,7 @@ export function useMediaActions() { api.playlists.getOrderedArtistSongIds, { artistId: artistId as Id<"artists"> } ) - void runQueueAction(action, songIds) + void runQueueAction(action, songIds.flat()) break } } diff --git a/packages/convex/client/favorites.ts b/packages/convex/client/favorites.ts index 69d20eb..a905252 100644 --- a/packages/convex/client/favorites.ts +++ b/packages/convex/client/favorites.ts @@ -1,63 +1,54 @@ import type { OptimisticLocalStore, OptimisticUpdate } from "convex/browser" -import type { FunctionArgs, FunctionReference } from "convex/server" +import type { + FunctionArgs, + FunctionReference, + FunctionReturnType, +} from "convex/server" import { api } from "../convex/_generated/api" -import type { Id } from "../convex/_generated/dataModel" +import { chunk } from "../convex/limits" -type FavoriteIds = { - favoriteSongs: Id<"songs">[] - favoriteAlbums: Id<"albums">[] - favoriteArtists: Id<"artists">[] -} +type UserFavorites = FunctionReturnType +type FavoriteList = keyof UserFavorites -type FavoriteList = keyof FavoriteIds - -function patchFavorites( +/** Rewrites one list of the cached `getUserFavorites` result, if loaded. */ +function patchFavorites( store: OptimisticLocalStore, - list: List, - update: (ids: FavoriteIds[List]) => FavoriteIds[List] + list: FavoriteList, + update: (ids: string[]) => string[] ) { const current = store.getQuery(api.favorites.getUserFavorites, {}) if (!current) return - const next = update(current[list]) - if (next !== current[list]) { - store.setQuery( - api.favorites.getUserFavorites, - {}, - { - ...current, - [list]: next, - } - ) + const ids: string[] = current[list].flat() + const next = update(ids) + if (next !== ids) { + // `next` only holds ids of this list's kind, which TS cannot follow here. + store.setQuery(api.favorites.getUserFavorites, {}, { + ...current, + [list]: chunk(next), + } as UserFavorites) } } -function addFavorites( +function addFavorites( store: OptimisticLocalStore, - list: List, - ids: FavoriteIds[List] + list: FavoriteList, + ids: string[] ) { patchFavorites(store, list, (current) => { - const added = ids.filter((id) => !(current as string[]).includes(id)) - return added.length === 0 - ? current - : ([...new Set([...current, ...added])] as FavoriteIds[List]) + const added = ids.filter((id) => !current.includes(id)) + return added.length === 0 ? current : [...new Set([...current, ...added])] }) } -function removeFavorites( +function removeFavorites( store: OptimisticLocalStore, - list: List, - ids: FavoriteIds[List] + list: FavoriteList, + ids: string[] ) { - const removed = new Set(ids) - patchFavorites( - store, - list, - (current) => - (current as string[]).filter( - (id) => !removed.has(id) - ) as FavoriteIds[List] + const removed = new Set(ids) + patchFavorites(store, list, (current) => + current.filter((id) => !removed.has(id)) ) } diff --git a/packages/convex/convex/app-intent-playlists.test.cjs b/packages/convex/convex/app-intent-playlists.test.cjs index 701d969..3229c5a 100644 --- a/packages/convex/convex/app-intent-playlists.test.cjs +++ b/packages/convex/convex/app-intent-playlists.test.cjs @@ -29,7 +29,7 @@ function playlistMutation() { mutation: (definition) => definition, query: (definition) => definition, } - if (name === "./library") return {} + if (name === "./library" || name === "./limits") return {} throw new Error(`Unexpected import ${name}`) }, }) diff --git a/packages/convex/convex/favoriteStore.ts b/packages/convex/convex/favoriteStore.ts index a75d1a2..749937a 100644 --- a/packages/convex/convex/favoriteStore.ts +++ b/packages/convex/convex/favoriteStore.ts @@ -1,7 +1,10 @@ import type { WithoutSystemFields } from "convex/server" -import { ConvexError } from "convex/values" +import { ConvexError, v } from "convex/values" +import { internal } from "./_generated/api" import type { Doc, Id } from "./_generated/dataModel" import type { MutationCtx, QueryCtx } from "./_generated/server" +import { internalMutation } from "./_generated/server" +import { chunk, type Chunked } from "./limits" export type FavoriteKind = Doc<"favorites">["kind"] @@ -9,13 +12,18 @@ type FavoriteEntityId = { [Kind in FavoriteKind]: Extract, { kind: Kind }>["entityId"] } -/** The `favorites.getUserFavorites` shape, kept from when these lived on `users`. */ +/** A user's favorites, each list oldest first. */ export type FavoriteIds = { favoriteSongs: Id<"songs">[] favoriteAlbums: Id<"albums">[] favoriteArtists: Id<"artists">[] } +/** `FavoriteIds` as `favorites.getUserFavorites` returns it. */ +export type FavoriteIdChunks = { + [List in keyof FavoriteIds]: Chunked +} + function legacyFavorites(user: Doc<"users"> | null): FavoriteIds { return { favoriteSongs: user?.favoriteSongs ?? [], @@ -69,25 +77,59 @@ export async function readFavorites( return legacyFavorites(await ctx.db.get(userId)) } -function favoriteRow( +/** + * Above this many ids, one scan of the user's list beats a lookup per id, and + * keeps bulk writes under Convex's 4,096 reads per function. + */ +const MAX_POINT_LOOKUPS = 64 + +/** Maps each of `ids` the user has favorited to its row. */ +async function favoriteRows( ctx: QueryCtx, userId: Id<"users">, - entityId: FavoriteEntityId[FavoriteKind] -) { - return ctx.db - .query("favorites") - .withIndex("by_user_entity", (q) => - q.eq("userId", userId).eq("entityId", entityId) + kind: Kind, + ids: ReadonlySet +): Promise>> { + const rows = + ids.size <= MAX_POINT_LOOKUPS + ? await Promise.all( + [...ids].map((entityId) => + ctx.db + .query("favorites") + .withIndex("by_user_entity", (q) => + q.eq("userId", userId).eq("entityId", entityId) + ) + .unique() + ) + ) + : await ctx.db + .query("favorites") + .withIndex("by_user_kind", (q) => + q.eq("userId", userId).eq("kind", kind) + ) + .collect() + return new Map( + rows.flatMap((row) => + row && ids.has(row.entityId as FavoriteEntityId[Kind]) + ? [[row.entityId, row._id] as const] + : [] ) - .unique() + ) } -export async function isFavorite( +/** True when `ids` is non-empty and the user has favorited every one. */ +export async function allFavorited( ctx: QueryCtx, userId: Id<"users">, - entityId: FavoriteEntityId[FavoriteKind] + kind: Kind, + ids: FavoriteEntityId[Kind][] ) { - return (await favoriteRow(ctx, userId, entityId)) !== null + const unique = new Set(ids) + if (unique.size === 0) { + return false + } + const rows = await favoriteRows(ctx, userId, kind, unique) + return rows.size === unique.size } /** Appends ids that are not favorited yet, keeping existing ones in place. */ @@ -97,8 +139,10 @@ export async function addFavorites( kind: Kind, ids: FavoriteEntityId[Kind][] ) { - for (const entityId of new Set(ids)) { - if (!(await isFavorite(ctx, userId, entityId))) { + const unique = new Set(ids) + const existing = await favoriteRows(ctx, userId, kind, unique) + for (const entityId of unique) { + if (!existing.has(entityId)) { // TS cannot tie a generic `kind` to its `entityId` union member. await ctx.db.insert("favorites", { userId, @@ -109,19 +153,52 @@ export async function addFavorites( } } -export async function removeFavorites( +export async function removeFavorites( ctx: MutationCtx, userId: Id<"users">, - ids: Iterable + kind: Kind, + ids: FavoriteEntityId[Kind][] ) { - for (const entityId of new Set(ids)) { - const row = await favoriteRow(ctx, userId, entityId) - if (row) { - await ctx.db.delete(row._id) - } + const rows = await favoriteRows(ctx, userId, kind, new Set(ids)) + await deleteFavoriteRows(ctx, [...rows.values()]) +} + +/** + * Rows one call deletes before scheduling the rest. Convex counts each delete + * as a read, so unfavoriting a huge artist cannot happen in one mutation. + */ +const MAX_INLINE_DELETES = 1_000 + +async function deleteFavoriteRows(ctx: MutationCtx, rowIds: Id<"favorites">[]) { + for (const rowId of rowIds.slice(0, MAX_INLINE_DELETES)) { + await ctx.db.delete(rowId) + } + for (const rest of chunk(rowIds.slice(MAX_INLINE_DELETES))) { + await ctx.scheduler.runAfter(0, internal.favoriteStore.deleteRows, { + rowIds: rest, + }) } } +/** Finishes deletes `deleteFavoriteRows` could not fit in one mutation. */ +export const deleteRows = internalMutation({ + args: { rowIds: v.array(v.id("favorites")) }, + handler: async (ctx, { rowIds }) => { + // Rows can disappear in between, e.g. when the scanner purges a song. + for (const rowId of rowIds.slice(0, MAX_INLINE_DELETES)) { + if (await ctx.db.get(rowId)) { + await ctx.db.delete(rowId) + } + } + const rest = rowIds.slice(MAX_INLINE_DELETES) + if (rest.length > 0) { + await ctx.scheduler.runAfter(0, internal.favoriteStore.deleteRows, { + rowIds: rest, + }) + } + }, +}) + /** * Moves a user's legacy favorite arrays into `favorites` rows, keeping their * order, then clears them. Safe to call repeatedly. @@ -180,13 +257,18 @@ export async function deleteUserFavorites( ctx: MutationCtx, userId: Id<"users"> ) { - for (const kind of ["song", "album", "artist"] as const) { - const rows = await ctx.db - .query("favorites") - .withIndex("by_user_kind", (q) => q.eq("userId", userId).eq("kind", kind)) - .collect() - for (const row of rows) { - await ctx.db.delete(row._id) - } - } + const rows = await Promise.all( + (["song", "album", "artist"] as const).map((kind) => + ctx.db + .query("favorites") + .withIndex("by_user_kind", (q) => + q.eq("userId", userId).eq("kind", kind) + ) + .collect() + ) + ) + await deleteFavoriteRows( + ctx, + rows.flat().map((row) => row._id) + ) } diff --git a/packages/convex/convex/favorites.ts b/packages/convex/convex/favorites.ts index 960407f..b1ba701 100644 --- a/packages/convex/convex/favorites.ts +++ b/packages/convex/convex/favorites.ts @@ -5,14 +5,16 @@ import type { MutationCtx, QueryCtx } from "./_generated/server" import { mutation, query } from "./_generated/server" import { addFavorites, - isFavorite, + allFavorited, readFavorites, removeFavorites, requireMigratedFavorites, - type FavoriteIds, + type FavoriteIdChunks, } from "./favoriteStore" -import type { BrowsePlaylistDetail, BrowsePlaylistSummary } from "./library" +import type { BrowsePlaylistSummary } from "./library" import { browseSongsFromIds } from "./library" +import { chunk } from "./limits" +import type { BrowseFavoritesDetail } from "./types" /** Resolves the signed-in user and moves any legacy favorites into rows. */ async function requireFavoritesUser(ctx: MutationCtx) { @@ -48,30 +50,13 @@ async function getSongIdsForArtist(ctx: QueryCtx, artistId: Id<"artists">) { return songs.map((song) => song._id) } -/** True when `ids` is non-empty and every one of them is favorited. */ -async function allFavorited( - ctx: QueryCtx, - userId: Id<"users">, - ids: (Id<"songs"> | Id<"albums">)[] -) { - if (ids.length === 0) { - return false - } - for (const id of ids) { - if (!(await isFavorite(ctx, userId, id))) { - return false - } - } - return true -} - async function promoteArtistIfAllAlbumsFavorited( ctx: MutationCtx, userId: Id<"users">, artistId: Id<"artists"> ) { const albumIds = await getAlbumIdsForArtist(ctx, artistId) - if (await allFavorited(ctx, userId, albumIds)) { + if (await allFavorited(ctx, userId, "album", albumIds)) { await addFavorites(ctx, userId, "artist", [artistId]) } } @@ -84,7 +69,7 @@ async function cascadeUpFromSongs( ) { for (const albumId of new Set(songs.map((song) => song.album))) { const songIds = await getSongIdsForAlbum(ctx, albumId) - if (await allFavorited(ctx, userId, songIds)) { + if (await allFavorited(ctx, userId, "song", songIds)) { await addFavorites(ctx, userId, "album", [albumId]) } } @@ -95,12 +80,17 @@ async function cascadeUpFromSongs( export const getUserFavorites = query({ args: {}, - handler: async (ctx): Promise => { + handler: async (ctx): Promise => { const userId = await getAuthUserId(ctx) if (userId === null) { return { favoriteSongs: [], favoriteAlbums: [], favoriteArtists: [] } } - return readFavorites(ctx, userId) + const favorites = await readFavorites(ctx, userId) + return { + favoriteSongs: chunk(favorites.favoriteSongs), + favoriteAlbums: chunk(favorites.favoriteAlbums), + favoriteArtists: chunk(favorites.favoriteArtists), + } }, }) @@ -135,7 +125,7 @@ async function browseFavoritesSummaryHandler( export const browseFavoritesPlaylistDetail = query({ args: {}, - handler: async (ctx): Promise => { + handler: async (ctx): Promise => { const summary = await browseFavoritesSummaryHandler(ctx) if (!summary) { return null @@ -149,7 +139,7 @@ export const browseFavoritesPlaylistDetail = query({ title: summary.title, owner: summary.owner, cover: summary.cover, - songs, + songs: chunk(songs), isOwner: summary.isOwner, isSubscribed: summary.isSubscribed, } @@ -218,15 +208,17 @@ async function unfavoriteSongsHandler( (song): song is Doc<"songs"> => song !== null ) - await removeFavorites(ctx, userId, songIds) + await removeFavorites(ctx, userId, "song", songIds) await removeFavorites( ctx, userId, + "album", songs.map((song) => song.album) ) await removeFavorites( ctx, userId, + "artist", songs.map((song) => song.albumArtist) ) } @@ -261,8 +253,14 @@ export const unfavoriteAlbum = mutation({ return } - await removeFavorites(ctx, userId, [albumId, album.artist]) - await removeFavorites(ctx, userId, await getSongIdsForAlbum(ctx, albumId)) + await removeFavorites(ctx, userId, "album", [albumId]) + await removeFavorites(ctx, userId, "artist", [album.artist]) + await removeFavorites( + ctx, + userId, + "song", + await getSongIdsForAlbum(ctx, albumId) + ) }, }) @@ -297,12 +295,18 @@ export const unfavoriteArtist = mutation({ args: { artistId: v.id("artists") }, handler: async (ctx, { artistId }) => { const userId = await requireFavoritesUser(ctx) - await removeFavorites(ctx, userId, [artistId]) + await removeFavorites(ctx, userId, "artist", [artistId]) await removeFavorites( ctx, userId, + "album", await getAlbumIdsForArtist(ctx, artistId) ) - await removeFavorites(ctx, userId, await getSongIdsForArtist(ctx, artistId)) + await removeFavorites( + ctx, + userId, + "song", + await getSongIdsForArtist(ctx, artistId) + ) }, }) diff --git a/packages/convex/convex/library.ts b/packages/convex/convex/library.ts index 4ee92ad..6477067 100644 --- a/packages/convex/convex/library.ts +++ b/packages/convex/convex/library.ts @@ -22,6 +22,7 @@ import type { SidebarPlaylists, } from "./types" import { purgeFavoriteEntities } from "./favoriteStore" +import { chunk, type Chunked } from "./limits" import { recordIntentCatalogChanges } from "./intentCatalogChanges" export type { @@ -155,10 +156,19 @@ function toBrowseAlbum( } } +/** + * Past this many songs, scanning the library takes fewer reads than a get per + * song, album and artist, which would hit Convex's 4,096 reads per function. + */ +const MAX_SONG_GETS = 1_000 + export async function browseSongsFromIds( ctx: QueryCtx, songIds: Id<"songs">[] ): Promise { + if (songIds.length > MAX_SONG_GETS) { + return browseSongsByScan(ctx, songIds) + } const songs = await Promise.all( songIds.map((songId) => ctx.db.get("songs", songId)) ) @@ -168,6 +178,27 @@ export async function browseSongsFromIds( ) } +async function browseSongsByScan( + ctx: QueryCtx, + songIds: Id<"songs">[] +): Promise { + const [songs, albums, artistMap] = await Promise.all([ + ctx.db.query("songs").collect(), + ctx.db.query("albums").collect(), + loadArtistMap(ctx), + ]) + const songMap = new Map(songs.map((song) => [song._id, song])) + const albumMap = new Map(albums.map((album) => [album._id, album])) + return songIds.flatMap((songId) => { + const song = songMap.get(songId) + const album = song && albumMap.get(song.album) + const albumArtist = song && artistMap.get(song.albumArtist) + return song && album && albumArtist + ? [toBrowseSong(song, album, albumArtist)] + : [] + }) +} + async function browseSongDocs( ctx: QueryCtx, songs: Doc<"songs">[] @@ -215,32 +246,34 @@ function toBrowseSong( export const browseAlbums = query({ args: {}, - handler: async (ctx) => { + handler: async (ctx): Promise> => { if (!(await isSignedIn(ctx))) { return [] } const artistMap = await loadArtistMap(ctx) const albums = await ctx.db.query("albums").collect() - return albums - .flatMap((album) => { - const artist = artistMap.get(album.artist) - return artist ? [toBrowseAlbum(album, artist)] : [] - }) - .sort((a, b) => a.title.localeCompare(b.title)) + return chunk( + albums + .flatMap((album) => { + const artist = artistMap.get(album.artist) + return artist ? [toBrowseAlbum(album, artist)] : [] + }) + .sort((a, b) => a.title.localeCompare(b.title)) + ) }, }) export const browseArtists = query({ args: {}, - handler: async (ctx) => { + handler: async (ctx): Promise> => { if (!(await isSignedIn(ctx))) { return [] } const artists = await ctx.db.query("artists").collect() - return artists - .map(toBrowseArtist) - .sort((a, b) => a.name.localeCompare(b.name)) + return chunk( + artists.map(toBrowseArtist).sort((a, b) => a.name.localeCompare(b.name)) + ) }, }) @@ -509,7 +542,9 @@ export const browseSortedSongIds = query({ }, handler: async (ctx, args) => { await requireSignedIn(ctx) - return getSortedLibrarySongIds(ctx, args.sortKey, args.sortDirection) + return chunk( + await getSortedLibrarySongIds(ctx, args.sortKey, args.sortDirection) + ) }, }) @@ -606,6 +641,12 @@ function matchScoreSongArtists( ) } +/** + * Results kept per search section. Broad queries like "a" match most of the + * library, which would overflow Convex's array limit and is too long to scroll. + */ +const MAX_SEARCH_RESULTS = 500 + export const searchLibrary = query({ args: { query: v.string() }, handler: async (ctx, args) => { @@ -701,9 +742,15 @@ export const searchLibrary = query({ const result: LibrarySearchResults = { topResults, - artists: artistResults.map((result) => result.item), - albums: albumResults.map((result) => result.item), - songs: songResults.map((result) => result.item), + artists: artistResults + .slice(0, MAX_SEARCH_RESULTS) + .map((result) => result.item), + albums: albumResults + .slice(0, MAX_SEARCH_RESULTS) + .map((result) => result.item), + songs: songResults + .slice(0, MAX_SEARCH_RESULTS) + .map((result) => result.item), } return result }, @@ -711,7 +758,7 @@ export const searchLibrary = query({ export const browseRecentlyAdded = query({ args: { limit: v.optional(v.number()) }, - handler: async (ctx, args) => { + handler: async (ctx, args): Promise> => { if (!(await isSignedIn(ctx))) { return [] } @@ -724,10 +771,12 @@ export const browseRecentlyAdded = query({ albums.map((album) => album.artist) ) - return albums.flatMap((album) => { - const artist = artistMap.get(album.artist) - return artist ? [toBrowseAlbum(album, artist)] : [] - }) + return chunk( + albums.flatMap((album) => { + const artist = artistMap.get(album.artist) + return artist ? [toBrowseAlbum(album, artist)] : [] + }) + ) }, }) diff --git a/packages/convex/convex/limits.ts b/packages/convex/convex/limits.ts index 8adb174..d4ae255 100644 --- a/packages/convex/convex/limits.ts +++ b/packages/convex/convex/limits.ts @@ -8,3 +8,20 @@ export const MAX_QUEUE_LENGTH = 5_000 /** Largest profile photo accepted, in bytes after base64 decoding. */ export const MAX_PROFILE_IMAGE_BYTES = 2 * 1024 * 1024 + +/** Most values Convex accepts in a single array. */ +export const MAX_ARRAY_LENGTH = 8_192 + +/** + * A list split into arrays of at most `MAX_ARRAY_LENGTH` values, so a query can + * return more than one array allows. Read it back with `.flat()`. + */ +export type Chunked = T[][] + +export function chunk(items: readonly T[]): Chunked { + const chunks: Chunked = [] + for (let start = 0; start < items.length; start += MAX_ARRAY_LENGTH) { + chunks.push(items.slice(start, start + MAX_ARRAY_LENGTH)) + } + return chunks +} diff --git a/packages/convex/convex/playlists.ts b/packages/convex/convex/playlists.ts index dd34f42..ab18f2c 100644 --- a/packages/convex/convex/playlists.ts +++ b/packages/convex/convex/playlists.ts @@ -3,6 +3,7 @@ import { ConvexError, v } from "convex/values" import type { Doc, Id } from "./_generated/dataModel" import type { MutationCtx, QueryCtx } from "./_generated/server" import { mutation, query } from "./_generated/server" +import { chunk } from "./limits" import type { BrowsePlaylistDetail, BrowsePlaylistSummary, @@ -290,7 +291,7 @@ export const getArtistSongIds = query({ .query("songs") .withIndex("by_artist", (q) => q.eq("albumArtist", args.artistId)) .collect() - return songs.map((song) => song._id) + return chunk(songs.map((song) => song._id)) }, }) @@ -321,7 +322,7 @@ export const getOrderedArtistSongIds = query({ return a.name.localeCompare(b.name) }) - return songs.map((song) => song._id) + return chunk(songs.map((song) => song._id)) }, }) diff --git a/packages/convex/convex/types.ts b/packages/convex/convex/types.ts index 6cd39de..160e8f5 100644 --- a/packages/convex/convex/types.ts +++ b/packages/convex/convex/types.ts @@ -1,4 +1,5 @@ import type { Id } from "./_generated/dataModel" +import type { Chunked } from "./limits" export type BrowseArtist = { id: Id<"artists"> @@ -75,6 +76,11 @@ export type BrowsePlaylistDetail = { isSubscribed: boolean } +/** Favorites can outgrow one array, so its songs come back chunked. */ +export type BrowseFavoritesDetail = Omit & { + songs: Chunked +} + export type BrowsePlaylistSummary = Omit & { songIds: Id<"songs">[] }