Files
otunes/packages/convex/client/download-queue.ts
T
joshuafhigginsandClaude Opus 5.5 ffafbdc7ad feat(mobile): keep downloads going in the background
iOS songs download through a background URL session, and on iOS 26+ a
continued processing task keeps the app running with a Live Activity.
Android keeps the app alive with a dataSync foreground service while songs
are queued.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 10:25:11 -05:00

574 lines
18 KiB
TypeScript

import {
DOWNLOAD_QUALITIES,
downloadPlan,
type DownloadQuality,
} from "../download-quality"
export type DownloadedSong = {
songId: string
/** The quality the file is in, after any fallback to High. */
quality: DownloadQuality
bytes: number
/** Whose cover to keep for offline use. */
albumId?: string
}
/** Artwork kept on the device for downloaded songs and playlists. */
export type DownloadedCover = { kind: "album" | "playlist"; id: string }
/**
* An album or playlist downloaded as a whole. Playlists stay in sync: songs
* added later download in `quality`, and songs taken out are removed.
*/
export type DownloadedCollection = {
kind: "album" | "playlist"
id: string
title: string
quality: DownloadQuality
/** Every song in the album or playlist, for syncing. */
songIds: string[]
/** Songs whose download was removed from this collection only. */
removedSongIds?: string[]
}
export type CollectionRef = Pick<DownloadedCollection, "kind" | "id">
/** Where songs are removed from: one collection, or Other Downloaded Songs. */
export type DownloadSource = CollectionRef | "other"
/** The songs a collection keeps downloaded. */
export function collectionSongIds(collection: DownloadedCollection) {
if (!collection.removedSongIds?.length) return collection.songIds
const removed = new Set(collection.removedSongIds)
return collection.songIds.filter((songId) => !removed.has(songId))
}
/**
* What a device remembers besides the audio: downloaded collections, and songs
* downloaded on their own ("Other Downloaded Songs"). A song leaves the other
* songs when its album is downloaded, but stays when a playlist holds it.
*/
export type DownloadIndex = {
collections: DownloadedCollection[]
otherSongIds: string[]
}
/** Where a platform keeps downloaded audio. */
export type DownloadStorage = {
list(): Promise<DownloadedSong[]>
/**
* Fetches `requested` from the server and stores it, recording `quality`
* (what the server falls back to). Returns the stored size. Must leave
* nothing behind when it throws or `signal` aborts.
*/
save(
song: {
songId: string
requested: DownloadQuality
quality: DownloadQuality
albumId?: string
},
signal: AbortSignal
): Promise<number>
/** Fetches and keeps a cover unless it is already kept. */
saveCover(cover: DownloadedCover): Promise<void>
/** Deletes every kept cover not in `keep`. */
pruneCovers(keep: readonly DownloadedCover[]): Promise<void>
remove(songIds: readonly string[]): Promise<void>
readIndex(): Promise<DownloadIndex>
writeIndex(index: DownloadIndex): Promise<void>
}
export type DownloadableSong = {
id: string
files: Record<string, unknown>
albumId?: string
}
export type DownloadStatus = {
downloaded: number
queued: number
total: number
}
export type DownloadBreakdown = {
bytes: number
songs: number
queued: number
byQuality: { quality: DownloadQuality; bytes: number; songs: number }[]
collections: {
collection: DownloadedCollection
bytes: number
status: DownloadStatus
/** Downloaded songs that fell back to High. */
fallbackCount: number
}[]
/** Other Downloaded Songs: songs downloaded on their own. */
otherSongs: { bytes: number; songIds: string[] }
}
export type SongDownloadState = "here" | "device" | "removed" | "none"
export type DownloadQueue = ReturnType<typeof createDownloadQueue>
/** The Favorites playlist has no ID or cover; downloads use this one. */
export const FAVORITES_ID = "favorites"
function collectionKey({ kind, id }: CollectionRef) {
return `${kind}:${id}`
}
/**
* Downloads songs a few at a time and tracks which songs, albums, and
* playlists are on this device. Read it from React with
* `useSyncExternalStore(subscribe, getVersion)`.
*/
export function createDownloadQueue(
storage: DownloadStorage,
options: { concurrency?: number } = {}
) {
let concurrency = options.concurrency ?? 2
const downloaded = new Map<string, DownloadedSong>()
const collections = new Map<string, DownloadedCollection>()
const otherSongIds = new Set<string>()
const waiting: { song: DownloadableSong; quality: DownloadQuality }[] = []
const running = new Map<string, AbortController>()
const listeners = new Set<() => void>()
const failureListeners = new Set<(songId: string, error: unknown) => void>()
let version = 0
let loading: Promise<void> | null = null
function notify() {
version += 1
for (const listener of listeners) listener()
}
function load() {
loading ??= Promise.all([storage.list(), storage.readIndex()]).then(
([songs, index]) => {
for (const song of songs) downloaded.set(song.songId, song)
for (const collection of index.collections) {
collections.set(collectionKey(collection), collection)
}
// Songs still queued when the app closed were never downloaded.
for (const songId of index.otherSongIds) {
if (downloaded.has(songId)) otherSongIds.add(songId)
}
// Retries covers that failed before (say, offline) and drops strays.
for (const cover of neededCovers()) saveCover(cover)
pruneCovers()
notify()
},
(error) => {
loading = null
throw error
}
)
return loading
}
function saveIndex() {
return storage.writeIndex({
collections: [...collections.values()],
otherSongIds: [...otherSongIds],
})
}
function isQueued(songId: string) {
return running.has(songId) || waiting.some(({ song }) => song.id === songId)
}
function status(songIds: readonly string[]): DownloadStatus {
let done = 0
let queued = 0
for (const songId of songIds) {
if (downloaded.has(songId)) done += 1
else if (isQueued(songId)) queued += 1
}
return { downloaded: done, queued, total: songIds.length }
}
function pump() {
while (running.size < concurrency && waiting.length > 0) {
const { song, quality } = waiting.shift()!
const controller = new AbortController()
const actual = downloadPlan(song.files, quality).quality
running.set(song.id, controller)
void storage
.save(
{
songId: song.id,
requested: quality,
quality: actual,
albumId: song.albumId,
},
controller.signal
)
.then(
(bytes) => {
if (controller.signal.aborted) return
downloaded.set(song.id, {
songId: song.id,
quality: actual,
bytes,
albumId: song.albumId,
})
if (song.albumId) {
saveCover({ kind: "album", id: song.albumId })
}
},
(error) => {
if (controller.signal.aborted) return
// Load drops it from the saved index, since it never downloaded.
otherSongIds.delete(song.id)
for (const listener of failureListeners) listener(song.id, error)
}
)
.finally(() => {
if (running.get(song.id) === controller) running.delete(song.id)
notify()
pump()
})
}
}
function enqueueLoaded(
songs: readonly DownloadableSong[],
quality: DownloadQuality
) {
for (const song of songs) {
if (downloaded.has(song.id) || isQueued(song.id)) continue
waiting.push({ song, quality })
}
}
/** Covers are best effort: a missing one falls back to the placeholder. */
function saveCover(cover: DownloadedCover) {
void storage.saveCover(cover).catch(() => {})
}
/** Album covers of downloaded songs, and covers of downloaded playlists. */
function neededCovers() {
const covers = new Map<string, DownloadedCover>()
for (const song of downloaded.values()) {
if (song.albumId) {
covers.set(`album:${song.albumId}`, { kind: "album", id: song.albumId })
}
}
for (const collection of collections.values()) {
if (collection.kind === "playlist" && collection.id !== FAVORITES_ID) {
covers.set(`playlist:${collection.id}`, {
kind: "playlist",
id: collection.id,
})
}
}
return [...covers.values()]
}
function pruneCovers() {
void storage.pruneCovers(neededCovers()).catch(() => {})
}
/** Cancels and deletes audio, leaving the index as it is. */
async function deleteAudio(songIds: readonly string[]) {
if (songIds.length === 0) return
const ids = new Set(songIds)
for (let index = waiting.length - 1; index >= 0; index -= 1) {
if (ids.has(waiting[index].song.id)) waiting.splice(index, 1)
}
for (const songId of ids) {
running.get(songId)?.abort()
running.delete(songId)
downloaded.delete(songId)
}
notify()
await storage.remove([...ids])
pruneCovers()
pump()
}
/** Song IDs kept by Other Downloaded Songs or a collection besides `except`. */
function songsHeldElsewhere(except?: string) {
const held = new Set(otherSongIds)
for (const [key, collection] of collections) {
if (key === except) continue
for (const songId of collectionSongIds(collection)) held.add(songId)
}
return held
}
/** Deletes the audio of songs nothing holds anymore. */
async function deleteUnheld(songIds: readonly string[]) {
const held = songsHeldElsewhere()
await deleteAudio(songIds.filter((songId) => !held.has(songId)))
}
/**
* Gives songs back to collections they were removed from. Returns the songs
* no collection took back.
*/
function restoreToCollections(songIds: readonly string[]) {
const unclaimed = new Set(songIds)
for (const [key, collection] of collections) {
const removed = collection.removedSongIds
if (!removed?.some((songId) => unclaimed.has(songId))) continue
collections.set(key, {
...collection,
removedSongIds: removed.filter((songId) => !unclaimed.has(songId)),
})
for (const songId of removed) unclaimed.delete(songId)
}
return unclaimed
}
/** Removes `songIds` from one collection's download. */
function removeFromCollection(key: string, songIds: readonly string[]) {
const collection = collections.get(key)
if (!collection) return
const members = new Set(collection.songIds)
const removed = new Set(collection.removedSongIds)
for (const songId of songIds) if (members.has(songId)) removed.add(songId)
collections.set(key, { ...collection, removedSongIds: [...removed] })
}
return {
load,
subscribe(listener: () => void) {
listeners.add(listener)
return () => {
listeners.delete(listener)
}
},
/** Changes whenever anything downloads, queues, or is removed. */
getVersion: () => version,
/** Called when a song fails to download; it is left undownloaded. */
onFailure(listener: (songId: string, error: unknown) => void) {
failureListeners.add(listener)
return () => {
failureListeners.delete(listener)
}
},
getDownloaded: (songId: string) => downloaded.get(songId),
getCollection: (ref: CollectionRef) => collections.get(collectionKey(ref)),
/**
* How a song is downloaded as seen from `from`. "here" means that download
* holds it (outside a downloaded collection, Other Downloaded Songs does).
* "device" means only another download holds it, and "removed" means it
* was removed from the downloaded collection `from` but is still on the
* device. Menus offer Remove only for "here"; the downloaded icon shows
* for "here" and "device".
*/
downloadState(songId: string, from?: DownloadSource): SongDownloadState {
if (!downloaded.has(songId)) return "none"
const collection =
from && from !== "other"
? collections.get(collectionKey(from))
: undefined
if (collection) {
return collection.removedSongIds?.includes(songId) ? "removed" : "here"
}
return otherSongIds.has(songId) ? "here" : "device"
},
collections: () => [...collections.values()],
isQueued,
status,
/** Songs downloading or waiting to. */
pendingCount: () => running.size + waiting.length,
/**
* How many songs download at once. Raising it starts waiting songs now;
* lowering it lets running songs finish.
*/
setConcurrency(limit: number) {
concurrency = limit
pump()
},
/** Songs downloaded on their own, in the order they were added. */
otherSongIds: () => [...otherSongIds],
/**
* Downloads songs on their own. Songs removed from a downloaded album or
* playlist go back to it; the rest join Other Downloaded Songs, including
* songs already on the device for another download.
*/
async enqueue(
songs: readonly DownloadableSong[],
quality: DownloadQuality
) {
await load()
const unclaimed = restoreToCollections(songs.map((song) => song.id))
for (const songId of unclaimed) otherSongIds.add(songId)
enqueueLoaded(songs, quality)
notify()
pump()
await saveIndex()
},
/**
* Downloads an album or playlist and remembers it for syncing. An album
* takes its songs out of Other Downloaded Songs.
*/
async downloadCollection(
collection: Omit<DownloadedCollection, "songIds">,
songs: readonly DownloadableSong[]
) {
await load()
if (collection.kind === "album") {
for (const song of songs) otherSongIds.delete(song.id)
}
if (collection.kind === "playlist" && collection.id !== FAVORITES_ID) {
saveCover({ kind: "playlist", id: collection.id })
}
collections.set(collectionKey(collection), {
kind: collection.kind,
id: collection.id,
title: collection.title,
quality: collection.quality,
songIds: songs.map((song) => song.id),
})
enqueueLoaded(songs, collection.quality)
notify()
pump()
await saveIndex()
},
/**
* Brings a downloaded playlist up to date with its current songs.
* `loadSongs` fetches file info for the songs that were added.
*/
async syncCollection(
ref: CollectionRef,
songIds: readonly string[],
loadSongs: (songIds: string[]) => Promise<DownloadableSong[]>
) {
await load()
const key = collectionKey(ref)
const collection = collections.get(key)
if (!collection) return
const before = new Set(collection.songIds)
const after = new Set(songIds)
const added = songIds.filter((songId) => !before.has(songId))
const dropped = collection.songIds.filter((songId) => !after.has(songId))
if (added.length === 0 && dropped.length === 0) return
collections.set(key, {
...collection,
songIds: [...songIds],
removedSongIds: collection.removedSongIds?.filter((songId) =>
after.has(songId)
),
})
await saveIndex()
await deleteUnheld(dropped)
if (added.length > 0) {
enqueueLoaded(await loadSongs(added), collection.quality)
notify()
pump()
}
},
/** Forgets a collection and removes its songs nothing else holds. */
async removeCollection(ref: CollectionRef) {
await load()
const key = collectionKey(ref)
const collection = collections.get(key)
if (!collection) return
collections.delete(key)
await saveIndex()
await deleteUnheld(collection.songIds)
pruneCovers()
},
/**
* Removes song downloads from one download: `from` when it is a downloaded
* collection, else Other Downloaded Songs. A song stays on the device while
* anything else holds it.
*/
async removeSongs(songIds: readonly string[], from?: DownloadSource) {
await load()
if (from && from !== "other" && collections.has(collectionKey(from))) {
removeFromCollection(collectionKey(from), songIds)
} else {
for (const songId of songIds) otherSongIds.delete(songId)
}
notify()
await saveIndex()
await deleteUnheld(songIds)
},
/** Empties Other Downloaded Songs, keeping songs a collection holds. */
async removeOtherSongs() {
await load()
const songIds = [...otherSongIds]
otherSongIds.clear()
await saveIndex()
await deleteUnheld(songIds)
},
async removeAll() {
await load()
collections.clear()
otherSongIds.clear()
await saveIndex()
await deleteAudio([
...downloaded.keys(),
...running.keys(),
...waiting.map(({ song }) => song.id),
])
},
/** Where download storage goes, for settings. */
breakdown(): DownloadBreakdown {
const byQuality = DOWNLOAD_QUALITIES.map((quality) => ({
quality,
bytes: 0,
songs: 0,
}))
let bytes = 0
for (const song of downloaded.values()) {
const entry = byQuality.find(({ quality }) => quality === song.quality)!
entry.bytes += song.bytes
entry.songs += 1
bytes += song.bytes
}
const collectionRows = [...collections.values()].map((collection) => {
let collectionBytes = 0
let fallbackCount = 0
const songIds = collectionSongIds(collection)
for (const songId of songIds) {
const song = downloaded.get(songId)
if (!song) continue
collectionBytes += song.bytes
if (song.quality !== collection.quality) fallbackCount += 1
}
return {
collection,
bytes: collectionBytes,
status: status(songIds),
fallbackCount,
}
})
const otherSongs = { bytes: 0, songIds: [] as string[] }
for (const songId of otherSongIds) {
const song = downloaded.get(songId)
if (!song) continue
otherSongs.bytes += song.bytes
otherSongs.songIds.push(songId)
}
return {
bytes,
songs: downloaded.size,
queued: running.size + waiting.length,
byQuality,
collections: collectionRows.sort((a, b) => b.bytes - a.bytes),
otherSongs,
}
},
}
}