fix(convex): scale library scans without changing search or playback (#107)
This commit was merged in pull request #107.
This commit is contained in:
1 parent
1d167d4726
commit
dbb774df0d
6 files changed
+910
-677
No files matched your search
+454
-466
@@ -61,512 +61,494 @@ async function cleanHlsStreamsOlderThan(maxIdleMs: number) {
|
||||
return { checked, removed, failed }
|
||||
}
|
||||
|
||||
async function coverPathIfExists(path: string): Promise<string | null> {
|
||||
return fs
|
||||
.access(path)
|
||||
.then(() => path)
|
||||
.catch(() => null)
|
||||
}
|
||||
const SCAN_PAGE_SIZE = 2000
|
||||
/** Outlives the 10 minute action timeout so a crashed scan frees the lock. */
|
||||
const SCAN_LOCK_LEASE_MS = 15 * 60 * 1000
|
||||
/** New files left unparsed after this are picked up by the next scan. */
|
||||
const SCAN_PARSE_BUDGET_MS = 6 * 60 * 1000
|
||||
const SCAN_BATCH_SIZE = 200
|
||||
const FS_CONCURRENCY = 16
|
||||
|
||||
async function syncArtistCover(
|
||||
ctx: ActionCtx,
|
||||
codec: string,
|
||||
dbCover: string | null,
|
||||
diskPath: string,
|
||||
id: Id<"artists">
|
||||
) {
|
||||
const onDisk = await coverPathIfExists(diskPath)
|
||||
if (onDisk !== null) {
|
||||
if (dbCover !== onDisk) {
|
||||
await ctx.runMutation(internal.library.updateArtistCover, {
|
||||
id,
|
||||
cover: onDisk,
|
||||
})
|
||||
type Page<T> = { page: T[]; isDone: boolean; continueCursor: string }
|
||||
|
||||
/** Reads every page of an internal paginated query. */
|
||||
async function collectPages<T>(
|
||||
load: (cursor: string | null) => Promise<Page<T>>
|
||||
): Promise<T[]> {
|
||||
const items: T[] = []
|
||||
let cursor: string | null = null
|
||||
for (;;) {
|
||||
const result = await load(cursor)
|
||||
items.push(...result.page)
|
||||
if (result.isDone) {
|
||||
return items
|
||||
}
|
||||
} else if (dbCover !== null && dbCover.includes(codec)) {
|
||||
await ctx.runMutation(internal.library.updateArtistCover, {
|
||||
id,
|
||||
cover: null,
|
||||
})
|
||||
cursor = result.continueCursor
|
||||
}
|
||||
}
|
||||
|
||||
async function syncAlbumCover(
|
||||
ctx: ActionCtx,
|
||||
codec: string,
|
||||
dbCover: string | null,
|
||||
diskPath: string,
|
||||
id: Id<"albums">
|
||||
) {
|
||||
const onDisk = await coverPathIfExists(diskPath)
|
||||
if (onDisk !== null) {
|
||||
if (dbCover !== onDisk) {
|
||||
await ctx.runMutation(internal.library.updateAlbumCover, {
|
||||
id,
|
||||
cover: onDisk,
|
||||
})
|
||||
/** Like `Promise.all(items.map(fn))` with at most `limit` calls in flight. */
|
||||
async function mapLimit<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length)
|
||||
let next = 0
|
||||
const worker = async () => {
|
||||
while (next < items.length) {
|
||||
const index = next++
|
||||
results[index] = await fn(items[index])
|
||||
}
|
||||
} else if (dbCover !== null && dbCover.includes(codec)) {
|
||||
await ctx.runMutation(internal.library.updateAlbumCover, {
|
||||
id,
|
||||
cover: null,
|
||||
})
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(limit, items.length) }, worker)
|
||||
)
|
||||
return results
|
||||
}
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
const chunks: T[][] = []
|
||||
for (let i = 0; i < items.length; i += size) {
|
||||
chunks.push(items.slice(i, i + size))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
async function readDirents(dir: string): Promise<fsSync.Dirent[]> {
|
||||
try {
|
||||
return await fs.readdir(dir, { withFileTypes: true })
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return []
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function purgeRemovedSongs(ctx: ActionCtx, ids: Id<"songs">[]) {
|
||||
if (ids.length === 0) {
|
||||
return
|
||||
}
|
||||
await ctx.runMutation(internal.library.purgeSongsFromUsersAndPlaylists, {
|
||||
ids,
|
||||
type DiskAlbum = {
|
||||
codec: string
|
||||
artistFolder: string
|
||||
albumFolder: string
|
||||
files: string[]
|
||||
}
|
||||
|
||||
type DiskLibrary = {
|
||||
/** Every audio file path. */
|
||||
files: Set<string>
|
||||
/** Every `folder.jpg` / `cover.jpg` path. */
|
||||
covers: Set<string>
|
||||
albums: DiskAlbum[]
|
||||
}
|
||||
|
||||
/** Lists `DATA_DIR/<codec>/<artist>/<album>/<file>` with one readdir per directory. */
|
||||
async function walkLibrary(codecs: string[]): Promise<DiskLibrary> {
|
||||
const files = new Set<string>()
|
||||
const covers = new Set<string>()
|
||||
|
||||
const artistDirs = (
|
||||
await Promise.all(
|
||||
codecs.map(async (codec) =>
|
||||
(await readDirents(`${DATA_DIR}/${codec}`))
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => ({ codec, artistFolder: dirent.name }))
|
||||
)
|
||||
)
|
||||
).flat()
|
||||
|
||||
const albumDirs = (
|
||||
await mapLimit(
|
||||
artistDirs,
|
||||
FS_CONCURRENCY,
|
||||
async ({ codec, artistFolder }) => {
|
||||
const artistPath = `${DATA_DIR}/${codec}/${artistFolder}`
|
||||
const dirents = await readDirents(artistPath)
|
||||
if (dirents.some((d) => !d.isDirectory() && d.name === "folder.jpg")) {
|
||||
covers.add(`${artistPath}/folder.jpg`)
|
||||
}
|
||||
return dirents
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => ({ codec, artistFolder, albumFolder: dirent.name }))
|
||||
}
|
||||
)
|
||||
).flat()
|
||||
|
||||
const albums = await mapLimit(albumDirs, FS_CONCURRENCY, async (album) => {
|
||||
const albumPath = `${DATA_DIR}/${album.codec}/${album.artistFolder}/${album.albumFolder}`
|
||||
const albumFiles: string[] = []
|
||||
for (const dirent of await readDirents(albumPath)) {
|
||||
if (dirent.isDirectory()) {
|
||||
continue
|
||||
}
|
||||
const filePath = `${albumPath}/${dirent.name}`
|
||||
if (dirent.name === "cover.jpg") {
|
||||
covers.add(filePath)
|
||||
} else {
|
||||
files.add(filePath)
|
||||
albumFiles.push(filePath)
|
||||
}
|
||||
}
|
||||
return { ...album, files: albumFiles }
|
||||
})
|
||||
|
||||
return { files, covers, albums }
|
||||
}
|
||||
|
||||
async function purgeRemovedAlbums(ctx: ActionCtx, ids: Id<"albums">[]) {
|
||||
if (ids.length === 0) {
|
||||
return
|
||||
/**
|
||||
* Keeps the current cover while its file exists, otherwise the first codec's
|
||||
* cover on disk. Covers outside the data directory are left alone.
|
||||
*/
|
||||
function pickCover(
|
||||
current: string | null,
|
||||
candidates: string[],
|
||||
covers: Set<string>
|
||||
): string | null {
|
||||
if (
|
||||
current !== null &&
|
||||
(covers.has(current) || !current.startsWith(`${DATA_DIR}/`))
|
||||
) {
|
||||
return current
|
||||
}
|
||||
await ctx.runMutation(internal.library.purgeAlbumsFromUsers, { ids })
|
||||
return candidates.find((candidate) => covers.has(candidate)) ?? null
|
||||
}
|
||||
|
||||
async function purgeRemovedArtists(ctx: ActionCtx, ids: Id<"artists">[]) {
|
||||
if (ids.length === 0) {
|
||||
return
|
||||
/** Untagged years are stored as 0 / "", which both clients treat as unknown. */
|
||||
function releaseDate(common: music.ICommonTagsResult) {
|
||||
if (common.date) {
|
||||
return common.date
|
||||
}
|
||||
await ctx.runMutation(internal.library.purgeArtistsFromUsers, { ids })
|
||||
return common.year ? `01-01-${common.year}` : ""
|
||||
}
|
||||
|
||||
async function removeCodecFromDatabase(ctx: ActionCtx, codec: string) {
|
||||
const allSongs = await ctx.runQuery(internal.library.getSongs)
|
||||
type NewSong = {
|
||||
name: string
|
||||
files: Record<string, string>
|
||||
artist: string | string[]
|
||||
length: number
|
||||
diskNumber: number | null
|
||||
track: number
|
||||
year: number
|
||||
date: string
|
||||
genres: string[]
|
||||
}
|
||||
|
||||
const songsToRemove: Id<"songs">[] = []
|
||||
const songsToUpdate: { id: Id<"songs">; files: Record<string, string> }[] = []
|
||||
for (const song of allSongs) {
|
||||
if (!(codec in song.files)) {
|
||||
type ImportEntry = {
|
||||
artist: { name: string; folder: string; cover: string | null }
|
||||
album: {
|
||||
name: string
|
||||
folder: string
|
||||
cover: string | null
|
||||
year: number
|
||||
date: string
|
||||
}
|
||||
songs: Map<string, NewSong>
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconciles the database with the files on disk. Reads the library once in
|
||||
* pages, walks the disk once, parses only untracked files and writes only
|
||||
* what changed, so re-scanning an unchanged library makes no writes.
|
||||
*/
|
||||
async function scanLibrary(ctx: ActionCtx) {
|
||||
const startedAt = Date.now()
|
||||
const codecs = (await fs.readdir(DATA_DIR, { withFileTypes: true }))
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => dirent.name)
|
||||
|
||||
const [artists, albums, songs, disk] = await Promise.all([
|
||||
collectPages((cursor) =>
|
||||
ctx.runQuery(internal.library.scanArtistsPage, {
|
||||
paginationOpts: { numItems: SCAN_PAGE_SIZE, cursor },
|
||||
})
|
||||
),
|
||||
collectPages((cursor) =>
|
||||
ctx.runQuery(internal.library.scanAlbumsPage, {
|
||||
paginationOpts: { numItems: SCAN_PAGE_SIZE, cursor },
|
||||
})
|
||||
),
|
||||
collectPages((cursor) =>
|
||||
ctx.runQuery(internal.library.scanSongsPage, {
|
||||
paginationOpts: { numItems: SCAN_PAGE_SIZE, cursor },
|
||||
})
|
||||
),
|
||||
walkLibrary(codecs),
|
||||
])
|
||||
|
||||
if (disk.files.size === 0 && songs.length > 0) {
|
||||
throw new Error(
|
||||
`No audio files found under ${DATA_DIR}; refusing to empty the library`
|
||||
)
|
||||
}
|
||||
|
||||
// Drop file paths that are gone from disk (including removed codecs). Songs
|
||||
// left without files are deleted below unless a new file matches them.
|
||||
const songFiles = new Map<Id<"songs">, Record<string, string>>()
|
||||
const changedSongs = new Set<Id<"songs">>()
|
||||
const songsByAlbum = new Map<Id<"albums">, Map<string, Id<"songs">>>()
|
||||
const tracked = new Set<string>()
|
||||
for (const song of songs) {
|
||||
const entries = Object.entries(song.files)
|
||||
const kept = entries.filter(([, filePath]) => disk.files.has(filePath))
|
||||
if (kept.length !== entries.length) {
|
||||
changedSongs.add(song._id)
|
||||
}
|
||||
songFiles.set(song._id, Object.fromEntries(kept))
|
||||
for (const [, filePath] of kept) {
|
||||
tracked.add(filePath)
|
||||
}
|
||||
const byName = songsByAlbum.get(song.album) ?? new Map()
|
||||
if (!byName.has(song.name)) {
|
||||
byName.set(song.name, song._id)
|
||||
}
|
||||
songsByAlbum.set(song.album, byName)
|
||||
}
|
||||
|
||||
// Parse untracked files only.
|
||||
const untracked = disk.albums.flatMap((album) =>
|
||||
album.files
|
||||
.filter((filePath) => !tracked.has(filePath))
|
||||
.map((filePath) => ({ ...album, filePath }))
|
||||
)
|
||||
const deadline = startedAt + SCAN_PARSE_BUDGET_MS
|
||||
const parsed = await mapLimit(untracked, FS_CONCURRENCY, async (file) => {
|
||||
if (Date.now() > deadline) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const metadata = await music.parseFile(file.filePath, {
|
||||
skipCovers: true,
|
||||
})
|
||||
return { ...file, metadata }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
})
|
||||
const skipped = parsed.filter((file) => file === null).length
|
||||
|
||||
const artistByFolder = new Map(
|
||||
artists.map((artist) => [artist.folder, artist])
|
||||
)
|
||||
const albumByKey = new Map(
|
||||
albums.map((album) => [`${album.artist}/${album.folder}`, album])
|
||||
)
|
||||
const imports = new Map<string, ImportEntry>()
|
||||
for (const file of parsed) {
|
||||
if (file === null) {
|
||||
continue
|
||||
}
|
||||
const { codec, artistFolder, albumFolder, filePath, metadata } = file
|
||||
const common = metadata.common
|
||||
const title = common.title || path.parse(filePath).name
|
||||
|
||||
const dbArtist = artistByFolder.get(artistFolder)
|
||||
const dbAlbum = dbArtist
|
||||
? albumByKey.get(`${dbArtist._id}/${albumFolder}`)
|
||||
: undefined
|
||||
const existingId = dbAlbum && songsByAlbum.get(dbAlbum._id)?.get(title)
|
||||
if (existingId) {
|
||||
// A same-titled sibling must not steal a path that still exists.
|
||||
const files = songFiles.get(existingId)!
|
||||
if (!(codec in files)) {
|
||||
songFiles.set(existingId, { ...files, [codec]: filePath })
|
||||
changedSongs.add(existingId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
const { [codec]: _, ...update } = song.files
|
||||
if (Object.keys(update).length === 0) {
|
||||
songsToRemove.push(song._id)
|
||||
} else {
|
||||
songsToUpdate.push({
|
||||
id: song._id,
|
||||
files: update,
|
||||
})
|
||||
const key = `${artistFolder}/${albumFolder}`
|
||||
let entry = imports.get(key)
|
||||
if (!entry) {
|
||||
entry = {
|
||||
artist: {
|
||||
name: common.albumartist || artistFolder,
|
||||
folder: artistFolder,
|
||||
cover: pickCover(
|
||||
null,
|
||||
codecs.map((c) => `${DATA_DIR}/${c}/${artistFolder}/folder.jpg`),
|
||||
disk.covers
|
||||
),
|
||||
},
|
||||
album: {
|
||||
name: common.album || albumFolder,
|
||||
folder: albumFolder,
|
||||
cover: pickCover(
|
||||
null,
|
||||
codecs.map(
|
||||
(c) => `${DATA_DIR}/${c}/${artistFolder}/${albumFolder}/cover.jpg`
|
||||
),
|
||||
disk.covers
|
||||
),
|
||||
year: common.year || 0,
|
||||
date: releaseDate(common),
|
||||
},
|
||||
songs: new Map(),
|
||||
}
|
||||
imports.set(key, entry)
|
||||
}
|
||||
|
||||
const pending = entry.songs.get(title)
|
||||
if (pending) {
|
||||
pending.files[codec] ??= filePath
|
||||
continue
|
||||
}
|
||||
entry.songs.set(title, {
|
||||
name: title,
|
||||
files: { [codec]: filePath },
|
||||
artist:
|
||||
common.artist ?? common.artists ?? dbArtist?.name ?? entry.artist.name,
|
||||
length: Math.round(metadata.format.duration || 0),
|
||||
diskNumber: common.disk.of === 1 ? null : common.disk.no,
|
||||
track: common.track.no || 0,
|
||||
year: common.year || 0,
|
||||
date: releaseDate(common),
|
||||
genres: common.genre || [],
|
||||
})
|
||||
}
|
||||
|
||||
await ctx.runMutation(internal.library.changeFilesForSongs, {
|
||||
songs: songsToUpdate,
|
||||
})
|
||||
await purgeRemovedSongs(ctx, songsToRemove)
|
||||
await ctx.runMutation(internal.library.removeSongs, { ids: songsToRemove })
|
||||
|
||||
const allAlbums = await ctx.runQuery(internal.library.getAlbums)
|
||||
const removedSongIds = new Set(songsToRemove)
|
||||
const albumsWithSongs = new Set(
|
||||
allSongs.flatMap((song) =>
|
||||
removedSongIds.has(song._id) ? [] : [song.album]
|
||||
)
|
||||
const deletedSongs = songs.flatMap((song) =>
|
||||
Object.keys(songFiles.get(song._id)!).length === 0 ? [song._id] : []
|
||||
)
|
||||
const albumsToRemove = allAlbums
|
||||
const deletedSongSet = new Set(deletedSongs)
|
||||
const updatedSongs = [...changedSongs].filter((id) => !deletedSongSet.has(id))
|
||||
|
||||
const albumsWithSongs = new Set(
|
||||
songs.flatMap((song) => (deletedSongSet.has(song._id) ? [] : [song.album]))
|
||||
)
|
||||
const emptyAlbums = albums
|
||||
.filter((album) => !albumsWithSongs.has(album._id))
|
||||
.map((album) => album._id)
|
||||
await purgeRemovedAlbums(ctx, albumsToRemove)
|
||||
await ctx.runMutation(internal.library.removeAlbums, { ids: albumsToRemove })
|
||||
|
||||
const allArtists = await ctx.runQuery(internal.library.getArtists)
|
||||
const removedAlbumIds = new Set(albumsToRemove)
|
||||
const emptyAlbumSet = new Set(emptyAlbums)
|
||||
const artistsWithAlbums = new Set(
|
||||
allAlbums.flatMap((album) =>
|
||||
removedAlbumIds.has(album._id) ? [] : [album.artist]
|
||||
albums.flatMap((album) =>
|
||||
emptyAlbumSet.has(album._id) ? [] : [album.artist]
|
||||
)
|
||||
)
|
||||
const artistsToRemove = allArtists
|
||||
const emptyArtists = artists
|
||||
.filter((artist) => !artistsWithAlbums.has(artist._id))
|
||||
.map((artist) => artist._id)
|
||||
await purgeRemovedArtists(ctx, artistsToRemove)
|
||||
await ctx.runMutation(internal.library.removeArtists, {
|
||||
ids: artistsToRemove,
|
||||
const emptyArtistSet = new Set(emptyArtists)
|
||||
|
||||
// Albums and artists about to be deleted keep their cover.
|
||||
const artistById = new Map(artists.map((artist) => [artist._id, artist]))
|
||||
const artistCovers = artists.flatMap((artist) => {
|
||||
if (emptyArtistSet.has(artist._id)) {
|
||||
return []
|
||||
}
|
||||
const cover = pickCover(
|
||||
artist.cover,
|
||||
codecs.map((c) => `${DATA_DIR}/${c}/${artist.folder}/folder.jpg`),
|
||||
disk.covers
|
||||
)
|
||||
return cover === artist.cover ? [] : [{ id: artist._id, cover }]
|
||||
})
|
||||
const albumCovers = albums.flatMap((album) => {
|
||||
const artist = artistById.get(album.artist)
|
||||
if (!artist || emptyAlbumSet.has(album._id)) {
|
||||
return []
|
||||
}
|
||||
const cover = pickCover(
|
||||
album.cover,
|
||||
codecs.map(
|
||||
(c) => `${DATA_DIR}/${c}/${artist.folder}/${album.folder}/cover.jpg`
|
||||
),
|
||||
disk.covers
|
||||
)
|
||||
return cover === album.cover ? [] : [{ id: album._id, cover }]
|
||||
})
|
||||
|
||||
// Writes. Imports run before the empty-album/artist sweeps, and those
|
||||
// mutations re-check emptiness, so nothing that gained songs is deleted.
|
||||
for (const batch of chunk(updatedSongs, SCAN_BATCH_SIZE)) {
|
||||
await ctx.runMutation(internal.library.changeFilesForSongs, {
|
||||
songs: batch.map((id) => ({ id, files: songFiles.get(id)! })),
|
||||
})
|
||||
}
|
||||
for (const batch of chunk(deletedSongs, SCAN_BATCH_SIZE)) {
|
||||
await ctx.runMutation(internal.library.deleteSongs, { ids: batch })
|
||||
}
|
||||
|
||||
let importBatch: ImportEntry[] = []
|
||||
let importBatchSongs = 0
|
||||
const flushImports = async () => {
|
||||
if (importBatch.length === 0) {
|
||||
return
|
||||
}
|
||||
await ctx.runMutation(internal.library.importAlbums, {
|
||||
albums: importBatch.map((entry) => ({
|
||||
...entry,
|
||||
songs: [...entry.songs.values()],
|
||||
})),
|
||||
})
|
||||
importBatch = []
|
||||
importBatchSongs = 0
|
||||
}
|
||||
for (const entry of imports.values()) {
|
||||
importBatch.push(entry)
|
||||
importBatchSongs += entry.songs.size
|
||||
if (importBatchSongs >= SCAN_BATCH_SIZE) {
|
||||
await flushImports()
|
||||
}
|
||||
}
|
||||
await flushImports()
|
||||
|
||||
for (const batch of chunk(artistCovers, SCAN_BATCH_SIZE)) {
|
||||
await ctx.runMutation(internal.library.updateArtistCovers, {
|
||||
artists: batch,
|
||||
})
|
||||
}
|
||||
for (const batch of chunk(albumCovers, SCAN_BATCH_SIZE)) {
|
||||
await ctx.runMutation(internal.library.updateAlbumCovers, {
|
||||
albums: batch,
|
||||
})
|
||||
}
|
||||
|
||||
let removedAlbums = 0
|
||||
for (const batch of chunk(emptyAlbums, SCAN_BATCH_SIZE)) {
|
||||
removedAlbums += await ctx.runMutation(internal.library.deleteEmptyAlbums, {
|
||||
ids: batch,
|
||||
})
|
||||
}
|
||||
|
||||
let removedArtists = 0
|
||||
for (const batch of chunk(emptyArtists, SCAN_BATCH_SIZE)) {
|
||||
removedArtists += await ctx.runMutation(
|
||||
internal.library.deleteEmptyArtists,
|
||||
{ ids: batch }
|
||||
)
|
||||
}
|
||||
|
||||
const importedSongs = [...imports.values()].reduce(
|
||||
(total, entry) => total + entry.songs.size,
|
||||
0
|
||||
)
|
||||
const summary =
|
||||
`songs: ${importedSongs} added, ${updatedSongs.length} updated, ` +
|
||||
`${deletedSongs.length} removed; albums removed: ${removedAlbums}; ` +
|
||||
`artists removed: ${removedArtists}; covers updated: ` +
|
||||
`${artistCovers.length + albumCovers.length}; ` +
|
||||
`files deferred: ${skipped}; ${Date.now() - startedAt}ms`
|
||||
console.log(`library scan: ${summary}`)
|
||||
return summary
|
||||
}
|
||||
|
||||
/** Runs one scan under the library scan lock; `null` if one is already running. */
|
||||
export const scheduledUpdateLibrary = internalAction({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
const codecs = (await fs.readdir(DATA_DIR, { withFileTypes: true }))
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => dirent.name)
|
||||
|
||||
const dbCodecs = Array.from(
|
||||
new Map(
|
||||
(await ctx.runQuery(internal.library.getSongs))
|
||||
.map((song) => {
|
||||
return Object.keys(song.files)
|
||||
})
|
||||
.flat()
|
||||
.map((codec) => [codec, codec])
|
||||
).values()
|
||||
handler: async (ctx): Promise<string | null> => {
|
||||
const token = crypto.randomUUID()
|
||||
const acquired: boolean = await ctx.runMutation(
|
||||
internal.library.acquireLibraryScanLock,
|
||||
{ token, leaseMs: SCAN_LOCK_LEASE_MS }
|
||||
)
|
||||
|
||||
for (const dbCodec of dbCodecs) {
|
||||
if (!codecs.includes(dbCodec)) {
|
||||
await removeCodecFromDatabase(ctx, dbCodec)
|
||||
}
|
||||
if (!acquired) {
|
||||
console.log("library scan: already running, skipped")
|
||||
return null
|
||||
}
|
||||
|
||||
// Removals
|
||||
for (const codec of codecs) {
|
||||
const dbArtists = await ctx.runQuery(internal.library.getArtists)
|
||||
const dbAlbums = await ctx.runQuery(internal.library.getAlbums)
|
||||
const dbSongs = await ctx.runQuery(internal.library.getSongs)
|
||||
|
||||
const dirtySongs = dbSongs
|
||||
.filter((song) => {
|
||||
return codec in song.files && !fsSync.existsSync(song.files[codec])
|
||||
})
|
||||
.map((song) => {
|
||||
const { [codec]: _, ...update } = song.files
|
||||
return { id: song._id, files: update }
|
||||
})
|
||||
|
||||
const songsToDelete = dirtySongs.filter(
|
||||
(song) => Object.keys(song.files).length === 0
|
||||
)
|
||||
const songsToUpdate = dirtySongs.filter(
|
||||
(song) => Object.keys(song.files).length !== 0
|
||||
)
|
||||
const songsToDeleteIDs = songsToDelete.map((song) => song.id)
|
||||
|
||||
await purgeRemovedSongs(ctx, songsToDeleteIDs)
|
||||
await ctx.runMutation(internal.library.removeSongs, {
|
||||
ids: songsToDeleteIDs,
|
||||
})
|
||||
await ctx.runMutation(internal.library.changeFilesForSongs, {
|
||||
songs: songsToUpdate,
|
||||
})
|
||||
|
||||
const deletedSongIds = new Set(songsToDeleteIDs)
|
||||
const albumsWithSongs = new Set(
|
||||
dbSongs.flatMap((song) =>
|
||||
deletedSongIds.has(song._id) ? [] : [song.album]
|
||||
)
|
||||
)
|
||||
const albumsToDelete = dbAlbums
|
||||
.filter((album) => !albumsWithSongs.has(album._id))
|
||||
.map((album) => album._id)
|
||||
await purgeRemovedAlbums(ctx, albumsToDelete)
|
||||
await ctx.runMutation(internal.library.removeAlbums, {
|
||||
ids: albumsToDelete,
|
||||
})
|
||||
|
||||
const deletedAlbumIds = new Set(albumsToDelete)
|
||||
const artistsWithAlbums = new Set(
|
||||
dbAlbums.flatMap((album) =>
|
||||
deletedAlbumIds.has(album._id) ? [] : [album.artist]
|
||||
)
|
||||
)
|
||||
const artistsToDelete = dbArtists
|
||||
.filter((artist) => !artistsWithAlbums.has(artist._id))
|
||||
.map((artist) => artist._id)
|
||||
await purgeRemovedArtists(ctx, artistsToDelete)
|
||||
await ctx.runMutation(internal.library.removeArtists, {
|
||||
ids: artistsToDelete,
|
||||
})
|
||||
try {
|
||||
return await scanLibrary(ctx)
|
||||
} finally {
|
||||
await ctx.runMutation(internal.library.releaseLibraryScanLock, { token })
|
||||
}
|
||||
|
||||
// Additions
|
||||
for (const codec of codecs) {
|
||||
// Artists
|
||||
console.log(`${codec}: artists`)
|
||||
let dbArtists = await ctx.runQuery(internal.library.getArtists)
|
||||
const dbArtistFolders = dbArtists.map((artist) => artist.folder)
|
||||
const artists = (
|
||||
await fs.readdir(`${DATA_DIR}/${codec}`, { withFileTypes: true })
|
||||
)
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => dirent.name)
|
||||
const potentialArtists = artists.filter(
|
||||
(artist) => !dbArtistFolders.includes(artist)
|
||||
)
|
||||
|
||||
const newArtists = []
|
||||
for (const folder of potentialArtists) {
|
||||
const albums = (
|
||||
await fs.readdir(`${DATA_DIR}/${codec}/${folder}`, {
|
||||
withFileTypes: true,
|
||||
})
|
||||
)
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => dirent.name)
|
||||
|
||||
if (albums.length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
for (let i = 0; i < albums.length; i++) {
|
||||
const songs = (
|
||||
await fs.readdir(`${DATA_DIR}/${codec}/${folder}/${albums[i]}`, {
|
||||
withFileTypes: true,
|
||||
})
|
||||
)
|
||||
.filter(
|
||||
(dirent) => !dirent.isDirectory() && dirent.name !== "cover.jpg"
|
||||
)
|
||||
.map((dirent) => dirent.name)
|
||||
|
||||
if (songs.length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const metadata = await music.parseFile(
|
||||
`${DATA_DIR}/${codec}/${folder}/${albums[i]}/${songs[0]}`,
|
||||
{ skipCovers: true }
|
||||
)
|
||||
|
||||
const artistCoverPath = `${DATA_DIR}/${codec}/${folder}/folder.jpg`
|
||||
const cover = await coverPathIfExists(artistCoverPath)
|
||||
|
||||
newArtists.push({
|
||||
name: metadata.common.albumartist || folder,
|
||||
folder,
|
||||
cover,
|
||||
})
|
||||
break
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
await ctx.runMutation(internal.library.createArtists, {
|
||||
artists: newArtists,
|
||||
})
|
||||
dbArtists = await ctx.runQuery(internal.library.getArtists)
|
||||
|
||||
// Albums
|
||||
console.log(`${codec}: albums`)
|
||||
let dbAlbums = await ctx.runQuery(internal.library.getAlbums)
|
||||
const newAlbums = []
|
||||
for (const dbArtist of dbArtists) {
|
||||
const dbAlbumsInArtist = dbAlbums.filter(
|
||||
(album) => album.artist === dbArtist._id
|
||||
)
|
||||
const dbAlbumFolders = dbAlbumsInArtist.map((album) => album.folder)
|
||||
|
||||
if (!fsSync.existsSync(`${DATA_DIR}/${codec}/${dbArtist.folder}`)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const albums = (
|
||||
await fs.readdir(`${DATA_DIR}/${codec}/${dbArtist.folder}`, {
|
||||
withFileTypes: true,
|
||||
})
|
||||
)
|
||||
.filter((dirent) => dirent.isDirectory())
|
||||
.map((dirent) => dirent.name)
|
||||
const potentialAlbums = albums.filter(
|
||||
(album) => !dbAlbumFolders.includes(album)
|
||||
)
|
||||
|
||||
for (const potentialAlbum of potentialAlbums) {
|
||||
const potentialAlbumPath = `${DATA_DIR}/${codec}/${dbArtist.folder}/${potentialAlbum}`
|
||||
if (!fsSync.existsSync(potentialAlbumPath)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const songs = (
|
||||
await fs.readdir(potentialAlbumPath, {
|
||||
withFileTypes: true,
|
||||
})
|
||||
)
|
||||
.filter(
|
||||
(dirent) => !dirent.isDirectory() && dirent.name !== "cover.jpg"
|
||||
)
|
||||
.map((dirent) => dirent.name)
|
||||
|
||||
if (songs.length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const metadata = await music.parseFile(
|
||||
`${potentialAlbumPath}/${songs[0]}`,
|
||||
{ skipCovers: true }
|
||||
)
|
||||
|
||||
const albumCoverPath = `${DATA_DIR}/${codec}/${dbArtist.folder}/${potentialAlbum}/cover.jpg`
|
||||
const cover = await coverPathIfExists(albumCoverPath)
|
||||
|
||||
newAlbums.push({
|
||||
name: metadata.common.album || potentialAlbum,
|
||||
folder: potentialAlbum,
|
||||
cover,
|
||||
year: metadata.common.year || 2026,
|
||||
date:
|
||||
metadata.common.date || `01-01-${metadata.common.year || 2026}`,
|
||||
artist: dbArtist._id,
|
||||
})
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
await ctx.runMutation(internal.library.createAlbums, {
|
||||
albums: newAlbums,
|
||||
})
|
||||
dbAlbums = await ctx.runQuery(internal.library.getAlbums)
|
||||
const dbSongs = await ctx.runQuery(internal.library.getSongs)
|
||||
|
||||
const dbAlbumsArtists = dbAlbums.map((album) => {
|
||||
return {
|
||||
...album,
|
||||
artist: dbArtists.find((artist) => artist._id === album.artist)!,
|
||||
}
|
||||
})
|
||||
|
||||
// Covers
|
||||
console.log(`${codec}: covers`)
|
||||
for (const dbArtist of dbArtists) {
|
||||
const artistPath = `${DATA_DIR}/${codec}/${dbArtist.folder}`
|
||||
if (!fsSync.existsSync(artistPath)) {
|
||||
continue
|
||||
}
|
||||
await syncArtistCover(
|
||||
ctx,
|
||||
codec,
|
||||
dbArtist.cover,
|
||||
`${artistPath}/folder.jpg`,
|
||||
dbArtist._id
|
||||
)
|
||||
}
|
||||
|
||||
for (const dbAlbum of dbAlbumsArtists) {
|
||||
const artistPath = `${DATA_DIR}/${codec}/${dbAlbum.artist.folder}`
|
||||
const albumPath = `${artistPath}/${dbAlbum.folder}`
|
||||
if (!fsSync.existsSync(artistPath) || !fsSync.existsSync(albumPath)) {
|
||||
continue
|
||||
}
|
||||
await syncAlbumCover(
|
||||
ctx,
|
||||
codec,
|
||||
dbAlbum.cover,
|
||||
`${albumPath}/cover.jpg`,
|
||||
dbAlbum._id
|
||||
)
|
||||
}
|
||||
|
||||
// Songs
|
||||
console.log(`${codec}: songs`)
|
||||
|
||||
const newSongs = []
|
||||
const updateSongs = []
|
||||
for (const dbAlbum of dbAlbumsArtists) {
|
||||
const dbSongsInAlbum = dbSongs.filter(
|
||||
(song) => song.album === dbAlbum._id
|
||||
)
|
||||
const dbAlreadyTracked = dbSongsInAlbum
|
||||
.filter((song) => codec in song.files)
|
||||
.map((song) => song.files[codec])
|
||||
|
||||
const artistPath = `${DATA_DIR}/${codec}/${dbAlbum.artist.folder}`
|
||||
const albumPath = `${artistPath}/${dbAlbum.folder}`
|
||||
if (!fsSync.existsSync(artistPath) || !fsSync.existsSync(albumPath)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const songs = (await fs.readdir(albumPath, { withFileTypes: true }))
|
||||
.filter(
|
||||
(dirent) => !dirent.isDirectory() && dirent.name !== "cover.jpg"
|
||||
)
|
||||
.map((dirent) => dirent.name)
|
||||
.filter((song) => !dbAlreadyTracked.includes(`${albumPath}/${song}`))
|
||||
|
||||
for (const song of songs) {
|
||||
try {
|
||||
const path = `${DATA_DIR}/${codec}/${dbAlbum.artist.folder}/${dbAlbum.folder}/${song}`
|
||||
const metadata = await music.parseFile(path, { skipCovers: true })
|
||||
|
||||
if (!metadata.common.title) {
|
||||
continue
|
||||
}
|
||||
|
||||
const existingSong = dbSongsInAlbum.find(
|
||||
(dbSong) => dbSong.name === metadata.common.title
|
||||
)
|
||||
|
||||
if (existingSong) {
|
||||
if (
|
||||
!(codec in existingSong.files) ||
|
||||
existingSong.files[codec] !== path
|
||||
) {
|
||||
const update = { [codec]: path, ...existingSong.files }
|
||||
updateSongs.push({
|
||||
id: existingSong._id,
|
||||
files: update,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
let files = { [codec]: path }
|
||||
|
||||
if (codec !== "atmos") {
|
||||
const artistPath = `${DATA_DIR}/atmos/${dbAlbum.artist.folder}`
|
||||
const albumPath = `${artistPath}/${dbAlbum.folder}`
|
||||
const path = `${albumPath}/${song}`
|
||||
|
||||
if (fsSync.existsSync(path)) {
|
||||
files = { atmos: path, ...files }
|
||||
}
|
||||
}
|
||||
if (codec !== "lossless") {
|
||||
const artistPath = `${DATA_DIR}/lossless/${dbAlbum.artist.folder}`
|
||||
const albumPath = `${artistPath}/${dbAlbum.folder}`
|
||||
const path = `${albumPath}/${song}`
|
||||
|
||||
if (fsSync.existsSync(path)) {
|
||||
files = { lossless: path, ...files }
|
||||
}
|
||||
}
|
||||
if (codec !== "normal") {
|
||||
const artistPath = `${DATA_DIR}/normal/${dbAlbum.artist.folder}`
|
||||
const albumPath = `${artistPath}/${dbAlbum.folder}`
|
||||
const path = `${albumPath}/${song}`
|
||||
|
||||
if (fsSync.existsSync(path)) {
|
||||
files = { normal: path, ...files }
|
||||
}
|
||||
}
|
||||
|
||||
newSongs.push({
|
||||
name: metadata.common.title,
|
||||
files,
|
||||
album: dbAlbum._id,
|
||||
artist:
|
||||
metadata.common.artist ??
|
||||
metadata.common.artists ??
|
||||
dbAlbum.artist.name,
|
||||
albumArtist: dbAlbum.artist._id,
|
||||
length: Math.round(metadata.format.duration || 0),
|
||||
diskNumber:
|
||||
metadata.common.disk.of === 1
|
||||
? null
|
||||
: metadata.common.disk.no,
|
||||
track: metadata.common.track.no || 0,
|
||||
year: metadata.common.year || 2026,
|
||||
date:
|
||||
metadata.common.date ||
|
||||
`01-01-${metadata.common.year || 2026}`,
|
||||
genres: metadata.common.genre || [],
|
||||
})
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
await ctx.runMutation(internal.library.createSongs, { songs: newSongs })
|
||||
await ctx.runMutation(internal.library.changeFilesForSongs, {
|
||||
songs: updateSongs,
|
||||
})
|
||||
}
|
||||
|
||||
return "success"
|
||||
},
|
||||
})
|
||||
|
||||
@@ -574,7 +556,13 @@ export const updateLibrary = action({
|
||||
args: {},
|
||||
handler: async (ctx): Promise<string> => {
|
||||
await requireAdmin(ctx)
|
||||
return ctx.runAction(internal.actions.scheduledUpdateLibrary)
|
||||
const result: string | null = await ctx.runAction(
|
||||
internal.actions.scheduledUpdateLibrary
|
||||
)
|
||||
if (result === null) {
|
||||
throw new ConvexError("A library scan is already running")
|
||||
}
|
||||
return result
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ async function songEntities(ctx: QueryCtx, songs: Doc<"songs">[]) {
|
||||
albumId: album._id,
|
||||
duration: String(Math.max(0, Math.round(song.length))),
|
||||
genres: song.genres.join(", "),
|
||||
year: String(song.year),
|
||||
year: song.year ? String(song.year) : "",
|
||||
track: String(song.track),
|
||||
disk: String(song.diskNumber ?? ""),
|
||||
},
|
||||
@@ -112,7 +112,7 @@ async function albumEntities(ctx: QueryCtx, albums: Doc<"albums">[]) {
|
||||
metadata: {
|
||||
artistName: artist.name,
|
||||
artistId: artist._id,
|
||||
year: String(album.year),
|
||||
year: album.year ? String(album.year) : "",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
const { test } = require("node:test")
|
||||
const assert = require("node:assert/strict")
|
||||
const fs = require("node:fs/promises")
|
||||
const path = require("node:path")
|
||||
const os = require("node:os")
|
||||
const vm = require("node:vm")
|
||||
const ts = require("typescript")
|
||||
|
||||
// Exercise the action against real directory listings without a backend or
|
||||
// audio decoder. Metadata is fixed so renames and same-title copies match.
|
||||
async function scanner(root) {
|
||||
const module = { exports: {} }
|
||||
const source = ts.transpileModule(
|
||||
await fs.readFile(`${__dirname}/actions.ts`, "utf8"),
|
||||
{
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2022,
|
||||
},
|
||||
}
|
||||
).outputText
|
||||
const library = new Proxy({}, { get: (_, name) => name })
|
||||
vm.runInNewContext(source, {
|
||||
module,
|
||||
exports: module.exports,
|
||||
process: { env: { DATA: root } },
|
||||
crypto: require("node:crypto"),
|
||||
console: { log() {} },
|
||||
require(name) {
|
||||
if (name.startsWith("node:")) return require(name)
|
||||
if (name === "convex/values") return require(name)
|
||||
if (name === "./_generated/api") return { api: {}, internal: { library } }
|
||||
if (name === "./_generated/server")
|
||||
return { action: (x) => x, internalAction: (x) => x }
|
||||
if (name === "music-metadata")
|
||||
return {
|
||||
parseFile: async () => ({
|
||||
common: { title: "Song", disk: {}, track: {} },
|
||||
format: { duration: 60 },
|
||||
}),
|
||||
}
|
||||
throw new Error(`Unexpected import ${name}`)
|
||||
},
|
||||
})
|
||||
return module.exports.scheduledUpdateLibrary.handler
|
||||
}
|
||||
|
||||
test("rescans keep tracked songs and covers stable, and a rename keeps the song ID", async (t) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "otunes-scan-"))
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }))
|
||||
const folder = (codec) => path.join(root, codec, "The Abnormals", "Album")
|
||||
for (const codec of ["normal", "lossless"]) {
|
||||
await fs.mkdir(folder(codec), { recursive: true })
|
||||
await fs.writeFile(path.join(folder(codec), "song.mp3"), "audio")
|
||||
await fs.writeFile(path.join(folder(codec), "cover.jpg"), "cover")
|
||||
await fs.writeFile(path.join(folder(codec), "..", "folder.jpg"), "cover")
|
||||
}
|
||||
const artists = [
|
||||
{
|
||||
_id: "artist",
|
||||
name: "The Abnormals",
|
||||
folder: "The Abnormals",
|
||||
cover: path.join(root, "lossless", "The Abnormals", "folder.jpg"),
|
||||
},
|
||||
]
|
||||
const albums = [
|
||||
{
|
||||
_id: "album",
|
||||
artist: "artist",
|
||||
folder: "Album",
|
||||
cover: path.join(folder("lossless"), "cover.jpg"),
|
||||
},
|
||||
]
|
||||
const songs = [
|
||||
{
|
||||
_id: "song",
|
||||
name: "Song",
|
||||
album: "album",
|
||||
files: Object.fromEntries(
|
||||
["normal", "lossless"].map((c) => [c, path.join(folder(c), "song.mp3")])
|
||||
),
|
||||
},
|
||||
]
|
||||
const writes = []
|
||||
let locked = false
|
||||
const ctx = {
|
||||
async runQuery(name) {
|
||||
const page = {
|
||||
scanArtistsPage: artists,
|
||||
scanAlbumsPage: albums,
|
||||
scanSongsPage: songs,
|
||||
}[name]
|
||||
assert.ok(page, name)
|
||||
return { page, isDone: true, continueCursor: "" }
|
||||
},
|
||||
async runMutation(name, args) {
|
||||
if (name === "acquireLibraryScanLock") return !locked
|
||||
if (name === "releaseLibraryScanLock") return
|
||||
writes.push({ name, args })
|
||||
if (name === "changeFilesForSongs") {
|
||||
for (const update of args.songs)
|
||||
songs.find((s) => s._id === update.id).files = update.files
|
||||
return
|
||||
}
|
||||
throw new Error(`Unexpected write ${name}`)
|
||||
},
|
||||
}
|
||||
const scan = await scanner(root)
|
||||
await scan(ctx)
|
||||
assert.equal(writes.length, 0)
|
||||
await fs.writeFile(path.join(folder("normal"), "copy.mp3"), "audio")
|
||||
await scan(ctx)
|
||||
await scan(ctx)
|
||||
assert.equal(writes.length, 0)
|
||||
await fs.rm(path.join(folder("normal"), "copy.mp3"))
|
||||
await fs.rename(
|
||||
path.join(folder("normal"), "song.mp3"),
|
||||
path.join(folder("normal"), "renamed.mp3")
|
||||
)
|
||||
await scan(ctx)
|
||||
assert.equal(writes.length, 1)
|
||||
assert.equal(writes[0].args.songs[0].id, "song")
|
||||
assert.equal(
|
||||
songs[0].files.normal,
|
||||
path.join(folder("normal"), "renamed.mp3")
|
||||
)
|
||||
await scan(ctx)
|
||||
assert.equal(writes.length, 1)
|
||||
locked = true
|
||||
assert.equal(await scan(ctx), null)
|
||||
assert.equal(writes.length, 1)
|
||||
locked = false
|
||||
for (const codec of ["normal", "lossless"]) {
|
||||
await fs.rm(folder(codec), { recursive: true })
|
||||
}
|
||||
await assert.rejects(scan(ctx), /refusing to empty the library/)
|
||||
assert.equal(writes.length, 1)
|
||||
})
|
||||
+310
-209
@@ -3,7 +3,7 @@ import { getAuthUserId } from "@convex-dev/auth/server"
|
||||
import { internalMutation, internalQuery, query } from "./_generated/server"
|
||||
import { ConvexError, v } from "convex/values"
|
||||
import type { Doc, Id } from "./_generated/dataModel"
|
||||
import type { QueryCtx } from "./_generated/server"
|
||||
import type { MutationCtx, QueryCtx } from "./_generated/server"
|
||||
import type { PaginationOptions, PaginationResult } from "convex/server"
|
||||
import type {
|
||||
BrowseAlbum,
|
||||
@@ -545,7 +545,7 @@ export const browseAlbumDetail = query({
|
||||
|
||||
return {
|
||||
...toBrowseAlbum(album, artist),
|
||||
year: primarySong?.year ?? null,
|
||||
year: primarySong?.year || null,
|
||||
genre: primarySong?.genres[0] ?? null,
|
||||
releaseDate: album.date,
|
||||
hasLossless: songs.some((song) => "lossless" in song.files),
|
||||
@@ -845,27 +845,55 @@ export const browseRecentlyPlayedPaginated = query({
|
||||
},
|
||||
})
|
||||
|
||||
export const getArtists = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
return await ctx.db.query("artists").collect()
|
||||
export const scanArtistsPage = internalQuery({
|
||||
args: { paginationOpts: paginationOptsValidator },
|
||||
handler: async (ctx, args) => {
|
||||
const result = await ctx.db.query("artists").paginate(args.paginationOpts)
|
||||
return {
|
||||
...result,
|
||||
page: result.page.map(({ _id, name, folder, cover }) => ({
|
||||
_id,
|
||||
name,
|
||||
folder,
|
||||
cover,
|
||||
})),
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const getAlbums = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
return await ctx.db.query("albums").collect()
|
||||
export const scanAlbumsPage = internalQuery({
|
||||
args: { paginationOpts: paginationOptsValidator },
|
||||
handler: async (ctx, args) => {
|
||||
const result = await ctx.db.query("albums").paginate(args.paginationOpts)
|
||||
return {
|
||||
...result,
|
||||
page: result.page.map(({ _id, folder, cover, artist }) => ({
|
||||
_id,
|
||||
folder,
|
||||
cover,
|
||||
artist,
|
||||
})),
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const getSongs = internalQuery({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
return await ctx.db.query("songs").collect()
|
||||
export const scanSongsPage = internalQuery({
|
||||
args: { paginationOpts: paginationOptsValidator },
|
||||
handler: async (ctx, args) => {
|
||||
const result = await ctx.db.query("songs").paginate(args.paginationOpts)
|
||||
return {
|
||||
...result,
|
||||
page: result.page.map(({ _id, name, files, album }) => ({
|
||||
_id,
|
||||
name,
|
||||
files,
|
||||
album,
|
||||
})),
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
/** Public: share-link previews load covers without a session. */
|
||||
export const getArtistCover = query({
|
||||
args: { id: v.id("artists") },
|
||||
@@ -962,225 +990,260 @@ export const getSongsForPlayback = query({
|
||||
},
|
||||
})
|
||||
|
||||
export const createArtists = internalMutation({
|
||||
args: {
|
||||
artists: v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
folder: v.string(),
|
||||
cover: v.nullable(v.string()),
|
||||
})
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
for (const artist of args.artists) {
|
||||
const existing = await ctx.db
|
||||
.query("artists")
|
||||
.withIndex("by_folder", (q) => q.eq("folder", artist.folder))
|
||||
.first()
|
||||
if (existing) {
|
||||
continue
|
||||
}
|
||||
await ctx.db.insert("artists", artist)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const createAlbums = internalMutation({
|
||||
/**
|
||||
* Creates (or reuses, matched by folder) the artist and album for each entry
|
||||
* together with its songs, so an album never exists without songs.
|
||||
*/
|
||||
export const importAlbums = internalMutation({
|
||||
args: {
|
||||
albums: v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
folder: v.string(),
|
||||
cover: v.nullable(v.string()),
|
||||
year: v.number(),
|
||||
date: v.string(),
|
||||
artist: v.id("artists"),
|
||||
artist: v.object({
|
||||
name: v.string(),
|
||||
folder: v.string(),
|
||||
cover: v.nullable(v.string()),
|
||||
}),
|
||||
album: v.object({
|
||||
name: v.string(),
|
||||
folder: v.string(),
|
||||
cover: v.nullable(v.string()),
|
||||
year: v.number(),
|
||||
date: v.string(),
|
||||
}),
|
||||
songs: v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
files: v.record(v.string(), v.string()),
|
||||
artist: v.union(v.string(), v.array(v.string())),
|
||||
length: v.number(),
|
||||
diskNumber: v.nullable(v.number()),
|
||||
track: v.number(),
|
||||
year: v.number(),
|
||||
date: v.string(),
|
||||
genres: v.array(v.string()),
|
||||
})
|
||||
),
|
||||
})
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
for (const album of args.albums) {
|
||||
const existing = await ctx.db
|
||||
.query("albums")
|
||||
.withIndex("by_all", (q) =>
|
||||
q.eq("artist", album.artist).eq("folder", album.folder)
|
||||
)
|
||||
.first()
|
||||
if (existing) {
|
||||
for (const entry of args.albums) {
|
||||
if (entry.songs.length === 0) {
|
||||
continue
|
||||
}
|
||||
await ctx.db.insert("albums", album)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const createSongs = internalMutation({
|
||||
args: {
|
||||
songs: v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
files: v.record(v.string(), v.string()),
|
||||
album: v.id("albums"),
|
||||
artist: v.union(v.string(), v.array(v.string())),
|
||||
albumArtist: v.id("artists"),
|
||||
length: v.number(),
|
||||
diskNumber: v.nullable(v.number()),
|
||||
track: v.number(),
|
||||
year: v.number(),
|
||||
date: v.string(),
|
||||
genres: v.array(v.string()),
|
||||
})
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
for (const song of args.songs) {
|
||||
const existing = await ctx.db
|
||||
.query("songs")
|
||||
.withIndex("by_all", (q) =>
|
||||
q
|
||||
.eq("name", song.name)
|
||||
.eq("albumArtist", song.albumArtist)
|
||||
.eq("album", song.album)
|
||||
)
|
||||
.first()
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, {
|
||||
files: { ...existing.files, ...song.files },
|
||||
const artistId =
|
||||
(
|
||||
await ctx.db
|
||||
.query("artists")
|
||||
.withIndex("by_folder", (q) => q.eq("folder", entry.artist.folder))
|
||||
.first()
|
||||
)?._id ?? (await ctx.db.insert("artists", entry.artist))
|
||||
|
||||
const albumId =
|
||||
(
|
||||
await ctx.db
|
||||
.query("albums")
|
||||
.withIndex("by_all", (q) =>
|
||||
q.eq("artist", artistId).eq("folder", entry.album.folder)
|
||||
)
|
||||
.first()
|
||||
)?._id ??
|
||||
(await ctx.db.insert("albums", { ...entry.album, artist: artistId }))
|
||||
|
||||
for (const song of entry.songs) {
|
||||
const existing = await ctx.db
|
||||
.query("songs")
|
||||
.withIndex("by_all", (q) =>
|
||||
q
|
||||
.eq("name", song.name)
|
||||
.eq("albumArtist", artistId)
|
||||
.eq("album", albumId)
|
||||
)
|
||||
.first()
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, {
|
||||
files: { ...existing.files, ...song.files },
|
||||
})
|
||||
continue
|
||||
}
|
||||
await ctx.db.insert("songs", {
|
||||
...song,
|
||||
album: albumId,
|
||||
albumArtist: artistId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
await ctx.db.insert("songs", song)
|
||||
}
|
||||
|
||||
return
|
||||
},
|
||||
})
|
||||
|
||||
export const purgeSongsFromUsersAndPlaylists = internalMutation({
|
||||
/** Deletes songs and removes them from every playlist, favorite and queue. */
|
||||
export const deleteSongs = internalMutation({
|
||||
args: { ids: v.array(v.id("songs")) },
|
||||
handler: async (ctx, args) => {
|
||||
const remove = new Set(args.ids)
|
||||
if (remove.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const playlist of await ctx.db.query("playlists").collect()) {
|
||||
if (playlist.songs.some((songId) => remove.has(songId))) {
|
||||
await ctx.db.patch(playlist._id, {
|
||||
songs: playlist.songs.filter((songId) => !remove.has(songId)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const user of await ctx.db.query("users").collect()) {
|
||||
const favoriteSongs = user.favoriteSongs
|
||||
const songQueue = user.songQueue
|
||||
const queueHistory = user.queueHistory
|
||||
const patch: Partial<
|
||||
Pick<
|
||||
Doc<"users">,
|
||||
"favoriteSongs" | "songQueue" | "queueHistory" | "currentSongProgress"
|
||||
>
|
||||
> = {}
|
||||
|
||||
if (favoriteSongs?.some((songId) => remove.has(songId))) {
|
||||
patch.favoriteSongs = favoriteSongs.filter(
|
||||
(songId) => !remove.has(songId)
|
||||
)
|
||||
}
|
||||
if (songQueue?.some((songId) => remove.has(songId))) {
|
||||
const currentRemoved = remove.has(songQueue[0])
|
||||
patch.songQueue = songQueue.filter((songId) => !remove.has(songId))
|
||||
if (currentRemoved) {
|
||||
patch.currentSongProgress = 0
|
||||
}
|
||||
}
|
||||
if (queueHistory?.some((songId) => remove.has(songId))) {
|
||||
patch.queueHistory = queueHistory.filter(
|
||||
(songId) => !remove.has(songId)
|
||||
)
|
||||
}
|
||||
if (Object.keys(patch).length > 0) {
|
||||
await ctx.db.patch(user._id, patch)
|
||||
}
|
||||
for (const id of args.ids) {
|
||||
await ctx.db.delete("songs", id)
|
||||
}
|
||||
await purgeSongsFromUsersAndPlaylists(ctx, new Set(args.ids))
|
||||
await recordIntentCatalogChanges(ctx, "song", args.ids, true)
|
||||
},
|
||||
})
|
||||
|
||||
export const purgeAlbumsFromUsers = internalMutation({
|
||||
/** Deletes the albums that still have no songs when this mutation runs. */
|
||||
export const deleteEmptyAlbums = internalMutation({
|
||||
args: { ids: v.array(v.id("albums")) },
|
||||
handler: async (ctx, args) => {
|
||||
const remove = new Set(args.ids)
|
||||
if (remove.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const user of await ctx.db.query("users").collect()) {
|
||||
const favoriteAlbums = user.favoriteAlbums
|
||||
const recentlyPlayed = user.recentlyPlayed
|
||||
const patch: Partial<
|
||||
Pick<Doc<"users">, "favoriteAlbums" | "recentlyPlayed">
|
||||
> = {}
|
||||
if (favoriteAlbums?.some((albumId) => remove.has(albumId))) {
|
||||
patch.favoriteAlbums = favoriteAlbums.filter(
|
||||
(albumId) => !remove.has(albumId)
|
||||
)
|
||||
}
|
||||
if (
|
||||
recentlyPlayed?.some((itemId) => remove.has(itemId as Id<"albums">))
|
||||
) {
|
||||
patch.recentlyPlayed = recentlyPlayed.filter(
|
||||
(itemId) => !remove.has(itemId as Id<"albums">)
|
||||
)
|
||||
}
|
||||
if (Object.keys(patch).length > 0) {
|
||||
await ctx.db.patch(user._id, patch)
|
||||
const deleted = new Set<Id<"albums">>()
|
||||
for (const id of args.ids) {
|
||||
const song = await ctx.db
|
||||
.query("songs")
|
||||
.withIndex("by_album", (q) => q.eq("album", id))
|
||||
.first()
|
||||
if (song === null && (await ctx.db.get("albums", id)) !== null) {
|
||||
await ctx.db.delete("albums", id)
|
||||
deleted.add(id)
|
||||
}
|
||||
}
|
||||
await purgeAlbumsFromUsers(ctx, deleted)
|
||||
await recordIntentCatalogChanges(ctx, "album", [...deleted], true)
|
||||
return deleted.size
|
||||
},
|
||||
})
|
||||
|
||||
export const purgeArtistsFromUsers = internalMutation({
|
||||
/** Deletes the artists that still have no albums when this mutation runs. */
|
||||
export const deleteEmptyArtists = internalMutation({
|
||||
args: { ids: v.array(v.id("artists")) },
|
||||
handler: async (ctx, args) => {
|
||||
const remove = new Set(args.ids)
|
||||
if (remove.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const user of await ctx.db.query("users").collect()) {
|
||||
const favoriteArtists = user.favoriteArtists
|
||||
if (favoriteArtists?.some((artistId) => remove.has(artistId))) {
|
||||
await ctx.db.patch(user._id, {
|
||||
favoriteArtists: favoriteArtists.filter(
|
||||
(artistId) => !remove.has(artistId)
|
||||
),
|
||||
})
|
||||
const deleted = new Set<Id<"artists">>()
|
||||
for (const id of args.ids) {
|
||||
const album = await ctx.db
|
||||
.query("albums")
|
||||
.withIndex("by_artist", (q) => q.eq("artist", id))
|
||||
.first()
|
||||
if (album === null && (await ctx.db.get("artists", id)) !== null) {
|
||||
await ctx.db.delete("artists", id)
|
||||
deleted.add(id)
|
||||
}
|
||||
}
|
||||
await purgeArtistsFromUsers(ctx, deleted)
|
||||
await recordIntentCatalogChanges(ctx, "artist", [...deleted], true)
|
||||
return deleted.size
|
||||
},
|
||||
})
|
||||
|
||||
export const removeArtists = internalMutation({
|
||||
args: { ids: v.array(v.id("artists")) },
|
||||
handler: async (ctx, args) => {
|
||||
for (const id of args.ids) {
|
||||
await ctx.db.delete("artists", id)
|
||||
}
|
||||
await recordIntentCatalogChanges(ctx, "artist", args.ids, true)
|
||||
},
|
||||
})
|
||||
async function purgeSongsFromUsersAndPlaylists(
|
||||
ctx: MutationCtx,
|
||||
remove: Set<Id<"songs">>
|
||||
) {
|
||||
if (remove.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
export const removeAlbums = internalMutation({
|
||||
args: { ids: v.array(v.id("albums")) },
|
||||
handler: async (ctx, args) => {
|
||||
for (const id of args.ids) {
|
||||
await ctx.db.delete("albums", id)
|
||||
for (const playlist of await ctx.db.query("playlists").collect()) {
|
||||
if (playlist.songs.some((songId) => remove.has(songId))) {
|
||||
await ctx.db.patch(playlist._id, {
|
||||
songs: playlist.songs.filter((songId) => !remove.has(songId)),
|
||||
})
|
||||
}
|
||||
await recordIntentCatalogChanges(ctx, "album", args.ids, true)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
for (const state of await ctx.db.query("playbackStates").collect()) {
|
||||
if (
|
||||
state.songQueue.some((songId) => remove.has(songId)) ||
|
||||
state.queueHistory.some((songId) => remove.has(songId))
|
||||
) {
|
||||
await ctx.db.patch(state._id, {
|
||||
songQueue: state.songQueue.filter((songId) => !remove.has(songId)),
|
||||
queueHistory: state.queueHistory.filter(
|
||||
(songId) => !remove.has(songId)
|
||||
),
|
||||
revision: state.revision + 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy queue fields stay until `migrations.migrateUsers` clears them.
|
||||
for (const user of await ctx.db.query("users").collect()) {
|
||||
const favoriteSongs = user.favoriteSongs
|
||||
const songQueue = user.songQueue
|
||||
const queueHistory = user.queueHistory
|
||||
const patch: Partial<
|
||||
Pick<
|
||||
Doc<"users">,
|
||||
"favoriteSongs" | "songQueue" | "queueHistory" | "currentSongProgress"
|
||||
>
|
||||
> = {}
|
||||
|
||||
if (favoriteSongs?.some((songId) => remove.has(songId))) {
|
||||
patch.favoriteSongs = favoriteSongs.filter(
|
||||
(songId) => !remove.has(songId)
|
||||
)
|
||||
}
|
||||
if (songQueue?.some((songId) => remove.has(songId))) {
|
||||
const currentRemoved = remove.has(songQueue[0])
|
||||
patch.songQueue = songQueue.filter((songId) => !remove.has(songId))
|
||||
if (currentRemoved) {
|
||||
patch.currentSongProgress = 0
|
||||
}
|
||||
}
|
||||
if (queueHistory?.some((songId) => remove.has(songId))) {
|
||||
patch.queueHistory = queueHistory.filter((songId) => !remove.has(songId))
|
||||
}
|
||||
if (Object.keys(patch).length > 0) {
|
||||
await ctx.db.patch(user._id, patch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function purgeAlbumsFromUsers(
|
||||
ctx: MutationCtx,
|
||||
remove: Set<Id<"albums">>
|
||||
) {
|
||||
if (remove.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const user of await ctx.db.query("users").collect()) {
|
||||
const favoriteAlbums = user.favoriteAlbums
|
||||
const recentlyPlayed = user.recentlyPlayed
|
||||
const patch: Partial<
|
||||
Pick<Doc<"users">, "favoriteAlbums" | "recentlyPlayed">
|
||||
> = {}
|
||||
if (favoriteAlbums?.some((albumId) => remove.has(albumId))) {
|
||||
patch.favoriteAlbums = favoriteAlbums.filter(
|
||||
(albumId) => !remove.has(albumId)
|
||||
)
|
||||
}
|
||||
if (recentlyPlayed?.some((itemId) => remove.has(itemId as Id<"albums">))) {
|
||||
patch.recentlyPlayed = recentlyPlayed.filter(
|
||||
(itemId) => !remove.has(itemId as Id<"albums">)
|
||||
)
|
||||
}
|
||||
if (Object.keys(patch).length > 0) {
|
||||
await ctx.db.patch(user._id, patch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function purgeArtistsFromUsers(
|
||||
ctx: MutationCtx,
|
||||
remove: Set<Id<"artists">>
|
||||
) {
|
||||
if (remove.size === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const user of await ctx.db.query("users").collect()) {
|
||||
const favoriteArtists = user.favoriteArtists
|
||||
if (favoriteArtists?.some((artistId) => remove.has(artistId))) {
|
||||
await ctx.db.patch(user._id, {
|
||||
favoriteArtists: favoriteArtists.filter(
|
||||
(artistId) => !remove.has(artistId)
|
||||
),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const changeFilesForSongs = internalMutation({
|
||||
args: {
|
||||
@@ -1196,26 +1259,64 @@ export const changeFilesForSongs = internalMutation({
|
||||
},
|
||||
})
|
||||
|
||||
export const updateArtistCover = internalMutation({
|
||||
args: { id: v.id("artists"), cover: v.nullable(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
return await ctx.db.patch("artists", args.id, { cover: args.cover })
|
||||
export const updateArtistCovers = internalMutation({
|
||||
args: {
|
||||
artists: v.array(
|
||||
v.object({ id: v.id("artists"), cover: v.nullable(v.string()) })
|
||||
),
|
||||
},
|
||||
})
|
||||
|
||||
export const updateAlbumCover = internalMutation({
|
||||
args: { id: v.id("albums"), cover: v.nullable(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
return await ctx.db.patch("albums", args.id, { cover: args.cover })
|
||||
for (const { id, cover } of args.artists) {
|
||||
if ((await ctx.db.get("artists", id)) !== null) {
|
||||
await ctx.db.patch("artists", id, { cover })
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
export const removeSongs = internalMutation({
|
||||
args: { ids: v.array(v.id("songs")) },
|
||||
export const updateAlbumCovers = internalMutation({
|
||||
args: {
|
||||
albums: v.array(
|
||||
v.object({ id: v.id("albums"), cover: v.nullable(v.string()) })
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
for (const { id, cover } of args.albums) {
|
||||
if ((await ctx.db.get("albums", id)) !== null) {
|
||||
await ctx.db.patch("albums", id, { cover })
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
/**
|
||||
* Takes the single library scan lease if it is free or expired. The lease
|
||||
* outlives the action timeout so a crashed scan cannot hold it forever.
|
||||
*/
|
||||
export const acquireLibraryScanLock = internalMutation({
|
||||
args: { token: v.string(), leaseMs: v.number() },
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now()
|
||||
const lock = await ctx.db.query("libraryScanLock").first()
|
||||
if (lock && lock.token !== args.token && lock.expiresAt > now) {
|
||||
return false
|
||||
}
|
||||
const expiresAt = now + args.leaseMs
|
||||
if (lock) {
|
||||
await ctx.db.patch(lock._id, { token: args.token, expiresAt })
|
||||
} else {
|
||||
await ctx.db.insert("libraryScanLock", { token: args.token, expiresAt })
|
||||
}
|
||||
return true
|
||||
},
|
||||
})
|
||||
|
||||
export const releaseLibraryScanLock = internalMutation({
|
||||
args: { token: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
for (const id of args.ids) {
|
||||
await ctx.db.delete("songs", id)
|
||||
const lock = await ctx.db.query("libraryScanLock").first()
|
||||
if (lock?.token === args.token) {
|
||||
await ctx.db.delete(lock._id)
|
||||
}
|
||||
await recordIntentCatalogChanges(ctx, "song", args.ids, true)
|
||||
},
|
||||
})
|
||||
@@ -78,6 +78,10 @@ export default defineSchema({
|
||||
appSettings: defineTable({
|
||||
disableAccountCreation: v.boolean(),
|
||||
}),
|
||||
libraryScanLock: defineTable({
|
||||
token: v.string(),
|
||||
expiresAt: v.number(),
|
||||
}),
|
||||
artists: defineTable({
|
||||
name: v.string(),
|
||||
folder: v.string(),
|
||||
|
||||
@@ -33,8 +33,10 @@ export type BrowseAlbumTrack = {
|
||||
}
|
||||
|
||||
export type BrowseAlbumDetail = BrowseAlbum & {
|
||||
/** `null` when the files have no year tag; clients hide it. */
|
||||
year: number | null
|
||||
genre: string | null
|
||||
/** Empty when unknown; clients hide it. */
|
||||
releaseDate: string
|
||||
hasLossless: boolean
|
||||
hasAtmos: boolean
|
||||
|
||||
Reference in new issue
Block a user