feat: fuzzy search for songs and lyrics
Lyrics search kept only the top 50 search index hits before checking lines, so phrases of common words like "let your people go" lost the song. Check the whole index page instead, and match titles, artists, albums and lyrics with typo tolerance, ignoring accents and apostrophes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
332a7aed1f
commit
19894805df
5 files changed
+165
-52
No files matched your search
@@ -0,0 +1,93 @@
|
|||||||
|
/**
|
||||||
|
* Typo-tolerant word matching, shared by library search, lyrics search and
|
||||||
|
* in-list song filters.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lowercase words without accents or apostrophes, so "Don't Stop Beyoncé"
|
||||||
|
* gives ["dont", "stop", "beyonce"].
|
||||||
|
*/
|
||||||
|
export function searchWords(text: string): string[] {
|
||||||
|
return (
|
||||||
|
text
|
||||||
|
.normalize("NFKD")
|
||||||
|
.replace(/\p{M}/gu, "")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/['’]/g, "")
|
||||||
|
.match(/[\p{L}\p{N}]+/gu) ?? []
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Typos a query word may hold. Short words get none, or they match too much. */
|
||||||
|
function allowedTypos(term: string) {
|
||||||
|
if (term.length < 4) return 0
|
||||||
|
return term.length < 8 ? 1 : 2
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Edits (insert, delete, substitute or swap neighbors) turning `a` into `b`,
|
||||||
|
* or `max + 1` once it is sure to exceed `max`.
|
||||||
|
*/
|
||||||
|
function editDistance(a: string, b: string, max: number): number {
|
||||||
|
if (Math.abs(a.length - b.length) > max) return max + 1
|
||||||
|
let before: number[] = []
|
||||||
|
let previous = Array.from({ length: b.length + 1 }, (_, j) => j)
|
||||||
|
for (let i = 1; i <= a.length; i++) {
|
||||||
|
const current = [i]
|
||||||
|
let rowMin = i
|
||||||
|
for (let j = 1; j <= b.length; j++) {
|
||||||
|
const cost = a[i - 1] === b[j - 1] ? 0 : 1
|
||||||
|
let distance = Math.min(
|
||||||
|
previous[j]! + 1,
|
||||||
|
current[j - 1]! + 1,
|
||||||
|
previous[j - 1]! + cost
|
||||||
|
)
|
||||||
|
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
|
||||||
|
distance = Math.min(distance, before[j - 2]! + 1)
|
||||||
|
}
|
||||||
|
current.push(distance)
|
||||||
|
rowMin = Math.min(rowMin, distance)
|
||||||
|
}
|
||||||
|
if (rowMin > max) return max + 1
|
||||||
|
before = previous
|
||||||
|
previous = current
|
||||||
|
}
|
||||||
|
return previous[b.length]!
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether `word` is query `term` give or take a few typos. With `prefix` the
|
||||||
|
* word may also just start with the term, as when it is still being typed.
|
||||||
|
*/
|
||||||
|
function wordMatchesTerm(term: string, word: string, prefix: boolean) {
|
||||||
|
const typos = allowedTypos(term)
|
||||||
|
if (typos === 0) return false
|
||||||
|
if (editDistance(term, word, typos) <= typos) return true
|
||||||
|
return (
|
||||||
|
prefix &&
|
||||||
|
word.length > term.length &&
|
||||||
|
editDistance(term, word.slice(0, term.length), typos) <= typos
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How every query term (from `searchWords`) appears among `words`: "exact"
|
||||||
|
* when each is a word, "fuzzy" when some only match give or take typos, or
|
||||||
|
* null when one is missing. The last term may be a word's prefix.
|
||||||
|
*/
|
||||||
|
export function matchWords(
|
||||||
|
words: readonly string[],
|
||||||
|
terms: readonly string[]
|
||||||
|
): "exact" | "fuzzy" | null {
|
||||||
|
let quality: "exact" | "fuzzy" = "exact"
|
||||||
|
for (const [index, term] of terms.entries()) {
|
||||||
|
const prefix = index === terms.length - 1
|
||||||
|
const exact = words.some((word) =>
|
||||||
|
prefix ? word.startsWith(term) : word === term
|
||||||
|
)
|
||||||
|
if (exact) continue
|
||||||
|
if (!words.some((word) => wordMatchesTerm(term, word, prefix))) return null
|
||||||
|
quality = "fuzzy"
|
||||||
|
}
|
||||||
|
return terms.length > 0 ? quality : null
|
||||||
|
}
|
||||||
@@ -5,6 +5,8 @@
|
|||||||
* the same.
|
* the same.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { matchWords, searchWords } from "./fuzzy-search"
|
||||||
|
|
||||||
export type LyricWord = {
|
export type LyricWord = {
|
||||||
/** Includes its trailing space, so a line's words join back into its text. */
|
/** Includes its trailing space, so a line's words join back into its text. */
|
||||||
text: string
|
text: string
|
||||||
@@ -212,32 +214,32 @@ export function followLyrics(
|
|||||||
return () => clearTimeout(timer)
|
return () => clearTimeout(timer)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Lowercase words of a search, for matching lyrics the way the search index does. */
|
|
||||||
function searchTerms(query: string): string[] {
|
|
||||||
return query.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? []
|
|
||||||
}
|
|
||||||
|
|
||||||
const SNIPPET_LEAD = 24
|
const SNIPPET_LEAD = 24
|
||||||
|
|
||||||
export type LyricsMatch = {
|
export type LyricsMatch = {
|
||||||
/** The matched line, trimmed to start near the match. */
|
/** The matched line, trimmed to start near the match. */
|
||||||
snippet: string
|
snippet: string
|
||||||
/** Whether the line holds the query as typed, not just its words. */
|
/**
|
||||||
exact: boolean
|
* 2 when the line holds the query as typed, 1 when it holds every word of
|
||||||
|
* it, 0 when it does give or take typos.
|
||||||
|
*/
|
||||||
|
score: 0 | 1 | 2
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Where a search matched the lyrics: a line holding the query as typed, or
|
* Where a search matched the lyrics: the best line holding the query as
|
||||||
* else a line holding every word of it (the last may be a prefix, as when
|
* typed, or else every word of it (see `matchWords`). Null when no single
|
||||||
* typing). Null when no single line does.
|
* line does.
|
||||||
*/
|
*/
|
||||||
export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
||||||
const needle = query.trim().toLowerCase()
|
const needle = query.trim().toLowerCase()
|
||||||
const terms = searchTerms(needle)
|
const terms = searchWords(needle)
|
||||||
const last = terms.at(-1)
|
// Most songs miss a word entirely, so skip them before checking lines.
|
||||||
if (!needle || last === undefined) return null
|
if (!needle || !matchWords([...new Set(searchWords(plain))], terms)) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
let loose: string | null = null
|
let loose: LyricsMatch | null = null
|
||||||
for (const line of plain.split("\n")) {
|
for (const line of plain.split("\n")) {
|
||||||
const column = line.toLowerCase().indexOf(needle)
|
const column = line.toLowerCase().indexOf(needle)
|
||||||
if (column !== -1) {
|
if (column !== -1) {
|
||||||
@@ -245,15 +247,13 @@ export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
|||||||
column > SNIPPET_LEAD
|
column > SNIPPET_LEAD
|
||||||
? `…${line.slice(column - SNIPPET_LEAD).trimStart()}`
|
? `…${line.slice(column - SNIPPET_LEAD).trimStart()}`
|
||||||
: line.trim()
|
: line.trim()
|
||||||
return { snippet, exact: true }
|
return { snippet, score: 2 }
|
||||||
}
|
}
|
||||||
if (loose === null) {
|
if (loose?.score === 1) continue
|
||||||
const words = searchTerms(line)
|
const quality = matchWords(searchWords(line), terms)
|
||||||
const hasAll =
|
if (quality === "exact" || (quality && !loose)) {
|
||||||
terms.slice(0, -1).every((term) => words.includes(term)) &&
|
loose = { snippet: line.trim(), score: quality === "exact" ? 1 : 0 }
|
||||||
words.some((word) => word.startsWith(last))
|
|
||||||
if (hasAll) loose = line.trim()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return loose === null ? null : { snippet: loose, exact: false }
|
return loose
|
||||||
}
|
}
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { matchWords, searchWords } from "./fuzzy-search"
|
||||||
|
|
||||||
type SearchableSong = {
|
type SearchableSong = {
|
||||||
id: string
|
id: string
|
||||||
title: string
|
title: string
|
||||||
@@ -6,9 +8,9 @@ type SearchableSong = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Case-insensitive match on title, artist, or album, plus the songs in
|
* Case-insensitive match on title, artist, or album, give or take typos (see
|
||||||
* `lyricsMatches` (from `api.lyrics.matchingSongIds`). A blank query keeps
|
* `matchWords`), plus the songs in `lyricsMatches` (from
|
||||||
* every song.
|
* `api.lyrics.matchingSongIds`). A blank query keeps every song.
|
||||||
*/
|
*/
|
||||||
export function filterSongs<T extends SearchableSong>(
|
export function filterSongs<T extends SearchableSong>(
|
||||||
songs: T[],
|
songs: T[],
|
||||||
@@ -19,12 +21,14 @@ export function filterSongs<T extends SearchableSong>(
|
|||||||
if (!normalized) {
|
if (!normalized) {
|
||||||
return songs
|
return songs
|
||||||
}
|
}
|
||||||
|
const terms = searchWords(normalized)
|
||||||
const byLyrics = new Set(lyricsMatches)
|
const byLyrics = new Set(lyricsMatches)
|
||||||
return songs.filter(
|
return songs.filter((song) => {
|
||||||
(song) =>
|
if (byLyrics.has(song.id)) return true
|
||||||
byLyrics.has(song.id) ||
|
const fields = [song.title, song.artist.name, song.album.title]
|
||||||
[song.title, song.artist.name, song.album.title].some((value) =>
|
return (
|
||||||
value.toLowerCase().includes(normalized)
|
fields.some((value) => value.toLowerCase().includes(normalized)) ||
|
||||||
)
|
matchWords(searchWords(fields.join(" ")), terms) !== null
|
||||||
)
|
)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
@@ -27,6 +27,7 @@ import { chunk, type Chunked } from "./limits"
|
|||||||
import { recordIntentCatalogChanges } from "./intentCatalogChanges"
|
import { recordIntentCatalogChanges } from "./intentCatalogChanges"
|
||||||
import { playlistCoverVersion } from "./playlistCover"
|
import { playlistCoverVersion } from "./playlistCover"
|
||||||
import { deleteSongLyrics, searchSongLyrics, writeSongLyrics } from "./lyrics"
|
import { deleteSongLyrics, searchSongLyrics, writeSongLyrics } from "./lyrics"
|
||||||
|
import { matchWords, searchWords } from "../client/fuzzy-search"
|
||||||
|
|
||||||
export type {
|
export type {
|
||||||
BrowseAlbum,
|
BrowseAlbum,
|
||||||
@@ -618,7 +619,16 @@ export const browseArtistDetail = query({
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
function matchScore(text: string, query: string): number {
|
/**
|
||||||
|
* How well `text` matches a lowercase search, from 100 (the whole text) down
|
||||||
|
* to 20 (every word, give or take typos); 0 is no match. `terms` are the
|
||||||
|
* query's `searchWords`.
|
||||||
|
*/
|
||||||
|
function matchScore(
|
||||||
|
text: string,
|
||||||
|
query: string,
|
||||||
|
terms: readonly string[]
|
||||||
|
): number {
|
||||||
const lower = text.toLowerCase()
|
const lower = text.toLowerCase()
|
||||||
if (lower === query) {
|
if (lower === query) {
|
||||||
return 100
|
return 100
|
||||||
@@ -632,15 +642,20 @@ function matchScore(text: string, query: string): number {
|
|||||||
if (lower.includes(query)) {
|
if (lower.includes(query)) {
|
||||||
return 40
|
return 40
|
||||||
}
|
}
|
||||||
return 0
|
const words = matchWords(searchWords(text), terms)
|
||||||
|
if (words === "exact") {
|
||||||
|
return 30
|
||||||
|
}
|
||||||
|
return words === "fuzzy" ? 20 : 0
|
||||||
}
|
}
|
||||||
|
|
||||||
function matchScoreSongArtists(
|
function matchScoreSongArtists(
|
||||||
artist: string | string[],
|
artist: string | string[],
|
||||||
query: string
|
query: string,
|
||||||
|
terms: readonly string[]
|
||||||
): number {
|
): number {
|
||||||
return getSongArtistNames(artist).reduce(
|
return getSongArtistNames(artist).reduce(
|
||||||
(best, name) => Math.max(best, matchScore(name, query)),
|
(best, name) => Math.max(best, matchScore(name, query, terms)),
|
||||||
0
|
0
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -658,6 +673,7 @@ export const searchLibrary = query({
|
|||||||
args: { query: v.string() },
|
args: { query: v.string() },
|
||||||
handler: async (ctx, args) => {
|
handler: async (ctx, args) => {
|
||||||
const queryText = args.query.trim().toLowerCase()
|
const queryText = args.query.trim().toLowerCase()
|
||||||
|
const terms = searchWords(queryText)
|
||||||
if (!queryText || !(await isSignedIn(ctx))) {
|
if (!queryText || !(await isSignedIn(ctx))) {
|
||||||
return {
|
return {
|
||||||
topResults: [],
|
topResults: [],
|
||||||
@@ -678,7 +694,7 @@ export const searchLibrary = query({
|
|||||||
.map((artist) => ({
|
.map((artist) => ({
|
||||||
kind: "artist" as const,
|
kind: "artist" as const,
|
||||||
item: toBrowseArtist(artist),
|
item: toBrowseArtist(artist),
|
||||||
score: matchScore(artist.name, queryText),
|
score: matchScore(artist.name, queryText, terms),
|
||||||
}))
|
}))
|
||||||
.filter((result) => result.score > 0)
|
.filter((result) => result.score > 0)
|
||||||
.sort(
|
.sort(
|
||||||
@@ -692,8 +708,8 @@ export const searchLibrary = query({
|
|||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
const score = Math.max(
|
const score = Math.max(
|
||||||
matchScore(album.name, queryText),
|
matchScore(album.name, queryText, terms),
|
||||||
matchScore(artist.name, queryText)
|
matchScore(artist.name, queryText, terms)
|
||||||
)
|
)
|
||||||
if (score === 0) {
|
if (score === 0) {
|
||||||
return []
|
return []
|
||||||
@@ -718,10 +734,10 @@ export const searchLibrary = query({
|
|||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
const score = Math.max(
|
const score = Math.max(
|
||||||
matchScore(song.name, queryText),
|
matchScore(song.name, queryText, terms),
|
||||||
matchScoreSongArtists(song.artist, queryText),
|
matchScoreSongArtists(song.artist, queryText, terms),
|
||||||
matchScore(album.name, queryText),
|
matchScore(album.name, queryText, terms),
|
||||||
matchScore(albumArtist.name, queryText)
|
matchScore(albumArtist.name, queryText, terms)
|
||||||
)
|
)
|
||||||
if (score === 0) {
|
if (score === 0) {
|
||||||
return []
|
return []
|
||||||
|
|||||||
@@ -62,10 +62,10 @@ export async function deleteSongLyrics(ctx: MutationCtx, songId: Id<"songs">) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Songs with a lyrics line matching `query` (see `matchLyrics`), with that
|
* Up to `limit` songs with a lyrics line matching `query` (see `matchLyrics`),
|
||||||
* line. Lines holding the query as typed come first, then the search index's
|
* with that line, best matches first. The search index matches any word
|
||||||
* relevance order. The index matches words anywhere in the lyrics, so its
|
* anywhere in the lyrics and ranks common words poorly, so its whole page is
|
||||||
* results are checked line by line.
|
* checked line by line before trimming.
|
||||||
*/
|
*/
|
||||||
export async function searchSongLyrics(
|
export async function searchSongLyrics(
|
||||||
ctx: QueryCtx,
|
ctx: QueryCtx,
|
||||||
@@ -77,15 +77,15 @@ export async function searchSongLyrics(
|
|||||||
const rows = await ctx.db
|
const rows = await ctx.db
|
||||||
.query("lyricsSearch")
|
.query("lyricsSearch")
|
||||||
.withSearchIndex("search_plain", (q) => q.search("plain", text))
|
.withSearchIndex("search_plain", (q) => q.search("plain", text))
|
||||||
.take(limit)
|
.take(MAX_LYRICS_MATCHES)
|
||||||
const matches = rows.flatMap((row) => {
|
return rows
|
||||||
|
.flatMap((row) => {
|
||||||
const match = matchLyrics(row.plain, text)
|
const match = matchLyrics(row.plain, text)
|
||||||
return match ? [{ song: row.song, ...match }] : []
|
return match ? [{ song: row.song, ...match }] : []
|
||||||
})
|
})
|
||||||
return [
|
.sort((a, b) => b.score - a.score)
|
||||||
...matches.filter((match) => match.exact),
|
.slice(0, limit)
|
||||||
...matches.filter((match) => !match.exact),
|
.map(({ song, snippet }) => ({ song, snippet }))
|
||||||
].map(({ song, snippet }) => ({ song, snippet }))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The song's lyrics tag (see `client/lyrics.ts`), or null without lyrics. */
|
/** The song's lyrics tag (see `client/lyrics.ts`), or null without lyrics. */
|
||||||
|
|||||||
Reference in new issue
Block a user