feat: fuzzy search for songs and lyrics
Lyrics search kept only the top 50 search index hits before checking lines, so phrases of common words like "let your people go" lost the song. Check the whole index page instead, and match titles, artists, albums and lyrics with typo tolerance, ignoring accents and apostrophes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
332a7aed1f
commit
19894805df
5 files changed
+165
-52
No files matched your search
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Typo-tolerant word matching, shared by library search, lyrics search and
|
||||
* in-list song filters.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Lowercase words without accents or apostrophes, so "Don't Stop Beyoncé"
|
||||
* gives ["dont", "stop", "beyonce"].
|
||||
*/
|
||||
export function searchWords(text: string): string[] {
|
||||
return (
|
||||
text
|
||||
.normalize("NFKD")
|
||||
.replace(/\p{M}/gu, "")
|
||||
.toLowerCase()
|
||||
.replace(/['’]/g, "")
|
||||
.match(/[\p{L}\p{N}]+/gu) ?? []
|
||||
)
|
||||
}
|
||||
|
||||
/** Typos a query word may hold. Short words get none, or they match too much. */
|
||||
function allowedTypos(term: string) {
|
||||
if (term.length < 4) return 0
|
||||
return term.length < 8 ? 1 : 2
|
||||
}
|
||||
|
||||
/**
|
||||
* Edits (insert, delete, substitute or swap neighbors) turning `a` into `b`,
|
||||
* or `max + 1` once it is sure to exceed `max`.
|
||||
*/
|
||||
function editDistance(a: string, b: string, max: number): number {
|
||||
if (Math.abs(a.length - b.length) > max) return max + 1
|
||||
let before: number[] = []
|
||||
let previous = Array.from({ length: b.length + 1 }, (_, j) => j)
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
const current = [i]
|
||||
let rowMin = i
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1
|
||||
let distance = Math.min(
|
||||
previous[j]! + 1,
|
||||
current[j - 1]! + 1,
|
||||
previous[j - 1]! + cost
|
||||
)
|
||||
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
|
||||
distance = Math.min(distance, before[j - 2]! + 1)
|
||||
}
|
||||
current.push(distance)
|
||||
rowMin = Math.min(rowMin, distance)
|
||||
}
|
||||
if (rowMin > max) return max + 1
|
||||
before = previous
|
||||
previous = current
|
||||
}
|
||||
return previous[b.length]!
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `word` is query `term` give or take a few typos. With `prefix` the
|
||||
* word may also just start with the term, as when it is still being typed.
|
||||
*/
|
||||
function wordMatchesTerm(term: string, word: string, prefix: boolean) {
|
||||
const typos = allowedTypos(term)
|
||||
if (typos === 0) return false
|
||||
if (editDistance(term, word, typos) <= typos) return true
|
||||
return (
|
||||
prefix &&
|
||||
word.length > term.length &&
|
||||
editDistance(term, word.slice(0, term.length), typos) <= typos
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* How every query term (from `searchWords`) appears among `words`: "exact"
|
||||
* when each is a word, "fuzzy" when some only match give or take typos, or
|
||||
* null when one is missing. The last term may be a word's prefix.
|
||||
*/
|
||||
export function matchWords(
|
||||
words: readonly string[],
|
||||
terms: readonly string[]
|
||||
): "exact" | "fuzzy" | null {
|
||||
let quality: "exact" | "fuzzy" = "exact"
|
||||
for (const [index, term] of terms.entries()) {
|
||||
const prefix = index === terms.length - 1
|
||||
const exact = words.some((word) =>
|
||||
prefix ? word.startsWith(term) : word === term
|
||||
)
|
||||
if (exact) continue
|
||||
if (!words.some((word) => wordMatchesTerm(term, word, prefix))) return null
|
||||
quality = "fuzzy"
|
||||
}
|
||||
return terms.length > 0 ? quality : null
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
* the same.
|
||||
*/
|
||||
|
||||
import { matchWords, searchWords } from "./fuzzy-search"
|
||||
|
||||
export type LyricWord = {
|
||||
/** Includes its trailing space, so a line's words join back into its text. */
|
||||
text: string
|
||||
@@ -212,32 +214,32 @@ export function followLyrics(
|
||||
return () => clearTimeout(timer)
|
||||
}
|
||||
|
||||
/** Lowercase words of a search, for matching lyrics the way the search index does. */
|
||||
function searchTerms(query: string): string[] {
|
||||
return query.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? []
|
||||
}
|
||||
|
||||
const SNIPPET_LEAD = 24
|
||||
|
||||
export type LyricsMatch = {
|
||||
/** The matched line, trimmed to start near the match. */
|
||||
snippet: string
|
||||
/** Whether the line holds the query as typed, not just its words. */
|
||||
exact: boolean
|
||||
/**
|
||||
* 2 when the line holds the query as typed, 1 when it holds every word of
|
||||
* it, 0 when it does give or take typos.
|
||||
*/
|
||||
score: 0 | 1 | 2
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a search matched the lyrics: a line holding the query as typed, or
|
||||
* else a line holding every word of it (the last may be a prefix, as when
|
||||
* typing). Null when no single line does.
|
||||
* Where a search matched the lyrics: the best line holding the query as
|
||||
* typed, or else every word of it (see `matchWords`). Null when no single
|
||||
* line does.
|
||||
*/
|
||||
export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
||||
const needle = query.trim().toLowerCase()
|
||||
const terms = searchTerms(needle)
|
||||
const last = terms.at(-1)
|
||||
if (!needle || last === undefined) return null
|
||||
const terms = searchWords(needle)
|
||||
// Most songs miss a word entirely, so skip them before checking lines.
|
||||
if (!needle || !matchWords([...new Set(searchWords(plain))], terms)) {
|
||||
return null
|
||||
}
|
||||
|
||||
let loose: string | null = null
|
||||
let loose: LyricsMatch | null = null
|
||||
for (const line of plain.split("\n")) {
|
||||
const column = line.toLowerCase().indexOf(needle)
|
||||
if (column !== -1) {
|
||||
@@ -245,15 +247,13 @@ export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
||||
column > SNIPPET_LEAD
|
||||
? `…${line.slice(column - SNIPPET_LEAD).trimStart()}`
|
||||
: line.trim()
|
||||
return { snippet, exact: true }
|
||||
return { snippet, score: 2 }
|
||||
}
|
||||
if (loose === null) {
|
||||
const words = searchTerms(line)
|
||||
const hasAll =
|
||||
terms.slice(0, -1).every((term) => words.includes(term)) &&
|
||||
words.some((word) => word.startsWith(last))
|
||||
if (hasAll) loose = line.trim()
|
||||
if (loose?.score === 1) continue
|
||||
const quality = matchWords(searchWords(line), terms)
|
||||
if (quality === "exact" || (quality && !loose)) {
|
||||
loose = { snippet: line.trim(), score: quality === "exact" ? 1 : 0 }
|
||||
}
|
||||
}
|
||||
return loose === null ? null : { snippet: loose, exact: false }
|
||||
return loose
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { matchWords, searchWords } from "./fuzzy-search"
|
||||
|
||||
type SearchableSong = {
|
||||
id: string
|
||||
title: string
|
||||
@@ -6,9 +8,9 @@ type SearchableSong = {
|
||||
}
|
||||
|
||||
/**
|
||||
* Case-insensitive match on title, artist, or album, plus the songs in
|
||||
* `lyricsMatches` (from `api.lyrics.matchingSongIds`). A blank query keeps
|
||||
* every song.
|
||||
* Case-insensitive match on title, artist, or album, give or take typos (see
|
||||
* `matchWords`), plus the songs in `lyricsMatches` (from
|
||||
* `api.lyrics.matchingSongIds`). A blank query keeps every song.
|
||||
*/
|
||||
export function filterSongs<T extends SearchableSong>(
|
||||
songs: T[],
|
||||
@@ -19,12 +21,14 @@ export function filterSongs<T extends SearchableSong>(
|
||||
if (!normalized) {
|
||||
return songs
|
||||
}
|
||||
const terms = searchWords(normalized)
|
||||
const byLyrics = new Set(lyricsMatches)
|
||||
return songs.filter(
|
||||
(song) =>
|
||||
byLyrics.has(song.id) ||
|
||||
[song.title, song.artist.name, song.album.title].some((value) =>
|
||||
value.toLowerCase().includes(normalized)
|
||||
)
|
||||
return songs.filter((song) => {
|
||||
if (byLyrics.has(song.id)) return true
|
||||
const fields = [song.title, song.artist.name, song.album.title]
|
||||
return (
|
||||
fields.some((value) => value.toLowerCase().includes(normalized)) ||
|
||||
matchWords(searchWords(fields.join(" ")), terms) !== null
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import { chunk, type Chunked } from "./limits"
|
||||
import { recordIntentCatalogChanges } from "./intentCatalogChanges"
|
||||
import { playlistCoverVersion } from "./playlistCover"
|
||||
import { deleteSongLyrics, searchSongLyrics, writeSongLyrics } from "./lyrics"
|
||||
import { matchWords, searchWords } from "../client/fuzzy-search"
|
||||
|
||||
export type {
|
||||
BrowseAlbum,
|
||||
@@ -618,7 +619,16 @@ export const browseArtistDetail = query({
|
||||
},
|
||||
})
|
||||
|
||||
function matchScore(text: string, query: string): number {
|
||||
/**
|
||||
* How well `text` matches a lowercase search, from 100 (the whole text) down
|
||||
* to 20 (every word, give or take typos); 0 is no match. `terms` are the
|
||||
* query's `searchWords`.
|
||||
*/
|
||||
function matchScore(
|
||||
text: string,
|
||||
query: string,
|
||||
terms: readonly string[]
|
||||
): number {
|
||||
const lower = text.toLowerCase()
|
||||
if (lower === query) {
|
||||
return 100
|
||||
@@ -632,15 +642,20 @@ function matchScore(text: string, query: string): number {
|
||||
if (lower.includes(query)) {
|
||||
return 40
|
||||
}
|
||||
return 0
|
||||
const words = matchWords(searchWords(text), terms)
|
||||
if (words === "exact") {
|
||||
return 30
|
||||
}
|
||||
return words === "fuzzy" ? 20 : 0
|
||||
}
|
||||
|
||||
function matchScoreSongArtists(
|
||||
artist: string | string[],
|
||||
query: string
|
||||
query: string,
|
||||
terms: readonly string[]
|
||||
): number {
|
||||
return getSongArtistNames(artist).reduce(
|
||||
(best, name) => Math.max(best, matchScore(name, query)),
|
||||
(best, name) => Math.max(best, matchScore(name, query, terms)),
|
||||
0
|
||||
)
|
||||
}
|
||||
@@ -658,6 +673,7 @@ export const searchLibrary = query({
|
||||
args: { query: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const queryText = args.query.trim().toLowerCase()
|
||||
const terms = searchWords(queryText)
|
||||
if (!queryText || !(await isSignedIn(ctx))) {
|
||||
return {
|
||||
topResults: [],
|
||||
@@ -678,7 +694,7 @@ export const searchLibrary = query({
|
||||
.map((artist) => ({
|
||||
kind: "artist" as const,
|
||||
item: toBrowseArtist(artist),
|
||||
score: matchScore(artist.name, queryText),
|
||||
score: matchScore(artist.name, queryText, terms),
|
||||
}))
|
||||
.filter((result) => result.score > 0)
|
||||
.sort(
|
||||
@@ -692,8 +708,8 @@ export const searchLibrary = query({
|
||||
return []
|
||||
}
|
||||
const score = Math.max(
|
||||
matchScore(album.name, queryText),
|
||||
matchScore(artist.name, queryText)
|
||||
matchScore(album.name, queryText, terms),
|
||||
matchScore(artist.name, queryText, terms)
|
||||
)
|
||||
if (score === 0) {
|
||||
return []
|
||||
@@ -718,10 +734,10 @@ export const searchLibrary = query({
|
||||
return []
|
||||
}
|
||||
const score = Math.max(
|
||||
matchScore(song.name, queryText),
|
||||
matchScoreSongArtists(song.artist, queryText),
|
||||
matchScore(album.name, queryText),
|
||||
matchScore(albumArtist.name, queryText)
|
||||
matchScore(song.name, queryText, terms),
|
||||
matchScoreSongArtists(song.artist, queryText, terms),
|
||||
matchScore(album.name, queryText, terms),
|
||||
matchScore(albumArtist.name, queryText, terms)
|
||||
)
|
||||
if (score === 0) {
|
||||
return []
|
||||
|
||||
@@ -62,10 +62,10 @@ export async function deleteSongLyrics(ctx: MutationCtx, songId: Id<"songs">) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Songs with a lyrics line matching `query` (see `matchLyrics`), with that
|
||||
* line. Lines holding the query as typed come first, then the search index's
|
||||
* relevance order. The index matches words anywhere in the lyrics, so its
|
||||
* results are checked line by line.
|
||||
* Up to `limit` songs with a lyrics line matching `query` (see `matchLyrics`),
|
||||
* with that line, best matches first. The search index matches any word
|
||||
* anywhere in the lyrics and ranks common words poorly, so its whole page is
|
||||
* checked line by line before trimming.
|
||||
*/
|
||||
export async function searchSongLyrics(
|
||||
ctx: QueryCtx,
|
||||
@@ -77,15 +77,15 @@ export async function searchSongLyrics(
|
||||
const rows = await ctx.db
|
||||
.query("lyricsSearch")
|
||||
.withSearchIndex("search_plain", (q) => q.search("plain", text))
|
||||
.take(limit)
|
||||
const matches = rows.flatMap((row) => {
|
||||
.take(MAX_LYRICS_MATCHES)
|
||||
return rows
|
||||
.flatMap((row) => {
|
||||
const match = matchLyrics(row.plain, text)
|
||||
return match ? [{ song: row.song, ...match }] : []
|
||||
})
|
||||
return [
|
||||
...matches.filter((match) => match.exact),
|
||||
...matches.filter((match) => !match.exact),
|
||||
].map(({ song, snippet }) => ({ song, snippet }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
.map(({ song, snippet }) => ({ song, snippet }))
|
||||
}
|
||||
|
||||
/** The song's lyrics tag (see `client/lyrics.ts`), or null without lyrics. */
|
||||
|
||||
Reference in new issue
Block a user