feat: fuzzy search for songs and lyrics (#140)
This commit was merged in pull request #140.
This commit is contained in:
1 parent
332a7aed1f
commit
6e2794df66
5 files changed
+165
-52
No files matched your search
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Typo-tolerant word matching, shared by library search, lyrics search and
|
||||
* in-list song filters.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Lowercase words without accents or apostrophes, so "Don't Stop Beyoncé"
|
||||
* gives ["dont", "stop", "beyonce"].
|
||||
*/
|
||||
export function searchWords(text: string): string[] {
|
||||
return (
|
||||
text
|
||||
.normalize("NFKD")
|
||||
.replace(/\p{M}/gu, "")
|
||||
.toLowerCase()
|
||||
.replace(/['’]/g, "")
|
||||
.match(/[\p{L}\p{N}]+/gu) ?? []
|
||||
)
|
||||
}
|
||||
|
||||
/** Typos a query word may hold. Short words get none, or they match too much. */
|
||||
function allowedTypos(term: string) {
|
||||
if (term.length < 4) return 0
|
||||
return term.length < 8 ? 1 : 2
|
||||
}
|
||||
|
||||
/**
|
||||
* Edits (insert, delete, substitute or swap neighbors) turning `a` into `b`,
|
||||
* or `max + 1` once it is sure to exceed `max`.
|
||||
*/
|
||||
function editDistance(a: string, b: string, max: number): number {
|
||||
if (Math.abs(a.length - b.length) > max) return max + 1
|
||||
let before: number[] = []
|
||||
let previous = Array.from({ length: b.length + 1 }, (_, j) => j)
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
const current = [i]
|
||||
let rowMin = i
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1
|
||||
let distance = Math.min(
|
||||
previous[j]! + 1,
|
||||
current[j - 1]! + 1,
|
||||
previous[j - 1]! + cost
|
||||
)
|
||||
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
|
||||
distance = Math.min(distance, before[j - 2]! + 1)
|
||||
}
|
||||
current.push(distance)
|
||||
rowMin = Math.min(rowMin, distance)
|
||||
}
|
||||
if (rowMin > max) return max + 1
|
||||
before = previous
|
||||
previous = current
|
||||
}
|
||||
return previous[b.length]!
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `word` is query `term` give or take a few typos. With `prefix` the
|
||||
* word may also just start with the term, as when it is still being typed.
|
||||
*/
|
||||
function wordMatchesTerm(term: string, word: string, prefix: boolean) {
|
||||
const typos = allowedTypos(term)
|
||||
if (typos === 0) return false
|
||||
if (editDistance(term, word, typos) <= typos) return true
|
||||
return (
|
||||
prefix &&
|
||||
word.length > term.length &&
|
||||
editDistance(term, word.slice(0, term.length), typos) <= typos
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* How every query term (from `searchWords`) appears among `words`: "exact"
|
||||
* when each is a word, "fuzzy" when some only match give or take typos, or
|
||||
* null when one is missing. The last term may be a word's prefix.
|
||||
*/
|
||||
export function matchWords(
|
||||
words: readonly string[],
|
||||
terms: readonly string[]
|
||||
): "exact" | "fuzzy" | null {
|
||||
let quality: "exact" | "fuzzy" = "exact"
|
||||
for (const [index, term] of terms.entries()) {
|
||||
const prefix = index === terms.length - 1
|
||||
const exact = words.some((word) =>
|
||||
prefix ? word.startsWith(term) : word === term
|
||||
)
|
||||
if (exact) continue
|
||||
if (!words.some((word) => wordMatchesTerm(term, word, prefix))) return null
|
||||
quality = "fuzzy"
|
||||
}
|
||||
return terms.length > 0 ? quality : null
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
* the same.
|
||||
*/
|
||||
|
||||
import { matchWords, searchWords } from "./fuzzy-search"
|
||||
|
||||
export type LyricWord = {
|
||||
/** Includes its trailing space, so a line's words join back into its text. */
|
||||
text: string
|
||||
@@ -212,32 +214,32 @@ export function followLyrics(
|
||||
return () => clearTimeout(timer)
|
||||
}
|
||||
|
||||
/** Lowercase words of a search, for matching lyrics the way the search index does. */
|
||||
function searchTerms(query: string): string[] {
|
||||
return query.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? []
|
||||
}
|
||||
|
||||
const SNIPPET_LEAD = 24
|
||||
|
||||
export type LyricsMatch = {
|
||||
/** The matched line, trimmed to start near the match. */
|
||||
snippet: string
|
||||
/** Whether the line holds the query as typed, not just its words. */
|
||||
exact: boolean
|
||||
/**
|
||||
* 2 when the line holds the query as typed, 1 when it holds every word of
|
||||
* it, 0 when it does give or take typos.
|
||||
*/
|
||||
score: 0 | 1 | 2
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a search matched the lyrics: a line holding the query as typed, or
|
||||
* else a line holding every word of it (the last may be a prefix, as when
|
||||
* typing). Null when no single line does.
|
||||
* Where a search matched the lyrics: the best line holding the query as
|
||||
* typed, or else every word of it (see `matchWords`). Null when no single
|
||||
* line does.
|
||||
*/
|
||||
export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
||||
const needle = query.trim().toLowerCase()
|
||||
const terms = searchTerms(needle)
|
||||
const last = terms.at(-1)
|
||||
if (!needle || last === undefined) return null
|
||||
const terms = searchWords(needle)
|
||||
// Most songs miss a word entirely, so skip them before checking lines.
|
||||
if (!needle || !matchWords([...new Set(searchWords(plain))], terms)) {
|
||||
return null
|
||||
}
|
||||
|
||||
let loose: string | null = null
|
||||
let loose: LyricsMatch | null = null
|
||||
for (const line of plain.split("\n")) {
|
||||
const column = line.toLowerCase().indexOf(needle)
|
||||
if (column !== -1) {
|
||||
@@ -245,15 +247,13 @@ export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
||||
column > SNIPPET_LEAD
|
||||
? `…${line.slice(column - SNIPPET_LEAD).trimStart()}`
|
||||
: line.trim()
|
||||
return { snippet, exact: true }
|
||||
return { snippet, score: 2 }
|
||||
}
|
||||
if (loose === null) {
|
||||
const words = searchTerms(line)
|
||||
const hasAll =
|
||||
terms.slice(0, -1).every((term) => words.includes(term)) &&
|
||||
words.some((word) => word.startsWith(last))
|
||||
if (hasAll) loose = line.trim()
|
||||
if (loose?.score === 1) continue
|
||||
const quality = matchWords(searchWords(line), terms)
|
||||
if (quality === "exact" || (quality && !loose)) {
|
||||
loose = { snippet: line.trim(), score: quality === "exact" ? 1 : 0 }
|
||||
}
|
||||
}
|
||||
return loose === null ? null : { snippet: loose, exact: false }
|
||||
return loose
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { matchWords, searchWords } from "./fuzzy-search"
|
||||
|
||||
type SearchableSong = {
|
||||
id: string
|
||||
title: string
|
||||
@@ -6,9 +8,9 @@ type SearchableSong = {
|
||||
}
|
||||
|
||||
/**
|
||||
* Case-insensitive match on title, artist, or album, plus the songs in
|
||||
* `lyricsMatches` (from `api.lyrics.matchingSongIds`). A blank query keeps
|
||||
* every song.
|
||||
* Case-insensitive match on title, artist, or album, give or take typos (see
|
||||
* `matchWords`), plus the songs in `lyricsMatches` (from
|
||||
* `api.lyrics.matchingSongIds`). A blank query keeps every song.
|
||||
*/
|
||||
export function filterSongs<T extends SearchableSong>(
|
||||
songs: T[],
|
||||
@@ -19,12 +21,14 @@ export function filterSongs<T extends SearchableSong>(
|
||||
if (!normalized) {
|
||||
return songs
|
||||
}
|
||||
const terms = searchWords(normalized)
|
||||
const byLyrics = new Set(lyricsMatches)
|
||||
return songs.filter(
|
||||
(song) =>
|
||||
byLyrics.has(song.id) ||
|
||||
[song.title, song.artist.name, song.album.title].some((value) =>
|
||||
value.toLowerCase().includes(normalized)
|
||||
)
|
||||
return songs.filter((song) => {
|
||||
if (byLyrics.has(song.id)) return true
|
||||
const fields = [song.title, song.artist.name, song.album.title]
|
||||
return (
|
||||
fields.some((value) => value.toLowerCase().includes(normalized)) ||
|
||||
matchWords(searchWords(fields.join(" ")), terms) !== null
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import { chunk, type Chunked } from "./limits"
|
||||
import { recordIntentCatalogChanges } from "./intentCatalogChanges"
|
||||
import { playlistCoverVersion } from "./playlistCover"
|
||||
import { deleteSongLyrics, searchSongLyrics, writeSongLyrics } from "./lyrics"
|
||||
import { matchWords, searchWords } from "../client/fuzzy-search"
|
||||
|
||||
export type {
|
||||
BrowseAlbum,
|
||||
@@ -618,7 +619,16 @@ export const browseArtistDetail = query({
|
||||
},
|
||||
})
|
||||
|
||||
function matchScore(text: string, query: string): number {
|
||||
/**
|
||||
* How well `text` matches a lowercase search, from 100 (the whole text) down
|
||||
* to 20 (every word, give or take typos); 0 is no match. `terms` are the
|
||||
* query's `searchWords`.
|
||||
*/
|
||||
function matchScore(
|
||||
text: string,
|
||||
query: string,
|
||||
terms: readonly string[]
|
||||
): number {
|
||||
const lower = text.toLowerCase()
|
||||
if (lower === query) {
|
||||
return 100
|
||||
@@ -632,15 +642,20 @@ function matchScore(text: string, query: string): number {
|
||||
if (lower.includes(query)) {
|
||||
return 40
|
||||
}
|
||||
return 0
|
||||
const words = matchWords(searchWords(text), terms)
|
||||
if (words === "exact") {
|
||||
return 30
|
||||
}
|
||||
return words === "fuzzy" ? 20 : 0
|
||||
}
|
||||
|
||||
function matchScoreSongArtists(
|
||||
artist: string | string[],
|
||||
query: string
|
||||
query: string,
|
||||
terms: readonly string[]
|
||||
): number {
|
||||
return getSongArtistNames(artist).reduce(
|
||||
(best, name) => Math.max(best, matchScore(name, query)),
|
||||
(best, name) => Math.max(best, matchScore(name, query, terms)),
|
||||
0
|
||||
)
|
||||
}
|
||||
@@ -658,6 +673,7 @@ export const searchLibrary = query({
|
||||
args: { query: v.string() },
|
||||
handler: async (ctx, args) => {
|
||||
const queryText = args.query.trim().toLowerCase()
|
||||
const terms = searchWords(queryText)
|
||||
if (!queryText || !(await isSignedIn(ctx))) {
|
||||
return {
|
||||
topResults: [],
|
||||
@@ -678,7 +694,7 @@ export const searchLibrary = query({
|
||||
.map((artist) => ({
|
||||
kind: "artist" as const,
|
||||
item: toBrowseArtist(artist),
|
||||
score: matchScore(artist.name, queryText),
|
||||
score: matchScore(artist.name, queryText, terms),
|
||||
}))
|
||||
.filter((result) => result.score > 0)
|
||||
.sort(
|
||||
@@ -692,8 +708,8 @@ export const searchLibrary = query({
|
||||
return []
|
||||
}
|
||||
const score = Math.max(
|
||||
matchScore(album.name, queryText),
|
||||
matchScore(artist.name, queryText)
|
||||
matchScore(album.name, queryText, terms),
|
||||
matchScore(artist.name, queryText, terms)
|
||||
)
|
||||
if (score === 0) {
|
||||
return []
|
||||
@@ -718,10 +734,10 @@ export const searchLibrary = query({
|
||||
return []
|
||||
}
|
||||
const score = Math.max(
|
||||
matchScore(song.name, queryText),
|
||||
matchScoreSongArtists(song.artist, queryText),
|
||||
matchScore(album.name, queryText),
|
||||
matchScore(albumArtist.name, queryText)
|
||||
matchScore(song.name, queryText, terms),
|
||||
matchScoreSongArtists(song.artist, queryText, terms),
|
||||
matchScore(album.name, queryText, terms),
|
||||
matchScore(albumArtist.name, queryText, terms)
|
||||
)
|
||||
if (score === 0) {
|
||||
return []
|
||||
|
||||
@@ -62,10 +62,10 @@ export async function deleteSongLyrics(ctx: MutationCtx, songId: Id<"songs">) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Songs with a lyrics line matching `query` (see `matchLyrics`), with that
|
||||
* line. Lines holding the query as typed come first, then the search index's
|
||||
* relevance order. The index matches words anywhere in the lyrics, so its
|
||||
* results are checked line by line.
|
||||
* Up to `limit` songs with a lyrics line matching `query` (see `matchLyrics`),
|
||||
* with that line, best matches first. The search index matches any word
|
||||
* anywhere in the lyrics and ranks common words poorly, so its whole page is
|
||||
* checked line by line before trimming.
|
||||
*/
|
||||
export async function searchSongLyrics(
|
||||
ctx: QueryCtx,
|
||||
@@ -77,15 +77,15 @@ export async function searchSongLyrics(
|
||||
const rows = await ctx.db
|
||||
.query("lyricsSearch")
|
||||
.withSearchIndex("search_plain", (q) => q.search("plain", text))
|
||||
.take(limit)
|
||||
const matches = rows.flatMap((row) => {
|
||||
.take(MAX_LYRICS_MATCHES)
|
||||
return rows
|
||||
.flatMap((row) => {
|
||||
const match = matchLyrics(row.plain, text)
|
||||
return match ? [{ song: row.song, ...match }] : []
|
||||
})
|
||||
return [
|
||||
...matches.filter((match) => match.exact),
|
||||
...matches.filter((match) => !match.exact),
|
||||
].map(({ song, snippet }) => ({ song, snippet }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
.map(({ song, snippet }) => ({ song, snippet }))
|
||||
}
|
||||
|
||||
/** The song's lyrics tag (see `client/lyrics.ts`), or null without lyrics. */
|
||||
|
||||
Reference in new issue
Block a user