feat: fuzzy search for songs and lyrics (#140)
This commit was merged in pull request #140.
This commit is contained in:
1 parent
332a7aed1f
commit
6e2794df66
5 files changed
+169
-56
No files matched your search
@@ -0,0 +1,93 @@
|
|||||||
|
/**
|
||||||
|
* Typo-tolerant word matching, shared by library search, lyrics search and
|
||||||
|
* in-list song filters.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lowercase words without accents or apostrophes, so "Don't Stop Beyoncé"
|
||||||
|
* gives ["dont", "stop", "beyonce"].
|
||||||
|
*/
|
||||||
|
export function searchWords(text: string): string[] {
|
||||||
|
return (
|
||||||
|
text
|
||||||
|
.normalize("NFKD")
|
||||||
|
.replace(/\p{M}/gu, "")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/['’]/g, "")
|
||||||
|
.match(/[\p{L}\p{N}]+/gu) ?? []
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Typos a query word may hold. Short words get none, or they match too much. */
|
||||||
|
function allowedTypos(term: string) {
|
||||||
|
if (term.length < 4) return 0
|
||||||
|
return term.length < 8 ? 1 : 2
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Edits (insert, delete, substitute or swap neighbors) turning `a` into `b`,
|
||||||
|
* or `max + 1` once it is sure to exceed `max`.
|
||||||
|
*/
|
||||||
|
function editDistance(a: string, b: string, max: number): number {
|
||||||
|
if (Math.abs(a.length - b.length) > max) return max + 1
|
||||||
|
let before: number[] = []
|
||||||
|
let previous = Array.from({ length: b.length + 1 }, (_, j) => j)
|
||||||
|
for (let i = 1; i <= a.length; i++) {
|
||||||
|
const current = [i]
|
||||||
|
let rowMin = i
|
||||||
|
for (let j = 1; j <= b.length; j++) {
|
||||||
|
const cost = a[i - 1] === b[j - 1] ? 0 : 1
|
||||||
|
let distance = Math.min(
|
||||||
|
previous[j]! + 1,
|
||||||
|
current[j - 1]! + 1,
|
||||||
|
previous[j - 1]! + cost
|
||||||
|
)
|
||||||
|
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
|
||||||
|
distance = Math.min(distance, before[j - 2]! + 1)
|
||||||
|
}
|
||||||
|
current.push(distance)
|
||||||
|
rowMin = Math.min(rowMin, distance)
|
||||||
|
}
|
||||||
|
if (rowMin > max) return max + 1
|
||||||
|
before = previous
|
||||||
|
previous = current
|
||||||
|
}
|
||||||
|
return previous[b.length]!
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether `word` is query `term` give or take a few typos. With `prefix` the
|
||||||
|
* word may also just start with the term, as when it is still being typed.
|
||||||
|
*/
|
||||||
|
function wordMatchesTerm(term: string, word: string, prefix: boolean) {
|
||||||
|
const typos = allowedTypos(term)
|
||||||
|
if (typos === 0) return false
|
||||||
|
if (editDistance(term, word, typos) <= typos) return true
|
||||||
|
return (
|
||||||
|
prefix &&
|
||||||
|
word.length > term.length &&
|
||||||
|
editDistance(term, word.slice(0, term.length), typos) <= typos
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How every query term (from `searchWords`) appears among `words`: "exact"
|
||||||
|
* when each is a word, "fuzzy" when some only match give or take typos, or
|
||||||
|
* null when one is missing. The last term may be a word's prefix.
|
||||||
|
*/
|
||||||
|
export function matchWords(
|
||||||
|
words: readonly string[],
|
||||||
|
terms: readonly string[]
|
||||||
|
): "exact" | "fuzzy" | null {
|
||||||
|
let quality: "exact" | "fuzzy" = "exact"
|
||||||
|
for (const [index, term] of terms.entries()) {
|
||||||
|
const prefix = index === terms.length - 1
|
||||||
|
const exact = words.some((word) =>
|
||||||
|
prefix ? word.startsWith(term) : word === term
|
||||||
|
)
|
||||||
|
if (exact) continue
|
||||||
|
if (!words.some((word) => wordMatchesTerm(term, word, prefix))) return null
|
||||||
|
quality = "fuzzy"
|
||||||
|
}
|
||||||
|
return terms.length > 0 ? quality : null
|
||||||
|
}
|
||||||
@@ -5,6 +5,8 @@
|
|||||||
* the same.
|
* the same.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { matchWords, searchWords } from "./fuzzy-search"
|
||||||
|
|
||||||
export type LyricWord = {
|
export type LyricWord = {
|
||||||
/** Includes its trailing space, so a line's words join back into its text. */
|
/** Includes its trailing space, so a line's words join back into its text. */
|
||||||
text: string
|
text: string
|
||||||
@@ -212,32 +214,32 @@ export function followLyrics(
|
|||||||
return () => clearTimeout(timer)
|
return () => clearTimeout(timer)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Lowercase words of a search, for matching lyrics the way the search index does. */
|
|
||||||
function searchTerms(query: string): string[] {
|
|
||||||
return query.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? []
|
|
||||||
}
|
|
||||||
|
|
||||||
const SNIPPET_LEAD = 24
|
const SNIPPET_LEAD = 24
|
||||||
|
|
||||||
export type LyricsMatch = {
|
export type LyricsMatch = {
|
||||||
/** The matched line, trimmed to start near the match. */
|
/** The matched line, trimmed to start near the match. */
|
||||||
snippet: string
|
snippet: string
|
||||||
/** Whether the line holds the query as typed, not just its words. */
|
/**
|
||||||
exact: boolean
|
* 2 when the line holds the query as typed, 1 when it holds every word of
|
||||||
|
* it, 0 when it does give or take typos.
|
||||||
|
*/
|
||||||
|
score: 0 | 1 | 2
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Where a search matched the lyrics: a line holding the query as typed, or
|
* Where a search matched the lyrics: the best line holding the query as
|
||||||
* else a line holding every word of it (the last may be a prefix, as when
|
* typed, or else every word of it (see `matchWords`). Null when no single
|
||||||
* typing). Null when no single line does.
|
* line does.
|
||||||
*/
|
*/
|
||||||
export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
||||||
const needle = query.trim().toLowerCase()
|
const needle = query.trim().toLowerCase()
|
||||||
const terms = searchTerms(needle)
|
const terms = searchWords(needle)
|
||||||
const last = terms.at(-1)
|
// Most songs miss a word entirely, so skip them before checking lines.
|
||||||
if (!needle || last === undefined) return null
|
if (!needle || !matchWords([...new Set(searchWords(plain))], terms)) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
let loose: string | null = null
|
let loose: LyricsMatch | null = null
|
||||||
for (const line of plain.split("\n")) {
|
for (const line of plain.split("\n")) {
|
||||||
const column = line.toLowerCase().indexOf(needle)
|
const column = line.toLowerCase().indexOf(needle)
|
||||||
if (column !== -1) {
|
if (column !== -1) {
|
||||||
@@ -245,15 +247,13 @@ export function matchLyrics(plain: string, query: string): LyricsMatch | null {
|
|||||||
column > SNIPPET_LEAD
|
column > SNIPPET_LEAD
|
||||||
? `…${line.slice(column - SNIPPET_LEAD).trimStart()}`
|
? `…${line.slice(column - SNIPPET_LEAD).trimStart()}`
|
||||||
: line.trim()
|
: line.trim()
|
||||||
return { snippet, exact: true }
|
return { snippet, score: 2 }
|
||||||
}
|
}
|
||||||
if (loose === null) {
|
if (loose?.score === 1) continue
|
||||||
const words = searchTerms(line)
|
const quality = matchWords(searchWords(line), terms)
|
||||||
const hasAll =
|
if (quality === "exact" || (quality && !loose)) {
|
||||||
terms.slice(0, -1).every((term) => words.includes(term)) &&
|
loose = { snippet: line.trim(), score: quality === "exact" ? 1 : 0 }
|
||||||
words.some((word) => word.startsWith(last))
|
|
||||||
if (hasAll) loose = line.trim()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return loose === null ? null : { snippet: loose, exact: false }
|
return loose
|
||||||
}
|
}
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { matchWords, searchWords } from "./fuzzy-search"
|
||||||
|
|
||||||
type SearchableSong = {
|
type SearchableSong = {
|
||||||
id: string
|
id: string
|
||||||
title: string
|
title: string
|
||||||
@@ -6,9 +8,9 @@ type SearchableSong = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Case-insensitive match on title, artist, or album, plus the songs in
|
* Case-insensitive match on title, artist, or album, give or take typos (see
|
||||||
* `lyricsMatches` (from `api.lyrics.matchingSongIds`). A blank query keeps
|
* `matchWords`), plus the songs in `lyricsMatches` (from
|
||||||
* every song.
|
* `api.lyrics.matchingSongIds`). A blank query keeps every song.
|
||||||
*/
|
*/
|
||||||
export function filterSongs<T extends SearchableSong>(
|
export function filterSongs<T extends SearchableSong>(
|
||||||
songs: T[],
|
songs: T[],
|
||||||
@@ -19,12 +21,14 @@ export function filterSongs<T extends SearchableSong>(
|
|||||||
if (!normalized) {
|
if (!normalized) {
|
||||||
return songs
|
return songs
|
||||||
}
|
}
|
||||||
|
const terms = searchWords(normalized)
|
||||||
const byLyrics = new Set(lyricsMatches)
|
const byLyrics = new Set(lyricsMatches)
|
||||||
return songs.filter(
|
return songs.filter((song) => {
|
||||||
(song) =>
|
if (byLyrics.has(song.id)) return true
|
||||||
byLyrics.has(song.id) ||
|
const fields = [song.title, song.artist.name, song.album.title]
|
||||||
[song.title, song.artist.name, song.album.title].some((value) =>
|
return (
|
||||||
value.toLowerCase().includes(normalized)
|
fields.some((value) => value.toLowerCase().includes(normalized)) ||
|
||||||
)
|
matchWords(searchWords(fields.join(" ")), terms) !== null
|
||||||
)
|
)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
@@ -27,6 +27,7 @@ import { chunk, type Chunked } from "./limits"
|
|||||||
import { recordIntentCatalogChanges } from "./intentCatalogChanges"
|
import { recordIntentCatalogChanges } from "./intentCatalogChanges"
|
||||||
import { playlistCoverVersion } from "./playlistCover"
|
import { playlistCoverVersion } from "./playlistCover"
|
||||||
import { deleteSongLyrics, searchSongLyrics, writeSongLyrics } from "./lyrics"
|
import { deleteSongLyrics, searchSongLyrics, writeSongLyrics } from "./lyrics"
|
||||||
|
import { matchWords, searchWords } from "../client/fuzzy-search"
|
||||||
|
|
||||||
export type {
|
export type {
|
||||||
BrowseAlbum,
|
BrowseAlbum,
|
||||||
@@ -618,7 +619,16 @@ export const browseArtistDetail = query({
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
function matchScore(text: string, query: string): number {
|
/**
|
||||||
|
* How well `text` matches a lowercase search, from 100 (the whole text) down
|
||||||
|
* to 20 (every word, give or take typos); 0 is no match. `terms` are the
|
||||||
|
* query's `searchWords`.
|
||||||
|
*/
|
||||||
|
function matchScore(
|
||||||
|
text: string,
|
||||||
|
query: string,
|
||||||
|
terms: readonly string[]
|
||||||
|
): number {
|
||||||
const lower = text.toLowerCase()
|
const lower = text.toLowerCase()
|
||||||
if (lower === query) {
|
if (lower === query) {
|
||||||
return 100
|
return 100
|
||||||
@@ -632,15 +642,20 @@ function matchScore(text: string, query: string): number {
|
|||||||
if (lower.includes(query)) {
|
if (lower.includes(query)) {
|
||||||
return 40
|
return 40
|
||||||
}
|
}
|
||||||
return 0
|
const words = matchWords(searchWords(text), terms)
|
||||||
|
if (words === "exact") {
|
||||||
|
return 30
|
||||||
|
}
|
||||||
|
return words === "fuzzy" ? 20 : 0
|
||||||
}
|
}
|
||||||
|
|
||||||
function matchScoreSongArtists(
|
function matchScoreSongArtists(
|
||||||
artist: string | string[],
|
artist: string | string[],
|
||||||
query: string
|
query: string,
|
||||||
|
terms: readonly string[]
|
||||||
): number {
|
): number {
|
||||||
return getSongArtistNames(artist).reduce(
|
return getSongArtistNames(artist).reduce(
|
||||||
(best, name) => Math.max(best, matchScore(name, query)),
|
(best, name) => Math.max(best, matchScore(name, query, terms)),
|
||||||
0
|
0
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -658,6 +673,7 @@ export const searchLibrary = query({
|
|||||||
args: { query: v.string() },
|
args: { query: v.string() },
|
||||||
handler: async (ctx, args) => {
|
handler: async (ctx, args) => {
|
||||||
const queryText = args.query.trim().toLowerCase()
|
const queryText = args.query.trim().toLowerCase()
|
||||||
|
const terms = searchWords(queryText)
|
||||||
if (!queryText || !(await isSignedIn(ctx))) {
|
if (!queryText || !(await isSignedIn(ctx))) {
|
||||||
return {
|
return {
|
||||||
topResults: [],
|
topResults: [],
|
||||||
@@ -678,7 +694,7 @@ export const searchLibrary = query({
|
|||||||
.map((artist) => ({
|
.map((artist) => ({
|
||||||
kind: "artist" as const,
|
kind: "artist" as const,
|
||||||
item: toBrowseArtist(artist),
|
item: toBrowseArtist(artist),
|
||||||
score: matchScore(artist.name, queryText),
|
score: matchScore(artist.name, queryText, terms),
|
||||||
}))
|
}))
|
||||||
.filter((result) => result.score > 0)
|
.filter((result) => result.score > 0)
|
||||||
.sort(
|
.sort(
|
||||||
@@ -692,8 +708,8 @@ export const searchLibrary = query({
|
|||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
const score = Math.max(
|
const score = Math.max(
|
||||||
matchScore(album.name, queryText),
|
matchScore(album.name, queryText, terms),
|
||||||
matchScore(artist.name, queryText)
|
matchScore(artist.name, queryText, terms)
|
||||||
)
|
)
|
||||||
if (score === 0) {
|
if (score === 0) {
|
||||||
return []
|
return []
|
||||||
@@ -718,10 +734,10 @@ export const searchLibrary = query({
|
|||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
const score = Math.max(
|
const score = Math.max(
|
||||||
matchScore(song.name, queryText),
|
matchScore(song.name, queryText, terms),
|
||||||
matchScoreSongArtists(song.artist, queryText),
|
matchScoreSongArtists(song.artist, queryText, terms),
|
||||||
matchScore(album.name, queryText),
|
matchScore(album.name, queryText, terms),
|
||||||
matchScore(albumArtist.name, queryText)
|
matchScore(albumArtist.name, queryText, terms)
|
||||||
)
|
)
|
||||||
if (score === 0) {
|
if (score === 0) {
|
||||||
return []
|
return []
|
||||||
|
|||||||
@@ -62,10 +62,10 @@ export async function deleteSongLyrics(ctx: MutationCtx, songId: Id<"songs">) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Songs with a lyrics line matching `query` (see `matchLyrics`), with that
|
* Up to `limit` songs with a lyrics line matching `query` (see `matchLyrics`),
|
||||||
* line. Lines holding the query as typed come first, then the search index's
|
* with that line, best matches first. The search index matches any word
|
||||||
* relevance order. The index matches words anywhere in the lyrics, so its
|
* anywhere in the lyrics and ranks common words poorly, so its whole page is
|
||||||
* results are checked line by line.
|
* checked line by line before trimming.
|
||||||
*/
|
*/
|
||||||
export async function searchSongLyrics(
|
export async function searchSongLyrics(
|
||||||
ctx: QueryCtx,
|
ctx: QueryCtx,
|
||||||
@@ -77,15 +77,15 @@ export async function searchSongLyrics(
|
|||||||
const rows = await ctx.db
|
const rows = await ctx.db
|
||||||
.query("lyricsSearch")
|
.query("lyricsSearch")
|
||||||
.withSearchIndex("search_plain", (q) => q.search("plain", text))
|
.withSearchIndex("search_plain", (q) => q.search("plain", text))
|
||||||
.take(limit)
|
.take(MAX_LYRICS_MATCHES)
|
||||||
const matches = rows.flatMap((row) => {
|
return rows
|
||||||
const match = matchLyrics(row.plain, text)
|
.flatMap((row) => {
|
||||||
return match ? [{ song: row.song, ...match }] : []
|
const match = matchLyrics(row.plain, text)
|
||||||
})
|
return match ? [{ song: row.song, ...match }] : []
|
||||||
return [
|
})
|
||||||
...matches.filter((match) => match.exact),
|
.sort((a, b) => b.score - a.score)
|
||||||
...matches.filter((match) => !match.exact),
|
.slice(0, limit)
|
||||||
].map(({ song, snippet }) => ({ song, snippet }))
|
.map(({ song, snippet }) => ({ song, snippet }))
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The song's lyrics tag (see `client/lyrics.ts`), or null without lyrics. */
|
/** The song's lyrics tag (see `client/lyrics.ts`), or null without lyrics. */
|
||||||
|
|||||||
Reference in new issue
Block a user