Compare commits
1
Commits
No files matched your search
+5
-6
@@ -54,12 +54,11 @@ playwright.config.*
|
||||
*~
|
||||
*.log
|
||||
|
||||
# Environment variables (only commit template files)
|
||||
.env
|
||||
.env*.local
|
||||
.env.development
|
||||
.env.test
|
||||
.env.production.local
|
||||
# Environment variables, in any workspace. Next.js inlines NEXT_PUBLIC_* values
|
||||
# into the bundle and the Convex CLI reads .env.local on deploy.
|
||||
**/.env
|
||||
**/.env.*
|
||||
!**/.env.example
|
||||
|
||||
# Docker configuration files (not needed inside build context)
|
||||
Dockerfile*
|
||||
|
||||
+6
-1
@@ -195,7 +195,12 @@ RUN node --version \
|
||||
&& ldd /convex/generate_key \
|
||||
&& ! ldd /convex/convex-local-backend /convex/generate_key | grep -q 'not found'
|
||||
|
||||
RUN useradd --system --user-group --home-dir /app --shell /usr/sbin/nologin otunes
|
||||
# The backend gets its own user so the web server cannot read the database or
|
||||
# instance secret. It joins the otunes group to clean up the web server's HLS
|
||||
# cache in /tmp/otunes.
|
||||
RUN useradd --system --user-group --home-dir /app --shell /usr/sbin/nologin otunes \
|
||||
&& useradd --system --user-group --groups otunes --create-home --home-dir /home/convex \
|
||||
--shell /usr/sbin/nologin convex
|
||||
|
||||
COPY --from=convex-functions /app /opt/otunes/convex
|
||||
COPY docker/generate-auth-keys.mjs /opt/otunes/generate-auth-keys.mjs
|
||||
|
||||
@@ -3,11 +3,11 @@ import type { Id } from "@otunes/convex/_generated/dataModel"
|
||||
import { audioContentTypeForPath } from "@/lib/audio-content-type"
|
||||
import { seekableLosslessFile } from "@/lib/server-lossless"
|
||||
import { getConvexClient } from "@/lib/convex-server"
|
||||
import { audioInfo } from "@/lib/server-audio-info"
|
||||
import { ffmpegToReadableStream, streamFile } from "@/lib/server-audio-stream"
|
||||
import { spawnFfmpeg } from "@/lib/server-ffmpeg"
|
||||
import { NextRequest, NextResponse, userAgent } from "next/server"
|
||||
import { spawn } from "node:child_process"
|
||||
import fs from "node:fs/promises"
|
||||
import * as music from "music-metadata"
|
||||
|
||||
function parseByteRange(rangeHeader: string | null, fileSize: number) {
|
||||
if (!rangeHeader) {
|
||||
@@ -151,15 +151,14 @@ export async function GET(
|
||||
transcodeNeeded = false
|
||||
}
|
||||
|
||||
const metadata = await music.parseFile(song.files[originalCodec])
|
||||
|
||||
if (!transcodeNeeded && originalCodec === "lossless") {
|
||||
// Native iOS playback does not identify itself as Safari.
|
||||
const acceptsAlac =
|
||||
browser.name === "Safari" ||
|
||||
request.nextUrl.searchParams.get("losslessFormat") === "alac"
|
||||
if (
|
||||
acceptsAlac &&
|
||||
(await audioInfo(song.files[originalCodec])).codec === "ALAC"
|
||||
) {
|
||||
if (acceptsAlac && metadata.format.codec === "ALAC") {
|
||||
try {
|
||||
return await streamFileResponse(
|
||||
request,
|
||||
@@ -184,6 +183,11 @@ export async function GET(
|
||||
(originalCodec === "atmos" || originalCodec === "normal")
|
||||
) {
|
||||
// TODO: Dolby Atmos doesn't work on Google Chrome
|
||||
console.log(
|
||||
metadata.format.codec,
|
||||
metadata.format.container,
|
||||
metadata.format.codecProfile
|
||||
)
|
||||
try {
|
||||
return await streamFileResponse(
|
||||
request,
|
||||
@@ -196,6 +200,7 @@ export async function GET(
|
||||
}
|
||||
|
||||
const ffmpegArgs = [
|
||||
"-nostdin",
|
||||
"-i",
|
||||
song.files[originalCodec],
|
||||
"-c:a",
|
||||
@@ -209,8 +214,8 @@ export async function GET(
|
||||
|
||||
try {
|
||||
// TODO: Fix seeking for this
|
||||
const ffmpeg = await spawnFfmpeg(ffmpegArgs, request.signal)
|
||||
const stream = ffmpegToReadableStream(ffmpeg)
|
||||
const ffmpegProcess = spawn("ffmpeg", ffmpegArgs)
|
||||
const stream = ffmpegToReadableStream(ffmpegProcess)
|
||||
|
||||
return new NextResponse(stream, {
|
||||
status: 200,
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
import fs from "node:fs/promises"
|
||||
import * as music from "music-metadata"
|
||||
|
||||
const MAX_ENTRIES = 1000
|
||||
|
||||
export type AudioInfo = {
|
||||
codec: string | undefined
|
||||
duration: number | undefined
|
||||
}
|
||||
|
||||
const cache = new Map<string, Promise<AudioInfo>>()
|
||||
|
||||
/** Codec info for a file, parsed once per file version. */
|
||||
export async function audioInfo(file: string): Promise<AudioInfo> {
|
||||
const stat = await fs.stat(file)
|
||||
const key = `${file}\0${stat.size}\0${stat.mtimeMs}`
|
||||
|
||||
const cached = cache.get(key)
|
||||
if (cached) {
|
||||
cache.delete(key)
|
||||
cache.set(key, cached)
|
||||
return cached
|
||||
}
|
||||
|
||||
const info = music.parseFile(file).then(({ format }) => ({
|
||||
codec: format.codec,
|
||||
duration: format.duration,
|
||||
}))
|
||||
cache.set(key, info)
|
||||
info.catch(() => {
|
||||
if (cache.get(key) === info) {
|
||||
cache.delete(key)
|
||||
}
|
||||
})
|
||||
|
||||
if (cache.size > MAX_ENTRIES) {
|
||||
cache.delete(cache.keys().next().value!)
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
@@ -1,96 +1,96 @@
|
||||
import { createReadStream } from "node:fs"
|
||||
import type { Readable } from "node:stream"
|
||||
import type { FfmpegProcess } from "@/lib/server-ffmpeg"
|
||||
|
||||
const HIGH_WATER_MARK_BYTES = 256 * 1024
|
||||
import { type ChildProcessWithoutNullStreams } from "node:child_process"
|
||||
import { createReadStream, ReadStream } from "node:fs"
|
||||
|
||||
export type ByteRange = {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a Node stream, pausing it while the consumer is behind. `cancel` runs
|
||||
* when the consumer goes away (seek, track change, disconnect). `finish` runs
|
||||
* when the source ends and may reject to error the stream.
|
||||
*/
|
||||
function toWebStream(
|
||||
source: Readable,
|
||||
{
|
||||
cancel,
|
||||
finish,
|
||||
}: { cancel: () => void; finish?: () => Promise<void> }
|
||||
): ReadableStream<Uint8Array> {
|
||||
let done = false
|
||||
|
||||
return new ReadableStream<Uint8Array>(
|
||||
{
|
||||
start(controller) {
|
||||
const fail = (error: unknown) => {
|
||||
if (done) {
|
||||
return
|
||||
}
|
||||
done = true
|
||||
controller.error(error)
|
||||
cancel()
|
||||
}
|
||||
|
||||
source.on("data", (chunk: Buffer) => {
|
||||
if (done) {
|
||||
return
|
||||
}
|
||||
controller.enqueue(new Uint8Array(chunk))
|
||||
if ((controller.desiredSize ?? 0) <= 0) {
|
||||
source.pause()
|
||||
}
|
||||
})
|
||||
source.once("error", fail)
|
||||
source.once("end", () => {
|
||||
if (done) {
|
||||
return
|
||||
}
|
||||
void (finish?.() ?? Promise.resolve()).then(() => {
|
||||
if (!done) {
|
||||
done = true
|
||||
controller.close()
|
||||
}
|
||||
}, fail)
|
||||
})
|
||||
},
|
||||
pull() {
|
||||
source.resume()
|
||||
},
|
||||
cancel() {
|
||||
if (done) {
|
||||
return
|
||||
}
|
||||
done = true
|
||||
cancel()
|
||||
},
|
||||
},
|
||||
new ByteLengthQueuingStrategy({ highWaterMark: HIGH_WATER_MARK_BYTES })
|
||||
)
|
||||
async function* nodeStreamToIterator(stream: ReadStream) {
|
||||
for await (const chunk of stream) {
|
||||
yield new Uint8Array(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
export function streamFile(path: string, range?: ByteRange) {
|
||||
const file = createReadStream(path, range)
|
||||
return toWebStream(file, { cancel: () => file.destroy() })
|
||||
}
|
||||
|
||||
/** Streams ffmpeg stdout and kills ffmpeg when the consumer goes away. */
|
||||
export function ffmpegToReadableStream({
|
||||
child,
|
||||
exited,
|
||||
stderrTail,
|
||||
}: FfmpegProcess) {
|
||||
return toWebStream(child.stdout, {
|
||||
cancel: () => child.kill("SIGKILL"),
|
||||
finish: async () => {
|
||||
const code = await exited
|
||||
if (code !== 0) {
|
||||
console.error(`ffmpeg transcode exited with code ${code}`, stderrTail())
|
||||
throw new Error(`ffmpeg exited with code ${code}`)
|
||||
// @ts-expect-error trust me bro
|
||||
function iteratorToStream(iterator) {
|
||||
return new ReadableStream({
|
||||
async pull(controller) {
|
||||
const { value, done } = await iterator.next()
|
||||
if (done) {
|
||||
controller.close()
|
||||
} else {
|
||||
controller.enqueue(value)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export function streamFile(path: string, range?: ByteRange): ReadableStream {
|
||||
const nodeStream = createReadStream(path, range)
|
||||
return iteratorToStream(nodeStreamToIterator(nodeStream))
|
||||
}
|
||||
|
||||
export function ffmpegToReadableStream(
|
||||
ffmpegProcess: ChildProcessWithoutNullStreams
|
||||
) {
|
||||
let closed = false
|
||||
|
||||
return new ReadableStream({
|
||||
start(controller) {
|
||||
const finalize = (err?: Error) => {
|
||||
if (closed) {
|
||||
return
|
||||
}
|
||||
closed = true
|
||||
|
||||
try {
|
||||
if (err) {
|
||||
controller.error(err)
|
||||
} else {
|
||||
controller.close()
|
||||
}
|
||||
} catch {
|
||||
// Consumer may have already closed the stream (e.g. track change).
|
||||
}
|
||||
}
|
||||
|
||||
ffmpegProcess.stdout.on("data", (chunk: Buffer) => {
|
||||
if (closed) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
controller.enqueue(new Uint8Array(chunk))
|
||||
} catch {
|
||||
closed = true
|
||||
ffmpegProcess.kill("SIGKILL")
|
||||
}
|
||||
})
|
||||
|
||||
ffmpegProcess.stderr.on("data", (data) => {
|
||||
console.error(`FFmpeg log: ${data.toString()}`)
|
||||
})
|
||||
|
||||
ffmpegProcess.stdout.on("end", () => finalize())
|
||||
ffmpegProcess.stdout.on("error", (err) => finalize(err))
|
||||
|
||||
ffmpegProcess.on("close", (code) => {
|
||||
if (code !== 0 && !closed) {
|
||||
finalize(new Error(`ffmpeg exited with code ${code}`))
|
||||
} else {
|
||||
finalize()
|
||||
}
|
||||
})
|
||||
|
||||
ffmpegProcess.on("error", (err) => finalize(err))
|
||||
},
|
||||
cancel() {
|
||||
if (closed) {
|
||||
return
|
||||
}
|
||||
closed = true
|
||||
ffmpegProcess.kill("SIGKILL")
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
import fs from "node:fs/promises"
|
||||
import os from "node:os"
|
||||
import path from "node:path"
|
||||
|
||||
const DEFAULT_MAX_MB = 2048
|
||||
const ORPHAN_MAX_AGE_MS = 60 * 60 * 1000
|
||||
/** Prefix for in-progress entries. Pruning removes old unpinned ones as abandoned. */
|
||||
export const CACHE_ORPHAN_PREFIX = "convert-"
|
||||
|
||||
type CacheState = {
|
||||
pinned: Map<string, number>
|
||||
pruning: Promise<void> | null
|
||||
}
|
||||
|
||||
const globalState = globalThis as typeof globalThis & {
|
||||
__otunesCache?: CacheState
|
||||
}
|
||||
const state = (globalState.__otunesCache ??= {
|
||||
pinned: new Map(),
|
||||
pruning: null,
|
||||
})
|
||||
|
||||
/**
|
||||
* Transcode cache root. Set OTUNES_CACHE_DIR to a persistent volume to keep
|
||||
* transcodes across restarts.
|
||||
*/
|
||||
export function cacheRoot() {
|
||||
return process.env.OTUNES_CACHE_DIR || path.join(os.tmpdir(), "otunes-cache")
|
||||
}
|
||||
|
||||
export function cacheDir(kind: "hls" | "lossless") {
|
||||
return path.join(cacheRoot(), kind)
|
||||
}
|
||||
|
||||
function maxCacheBytes() {
|
||||
const configured = Number(process.env.OTUNES_CACHE_MAX_MB)
|
||||
const mb = Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_MAX_MB
|
||||
return mb * 1024 * 1024
|
||||
}
|
||||
|
||||
/** Pinned entries are never pruned. Pins are reference counted. */
|
||||
export function pinCacheEntry(entry: string) {
|
||||
state.pinned.set(entry, (state.pinned.get(entry) ?? 0) + 1)
|
||||
}
|
||||
|
||||
export function unpinCacheEntry(entry: string) {
|
||||
const count = state.pinned.get(entry) ?? 0
|
||||
if (count <= 1) {
|
||||
state.pinned.delete(entry)
|
||||
} else {
|
||||
state.pinned.set(entry, count - 1)
|
||||
}
|
||||
}
|
||||
|
||||
/** Marks an entry as recently used so pruning removes it last. */
|
||||
export async function touchCacheEntry(entry: string) {
|
||||
const now = new Date()
|
||||
await fs.utimes(entry, now, now)
|
||||
}
|
||||
|
||||
async function entrySize(entry: string): Promise<number> {
|
||||
const stat = await fs.stat(entry).catch(() => null)
|
||||
if (!stat) {
|
||||
return 0
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
return stat.size
|
||||
}
|
||||
const names = await fs.readdir(entry).catch(() => [])
|
||||
const sizes = await Promise.all(
|
||||
names.map((name) => entrySize(path.join(entry, name)))
|
||||
)
|
||||
return sizes.reduce((total, size) => total + size, 0)
|
||||
}
|
||||
|
||||
async function prune() {
|
||||
const entries: {
|
||||
path: string
|
||||
size: number
|
||||
mtimeMs: number
|
||||
orphan: boolean
|
||||
}[] = []
|
||||
|
||||
for (const kind of ["hls", "lossless"] as const) {
|
||||
const dir = cacheDir(kind)
|
||||
const names = await fs.readdir(dir).catch(() => [])
|
||||
for (const name of names) {
|
||||
const entry = path.join(dir, name)
|
||||
const stat = await fs.stat(entry).catch(() => null)
|
||||
if (!stat) {
|
||||
continue
|
||||
}
|
||||
entries.push({
|
||||
path: entry,
|
||||
size: await entrySize(entry),
|
||||
mtimeMs: stat.mtimeMs,
|
||||
orphan: name.startsWith(CACHE_ORPHAN_PREFIX),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const maxBytes = maxCacheBytes()
|
||||
const now = Date.now()
|
||||
let total = entries.reduce((sum, entry) => sum + entry.size, 0)
|
||||
entries.sort((a, b) => a.mtimeMs - b.mtimeMs)
|
||||
|
||||
for (const entry of entries) {
|
||||
if (state.pinned.has(entry.path)) {
|
||||
continue
|
||||
}
|
||||
const remove = entry.orphan
|
||||
? now - entry.mtimeMs > ORPHAN_MAX_AGE_MS
|
||||
: total > maxBytes
|
||||
if (remove) {
|
||||
await fs.rm(entry.path, { recursive: true, force: true }).catch(() => {})
|
||||
total -= entry.size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes least recently used entries until the cache fits under
|
||||
* OTUNES_CACHE_MAX_MB, plus abandoned in-progress entries.
|
||||
*/
|
||||
export function pruneCache() {
|
||||
state.pruning ??= prune()
|
||||
.catch((error) => console.error("Transcode cache prune failed", error))
|
||||
.finally(() => {
|
||||
state.pruning = null
|
||||
})
|
||||
return state.pruning
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"
|
||||
import os from "node:os"
|
||||
|
||||
const STDERR_TAIL_BYTES = 4096
|
||||
|
||||
export type FfmpegProcess = {
|
||||
child: ChildProcessWithoutNullStreams
|
||||
/** Resolves with the exit code, or null when killed by a signal. */
|
||||
exited: Promise<number | null>
|
||||
stderrTail: () => string
|
||||
}
|
||||
|
||||
type Limiter = {
|
||||
running: number
|
||||
waiting: (() => void)[]
|
||||
}
|
||||
|
||||
// Kept on globalThis so dev reloads share one limit with processes they spawned.
|
||||
const globalState = globalThis as typeof globalThis & {
|
||||
__otunesFfmpegLimiter?: Limiter
|
||||
}
|
||||
const limiter = (globalState.__otunesFfmpegLimiter ??= {
|
||||
running: 0,
|
||||
waiting: [],
|
||||
})
|
||||
|
||||
function maxConcurrentFfmpeg() {
|
||||
const configured = Number(process.env.OTUNES_MAX_FFMPEG)
|
||||
if (Number.isInteger(configured) && configured > 0) {
|
||||
return configured
|
||||
}
|
||||
return Math.max(2, os.availableParallelism())
|
||||
}
|
||||
|
||||
async function acquireSlot(signal?: AbortSignal) {
|
||||
signal?.throwIfAborted()
|
||||
if (limiter.running < maxConcurrentFfmpeg()) {
|
||||
limiter.running += 1
|
||||
return
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const grant = () => {
|
||||
signal?.removeEventListener("abort", onAbort)
|
||||
resolve()
|
||||
}
|
||||
const onAbort = () => {
|
||||
const index = limiter.waiting.indexOf(grant)
|
||||
if (index !== -1) {
|
||||
limiter.waiting.splice(index, 1)
|
||||
}
|
||||
reject(signal?.reason)
|
||||
}
|
||||
limiter.waiting.push(grant)
|
||||
signal?.addEventListener("abort", onAbort, { once: true })
|
||||
})
|
||||
}
|
||||
|
||||
function releaseSlot() {
|
||||
const next = limiter.waiting.shift()
|
||||
if (next) {
|
||||
// The slot passes straight to the next waiter.
|
||||
next()
|
||||
} else {
|
||||
limiter.running -= 1
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawns ffmpeg once a slot under the concurrency cap is free. The slot is
|
||||
* held until the process exits. Rejects if `signal` aborts while queued.
|
||||
*/
|
||||
export async function spawnFfmpeg(
|
||||
args: string[],
|
||||
signal?: AbortSignal
|
||||
): Promise<FfmpegProcess> {
|
||||
await acquireSlot(signal)
|
||||
|
||||
let child: ChildProcessWithoutNullStreams
|
||||
try {
|
||||
child = spawn("ffmpeg", [
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
...args,
|
||||
])
|
||||
} catch (error) {
|
||||
releaseSlot()
|
||||
throw error
|
||||
}
|
||||
|
||||
// stderr must be drained or ffmpeg blocks once the pipe buffer fills.
|
||||
let stderr = ""
|
||||
child.stderr.on("data", (data: Buffer) => {
|
||||
stderr = (stderr + data.toString()).slice(-STDERR_TAIL_BYTES)
|
||||
})
|
||||
|
||||
const exited = new Promise<number | null>((resolve) => {
|
||||
let released = false
|
||||
const finish = (code: number | null) => {
|
||||
if (!released) {
|
||||
released = true
|
||||
releaseSlot()
|
||||
}
|
||||
resolve(code)
|
||||
}
|
||||
child.once("error", () => finish(1))
|
||||
child.once("close", (code) => finish(code))
|
||||
})
|
||||
|
||||
return { child, exited, stderrTail: () => stderr.trim() }
|
||||
}
|
||||
+93
-183
@@ -2,15 +2,8 @@ import type { Doc } from "@otunes/convex/_generated/dataModel"
|
||||
import crypto from "node:crypto"
|
||||
import fs from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { audioInfo } from "@/lib/server-audio-info"
|
||||
import {
|
||||
cacheDir,
|
||||
pinCacheEntry,
|
||||
pruneCache,
|
||||
touchCacheEntry,
|
||||
unpinCacheEntry,
|
||||
} from "@/lib/server-cache"
|
||||
import { spawnFfmpeg, type FfmpegProcess } from "@/lib/server-ffmpeg"
|
||||
import { spawn } from "node:child_process"
|
||||
import * as music from "music-metadata"
|
||||
|
||||
export const HLS_SEGMENT_SECONDS = 6
|
||||
const HLS_MAX_IDLE_MS = 30 * 60 * 1000
|
||||
@@ -24,28 +17,11 @@ type HlsJob = {
|
||||
lastTouchedAt: number
|
||||
processExited: boolean
|
||||
processExitCode: number | null
|
||||
process: FfmpegProcess | null
|
||||
abort: AbortController
|
||||
evicted: boolean
|
||||
}
|
||||
|
||||
type HlsState = {
|
||||
/** In-flight and ready jobs by cache key, so overlapping requests share one ffmpeg. */
|
||||
pending: Map<string, Promise<HlsJob>>
|
||||
jobs: Map<string, HlsJob>
|
||||
jobsByLookup: Map<string, HlsJob>
|
||||
lastCleanupAt: number
|
||||
}
|
||||
|
||||
const globalState = globalThis as typeof globalThis & {
|
||||
__otunesHls?: HlsState
|
||||
}
|
||||
const state = (globalState.__otunesHls ??= {
|
||||
pending: new Map(),
|
||||
jobs: new Map(),
|
||||
jobsByLookup: new Map(),
|
||||
lastCleanupAt: 0,
|
||||
})
|
||||
const jobs = new Map<string, HlsJob>()
|
||||
const jobsByLookup = new Map<string, HlsJob>()
|
||||
let lastCleanupAt = 0
|
||||
|
||||
export function qualityKbps(quality: string | null) {
|
||||
if (quality === "low") {
|
||||
@@ -74,6 +50,10 @@ export function selectHlsSourceCodec(
|
||||
return sourceCodec
|
||||
}
|
||||
|
||||
function hlsCacheRoot() {
|
||||
return "/tmp/otunes"
|
||||
}
|
||||
|
||||
function hlsLookupKey(
|
||||
songId: string,
|
||||
quality: string | null,
|
||||
@@ -108,38 +88,31 @@ export function hlsSegmentPath(job: HlsJob, segmentIndex: number) {
|
||||
|
||||
async function touchHlsJob(job: HlsJob) {
|
||||
job.lastTouchedAt = Date.now()
|
||||
await touchCacheEntry(job.dir).catch(() => {})
|
||||
await fs.utimes(job.dir, new Date(job.lastTouchedAt), new Date(job.lastTouchedAt)).catch(
|
||||
() => {}
|
||||
)
|
||||
}
|
||||
|
||||
function forgetJob(job: HlsJob) {
|
||||
jobs.delete(job.key)
|
||||
if (jobsByLookup.get(job.lookupKey) === job) {
|
||||
jobsByLookup.delete(job.lookupKey)
|
||||
}
|
||||
}
|
||||
|
||||
async function evictJob(job: HlsJob) {
|
||||
if (job.evicted) {
|
||||
return
|
||||
}
|
||||
job.evicted = true
|
||||
state.pending.delete(job.key)
|
||||
state.jobs.delete(job.key)
|
||||
if (state.jobsByLookup.get(job.lookupKey) === job) {
|
||||
state.jobsByLookup.delete(job.lookupKey)
|
||||
}
|
||||
|
||||
job.abort.abort()
|
||||
job.process?.child.kill("SIGKILL")
|
||||
const complete = job.processExited && job.processExitCode === 0
|
||||
if (!complete) {
|
||||
await job.process?.exited
|
||||
await fs.rm(job.dir, { recursive: true, force: true }).catch(() => {})
|
||||
}
|
||||
unpinCacheEntry(job.dir)
|
||||
forgetJob(job)
|
||||
await fs.rm(job.dir, { recursive: true, force: true }).catch(() => {})
|
||||
}
|
||||
|
||||
async function cleanupIdleHlsJobs() {
|
||||
const now = Date.now()
|
||||
if (now - state.lastCleanupAt < 60_000) {
|
||||
if (now - lastCleanupAt < 60_000) {
|
||||
return
|
||||
}
|
||||
state.lastCleanupAt = now
|
||||
lastCleanupAt = now
|
||||
|
||||
for (const job of [...state.jobs.values()]) {
|
||||
for (const job of [...jobs.values()]) {
|
||||
if (now - job.lastTouchedAt > HLS_MAX_IDLE_MS) {
|
||||
await evictJob(job)
|
||||
}
|
||||
@@ -151,7 +124,7 @@ export function getExistingHlsJob(
|
||||
quality: string | null,
|
||||
preferAtmos: boolean
|
||||
) {
|
||||
return state.jobsByLookup.get(hlsLookupKey(songId, quality, preferAtmos)) ?? null
|
||||
return jobsByLookup.get(hlsLookupKey(songId, quality, preferAtmos)) ?? null
|
||||
}
|
||||
|
||||
export async function waitForHlsSegment(
|
||||
@@ -164,22 +137,18 @@ export async function waitForHlsSegment(
|
||||
const startedAt = Date.now()
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
// Read before checking the file so a segment renamed just before exit is found.
|
||||
const exited = job.processExited
|
||||
try {
|
||||
// ffmpeg writes to a .tmp file and renames it when complete.
|
||||
await fs.access(segmentPath)
|
||||
return segmentPath
|
||||
} catch {
|
||||
if (job.evicted) {
|
||||
throw new Error("HLS job was evicted")
|
||||
const firstStat = await fs.stat(segmentPath)
|
||||
if (firstStat.size > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||
const secondStat = await fs.stat(segmentPath)
|
||||
if (firstStat.size === secondStat.size) {
|
||||
return segmentPath
|
||||
}
|
||||
}
|
||||
if (exited) {
|
||||
throw new Error(
|
||||
job.processExitCode === 0
|
||||
? "HLS segment does not exist"
|
||||
: `HLS transcode failed with code ${job.processExitCode}`
|
||||
)
|
||||
} catch {
|
||||
if (job.processExited && job.processExitCode !== 0) {
|
||||
throw new Error(`HLS transcode failed with code ${job.processExitCode}`)
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||
@@ -188,15 +157,52 @@ export async function waitForHlsSegment(
|
||||
throw new Error("Timed out waiting for HLS segment")
|
||||
}
|
||||
|
||||
async function hasCompletePlaylist(dir: string) {
|
||||
const playlist = await fs
|
||||
.readFile(path.join(dir, "index.m3u8"), "utf8")
|
||||
.catch(() => "")
|
||||
return playlist.includes("#EXT-X-ENDLIST")
|
||||
}
|
||||
export async function ensureHlsJob(
|
||||
song: Doc<"songs">,
|
||||
quality: string | null,
|
||||
preferAtmos: boolean
|
||||
) {
|
||||
await cleanupIdleHlsJobs()
|
||||
|
||||
const kbps = qualityKbps(quality)
|
||||
const sourceCodec = selectHlsSourceCodec(song, quality, preferAtmos)
|
||||
const sourcePath = song.files[sourceCodec]
|
||||
const key = await hlsJobKey(song, sourcePath, kbps)
|
||||
const lookupKey = hlsLookupKey(song._id, quality, preferAtmos)
|
||||
const existingJob = jobs.get(key)
|
||||
|
||||
if (existingJob) {
|
||||
try {
|
||||
await fs.access(existingJob.dir)
|
||||
await touchHlsJob(existingJob)
|
||||
return existingJob
|
||||
} catch {
|
||||
forgetJob(existingJob)
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = await music.parseFile(sourcePath)
|
||||
const duration = metadata.format.duration ?? song.length
|
||||
const segmentCount = Math.max(1, Math.ceil(duration / HLS_SEGMENT_SECONDS))
|
||||
const dir = path.join(hlsCacheRoot(), key)
|
||||
await fs.mkdir(dir, { recursive: true })
|
||||
|
||||
const job: HlsJob = {
|
||||
key,
|
||||
lookupKey,
|
||||
dir,
|
||||
duration,
|
||||
segmentCount,
|
||||
lastTouchedAt: Date.now(),
|
||||
processExited: false,
|
||||
processExitCode: null,
|
||||
}
|
||||
jobs.set(key, job)
|
||||
jobsByLookup.set(lookupKey, job)
|
||||
await touchHlsJob(job)
|
||||
|
||||
function startTranscode(job: HlsJob, sourcePath: string, kbps: number) {
|
||||
const ffmpegArgs = [
|
||||
"-nostdin",
|
||||
"-i",
|
||||
sourcePath,
|
||||
"-vn",
|
||||
@@ -216,121 +222,25 @@ function startTranscode(job: HlsJob, sourcePath: string, kbps: number) {
|
||||
"0",
|
||||
"-hls_segment_type",
|
||||
"mpegts",
|
||||
"-hls_flags",
|
||||
"temp_file",
|
||||
"-hls_segment_filename",
|
||||
path.join(job.dir, "segment_%05d.ts"),
|
||||
path.join(job.dir, "index.m3u8"),
|
||||
path.join(dir, "segment_%05d.ts"),
|
||||
path.join(dir, "index.m3u8"),
|
||||
]
|
||||
const ffmpegProcess = spawn("ffmpeg", ffmpegArgs)
|
||||
|
||||
spawnFfmpeg(ffmpegArgs, job.abort.signal).then(
|
||||
async (ffmpeg) => {
|
||||
job.process = ffmpeg
|
||||
if (job.evicted) {
|
||||
ffmpeg.child.kill("SIGKILL")
|
||||
}
|
||||
const code = await ffmpeg.exited
|
||||
job.process = null
|
||||
job.processExited = true
|
||||
job.processExitCode = code
|
||||
if (job.evicted) {
|
||||
return
|
||||
}
|
||||
if (code === 0) {
|
||||
void pruneCache()
|
||||
} else {
|
||||
console.error(`ffmpeg HLS exited with code ${code}`, ffmpeg.stderrTail())
|
||||
}
|
||||
},
|
||||
() => {
|
||||
job.processExited = true
|
||||
job.processExitCode = 1
|
||||
}
|
||||
)
|
||||
}
|
||||
ffmpegProcess.stderr.on("data", (data) => {
|
||||
console.error(`FFmpeg HLS log: ${data.toString()}`)
|
||||
})
|
||||
|
||||
async function createHlsJob(
|
||||
song: Doc<"songs">,
|
||||
sourcePath: string,
|
||||
kbps: number,
|
||||
key: string,
|
||||
lookupKey: string
|
||||
): Promise<HlsJob> {
|
||||
const info = await audioInfo(sourcePath)
|
||||
const duration = info.duration ?? song.length
|
||||
const segmentCount = Math.max(1, Math.ceil(duration / HLS_SEGMENT_SECONDS))
|
||||
const dir = path.join(cacheDir("hls"), key)
|
||||
pinCacheEntry(dir)
|
||||
ffmpegProcess.on("close", (code) => {
|
||||
job.processExited = true
|
||||
job.processExitCode = code
|
||||
})
|
||||
|
||||
try {
|
||||
const complete = await hasCompletePlaylist(dir)
|
||||
if (!complete) {
|
||||
await fs.rm(dir, { recursive: true, force: true })
|
||||
await fs.mkdir(dir, { recursive: true })
|
||||
}
|
||||
ffmpegProcess.on("error", () => {
|
||||
job.processExited = true
|
||||
job.processExitCode = 1
|
||||
})
|
||||
|
||||
const job: HlsJob = {
|
||||
key,
|
||||
lookupKey,
|
||||
dir,
|
||||
duration,
|
||||
segmentCount,
|
||||
lastTouchedAt: Date.now(),
|
||||
processExited: complete,
|
||||
processExitCode: complete ? 0 : null,
|
||||
process: null,
|
||||
abort: new AbortController(),
|
||||
evicted: false,
|
||||
}
|
||||
state.jobs.set(key, job)
|
||||
await touchHlsJob(job)
|
||||
|
||||
if (!complete) {
|
||||
startTranscode(job, sourcePath, kbps)
|
||||
}
|
||||
return job
|
||||
} catch (error) {
|
||||
unpinCacheEntry(dir)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureHlsJob(
|
||||
song: Doc<"songs">,
|
||||
quality: string | null,
|
||||
preferAtmos: boolean
|
||||
): Promise<HlsJob> {
|
||||
await cleanupIdleHlsJobs()
|
||||
|
||||
const kbps = qualityKbps(quality)
|
||||
const sourceCodec = selectHlsSourceCodec(song, quality, preferAtmos)
|
||||
const sourcePath = song.files[sourceCodec]
|
||||
const key = await hlsJobKey(song, sourcePath, kbps)
|
||||
const lookupKey = hlsLookupKey(song._id, quality, preferAtmos)
|
||||
|
||||
let pending = state.pending.get(key)
|
||||
if (!pending) {
|
||||
pending = createHlsJob(song, sourcePath, kbps, key, lookupKey)
|
||||
state.pending.set(key, pending)
|
||||
const created = pending
|
||||
created.catch(() => {
|
||||
if (state.pending.get(key) === created) {
|
||||
state.pending.delete(key)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const job = await pending
|
||||
const exists = await fs.access(job.dir).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
if (job.evicted || !exists) {
|
||||
await evictJob(job)
|
||||
return ensureHlsJob(song, quality, preferAtmos)
|
||||
}
|
||||
|
||||
state.jobsByLookup.set(lookupKey, job)
|
||||
await touchHlsJob(job)
|
||||
return job
|
||||
}
|
||||
@@ -1,15 +1,8 @@
|
||||
import { spawn } from "node:child_process"
|
||||
import { createHash } from "node:crypto"
|
||||
import fs from "node:fs/promises"
|
||||
import os from "node:os"
|
||||
import path from "node:path"
|
||||
import {
|
||||
CACHE_ORPHAN_PREFIX,
|
||||
cacheDir,
|
||||
pinCacheEntry,
|
||||
pruneCache,
|
||||
touchCacheEntry,
|
||||
unpinCacheEntry,
|
||||
} from "@/lib/server-cache"
|
||||
import { spawnFfmpeg } from "@/lib/server-ffmpeg"
|
||||
|
||||
const conversions = new Map<string, Promise<string>>()
|
||||
|
||||
@@ -34,38 +27,56 @@ export async function seekableLosslessFile(source: string): Promise<string> {
|
||||
}
|
||||
|
||||
async function convert(source: string, key: string): Promise<string> {
|
||||
const dir = cacheDir("lossless")
|
||||
const dir = path.join(os.tmpdir(), "otunes-lossless")
|
||||
await fs.mkdir(dir, { recursive: true })
|
||||
const cutoff = Date.now() - 30 * 60 * 1000
|
||||
await Promise.all(
|
||||
(await fs.readdir(dir))
|
||||
.filter((name) => name.endsWith(".flac"))
|
||||
.map(async (name) => {
|
||||
const file = path.join(dir, name)
|
||||
const stat = await fs.stat(file).catch(() => null)
|
||||
if (stat && stat.mtimeMs < cutoff) await fs.rm(file, { force: true })
|
||||
})
|
||||
)
|
||||
const output = path.join(dir, `${key}.flac`)
|
||||
try {
|
||||
// Every range request touches the file so it stays cached while playing.
|
||||
await touchCacheEntry(output)
|
||||
await fs.access(output)
|
||||
return output
|
||||
} catch {
|
||||
// Publish only completed conversions so range requests always see a complete file.
|
||||
}
|
||||
const temporaryDir = await fs.mkdtemp(path.join(dir, CACHE_ORPHAN_PREFIX))
|
||||
const temporary = path.join(temporaryDir, "audio.flac")
|
||||
pinCacheEntry(temporaryDir)
|
||||
const temporary = path.join(
|
||||
await fs.mkdtemp(path.join(dir, "convert-")),
|
||||
"audio.flac"
|
||||
)
|
||||
try {
|
||||
const ffmpeg = await spawnFfmpeg([
|
||||
"-i",
|
||||
source,
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-c:a",
|
||||
"flac",
|
||||
temporary,
|
||||
])
|
||||
const code = await ffmpeg.exited
|
||||
if (code !== 0) {
|
||||
throw new Error(`Lossless conversion failed: ${code}: ${ffmpeg.stderrTail()}`)
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-nostdin",
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
source,
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-c:a",
|
||||
"flac",
|
||||
temporary,
|
||||
],
|
||||
{ stdio: "ignore" }
|
||||
)
|
||||
child.on("error", reject)
|
||||
child.on("close", (code) => {
|
||||
if (code === 0) resolve()
|
||||
else reject(new Error(`Lossless conversion failed: ${code}`))
|
||||
})
|
||||
})
|
||||
await fs.rename(temporary, output)
|
||||
return output
|
||||
} finally {
|
||||
unpinCacheEntry(temporaryDir)
|
||||
await fs.rm(temporaryDir, { recursive: true, force: true })
|
||||
await fs.rm(path.dirname(temporary), { recursive: true, force: true })
|
||||
}
|
||||
void pruneCache()
|
||||
return output
|
||||
}
|
||||
+27
-6
@@ -46,22 +46,34 @@ stop() {
|
||||
}
|
||||
trap 'stop; exit 0' TERM INT
|
||||
|
||||
as_convex=(setpriv --reuid=convex --regid=convex --init-groups env HOME=/home/convex)
|
||||
|
||||
convex_cli() {
|
||||
(
|
||||
cd "$FUNCTIONS_DIR"
|
||||
unset CONVEX_DEPLOYMENT CONVEX_DEPLOY_KEY
|
||||
CONVEX_SELF_HOSTED_URL="$BACKEND_URL" \
|
||||
CONVEX_SELF_HOSTED_ADMIN_KEY="$admin_key" \
|
||||
"$NODE" node_modules/convex/bin/main.js "$@"
|
||||
"${as_convex[@]}" "$NODE" node_modules/convex/bin/main.js "$@"
|
||||
)
|
||||
}
|
||||
|
||||
# Only root (the backend) may read the database and instance secret.
|
||||
# Only the convex user may read the database and instance secret. Volumes from
|
||||
# images that ran the backend as root are still owned by root, and
|
||||
# read_credentials.sh generates a new instance secret if it cannot read the old one.
|
||||
mkdir -p "$CONVEX_DATA_DIR"
|
||||
find "$CONVEX_DATA_DIR" \( ! -user convex -o ! -group convex \) \
|
||||
-exec chown -h convex:convex {} +
|
||||
chmod 700 "$CONVEX_DATA_DIR"
|
||||
|
||||
# The web server writes HLS streams here and a Convex cron deletes stale ones.
|
||||
mkdir -p /tmp/otunes
|
||||
chown otunes:otunes /tmp/otunes
|
||||
chmod -R g+rwX /tmp/otunes
|
||||
chmod g+s,o-rwx /tmp/otunes
|
||||
|
||||
log "Starting Convex backend"
|
||||
(cd /convex && umask 077 && exec ./run_backend.sh) &
|
||||
(cd /convex && umask 077 && exec "${as_convex[@]}" ./run_backend.sh) &
|
||||
backend_pid=$!
|
||||
|
||||
for _ in $(seq 1 120); do
|
||||
@@ -80,17 +92,24 @@ if ! curl -fsS "$BACKEND_URL/version" >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
admin_key=$(cd /convex && ./generate_admin_key.sh)
|
||||
admin_key=$(cd /convex && umask 077 && "${as_convex[@]}" ./generate_admin_key.sh)
|
||||
|
||||
log "Deploying Convex functions"
|
||||
convex_cli deploy --typecheck disable --codegen disable
|
||||
|
||||
if [ -z "$(convex_cli env get JWKS 2>/dev/null || true)" ]; then
|
||||
# `env get` exits 0 with no output when the variable is unset. Any other failure
|
||||
# must not be read as "no keys": regenerating them signs every user out.
|
||||
if ! jwks=$(convex_cli env get JWKS); then
|
||||
log "Could not read Convex Auth signing keys"
|
||||
stop
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$jwks" ]; then
|
||||
log "Generating Convex Auth signing keys"
|
||||
"$NODE" /opt/otunes/generate-auth-keys.mjs | convex_cli env set --force
|
||||
fi
|
||||
|
||||
unset admin_key
|
||||
unset admin_key jwks
|
||||
|
||||
log "Starting web server"
|
||||
(
|
||||
@@ -100,6 +119,8 @@ log "Starting web server"
|
||||
# Server-side Convex helpers fall back to this at runtime. The browser bundle
|
||||
# is built without it, so browsers still derive their own URL.
|
||||
export NEXT_PUBLIC_CONVEX_URL="$BACKEND_URL"
|
||||
# Lets the convex group member delete stale HLS streams.
|
||||
umask 0002
|
||||
exec setpriv --reuid=otunes --regid=otunes --init-groups \
|
||||
"$NODE" apps/web/server.js
|
||||
) &
|
||||
|
||||
Reference in new issue
Block a user