From 140dddc3836d0b473f8a23e49704cc3e9d9f0f54 Mon Sep 17 00:00:00 2001 From: Joshua Higgins Date: Thu, 19 Nov 2026 13:15:21 -0500 Subject: [PATCH] feat: playlist cover artwork Playlists draw a 2x2 of their first albums automatically. Owners can pick from random grids, reshuffle, or upload an image on web and mobile. Covers are rendered once into Convex storage and redrawn only when the playlist's albums change. Web and iPad sidebars show playlist covers. Co-Authored-By: Claude Opus 5.5 --- .../app/(app)/(tabs)/home/playlist/[id].tsx | 16 +- apps/mobile/app/(app)/_layout.tsx | 16 + apps/mobile/app/(app)/playlist-cover.tsx | 391 ++++++++++++++++++ apps/mobile/components/media-cards.tsx | 6 +- apps/mobile/components/media-context-menu.tsx | 8 +- apps/mobile/components/media-menu.android.tsx | 1 + apps/mobile/components/tablet-sidebar.tsx | 35 +- apps/mobile/lib/urls.ts | 5 +- apps/mobile/native/TabletSidebar.swift | 118 +++++- apps/web/app/api/playlist/[id]/cover/route.ts | 17 +- apps/web/app/api/user/[id]/avatar/route.ts | 13 +- apps/web/components/app-sidebar.tsx | 32 +- apps/web/components/options-modal.tsx | 2 +- apps/web/components/playlist-card.tsx | 4 +- apps/web/components/playlist-cover-dialog.tsx | 215 ++++++++++ apps/web/components/playlist-view.tsx | 48 ++- apps/web/lib/convex-server.ts | 11 + apps/web/lib/cover-urls.ts | 13 +- ...esize-profile-image.ts => resize-image.ts} | 38 +- apps/web/lib/serve-cover.ts | 23 ++ docker/entrypoint.sh | 1 + packages/convex/client/playlist-cover.ts | 52 +++ packages/convex/convex/_generated/api.d.ts | 6 + packages/convex/convex/images.ts | 29 ++ packages/convex/convex/library.ts | 11 +- packages/convex/convex/limits.ts | 7 +- packages/convex/convex/migrations.ts | 29 ++ packages/convex/convex/playlistCover.ts | 161 ++++++++ packages/convex/convex/playlistCoverImages.ts | 123 ++++++ packages/convex/convex/playlists.ts | 77 +++- packages/convex/convex/schema.ts | 7 + packages/convex/convex/users.ts | 28 +- packages/convex/package.json | 5 + pnpm-lock.yaml | 3 + 34 files changed, 1459 insertions(+), 92 deletions(-) create mode 100644 apps/mobile/app/(app)/playlist-cover.tsx create mode 100644 apps/web/components/playlist-cover-dialog.tsx rename apps/web/lib/{resize-profile-image.ts => resize-image.ts} (52%) create mode 100644 packages/convex/client/playlist-cover.ts create mode 100644 packages/convex/convex/images.ts create mode 100644 packages/convex/convex/playlistCover.ts create mode 100644 packages/convex/convex/playlistCoverImages.ts diff --git a/apps/mobile/app/(app)/(tabs)/home/playlist/[id].tsx b/apps/mobile/app/(app)/(tabs)/home/playlist/[id].tsx index 0450db5..c3c7830 100644 --- a/apps/mobile/app/(app)/(tabs)/home/playlist/[id].tsx +++ b/apps/mobile/app/(app)/(tabs)/home/playlist/[id].tsx @@ -68,6 +68,7 @@ const OWNER_MENU_ACTIONS: MediaAction[] = [ "play-last", "shuffle", "favorite", + "change-cover", "delete-playlist", ] @@ -83,7 +84,11 @@ export default function PlaylistScreen() { const songs = playlist?.songs const coverUri = config && playlistId - ? playlistCoverUrl(config.serverUrl, playlist?.id ?? playlistId) + ? playlistCoverUrl( + config.serverUrl, + playlist?.id ?? playlistId, + playlist?.cover ?? null + ) : "" const { palette: artworkPalette, settled } = useCoverPalette( coverUri, @@ -261,6 +266,15 @@ function PlaylistContent({ void handleDelete() return } + if (action === "change-cover") { + if (playlistId) { + router.push({ + pathname: "/playlist-cover", + params: { id: playlistId }, + }) + } + return + } void handlePlaylistDetailAction(songIds, action) } diff --git a/apps/mobile/app/(app)/_layout.tsx b/apps/mobile/app/(app)/_layout.tsx index 2424e6c..6aca6ec 100644 --- a/apps/mobile/app/(app)/_layout.tsx +++ b/apps/mobile/app/(app)/_layout.tsx @@ -106,6 +106,22 @@ export default function AppLayout() { contentStyle: { backgroundColor: "#000" }, }} /> + [] } + | { kind: "upload"; uri: string; image: string } + +const STRIP_GAP = spacing.sm +/** Selection border and padding around each thumb, both sides. */ +const THUMB_CHROME = 6 + +/** + * Full screen cover editor for a playlist owner: a large preview of the + * selected cover, the current cover pinned apart from four random album + * grids, and Save in the header. Opened from the playlist's menu. + */ +export default function PlaylistCoverScreen() { + const { id } = useLocalSearchParams<{ id: string }>() + const playlistId = id as Id<"playlists"> + const playlist = useCachedQuery(api.playlists.browsePlaylistDetail, { + id: playlistId, + }) + const { config } = useServerConfig() + const colors = useThemeColors() + const { width } = useWindowDimensions() + const setCoverAlbums = useMutation(api.playlists.setPlaylistCoverAlbums) + const uploadCover = useMutation(api.playlists.uploadPlaylistCover) + + const albums = useMemo( + () => (playlist ? playlistCoverAlbums(playlist.songs) : []), + [playlist] + ) + const [options, setOptions] = useState[][] | null>(null) + const [selected, setSelected] = useState(null) + const [saving, setSaving] = useState(false) + + // Draw options once the songs arrive. + if (options === null && playlist) { + setOptions(randomPlaylistCovers(albums)) + } + + const hasCurrent = playlist?.cover != null + const choice: Choice | null = + selected ?? + (hasCurrent + ? { kind: "current" } + : options?.[0] + ? { kind: "albums", albumIds: options[0] } + : null) + + async function pickImage() { + const result = await ImagePicker.launchImageLibraryAsync({ + mediaTypes: ["images"], + allowsEditing: true, + aspect: [1, 1], + quality: 0.7, + base64: true, + }) + const asset = result.canceled ? undefined : result.assets[0] + if (!asset?.base64) return + setSelected({ + kind: "upload", + uri: asset.uri, + image: `data:${asset.mimeType ?? "image/jpeg"};base64,${asset.base64}`, + }) + } + + async function save() { + if (!choice || choice.kind === "current") { + router.back() + return + } + setSaving(true) + try { + if (choice.kind === "albums") { + await setCoverAlbums({ playlistId, albumIds: choice.albumIds }) + } else { + await uploadCover({ playlistId, image: choice.image }) + } + router.back() + } catch (error) { + showError( + error instanceof Error ? error.message : "Failed to change cover" + ) + setSaving(false) + } + } + + const header = ( + ( + router.back()} /> + ), + headerRight: () => ( + void save()} + /> + ), + }} + /> + ) + + if (!playlist || !config || !options) { + return ( + <> + {header} + + + ) + } + + const serverUrl = config.serverUrl + const currentUri = playlistCoverUrl(serverUrl, playlistId, playlist.cover) + const previewSize = Math.min(width - spacing.lg * 2, 360) + // One row: current | uploaded? | options, thumbs at most 72pt. + const thumbCount = + options.length + (hasCurrent ? 1 : 0) + (choice?.kind === "upload" ? 1 : 0) + const rowItems = thumbCount + (hasCurrent ? 1 : 0) + const thumbSize = Math.min( + 72, + Math.floor( + (width - spacing.lg * 2 - STRIP_GAP * (rowItems - 1) - 1) / + Math.max(thumbCount, 1) + ) - THUMB_CHROME + ) + + function renderChoice(item: Choice, size: number) { + if (item.kind === "current") { + return ( + + ) + } + if (item.kind === "upload") { + return ( + + ) + } + return ( + + ) + } + + const isSelected = (item: Choice) => + choice?.kind === item.kind && + (item.kind !== "albums" || + (choice.kind === "albums" && choice.albumIds === item.albumIds)) + + return ( + + {header} + + {choice ? renderChoice(choice, previewSize) : null} + + + + {hasCurrent ? ( + <> + setSelected({ kind: "current" })} + > + {renderChoice({ kind: "current" }, thumbSize)} + + + + ) : null} + {choice?.kind === "upload" ? ( + + {renderChoice(choice, thumbSize)} + + ) : null} + {options.map((albumIds) => { + const item: Choice = { kind: "albums", albumIds } + return ( + setSelected(item)} + > + {renderChoice(item, thumbSize)} + + ) + })} + + + {albums.length === 0 ? ( + + No album artwork in this playlist yet. + + ) : null} + + + {albums.length >= COVER_GRID_ALBUMS ? ( + + ) : null} + + { + const file = event.target.files?.[0] + event.target.value = "" + if (file) void upload(file) + }} + /> + + + + ) +} + +function CoverButton({ + label, + selected = false, + onClick, + children, +}: { + label: string + selected?: boolean + onClick: () => void + children: React.ReactNode +}) { + return ( + + ) +} + +/** Preview of a generated cover: one album alone, or four tiled 2x2. */ +function AlbumGrid({ albumIds }: { albumIds: Id<"albums">[] }) { + const grid = albumIds.length > 1 + return ( +
+ {albumIds.map((albumId) => ( +
+ +
+ ))} +
+ ) +} diff --git a/apps/web/components/playlist-view.tsx b/apps/web/components/playlist-view.tsx index e1a2eb1..637ccac 100644 --- a/apps/web/components/playlist-view.tsx +++ b/apps/web/components/playlist-view.tsx @@ -6,7 +6,16 @@ import * as React from "react" import Image from "next/image" import { useRouter } from "next/navigation" import { useMutation } from "convex/react" -import { Play, SearchIcon, Share2, Shuffle, Star, Trash2 } from "lucide-react" +import { + ImageIcon, + Pencil, + Play, + SearchIcon, + Share2, + Shuffle, + Star, + Trash2, +} from "lucide-react" import { toast } from "sonner" import { useFavorites } from "@/components/favorites-provider" @@ -15,6 +24,7 @@ import { usePlaybackActions, } from "@/components/playback-provider" import { MediaCollectionFooter } from "@/components/media-collection-footer" +import { PlaylistCoverDialog } from "@/components/playlist-cover-dialog" import { MediaContextMenu, type MediaContextAction, @@ -97,6 +107,8 @@ export function PlaylistView({ playlist }: { playlist: BrowsePlaylistDetail }) { const [dragIndex, setDragIndex] = React.useState(null) const [isSubscribed, setIsSubscribed] = React.useState(playlist.isSubscribed) const [subscriptionPending, setSubscriptionPending] = React.useState(false) + const [coverDialogOpen, setCoverDialogOpen] = React.useState(false) + const canEditCover = isOwner && !isFavorites && playlist.id !== null if (syncedPlaylist !== playlist) { setSyncedPlaylist(playlist) @@ -291,21 +303,32 @@ export function PlaylistView({ playlist }: { playlist: BrowsePlaylistDetail }) {
-
+
{isFavorites ? (
) : ( {playlist.title} )} + {canEditCover ? ( + + ) : null}
@@ -385,6 +408,13 @@ export function PlaylistView({ playlist }: { playlist: BrowsePlaylistDetail }) { {isOwner && !isFavorites ? ( <> + setCoverDialogOpen(true)} + > + Change Cover + +
+ {canEditCover ? ( + + ) : null}
) } diff --git a/apps/web/lib/convex-server.ts b/apps/web/lib/convex-server.ts index 01d872b..aaee01f 100644 --- a/apps/web/lib/convex-server.ts +++ b/apps/web/lib/convex-server.ts @@ -24,3 +24,14 @@ export function getConvexClient() { } return client } + +/** + * Downloads a Convex storage file. The backend builds storage URLs from its + * public origin, which may not be reachable from here, so the same path is + * fetched from the internal URL. `null` when it is missing. + */ +export async function fetchConvexStorageFile(storageUrl: string) { + const { pathname, search } = new URL(storageUrl) + const response = await fetch(new URL(pathname + search, getConvexServerUrl())) + return response.ok ? Buffer.from(await response.arrayBuffer()) : null +} diff --git a/apps/web/lib/cover-urls.ts b/apps/web/lib/cover-urls.ts index 3e44888..8585295 100644 --- a/apps/web/lib/cover-urls.ts +++ b/apps/web/lib/cover-urls.ts @@ -12,6 +12,15 @@ export function albumCoverUrl(albumId: string, size?: CoverImageSize) { return withCoverImageSize(`/api/album/${albumId}/cover`, size) } -export function playlistCoverUrl(playlistId: string, size?: CoverImageSize) { - return withCoverImageSize(`/api/playlist/${playlistId}/cover`, size) +/** `version` is the summary's `cover`; it changes with the artwork. */ +export function playlistCoverUrl( + playlistId: string, + version: string | null, + size?: CoverImageSize +) { + const params = new URLSearchParams() + if (version) params.set("v", version) + if (size) params.set("size", String(size)) + const query = params.toString() + return `/api/playlist/${playlistId}/cover${query ? `?${query}` : ""}` } diff --git a/apps/web/lib/resize-profile-image.ts b/apps/web/lib/resize-image.ts similarity index 52% rename from apps/web/lib/resize-profile-image.ts rename to apps/web/lib/resize-image.ts index 50fde2b..f0dde4e 100644 --- a/apps/web/lib/resize-profile-image.ts +++ b/apps/web/lib/resize-image.ts @@ -1,9 +1,12 @@ -const PROFILE_IMAGE_SIZE = 256 - /** - * Downscales an image to 256×256 (center-cropped cover) and returns a base64 data URL. + * Downscales an image to `size`×`size` (center-cropped cover) and returns a + * base64 data URL. JPEG, unless `keepLossless` and the file is PNG or WebP. */ -export function resizeProfileImageToDataUrl(file: File): Promise { +export function resizeImageToDataUrl( + file: File, + size: number, + { keepLossless = false } = {} +): Promise { return new Promise((resolve, reject) => { const url = URL.createObjectURL(file) const image = new Image() @@ -12,8 +15,8 @@ export function resizeProfileImageToDataUrl(file: File): Promise { URL.revokeObjectURL(url) const canvas = document.createElement("canvas") - canvas.width = PROFILE_IMAGE_SIZE - canvas.height = PROFILE_IMAGE_SIZE + canvas.width = size + canvas.height = size const ctx = canvas.getContext("2d") if (!ctx) { @@ -21,19 +24,20 @@ export function resizeProfileImageToDataUrl(file: File): Promise { return } - const scale = Math.max( - PROFILE_IMAGE_SIZE / image.width, - PROFILE_IMAGE_SIZE / image.height - ) + const scale = Math.max(size / image.width, size / image.height) const width = image.width * scale const height = image.height * scale - const x = (PROFILE_IMAGE_SIZE - width) / 2 - const y = (PROFILE_IMAGE_SIZE - height) / 2 - - ctx.drawImage(image, x, y, width, height) + ctx.drawImage( + image, + (size - width) / 2, + (size - height) / 2, + width, + height + ) const outputType = - file.type === "image/png" || file.type === "image/webp" + keepLossless && + (file.type === "image/png" || file.type === "image/webp") ? "image/png" : "image/jpeg" resolve( @@ -52,3 +56,7 @@ export function resizeProfileImageToDataUrl(file: File): Promise { image.src = url }) } + +export function resizeProfileImageToDataUrl(file: File) { + return resizeImageToDataUrl(file, 256, { keepLossless: true }) +} diff --git a/apps/web/lib/serve-cover.ts b/apps/web/lib/serve-cover.ts index c08f7fa..7d6e5b0 100644 --- a/apps/web/lib/serve-cover.ts +++ b/apps/web/lib/serve-cover.ts @@ -34,6 +34,29 @@ function requestedCoverSize(request: Request): number | undefined { return COVER_IMAGE_SIZES.has(size) ? size : undefined } +/** + * Serves stored playlist artwork as a square WebP, `?size=` pixels a side + * (1024 by default). URLs + * carry the artwork's version (`?v=`), so new artwork gets a new URL and + * versioned responses can be cached for good. + */ +export async function serveStoredCover(image: Buffer, request: Request) { + const size = requestedCoverSize(request) ?? 1024 + const data = await sharp(image) + .rotate() + .resize(size, size, { fit: "cover", withoutEnlargement: true }) + .webp({ quality: 80 }) + .toBuffer() + return new Response(new Uint8Array(data), { + headers: { + "Content-Type": "image/webp", + "Cache-Control": new URL(request.url).searchParams.has("v") + ? "public, max-age=31536000, immutable" + : COVER_CACHE_CONTROL, + }, + }) +} + export async function serveCoverFile( coverPath: string | null | undefined, request: Request diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 93a77b1..898eaa0 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -99,6 +99,7 @@ convex_cli deploy --typecheck disable --codegen disable log "Running data migrations" convex_cli run migrations:migrateUsers +convex_cli run migrations:migratePlaylists # `env get` exits 0 with no output when the variable is unset. Any other failure # must not be read as "no keys": regenerating them signs every user out. diff --git a/packages/convex/client/playlist-cover.ts b/packages/convex/client/playlist-cover.ts new file mode 100644 index 0000000..c431d0f --- /dev/null +++ b/packages/convex/client/playlist-cover.ts @@ -0,0 +1,52 @@ +import type { Id } from "../convex/_generated/dataModel" +import { COVER_GRID_ALBUMS } from "../convex/limits" +import type { BrowseSong } from "../convex/types" + +export { COVER_GRID_ALBUMS } + +export type CoverAlbum = BrowseSong["album"] + +/** Distinct albums that have artwork, in playlist order. */ +export function playlistCoverAlbums(songs: readonly BrowseSong[]) { + const albums = new Map() + for (const song of songs) { + if (song.album.cover !== null && !albums.has(song.album.id)) { + albums.set(song.album.id, song.album) + } + } + return [...albums.values()] +} + +/** `count` distinct items in random order (Fisher-Yates prefix). */ +function sample(items: readonly T[], count: number) { + const pool = [...items] + for (let index = 0; index < count; index++) { + const swap = index + Math.floor(Math.random() * (pool.length - index)) + ;[pool[index], pool[swap]] = [pool[swap], pool[index]] + } + return pool.slice(0, count) +} + +/** + * Cover options to pick from, as album ids for `setPlaylistCoverAlbums`. + * With four or more albums: `count` random 2x2 grids, avoiding repeats. + * With fewer: each album's cover alone. + */ +export function randomPlaylistCovers( + albums: readonly CoverAlbum[], + count = 4 +): Id<"albums">[][] { + const ids = albums.map((album) => album.id) + if (ids.length < COVER_GRID_ALBUMS) return ids.map((id) => [id]) + + const options = new Map[]>() + for ( + let attempt = 0; + options.size < count && attempt < count * 10; + attempt++ + ) { + const option = sample(ids, COVER_GRID_ALBUMS) + options.set(option.join(","), option) + } + return [...options.values()] +} diff --git a/packages/convex/convex/_generated/api.d.ts b/packages/convex/convex/_generated/api.d.ts index 5298bfa..3c98aaf 100644 --- a/packages/convex/convex/_generated/api.d.ts +++ b/packages/convex/convex/_generated/api.d.ts @@ -14,6 +14,7 @@ import type * as crons from "../crons.js"; import type * as favoriteStore from "../favoriteStore.js"; import type * as favorites from "../favorites.js"; import type * as http from "../http.js"; +import type * as images from "../images.js"; import type * as intentCatalog from "../intentCatalog.js"; import type * as intentCatalogChanges from "../intentCatalogChanges.js"; import type * as library from "../library.js"; @@ -21,6 +22,8 @@ import type * as limits from "../limits.js"; import type * as migrations from "../migrations.js"; import type * as playback from "../playback.js"; import type * as playbackState from "../playbackState.js"; +import type * as playlistCover from "../playlistCover.js"; +import type * as playlistCoverImages from "../playlistCoverImages.js"; import type * as playlists from "../playlists.js"; import type * as types from "../types.js"; import type * as users from "../users.js"; @@ -38,6 +41,7 @@ declare const fullApi: ApiFromModules<{ favoriteStore: typeof favoriteStore; favorites: typeof favorites; http: typeof http; + images: typeof images; intentCatalog: typeof intentCatalog; intentCatalogChanges: typeof intentCatalogChanges; library: typeof library; @@ -45,6 +49,8 @@ declare const fullApi: ApiFromModules<{ migrations: typeof migrations; playback: typeof playback; playbackState: typeof playbackState; + playlistCover: typeof playlistCover; + playlistCoverImages: typeof playlistCoverImages; playlists: typeof playlists; types: typeof types; users: typeof users; diff --git a/packages/convex/convex/images.ts b/packages/convex/convex/images.ts new file mode 100644 index 0000000..43d8c58 --- /dev/null +++ b/packages/convex/convex/images.ts @@ -0,0 +1,29 @@ +import { ConvexError } from "convex/values" +import { MAX_IMAGE_UPLOAD_BYTES } from "./limits" + +const DATA_URL_PATTERN = /^data:(image\/[\w.+-]+);base64,([A-Za-z0-9+/=]+)$/ + +/** + * Splits an image data URL, rejecting anything over the size cap. `label` + * names the image in error messages ("Profile photo"). + */ +export function parseImageDataUrl(image: string, label: string) { + const match = DATA_URL_PATTERN.exec(image) + if (!match) { + throw new ConvexError(`${label} must be an image`) + } + const [, contentType, base64] = match + if (Math.floor((base64.length * 3) / 4) > MAX_IMAGE_UPLOAD_BYTES) { + throw new ConvexError( + `${label} must be ${MAX_IMAGE_UPLOAD_BYTES / 1024 / 1024} MB or smaller` + ) + } + return { contentType, base64 } +} + +/** Decodes an image data URL for `ctx.storage.store`. */ +export function imageDataUrlToBlob(image: string, label: string) { + const { contentType, base64 } = parseImageDataUrl(image, label) + const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)) + return new Blob([bytes], { type: contentType }) +} diff --git a/packages/convex/convex/library.ts b/packages/convex/convex/library.ts index 6477067..4a2116b 100644 --- a/packages/convex/convex/library.ts +++ b/packages/convex/convex/library.ts @@ -24,6 +24,7 @@ import type { import { purgeFavoriteEntities } from "./favoriteStore" import { chunk, type Chunked } from "./limits" import { recordIntentCatalogChanges } from "./intentCatalogChanges" +import { playlistCoverVersion } from "./playlistCover" export type { BrowseAlbum, @@ -875,7 +876,7 @@ async function browseRecentlyPlayedItems(ctx: QueryCtx, limit?: number) { id: entry.playlist._id, title: entry.playlist.name, owner: ownerMap.get(entry.playlist.owner)?.name ?? "Unknown", - cover: entry.playlist.cover, + cover: playlistCoverVersion(entry.playlist), }) } @@ -959,11 +960,15 @@ export const getAlbumCover = query({ }, }) -/** Public: share-link previews load covers without a session. */ +/** + * Public: share-link previews load covers without a session. A storage URL + * the web server proxies, or `null` when the playlist has no artwork. + */ export const getPlaylistCover = query({ args: { id: v.id("playlists") }, handler: async (ctx, args) => { - return (await ctx.db.get("playlists", args.id))?.cover ?? null + const storageId = (await ctx.db.get("playlists", args.id))?.coverStorageId + return storageId ? await ctx.storage.getUrl(storageId) : null }, }) diff --git a/packages/convex/convex/limits.ts b/packages/convex/convex/limits.ts index d4ae255..1d005fe 100644 --- a/packages/convex/convex/limits.ts +++ b/packages/convex/convex/limits.ts @@ -6,8 +6,11 @@ /** Songs kept in the playback queue, including the current song. */ export const MAX_QUEUE_LENGTH = 5_000 -/** Largest profile photo accepted, in bytes after base64 decoding. */ -export const MAX_PROFILE_IMAGE_BYTES = 2 * 1024 * 1024 +/** Largest uploaded image (profile photo, playlist cover), in bytes after base64 decoding. */ +export const MAX_IMAGE_UPLOAD_BYTES = 2 * 1024 * 1024 + +/** Album covers a generated playlist cover tiles 2x2. */ +export const COVER_GRID_ALBUMS = 4 /** Most values Convex accepts in a single array. */ export const MAX_ARRAY_LENGTH = 8_192 diff --git a/packages/convex/convex/migrations.ts b/packages/convex/convex/migrations.ts index 132457e..593b85e 100644 --- a/packages/convex/convex/migrations.ts +++ b/packages/convex/convex/migrations.ts @@ -56,3 +56,32 @@ export const migrateFavorites = internalMutation({ } }, }) + +/** + * Draws automatic artwork for playlists that have none yet. Idempotent; the + * Docker entrypoint runs it on every boot. + */ +export const migratePlaylists = internalMutation({ + args: { cursor: v.optional(v.union(v.string(), v.null())) }, + handler: async (ctx, { cursor }) => { + const page = await ctx.db + .query("playlists") + .paginate({ cursor: cursor ?? null, numItems: BATCH_SIZE }) + + for (const playlist of page.page) { + if (!playlist.coverSource && !playlist.coverAlbums) { + await ctx.scheduler.runAfter( + 0, + internal.playlistCover.refreshPlaylistCover, + { playlistId: playlist._id } + ) + } + } + + if (!page.isDone) { + await ctx.scheduler.runAfter(0, internal.migrations.migratePlaylists, { + cursor: page.continueCursor, + }) + } + }, +}) diff --git a/packages/convex/convex/playlistCover.ts b/packages/convex/convex/playlistCover.ts new file mode 100644 index 0000000..4bb16a0 --- /dev/null +++ b/packages/convex/convex/playlistCover.ts @@ -0,0 +1,161 @@ +import { v } from "convex/values" +import { internal } from "./_generated/api" +import type { Doc, Id } from "./_generated/dataModel" +import type { MutationCtx } from "./_generated/server" +import { internalMutation, internalQuery } from "./_generated/server" +import { COVER_GRID_ALBUMS } from "./limits" + +/* + * Playlist artwork is a stored image (`coverStorageId`), drawn once and + * redrawn only when its inputs change: + * - automatic (no `coverSource`): the first albums in the playlist, kept in + * sync by `refreshAutomaticCover` whenever the songs change + * - "albums": albums the owner picked + * - "upload": an image the owner uploaded + * `coverAlbums` holds the albums behind the current or pending drawing: four + * tile 2x2, one shows alone. Drawing happens in `playlistCoverImages`. + */ + +/** + * What clients put in the cover URL (`?v=`) so new artwork gets a new URL. + * `null` when the playlist has no artwork. + */ +export function playlistCoverVersion(playlist: Doc<"playlists">) { + return playlist.coverStorageId ?? null +} + +/** Songs read when picking automatic artwork, so long single-album runs stay cheap. */ +const AUTO_COVER_SONG_SCAN = 50 + +/** First distinct album covers in playlist order: four for a grid, else one. */ +async function automaticCoverAlbums(ctx: MutationCtx, songIds: Id<"songs">[]) { + const seen = new Set>() + const albums: Id<"albums">[] = [] + for (const songId of songIds.slice(0, AUTO_COVER_SONG_SCAN)) { + const song = await ctx.db.get("songs", songId) + if (!song || seen.has(song.album)) continue + seen.add(song.album) + const album = await ctx.db.get("albums", song.album) + if (album?.cover) albums.push(album._id) + if (albums.length === COVER_GRID_ALBUMS) break + } + return albums.length === COVER_GRID_ALBUMS ? albums : albums.slice(0, 1) +} + +function sameAlbums(a: Id<"albums">[] | undefined, b: Id<"albums">[]) { + return a?.length === b.length && a.every((id, index) => id === b[index]) +} + +/** + * Points `coverAlbums` at new albums and queues a drawing of them. The old + * image stays up until the new one lands. + */ +export async function drawCoverAlbums( + ctx: MutationCtx, + playlist: Doc<"playlists">, + albumIds: Id<"albums">[], + coverSource: "albums" | undefined +) { + await ctx.db.patch(playlist._id, { coverAlbums: albumIds, coverSource }) + await ctx.scheduler.runAfter( + 0, + internal.playlistCoverImages.renderPlaylistCover, + { playlistId: playlist._id, albumIds } + ) +} + +/** + * Keeps automatic artwork in step with the playlist's first albums. Call + * after the songs change, with the new order. Draws only when the albums + * differ from the last drawing. + */ +export async function refreshAutomaticCover( + ctx: MutationCtx, + playlist: Doc<"playlists">, + songIds: Id<"songs">[] +) { + if (playlist.coverSource) return + const albums = await automaticCoverAlbums(ctx, songIds) + if (sameAlbums(playlist.coverAlbums, albums)) return + if (albums.length > 0) { + await drawCoverAlbums(ctx, playlist, albums, undefined) + return + } + if (playlist.coverStorageId) await ctx.storage.delete(playlist.coverStorageId) + await ctx.db.patch(playlist._id, { + coverAlbums: undefined, + coverStorageId: undefined, + }) +} + +/** Album cover file paths for a drawing, or `null` if any lost its artwork. */ +export const coverFilePaths = internalQuery({ + args: { albumIds: v.array(v.id("albums")) }, + handler: async (ctx, { albumIds }) => { + const albums = await Promise.all( + albumIds.map((id) => ctx.db.get("albums", id)) + ) + const paths = albums.flatMap((album) => (album?.cover ? [album.cover] : [])) + return paths.length === albumIds.length ? paths : null + }, +}) + +/** Deletes the image `storageId` replaces. */ +async function deleteReplacedImage( + ctx: MutationCtx, + playlist: Doc<"playlists">, + storageId: Id<"_storage"> +) { + if (playlist.coverStorageId && playlist.coverStorageId !== storageId) { + await ctx.storage.delete(playlist.coverStorageId) + } +} + +/** Saves a finished drawing unless the playlist moved on while it rendered. */ +export const attachRenderedCover = internalMutation({ + args: { + playlistId: v.id("playlists"), + albumIds: v.array(v.id("albums")), + storageId: v.id("_storage"), + }, + handler: async (ctx, { playlistId, albumIds, storageId }) => { + const playlist = await ctx.db.get("playlists", playlistId) + if ( + !playlist || + playlist.coverSource === "upload" || + !sameAlbums(playlist.coverAlbums, albumIds) + ) { + await ctx.storage.delete(storageId) + return + } + await deleteReplacedImage(ctx, playlist, storageId) + await ctx.db.patch(playlist._id, { coverStorageId: storageId }) + }, +}) + +/** Saves an uploaded image as the playlist's artwork. */ +export const attachUploadedCover = internalMutation({ + args: { playlistId: v.id("playlists"), storageId: v.id("_storage") }, + handler: async (ctx, { playlistId, storageId }) => { + const playlist = await ctx.db.get("playlists", playlistId) + if (!playlist) { + await ctx.storage.delete(storageId) + return + } + await deleteReplacedImage(ctx, playlist, storageId) + await ctx.db.patch(playlist._id, { + coverSource: "upload", + coverAlbums: undefined, + coverStorageId: storageId, + }) + }, +}) + +/** Draws automatic artwork for one playlist. Scheduled by `migratePlaylists`. */ +export const refreshPlaylistCover = internalMutation({ + args: { playlistId: v.id("playlists") }, + handler: async (ctx, { playlistId }) => { + const playlist = await ctx.db.get("playlists", playlistId) + if (playlist) await refreshAutomaticCover(ctx, playlist, playlist.songs) + }, +}) diff --git a/packages/convex/convex/playlistCoverImages.ts b/packages/convex/convex/playlistCoverImages.ts new file mode 100644 index 0000000..97c806f --- /dev/null +++ b/packages/convex/convex/playlistCoverImages.ts @@ -0,0 +1,123 @@ +"use node" + +import * as fs from "node:fs/promises" +import { v } from "convex/values" +import * as jpeg from "jpeg-js" +import { internal } from "./_generated/api" +import { internalAction } from "./_generated/server" +import { imageDataUrlToBlob } from "./images" + +/* + * Draws playlist artwork into storage. Pure JS on purpose: self-hosted Convex + * cannot install native packages like sharp for node actions. Album covers + * are always `cover.jpg` / `folder.jpg`, so JPEG in and out is enough. + */ + +/** Stored playlist artwork is square, this many pixels a side. */ +const COVER_SIZE = 1024 +const JPEG_QUALITY = 88 + +type Rgba = { width: number; height: number; data: Uint8Array } + +/** + * Center-crops `source` to a square and area-averages it into `target` at + * (`left`, `top`), `size` pixels a side. Averaging keeps downscaled covers + * smooth; tiny covers upscale blocky but intact. + */ +function drawSquare( + source: Rgba, + target: Rgba, + left: number, + top: number, + size: number +) { + const side = Math.min(source.width, source.height) + const offsetX = Math.floor((source.width - side) / 2) + const offsetY = Math.floor((source.height - side) / 2) + const scale = side / size + + for (let y = 0; y < size; y++) { + const y0 = offsetY + Math.floor(y * scale) + const y1 = Math.max(y0 + 1, offsetY + Math.floor((y + 1) * scale)) + for (let x = 0; x < size; x++) { + const x0 = offsetX + Math.floor(x * scale) + const x1 = Math.max(x0 + 1, offsetX + Math.floor((x + 1) * scale)) + let r = 0 + let g = 0 + let b = 0 + for (let sy = y0; sy < y1; sy++) { + let index = (sy * source.width + x0) * 4 + for (let sx = x0; sx < x1; sx++, index += 4) { + r += source.data[index] + g += source.data[index + 1] + b += source.data[index + 2] + } + } + const count = (x1 - x0) * (y1 - y0) + const out = ((top + y) * target.width + left + x) * 4 + target.data[out] = r / count + target.data[out + 1] = g / count + target.data[out + 2] = b / count + target.data[out + 3] = 255 + } + } +} + +/** One cover alone, or four tiled 2x2 left to right, top to bottom. */ +async function drawCover(paths: string[]) { + const canvas: Rgba = { + width: COVER_SIZE, + height: COVER_SIZE, + data: new Uint8Array(COVER_SIZE * COVER_SIZE * 4), + } + const tile = paths.length === 1 ? COVER_SIZE : COVER_SIZE / 2 + // One at a time: a decoded 3000px cover is ~36 MB. + for (const [index, path] of paths.entries()) { + const source = jpeg.decode(await fs.readFile(path), { useTArray: true }) + drawSquare( + source, + canvas, + (index % 2) * tile, + Math.floor(index / 2) * tile, + tile + ) + } + return jpeg.encode(canvas, JPEG_QUALITY).data +} + +/** Draws album artwork into storage. Scheduled by `drawCoverAlbums`. */ +export const renderPlaylistCover = internalAction({ + args: { playlistId: v.id("playlists"), albumIds: v.array(v.id("albums")) }, + handler: async (ctx, { playlistId, albumIds }) => { + const paths = await ctx.runQuery(internal.playlistCover.coverFilePaths, { + albumIds, + }) + if (!paths) return + const image = await drawCover(paths) + const storageId = await ctx.storage.store( + new Blob([new Uint8Array(image)], { type: "image/jpeg" }) + ) + await ctx.runMutation(internal.playlistCover.attachRenderedCover, { + playlistId, + albumIds, + storageId, + }) + }, +}) + +/** + * Stores an uploaded data URL as is; clients send it square. Scheduled by + * `uploadPlaylistCover`. + */ +export const storeUploadedCover = internalAction({ + args: { playlistId: v.id("playlists"), image: v.string() }, + handler: async (ctx, { playlistId, image }) => { + const storageId = await ctx.storage.store( + imageDataUrlToBlob(image, "Cover") + ) + await ctx.runMutation(internal.playlistCover.attachUploadedCover, { + playlistId, + storageId, + }) + }, +}) diff --git a/packages/convex/convex/playlists.ts b/packages/convex/convex/playlists.ts index ab18f2c..2b2fb3b 100644 --- a/packages/convex/convex/playlists.ts +++ b/packages/convex/convex/playlists.ts @@ -3,7 +3,14 @@ import { ConvexError, v } from "convex/values" import type { Doc, Id } from "./_generated/dataModel" import type { MutationCtx, QueryCtx } from "./_generated/server" import { mutation, query } from "./_generated/server" -import { chunk } from "./limits" +import { internal } from "./_generated/api" +import { parseImageDataUrl } from "./images" +import { chunk, COVER_GRID_ALBUMS } from "./limits" +import { + drawCoverAlbums, + playlistCoverVersion, + refreshAutomaticCover, +} from "./playlistCover" import type { BrowsePlaylistDetail, BrowsePlaylistSummary, @@ -25,7 +32,7 @@ function toPlaylistSummary( id: playlist._id, title: playlist.name, owner: ownerName, - cover: playlist.cover, + cover: playlistCoverVersion(playlist), } } @@ -106,7 +113,7 @@ async function browsePlaylistSummaryHandler( id: owner._id, name: owner.name, }, - cover: playlist.cover, + cover: playlistCoverVersion(playlist), isOwner, isSubscribed, songIds: playlist.songs, @@ -366,6 +373,8 @@ export const createPlaylist = mutation({ cover: null, songs: uniqueSongIds, }) + const playlist = await ctx.db.get("playlists", playlistId) + await refreshAutomaticCover(ctx, playlist!, uniqueSongIds) if (args.invocationId) await ctx.db.insert("appIntentPlaylists", { userId, @@ -388,9 +397,9 @@ export const addSongsToPlaylist = mutation({ const toAdd = uniqueIncoming.filter((songId) => !existing.has(songId)) if (toAdd.length > 0) { - await ctx.db.patch(playlist._id, { - songs: [...playlist.songs, ...toAdd], - }) + const songs = [...playlist.songs, ...toAdd] + await ctx.db.patch(playlist._id, { songs }) + await refreshAutomaticCover(ctx, playlist, songs) } return { @@ -408,9 +417,9 @@ export const removeSongsFromPlaylist = mutation({ handler: async (ctx, args) => { const { playlist } = await requirePlaylistOwner(ctx, args.playlistId) const remove = new Set(args.songIds) - await ctx.db.patch(playlist._id, { - songs: playlist.songs.filter((songId) => !remove.has(songId)), - }) + const songs = playlist.songs.filter((songId) => !remove.has(songId)) + await ctx.db.patch(playlist._id, { songs }) + await refreshAutomaticCover(ctx, playlist, songs) }, }) @@ -430,13 +439,14 @@ export const reorderPlaylistSongs = mutation({ throw new ConvexError("Invalid song order") } await ctx.db.patch(playlist._id, { songs: args.songIds }) + await refreshAutomaticCover(ctx, playlist, args.songIds) }, }) export const deletePlaylist = mutation({ args: { playlistId: v.id("playlists") }, handler: async (ctx, args) => { - await requirePlaylistOwner(ctx, args.playlistId) + const { playlist } = await requirePlaylistOwner(ctx, args.playlistId) const users = await ctx.db.query("users").collect() for (const user of users) { @@ -466,10 +476,57 @@ export const deletePlaylist = mutation({ }) } + if (playlist.coverStorageId) { + await ctx.storage.delete(playlist.coverStorageId) + } await ctx.db.delete("playlists", args.playlistId) }, }) +/** + * Draws these album covers as the playlist's artwork: one alone, or four + * tiled 2x2. Replaces automatic or uploaded artwork. + */ +export const setPlaylistCoverAlbums = mutation({ + args: { + playlistId: v.id("playlists"), + albumIds: v.array(v.id("albums")), + }, + handler: async (ctx, { playlistId, albumIds }) => { + const { playlist } = await requirePlaylistOwner(ctx, playlistId) + if ( + (albumIds.length !== 1 && albumIds.length !== COVER_GRID_ALBUMS) || + new Set(albumIds).size !== albumIds.length + ) { + throw new ConvexError("Pick one album or four different albums") + } + for (const albumId of albumIds) { + const album = await ctx.db.get("albums", albumId) + if (!album?.cover) { + throw new ConvexError("Album has no artwork") + } + } + await drawCoverAlbums(ctx, playlist, albumIds, "albums") + }, +}) + +/** + * Takes an image data URL. It is cropped into storage in the background and + * the cover version changes once it lands. + */ +export const uploadPlaylistCover = mutation({ + args: { playlistId: v.id("playlists"), image: v.string() }, + handler: async (ctx, { playlistId, image }) => { + await requirePlaylistOwner(ctx, playlistId) + parseImageDataUrl(image, "Cover") + await ctx.scheduler.runAfter( + 0, + internal.playlistCoverImages.storeUploadedCover, + { playlistId, image } + ) + }, +}) + export const subscribeToPlaylist = mutation({ args: { playlistId: v.id("playlists") }, handler: async (ctx, args) => { diff --git a/packages/convex/convex/schema.ts b/packages/convex/convex/schema.ts index 1690dcb..e7596fc 100644 --- a/packages/convex/convex/schema.ts +++ b/packages/convex/convex/schema.ts @@ -148,7 +148,14 @@ export default defineSchema({ playlists: defineTable({ name: v.string(), owner: v.id("users"), + /** Legacy and unused; artwork lives in `coverStorageId`. */ cover: v.nullable(v.string()), + // Artwork, see `playlistCover.ts`. No `coverSource` means automatic. + coverSource: v.optional(v.union(v.literal("albums"), v.literal("upload"))), + /** Albums behind the drawn artwork: four tile 2x2, one shows alone. */ + coverAlbums: v.optional(v.array(v.id("albums"))), + /** The drawn or uploaded artwork. */ + coverStorageId: v.optional(v.id("_storage")), songs: v.array(v.id("songs")), }).index("by_owner", ["owner"]), }) diff --git a/packages/convex/convex/users.ts b/packages/convex/convex/users.ts index 21c68e3..d32feca 100644 --- a/packages/convex/convex/users.ts +++ b/packages/convex/convex/users.ts @@ -15,7 +15,7 @@ import { } from "./_generated/server" import type { Doc, Id } from "./_generated/dataModel" import type { ActionCtx, MutationCtx } from "./_generated/server" -import { MAX_PROFILE_IMAGE_BYTES } from "./limits" +import { imageDataUrlToBlob, parseImageDataUrl } from "./images" import { deleteUserFavorites } from "./favoriteStore" import { deleteUserPlayback } from "./playbackState" @@ -138,6 +138,9 @@ async function deleteOwnedPlaylists(ctx: MutationCtx, userId: Id<"users">) { } for (const playlist of owned) { + if (playlist.coverStorageId) { + await ctx.storage.delete(playlist.coverStorageId) + } await ctx.db.delete(playlist._id) } } @@ -188,23 +191,6 @@ export const listUsers = query({ }, }) -const DATA_URL_PATTERN = /^data:(image\/[\w.+-]+);base64,([A-Za-z0-9+/=]+)$/ - -/** Splits an image data URL, rejecting anything over the size cap. */ -function parseImageDataUrl(image: string) { - const match = DATA_URL_PATTERN.exec(image) - if (!match) { - throw new ConvexError("Profile photo must be an image") - } - const [, contentType, base64] = match - if (Math.floor((base64.length * 3) / 4) > MAX_PROFILE_IMAGE_BYTES) { - throw new ConvexError( - `Profile photo must be ${MAX_PROFILE_IMAGE_BYTES / 1024 / 1024} MB or smaller` - ) - } - return { contentType, base64 } -} - /** * Path clients load the avatar from. The web server proxies it from Convex * storage, since the backend's storage URLs are not reachable from browsers @@ -231,7 +217,7 @@ export const updateProfileImage = mutation({ await setProfileImage(ctx, userId, null) return } - parseImageDataUrl(image) + parseImageDataUrl(image, "Profile photo") await ctx.scheduler.runAfter(0, internal.users.storeProfileImage, { userId, image, @@ -243,10 +229,8 @@ export const updateProfileImage = mutation({ export const storeProfileImage = internalAction({ args: { userId: v.id("users"), image: v.string() }, handler: async (ctx, { userId, image }) => { - const { contentType, base64 } = parseImageDataUrl(image) - const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)) const storageId = await ctx.storage.store( - new Blob([bytes], { type: contentType }) + imageDataUrlToBlob(image, "Profile photo") ) await ctx.runMutation(internal.users.attachProfileImage, { userId, diff --git a/packages/convex/package.json b/packages/convex/package.json index 3a53357..0c5a4cf 100644 --- a/packages/convex/package.json +++ b/packages/convex/package.json @@ -51,6 +51,10 @@ "types": "./client/playback-queue.ts", "default": "./client/playback-queue.ts" }, + "./client/playlist-cover": { + "types": "./client/playlist-cover.ts", + "default": "./client/playlist-cover.ts" + }, "./client/playback-store": { "types": "./client/playback-store.ts", "default": "./client/playback-store.ts" @@ -74,6 +78,7 @@ "dependencies": { "@convex-dev/auth": "^0.0.93", "convex": "^1.40.0", + "jpeg-js": "0.4.4", "music-metadata": "11.12.3" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fbbf705..8100a90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -515,6 +515,9 @@ importers: convex: specifier: ^1.40.0 version: 1.42.1(react@19.3.1) + jpeg-js: + specifier: 0.4.4 + version: 0.4.4 music-metadata: specifier: ^11.16.0 version: 11.16.1(supports-color@8.1.1)