From 62db772dada4fdb55c8bb9db093c82c9e6bd295e Mon Sep 17 00:00:00 2001 From: andwati Date: Sun, 30 Aug 2026 18:52:39 +0300 Subject: [PATCH] Add bookshelf ISBN/paper enrichment, fix status field name collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apps/cms/src/utils/bookshelf-covers.ts fetches book covers from Open Library by ISBN (real image, cached into apps/site/public/covers/) and backfills missing title/authors for papers from Semantic Scholar by DOI/arXiv id. Wired into the sync middleware in content-sync.ts: runs once per entry (cover_image already set means "done"), and only on live create/update through Strapi, not during the boot-time import (so a restart doesn't hammer external APIs). Also renames the bookshelf `status` field to `read_status` everywhere (schema, content/README.md, seed content, Astro zod schema, bookshelf page, sync mapping). Strapi's content-manager API treats a top-level `status` key as its own reserved draft/publish parameter even with draftAndPublish disabled, so an attribute literally named `status` silently rejected every write — caught this while testing the enrichment hook. Verified end-to-end: created a book by ISBN with no cover_image, confirmed a real cover image was downloaded and cached, cover_image was set on the entry and written to the file, and the cover/file were both removed on delete. --- TODO.md | 24 ++++-- .../content-types/bookshelf-entry/schema.json | 2 +- apps/cms/src/utils/bookshelf-covers.ts | 65 ++++++++++++++++ apps/cms/src/utils/content-sync.ts | 78 ++++++++++++++++--- apps/cms/types/generated/contentTypes.d.ts | 2 +- apps/site/src/content.config.ts | 2 +- apps/site/src/pages/bookshelf/index.astro | 2 +- content/README.md | 16 +++- .../bookshelf/attention-is-all-you-need.md | 2 +- ...and-interpretation-of-computer-programs.md | 2 +- 10 files changed, 167 insertions(+), 28 deletions(-) create mode 100644 apps/cms/src/utils/bookshelf-covers.ts diff --git a/TODO.md b/TODO.md index 42c09ee..6efc4fb 100644 --- a/TODO.md +++ b/TODO.md @@ -27,14 +27,22 @@ approved plan for full rationale on each decision referenced below. ## Content pipeline -- [ ] Strapi custom provider/lifecycle hooks that read + write the same - Markdown files in `content/` (Strapi's DB stays a disposable cache) -- [ ] Verify round-trip: edit in Strapi admin → file on disk updates, and - vice versa, without drift -- [ ] Bookshelf ISBN lookup hook (Open Library / Google Books) with local - cover caching -- [ ] Bookshelf paper lookup hook (Semantic Scholar / arXiv) with local - cover/thumbnail caching +- [x] Strapi custom provider/lifecycle hooks that read + write the same + Markdown files in `content/` (Strapi's DB stays a disposable cache) — + `apps/cms/src/utils/content-{file,sync}.ts`, wired in `src/index.ts` +- [x] Verify round-trip: edit in Strapi admin → file on disk updates, and + vice versa, without drift — verified create/update/delete via the + content-manager API, plus idempotent re-import across a restart +- [x] Bookshelf ISBN lookup hook with local cover caching — + `apps/cms/src/utils/bookshelf-covers.ts` fetches from Open Library by + ISBN and caches into `apps/site/public/covers/`; verified end-to-end + (real cover downloaded, `cover_image` synced to file, cleaned up on + delete) +- [x] Bookshelf paper lookup hook — Semantic Scholar backfills title/authors + by DOI/arXiv id when missing. No cover/thumbnail: neither Semantic + Scholar nor arXiv offers a reliable image source for papers, so + `cover_image` stays manually-set for papers (or gets one later from + the OG-image pipeline below) - [ ] OG image auto-generation extended from the legacy per-post script to all three content types (writings, portfolio, bookshelf) - [ ] `llms.txt` generation extended to cover all content types, not just diff --git a/apps/cms/src/api/bookshelf-entry/content-types/bookshelf-entry/schema.json b/apps/cms/src/api/bookshelf-entry/content-types/bookshelf-entry/schema.json index 8f394b2..07098e1 100644 --- a/apps/cms/src/api/bookshelf-entry/content-types/bookshelf-entry/schema.json +++ b/apps/cms/src/api/bookshelf-entry/content-types/bookshelf-entry/schema.json @@ -55,7 +55,7 @@ "min": 1, "max": 5 }, - "status": { + "read_status": { "type": "enumeration", "enum": ["reading", "read", "abandoned"] }, diff --git a/apps/cms/src/utils/bookshelf-covers.ts b/apps/cms/src/utils/bookshelf-covers.ts new file mode 100644 index 0000000..fd967aa --- /dev/null +++ b/apps/cms/src/utils/bookshelf-covers.ts @@ -0,0 +1,65 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +/** + * Covers are cached into the Astro app's public/ dir directly, so the built + * site serves them from /covers/.jpg with no runtime dependency on the + * source (Open Library, Semantic Scholar, ...). + */ +const COVERS_DIR = join(process.cwd(), "..", "site", "public", "covers"); + +async function download(url: string): Promise { + const res = await fetch(url); + if (!res.ok) return undefined; + const buf = Buffer.from(await res.arrayBuffer()); + // Open Library serves a tiny placeholder image for ISBNs it doesn't have a cover for. + if (buf.byteLength < 500) return undefined; + return buf; +} + +/** Fetches a book cover by ISBN from Open Library and caches it locally. */ +export async function fetchBookCover( + isbn: string, + slug: string, +): Promise { + const buf = await download( + `https://covers.openlibrary.org/b/isbn/${encodeURIComponent(isbn)}-L.jpg?default=false`, + ); + if (!buf) return undefined; + const file = join(COVERS_DIR, `${slug}.jpg`); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, buf); + return `/covers/${slug}.jpg`; +} + +interface PaperMetadata { + title?: string; + authors?: string[]; +} + +/** + * Fetches title/author metadata for a paper from Semantic Scholar by DOI or + * arXiv id, used to backfill fields left blank in Strapi. Semantic Scholar + * has no reliable thumbnail/cover source for papers, so unlike books this + * does not produce a cached image. + */ +export async function fetchPaperMetadata(identifier: { + doi?: string; + arxivId?: string; +}): Promise { + const id = identifier.doi + ? `DOI:${identifier.doi}` + : identifier.arxivId + ? `ARXIV:${identifier.arxivId}` + : undefined; + if (!id) return undefined; + const res = await fetch( + `https://api.semanticscholar.org/graph/v1/paper/${encodeURIComponent(id)}?fields=title,authors`, + ); + if (!res.ok) return undefined; + const data = (await res.json()) as { + title?: string; + authors?: { name: string }[]; + }; + return { title: data.title, authors: data.authors?.map((a) => a.name) }; +} diff --git a/apps/cms/src/utils/content-sync.ts b/apps/cms/src/utils/content-sync.ts index a7c1147..0a374e2 100644 --- a/apps/cms/src/utils/content-sync.ts +++ b/apps/cms/src/utils/content-sync.ts @@ -1,4 +1,5 @@ import { join } from "node:path"; +import { fetchBookCover, fetchPaperMetadata } from "./bookshelf-covers"; import { deleteEntry, readAllEntries, @@ -6,6 +7,8 @@ import { writeEntry, } from "./content-file"; +const BOOKSHELF_UID = "api::bookshelf-entry.bookshelf-entry"; + type Layout = "nested" | "flat"; interface SyncConfig { @@ -136,7 +139,7 @@ function buildConfigs(): Record { url: e.url, cover_image: e.cover_image, rating: e.rating, - status: e.status, + read_status: e.read_status, date_started: toTomlDate(e.date_started as string), date_finished: toTomlDate(e.date_finished as string), taxonomies: { tags: e.tags ?? [] }, @@ -156,7 +159,7 @@ function buildConfigs(): Record { url: fm.url, cover_image: fm.cover_image, rating: fm.rating, - status: fm.status, + read_status: fm.read_status, date_started: isoDate(fm.date_started), date_finished: isoDate(fm.date_finished), tags: taxonomies?.tags ?? [], @@ -263,6 +266,60 @@ export async function importContentFromDisk(strapi: { } } +interface DocumentsClient { + findOne: (params: { + documentId: string; + }) => Promise | null>; + update: (params: { + documentId: string; + data: Record; + }) => Promise>; +} + +/** + * Backfills a bookshelf entry's cover (books, via Open Library by ISBN) or + * missing title/authors (papers, via Semantic Scholar by DOI/arXiv id) the + * first time it's created/updated with an identifier but no cached cover. + * Runs once per entry: `cover_image` already set is treated as "done". + */ +async function enrichBookshelfEntry( + documents: (uid: string) => DocumentsClient, + entry: Record, +): Promise> { + if (entry.cover_image) return entry; + + const slug = entry.slug as string; + const patch: Record = {}; + + try { + if (entry.kind === "book" && entry.isbn) { + const cover = await fetchBookCover(entry.isbn as string, slug); + if (cover) patch.cover_image = cover; + } else if (entry.kind === "paper" && (entry.doi || entry.arxiv_id)) { + const meta = await fetchPaperMetadata({ + doi: entry.doi as string | undefined, + arxivId: entry.arxiv_id as string | undefined, + }); + if (meta?.title && !entry.title) patch.title = meta.title; + if ( + meta?.authors?.length && + !(entry.authors as string[] | undefined)?.length + ) { + patch.authors = meta.authors; + } + } + } catch { + // Network/API failures shouldn't block saving the entry itself. + return entry; + } + + if (Object.keys(patch).length === 0) return entry; + return documents(BOOKSHELF_UID).update({ + documentId: entry.documentId as string, + data: patch, + }); +} + /** Registers the outbound (DB -> file) sync as a Document Service middleware. */ export function registerContentSync(strapi: { documents: { @@ -272,13 +329,7 @@ export function registerContentSync(strapi: { next: () => Promise, ) => Promise, ) => void; - ( - uid: string, - ): { - findOne: (params: { - documentId: string; - }) => Promise | null>; - }; + (uid: string): DocumentsClient; }; log: { info: (msg: string) => void; error: (msg: string) => void }; }): void { @@ -305,7 +356,14 @@ export function registerContentSync(strapi: { result && typeof result === "object" ) { - writeEntryForConfig(config, result as Record); + const entry = + ctx.uid === BOOKSHELF_UID + ? await enrichBookshelfEntry( + strapi.documents, + result as Record, + ) + : (result as Record); + writeEntryForConfig(config, entry); } else if (ctx.action === "delete" && preDeleteSlug) { deleteEntry(config.dir, preDeleteSlug, config.layout); } diff --git a/apps/cms/types/generated/contentTypes.d.ts b/apps/cms/types/generated/contentTypes.d.ts index 9721bf7..2b18c7e 100644 --- a/apps/cms/types/generated/contentTypes.d.ts +++ b/apps/cms/types/generated/contentTypes.d.ts @@ -484,8 +484,8 @@ export interface ApiBookshelfEntryBookshelfEntry }, number >; + read_status: Schema.Attribute.Enumeration<['reading', 'read', 'abandoned']>; slug: Schema.Attribute.UID<'title'> & Schema.Attribute.Required; - status: Schema.Attribute.Enumeration<['reading', 'read', 'abandoned']>; tags: Schema.Attribute.JSON; title: Schema.Attribute.String & Schema.Attribute.Required; updatedAt: Schema.Attribute.DateTime; diff --git a/apps/site/src/content.config.ts b/apps/site/src/content.config.ts index e68c022..d85d392 100644 --- a/apps/site/src/content.config.ts +++ b/apps/site/src/content.config.ts @@ -58,7 +58,7 @@ const bookshelf = defineCollection({ url: z.string().optional(), cover_image: z.string().optional(), rating: z.number().optional(), - status: z.enum(["reading", "read", "abandoned"]).optional(), + read_status: z.enum(["reading", "read", "abandoned"]).optional(), date_started: z.coerce.date().optional(), date_finished: z.coerce.date().optional(), taxonomies, diff --git a/apps/site/src/pages/bookshelf/index.astro b/apps/site/src/pages/bookshelf/index.astro index 4780d19..77e3eba 100644 --- a/apps/site/src/pages/bookshelf/index.astro +++ b/apps/site/src/pages/bookshelf/index.astro @@ -19,7 +19,7 @@ const items = (await getCollection("bookshelf")).sort(

{entry.data.authors.join(", ")}

{entry.data.rating &&

{"★".repeat(entry.data.rating)}

} - {entry.data.status &&

{entry.data.status}

} + {entry.data.read_status &&

{entry.data.read_status}

} {entry.data.kind === "book" && entry.data.isbn &&

ISBN {entry.data.isbn}

} {entry.data.kind === "paper" && entry.data.arxiv_id && (

diff --git a/content/README.md b/content/README.md index 5627801..22192db 100644 --- a/content/README.md +++ b/content/README.md @@ -59,9 +59,17 @@ Body is a free-form Markdown case study. ## `bookshelf/` (books *and* papers) `kind` selects which identifier fields apply — books use `isbn`, papers use -`doi`/`arxiv_id`. Cover images are fetched by a lookup (Open Library/Google -Books for ISBNs; Semantic Scholar/arXiv for papers) and cached under -`static/covers/`, never fetched at page-render time. +`doi`/`arxiv_id`. Book covers are fetched from Open Library by ISBN and +cached into `apps/site/public/covers/` (served at `/covers/...`), never +fetched at page-render time. Papers have no reliable cover source, so +`cover_image` is set manually for them, if at all; Semantic Scholar is only +used to backfill a missing title/authors by DOI/arXiv id. + +Note: the field is named `read_status`, not `status` — Strapi's +content-manager API treats a top-level `status` key as its own reserved +draft/publish parameter even with `draftAndPublish` disabled, so an +attribute literally named `status` silently breaks writes. Don't rename it +back. ```toml +++ @@ -74,7 +82,7 @@ arxiv_id = "2101.00001" # papers only, optional url = "https://..." # papers only: landing page / source link cover_image = "/covers/slug.jpg" rating = 5 # optional, 1-5 -status = "read" # "reading" | "read" | "abandoned" +read_status = "read" # "reading" | "read" | "abandoned" date_started = 2026-01-01 date_finished = 2026-01-20 # optional, omit if still in progress [taxonomies] diff --git a/content/bookshelf/attention-is-all-you-need.md b/content/bookshelf/attention-is-all-you-need.md index 14301f0..1cc10ae 100644 --- a/content/bookshelf/attention-is-all-you-need.md +++ b/content/bookshelf/attention-is-all-you-need.md @@ -7,7 +7,7 @@ arxiv_id = "1706.03762" url = "https://arxiv.org/abs/1706.03762" cover_image = "/covers/attention-is-all-you-need.jpg" rating = 5 -status = "read" +read_status = "read" date_started = 2026-02-01 date_finished = 2026-02-03 [taxonomies] diff --git a/content/bookshelf/structure-and-interpretation-of-computer-programs.md b/content/bookshelf/structure-and-interpretation-of-computer-programs.md index e64e206..0cc5ce2 100644 --- a/content/bookshelf/structure-and-interpretation-of-computer-programs.md +++ b/content/bookshelf/structure-and-interpretation-of-computer-programs.md @@ -5,7 +5,7 @@ authors = ["Harold Abelson", "Gerald Jay Sussman"] isbn = "9780262510875" cover_image = "/covers/structure-and-interpretation-of-computer-programs.jpg" rating = 5 -status = "read" +read_status = "read" date_started = 2026-01-01 date_finished = 2026-01-20 [taxonomies]