Extend llms.txt to all content types; fix a gitignore bug

scripts/generate-llms.astro-site.mjs covers writings, portfolio, bookshelf, and blogs (unblocked now that site.config.ts exists from the feeds/meta work), wired into apps/site's build script. Also fixes .gitignore: the bare `public` pattern (meant to ignore the Zola build's root-level output directory) was matching at any depth, so apps/site/public/ and apps/cms/public/ were being silently excluded from every commit despite `git add` reporting them as staged for directory adds (the ignore-file warning only surfaces for an explicitly named path, not a recursive directory add — easy to miss). Scoped to `/public` so only the root output dir is ignored. Picks up robots.txt, the Astro scaffold's favicon, llms.txt, and Strapi's own public/robots.txt, none of which were actually tracked until now.
2026-10-22 02:54:26 +00:00 · 2026-08-30 19:18:43 +03:00
parent 529a0efbfe
commit 2fdd0052b5
10 changed files with 235 additions and 17 deletions
@@ -0,0 +1,127 @@
+// Generates apps/site/public/llms.txt for the new Astro site, covering all
+// four content types. Run before `astro build` (wired into
+// apps/site/package.json's build script) so it's included as a static file.
+//
+// Site metadata is duplicated from apps/site/src/site.config.ts rather than
+// imported — this script runs as plain Node outside Vite/Astro's pipeline,
+// and the config is small enough that duplicating it beats adding a
+// cross-module-system import.
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse as parseToml } from "smol-toml";
+
+const baseUrl = "https://andwati.com";
+const siteTitle = "Thoughts and Musings";
+const summary =
+  "Ian Andwati's personal digital-legacy site covering computer science, mathematics, life, and beliefs — writings, portfolio, and a curated bookshelf.";
+
+// Resolved relative to this file, not process.cwd() — this script runs as
+// part of apps/site's build script, whose cwd is apps/site, not the repo
+// root where content/ lives.
+const root = path.join(path.dirname(fileURLToPath(import.meta.url)), "..");
+const contentDir = path.join(root, "content");
+const outFile = path.join(root, "apps", "site", "public", "llms.txt");
+
+const FRONTMATTER_RE = /^\+\+\+\r?\n([\s\S]*?)\r?\n\+\+\+\r?\n?([\s\S]*)$/;
+
+function walk(dir) {
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) return walk(full);
+    return entry.isFile() && entry.name.endsWith(".md") ? [full] : [];
+  });
+}
+
+function slugFor(dir, file) {
+  const rel = path.relative(dir, file).split(path.sep).join("/");
+  return rel.endsWith("/index.md")
+    ? rel.slice(0, -"/index.md".length)
+    : rel.slice(0, -".md".length);
+}
+
+function readEntries(typeDir) {
+  return walk(typeDir)
+    .map((file) => {
+      const raw = fs.readFileSync(file, "utf8");
+      const match = raw.match(FRONTMATTER_RE);
+      if (!match) return null;
+      const frontmatter = parseToml(match[1]);
+      return { slug: slugFor(typeDir, file), frontmatter };
+    })
+    .filter(Boolean);
+}
+
+const writings = readEntries(path.join(contentDir, "writings"))
+  .filter((e) => !e.frontmatter.draft)
+  .sort((a, b) =>
+    String(b.frontmatter.date).localeCompare(String(a.frontmatter.date)),
+  );
+const portfolio = readEntries(path.join(contentDir, "portfolio")).filter(
+  (e) => !e.frontmatter.draft,
+);
+const bookshelf = readEntries(path.join(contentDir, "bookshelf"));
+const blogs = readEntries(path.join(contentDir, "blogs"));
+
+function link(title, url, description) {
+  return `- [${title}](${url})${description ? `: ${description.replace(/\s+/g, " ").trim()}` : ""}`;
+}
+
+const lines = [
+  `# ${siteTitle}`,
+  "",
+  `> ${summary}`,
+  "",
+  "## Main pages",
+  "",
+  `- [Home](${baseUrl}/): Introduction and latest writings.`,
+  `- [Writings](${baseUrl}/writings/): Long-form essays and technical posts.`,
+  `- [Portfolio](${baseUrl}/portfolio/): Projects and professional work.`,
+  `- [Bookshelf](${baseUrl}/bookshelf/): Books and papers read, with notes.`,
+  `- [Blogs](${baseUrl}/blogs/): Curated external blogs.`,
+  `- [Tags](${baseUrl}/tags/): Writings grouped by topic.`,
+  "",
+  "## Writings",
+  "",
+  ...writings.map((e) =>
+    link(
+      e.frontmatter.title,
+      `${baseUrl}/writings/${e.slug}/`,
+      e.frontmatter.description,
+    ),
+  ),
+  "",
+  "## Portfolio",
+  "",
+  ...portfolio.map((e) =>
+    link(e.frontmatter.title, `${baseUrl}/portfolio/`, e.frontmatter.outcome),
+  ),
+  "",
+  "## Bookshelf",
+  "",
+  ...bookshelf.map((e) =>
+    link(
+      e.frontmatter.title,
+      `${baseUrl}/bookshelf/`,
+      (e.frontmatter.authors ?? []).join(", "),
+    ),
+  ),
+  "",
+  "## Blogs followed",
+  "",
+  ...blogs.map((e) => link(e.frontmatter.title, e.frontmatter.url)),
+  "",
+  "## Feeds and discovery",
+  "",
+  `- [RSS feed](${baseUrl}/rss.xml): Full writings feed in RSS format.`,
+  `- [JSON feed](${baseUrl}/feed.json): Full writings feed in JSON Feed format.`,
+  `- [Sitemap](${baseUrl}/sitemap-index.xml): Machine-readable index of public pages.`,
+  "",
+];
+
+fs.mkdirSync(path.dirname(outFile), { recursive: true });
+fs.writeFileSync(outFile, lines.join("\n"));
+console.log(
+  `Generated ${path.relative(root, outFile)} (${writings.length} writings, ${portfolio.length} portfolio, ${bookshelf.length} bookshelf, ${blogs.length} blogs).`,
+);