From 2fdd0052b57edb176c0627f1ca8fefcc7e2a5835 Mon Sep 17 00:00:00 2001 From: andwati Date: Sun, 30 Aug 2026 19:18:43 +0300 Subject: [PATCH] Extend llms.txt to all content types; fix a gitignore bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/generate-llms.astro-site.mjs covers writings, portfolio, bookshelf, and blogs (unblocked now that site.config.ts exists from the feeds/meta work), wired into apps/site's build script. Also fixes .gitignore: the bare `public` pattern (meant to ignore the Zola build's root-level output directory) was matching at any depth, so apps/site/public/ and apps/cms/public/ were being silently excluded from every commit despite `git add` reporting them as staged for directory adds (the ignore-file warning only surfaces for an explicitly named path, not a recursive directory add — easy to miss). Scoped to `/public` so only the root output dir is ignored. Picks up robots.txt, the Astro scaffold's favicon, llms.txt, and Strapi's own public/robots.txt, none of which were actually tracked until now. --- .gitignore | 2 +- TODO.md | 50 +++++++---- apps/cms/public/robots.txt | 3 + apps/cms/public/uploads/.gitkeep | 0 apps/site/package.json | 3 +- apps/site/public/favicon.ico | Bin 0 -> 655 bytes apps/site/public/favicon.svg | 9 ++ apps/site/public/llms.txt | 54 ++++++++++++ apps/site/public/robots.txt | 4 + scripts/generate-llms.astro-site.mjs | 127 +++++++++++++++++++++++++++ 10 files changed, 235 insertions(+), 17 deletions(-) create mode 100644 apps/cms/public/robots.txt create mode 100644 apps/cms/public/uploads/.gitkeep create mode 100644 apps/site/public/favicon.ico create mode 100644 apps/site/public/favicon.svg create mode 100644 apps/site/public/llms.txt create mode 100644 apps/site/public/robots.txt create mode 100644 scripts/generate-llms.astro-site.mjs diff --git a/.gitignore b/.gitignore index 5e01026..ba60804 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -public +/public node_modules manimations/*.* /zola diff --git a/TODO.md b/TODO.md index 36f03c0..ca8fd48 100644 --- a/TODO.md +++ b/TODO.md @@ -49,19 +49,27 @@ approved plan for full rationale on each decision referenced below. to the old "CASE FILE" hacker-terminal branding being fully replaced; building this now would just be redone once the new visual identity lands -- [ ] `llms.txt` generation extended to cover all content types, not just - posts — **blocked on site config**: `scripts/generate-llms.mjs` reads - site metadata (base_url, description, nav) from `zola.toml`, which has - no equivalent yet in the new site (Astro site config isn't decided) +- [x] `llms.txt` generation extended to cover all content types — unblocked + once `site.config.ts` existed (built above for feeds/meta anyway): + `scripts/generate-llms.astro-site.mjs` reads all four content + directories directly, wired into `apps/site`'s build script ## Design system - [ ] New typography scale + color system (full reset, not inherited from - `sass/_theme.scss`) -- [ ] Motion primitives: CSS + native Web Animations API spring utility -- [ ] Base layout/components replacing the placeholder `Base.astro` -- [ ] Homepage redesign around the Three.js centerpiece as hero -- [ ] New 404 page concept (terminal easter egg retired) + `sass/_theme.scss`) — **needs your input**: aesthetic/brand decisions, + not something to guess at +- [x] Motion primitives: CSS + native Web Animations API spring utility — + `apps/site/src/lib/spring.ts` (damped-oscillator keyframes, + `prefers-reduced-motion` aware). Not wired into any component yet — + that comes with the design system below +- [ ] Base layout/components replacing the placeholder `Base.astro` — + **needs your input**: depends on the typography/color decision above +- [ ] Homepage redesign around the Three.js centerpiece as hero — **needs + your input**: depends on the constellation data model decision below +- [x] New 404 page concept (terminal easter egg retired) — "Page Fault", + a SIGSEGV/memory-fault framing; plain/unstyled pending the design + system, same as the rest of the current placeholder layout ## Three.js centerpiece @@ -108,12 +116,24 @@ approved plan for full rationale on each decision referenced below. ## Migration from the legacy Zola site -- [ ] Script/process to convert `content/posts/*.md` into the new - `content/writings/` schema -- [ ] Preserve slugs/URLs or add redirects (mechanism TBD — Astro - middleware vs. `static-web-server`/`sws.toml` rules) -- [ ] Migrate `content/archive/`, `content/about.md` equivalents -- [ ] Verify RSS subscribers and inbound links survive the cutover +- [x] Script/process to convert `content/posts/*.md` into the new + `content/writings/` schema — `scripts/migrate-posts-to-writings.mjs`, + additive only (`content/posts/` untouched), re-runnable/idempotent. + 2 of 20 posts (`from-c-to-machine-code`, + `self-hosting-gitea-and-mirroring-github`) use note/tip/warning/danger + callout shortcodes with no Astro equivalent yet — flagged by the + script, needs a real component once the design system exists + (**not blocking**, just noted for later) +- [x] Preserve slugs/URLs via redirects — the script generates 301s from + `/posts//` to `/writings//` into `sws.toml`, idempotently +- [ ] Migrate `content/archive/`, `content/about.md` equivalents — holding + off: the Astro site doesn't have archive/about pages built yet, and + "about" copy/framing is exactly the kind of thing you'd want to + review/rewrite for the new "digital legacy" identity, not port as-is +- [ ] Verify RSS subscribers and inbound links survive the cutover — can + only really be checked once this is deployed live (**needs your + input**: this is a production-cutover verification, not a build-time + one) ## Infra / deployment diff --git a/apps/cms/public/robots.txt b/apps/cms/public/robots.txt new file mode 100644 index 0000000..ff5d316 --- /dev/null +++ b/apps/cms/public/robots.txt @@ -0,0 +1,3 @@ +# To prevent search engines from seeing the site altogether, uncomment the next two lines: +# User-Agent: * +# Disallow: / diff --git a/apps/cms/public/uploads/.gitkeep b/apps/cms/public/uploads/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/site/package.json b/apps/site/package.json index 52cc096..443fd2c 100644 --- a/apps/site/package.json +++ b/apps/site/package.json @@ -7,7 +7,8 @@ }, "scripts": { "dev": "astro dev", - "build": "astro build && pagefind --site dist", + "generate:llms": "node ../../scripts/generate-llms.astro-site.mjs", + "build": "pnpm run generate:llms && astro build && pagefind --site dist", "preview": "astro preview", "check": "astro check", "astro": "astro" diff --git a/apps/site/public/favicon.ico b/apps/site/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..7f48a94d16071d6c8d06478c7458ab12e675019c GIT binary patch literal 655 zcmV;A0&x9_P)Rl-XF(A`bsas&GH{e7U1}Ri zJr5jR8B2*Jd6$=$AqgTM2o2FV$WZ9|#jJ3mmpEs{jB0ps@*Kxv}=RB|IJih8Z&fqwCG`%bN0000#bW%=J zQ=IH#a_&L{B{_6Lu_3m>0bMN%+@aOmN_3G~H^8EGi>+bXO=;-|Z`uFnf==AdP z{Oj-S=ltmI=<4`LcLE*&009F@L_t(|+I`d4ZUZ3@1<*Uo7H^LoCw6-8z4wsbd;b4l zA}zMFtOw2mLX6O5Mgl}(5P=uOM4%=tnuHiuAp%(G<c=npm$Fz%eL + + + diff --git a/apps/site/public/llms.txt b/apps/site/public/llms.txt new file mode 100644 index 0000000..848280f --- /dev/null +++ b/apps/site/public/llms.txt @@ -0,0 +1,54 @@ +# Thoughts and Musings + +> Ian Andwati's personal digital-legacy site covering computer science, mathematics, life, and beliefs — writings, portfolio, and a curated bookshelf. + +## Main pages + +- [Home](https://andwati.com/): Introduction and latest writings. +- [Writings](https://andwati.com/writings/): Long-form essays and technical posts. +- [Portfolio](https://andwati.com/portfolio/): Projects and professional work. +- [Bookshelf](https://andwati.com/bookshelf/): Books and papers read, with notes. +- [Blogs](https://andwati.com/blogs/): Curated external blogs. +- [Tags](https://andwati.com/tags/): Writings grouped by topic. + +## Writings + +- [What is pwn?](https://andwati.com/writings/what-is-pwn/): No not that one +- [Export default and module.exports in Javascript](https://andwati.com/writings/javascript-exporting-modules/): Modules are the building blocks of modern JavaScript applications, allowing for the encapsulation of code into reusable components. Two primary systems govern the exportation of these modules: export default in ES6 and module.exports in CommonJS. +- [cognitive inertia](https://andwati.com/writings/inertia/): inertia +- [Self-Hosting Gitea and Automating GitHub Mirrors](https://andwati.com/writings/self-hosting-gitea-and-mirroring-github/): Self-Hosting Gitea and Automating GitHub Mirrors (and Everything Cloudflare Broke Along the Way) +- [From C to Machine Code: What Actually Runs](https://andwati.com/writings/from-c-to-machine-code/): Before diving into how to break a program, we need to understand what the computer is actually executing +- [Semantic Versioning(SemVer) and Conventional Commits](https://andwati.com/writings/semantic-version-and-conventional-commits/): Adopting best practices in your git and build workflows. +- [Integrating Netlify CMS](https://andwati.com/writings/integrating-netlify/): Integrating netlify cms +- [Install PostgreSQL on Arch | Manjaro | Garuda Linux](https://andwati.com/writings/install-postgresql-15-on-arch-manjaro-garuda-linux/): Getting PostgreSQL up and running on Arch Linux-based distros +- [A Fresh Coat Of Paint](https://andwati.com/writings/a-fresh-coat-of-paint/): Migrating to Zola +- [What say you?](https://andwati.com/writings/utterances-comments/): Adding a comment system by utterances +- [The One Thing Nobody Tells You About WiFi Hotspots on Arch Linux](https://andwati.com/writings/wifi-hotspot-arch/): A minimal Arch install is great until NetworkManager silently fails to find a binary you didn't know you needed +- [Setting up Obsidian sync in Windows](https://andwati.com/writings/obsidian-sync/): Syncing Obsidian in Windows with Windows Task Scheduler with git +- [The Single Byte That Kills Your Exploit: Understanding Endianness](https://andwati.com/writings/the-single-byte-that-kills-your-exploit-understanding-endianness/): Problem: Computers speak different languages, like people +- [Why This Site Exists](https://andwati.com/writings/example-first-post/): A short note on what this new site is, and why it's more than a tech blog. +- [mmuctf 2025 writeup](https://andwati.com/writings/mmuctf2025/): mmuctf 2025 challenges writeup +- [Configure Vue.js app With Tailwind CSS](https://andwati.com/writings/vuejs-tailwind/): Tailwindcss in vuejs +- [Subresource Integrity](https://andwati.com/writings/subresource-integrity/): Understanding Subresource Integrity +- [ScriptCTF 2025 Writeup](https://andwati.com/writings/scriptctf2025/): ScriptCTF 2025 Writeup +- [Django Secret Key Tutorial](https://andwati.com/writings/generate-django-secret-key/): Managing secrets +- [Hello World](https://andwati.com/writings/hello-world/): Hello world + +## Portfolio + +- [Example Project](https://andwati.com/portfolio/): Established the portfolio content schema. + +## Bookshelf + +- [Attention Is All You Need](https://andwati.com/bookshelf/): Ashish Vaswani, Noam Shazeer, Niki Parmar +- [Structure and Interpretation of Computer Programs](https://andwati.com/bookshelf/): Harold Abelson, Gerald Jay Sussman + +## Blogs followed + +- [Example Blog](https://example.com) + +## Feeds and discovery + +- [RSS feed](https://andwati.com/rss.xml): Full writings feed in RSS format. +- [JSON feed](https://andwati.com/feed.json): Full writings feed in JSON Feed format. +- [Sitemap](https://andwati.com/sitemap-index.xml): Machine-readable index of public pages. diff --git a/apps/site/public/robots.txt b/apps/site/public/robots.txt new file mode 100644 index 0000000..09d32b3 --- /dev/null +++ b/apps/site/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://andwati.com/sitemap-index.xml diff --git a/scripts/generate-llms.astro-site.mjs b/scripts/generate-llms.astro-site.mjs new file mode 100644 index 0000000..5056f7b --- /dev/null +++ b/scripts/generate-llms.astro-site.mjs @@ -0,0 +1,127 @@ +// Generates apps/site/public/llms.txt for the new Astro site, covering all +// four content types. Run before `astro build` (wired into +// apps/site/package.json's build script) so it's included as a static file. +// +// Site metadata is duplicated from apps/site/src/site.config.ts rather than +// imported — this script runs as plain Node outside Vite/Astro's pipeline, +// and the config is small enough that duplicating it beats adding a +// cross-module-system import. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseToml } from "smol-toml"; + +const baseUrl = "https://andwati.com"; +const siteTitle = "Thoughts and Musings"; +const summary = + "Ian Andwati's personal digital-legacy site covering computer science, mathematics, life, and beliefs — writings, portfolio, and a curated bookshelf."; + +// Resolved relative to this file, not process.cwd() — this script runs as +// part of apps/site's build script, whose cwd is apps/site, not the repo +// root where content/ lives. +const root = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const contentDir = path.join(root, "content"); +const outFile = path.join(root, "apps", "site", "public", "llms.txt"); + +const FRONTMATTER_RE = /^\+\+\+\r?\n([\s\S]*?)\r?\n\+\+\+\r?\n?([\s\S]*)$/; + +function walk(dir) { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) return walk(full); + return entry.isFile() && entry.name.endsWith(".md") ? [full] : []; + }); +} + +function slugFor(dir, file) { + const rel = path.relative(dir, file).split(path.sep).join("/"); + return rel.endsWith("/index.md") + ? rel.slice(0, -"/index.md".length) + : rel.slice(0, -".md".length); +} + +function readEntries(typeDir) { + return walk(typeDir) + .map((file) => { + const raw = fs.readFileSync(file, "utf8"); + const match = raw.match(FRONTMATTER_RE); + if (!match) return null; + const frontmatter = parseToml(match[1]); + return { slug: slugFor(typeDir, file), frontmatter }; + }) + .filter(Boolean); +} + +const writings = readEntries(path.join(contentDir, "writings")) + .filter((e) => !e.frontmatter.draft) + .sort((a, b) => + String(b.frontmatter.date).localeCompare(String(a.frontmatter.date)), + ); +const portfolio = readEntries(path.join(contentDir, "portfolio")).filter( + (e) => !e.frontmatter.draft, +); +const bookshelf = readEntries(path.join(contentDir, "bookshelf")); +const blogs = readEntries(path.join(contentDir, "blogs")); + +function link(title, url, description) { + return `- [${title}](${url})${description ? `: ${description.replace(/\s+/g, " ").trim()}` : ""}`; +} + +const lines = [ + `# ${siteTitle}`, + "", + `> ${summary}`, + "", + "## Main pages", + "", + `- [Home](${baseUrl}/): Introduction and latest writings.`, + `- [Writings](${baseUrl}/writings/): Long-form essays and technical posts.`, + `- [Portfolio](${baseUrl}/portfolio/): Projects and professional work.`, + `- [Bookshelf](${baseUrl}/bookshelf/): Books and papers read, with notes.`, + `- [Blogs](${baseUrl}/blogs/): Curated external blogs.`, + `- [Tags](${baseUrl}/tags/): Writings grouped by topic.`, + "", + "## Writings", + "", + ...writings.map((e) => + link( + e.frontmatter.title, + `${baseUrl}/writings/${e.slug}/`, + e.frontmatter.description, + ), + ), + "", + "## Portfolio", + "", + ...portfolio.map((e) => + link(e.frontmatter.title, `${baseUrl}/portfolio/`, e.frontmatter.outcome), + ), + "", + "## Bookshelf", + "", + ...bookshelf.map((e) => + link( + e.frontmatter.title, + `${baseUrl}/bookshelf/`, + (e.frontmatter.authors ?? []).join(", "), + ), + ), + "", + "## Blogs followed", + "", + ...blogs.map((e) => link(e.frontmatter.title, e.frontmatter.url)), + "", + "## Feeds and discovery", + "", + `- [RSS feed](${baseUrl}/rss.xml): Full writings feed in RSS format.`, + `- [JSON feed](${baseUrl}/feed.json): Full writings feed in JSON Feed format.`, + `- [Sitemap](${baseUrl}/sitemap-index.xml): Machine-readable index of public pages.`, + "", +]; + +fs.mkdirSync(path.dirname(outFile), { recursive: true }); +fs.writeFileSync(outFile, lines.join("\n")); +console.log( + `Generated ${path.relative(root, outFile)} (${writings.length} writings, ${portfolio.length} portfolio, ${bookshelf.length} bookshelf, ${blogs.length} blogs).`, +);