zitadel/apps/docs/scripts/fetch-remote-content.mjs

import fs from 'fs';
import path, { join, dirname, resolve } from 'path';
import { spawn, execSync } from 'child_process';
import { fileURLToPath } from 'url';
import semver from 'semver';
import { Readable } from 'stream';

const FALLBACK_VERSION = 'v4.10.0';
const FALLBACK_BRANCH = 'main';
const REPO = 'zitadel/zitadel';
const CUTOFF = '4.10.0';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT_DIR = join(__dirname, '..');
const PROTO_DIR = join(ROOT_DIR, '../../proto');
const CONTENT_DIR = join(ROOT_DIR, 'content');
const PUBLIC_DIR = join(ROOT_DIR, 'public');
const VERSIONS_FILE = join(ROOT_DIR, 'content/versions.json');
const CONTENT_LATEST_DIR = join(ROOT_DIR, 'content');

console.log(`[fetch-docs] __dirname: ${__dirname}`);
console.log(`[fetch-docs] ROOT_DIR: ${ROOT_DIR}`);
console.log(`[fetch-docs] PROTO_DIR: ${PROTO_DIR}`);
console.log(`[fetch-docs] CONTENT_DIR: ${CONTENT_DIR}`);

// --- Helper Functions ---

// Sanitize logs to prevent log injection (CWE-117)
export function safeLog(str) {
  return str ? String(str).replace(/[\n\r]/g, '') : '';
}

// Validate refs to prevent command injection or unsafe URL construction
export function isValidRef(ref) {
  // Allow alphanumeric, dots, dashes, underscores, slashes, @ (dependabot), and +
  // But explicitly disallow ".." to prevent traversal
  if (ref.includes('..')) return false;
  return /^[a-zA-Z0-9._\-/@+]+$/.test(ref);
}

// Caches result to avoid redundant git/env checks.
let cachedRef = null;
function getCurrentRef() {
  if (cachedRef) return cachedRef;

  if (process.env.VERCEL_GIT_COMMIT_REF) {
    const ref = process.env.VERCEL_GIT_COMMIT_REF;
    if (!isValidRef(ref)) {
       console.warn(`[ref] Invalid VERCEL_GIT_COMMIT_REF: ${safeLog(ref)}, falling through...`);
    } else {
       console.log(`[ref] Detected Vercel Branch: ${safeLog(ref)}`);
       cachedRef = ref;
       return cachedRef;
    }
  }

  if (process.env.GITHUB_REF_NAME) {
    const ref = process.env.GITHUB_REF_NAME;
    if (!isValidRef(ref)) {
       console.warn(`[ref] Invalid GITHUB_REF_NAME: ${safeLog(ref)}, falling through...`);
    } else {
       console.log(`[ref] Detected GitHub Action Branch: ${safeLog(ref)}`);
       cachedRef = ref;
       return cachedRef;
    }
  }

  try {
    const branch = execSync('git branch --show-current').toString().trim();
    if (branch) {
      cachedRef = branch;
      return cachedRef;
    }
  } catch (e) {
    // Ignore git errors
  }
  console.log(`[ref] Defaulting to ${FALLBACK_BRANCH}`);
  cachedRef = FALLBACK_BRANCH;
  return cachedRef;
}

export function resetCache() {
  cachedRef = null;
}

async function fetchTags() {
  const token = process.env.GITHUB_TOKEN;
  const headers = { 'User-Agent': 'node-fetch' };
  if (token) headers['Authorization'] = `token ${token}`;

  const url = `https://api.github.com/repos/${REPO}/tags?per_page=100`;
  console.log(`Fetching tags from ${url}...`);
  const res = await fetch(url, { headers });
  if (!res.ok) {
    const body = await res.text();
    throw new Error(`Failed to fetch tags: ${res.statusText} - ${body}`);
  }
  const tags = await res.json();
  console.log(`Fetched ${tags.length} tags.`);
  return tags;
}

export function filterVersions(tags) {
  console.log(`Filtering tags with cutoff strictly > ${CUTOFF}...`);
  const versions = tags
    .map(t => t.name)
    .filter(v => {
      const valid = semver.valid(v);
      if (!valid) return false;
      // Strict cutoff: Do not fetch or build anything older (including that version)
      return semver.gt(v, CUTOFF);
    })
    .sort((a, b) => semver.rcompare(a, b));

  console.log(`Found ${versions.length} versions matching criteria.`);

  const groups = new Map();
  for (const v of versions) {
    const majorMinor = `${semver.major(v)}.${semver.minor(v)}`;
    if (!groups.has(majorMinor)) {
      groups.set(majorMinor, v);
    }
  }

  const result = Array.from(groups.values()).slice(0, 3);
  console.log(`Selected versions: ${result.join(', ')}`);
  return result;
}

// Safely copy a directory, avoiding recursive version folders
function copyDirectorySafely(src, dest) {
  if (!fs.existsSync(src)) return;
  fs.mkdirSync(dest, { recursive: true });

  const items = fs.readdirSync(src);
  // Matches v4.10, v4.10.0, etc.
  const versionDirPattern = /^v\d+(\.\d+){1,2}$/;

  for (const item of items) {
    // Avoid copying versioned folders to prevent recursion
    let isVersionDir = false;
    try {
      if (versionDirPattern.test(item) && fs.statSync(join(src, item)).isDirectory()) {
        isVersionDir = true;
      }
    } catch (e) {
      // Ignore stat errors
    }

    if (isVersionDir) continue;
    if (item === 'versions.json') continue; // Skip manifest

    fs.cpSync(join(src, item), join(dest, item), { recursive: true });
  }
}

async function downloadVersion(tag, sourceRef) {
  if (!isValidRef(sourceRef)) {
     throw new Error(`Invalid sourceRef: ${safeLog(sourceRef)}`);
  }

  const currentRef = getCurrentRef();
  const isLocal = sourceRef === currentRef;
  const tempDir = join(ROOT_DIR, `.temp/${tag}`); // Extract to tag-specific temp to avoid collisions
  fs.mkdirSync(tempDir, { recursive: true });

  try {
    if (isLocal) {
        console.log(`[local] Copying local content for ${tag} (ref: ${safeLog(sourceRef)})...`);
        
        // Copy content
        copyDirectorySafely(join(ROOT_DIR, 'content'), join(tempDir, 'apps/docs/content'));
        
        // Copy public
        copyDirectorySafely(join(ROOT_DIR, 'public'), join(tempDir, 'apps/docs/public'));

        // Copy external files
        const repoRoot = resolve(ROOT_DIR, '../..');
        const tempCmd = join(tempDir, 'cmd');
        fs.mkdirSync(tempCmd, { recursive: true });

        const defaultsPath = join(repoRoot, 'cmd/defaults.yaml');
        if (fs.existsSync(defaultsPath)) {
            fs.cpSync(defaultsPath, join(tempCmd, 'defaults.yaml'));
        }

        const stepsPath = join(repoRoot, 'cmd/setup/steps.yaml');
        if (fs.existsSync(stepsPath)) {
            fs.mkdirSync(join(tempCmd, 'setup'), { recursive: true });
            fs.cpSync(stepsPath, join(tempCmd, 'setup/steps.yaml'));
        }

    } else {
        const isBranch = sourceRef === 'main' || sourceRef === 'master' || !sourceRef.startsWith('v');
        const typeSegment = isBranch ? 'heads' : 'tags';
        const url = `https://github.com/${REPO}/archive/refs/${typeSegment}/${sourceRef}.tar.gz`;

        console.log(`Downloading content for ${tag} (using source: ${safeLog(sourceRef)})...`);
        const res = await fetch(url);
        if (!res.ok) throw new Error(`Failed to download ${url}: ${res.statusText}`);

        // Build tar arguments to extract only the docs content directory from the GitHub archive.
        const tarArgsWildcard = [
            '-xz',
            '-C', tempDir,
            '--strip-components=1'
        ];

        // GNU tar (Linux) requires --wildcards for patterns, BSD tar (macOS) does not support it (and uses patterns by default)
        if (process.platform !== 'darwin') {
            tarArgsWildcard.push('--wildcards');
        }

        tarArgsWildcard.push(
            '*/apps/docs/content',
            '*/apps/docs/public',
            '*/cmd/defaults.yaml',
            '*/cmd/setup/steps.yaml'
        );

        await new Promise((resolve, reject) => {
            const tar = spawn('tar', tarArgsWildcard);
            Readable.fromWeb(res.body).pipe(tar.stdin);
            tar.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`tar exited ${code}`))));
            tar.stderr.on('data', d => {
                const msg = d.toString();
                if (!msg.includes('Not found in archive')) console.error(msg);
            });
        });
    }

    // Move to final destination
    const versionSlug = `v${semver.major(tag)}.${semver.minor(tag)}`;
    const contentDest = join(CONTENT_DIR, versionSlug);
    const publicDest = join(PUBLIC_DIR, versionSlug);

    fs.mkdirSync(dirname(contentDest), { recursive: true });
    fs.mkdirSync(dirname(publicDest), { recursive: true });

    fs.rmSync(contentDest, { recursive: true, force: true });
    fs.rmSync(publicDest, { recursive: true, force: true });

    if (fs.existsSync(join(tempDir, 'apps/docs/content'))) {
        fs.renameSync(join(tempDir, 'apps/docs/content'), contentDest);
    } else {
         // Fallback warning
         console.warn(`[warn] apps/docs/content not found in archive for ${tag} (ref: ${safeLog(sourceRef)})`);
    }
    
    // Handle external files
    const externalDir = join(contentDest, '_external/cmd');
    fs.mkdirSync(externalDir, { recursive: true });
    
    if (fs.existsSync(join(tempDir, 'cmd/defaults.yaml'))) {
        fs.cpSync(join(tempDir, 'cmd/defaults.yaml'), join(externalDir, 'defaults.yaml'));
    }
    if (fs.existsSync(join(tempDir, 'cmd/setup/steps.yaml'))) {
        fs.mkdirSync(join(externalDir, 'setup'), { recursive: true });
        fs.cpSync(join(tempDir, 'cmd/setup/steps.yaml'), join(externalDir, 'setup/steps.yaml'));
    }

    if (fs.existsSync(join(tempDir, 'apps/docs/public'))) {
        fs.renameSync(join(tempDir, 'apps/docs/public'), publicDest);
    }

  } catch (err) {
    console.error(`[error] Failed to process version ${tag}: ${err.message}`);
    throw err;
  } finally {
    // Always clean up temp dir
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
}

async function downloadFileContent(tagOrBranch, repoPath) {
  const currentRef = getCurrentRef();
  if (tagOrBranch === currentRef) {
    console.log(`[local] Reading local file content for: ${repoPath}`);
    const repoRoot = resolve(ROOT_DIR, '../..');
    
    let decodedRepoPath;
    try {
        decodedRepoPath = decodeURIComponent(repoPath.replace(/\\/g, '/'));
    } catch (e) {
        // If decoding fails (malformed escape sequences), fall back to original
        decodedRepoPath = repoPath;
    }
    const normalizedRepoPath = decodedRepoPath.replace(/\\/g, '/');

    const localPath = resolve(repoRoot, normalizedRepoPath);
    
    // Secure Check: Ensure the resolved path actually starts with the repo root
    // strict check including separator to avoid partial matches (e.g. /opt/repo matching /opt/repo-hack)
    const secureRepoRoot = repoRoot.endsWith(path.sep) ? repoRoot : repoRoot + path.sep;
    if (!localPath.startsWith(secureRepoRoot) && localPath !== repoRoot) {
      console.warn(`[local] Refusing to read file outside repo root: ${localPath}`);
      return null;
    }

    if (fs.existsSync(localPath)) {
      return fs.readFileSync(localPath, 'utf8');
    }
    return null;
  }

  if (!isValidRef(tagOrBranch)) return null;

  const url = `https://raw.githubusercontent.com/${REPO}/${tagOrBranch}/${repoPath}`;
  const res = await fetch(url);
  if (!res.ok) return null;
  return await res.text();
}

async function fixRelativeImports(versionDir, tagOrBranch) {
  if (!fs.existsSync(versionDir)) return;
  const files = fs.readdirSync(versionDir, { recursive: true });

  const rewritePath = (filePath, originalRelPath) => {
    const versionFolder = path.basename(versionDir);
    const relativePathInContent = filePath.split(join('content', versionFolder))[1];
    if (!relativePathInContent) return null;

    const originalFilePath = join(CONTENT_LATEST_DIR, relativePathInContent);
    const originalDir = dirname(originalFilePath);
    const absoluteTarget = resolve(originalDir, originalRelPath);
    const projectRoot = resolve(ROOT_DIR, '../..');

    if (absoluteTarget.startsWith(PUBLIC_DIR)) {
      const relToPublic = absoluteTarget.slice(PUBLIC_DIR.length + 1);
      const newTargetAbs = join(PUBLIC_DIR, versionFolder, relToPublic);
      const newRelPath = path.relative(dirname(filePath), newTargetAbs);
      return newRelPath.split(path.sep).join('/');
    }

    if (absoluteTarget.startsWith(projectRoot) && !absoluteTarget.startsWith(CONTENT_LATEST_DIR) && !absoluteTarget.startsWith(PUBLIC_DIR)) {
      return null;
    }
    if (originalRelPath.includes('cmd/defaults.yaml') || originalRelPath.includes('cmd/setup/steps.yaml')) {
      return null;
    }

    if (absoluteTarget.startsWith(ROOT_DIR) && !absoluteTarget.startsWith(CONTENT_LATEST_DIR)) {
      const newRelPath = path.relative(dirname(filePath), absoluteTarget);
      return newRelPath.split(path.sep).join('/');
    }
    return null;
  };

  // We'll traverse all files to fix links/imports
  for (const file of files) {
    const filePath = join(versionDir, file);
    if (!fs.statSync(filePath).isFile()) continue;
    if (!filePath.endsWith('.mdx') && !filePath.endsWith('.md')) continue;

    let content = fs.readFileSync(filePath, 'utf8');
    let changed = false;

    // --- Replacements ---

    // 1. Imports: import ... from '...'
    const importRegex = /(import\s+.*?\s+from\s+['"])([^'"]+)(['"])/g;
    content = content.replace(importRegex, (match, p1, p2, p3) => {
      if (!p2.startsWith('.')) return match; // Only relative
      const rewritten = rewritePath(filePath, p2);
      if (rewritten && rewritten !== p2) {
        changed = true;
        return `${p1}${rewritten}${p3}`;
      }
      return match;
    });

    // 2. Markdown Images: ![alt](src)
    const mdImgRegex = /(!\[.*?\]\()([^\)]+)(\))/g;
    content = content.replace(mdImgRegex, (match, p1, p2, p3) => {
      if (!p2.startsWith('.')) return match;
      const rewritten = rewritePath(filePath, p2);
      if (rewritten && rewritten !== p2) {
        changed = true;
        return `${p1}${rewritten}${p3}`;
      }
      return match;
    });

    // 3. HTML Attributes: src="..." or href="..."
    const htmlAttrRegex = /(src|href)=['"]([^'"]+)['"]/g;
    content = content.replace(htmlAttrRegex, (match, attr, val) => {
      if (!val.startsWith('.')) return match;
      const rewritten = rewritePath(filePath, val);
      if (rewritten && rewritten !== val) {
        changed = true;
        const quote = match.includes("'") ? "'" : '"';
        return `${attr}=${quote}${rewritten}${quote}`;
      }
      return match;
    });

    // --- Special handling for the "download external file" case ---
    // Scan for strictly external files (like ../cmd/defaults.yaml) first
    const importRegexForDownload = /import\s+.*\s+from\s+['"](\.\.\/(\.\.\/)+[^'"]+)['"]/g;
    let match;
    while ((match = importRegexForDownload.exec(content)) !== null) {
      const relPath = match[1];
      const versionFolder = path.basename(versionDir);
      const relativePathInContent = filePath.split(join('content', versionFolder))[1];
      const originalFilePath = join(CONTENT_LATEST_DIR, relativePathInContent);
      const absoluteImportTarget = resolve(dirname(originalFilePath), relPath);
      const projectRoot = resolve(ROOT_DIR, '../..');

      // If it points to cmd/ or similar external
      if (absoluteImportTarget.startsWith(projectRoot) && !absoluteImportTarget.startsWith(CONTENT_LATEST_DIR) && !absoluteImportTarget.startsWith(PUBLIC_DIR)) {
        const repoRoot = resolve(ROOT_DIR, '../..');

        let relativeToRepoRoot;
        if (absoluteImportTarget.startsWith(join(repoRoot, 'apps'))) {
          relativeToRepoRoot = absoluteImportTarget.replace(join(repoRoot, 'apps') + '/', '');
        } else {
          relativeToRepoRoot = absoluteImportTarget.replace(repoRoot + '/', '');
        }

        const localPathInVersion = join(versionDir, '_external', relativeToRepoRoot);

        // Download if missing
        if (!fs.existsSync(localPathInVersion)) {
          console.log(`[fix-imports] Downloading external: ${relativeToRepoRoot}`);
          const fileContent = await downloadFileContent(tagOrBranch, relativeToRepoRoot);
          if (fileContent) {
            fs.mkdirSync(dirname(localPathInVersion), { recursive: true });
            fs.writeFileSync(localPathInVersion, fileContent);
          }
        }

        // Rewrite to local path
        const newRelPath = path.relative(dirname(filePath), localPathInVersion).split(path.sep).join('/');
        const finalPath = newRelPath.startsWith('.') ? newRelPath : './' + newRelPath;
        // Perform replacement
        const newImport = match[0].replace(relPath, finalPath);
        content = content.replace(match[0], newImport);
        changed = true;
      }
    }

    if (changed) {
      console.log(`[fix-relative] Updated ${file}`);
      fs.writeFileSync(filePath, content);
    }
  }
}

function getLocalVersion() {
    const vercelBranch = process.env.VERCEL_GIT_COMMIT_REF;
    let branch = vercelBranch;
    if (!branch) {
        try {
            branch = execSync('git branch --show-current').toString().trim();
        } catch (e) {}
    }

    if (branch && branch !== 'main' && branch !== 'master') {
        return { label: branch, isUnreleased: true };
    }
    if (branch === 'main' || branch === 'master') {
        return { label: 'ZITADEL Docs', isUnreleased: false };
    }

    try {
        const tag = execSync('git describe --tags --abbrev=0').toString().trim();
        if (semver.valid(tag) && semver.gt(tag, CUTOFF)) {
            return { label: tag, isUnreleased: false };
        }
    } catch (e) { }

    return { label: 'v4.11.0', isUnreleased: true };
}

async function run() {
  console.log('Starting version discovery...');
  const tags = await fetchTags();
  const selectedTags = filterVersions(tags);

  let localVer = getLocalVersion();
  let others = selectedTags;

  console.log(`Latest version (Local): ${localVer.label} (Unreleased: ${localVer.isUnreleased})`);

  // Conditional Fallback: If no versions found > 4.10.0, inject v4.10.0
  if (others.length === 0) {
    console.log(`[fallback] No versions found strictly > ${CUTOFF}. Injecting ${FALLBACK_VERSION} as fallback.`);
    others.push(FALLBACK_VERSION);
  }

  console.log(`Older versions to fetch: ${others.join(', ') || 'None'}`);

  await Promise.all(others.map(async (tag) => {
    let sourceRef = tag;

    // Explicit logic for version 4.10.x to use active branch (Faking 4.10.x)
    // This prevents fetching incompatible legacy docs for 4.10.1 etc.
    if (tag === FALLBACK_VERSION || (semver.major(tag) === 4 && semver.minor(tag) === 10)) {
      console.log(`[fake-override] Version ${tag} matches 4.10.x. Using fallback source (main/current) instead of tag.`);
      sourceRef = getCurrentRef();
    }

    const versionSlug = `v${semver.major(tag)}.${semver.minor(tag)}`;
    const contentDest = join(CONTENT_DIR, versionSlug);

    // Simple cache check: if directory exists and looks populated, skip
    // We could check for a specific file like meta.json or similar if we wanted to be more robust
    if (fs.existsSync(contentDest)) {
      console.log(`[skip] Version ${versionSlug} already exists. Skipping download.`);
    } else {
        await downloadVersion(tag, sourceRef);
        // Correctly pass sourceRef here so external files are fetched from the same place (local or remote)
        await fixRelativeImports(contentDest, sourceRef);
    }
  }));

  const versionsJson = [
    {
      param: 'latest',
      label: localVer.isUnreleased ? `${localVer.label} (Unreleased)` : `${localVer.label} (Latest)`,
      url: '/docs',
      ref: 'local',
      refType: 'local'
    }
  ];

  for (const tag of others) {
    const v = `v${semver.major(tag)}.${semver.minor(tag)}`;
    const versionSlug = `v${semver.major(tag)}${semver.minor(tag)}x`;
    const targetUrl = `https://docs-git-${versionSlug}-zitadel.vercel.app/docs`;
    versionsJson.push({
      param: v,
      label: v,
      url: `/docs/${v}`,
      ref: tag,
      refType: 'tag',
      target: targetUrl
    });
  }

  fs.writeFileSync(VERSIONS_FILE, JSON.stringify(versionsJson, null, 2));
  console.log('versions.json generated successfully.');
}

if (process.argv[1] === fileURLToPath(import.meta.url)) {
  run().catch(err => {
    console.error(err);
    process.exit(1);
  });
}

export { getCurrentRef, downloadVersion, downloadFileContent };