Self-Hosting Go Modules

#!/usr/bin/env tsx

import fs from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';

import JSZip from 'jszip';
import { $, argv, cd } from 'zx';


const FILE_MATCHER_IGNORE_LINE_RX = /^(\s*?|(\/.*?))$/;


function patternToRx(pattern: string) {
  const patternAsRxString =
    pattern
      .split("**/")
      .map(
        part =>
          part
            .split("")
            .map(c => c === '*' ? '[^/]+?' : `[${c}]`)
            .join("")
      )
      .join("([^/]+?\\/)*?")
  return new RegExp(`^${patternAsRxString}$`, "i");
}


const modules: string[] =
  (
    Array.isArray(argv.mod)
      ? argv.mod
      : typeof argv.mod === 'string'
        ? [argv.mod]
        : []
  )
    .filter(Boolean);
const include: string[] =
  (
    Array.isArray(argv.include)
      ? argv.include
      : typeof argv.include === 'string'
        ? [argv.include]
        : []
  )
    .filter(Boolean);
const includeListFile: string[] =
  (
    Array.isArray(argv.includeListFile)
      ? argv.includeListFile
      : typeof argv.includeListFile === 'string'
        ? [argv.includeListFile]
        : []
  )
    .filter(Boolean);
const exclude: string[] =
  (
    Array.isArray(argv.exclude)
      ? argv.exclude
      : typeof argv.exclude === 'string'
        ? [argv.exclude]
        : []
  )
    .filter(Boolean);
const excludeListFile: string[] =
  (
    Array.isArray(argv.excludeListFile)
      ? argv.excludeListFile
      : typeof argv.excludeListFile === 'string'
        ? [argv.excludeListFile]
        : []
  )
    .filter(Boolean);
const outDir: string = typeof argv.outDir === 'string' ? path.resolve(argv.outDir) : '';
const skipBuiltinExcludes: boolean = typeof argv.skipBuiltinExcludes !== 'undefined';
const skipBuiltinIncludes: boolean = typeof argv.skipBuiltinIncludes !== 'undefined';
const showSkips: boolean = typeof argv.showSkips !== 'undefined';

if (modules.length === 0) {
  console.error("No modules specified");
  process.exit(1);
}

if (outDir === '') {
  console.error("No out directory specified");
  process.exit(1);
}

// Clean -- if needed -- and create the output directory.
const needsClean = await (async () => {
  try {
    await fs.access(outDir, fs.constants.R_OK)

    return true;
  } catch (e) {
    return false;
  }
})();
try {
  if (needsClean) {
    if (argv.clean) {
      await fs.unlink(outDir);
    } else {
      console.error("Output directory already exists. Add the --clean flag to overwrite it.");
      process.exit(1);
    }
  }
} catch (e) {
  console.error("Unable to clean output directory:", e);
  process.exit(1);
}
try {
  await fs.mkdir(outDir, { recursive: true });
} catch (e) {
  console.error("Error creating output directory:", e);
  process.exit(1);
}

const pwd = (await $`pwd`).text().trim();
for (const module of modules) {
  const modDir = path.resolve(pwd, module);
  console.log("Fetching code for module at:", modDir);
  cd(modDir);

  console.log("Collecting version tags for module at:", modDir);
  const versionTags = $`git tag --list 'v*' | sort`;

  for await (const versionTag of versionTags) {
    // Check out the version
    console.log("Checking out version tag:", versionTag);
    await $`git checkout ${versionTag}`;

    const goModPath = path.join(modDir, 'go.mod');
    try {
      await fs.access(goModPath, fs.constants.R_OK)
    } catch (e) {
      console.error("Error reading go.mod:", e);
      continue;
    }

    const goModContents = await fs.readFile(goModPath, { encoding: 'utf8' });
    const modulePath = (() => {
      for (const line of goModContents.split("\n")) {
        const lineParts = line.split(/\s+/).filter(Boolean);
        if (lineParts.length === 2 && lineParts[0] === 'module') {
          return lineParts[1];
        }
      }
      return null;
    })();

    if (modulePath === null) {
      console.error("Unable to locate module path in go.mod for module:", module);
      continue;
    }

    console.log("Creating output directory for module:", modulePath);

    const versionOutputBaseDir = path.join(outDir, modulePath, '@v');
    const versionListFile = path.join(versionOutputBaseDir, 'list');
    const latestVersionOutputFile = path.join(outDir, modulePath, '@latest');
    try {
      await fs.mkdir(versionOutputBaseDir, { recursive: true });
    } catch (e) {
      console.error("Error creating module version directory:", e);
      continue;
    }

    console.log("Creating version list for module:", modulePath);
    await $`touch ${versionListFile}`;

    // Add version mod file
    const versionModFile = path.join(versionOutputBaseDir, `${versionTag}.mod`);
    try {
      await fs.copyFile(goModPath, versionModFile);
    } catch (e) {
      console.error("Error copying go.mod:", e);
      continue;
    }

    // Add files for this module version
    const conrExcludeRxs: RegExp[] = exclude.map(patternToRx);
    if (!skipBuiltinExcludes) {
      conrExcludeRxs.push(patternToRx('.git/**/*'));
      conrExcludeRxs.push(patternToRx('.gitignore'));
      conrExcludeRxs.push(patternToRx('**/*_test.go'));
    }
    for (const excludeFile of excludeListFile) {
      const excludeConrPath = path.resolve(modDir, excludeFile);
      const excludeConrContents = await (async () => {
        try {
          return await fs.readFile(excludeConrPath, {encoding: 'utf8'});
        } catch (e) {
          console.warn('Unable to read exclude.conr');
          return null;
        }
      })();
      if (excludeConrContents !== null) {
        excludeConrContents
          .split("\n")
          .filter(line => !FILE_MATCHER_IGNORE_LINE_RX.test(line))
          .map(pattern => patternToRx(pattern))
          .forEach(rx => {
            conrExcludeRxs.push(rx);
          });
      }
    }
    const conrIncludeRxs: RegExp[] = include.map(patternToRx);
    if (!skipBuiltinIncludes) {
      conrIncludeRxs.push(patternToRx('**/*.go'));
    }
    for (const includeFile of includeListFile) {
      const includeConrPath = path.resolve(modDir, includeFile);
      const includeConrContents = await (async () => {
        try {
          await fs.stat(includeConrPath);
        } catch (e) {
          return null;
        }
        try {
          return await fs.readFile(includeConrPath, {encoding: 'utf8'});
        } catch (e) {
          console.warn('Unable to read include.conr');
          return null;
        }
      })();
      if (includeConrContents !== null) {
        includeConrContents
          .split("\n")
          .filter(line => !FILE_MATCHER_IGNORE_LINE_RX.test(line))
          .map(pattern => patternToRx(pattern))
          .forEach(rx => {
            conrIncludeRxs.push(rx);
          });
      }
    }
    const zip = new JSZip();
    const allFileEntries = await fs.readdir(modDir, {
      encoding: 'utf8',
      recursive: true,
      withFileTypes: true,
    });
    iterateFiles:
    for (const fileEntry of allFileEntries) {
      if (fileEntry.isDirectory()) {
        continue;
      }

      const filePath = path.join(fileEntry.parentPath, fileEntry.name);
      const zipPathSuffix = filePath.substring(modDir.length + 1);
      if (conrIncludeRxs.length > 0) {
        const shouldInclude = (() => {
          for (const includeRx of conrIncludeRxs) {
            if (includeRx.test(zipPathSuffix)) {
              return true;
            }
          }
          return false;
        })();
        if (!shouldInclude) {
          if (showSkips) {
            console.log(` [skip] ${zipPathSuffix} does not match any:`, conrIncludeRxs);
          }
          continue iterateFiles;
        }
      }
      if (conrExcludeRxs.length > 0) {
        for (const excludeRx of conrExcludeRxs) {
          if (excludeRx.test(zipPathSuffix)) {
            if (showSkips) {
              console.log(` [skip] ${zipPathSuffix} due to rx:`, `excludeRx=${excludeRx}`);
            }
            continue iterateFiles;
          }
        }
      }
      // Set the proper paths for the zip files
      const zipPathPrefix = `${modulePath}@${versionTag}`;
      const zipPath = `${zipPathPrefix}/${zipPathSuffix}`;
      console.log(`+ ${versionTag}.zip <= ${zipPath} <= ${filePath}`);
      zip.file(zipPath, fs.readFile(filePath), { createFolders: false });
    }
    const versionZipFile = path.join(versionOutputBaseDir, `${versionTag}.zip`);
    console.log('Saving module version zip to:', versionZipFile);
    const buffer = await zip.generateAsync({ type: 'nodebuffer' });
    await fs.writeFile(versionZipFile, buffer);

    // Add to version list
    console.log('Appending version tag to:', versionListFile);
    await $`echo ${versionTag} >>${versionListFile}`;

    // Add version info
    const versionInfoFile = path.join(versionOutputBaseDir, `${versionTag}.info`);
    console.log('Creating version info file:', versionInfoFile);
    const versionInfo = JSON.stringify({
      Version: versionTag,
    });
    await $`echo ${versionInfo} >${versionInfoFile}`;

    // Overwrite latest (this is fine because we should have pre-sorted tags, with later versions coming later)
    console.log('Setting @latest:', latestVersionOutputFile);
    await fs.copyFile(versionInfoFile, latestVersionOutputFile);
  }

  cd(pwd);
}