[GitWikiWeb] / build.js

// 🐙🕸️ GitWikiWeb ∷ build.js
// ====================================================================
//
// Copyright © 2023 Lady [@ Lady’s Computer].
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at <https://mozilla.org/MPL/2.0/>.
//
// --------------------------------------------------------------------
//
// A script for generating static wiki pages from a git repository.
//
// First, clone this repository to your machine in an accessible
// location (for example, `/srv/git/GitWikiWeb`). A bare repository is
// fine; customize the templates and stylesheets as you like. Then, use
// this file as the post‐receive hook for your wiki as follows :—
//
//     #!/usr/bin/env -S sh
//     export GITWIKIWEB=/srv/git/GitWikiWeb
//     git archive --remote=$GITWIKIWEB HEAD build.js \
//       | tar -xO \
//       | deno run -A - ~/public/wiki $GITWIKIWEB
//
// The directory `~/public/wiki` (or whatever you specify as the first
// argument to `deno run -A -`) **will be deleted** and a new static
// wiki will be generated in its place. This script is not very smart
// (yet) and cannot selectively determine which pages will need
// updating. It just wipes and regenerates the whole thing.
//
// This script will make a number of requests to `$GITWIKIWEB` to
// download the latest templates, stylesheets, ⁊·c from this
// repository. Consequently, it is best that you set it to a repository
// you control and can ensure uptime for—ideally one local to the
// computer hosting the wiki.

import {
  emptyDir,
  ensureDir,
} from "https://deno.land/std@0.195.0/fs/mod.ts";
import djot from "npm:@djot/djot@0.2.3";
import { Parser } from "npm:htmlparser2@9.0.0";
import { DomHandler, Element, Text } from "npm:domhandler@5.0.3";
import * as domutils from "npm:domutils@3.1.0";
import domSerializer from "npm:dom-serializer@2.0.0";

const DESTINATION = Deno.args[0] ?? "~/public/wiki";
const REMOTE = Deno.args[1] ?? "/srv/git/GitWikiWeb";

const READ_ONLY = {
  configurable: false,
  enumerable: true,
  writable: false,
};

const rawBlock = (strings, ...substitutions) => ({
  tag: "raw_block",
  format: "html",
  text: String.raw(strings, substitutions),
});
const rawInline = (strings, ...substitutions) => ({
  tag: "raw_inline",
  format: "html",
  text: String.raw(strings, substitutions),
});
const str = (strings, ...substitutions) => ({
  tag: "str",
  text: String.raw(strings, substitutions),
});

const getDOM = (source) => {
  let result;
  const handler = new DomHandler((error, dom) => {
    if (error) {
      throw new Error("GitWikiWeb: Failed to process DOM.", {
        cause: error,
      });
    } else {
      result = dom;
    }
  });
  const parser = new Parser(handler);
  parser.write(source);
  parser.end();
  return result;
};

const getRemoteContent = async (pathName) => {
  const getArchive = new Deno.Command("git", {
    args: ["archive", `--remote=${REMOTE}`, "HEAD", pathName],
    stdout: "piped",
    stderr: "piped",
  }).spawn();
  const untar = new Deno.Command("tar", {
    args: ["-xO"],
    stdin: "piped",
    stdout: "piped",
    stderr: "piped",
  }).spawn();
  getArchive.stdout.pipeTo(untar.stdin);
  const [
    err1,
    getArchiveStatus,
    result,
    err2,
    untarStatus,
  ] = await Promise.allSettled([
    new Response(getArchive.stderr).text(),
    getArchive.status,
    new Response(untar.stdout).text(),
    new Response(untar.stderr).text(),
    untar.status,
  ]).then(logErrorsAndCollectResults);
  if (err1 || err2) {
    console.error(err1 + err2);
  } else {
    /* do nothing */
  }
  if (!getArchiveStatus.success) {
    throw new Error(
      `GitWikiWeb: git archive returned nonzero exit code: ${getArchiveStatus.code}.`,
    );
  } else if (!untarStatus.success) {
    throw new Error(
      `GitWikiWeb: tar returned nonzero exit code: ${untarStatus.code}.`,
    );
  } else {
    return result || "";
  }
};

const logErrorsAndCollectResults = (results) =>
  results.map(({ value, reason }) => {
    if (reason) {
      console.error(reason);
      return undefined;
    } else {
      return value;
    }
  });

const getReferenceFromPath = (path) =>
  /Sources\/([A-Z][0-9A-Za-z]*\/[A-Z][0-9A-Za-z]*)\.djot$/u.exec(path)
    ?.[1]?.replace?.("/", ":");

const listOfInternalLinks = (references, wrapper = ($) => $) => ({
  tag: "bullet_list",
  tight: true,
  style: "*",
  children: Array.from(
    references,
    (reference) => {
      const [namespace, pageName] = splitReference(reference);
      return {
        tag: "list_item",
        children: [{
          tag: "para",
          children: [wrapper({
            tag: "link",
            attributes: {
              "data-realm": "internal",
              "data-pagename": pageName,
              "data-namespace": namespace,
            },
            reference,
            children: [],
          })],
        }],
      };
    },
  ),
});

const diffReferences = async (hash) => {
  const diff = new Deno.Command("git", {
    args: [
      "diff-tree",
      "-r",
      "-z",
      "--name-only",
      "--no-renames",
      "--diff-filter=AM",
      hash,
    ],
    stdout: "piped",
    stderr: "piped",
  }).spawn();
  const [diffNames] = await Promise.allSettled([
    new Response(diff.stdout).text(),
    new Response(diff.stderr).text(),
  ]).then(logErrorsAndCollectResults);
  return references(diffNames.split("\0")); // returns an iterable
};

function* references(paths) {
  for (const path of paths) {
    const reference = getReferenceFromPath(path);
    if (reference) {
      yield reference;
    } else {
      /* do nothing */
    }
  }
}

const splitReference = (reference) => {
  const colonIndex = reference.indexOf(":");
  return [
    reference.substring(0, colonIndex),
    reference.substring(colonIndex + 1),
  ];
};

class GitWikiWebPage {
  #internalLinks = new Set();
  #externalLinks = new Map();

  constructor(namespace, name, ast, source) {
    const internalLinks = this.#internalLinks;
    const externalLinks = this.#externalLinks;
    const sections = Object.create(null);
    djot.applyFilter(ast, () => {
      let titleSoFar = null; // used to collect strs from headings
      return {
        doc: {
          enter: (_) => {},
          exit: (e) => {
            const links_section = [];
            if (internalLinks.size || externalLinks.size) {
              links_section.push(
                rawBlock`<footer>`,
                rawBlock`<nav id="links">`,
                {
                  tag: "heading",
                  level: 2,
                  children: [str`this page contains links`],
                },
              );
              if (internalLinks.size) {
                links_section.push(
                  rawBlock`<details>`,
                  rawBlock`<summary>on this wiki</summary>`,
                  listOfInternalLinks(internalLinks),
                  rawBlock`</details>`,
                );
              } else {
                /* do nothing */
              }
              if (externalLinks.size) {
                links_section.push(
                  rawBlock`<details>`,
                  rawBlock`<summary>elsewhere on the Web</summary>`,
                  {
                    tag: "bullet_list",
                    tight: true,
                    style: "*",
                    children: Array.from(
                      externalLinks,
                      ([destination, text]) => ({
                        tag: "list_item",
                        children: [{
                          tag: "para",
                          children: [{
                            tag: "link",
                            attributes: { "data-realm": "external" },
                            destination,
                            children: text
                              ? [
                                rawInline`<cite>`,
                                str`${text}`,
                                rawInline`</cite>`,
                              ]
                              : [
                                rawInline`<code>`,
                                str`${destination}`,
                                rawInline`</code>`,
                              ],
                          }],
                        }],
                      }),
                    ),
                  },
                  rawBlock`</details>`,
                );
              } else {
                /* do nothing */
              }
              links_section.push(
                rawBlock`</nav>`,
                rawBlock`</footer>`,
              );
            } else {
              /* do nothing */
            }
            e.children.push(...links_section);
          },
        },
        hard_break: {
          enter: (_) => {
            if (titleSoFar != null) {
              titleSoFar += " ";
            } else {
              /* do nothing */
            }
          },
          exit: (_) => {},
        },
        heading: {
          enter: (_) => {
            titleSoFar = "";
          },
          exit: (e) => {
            e.attributes ??= {};
            const { attributes } = e;
            attributes.title ??= titleSoFar;
            titleSoFar = null;
          },
        },
        link: {
          enter: (e) => {
            e.attributes ??= {};
            const { attributes, reference, destination } = e;
            if (
              /^(?:[A-Z][0-9A-Za-z]*|[@#])?:(?:[A-Z][0-9A-Za-z]*)?$/u
                .test(reference ?? "")
            ) {
              const [namespacePrefix, pageName] = splitReference(
                reference,
              );
              const expandedNamespace = {
                "": "Page",
                "@": "Editor",
                "#": "Category",
              }[namespacePrefix] ?? namespacePrefix;
              const resolvedReference = pageName == ""
                ? `Namespace:${expandedNamespace}`
                : `${expandedNamespace}:${pageName}`;
              this.#internalLinks.add(resolvedReference);
              e.reference = resolvedReference;
              attributes["data-realm"] = "internal";
              attributes["data-pagename"] = pageName;
              attributes["data-namespace"] = expandedNamespace;
            } else {
              attributes["data-realm"] = "external";
              const remote = destination ??
                ast.references[reference]?.destination;
              if (remote) {
                externalLinks.set(remote, attributes?.title);
              } else {
                /* do nothing */
              }
            }
          },
        },
        non_breaking_space: {
          enter: (_) => {
            if (titleSoFar != null) {
              titleSoFar += "\xA0";
            } else {
              /* do nothing */
            }
          },
          exit: (_) => {},
        },
        section: {
          enter: (_) => {},
          exit: (e) => {
            e.attributes ??= {};
            const { attributes, children } = e;
            const heading = children.find(({ tag }) =>
              tag == "heading"
            );
            const title = (() => {
              if (heading?.attributes?.title) {
                const result = heading.attributes.title;
                delete heading.attributes.title;
                return result;
              } else {
                return heading.level == 1
                  ? `${namespace}:${name}`
                  : "untitled section";
              }
            })();
            const variantTitles = Object.create(null);
            for (const attr in attributes) {
              if (attr.startsWith("v-")) {
                Object.defineProperty(
                  variantTitles,
                  attr.substring(2),
                  { ...READ_ONLY, value: attributes[attr] },
                );
                delete attributes[attr];
              } else {
                continue;
              }
            }
            const definition = Object.create(null, {
              title: { ...READ_ONLY, value: title },
              variantTitles: {
                ...READ_ONLY,
                value: Object.preventExtensions(variantTitles),
              },
            });
            if (heading.level == 1 && !("main" in sections)) {
              attributes.id = "main";
              heading.attributes ??= {};
              heading.attributes.class = "main";
            } else {
              /* do nothing */
            }
            try {
              Object.defineProperty(
                sections,
                attributes.id,
                {
                  ...READ_ONLY,
                  writable: true,
                  value: Object.preventExtensions(definition),
                },
              );
            } catch (cause) {
              throw new Error(
                `GitWikiWeb: A section with the provided @id already exists: ${attributes.id}`,
                { cause },
              );
            }
          },
        },
        soft_break: {
          enter: (_) => {
            if (titleSoFar != null) {
              titleSoFar += " ";
            } else {
              /* do nothing */
            }
          },
          exit: (_) => {},
        },
        str: {
          enter: ({ text }) => {
            if (titleSoFar != null) {
              titleSoFar += text;
            } else {
              /* do nothing */
            }
          },
          exit: (_) => {},
        },
      };
    });
    Object.defineProperties(this, {
      ast: { ...READ_ONLY, value: ast },
      namespace: { ...READ_ONLY, value: namespace },
      name: { ...READ_ONLY, value: name },
      sections: {
        ...READ_ONLY,
        value: Object.preventExtensions(sections),
      },
      source: { ...READ_ONLY, value: source },
    });
  }

  *externalLinks() {
    yield* this.#externalLinks;
  }

  *internalLinks() {
    yield* this.#internalLinks;
  }
}

{
  const ls = new Deno.Command("git", {
    args: ["ls-tree", "-rz", "live"],
    stdout: "piped",
    stderr: "piped",
  }).spawn();
  const [
    objects,
    lserr,
    lsstatus,
  ] = await Promise.allSettled([
    new Response(ls.stdout).text().then((lsout) =>
      lsout
        .split("\0")
        .slice(0, -1) // drop the last entry; it is empty
        .map(($) => $.split(/\s+/g))
    ),
    new Response(ls.stderr).text(),
    ls.status,
  ]).then(logErrorsAndCollectResults);
  if (lserr) {
    console.error(lserr);
  } else {
    /* do nothing */
  }
  if (!lsstatus.success) {
    throw new Error(
      `GitWikiWeb: git ls-tree returned nonzero exit code: ${lsstatus.code}.`,
    );
  } else {
    const requiredButMissingPages = new Map([
      ["Special:FrontPage", "front page"],
      ["Special:NotFound", "not found"],
      ["Special:RecentlyChanged", "recently changed"],
    ]);
    const pages = new Map();
    const promises = [emptyDir(DESTINATION)];
    for (const object of objects) {
      const hash = object[2];
      const path = object[3];
      const reference = getReferenceFromPath(path);
      if (reference == null) {
        continue;
      } else {
        const [namespace, pageName] = splitReference(reference);
        const cat = new Deno.Command("git", {
          args: ["cat-file", "blob", hash],
          stdout: "piped",
          stderr: "piped",
        }).spawn();
        const promise = Promise.allSettled([
          new Response(cat.stdout).text(),
          new Response(cat.stderr).text(),
          cat.status,
        ]).then(logErrorsAndCollectResults).then(
          ([source, caterr, catstatus]) => {
            if (caterr) {
              console.error(caterr);
            } else {
              /* do nothing */
            }
            if (!catstatus.success) {
              throw new Error(
                `GitWikiWeb: git cat-file returned nonzero exit code: ${catstatus.code}.`,
              );
            } else {
              const page = new GitWikiWebPage(
                namespace,
                pageName,
                djot.parse(source, {
                  warn: ($) =>
                    console.warn(`Djot(${reference}): ${$.render()}`),
                }),
                source,
              );
              const reference = `${namespace}:${pageName}`;
              pages.set(reference, page);
              requiredButMissingPages.delete(reference);
            }
          },
        );
        promises.push(promise);
      }
    }
    for (const [reference, defaultTitle] of requiredButMissingPages) {
      const [namespace, pageName] = splitReference(reference);
      const source = `# ${defaultTitle}\n`;
      const page = new GitWikiWebPage(
        namespace,
        pageName,
        djot.parse(source, {
          warn: ($) =>
            console.warn(`Djot(${reference}): ${$.render()}`),
        }),
        source,
      );
      pages.set(reference, page);
    }
    await Promise.allSettled(promises).then(
      logErrorsAndCollectResults,
    );
    const [template, recentlyChanged] = await Promise.allSettled([
      getRemoteContent("template.html"),
      (async () => {
        const dateParse = new Deno.Command("git", {
          args: ["rev-parse", "--after=1 week ago"],
          stdout: "piped",
          stderr: "piped",
        }).spawn();
        const [maxAge] = await Promise.allSettled([
          new Response(dateParse.stdout).text(),
          new Response(dateParse.stderr).text(),
        ]).then(logErrorsAndCollectResults);
        let commit;
        if (!maxAge) {
          /* do nothing */
        } else {
          const revList = new Deno.Command("git", {
            args: ["rev-list", maxAge, "--reverse", "HEAD"],
            stdout: "piped",
            stderr: "piped",
          }).spawn();
          [commit] = await Promise.allSettled([
            new Response(revList.stdout).text().then((list) =>
              list.split("\n")[0]
            ),
            new Response(revList.stderr).text(),
          ]).then(logErrorsAndCollectResults);
        }
        if (!commit) {
          const revList2 = new Deno.Command("git", {
            args: ["rev-list", "--max-count=1", "HEAD^"],
            stdout: "piped",
            stderr: "piped",
          }).spawn();
          [commit] = await Promise.allSettled([
            new Response(revList2.stdout).text().then((list) =>
              list.trim()
            ),
            new Response(revList2.stderr).text(),
          ]).then(logErrorsAndCollectResults);
        } else {
          /* do nothing */
        }
        const results = new Array(6);
        const seen = new Set();
        let recency = 5;
        let current;
        do {
          const show = new Deno.Command("git", {
            args: [
              "show",
              "-s",
              "--format=%H%x00%cI%x00%cD",
              recency ? `HEAD~${5 - recency}` : commit,
            ],
            stdout: "piped",
            stderr: "piped",
          }).spawn();
          const [
            [hash, dateTime, humanReadable],
          ] = await Promise.allSettled([
            new Response(show.stdout).text().then((rev) =>
              rev.trim().split("\0")
            ),
            new Response(show.stderr).text(),
          ]).then(logErrorsAndCollectResults);
          const refs = [];
          current = hash;
          for (const ref of (await diffReferences(current))) {
            if (seen.has(ref)) {
              /* do nothing */
            } else {
              refs.push(ref);
              seen.add(ref);
            }
          }
          results[recency] = { dateTime, humanReadable, refs };
        } while (recency-- > 0 && current && current != commit);
        return results;
      })(),
      ...Array.from(
        pages.keys(),
        (name) => ensureDir(`${DESTINATION}/${name}`),
      ),
      ["style.css"].map((dependency) =>
        getRemoteContent(dependency).then((source) =>
          Deno.writeTextFile(
            `${DESTINATION}/${dependency}`,
            source,
            { createNew: true },
          )
        )
      ),
    ]).then(logErrorsAndCollectResults);
    promises.length = 0;
    const redLinks = (() => {
      const result = new Set();
      for (const page of pages.values()) {
        for (const link of page.internalLinks()) {
          if (pages.has(link)) {
            continue;
          } else {
            result.add(link);
          }
        }
      }
      return result;
    })();
    for (
      const [pageRef, { ast, namespace, sections, source }] of pages
    ) {
      const title = sections.main?.title ?? pageRef;
      djot.applyFilter(ast, () => {
        let isNavigationPage = true;
        return {
          doc: {
            enter: (e) => {
              const { content, navigation } = (() => {
                const navigation = [];
                if (pageRef == "Special:RecentlyChanged") {
                  navigation.push({
                    tag: "bullet_list",
                    attributes: { class: "recent-changes" },
                    tight: true,
                    style: "*",
                    children: Array.from(function* () {
                      for (
                        const [index, result] of recentlyChanged
                          .entries()
                      ) {
                        if (result != null) {
                          const {
                            dateTime,
                            humanReadable,
                            refs,
                          } = result;
                          yield* listOfInternalLinks(refs, (link) => ({
                            tag: index ? "strong" : "span",
                            attributes: { "data-recency": `${index}` },
                            children: [
                              link,
                              str` `,
                              rawInline`<small>(<time dateTime="${dateTime}">`,
                              str`${humanReadable}`,
                              rawInline`</time>)</small>`,
                            ],
                          })).children;
                        } else {
                          /* do nothing */
                        }
                      }
                    }()).reverse(),
                  });
                } else {
                  isNavigationPage = false;
                  return { content: e.children, navigation };
                }
                return {
                  content: [
                    {
                      tag: "heading",
                      attributes: {
                        class: "main",
                        generated: "", // will be removed later
                      },
                      level: 1,
                      children: [str`${title}`],
                    },
                    rawBlock`<details id="navigation-about" open="">`,
                    rawBlock`<summary>about this listing</summary>`,
                    rawBlock`<article>`,
                    ...e.children,
                    rawBlock`</article>`,
                    rawBlock`</details>`,
                  ],
                  navigation: [
                    rawBlock`<nav id="navigation">`,
                    ...navigation,
                    rawBlock`</nav>`,
                  ],
                };
              })();
              e.children = [
                rawBlock`<article>`,
                ...content,
                ...navigation,
                rawBlock`</article>`,
              ];
            },
            exit: (_) => {},
          },
          heading: {
            enter: (e) => {
              const attributes = e.attributes ?? Object.create(null);
              if (
                isNavigationPage && e.level == 1 &&
                attributes?.class == "main"
              ) {
                if ("generated" in attributes) {
                  delete attributes.generated;
                } else {
                  return { stop: [] };
                }
              } else {
                /* do nothing */
              }
            },
            exit: (e) => {
              if (e.level == 1 && e.attributes?.class == "main") {
                return [
                  rawBlock`<header class="main">`,
                  e,
                  { tag: "verbatim", text: pageRef },
                  rawBlock`</header>`,
                ];
              } else {
                /* do nothing */
              }
            },
          },
          link: {
            enter: (_) => {},
            exit: (e) => {
              e.attributes ??= {};
              const { attributes, children, reference } = e;
              if (attributes["data-realm"] == "internal") {
                delete e.reference;
                if (redLinks.has(reference)) {
                  e.destination =
                    `/Special:NotFound?path=/${reference}`;
                  attributes["data-notfound"] = "";
                } else {
                  e.destination = `/${reference}`;
                }
                if (children.length == 0) {
                  const section =
                    pages.get(reference)?.sections?.main ??
                      Object.create(null);
                  const { v } = attributes;
                  if (v == null) {
                    children.push(
                      str`${section.title ?? reference}`,
                    );
                  } else {
                    delete attributes.v;
                    children.push(
                      str`${
                        section.variantTitles?.[v] ?? section.title ??
                          reference
                      }`,
                    );
                  }
                }
              } else {
                if (children.length == 0 && "title" in attributes) {
                  children.push(
                    rawInline`<cite>`,
                    str`${attributes.title}`,
                    rawInline`</cite>`,
                  );
                }
              }
              if (
                (attributes.class ?? "").split(/\s/gu).includes("sig")
              ) {
                return {
                  tag: "span",
                  attributes: { class: "sig" },
                  children: [str`—${"\xA0"}`, e],
                };
              } else {
                /* do nothing */
              }
            },
          },
          section: {
            enter: (_) => {},
            exit: (e) => {
              if (e.children.length < 1) {
                // The heading for this section was removed and it had
                // no other children.
                return [];
              } else {
                /* do nothing */
              }
            },
          },
        };
      });
      const doc = getDOM(template);
      const result = getDOM(`${djot.renderHTML(ast)}`);
      const headElement = domutils.findOne(
        (node) => node.name == "head",
        doc,
      );
      const titleElement = domutils.findOne(
        (node) => node.name == "title",
        headElement,
      );
      const contentElement = domutils.findOne(
        (node) => node.name == "gitwikiweb-content",
        doc,
      );
      if (headElement == null) {
        throw new Error(
          "GitWikiWeb: Template must explicitly include a <head> element.",
        );
      } else {
        domutils.appendChild(
          headElement,
          new Element("link", {
            rel: "source",
            type: "text/x.djot",
            href: `/${pageRef}/source.djot`,
          }),
        );
        if (titleElement == null) {
          domutils.prependChild(
            headElement,
            new Element("title", {}, [new Text(title)]),
          );
        } else {
          domutils.prependChild(titleElement, new Text(`${title} | `));
        }
      }
      if (contentElement == null) {
        throw new Error(
          "GitWikiWeb: Template did not include a <gitwikiweb-content> element.",
        );
      } else {
        for (const node of result) {
          domutils.prepend(contentElement, node);
        }
        domutils.removeElement(contentElement);
      }
      promises.push(
        Deno.writeTextFile(
          `${DESTINATION}/${pageRef}/index.html`,
          domSerializer(doc, {
            emptyAttrs: true,
            encodeEntities: "utf8",
            selfClosingTags: true,
            xmlMode: false,
          }),
          { createNew: true },
        ),
      );
      promises.push(
        Deno.writeTextFile(
          `${DESTINATION}/${pageRef}/source.djot`,
          source,
          { createNew: true },
        ),
      );
    }
    await Promise.allSettled(promises).then(
      logErrorsAndCollectResults,
    );
    console.log(`GitWikiWeb: Wrote ${pages.size} page(s).`);
  }
}