1 // 🐙🕸️ GitWikiWeb ∷ build.js
2 // ====================================================================
4 // Copyright © 2023 Lady [@ Lady’s Computer].
6 // This Source Code Form is subject to the terms of the Mozilla Public
7 // License, v. 2.0. If a copy of the MPL was not distributed with this
8 // file, You can obtain one at <https://mozilla.org/MPL/2.0/>.
10 // --------------------------------------------------------------------
12 // A script for generating static wiki pages from a git repository.
14 // First, clone this repository to your machine in an accessible
15 // location (for example, `/srv/git/GitWikiWeb`). A bare repository is
16 // fine; customize the templates and stylesheets as you like. Then, use
17 // this file as the post‐receive hook for your wiki as follows :—
19 // #!/usr/bin/env -S sh
20 // export GITWIKIWEB=/srv/git/GitWikiWeb
21 // git archive --remote=$GITWIKIWEB HEAD build.js \
23 // | deno run -A - ~/public/wiki $GITWIKIWEB current
25 // The directory `~/public/wiki` (or whatever you specify as the first
26 // argument to `deno run -A -`) **will be deleted** and a new static
27 // wiki will be generated in its place. This script is not very smart
28 // (yet) and cannot selectively determine which pages will need
29 // updating. It just wipes and regenerates the whole thing.
31 // This script will make a number of requests to `$GITWIKIWEB` to
32 // download the latest templates, stylesheets, ⁊·c from this
33 // repository. Consequently, it is best that you set it to a repository
34 // you control and can ensure uptime for—ideally one local to the
35 // computer hosting the wiki.
40 } from "https://deno.land/std@0.196.0/fs/mod.ts";
44 } from "https://deno.land/std@0.196.0/yaml/mod.ts";
45 import djot
from "npm:@djot/djot@0.2.3";
46 import { Parser
} from "npm:htmlparser2@9.0.0";
47 import { DomHandler
, Element
, Text
} from "npm:domhandler@5.0.3";
48 import * as domutils
from "npm:domutils@3.1.0";
49 import domSerializer
from "npm:dom-serializer@2.0.0";
51 const DESTINATION
= Deno
.args
[0] ?? "~/public/wiki";
52 const REMOTE
= Deno
.args
[1] ?? "/srv/git/GitWikiWeb";
53 const REV
= Deno
.args
[2] ?? "HEAD";
61 const NIL
= Object
.preventExtensions(Object
.create(null));
63 const rawBlock
= (strings
, ...substitutions
) => ({
66 text
: String
.raw(strings
, substitutions
),
68 const rawInline
= (strings
, ...substitutions
) => ({
71 text
: String
.raw(strings
, substitutions
),
73 const str
= (strings
, ...substitutions
) => ({
75 text
: String
.raw(strings
, substitutions
),
78 const getDOM
= (source
) => {
80 const handler
= new DomHandler((error
, dom
) => {
82 throw new Error("GitWikiWeb: Failed to process DOM.", {
89 const parser
= new Parser(handler
);
95 const getRemoteContent
= async (pathName
) => {
96 const getArchive
= new Deno
.Command("git", {
97 args
: ["archive", `--remote=${REMOTE}`, REV
, pathName
],
101 const untar
= new Deno
.Command("tar", {
107 getArchive
.stdout
.pipeTo(untar
.stdin
);
114 ] = await Promise
.allSettled([
115 new Response(getArchive
.stderr
).text(),
117 new Response(untar
.stdout
).text(),
118 new Response(untar
.stderr
).text(),
120 ]).then(logErrorsAndCollectResults
);
122 console
.error(err1
+ err2
);
126 if (!getArchiveStatus
.success
) {
128 `GitWikiWeb: git archive returned nonzero exit code: ${getArchiveStatus.code}.`,
130 } else if (!untarStatus
.success
) {
132 `GitWikiWeb: tar returned nonzero exit code: ${untarStatus.code}.`,
139 const logErrorsAndCollectResults
= (results
) =>
140 results
.map(({ value
, reason
}) => {
142 console
.error(reason
);
149 const getReferenceFromPath
= (path
) =>
150 /Sources\/([A-Z][0-9A-Za-z]*(?:\/[A-Z][0-9A-Za-z]*)+)\.djot$/u
151 .exec(path
)?.[1]?.replace
?.("/", ":"); // only replaces first slash
153 const listOfInternalLinks
= (references
, wrapper
= ($) => $) => ({
157 children
: Array
.from(
160 const [namespace, pageName
] = splitReference(reference
);
168 "data-realm": "internal",
169 "data-pagename": pageName
,
170 "data-namespace": namespace,
181 const diffReferences
= async (hash
, againstHead
= false) => {
182 const diff
= new Deno
.Command("git", {
190 ...(againstHead
? [hash
, "HEAD"] : [hash
]),
195 const [diffNames
] = await Promise
.allSettled([
196 new Response(diff
.stdout
).text(),
197 new Response(diff
.stderr
).text(),
198 ]).then(logErrorsAndCollectResults
);
199 return references(diffNames
.split("\0")); // returns an iterable
202 function* references(paths
) {
203 for (const path
of paths
) {
204 const reference
= getReferenceFromPath(path
);
213 const splitReference
= (reference
) => {
214 const colonIndex
= reference
.indexOf(":");
216 reference
.substring(0, colonIndex
),
217 reference
.substring(colonIndex
+ 1),
221 class GitWikiWebPage
{
222 #internalLinks
= new Set();
223 #externalLinks
= new Map();
225 constructor(namespace, name
, ast
, source
, config
) {
226 const internalLinks
= this.#internalLinks
;
227 const externalLinks
= this.#externalLinks
;
228 const sections
= Object
.create(null);
229 djot
.applyFilter(ast
, () => {
230 let titleSoFar
= null; // used to collect strs from headings
234 if (titleSoFar
!= null) {
248 const { attributes
} = e
;
249 attributes
.title
??= titleSoFar
;
256 const { attributes
, reference
, destination
} = e
;
258 /^(?:[A-Z][0-9A-Za-z]*|[@#])?:(?:[A-Z][0-9A-Za-z]*(?:\/[A-Z][0-9A-Za-z]*)*)?$/u
259 .test(reference
?? "")
261 const [namespacePrefix
, pageName
] = splitReference(
264 const expandedNamespace
= {
268 }[namespacePrefix
] ?? namespacePrefix
;
269 const resolvedReference
= pageName
== ""
270 ? `Namespace:${expandedNamespace}`
271 : `${expandedNamespace}:${pageName}`;
272 e
.reference
= resolvedReference
;
273 attributes
["data-realm"] = "internal";
274 attributes
["data-pagename"] = pageName
;
275 attributes
["data-namespace"] = expandedNamespace
;
277 resolvedReference
.startsWith("Editor:") &&
278 (attributes
.class ?? "").split(/\s/gu).includes("sig")
280 // This is a special internal link; do not record it.
283 // This is a non‐special internal link; record it.
284 internalLinks
.add(resolvedReference
);
287 attributes
["data-realm"] = "external";
288 const remote
= destination
??
289 ast
.references
[reference
]?.destination
;
291 externalLinks
.set(remote
, attributes
?.title
);
298 non_breaking_space
: {
300 if (titleSoFar
!= null) {
301 titleSoFar
+= "\xA0";
312 const { attributes
, children
} = e
;
313 const heading
= children
.find(({ tag
}) =>
316 const title
= (() => {
317 if (heading
?.attributes
?.title
) {
318 const result
= heading
.attributes
.title
;
319 delete heading
.attributes
.title
;
322 return heading
.level
== 1
323 ? `${namespace}:${name}`
324 : "untitled section";
327 const variantTitles
= Object
.create(null);
328 for (const attr
in attributes
) {
329 if (attr
.startsWith("v-")) {
330 Object
.defineProperty(
333 { ...READ_ONLY
, value
: attributes
[attr
] },
335 delete attributes
[attr
];
340 const definition
= Object
.create(null, {
341 title
: { ...READ_ONLY
, value
: title
},
344 value
: Object
.preventExtensions(variantTitles
),
347 if (heading
.level
== 1 && !("main" in sections
)) {
348 attributes
.id
= "main";
349 heading
.attributes
??= {};
350 heading
.attributes
.class = "main";
355 Object
.defineProperty(
361 value
: Object
.preventExtensions(definition
),
366 `GitWikiWeb: A section with the provided @id already exists: ${attributes.id}`,
374 if (titleSoFar
!= null) {
383 enter
: ({ text
}) => {
384 if (titleSoFar
!= null) {
395 const codepoint
= /^U\+([0-9A-Fa-f]+)$/u.exec(alias
)?.[1];
398 String.fromCodePoint(parseInt(codepoint, 16))
401 const resolved
= config
.symbols
?.[alias
];
402 return resolved
!= null ? str
`${resolved}` : e
;
408 Object
.defineProperties(this, {
409 ast
: { ...READ_ONLY
, value
: ast
},
410 namespace: { ...READ_ONLY
, value
: namespace },
411 name
: { ...READ_ONLY
, value
: name
},
414 value
: Object
.preventExtensions(sections
),
416 source
: { ...READ_ONLY
, value
: source
},
421 yield* this.#externalLinks
;
425 yield* this.#internalLinks
;
430 // Patches for Djot HTML renderer.
431 const { HTMLRenderer
: { prototype: htmlRendererPrototype
} } = djot
;
432 const { inTags
: upstreamInTags
} = htmlRendererPrototype
;
433 htmlRendererPrototype
.inTags = function (
437 extraAttrs
= undefined,
439 const attributes
= node
.attributes
?? NIL
;
440 if ("as" in attributes
) {
441 const newTag
= attributes
.as
;
442 delete attributes
.as
;
443 return upstreamInTags
.call(
451 return upstreamInTags
.call(
462 const config
= await
getRemoteContent("config.yaml").then((yaml
) =>
463 parseYaml(yaml
, { schema
: JSON_SCHEMA
})
465 const ls
= new Deno
.Command("git", {
466 args
: ["ls-tree", "-rz", "HEAD"],
474 ] = await Promise
.allSettled([
475 new Response(ls
.stdout
).text().then((lsout
) =>
478 .slice(0, -1) // drop the last entry; it is empty
479 .map(($) => $.split(/\s+/g))
481 new Response(ls
.stderr
).text(),
483 ]).then(logErrorsAndCollectResults
);
485 console
.error(lserr
);
489 if (!lsstatus
.success
) {
491 `GitWikiWeb: git ls-tree returned nonzero exit code: ${lsstatus.code}.`,
494 const requiredButMissingPages
= new Map([
495 ["Special:FrontPage", "front page"],
496 ["Special:NotFound", "not found"],
497 ["Special:RecentlyChanged", "recently changed"],
499 const pages
= new Map();
500 const promises
= [emptyDir(DESTINATION
)];
501 for (const object
of objects
) {
502 const hash
= object
[2];
503 const path
= object
[3];
504 const reference
= getReferenceFromPath(path
);
505 if (reference
== null) {
508 const [namespace, pageName
] = splitReference(reference
);
509 const cat
= new Deno
.Command("git", {
510 args
: ["cat-file", "blob", hash
],
514 const promise
= Promise
.allSettled([
515 new Response(cat
.stdout
).text(),
516 new Response(cat
.stderr
).text(),
518 ]).then(logErrorsAndCollectResults
).then(
519 ([source
, caterr
, catstatus
]) => {
521 console
.error(caterr
);
525 if (!catstatus
.success
) {
527 `GitWikiWeb: git cat-file returned nonzero exit code: ${catstatus.code}.`,
530 const reference
= `${namespace}:${pageName}`;
531 const page
= new GitWikiWebPage(
536 console
.warn(`Djot(${reference}): ${$.render()}`),
541 pages
.set(reference
, page
);
542 requiredButMissingPages
.delete(reference
);
546 promises
.push(promise
);
549 for (const [reference
, defaultTitle
] of requiredButMissingPages
) {
550 const [namespace, pageName
] = splitReference(reference
);
551 const source
= `# ${defaultTitle}\n`;
552 const page
= new GitWikiWebPage(
557 console
.warn(`Djot(${reference}): ${$.render()}`),
562 pages
.set(reference
, page
);
564 await Promise
.allSettled(promises
).then(
565 logErrorsAndCollectResults
,
567 const [template
, recentlyChanged
] = await Promise
.allSettled([
568 getRemoteContent("template.html"),
570 const dateParse
= new Deno
.Command("git", {
571 args
: ["rev-parse", "--after=1 week ago"],
575 const [maxAge
] = await Promise
.allSettled([
576 new Response(dateParse
.stdout
).text(),
577 new Response(dateParse
.stderr
).text(),
578 ]).then(logErrorsAndCollectResults
);
583 const revList
= new Deno
.Command("git", {
584 args
: ["rev-list", maxAge
, "--reverse", "HEAD"],
588 [commit
] = await Promise
.allSettled([
589 new Response(revList
.stdout
).text().then((list
) =>
592 new Response(revList
.stderr
).text(),
593 ]).then(logErrorsAndCollectResults
);
596 const revList2
= new Deno
.Command("git", {
597 args
: ["rev-list", "--max-count=1", "HEAD^"],
601 [commit
] = await Promise
.allSettled([
602 new Response(revList2
.stdout
).text().then((list
) =>
605 new Response(revList2
.stderr
).text(),
606 ]).then(logErrorsAndCollectResults
);
610 const results
= new Array(6);
611 const seen
= new Set();
612 const maxRecency
= Math
.max(config
.max_recency
| 0, 0);
613 let recency
= maxRecency
;
616 const show
= new Deno
.Command("git", {
620 "--format=%H%x00%cI%x00%cD",
621 recency
? `HEAD~${maxRecency - recency}` : commit
,
627 [hash
, dateTime
, humanReadable
],
628 ] = await Promise
.allSettled([
629 new Response(show
.stdout
).text().then((rev
) =>
630 rev
.trim().split("\0")
632 new Response(show
.stderr
).text(),
633 ]).then(logErrorsAndCollectResults
);
637 const ref
of (await
diffReferences(current
, !recency
))
646 results
[recency
] = { dateTime
, hash
, humanReadable
, refs
};
647 } while (recency
-- > 0 && current
&& current
!= commit
);
652 (name
) => ensureDir(`${DESTINATION}/${name}`),
654 ["style.css"].map((dependency
) =>
655 getRemoteContent(dependency
).then((source
) =>
657 `${DESTINATION}/${dependency}`,
663 ]).then(logErrorsAndCollectResults
);
665 const { redLinks
, subpages
} = (() => {
666 const redLinks
= new Set();
667 const subpages
= new Map();
668 for (const [pageRef
, page
] of pages
) {
669 let superRef
= pageRef
;
671 (superRef
= superRef
.substring(0, superRef
.indexOf("/")))
673 // Iterate over potential superpages and record them if they
675 if (pages
.has(superRef
)) {
676 // There is a superpage for the current page; record it.
677 if (subpages
.has(superRef
)) {
678 // The identified superpage already has other subpages.
679 subpages
.get(superRef
).add(pageRef
);
681 // The identified superpage does not already have other
683 subpages
.set(superRef
, new Set([pageRef
]));
687 // The superpage for the current page has not been found
692 for (const link
of page
.internalLinks()) {
693 // Iterate over the internal links of the current page and
694 // ensure they are all defined.
695 if (pages
.has(link
)) {
696 // The link was defined.
699 // The link was not defined; it is a redlink.
704 return { redLinks
, subpages
};
706 for (const [pageRef
, page
] of pages
) {
707 const { ast
, sections
, source
} = page
;
708 const title
= sections
.main
?.title
?? pageRef
;
709 const internalLinks
= new Set(page
.internalLinks());
710 const externalLinks
= new Map(page
.externalLinks());
711 const subpageRefs
= subpages
.get(pageRef
) ?? new Set();
712 djot
.applyFilter(ast
, () => {
713 let isNavigationPage
= true;
717 const seeAlsoSection
= [];
718 const linksSection
= [];
719 if (subpageRefs
.size
) {
721 rawBlock
`<nav id="seealso">`,
729 rawBlock
`<section id="subpages">`,
737 listOfInternalLinks(subpageRefs
),
738 rawBlock
`</section>`,
744 if (internalLinks
.size
|| externalLinks
.size
) {
746 rawBlock
`<nav id="links">`,
750 children
: [str
`this page contains links`],
753 if (internalLinks
.size
) {
755 rawBlock
`<details open="">`,
756 rawBlock
`<summary>on this wiki</summary>`,
757 listOfInternalLinks(internalLinks
),
758 rawBlock
`</details>`,
763 if (externalLinks
.size
) {
765 rawBlock
`<details open="">`,
766 rawBlock
`<summary>elsewhere on the Web</summary>`,
771 children
: Array
.from(
773 ([destination
, text
]) => ({
779 attributes
: { "data-realm": "external" },
797 rawBlock
`</details>`,
808 const childrenAndLinks
= [
812 rawBlock
`${"\uFFFF"}`, // footnote placeholder
816 const { content
, navigation
} = (() => {
817 const navigation
= [];
818 if (pageRef
== "Special:RecentlyChanged") {
821 attributes
: { class: "recent-changes" },
824 children
: Array
.from(function* () {
826 const [index
, result
] of recentlyChanged
829 if (result
!= null) {
835 yield* listOfInternalLinks(refs
, (link
) => ({
836 tag
: index
== 0 ? "span" : "strong",
837 attributes
: { "data-recency": `${index}` },
840 ...(index
== 0 ? [] : [
842 rawInline
`<small>(<time dateTime="${dateTime}">`,
843 str
`${humanReadable}`,
844 rawInline
`</time>)</small>`,
855 isNavigationPage
= false;
856 return { content
: childrenAndLinks
, navigation
};
864 generated
: "", // will be removed later
867 children
: [str
`${title}`],
869 rawBlock
`<details id="navigation-about" open="">`,
870 rawBlock
`<summary>about this listing</summary>`,
873 rawBlock
`</article>`,
874 rawBlock
`</details>`,
877 rawBlock
`<nav id="navigation">`,
887 rawBlock
`</article>`,
894 const attributes
= e
.attributes
?? NIL
;
896 isNavigationPage
&& e
.level
== 1 &&
897 attributes
?.class == "main"
899 if ("generated" in attributes
) {
900 delete attributes
.generated
;
909 if (e
.level
== 1 && e
.attributes
?.class == "main") {
911 rawBlock
`<header class="main">`,
913 { tag
: "verbatim", text
: pageRef
},
925 const { attributes
, children
, reference
} = e
;
926 if (attributes
["data-realm"] == "internal") {
928 if (redLinks
.has(reference
)) {
930 `/Special:NotFound?path=/${reference}`;
931 attributes
["data-notfound"] = "";
933 e
.destination
= `/${reference}`;
935 if (children
.length
== 0) {
937 pages
.get(reference
)?.sections
?.main
?? NIL
;
938 const { v
} = attributes
;
941 str
`${section.title ?? reference}`,
947 section.variantTitles?.[v] ?? section.title ??
954 if (children
.length
== 0 && "title" in attributes
) {
957 str
`${attributes.title}`,
963 (attributes
.class ?? "").split(/\s/gu).includes("sig")
967 attributes
: { class: "sig" },
968 children
: [str
`—${"\xA0"}`, e
],
978 if (e
.children
.length
< 1) {
979 // The heading for this section was removed and it had
980 // no other children.
989 const renderedAST
= djot
.renderAST(ast
);
990 const doc
= getDOM(template
);
991 const result
= getDOM(djot
.renderHTML(ast
, {
993 raw_block
: (node
, context
) => {
994 if (node
.format
== "html" && node
.text
== "\uFFFF") {
995 if (context
.nextFootnoteIndex
> 1) {
996 const result
= context
.renderNotes(ast
.footnotes
);
997 context
.nextFootnoteIndex
= 1;
1003 return context
.renderAstNodeDefault(node
);
1008 const headElement
= domutils
.findOne(
1009 (node
) => node
.name
== "head",
1012 const titleElement
= domutils
.findOne(
1013 (node
) => node
.name
== "title",
1016 const contentElement
= domutils
.findOne(
1017 (node
) => node
.name
== "gitwikiweb-content",
1020 if (headElement
== null) {
1022 "GitWikiWeb: Template must explicitly include a <head> element.",
1025 domutils
.appendChild(
1027 new Element("link", {
1029 type
: "text/x.djot",
1030 href
: `/${pageRef}/source.djot`,
1033 if (titleElement
== null) {
1034 domutils
.prependChild(
1036 new Element("title", {}, [new Text(title
)]),
1039 domutils
.prependChild(titleElement
, new Text(`${title} | `));
1042 if (contentElement
== null) {
1044 "GitWikiWeb: Template did not include a <gitwikiweb-content> element.",
1047 for (const node
of [...result
]) {
1048 domutils
.prepend(contentElement
, node
);
1050 domutils
.removeElement(contentElement
);
1054 `${DESTINATION}/${pageRef}/index.html`,
1055 domSerializer(doc
, {
1057 encodeEntities
: "utf8",
1058 selfClosingTags
: true,
1061 { createNew
: true },
1066 `${DESTINATION}/${pageRef}/index.ast`,
1068 { createNew
: true },
1073 `${DESTINATION}/${pageRef}/source.djot`,
1075 { createNew
: true },
1079 await Promise
.allSettled(promises
).then(
1080 logErrorsAndCollectResults
,
1082 console
.log(`GitWikiWeb: Wrote ${pages.size} page(s).`);