1 // 🐙🕸️ GitWikiWeb ∷ build.js
2 // ====================================================================
4 // Copyright © 2023 Lady [@ Lady’s Computer].
6 // This Source Code Form is subject to the terms of the Mozilla Public
7 // License, v. 2.0. If a copy of the MPL was not distributed with this
8 // file, You can obtain one at <https://mozilla.org/MPL/2.0/>.
10 // --------------------------------------------------------------------
12 // A script for generating static wiki pages from a git repository.
14 // First, clone this repository to your machine in an accessible
15 // location (for example, `/srv/git/GitWikiWeb`). A bare repository is
16 // fine; customize the templates and stylesheets as you like. Then, use
17 // this file as the post‐receive hook for your wiki as follows :—
19 // #!/usr/bin/env -S sh
20 // export GITWIKIWEB=/srv/git/GitWikiWeb
21 // git archive --remote=$GITWIKIWEB HEAD build.js \
23 // | deno run -A - ~/public/wiki $GITWIKIWEB
25 // The directory `~/public/wiki` (or whatever you specify as the first
26 // argument to `deno run -A -`) **will be deleted** and a new static
27 // wiki will be generated in its place. This script is not very smart
28 // (yet) and cannot selectively determine which pages will need
29 // updating. It just wipes and regenerates the whole thing.
31 // This script will make a number of requests to `$GITWIKIWEB` to
32 // download the latest templates, stylesheets, ⁊·c from this
33 // repository. Consequently, it is best that you set it to a repository
34 // you control and can ensure uptime for—ideally one local to the
35 // computer hosting the wiki.
40 } from "https://deno.land/std@0.195.0/fs/mod.ts";
41 import djot
from "npm:@djot/djot@0.2.3";
42 import { Parser
} from "npm:htmlparser2@9.0.0";
43 import { DomHandler
, Element
, Text
} from "npm:domhandler@5.0.3";
44 import * as domutils
from "npm:domutils@3.1.0";
45 import domSerializer
from "npm:dom-serializer@2.0.0";
47 const DESTINATION
= Deno
.args
[0] ?? "~/public/wiki";
48 const REMOTE
= Deno
.args
[1] ?? "/srv/git/GitWikiWeb";
56 const NIL
= Object
.preventExtensions(Object
.create(null));
58 const rawBlock
= (strings
, ...substitutions
) => ({
61 text
: String
.raw(strings
, substitutions
),
63 const rawInline
= (strings
, ...substitutions
) => ({
66 text
: String
.raw(strings
, substitutions
),
68 const str
= (strings
, ...substitutions
) => ({
70 text
: String
.raw(strings
, substitutions
),
73 const getDOM
= (source
) => {
75 const handler
= new DomHandler((error
, dom
) => {
77 throw new Error("GitWikiWeb: Failed to process DOM.", {
84 const parser
= new Parser(handler
);
90 const getRemoteContent
= async (pathName
) => {
91 const getArchive
= new Deno
.Command("git", {
92 args
: ["archive", `--remote=${REMOTE}`, "HEAD", pathName
],
96 const untar
= new Deno
.Command("tar", {
102 getArchive
.stdout
.pipeTo(untar
.stdin
);
109 ] = await Promise
.allSettled([
110 new Response(getArchive
.stderr
).text(),
112 new Response(untar
.stdout
).text(),
113 new Response(untar
.stderr
).text(),
115 ]).then(logErrorsAndCollectResults
);
117 console
.error(err1
+ err2
);
121 if (!getArchiveStatus
.success
) {
123 `GitWikiWeb: git archive returned nonzero exit code: ${getArchiveStatus.code}.`,
125 } else if (!untarStatus
.success
) {
127 `GitWikiWeb: tar returned nonzero exit code: ${untarStatus.code}.`,
134 const logErrorsAndCollectResults
= (results
) =>
135 results
.map(({ value
, reason
}) => {
137 console
.error(reason
);
144 const getReferenceFromPath
= (path
) =>
145 /Sources\/([A-Z][0-9A-Za-z]*\/[A-Z][0-9A-Za-z]*)\.djot$/u.exec(path
)
146 ?.[1]?.replace
?.("/", ":");
148 const listOfInternalLinks
= (references
, wrapper
= ($) => $) => ({
152 children
: Array
.from(
155 const [namespace, pageName
] = splitReference(reference
);
163 "data-realm": "internal",
164 "data-pagename": pageName
,
165 "data-namespace": namespace,
176 const diffReferences
= async (hash
) => {
177 const diff
= new Deno
.Command("git", {
190 const [diffNames
] = await Promise
.allSettled([
191 new Response(diff
.stdout
).text(),
192 new Response(diff
.stderr
).text(),
193 ]).then(logErrorsAndCollectResults
);
194 return references(diffNames
.split("\0")); // returns an iterable
197 function* references(paths
) {
198 for (const path
of paths
) {
199 const reference
= getReferenceFromPath(path
);
208 const splitReference
= (reference
) => {
209 const colonIndex
= reference
.indexOf(":");
211 reference
.substring(0, colonIndex
),
212 reference
.substring(colonIndex
+ 1),
216 class GitWikiWebPage
{
217 #internalLinks
= new Set();
218 #externalLinks
= new Map();
220 constructor(namespace, name
, ast
, source
) {
221 const internalLinks
= this.#internalLinks
;
222 const externalLinks
= this.#externalLinks
;
223 const sections
= Object
.create(null);
224 djot
.applyFilter(ast
, () => {
225 let titleSoFar
= null; // used to collect strs from headings
230 const links_section
= [];
231 if (internalLinks
.size
|| externalLinks
.size
) {
234 rawBlock
`<nav id="links">`,
238 children
: [str
`this page contains links`],
241 if (internalLinks
.size
) {
243 rawBlock
`<details open="">`,
244 rawBlock
`<summary>on this wiki</summary>`,
245 listOfInternalLinks(internalLinks
),
246 rawBlock
`</details>`,
251 if (externalLinks
.size
) {
253 rawBlock
`<details open="">`,
254 rawBlock
`<summary>elsewhere on the Web</summary>`,
259 children
: Array
.from(
261 ([destination
, text
]) => ({
267 attributes
: { "data-realm": "external" },
285 rawBlock
`</details>`,
297 e
.children
.push(...links_section
);
303 const attributes
= e
.attributes
?? NIL
;
304 const { as
} = attributes
;
306 delete attributes
.as
;
308 as
== "b" || as
== "cite" || as
== "i" || as
== "u"
325 if (titleSoFar
!= null) {
339 const { attributes
} = e
;
340 attributes
.title
??= titleSoFar
;
347 const { attributes
, reference
, destination
} = e
;
349 /^(?:[A-Z][0-9A-Za-z]*|[@#])?:(?:[A-Z][0-9A-Za-z]*)?$/u
350 .test(reference
?? "")
352 const [namespacePrefix
, pageName
] = splitReference(
355 const expandedNamespace
= {
359 }[namespacePrefix
] ?? namespacePrefix
;
360 const resolvedReference
= pageName
== ""
361 ? `Namespace:${expandedNamespace}`
362 : `${expandedNamespace}:${pageName}`;
363 this.#internalLinks
.add(resolvedReference
);
364 e
.reference
= resolvedReference
;
365 attributes
["data-realm"] = "internal";
366 attributes
["data-pagename"] = pageName
;
367 attributes
["data-namespace"] = expandedNamespace
;
369 attributes
["data-realm"] = "external";
370 const remote
= destination
??
371 ast
.references
[reference
]?.destination
;
373 externalLinks
.set(remote
, attributes
?.title
);
380 non_breaking_space
: {
382 if (titleSoFar
!= null) {
383 titleSoFar
+= "\xA0";
394 const { attributes
, children
} = e
;
395 const heading
= children
.find(({ tag
}) =>
398 const title
= (() => {
399 if (heading
?.attributes
?.title
) {
400 const result
= heading
.attributes
.title
;
401 delete heading
.attributes
.title
;
404 return heading
.level
== 1
405 ? `${namespace}:${name}`
406 : "untitled section";
409 const variantTitles
= Object
.create(null);
410 for (const attr
in attributes
) {
411 if (attr
.startsWith("v-")) {
412 Object
.defineProperty(
415 { ...READ_ONLY
, value
: attributes
[attr
] },
417 delete attributes
[attr
];
422 const definition
= Object
.create(null, {
423 title
: { ...READ_ONLY
, value
: title
},
426 value
: Object
.preventExtensions(variantTitles
),
429 if (heading
.level
== 1 && !("main" in sections
)) {
430 attributes
.id
= "main";
431 heading
.attributes
??= {};
432 heading
.attributes
.class = "main";
437 Object
.defineProperty(
443 value
: Object
.preventExtensions(definition
),
448 `GitWikiWeb: A section with the provided @id already exists: ${attributes.id}`,
456 if (titleSoFar
!= null) {
465 enter
: ({ text
}) => {
466 if (titleSoFar
!= null) {
477 const codepoint
= /^U\+([0-9A-Fa-f]+)$/u.exec(alias
)?.[1];
480 String.fromCodePoint(parseInt(codepoint, 16))
484 "--8": str
`${"—\u2060:\u202F"}`, // reverse puppyprick
485 "8--": str
`${"\u202F:\u2060—"}`, // forward puppyprick
486 sp
: rawInline
` `, // space
487 nbsp
: str
`${"\xA0"}`, // no‐break space
488 cgj
: str
`${"\u034F"}`, // combining grapheme joiner
489 ensp
: str
`${"\u2002"}`, // enspace
490 emsp
: str
`${"\u2003"}`, // emspace
491 figsp
: str
`${"\u2007"}`, // figure space
492 zwsp
: str
`${"\u200B"}`, // zero‐width space
493 zwnj
: str
`${"\u200C"}`, // zero‐width nonjoiner
494 zwj
: str
`${"\u200D"}`, // zero‐width joiner
495 nnbsp
: str
`${"\u202F"}`, // narrow no‐break space
496 mathsp
: str
`${"\u205F"}`, // math space
497 wj
: str
`${"\u2060"}`, // word joiner
498 fwsp
: str
`${"\u3000"}`, // fullwidth space
505 Object
.defineProperties(this, {
506 ast
: { ...READ_ONLY
, value
: ast
},
507 namespace: { ...READ_ONLY
, value
: namespace },
508 name
: { ...READ_ONLY
, value
: name
},
511 value
: Object
.preventExtensions(sections
),
513 source
: { ...READ_ONLY
, value
: source
},
518 yield* this.#externalLinks
;
522 yield* this.#internalLinks
;
527 const ls
= new Deno
.Command("git", {
528 args
: ["ls-tree", "-rz", "live"],
536 ] = await Promise
.allSettled([
537 new Response(ls
.stdout
).text().then((lsout
) =>
540 .slice(0, -1) // drop the last entry; it is empty
541 .map(($) => $.split(/\s+/g))
543 new Response(ls
.stderr
).text(),
545 ]).then(logErrorsAndCollectResults
);
547 console
.error(lserr
);
551 if (!lsstatus
.success
) {
553 `GitWikiWeb: git ls-tree returned nonzero exit code: ${lsstatus.code}.`,
556 const requiredButMissingPages
= new Map([
557 ["Special:FrontPage", "front page"],
558 ["Special:NotFound", "not found"],
559 ["Special:RecentlyChanged", "recently changed"],
561 const pages
= new Map();
562 const promises
= [emptyDir(DESTINATION
)];
563 for (const object
of objects
) {
564 const hash
= object
[2];
565 const path
= object
[3];
566 const reference
= getReferenceFromPath(path
);
567 if (reference
== null) {
570 const [namespace, pageName
] = splitReference(reference
);
571 const cat
= new Deno
.Command("git", {
572 args
: ["cat-file", "blob", hash
],
576 const promise
= Promise
.allSettled([
577 new Response(cat
.stdout
).text(),
578 new Response(cat
.stderr
).text(),
580 ]).then(logErrorsAndCollectResults
).then(
581 ([source
, caterr
, catstatus
]) => {
583 console
.error(caterr
);
587 if (!catstatus
.success
) {
589 `GitWikiWeb: git cat-file returned nonzero exit code: ${catstatus.code}.`,
592 const page
= new GitWikiWebPage(
597 console
.warn(`Djot(${reference}): ${$.render()}`),
601 const reference
= `${namespace}:${pageName}`;
602 pages
.set(reference
, page
);
603 requiredButMissingPages
.delete(reference
);
607 promises
.push(promise
);
610 for (const [reference
, defaultTitle
] of requiredButMissingPages
) {
611 const [namespace, pageName
] = splitReference(reference
);
612 const source
= `# ${defaultTitle}\n`;
613 const page
= new GitWikiWebPage(
618 console
.warn(`Djot(${reference}): ${$.render()}`),
622 pages
.set(reference
, page
);
624 await Promise
.allSettled(promises
).then(
625 logErrorsAndCollectResults
,
627 const [template
, recentlyChanged
] = await Promise
.allSettled([
628 getRemoteContent("template.html"),
630 const dateParse
= new Deno
.Command("git", {
631 args
: ["rev-parse", "--after=1 week ago"],
635 const [maxAge
] = await Promise
.allSettled([
636 new Response(dateParse
.stdout
).text(),
637 new Response(dateParse
.stderr
).text(),
638 ]).then(logErrorsAndCollectResults
);
643 const revList
= new Deno
.Command("git", {
644 args
: ["rev-list", maxAge
, "--reverse", "HEAD"],
648 [commit
] = await Promise
.allSettled([
649 new Response(revList
.stdout
).text().then((list
) =>
652 new Response(revList
.stderr
).text(),
653 ]).then(logErrorsAndCollectResults
);
656 const revList2
= new Deno
.Command("git", {
657 args
: ["rev-list", "--max-count=1", "HEAD^"],
661 [commit
] = await Promise
.allSettled([
662 new Response(revList2
.stdout
).text().then((list
) =>
665 new Response(revList2
.stderr
).text(),
666 ]).then(logErrorsAndCollectResults
);
670 const results
= new Array(6);
671 const seen
= new Set();
675 const show
= new Deno
.Command("git", {
679 "--format=%H%x00%cI%x00%cD",
680 recency
? `HEAD~${5 - recency}` : commit
,
686 [hash
, dateTime
, humanReadable
],
687 ] = await Promise
.allSettled([
688 new Response(show
.stdout
).text().then((rev
) =>
689 rev
.trim().split("\0")
691 new Response(show
.stderr
).text(),
692 ]).then(logErrorsAndCollectResults
);
695 for (const ref
of (await
diffReferences(current
))) {
703 results
[recency
] = { dateTime
, humanReadable
, refs
};
704 } while (recency
-- > 0 && current
&& current
!= commit
);
709 (name
) => ensureDir(`${DESTINATION}/${name}`),
711 ["style.css"].map((dependency
) =>
712 getRemoteContent(dependency
).then((source
) =>
714 `${DESTINATION}/${dependency}`,
720 ]).then(logErrorsAndCollectResults
);
722 const redLinks
= (() => {
723 const result
= new Set();
724 for (const page
of pages
.values()) {
725 for (const link
of page
.internalLinks()) {
726 if (pages
.has(link
)) {
736 const [pageRef
, { ast
, namespace, sections
, source
}] of pages
738 const title
= sections
.main
?.title
?? pageRef
;
739 djot
.applyFilter(ast
, () => {
740 let isNavigationPage
= true;
744 const { content
, navigation
} = (() => {
745 const navigation
= [];
746 if (pageRef
== "Special:RecentlyChanged") {
749 attributes
: { class: "recent-changes" },
752 children
: Array
.from(function* () {
754 const [index
, result
] of recentlyChanged
757 if (result
!= null) {
763 yield* listOfInternalLinks(refs
, (link
) => ({
764 tag
: index
== 0 ? "span" : "strong",
765 attributes
: { "data-recency": `${index}` },
768 ...(index
== 0 ? [] : [
770 rawInline
`<small>(<time dateTime="${dateTime}">`,
771 str
`${humanReadable}`,
772 rawInline
`</time>)</small>`,
783 isNavigationPage
= false;
784 return { content
: e
.children
, navigation
};
792 generated
: "", // will be removed later
795 children
: [str
`${title}`],
797 rawBlock
`<details id="navigation-about" open="">`,
798 rawBlock
`<summary>about this listing</summary>`,
801 rawBlock
`</article>`,
802 rawBlock
`</details>`,
805 rawBlock
`<nav id="navigation">`,
815 rawBlock
`</article>`,
822 const attributes
= e
.attributes
?? NIL
;
824 isNavigationPage
&& e
.level
== 1 &&
825 attributes
?.class == "main"
827 if ("generated" in attributes
) {
828 delete attributes
.generated
;
837 if (e
.level
== 1 && e
.attributes
?.class == "main") {
839 rawBlock
`<header class="main">`,
841 { tag
: "verbatim", text
: pageRef
},
853 const { attributes
, children
, reference
} = e
;
854 if (attributes
["data-realm"] == "internal") {
856 if (redLinks
.has(reference
)) {
858 `/Special:NotFound?path=/${reference}`;
859 attributes
["data-notfound"] = "";
861 e
.destination
= `/${reference}`;
863 if (children
.length
== 0) {
865 pages
.get(reference
)?.sections
?.main
?? NIL
;
866 const { v
} = attributes
;
869 str
`${section.title ?? reference}`,
875 section.variantTitles?.[v] ?? section.title ??
882 if (children
.length
== 0 && "title" in attributes
) {
885 str
`${attributes.title}`,
891 (attributes
.class ?? "").split(/\s/gu).includes("sig")
895 attributes
: { class: "sig" },
896 children
: [str
`—${"\xA0"}`, e
],
906 if (e
.children
.length
< 1) {
907 // The heading for this section was removed and it had
908 // no other children.
917 const doc
= getDOM(template
);
918 const result
= getDOM(`${djot.renderHTML(ast)}`);
919 const headElement
= domutils
.findOne(
920 (node
) => node
.name
== "head",
923 const titleElement
= domutils
.findOne(
924 (node
) => node
.name
== "title",
927 const contentElement
= domutils
.findOne(
928 (node
) => node
.name
== "gitwikiweb-content",
931 if (headElement
== null) {
933 "GitWikiWeb: Template must explicitly include a <head> element.",
936 domutils
.appendChild(
938 new Element("link", {
941 href
: `/${pageRef}/source.djot`,
944 if (titleElement
== null) {
945 domutils
.prependChild(
947 new Element("title", {}, [new Text(title
)]),
950 domutils
.prependChild(titleElement
, new Text(`${title} | `));
953 if (contentElement
== null) {
955 "GitWikiWeb: Template did not include a <gitwikiweb-content> element.",
958 for (const node
of result
) {
959 domutils
.prepend(contentElement
, node
);
961 domutils
.removeElement(contentElement
);
965 `${DESTINATION}/${pageRef}/index.html`,
968 encodeEntities
: "utf8",
969 selfClosingTags
: true,
977 `${DESTINATION}/${pageRef}/source.djot`,
983 await Promise
.allSettled(promises
).then(
984 logErrorsAndCollectResults
,
986 console
.log(`GitWikiWeb: Wrote ${pages.size} page(s).`);