1 // 🐙🕸️ GitWikiWeb ∷ build.js
2 // ====================================================================
4 // Copyright © 2023 Lady [@ Lady’s Computer].
6 // This Source Code Form is subject to the terms of the Mozilla Public
7 // License, v. 2.0. If a copy of the MPL was not distributed with this
8 // file, You can obtain one at <https://mozilla.org/MPL/2.0/>.
10 // --------------------------------------------------------------------
12 // A script for generating static wiki pages from a git repository.
14 // First, clone this repository to your machine in an accessible
15 // location (for example, `/srv/git/GitWikiWeb`). A bare repository is
16 // fine; customize the templates and stylesheets as you like. Then, use
17 // this file as the post‐receive hook for your wiki as follows :—
19 // #!/usr/bin/env -S sh
20 // export GITWIKIWEB=/srv/git/GitWikiWeb
21 // git archive --remote=$GITWIKIWEB HEAD build.js \
23 // | deno run -A - ~/public/wiki $GITWIKIWEB
25 // The directory `~/public/wiki` (or whatever you specify as the first
26 // argument to `deno run -A -`) **will be deleted** and a new static
27 // wiki will be generated in its place. This script is not very smart
28 // (yet) and cannot selectively determine which pages will need
29 // updating. It just wipes and regenerates the whole thing.
31 // This script will make a number of requests to `$GITWIKIWEB` to
32 // download the latest templates, stylesheets, ⁊·c from this
33 // repository. Consequently, it is best that you set it to a repository
34 // you control and can ensure uptime for—ideally one local to the
35 // computer hosting the wiki.
40 } from "https://deno.land/std@0.196.0/fs/mod.ts";
44 } from "https://deno.land/std@0.196.0/yaml/mod.ts";
45 import djot
from "npm:@djot/djot@0.2.3";
46 import { Parser
} from "npm:htmlparser2@9.0.0";
47 import { DomHandler
, Element
, Text
} from "npm:domhandler@5.0.3";
48 import * as domutils
from "npm:domutils@3.1.0";
49 import domSerializer
from "npm:dom-serializer@2.0.0";
51 const DESTINATION
= Deno
.args
[0] ?? "~/public/wiki";
52 const REMOTE
= Deno
.args
[1] ?? "/srv/git/GitWikiWeb";
60 const NIL
= Object
.preventExtensions(Object
.create(null));
62 const rawBlock
= (strings
, ...substitutions
) => ({
65 text
: String
.raw(strings
, substitutions
),
67 const rawInline
= (strings
, ...substitutions
) => ({
70 text
: String
.raw(strings
, substitutions
),
72 const str
= (strings
, ...substitutions
) => ({
74 text
: String
.raw(strings
, substitutions
),
77 const getDOM
= (source
) => {
79 const handler
= new DomHandler((error
, dom
) => {
81 throw new Error("GitWikiWeb: Failed to process DOM.", {
88 const parser
= new Parser(handler
);
94 const getRemoteContent
= async (pathName
) => {
95 const getArchive
= new Deno
.Command("git", {
96 args
: ["archive", `--remote=${REMOTE}`, "HEAD", pathName
],
100 const untar
= new Deno
.Command("tar", {
106 getArchive
.stdout
.pipeTo(untar
.stdin
);
113 ] = await Promise
.allSettled([
114 new Response(getArchive
.stderr
).text(),
116 new Response(untar
.stdout
).text(),
117 new Response(untar
.stderr
).text(),
119 ]).then(logErrorsAndCollectResults
);
121 console
.error(err1
+ err2
);
125 if (!getArchiveStatus
.success
) {
127 `GitWikiWeb: git archive returned nonzero exit code: ${getArchiveStatus.code}.`,
129 } else if (!untarStatus
.success
) {
131 `GitWikiWeb: tar returned nonzero exit code: ${untarStatus.code}.`,
138 const logErrorsAndCollectResults
= (results
) =>
139 results
.map(({ value
, reason
}) => {
141 console
.error(reason
);
148 const getReferenceFromPath
= (path
) =>
149 /Sources\/([A-Z][0-9A-Za-z]*\/[A-Z][0-9A-Za-z]*)\.djot$/u.exec(path
)
150 ?.[1]?.replace
?.("/", ":");
152 const listOfInternalLinks
= (references
, wrapper
= ($) => $) => ({
156 children
: Array
.from(
159 const [namespace, pageName
] = splitReference(reference
);
167 "data-realm": "internal",
168 "data-pagename": pageName
,
169 "data-namespace": namespace,
180 const diffReferences
= async (hash
, againstHead
= false) => {
181 const diff
= new Deno
.Command("git", {
189 ...(againstHead
? [hash
, "HEAD"] : [hash
]),
194 const [diffNames
] = await Promise
.allSettled([
195 new Response(diff
.stdout
).text(),
196 new Response(diff
.stderr
).text(),
197 ]).then(logErrorsAndCollectResults
);
198 return references(diffNames
.split("\0")); // returns an iterable
201 function* references(paths
) {
202 for (const path
of paths
) {
203 const reference
= getReferenceFromPath(path
);
212 const splitReference
= (reference
) => {
213 const colonIndex
= reference
.indexOf(":");
215 reference
.substring(0, colonIndex
),
216 reference
.substring(colonIndex
+ 1),
220 class GitWikiWebPage
{
221 #internalLinks
= new Set();
222 #externalLinks
= new Map();
224 constructor(namespace, name
, ast
, source
, config
) {
225 const internalLinks
= this.#internalLinks
;
226 const externalLinks
= this.#externalLinks
;
227 const sections
= Object
.create(null);
228 djot
.applyFilter(ast
, () => {
229 let titleSoFar
= null; // used to collect strs from headings
234 const links_section
= [];
235 if (internalLinks
.size
|| externalLinks
.size
) {
238 rawBlock
`<nav id="links">`,
242 children
: [str
`this page contains links`],
245 if (internalLinks
.size
) {
247 rawBlock
`<details open="">`,
248 rawBlock
`<summary>on this wiki</summary>`,
249 listOfInternalLinks(internalLinks
),
250 rawBlock
`</details>`,
255 if (externalLinks
.size
) {
257 rawBlock
`<details open="">`,
258 rawBlock
`<summary>elsewhere on the Web</summary>`,
263 children
: Array
.from(
265 ([destination
, text
]) => ({
271 attributes
: { "data-realm": "external" },
289 rawBlock
`</details>`,
301 e
.children
.push(...links_section
);
307 const attributes
= e
.attributes
?? NIL
;
308 const { as
} = attributes
;
310 delete attributes
.as
;
312 as
== "b" || as
== "cite" || as
== "i" || as
== "u"
329 if (titleSoFar
!= null) {
343 const { attributes
} = e
;
344 attributes
.title
??= titleSoFar
;
351 const { attributes
, reference
, destination
} = e
;
353 /^(?:[A-Z][0-9A-Za-z]*|[@#])?:(?:[A-Z][0-9A-Za-z]*)?$/u
354 .test(reference
?? "")
356 const [namespacePrefix
, pageName
] = splitReference(
359 const expandedNamespace
= {
363 }[namespacePrefix
] ?? namespacePrefix
;
364 const resolvedReference
= pageName
== ""
365 ? `Namespace:${expandedNamespace}`
366 : `${expandedNamespace}:${pageName}`;
367 this.#internalLinks
.add(resolvedReference
);
368 e
.reference
= resolvedReference
;
369 attributes
["data-realm"] = "internal";
370 attributes
["data-pagename"] = pageName
;
371 attributes
["data-namespace"] = expandedNamespace
;
373 attributes
["data-realm"] = "external";
374 const remote
= destination
??
375 ast
.references
[reference
]?.destination
;
377 externalLinks
.set(remote
, attributes
?.title
);
384 non_breaking_space
: {
386 if (titleSoFar
!= null) {
387 titleSoFar
+= "\xA0";
398 const { attributes
, children
} = e
;
399 const heading
= children
.find(({ tag
}) =>
402 const title
= (() => {
403 if (heading
?.attributes
?.title
) {
404 const result
= heading
.attributes
.title
;
405 delete heading
.attributes
.title
;
408 return heading
.level
== 1
409 ? `${namespace}:${name}`
410 : "untitled section";
413 const variantTitles
= Object
.create(null);
414 for (const attr
in attributes
) {
415 if (attr
.startsWith("v-")) {
416 Object
.defineProperty(
419 { ...READ_ONLY
, value
: attributes
[attr
] },
421 delete attributes
[attr
];
426 const definition
= Object
.create(null, {
427 title
: { ...READ_ONLY
, value
: title
},
430 value
: Object
.preventExtensions(variantTitles
),
433 if (heading
.level
== 1 && !("main" in sections
)) {
434 attributes
.id
= "main";
435 heading
.attributes
??= {};
436 heading
.attributes
.class = "main";
441 Object
.defineProperty(
447 value
: Object
.preventExtensions(definition
),
452 `GitWikiWeb: A section with the provided @id already exists: ${attributes.id}`,
460 if (titleSoFar
!= null) {
469 enter
: ({ text
}) => {
470 if (titleSoFar
!= null) {
481 const codepoint
= /^U\+([0-9A-Fa-f]+)$/u.exec(alias
)?.[1];
484 String.fromCodePoint(parseInt(codepoint, 16))
487 const resolved
= config
.symbols
?.[alias
];
488 return resolved
!= null ? str
`${resolved}` : e
;
494 Object
.defineProperties(this, {
495 ast
: { ...READ_ONLY
, value
: ast
},
496 namespace: { ...READ_ONLY
, value
: namespace },
497 name
: { ...READ_ONLY
, value
: name
},
500 value
: Object
.preventExtensions(sections
),
502 source
: { ...READ_ONLY
, value
: source
},
507 yield* this.#externalLinks
;
511 yield* this.#internalLinks
;
516 const config
= await
getRemoteContent("config.yaml").then((yaml
) =>
517 parseYaml(yaml
, { schema
: JSON_SCHEMA
})
519 const ls
= new Deno
.Command("git", {
520 args
: ["ls-tree", "-rz", "live"],
528 ] = await Promise
.allSettled([
529 new Response(ls
.stdout
).text().then((lsout
) =>
532 .slice(0, -1) // drop the last entry; it is empty
533 .map(($) => $.split(/\s+/g))
535 new Response(ls
.stderr
).text(),
537 ]).then(logErrorsAndCollectResults
);
539 console
.error(lserr
);
543 if (!lsstatus
.success
) {
545 `GitWikiWeb: git ls-tree returned nonzero exit code: ${lsstatus.code}.`,
548 const requiredButMissingPages
= new Map([
549 ["Special:FrontPage", "front page"],
550 ["Special:NotFound", "not found"],
551 ["Special:RecentlyChanged", "recently changed"],
553 const pages
= new Map();
554 const promises
= [emptyDir(DESTINATION
)];
555 for (const object
of objects
) {
556 const hash
= object
[2];
557 const path
= object
[3];
558 const reference
= getReferenceFromPath(path
);
559 if (reference
== null) {
562 const [namespace, pageName
] = splitReference(reference
);
563 const cat
= new Deno
.Command("git", {
564 args
: ["cat-file", "blob", hash
],
568 const promise
= Promise
.allSettled([
569 new Response(cat
.stdout
).text(),
570 new Response(cat
.stderr
).text(),
572 ]).then(logErrorsAndCollectResults
).then(
573 ([source
, caterr
, catstatus
]) => {
575 console
.error(caterr
);
579 if (!catstatus
.success
) {
581 `GitWikiWeb: git cat-file returned nonzero exit code: ${catstatus.code}.`,
584 const page
= new GitWikiWebPage(
589 console
.warn(`Djot(${reference}): ${$.render()}`),
594 const reference
= `${namespace}:${pageName}`;
595 pages
.set(reference
, page
);
596 requiredButMissingPages
.delete(reference
);
600 promises
.push(promise
);
603 for (const [reference
, defaultTitle
] of requiredButMissingPages
) {
604 const [namespace, pageName
] = splitReference(reference
);
605 const source
= `# ${defaultTitle}\n`;
606 const page
= new GitWikiWebPage(
611 console
.warn(`Djot(${reference}): ${$.render()}`),
616 pages
.set(reference
, page
);
618 await Promise
.allSettled(promises
).then(
619 logErrorsAndCollectResults
,
621 const [template
, recentlyChanged
] = await Promise
.allSettled([
622 getRemoteContent("template.html"),
624 const dateParse
= new Deno
.Command("git", {
625 args
: ["rev-parse", "--after=1 week ago"],
629 const [maxAge
] = await Promise
.allSettled([
630 new Response(dateParse
.stdout
).text(),
631 new Response(dateParse
.stderr
).text(),
632 ]).then(logErrorsAndCollectResults
);
637 const revList
= new Deno
.Command("git", {
638 args
: ["rev-list", maxAge
, "--reverse", "HEAD"],
642 [commit
] = await Promise
.allSettled([
643 new Response(revList
.stdout
).text().then((list
) =>
646 new Response(revList
.stderr
).text(),
647 ]).then(logErrorsAndCollectResults
);
650 const revList2
= new Deno
.Command("git", {
651 args
: ["rev-list", "--max-count=1", "HEAD^"],
655 [commit
] = await Promise
.allSettled([
656 new Response(revList2
.stdout
).text().then((list
) =>
659 new Response(revList2
.stderr
).text(),
660 ]).then(logErrorsAndCollectResults
);
664 const results
= new Array(6);
665 const seen
= new Set();
669 const show
= new Deno
.Command("git", {
673 "--format=%H%x00%cI%x00%cD",
674 recency
? `HEAD~${5 - recency}` : commit
,
680 [hash
, dateTime
, humanReadable
],
681 ] = await Promise
.allSettled([
682 new Response(show
.stdout
).text().then((rev
) =>
683 rev
.trim().split("\0")
685 new Response(show
.stderr
).text(),
686 ]).then(logErrorsAndCollectResults
);
690 const ref
of (await
diffReferences(current
, !recency
))
699 results
[recency
] = { dateTime
, hash
, humanReadable
, refs
};
700 } while (recency
-- > 0 && current
&& current
!= commit
);
705 (name
) => ensureDir(`${DESTINATION}/${name}`),
707 ["style.css"].map((dependency
) =>
708 getRemoteContent(dependency
).then((source
) =>
710 `${DESTINATION}/${dependency}`,
716 ]).then(logErrorsAndCollectResults
);
718 const redLinks
= (() => {
719 const result
= new Set();
720 for (const page
of pages
.values()) {
721 for (const link
of page
.internalLinks()) {
722 if (pages
.has(link
)) {
732 const [pageRef
, { ast
, namespace, sections
, source
}] of pages
734 const title
= sections
.main
?.title
?? pageRef
;
735 djot
.applyFilter(ast
, () => {
736 let isNavigationPage
= true;
740 const { content
, navigation
} = (() => {
741 const navigation
= [];
742 if (pageRef
== "Special:RecentlyChanged") {
745 attributes
: { class: "recent-changes" },
748 children
: Array
.from(function* () {
750 const [index
, result
] of recentlyChanged
753 if (result
!= null) {
759 yield* listOfInternalLinks(refs
, (link
) => ({
760 tag
: index
== 0 ? "span" : "strong",
761 attributes
: { "data-recency": `${index}` },
764 ...(index
== 0 ? [] : [
766 rawInline
`<small>(<time dateTime="${dateTime}">`,
767 str
`${humanReadable}`,
768 rawInline
`</time>)</small>`,
779 isNavigationPage
= false;
780 return { content
: e
.children
, navigation
};
788 generated
: "", // will be removed later
791 children
: [str
`${title}`],
793 rawBlock
`<details id="navigation-about" open="">`,
794 rawBlock
`<summary>about this listing</summary>`,
797 rawBlock
`</article>`,
798 rawBlock
`</details>`,
801 rawBlock
`<nav id="navigation">`,
811 rawBlock
`</article>`,
818 const attributes
= e
.attributes
?? NIL
;
820 isNavigationPage
&& e
.level
== 1 &&
821 attributes
?.class == "main"
823 if ("generated" in attributes
) {
824 delete attributes
.generated
;
833 if (e
.level
== 1 && e
.attributes
?.class == "main") {
835 rawBlock
`<header class="main">`,
837 { tag
: "verbatim", text
: pageRef
},
849 const { attributes
, children
, reference
} = e
;
850 if (attributes
["data-realm"] == "internal") {
852 if (redLinks
.has(reference
)) {
854 `/Special:NotFound?path=/${reference}`;
855 attributes
["data-notfound"] = "";
857 e
.destination
= `/${reference}`;
859 if (children
.length
== 0) {
861 pages
.get(reference
)?.sections
?.main
?? NIL
;
862 const { v
} = attributes
;
865 str
`${section.title ?? reference}`,
871 section.variantTitles?.[v] ?? section.title ??
878 if (children
.length
== 0 && "title" in attributes
) {
881 str
`${attributes.title}`,
887 (attributes
.class ?? "").split(/\s/gu).includes("sig")
891 attributes
: { class: "sig" },
892 children
: [str
`—${"\xA0"}`, e
],
902 if (e
.children
.length
< 1) {
903 // The heading for this section was removed and it had
904 // no other children.
913 const doc
= getDOM(template
);
914 const result
= getDOM(`${djot.renderHTML(ast)}`);
915 const headElement
= domutils
.findOne(
916 (node
) => node
.name
== "head",
919 const titleElement
= domutils
.findOne(
920 (node
) => node
.name
== "title",
923 const contentElement
= domutils
.findOne(
924 (node
) => node
.name
== "gitwikiweb-content",
927 if (headElement
== null) {
929 "GitWikiWeb: Template must explicitly include a <head> element.",
932 domutils
.appendChild(
934 new Element("link", {
937 href
: `/${pageRef}/source.djot`,
940 if (titleElement
== null) {
941 domutils
.prependChild(
943 new Element("title", {}, [new Text(title
)]),
946 domutils
.prependChild(titleElement
, new Text(`${title} | `));
949 if (contentElement
== null) {
951 "GitWikiWeb: Template did not include a <gitwikiweb-content> element.",
954 for (const node
of result
) {
955 domutils
.prepend(contentElement
, node
);
957 domutils
.removeElement(contentElement
);
961 `${DESTINATION}/${pageRef}/index.html`,
964 encodeEntities
: "utf8",
965 selfClosingTags
: true,
973 `${DESTINATION}/${pageRef}/source.djot`,
979 await Promise
.allSettled(promises
).then(
980 logErrorsAndCollectResults
,
982 console
.log(`GitWikiWeb: Wrote ${pages.size} page(s).`);