1 // 🐙🕸️ GitWikiWeb ∷ build.js 
   2 // ==================================================================== 
   4 // Copyright © 2023 Lady [@ Lady’s Computer]. 
   6 // This Source Code Form is subject to the terms of the Mozilla Public 
   7 // License, v. 2.0. If a copy of the MPL was not distributed with this 
   8 // file, You can obtain one at <https://mozilla.org/MPL/2.0/>. 
  10 // -------------------------------------------------------------------- 
  12 // A script for generating static wiki pages from a git repository. 
  14 // First, clone this repository to your machine in an accessible 
  15 // location (for example, `/srv/git/GitWikiWeb`). A bare repository is 
  16 // fine; customize the templates and stylesheets as you like. Then, use 
  17 // this file as the post‐receive hook for your wiki as follows :— 
  19 //     #!/usr/bin/env -S sh 
  20 //     export GITWIKIWEB=/srv/git/GitWikiWeb 
  21 //     git archive --remote=$GITWIKIWEB HEAD build.js \ 
  23 //       | deno run -A - ~/public/wiki $GITWIKIWEB 
  25 // The directory `~/public/wiki` (or whatever you specify as the first 
  26 // argument to `deno run -A -`) **will be deleted** and a new static 
  27 // wiki will be generated in its place. This script is not very smart 
  28 // (yet) and cannot selectively determine which pages will need 
  29 // updating. It just wipes and regenerates the whole thing. 
  31 // This script will make a number of requests to `$GITWIKIWEB` to 
  32 // download the latest templates, stylesheets, ⁊·c from this 
  33 // repository. Consequently, it is best that you set it to a repository 
  34 // you control and can ensure uptime for—ideally one local to the 
  35 // computer hosting the wiki. 
  40 } from "https://deno.land/std@0.196.0/fs/mod.ts"; 
  44 } from "https://deno.land/std@0.196.0/yaml/mod.ts"; 
  45 import djot 
from "npm:@djot/djot@0.2.3"; 
  46 import { Parser 
} from "npm:htmlparser2@9.0.0"; 
  47 import { DomHandler
, Element
, Text 
} from "npm:domhandler@5.0.3"; 
  48 import * as domutils 
from "npm:domutils@3.1.0"; 
  49 import domSerializer 
from "npm:dom-serializer@2.0.0"; 
  51 const DESTINATION 
= Deno
.args
[0] ?? "~/public/wiki"; 
  52 const REMOTE 
= Deno
.args
[1] ?? "/srv/git/GitWikiWeb"; 
  60 const NIL 
= Object
.preventExtensions(Object
.create(null)); 
  62 const rawBlock 
= (strings
, ...substitutions
) => ({ 
  65   text: String
.raw(strings
, substitutions
), 
  67 const rawInline 
= (strings
, ...substitutions
) => ({ 
  70   text: String
.raw(strings
, substitutions
), 
  72 const str 
= (strings
, ...substitutions
) => ({ 
  74   text: String
.raw(strings
, substitutions
), 
  77 const getDOM 
= (source
) => { 
  79   const handler 
= new DomHandler((error
, dom
) => { 
  81       throw new Error("GitWikiWeb: Failed to process DOM.", { 
  88   const parser 
= new Parser(handler
); 
  94 const getRemoteContent 
= async (pathName
) => { 
  95   const getArchive 
= new Deno
.Command("git", { 
  96     args: ["archive", `--remote=${REMOTE}`, "HEAD", pathName
], 
 100   const untar 
= new Deno
.Command("tar", { 
 106   getArchive
.stdout
.pipeTo(untar
.stdin
); 
 113   ] = await Promise
.allSettled([ 
 114     new Response(getArchive
.stderr
).text(), 
 116     new Response(untar
.stdout
).text(), 
 117     new Response(untar
.stderr
).text(), 
 119   ]).then(logErrorsAndCollectResults
); 
 121     console
.error(err1 
+ err2
); 
 125   if (!getArchiveStatus
.success
) { 
 127       `GitWikiWeb: git archive returned nonzero exit code: ${getArchiveStatus.code}.`, 
 129   } else if (!untarStatus
.success
) { 
 131       `GitWikiWeb: tar returned nonzero exit code: ${untarStatus.code}.`, 
 138 const logErrorsAndCollectResults 
= (results
) => 
 139   results
.map(({ value
, reason 
}) => { 
 141       console
.error(reason
); 
 148 const getReferenceFromPath 
= (path
) => 
 149   /Sources\/([A-Z][0-9A-Za-z]*\/[A-Z][0-9A-Za-z]*)\.djot$/u.exec(path
) 
 150     ?.[1]?.replace
?.("/", ":"); 
 152 const listOfInternalLinks 
= (references
, wrapper 
= ($) => $) => ({ 
 156   children: Array
.from( 
 159       const [namespace, pageName
] = splitReference(reference
); 
 167               "data-realm": "internal", 
 168               "data-pagename": pageName
, 
 169               "data-namespace": namespace, 
 180 const diffReferences 
= async (hash
, againstHead 
= false) => { 
 181   const diff 
= new Deno
.Command("git", { 
 189       ...(againstHead 
? [hash
, "HEAD"] : [hash
]), 
 194   const [diffNames
] = await Promise
.allSettled([ 
 195     new Response(diff
.stdout
).text(), 
 196     new Response(diff
.stderr
).text(), 
 197   ]).then(logErrorsAndCollectResults
); 
 198   return references(diffNames
.split("\0")); // returns an iterable 
 201 function* references(paths
) { 
 202   for (const path 
of paths
) { 
 203     const reference 
= getReferenceFromPath(path
); 
 212 const splitReference 
= (reference
) => { 
 213   const colonIndex 
= reference
.indexOf(":"); 
 215     reference
.substring(0, colonIndex
), 
 216     reference
.substring(colonIndex 
+ 1), 
 220 class GitWikiWebPage 
{ 
 221   #internalLinks 
= new Set(); 
 222   #externalLinks 
= new Map(); 
 224   constructor(namespace, name
, ast
, source
, config
) { 
 225     const internalLinks 
= this.#internalLinks
; 
 226     const externalLinks 
= this.#externalLinks
; 
 227     const sections 
= Object
.create(null); 
 228     djot
.applyFilter(ast
, () => { 
 229       let titleSoFar 
= null; // used to collect strs from headings 
 234             const links_section 
= []; 
 235             if (internalLinks
.size 
|| externalLinks
.size
) { 
 238                 rawBlock
`<nav id="links">`, 
 242                   children: [str
`this page contains links`], 
 245               if (internalLinks
.size
) { 
 247                   rawBlock
`<details open="">`, 
 248                   rawBlock
`<summary>on this wiki</summary>`, 
 249                   listOfInternalLinks(internalLinks
), 
 250                   rawBlock
`</details>`, 
 255               if (externalLinks
.size
) { 
 257                   rawBlock
`<details open="">`, 
 258                   rawBlock
`<summary>elsewhere on the Web</summary>`, 
 263                     children: Array
.from( 
 265                       ([destination
, text
]) => ({ 
 271                             attributes: { "data-realm": "external" }, 
 289                   rawBlock
`</details>`, 
 301             e
.children
.push(...links_section
); 
 307             const attributes 
= e
.attributes 
?? NIL
; 
 308             const { as 
} = attributes
; 
 310               delete attributes
.as
; 
 312                 as 
== "b" || as 
== "cite" || as 
== "i" || as 
== "u" 
 329             if (titleSoFar 
!= null) { 
 343             const { attributes 
} = e
; 
 344             attributes
.title 
??= titleSoFar
; 
 351             const { attributes
, reference
, destination 
} = e
; 
 353               /^(?:[A-Z][0-9A-Za-z]*|[@#])?:(?:[A-Z][0-9A-Za-z]*)?$/u 
 354                 .test(reference 
?? "") 
 356               const [namespacePrefix
, pageName
] = splitReference( 
 359               const expandedNamespace 
= { 
 363               }[namespacePrefix
] ?? namespacePrefix
; 
 364               const resolvedReference 
= pageName 
== "" 
 365                 ? `Namespace:${expandedNamespace}` 
 366                 : `${expandedNamespace}:${pageName}`; 
 367               this.#internalLinks
.add(resolvedReference
); 
 368               e
.reference 
= resolvedReference
; 
 369               attributes
["data-realm"] = "internal"; 
 370               attributes
["data-pagename"] = pageName
; 
 371               attributes
["data-namespace"] = expandedNamespace
; 
 373               attributes
["data-realm"] = "external"; 
 374               const remote 
= destination 
?? 
 375                 ast
.references
[reference
]?.destination
; 
 377                 externalLinks
.set(remote
, attributes
?.title
); 
 384         non_breaking_space: { 
 386             if (titleSoFar 
!= null) { 
 387               titleSoFar 
+= "\xA0"; 
 398             const { attributes
, children 
} = e
; 
 399             const heading 
= children
.find(({ tag 
}) => 
 402             const title 
= (() => { 
 403               if (heading
?.attributes
?.title
) { 
 404                 const result 
= heading
.attributes
.title
; 
 405                 delete heading
.attributes
.title
; 
 408                 return heading
.level 
== 1 
 409                   ? `${namespace}:${name}` 
 410                   : "untitled section"; 
 413             const variantTitles 
= Object
.create(null); 
 414             for (const attr 
in attributes
) { 
 415               if (attr
.startsWith("v-")) { 
 416                 Object
.defineProperty( 
 419                   { ...READ_ONLY
, value: attributes
[attr
] }, 
 421                 delete attributes
[attr
]; 
 426             const definition 
= Object
.create(null, { 
 427               title: { ...READ_ONLY
, value: title 
}, 
 430                 value: Object
.preventExtensions(variantTitles
), 
 433             if (heading
.level 
== 1 && !("main" in sections
)) { 
 434               attributes
.id 
= "main"; 
 435               heading
.attributes 
??= {}; 
 436               heading
.attributes
.class = "main"; 
 441               Object
.defineProperty( 
 447                   value: Object
.preventExtensions(definition
), 
 452                 `GitWikiWeb: A section with the provided @id already exists: ${attributes.id}`, 
 460             if (titleSoFar 
!= null) { 
 469           enter: ({ text 
}) => { 
 470             if (titleSoFar 
!= null) { 
 481             const codepoint 
= /^U\+([0-9A-Fa-f]+)$/u.exec(alias
)?.[1]; 
 484                 String.fromCodePoint(parseInt(codepoint, 16)) 
 487               const resolved 
= config
.symbols
?.[alias
]; 
 488               return resolved 
!= null ? str
`${resolved}` : e
; 
 494     Object
.defineProperties(this, { 
 495       ast: { ...READ_ONLY
, value: ast 
}, 
 496       namespace: { ...READ_ONLY
, value: namespace }, 
 497       name: { ...READ_ONLY
, value: name 
}, 
 500         value: Object
.preventExtensions(sections
), 
 502       source: { ...READ_ONLY
, value: source 
}, 
 507     yield* this.#externalLinks
; 
 511     yield* this.#internalLinks
; 
 516   const config 
= await 
getRemoteContent("config.yaml").then((yaml
) => 
 517     parseYaml(yaml
, { schema: JSON_SCHEMA 
}) 
 519   const ls 
= new Deno
.Command("git", { 
 520     args: ["ls-tree", "-rz", "live"], 
 528   ] = await Promise
.allSettled([ 
 529     new Response(ls
.stdout
).text().then((lsout
) => 
 532         .slice(0, -1) // drop the last entry; it is empty 
 533         .map(($) => $.split(/\s+/g)) 
 535     new Response(ls
.stderr
).text(), 
 537   ]).then(logErrorsAndCollectResults
); 
 539     console
.error(lserr
); 
 543   if (!lsstatus
.success
) { 
 545       `GitWikiWeb: git ls-tree returned nonzero exit code: ${lsstatus.code}.`, 
 548     const requiredButMissingPages 
= new Map([ 
 549       ["Special:FrontPage", "front page"], 
 550       ["Special:NotFound", "not found"], 
 551       ["Special:RecentlyChanged", "recently changed"], 
 553     const pages 
= new Map(); 
 554     const promises 
= [emptyDir(DESTINATION
)]; 
 555     for (const object 
of objects
) { 
 556       const hash 
= object
[2]; 
 557       const path 
= object
[3]; 
 558       const reference 
= getReferenceFromPath(path
); 
 559       if (reference 
== null) { 
 562         const [namespace, pageName
] = splitReference(reference
); 
 563         const cat 
= new Deno
.Command("git", { 
 564           args: ["cat-file", "blob", hash
], 
 568         const promise 
= Promise
.allSettled([ 
 569           new Response(cat
.stdout
).text(), 
 570           new Response(cat
.stderr
).text(), 
 572         ]).then(logErrorsAndCollectResults
).then( 
 573           ([source
, caterr
, catstatus
]) => { 
 575               console
.error(caterr
); 
 579             if (!catstatus
.success
) { 
 581                 `GitWikiWeb: git cat-file returned nonzero exit code: ${catstatus.code}.`, 
 584               const page 
= new GitWikiWebPage( 
 589                     console
.warn(`Djot(${reference}): ${$.render()}`), 
 594               const reference 
= `${namespace}:${pageName}`; 
 595               pages
.set(reference
, page
); 
 596               requiredButMissingPages
.delete(reference
); 
 600         promises
.push(promise
); 
 603     for (const [reference
, defaultTitle
] of requiredButMissingPages
) { 
 604       const [namespace, pageName
] = splitReference(reference
); 
 605       const source 
= `# ${defaultTitle}\n`; 
 606       const page 
= new GitWikiWebPage( 
 611             console
.warn(`Djot(${reference}): ${$.render()}`), 
 616       pages
.set(reference
, page
); 
 618     await Promise
.allSettled(promises
).then( 
 619       logErrorsAndCollectResults
, 
 621     const [template
, recentlyChanged
] = await Promise
.allSettled([ 
 622       getRemoteContent("template.html"), 
 624         const dateParse 
= new Deno
.Command("git", { 
 625           args: ["rev-parse", "--after=1 week ago"], 
 629         const [maxAge
] = await Promise
.allSettled([ 
 630           new Response(dateParse
.stdout
).text(), 
 631           new Response(dateParse
.stderr
).text(), 
 632         ]).then(logErrorsAndCollectResults
); 
 637           const revList 
= new Deno
.Command("git", { 
 638             args: ["rev-list", maxAge
, "--reverse", "HEAD"], 
 642           [commit
] = await Promise
.allSettled([ 
 643             new Response(revList
.stdout
).text().then((list
) => 
 646             new Response(revList
.stderr
).text(), 
 647           ]).then(logErrorsAndCollectResults
); 
 650           const revList2 
= new Deno
.Command("git", { 
 651             args: ["rev-list", "--max-count=1", "HEAD^"], 
 655           [commit
] = await Promise
.allSettled([ 
 656             new Response(revList2
.stdout
).text().then((list
) => 
 659             new Response(revList2
.stderr
).text(), 
 660           ]).then(logErrorsAndCollectResults
); 
 664         const results 
= new Array(6); 
 665         const seen 
= new Set(); 
 669           const show 
= new Deno
.Command("git", { 
 673               "--format=%H%x00%cI%x00%cD", 
 674               recency 
? `HEAD~${5 - recency}` : commit
, 
 680             [hash
, dateTime
, humanReadable
], 
 681           ] = await Promise
.allSettled([ 
 682             new Response(show
.stdout
).text().then((rev
) => 
 683               rev
.trim().split("\0") 
 685             new Response(show
.stderr
).text(), 
 686           ]).then(logErrorsAndCollectResults
); 
 690             const ref 
of (await 
diffReferences(current
, !recency
)) 
 699           results
[recency
] = { dateTime
, hash
, humanReadable
, refs 
}; 
 700         } while (recency
-- > 0 && current 
&& current 
!= commit
); 
 705         (name
) => ensureDir(`${DESTINATION}/${name}`), 
 707       ["style.css"].map((dependency
) => 
 708         getRemoteContent(dependency
).then((source
) => 
 710             `${DESTINATION}/${dependency}`, 
 716     ]).then(logErrorsAndCollectResults
); 
 718     const redLinks 
= (() => { 
 719       const result 
= new Set(); 
 720       for (const page 
of pages
.values()) { 
 721         for (const link 
of page
.internalLinks()) { 
 722           if (pages
.has(link
)) { 
 732       const [pageRef
, { ast
, namespace, sections
, source 
}] of pages
 
 734       const title 
= sections
.main
?.title 
?? pageRef
; 
 735       djot
.applyFilter(ast
, () => { 
 736         let isNavigationPage 
= true; 
 740               const { content
, navigation 
} = (() => { 
 741                 const navigation 
= []; 
 742                 if (pageRef 
== "Special:RecentlyChanged") { 
 745                     attributes: { class: "recent-changes" }, 
 748                     children: Array
.from(function* () { 
 750                         const [index
, result
] of recentlyChanged
 
 753                         if (result 
!= null) { 
 759                           yield* listOfInternalLinks(refs
, (link
) => ({ 
 760                             tag: index 
== 0 ? "span" : "strong", 
 761                             attributes: { "data-recency": `${index}` }, 
 764                               ...(index 
== 0 ? [] : [ 
 766                                 rawInline
`<small>(<time dateTime="${dateTime}">`, 
 767                                 str
`${humanReadable}`, 
 768                                 rawInline
`</time>)</small>`, 
 779                   isNavigationPage 
= false; 
 780                   return { content: e
.children
, navigation 
}; 
 788                         generated: "", // will be removed later 
 791                       children: [str
`${title}`], 
 793                     rawBlock
`<details id="navigation-about" open="">`, 
 794                     rawBlock
`<summary>about this listing</summary>`, 
 797                     rawBlock
`</article>`, 
 798                     rawBlock
`</details>`, 
 801                     rawBlock
`<nav id="navigation">`, 
 811                 rawBlock
`</article>`, 
 818               const attributes 
= e
.attributes 
?? NIL
; 
 820                 isNavigationPage 
&& e
.level 
== 1 && 
 821                 attributes
?.class == "main" 
 823                 if ("generated" in attributes
) { 
 824                   delete attributes
.generated
; 
 833               if (e
.level 
== 1 && e
.attributes
?.class == "main") { 
 835                   rawBlock
`<header class="main">`, 
 837                   { tag: "verbatim", text: pageRef 
}, 
 849               const { attributes
, children
, reference 
} = e
; 
 850               if (attributes
["data-realm"] == "internal") { 
 852                 if (redLinks
.has(reference
)) { 
 854                     `/Special:NotFound?path=/${reference}`; 
 855                   attributes
["data-notfound"] = ""; 
 857                   e
.destination 
= `/${reference}`; 
 859                 if (children
.length 
== 0) { 
 861                     pages
.get(reference
)?.sections
?.main 
?? NIL
; 
 862                   const { v 
} = attributes
; 
 865                       str
`${section.title ?? reference}`, 
 871                         section.variantTitles?.[v] ?? section.title ?? 
 878                 if (children
.length 
== 0 && "title" in attributes
) { 
 881                     str
`${attributes.title}`, 
 887                 (attributes
.class ?? "").split(/\s/gu).includes("sig") 
 891                   attributes: { class: "sig" }, 
 892                   children: [str
`—${"\xA0"}`, e
], 
 902               if (e
.children
.length 
< 1) { 
 903                 // The heading for this section was removed and it had 
 904                 // no other children. 
 913       const doc 
= getDOM(template
); 
 914       const result 
= getDOM(`${djot.renderHTML(ast)}`); 
 915       const headElement 
= domutils
.findOne( 
 916         (node
) => node
.name 
== "head", 
 919       const titleElement 
= domutils
.findOne( 
 920         (node
) => node
.name 
== "title", 
 923       const contentElement 
= domutils
.findOne( 
 924         (node
) => node
.name 
== "gitwikiweb-content", 
 927       if (headElement 
== null) { 
 929           "GitWikiWeb: Template must explicitly include a <head> element.", 
 932         domutils
.appendChild( 
 934           new Element("link", { 
 937             href: `/${pageRef}/source.djot`, 
 940         if (titleElement 
== null) { 
 941           domutils
.prependChild( 
 943             new Element("title", {}, [new Text(title
)]), 
 946           domutils
.prependChild(titleElement
, new Text(`${title} | `)); 
 949       if (contentElement 
== null) { 
 951           "GitWikiWeb: Template did not include a <gitwikiweb-content> element.", 
 954         for (const node 
of result
) { 
 955           domutils
.prepend(contentElement
, node
); 
 957         domutils
.removeElement(contentElement
); 
 961           `${DESTINATION}/${pageRef}/index.html`, 
 964             encodeEntities: "utf8", 
 965             selfClosingTags: true, 
 973           `${DESTINATION}/${pageRef}/source.djot`, 
 979     await Promise
.allSettled(promises
).then( 
 980       logErrorsAndCollectResults
, 
 982     console
.log(`GitWikiWeb: Wrote ${pages.size} page(s).`);