]> Lady’s Gitweb - GitWikiWeb/blob - build.js
da0f78391777e5bf058a43030e5ff13b0f539919
[GitWikiWeb] / build.js
1 // 🐙🕸️ GitWikiWeb ∷ build.js
2 // ====================================================================
3 //
4 // Copyright © 2023 Lady [@ Lady’s Computer].
5 //
6 // This Source Code Form is subject to the terms of the Mozilla Public
7 // License, v. 2.0. If a copy of the MPL was not distributed with this
8 // file, You can obtain one at <https://mozilla.org/MPL/2.0/>.
9 //
10 // --------------------------------------------------------------------
11 //
12 // A script for generating static wiki pages from a git repository.
13 //
14 // First, clone this repository to your machine in an accessible
15 // location (for example, `/srv/git/GitWikiWeb`). A bare repository is
16 // fine; customize the templates and stylesheets as you like. Then, use
17 // this file as the post‐receive hook for your wiki as follows :—
18 //
19 // #!/usr/bin/env -S sh
20 // export GITWIKIWEB=/srv/git/GitWikiWeb
21 // git archive --remote=$GITWIKIWEB HEAD build.js \
22 // | tar -xO \
23 // | deno run -A - ~/public/wiki $GITWIKIWEB
24 //
25 // The directory `~/public/wiki` (or whatever you specify as the first
26 // argument to `deno run -A -`) **will be deleted** and a new static
27 // wiki will be generated in its place. This script is not very smart
28 // (yet) and cannot selectively determine which pages will need
29 // updating. It just wipes and regenerates the whole thing.
30 //
31 // This script will make a number of requests to `$GITWIKIWEB` to
32 // download the latest templates, stylesheets, ⁊·c from this
33 // repository. Consequently, it is best that you set it to a repository
34 // you control and can ensure uptime for—ideally one local to the
35 // computer hosting the wiki.
36
37 import {
38 emptyDir,
39 ensureDir,
40 } from "https://deno.land/std@0.195.0/fs/mod.ts";
41 import djot from "npm:@djot/djot@0.2.3";
42 import { Parser } from "npm:htmlparser2@9.0.0";
43 import { DomHandler, Element, Text } from "npm:domhandler@5.0.3";
44 import * as domutils from "npm:domutils@3.1.0";
45 import domSerializer from "npm:dom-serializer@2.0.0";
46
47 const DESTINATION = Deno.args[0] ?? "~/public/wiki";
48 const REMOTE = Deno.args[1] ?? "/srv/git/GitWikiWeb";
49
50 const READ_ONLY = {
51 configurable: false,
52 enumerable: true,
53 writable: false,
54 };
55
56 const rawBlock = (strings, ...substitutions) => ({
57 tag: "raw_block",
58 format: "html",
59 text: String.raw(strings, substitutions),
60 });
61 const rawInline = (strings, ...substitutions) => ({
62 tag: "raw_inline",
63 format: "html",
64 text: String.raw(strings, substitutions),
65 });
66 const str = (strings, ...substitutions) => ({
67 tag: "str",
68 text: String.raw(strings, substitutions),
69 });
70
71 const getDOM = (source) => {
72 let result;
73 const handler = new DomHandler((error, dom) => {
74 if (error) {
75 throw new Error("GitWikiWeb: Failed to process DOM.", {
76 cause: error,
77 });
78 } else {
79 result = dom;
80 }
81 });
82 const parser = new Parser(handler);
83 parser.write(source);
84 parser.end();
85 return result;
86 };
87
88 const getRemoteContent = async (pathName) => {
89 const getArchive = new Deno.Command("git", {
90 args: ["archive", `--remote=${REMOTE}`, "HEAD", pathName],
91 stdout: "piped",
92 stderr: "piped",
93 }).spawn();
94 const untar = new Deno.Command("tar", {
95 args: ["-xO"],
96 stdin: "piped",
97 stdout: "piped",
98 stderr: "piped",
99 }).spawn();
100 getArchive.stdout.pipeTo(untar.stdin);
101 const [
102 err1,
103 getArchiveStatus,
104 result,
105 err2,
106 untarStatus,
107 ] = await Promise.allSettled([
108 new Response(getArchive.stderr).text(),
109 getArchive.status,
110 new Response(untar.stdout).text(),
111 new Response(untar.stderr).text(),
112 untar.status,
113 ]).then(logErrorsAndCollectResults);
114 if (err1 || err2) {
115 console.error(err1 + err2);
116 } else {
117 /* do nothing */
118 }
119 if (!getArchiveStatus.success) {
120 throw new Error(
121 `GitWikiWeb: git archive returned nonzero exit code: ${getArchiveStatus.code}.`,
122 );
123 } else if (!untarStatus.success) {
124 throw new Error(
125 `GitWikiWeb: tar returned nonzero exit code: ${untarStatus.code}.`,
126 );
127 } else {
128 return result || "";
129 }
130 };
131
132 const logErrorsAndCollectResults = (results) =>
133 results.map(({ value, reason }) => {
134 if (reason) {
135 console.error(reason);
136 return undefined;
137 } else {
138 return value;
139 }
140 });
141
142 const getReferenceFromPath = (path) =>
143 /Sources\/([A-Z][0-9A-Za-z]*\/[A-Z][0-9A-Za-z]*)\.djot$/u.exec(path)
144 ?.[1]?.replace?.("/", ":");
145
146 const listOfInternalLinks = (references) => ({
147 tag: "bullet_list",
148 tight: true,
149 style: "*",
150 children: Array.from(
151 references,
152 (reference) => {
153 const [namespace, pageName] = splitReference(reference);
154 return {
155 tag: "list_item",
156 children: [{
157 tag: "para",
158 children: [{
159 tag: "link",
160 attributes: {
161 "data-realm": "internal",
162 "data-pagename": pageName,
163 "data-namespace": namespace,
164 },
165 reference,
166 children: [],
167 }],
168 }],
169 };
170 },
171 ),
172 });
173
174 function* references(paths) {
175 for (const path of paths) {
176 const reference = getReferenceFromPath(path);
177 if (reference) {
178 yield reference;
179 } else {
180 /* do nothing */
181 }
182 }
183 }
184
185 const splitReference = (reference) => {
186 const colonIndex = reference.indexOf(":");
187 return [
188 reference.substring(0, colonIndex),
189 reference.substring(colonIndex + 1),
190 ];
191 };
192
193 class GitWikiWebPage {
194 #internalLinks = new Set();
195 #externalLinks = new Map();
196
197 constructor(namespace, name, ast, source) {
198 const internalLinks = this.#internalLinks;
199 const externalLinks = this.#externalLinks;
200 const sections = Object.create(null);
201 djot.applyFilter(ast, () => {
202 let titleSoFar = null; // used to collect strs from headings
203 return {
204 doc: {
205 enter: (_) => {},
206 exit: (e) => {
207 const links_section = [];
208 if (internalLinks.size || externalLinks.size) {
209 links_section.push(
210 rawBlock`<footer>`,
211 rawBlock`<nav id="links">`,
212 {
213 tag: "heading",
214 level: 2,
215 children: [str`this page contains links`],
216 },
217 );
218 if (internalLinks.size) {
219 links_section.push(
220 rawBlock`<details>`,
221 rawBlock`<summary>on this wiki</summary>`,
222 listOfInternalLinks(internalLinks),
223 rawBlock`</details>`,
224 );
225 } else {
226 /* do nothing */
227 }
228 if (externalLinks.size) {
229 links_section.push(
230 rawBlock`<details>`,
231 rawBlock`<summary>elsewhere on the Web</summary>`,
232 {
233 tag: "bullet_list",
234 tight: true,
235 style: "*",
236 children: Array.from(
237 externalLinks,
238 ([destination, text]) => ({
239 tag: "list_item",
240 children: [{
241 tag: "para",
242 children: [{
243 tag: "link",
244 attributes: { "data-realm": "external" },
245 destination,
246 children: text
247 ? [
248 rawInline`<cite>`,
249 str`${text}`,
250 rawInline`</cite>`,
251 ]
252 : [
253 rawInline`<code>`,
254 str`${destination}`,
255 rawInline`</code>`,
256 ],
257 }],
258 }],
259 }),
260 ),
261 },
262 rawBlock`</details>`,
263 );
264 } else {
265 /* do nothing */
266 }
267 links_section.push(
268 rawBlock`</nav>`,
269 rawBlock`</footer>`,
270 );
271 } else {
272 /* do nothing */
273 }
274 e.children.push(...links_section);
275 },
276 },
277 hard_break: {
278 enter: (_) => {
279 if (titleSoFar != null) {
280 titleSoFar += " ";
281 } else {
282 /* do nothing */
283 }
284 },
285 exit: (_) => {},
286 },
287 heading: {
288 enter: (_) => {
289 titleSoFar = "";
290 },
291 exit: (e) => {
292 e.attributes ??= {};
293 const { attributes } = e;
294 attributes.title ??= titleSoFar;
295 titleSoFar = null;
296 },
297 },
298 link: {
299 enter: (e) => {
300 e.attributes ??= {};
301 const { attributes, reference, destination } = e;
302 if (
303 /^(?:[A-Z][0-9A-Za-z]*|[@#])?:(?:[A-Z][0-9A-Za-z]*)?$/u
304 .test(reference ?? "")
305 ) {
306 const [namespacePrefix, pageName] = splitReference(
307 reference,
308 );
309 const expandedNamespace = {
310 "": "Page",
311 "@": "Editor",
312 "#": "Category",
313 }[namespacePrefix] ?? namespacePrefix;
314 const resolvedReference = pageName == ""
315 ? `Namespace:${expandedNamespace}`
316 : `${expandedNamespace}:${pageName}`;
317 this.#internalLinks.add(resolvedReference);
318 e.reference = resolvedReference;
319 attributes["data-realm"] = "internal";
320 attributes["data-pagename"] = pageName;
321 attributes["data-namespace"] = expandedNamespace;
322 } else {
323 attributes["data-realm"] = "external";
324 const remote = destination ??
325 ast.references[reference]?.destination;
326 if (remote) {
327 externalLinks.set(remote, attributes?.title);
328 } else {
329 /* do nothing */
330 }
331 }
332 },
333 },
334 non_breaking_space: {
335 enter: (_) => {
336 if (titleSoFar != null) {
337 titleSoFar += "\xA0";
338 } else {
339 /* do nothing */
340 }
341 },
342 exit: (_) => {},
343 },
344 section: {
345 enter: (_) => {},
346 exit: (e) => {
347 e.attributes ??= {};
348 const { attributes, children } = e;
349 const heading = children.find(({ tag }) =>
350 tag == "heading"
351 );
352 const title = (() => {
353 if (heading?.attributes?.title) {
354 const result = heading.attributes.title;
355 delete heading.attributes.title;
356 return result;
357 } else {
358 return heading.level == 1
359 ? `${namespace}:${name}`
360 : "untitled section";
361 }
362 })();
363 const variantTitles = Object.create(null);
364 for (const attr in attributes) {
365 if (attr.startsWith("v-")) {
366 Object.defineProperty(
367 variantTitles,
368 attr.substring(2),
369 { ...READ_ONLY, value: attributes[attr] },
370 );
371 delete attributes[attr];
372 } else {
373 continue;
374 }
375 }
376 const definition = Object.create(null, {
377 title: { ...READ_ONLY, value: title },
378 variantTitles: {
379 ...READ_ONLY,
380 value: Object.preventExtensions(variantTitles),
381 },
382 });
383 if (heading.level == 1 && !("main" in sections)) {
384 attributes.id = "main";
385 heading.attributes ??= {};
386 heading.attributes.class = "main";
387 } else {
388 /* do nothing */
389 }
390 try {
391 Object.defineProperty(
392 sections,
393 attributes.id,
394 {
395 ...READ_ONLY,
396 writable: true,
397 value: Object.preventExtensions(definition),
398 },
399 );
400 } catch (cause) {
401 throw new Error(
402 `GitWikiWeb: A section with the provided @id already exists: ${attributes.id}`,
403 { cause },
404 );
405 }
406 },
407 },
408 soft_break: {
409 enter: (_) => {
410 if (titleSoFar != null) {
411 titleSoFar += " ";
412 } else {
413 /* do nothing */
414 }
415 },
416 exit: (_) => {},
417 },
418 str: {
419 enter: ({ text }) => {
420 if (titleSoFar != null) {
421 titleSoFar += text;
422 } else {
423 /* do nothing */
424 }
425 },
426 exit: (_) => {},
427 },
428 };
429 });
430 Object.defineProperties(this, {
431 ast: { ...READ_ONLY, value: ast },
432 namespace: { ...READ_ONLY, value: namespace },
433 name: { ...READ_ONLY, value: name },
434 sections: {
435 ...READ_ONLY,
436 value: Object.preventExtensions(sections),
437 },
438 source: { ...READ_ONLY, value: source },
439 });
440 }
441
442 *externalLinks() {
443 yield* this.#externalLinks;
444 }
445
446 *internalLinks() {
447 yield* this.#internalLinks;
448 }
449 }
450
451 {
452 const ls = new Deno.Command("git", {
453 args: ["ls-tree", "-rz", "live"],
454 stdout: "piped",
455 stderr: "piped",
456 }).spawn();
457 const [
458 objects,
459 lserr,
460 lsstatus,
461 ] = await Promise.allSettled([
462 new Response(ls.stdout).text().then((lsout) =>
463 lsout
464 .split("\0")
465 .slice(0, -1) // drop the last entry; it is empty
466 .map(($) => $.split(/\s+/g))
467 ),
468 new Response(ls.stderr).text(),
469 ls.status,
470 ]).then(logErrorsAndCollectResults);
471 if (lserr) {
472 console.error(lserr);
473 } else {
474 /* do nothing */
475 }
476 if (!lsstatus.success) {
477 throw new Error(
478 `GitWikiWeb: git ls-tree returned nonzero exit code: ${lsstatus.code}.`,
479 );
480 } else {
481 const requiredButMissingPages = new Map([
482 ["Special:FrontPage", "front page"],
483 ["Special:NotFound", "not found"],
484 ["Special:RecentlyChanged", "recently changed"],
485 ]);
486 const pages = new Map();
487 const promises = [emptyDir(DESTINATION)];
488 for (const object of objects) {
489 const hash = object[2];
490 const path = object[3];
491 const reference = getReferenceFromPath(path);
492 if (reference == null) {
493 continue;
494 } else {
495 const [namespace, pageName] = splitReference(reference);
496 const cat = new Deno.Command("git", {
497 args: ["cat-file", "blob", hash],
498 stdout: "piped",
499 stderr: "piped",
500 }).spawn();
501 const promise = Promise.allSettled([
502 new Response(cat.stdout).text(),
503 new Response(cat.stderr).text(),
504 cat.status,
505 ]).then(logErrorsAndCollectResults).then(
506 ([source, caterr, catstatus]) => {
507 if (caterr) {
508 console.error(caterr);
509 } else {
510 /* do nothing */
511 }
512 if (!catstatus.success) {
513 throw new Error(
514 `GitWikiWeb: git cat-file returned nonzero exit code: ${catstatus.code}.`,
515 );
516 } else {
517 const page = new GitWikiWebPage(
518 namespace,
519 pageName,
520 djot.parse(source, {
521 warn: ($) =>
522 console.warn(`Djot(${reference}): ${$.render()}`),
523 }),
524 source,
525 );
526 const reference = `${namespace}:${pageName}`;
527 pages.set(reference, page);
528 requiredButMissingPages.delete(reference);
529 }
530 },
531 );
532 promises.push(promise);
533 }
534 }
535 for (const [reference, defaultTitle] of requiredButMissingPages) {
536 const [namespace, pageName] = splitReference(reference);
537 const source = `# ${defaultTitle}\n`;
538 const page = new GitWikiWebPage(
539 namespace,
540 pageName,
541 djot.parse(source, {
542 warn: ($) =>
543 console.warn(`Djot(${reference}): ${$.render()}`),
544 }),
545 source,
546 );
547 pages.set(reference, page);
548 }
549 await Promise.allSettled(promises).then(
550 logErrorsAndCollectResults,
551 );
552 const [template, recentlyChanged] = await Promise.allSettled([
553 getRemoteContent("template.html"),
554 (async () => {
555 const dateParse = new Deno.Command("git", {
556 args: ["rev-parse", "--after=1 week ago"],
557 stdout: "piped",
558 stderr: "piped",
559 }).spawn();
560 const [maxAge] = await Promise.allSettled([
561 new Response(dateParse.stdout).text(),
562 new Response(dateParse.stderr).text(),
563 ]).then(logErrorsAndCollectResults);
564 let commit;
565 if (!maxAge) {
566 /* do nothing */
567 } else {
568 const revList = new Deno.Command("git", {
569 args: ["rev-list", maxAge, "--reverse", "HEAD"],
570 stdout: "piped",
571 stderr: "piped",
572 }).spawn();
573 [commit] = await Promise.allSettled([
574 new Response(revList.stdout).text().then((list) =>
575 list.split("\n")[0]
576 ),
577 new Response(revList.stderr).text(),
578 ]).then(logErrorsAndCollectResults);
579 }
580 if (!commit) {
581 const revList2 = new Deno.Command("git", {
582 args: ["rev-list", "--max-count=1", "HEAD^"],
583 stdout: "piped",
584 stderr: "piped",
585 }).spawn();
586 [commit] = await Promise.allSettled([
587 new Response(revList2.stdout).text().then((list) =>
588 list.trim()
589 ),
590 new Response(revList2.stderr).text(),
591 ]).then(logErrorsAndCollectResults);
592 } else {
593 /* do nothing */
594 }
595 const diff = new Deno.Command("git", {
596 args: [
597 "diff",
598 "-z",
599 "--name-only",
600 "--no-renames",
601 "--diff-filter=AM",
602 commit,
603 ],
604 stdout: "piped",
605 stderr: "piped",
606 }).spawn();
607 const [diffNames] = await Promise.allSettled([
608 new Response(diff.stdout).text(),
609 new Response(diff.stderr).text(),
610 ]).then(logErrorsAndCollectResults);
611 return [...references(diffNames.split("\0"))];
612 })(),
613 ...Array.from(
614 pages.keys(),
615 (name) => ensureDir(`${DESTINATION}/${name}`),
616 ),
617 ["style.css"].map((dependency) =>
618 getRemoteContent(dependency).then((source) =>
619 Deno.writeTextFile(
620 `${DESTINATION}/${dependency}`,
621 source,
622 { createNew: true },
623 )
624 )
625 ),
626 ]).then(logErrorsAndCollectResults);
627 promises.length = 0;
628 const redLinks = (() => {
629 const result = new Set();
630 for (const page of pages.values()) {
631 for (const link of page.internalLinks()) {
632 if (pages.has(link)) {
633 continue;
634 } else {
635 result.add(link);
636 }
637 }
638 }
639 return result;
640 })();
641 for (
642 const [pageRef, { ast, namespace, sections, source }] of pages
643 ) {
644 const title = sections.main?.title ?? pageRef;
645 djot.applyFilter(ast, () => {
646 let isNavigationPage = true;
647 return {
648 doc: {
649 enter: (e) => {
650 const { content, navigation } = (() => {
651 const navigation = [];
652 if (pageRef == "Special:RecentlyChanged") {
653 navigation.push(
654 listOfInternalLinks(recentlyChanged),
655 );
656 } else {
657 isNavigationPage = false;
658 return { content: e.children, navigation };
659 }
660 return {
661 content: [
662 {
663 tag: "heading",
664 attributes: {
665 class: "main",
666 generated: "", // will be removed later
667 },
668 level: 1,
669 children: [str`${title}`],
670 },
671 rawBlock`<details open="">`,
672 rawBlock`<summary>about this listing</summary>`,
673 ...e.children,
674 rawBlock`</details>`,
675 ],
676 navigation: [
677 rawBlock`<nav id="navigation">`,
678 ...navigation,
679 rawBlock`</nav>`,
680 ],
681 };
682 })();
683 e.children = [
684 rawBlock`<article>`,
685 ...content,
686 ...navigation,
687 rawBlock`</article>`,
688 ];
689 },
690 exit: (_) => {},
691 },
692 heading: {
693 enter: (e) => {
694 const attributes = e.attributes ?? Object.create(null);
695 if (
696 isNavigationPage && e.level == 1 &&
697 attributes?.class == "main"
698 ) {
699 if ("generated" in attributes) {
700 delete attributes.generated;
701 } else {
702 return { stop: [] };
703 }
704 } else {
705 /* do nothing */
706 }
707 },
708 exit: (e) => {
709 if (e.level == 1 && e.attributes?.class == "main") {
710 return [
711 rawBlock`<header class="main">`,
712 e,
713 { tag: "verbatim", text: pageRef },
714 rawBlock`</header>`,
715 ];
716 } else {
717 /* do nothing */
718 }
719 },
720 },
721 link: {
722 enter: (_) => {},
723 exit: (e) => {
724 e.attributes ??= {};
725 const { attributes, children, reference } = e;
726 if (attributes["data-realm"] == "internal") {
727 delete e.reference;
728 if (redLinks.has(reference)) {
729 e.destination = `/Special:NotFound?path=/${reference}`;
730 attributes["data-notfound"] = "";
731 } else {
732 e.destination = `/${reference}`;
733 }
734 if (children.length == 0) {
735 const section =
736 pages.get(reference)?.sections?.main ??
737 Object.create(null);
738 const { v } = attributes;
739 if (v == null) {
740 children.push(
741 str`${section.title ?? reference}`,
742 );
743 } else {
744 delete attributes.v;
745 children.push(
746 str`${
747 section.variantTitles?.[v] ?? section.title ??
748 reference
749 }`,
750 );
751 }
752 }
753 } else {
754 if (children.length == 0 && "title" in attributes) {
755 children.push(
756 rawInline`<cite>`,
757 str`${attributes.title}`,
758 rawInline`</cite>`,
759 );
760 }
761 }
762 if (
763 (attributes.class ?? "").split(/\s/gu).includes("sig")
764 ) {
765 return {
766 tag: "span",
767 attributes: { class: "sig" },
768 children: [str`—${"\xA0"}`, e],
769 };
770 } else {
771 /* do nothing */
772 }
773 },
774 },
775 section: {
776 enter: (_) => {},
777 exit: (e) => {
778 if (e.children.length < 1) {
779 // The heading for this section was removed and it had
780 // no other children.
781 return [];
782 } else {
783 /* do nothing */
784 }
785 },
786 },
787 };
788 });
789 const doc = getDOM(template);
790 const result = getDOM(`${djot.renderHTML(ast)}`);
791 const headElement = domutils.findOne(
792 (node) => node.name == "head",
793 doc,
794 );
795 const titleElement = domutils.findOne(
796 (node) => node.name == "title",
797 headElement,
798 );
799 const contentElement = domutils.findOne(
800 (node) => node.name == "gitwikiweb-content",
801 doc,
802 );
803 if (headElement == null) {
804 throw new Error(
805 "GitWikiWeb: Template must explicitly include a <head> element.",
806 );
807 } else {
808 domutils.appendChild(
809 headElement,
810 new Element("link", {
811 rel: "source",
812 type: "text/x.djot",
813 href: `/${pageRef}/source.djot`,
814 }),
815 );
816 if (titleElement == null) {
817 domutils.prependChild(
818 headElement,
819 new Element("title", {}, [new Text(title)]),
820 );
821 } else {
822 domutils.prependChild(titleElement, new Text(`${title} | `));
823 }
824 }
825 if (contentElement == null) {
826 throw new Error(
827 "GitWikiWeb: Template did not include a <gitwikiweb-content> element.",
828 );
829 } else {
830 for (const node of result) {
831 domutils.prepend(contentElement, node);
832 }
833 domutils.removeElement(contentElement);
834 }
835 promises.push(
836 Deno.writeTextFile(
837 `${DESTINATION}/${pageRef}/index.html`,
838 domSerializer(doc, {
839 emptyAttrs: true,
840 encodeEntities: "utf8",
841 selfClosingTags: true,
842 xmlMode: false,
843 }),
844 { createNew: true },
845 ),
846 );
847 promises.push(
848 Deno.writeTextFile(
849 `${DESTINATION}/${pageRef}/source.djot`,
850 source,
851 { createNew: true },
852 ),
853 );
854 }
855 await Promise.allSettled(promises).then(
856 logErrorsAndCollectResults,
857 );
858 console.log(`GitWikiWeb: Wrote ${pages.size} page(s).`);
859 }
860 }
This page took 0.163597 seconds and 3 git commands to generate.