Lady’s Gitweb - x_status_git/blob - post-receive

   1 #!/usr/bin/env python3
   2 from datetime import datetime as dt, timezone
   3 from glob import iglob
   4 from itertools import starmap
   5 import json
   6 from os import mkdir
   7 from os.path import exists
   8 from pathlib import Path
   9 import re
  10 from shutil import copy2, rmtree
  11 from subprocess import run
  12 from sys import stdin
  13 from warnings import warn
  14 from xml.dom import XHTML_NAMESPACE
  15 from xml.dom.minidom import getDOMImplementation, parseString
  16
  17 GIT_DIRECTORY = "/home/USERNAME/Status.git"
  18 BUILD_DIRECTORY = "/home/USERNAME/status.site.example/.build"
  19 PUBLIC_DIRECTORY = "/home/USERNAME/status.site.example/public"
  20 PUBLIC_URL = "https://status.site.example"
  21 LANG = "en"
  22 LIVE_BRANCH = "live"
  23
  24 UTC = timezone.utc
  25 CURRENT_DATETIME = f"{dt.now(UTC).replace(tzinfo=None).isoformat(timespec='seconds')}Z"
  26 ATOM_NAMESPACE = "http://www.w3.org/2005/Atom"
  27
  28 if stdin.read().split()[-1] == f"refs/heads/{LIVE_BRANCH}":
  29
  30         print(f"This is an update to the '{LIVE_BRANCH}' branch; regenerating site…")
  31
  32         # Set up the build directory.
  33         if exists(BUILD_DIRECTORY):
  34                 rmtree(BUILD_DIRECTORY)
  35         cloneresult = run(["git", "clone", "--local", "--branch", LIVE_BRANCH, GIT_DIRECTORY, BUILD_DIRECTORY], capture_output=True, encoding="utf-8")
  36         # if cloneresult.stderr:
  37                 # print(cloneresult.stderr)
  38         cloneresult.check_returncode()
  39
  40         # Set up various containers.
  41         irimap = {}
  42         months = {}
  43         topics = {}
  44
  45         # Create an XML representation of the provided status text.
  46         def statusxml (text, version="1.0"):
  47                 doc = getDOMImplementation().createDocument(None, "article", None)
  48                 articleElt = doc.documentElement
  49                 articleElt.setAttribute("xmlns", XHTML_NAMESPACE)
  50                 articleElt.setAttribute("lang", LANG)
  51                 for para in text.split("\n\n"):
  52                         paraElt = articleElt.appendChild(doc.createElement("p"))
  53                         for component in re.findall(r'<[a-z]+:[^\s]*>(?:="[^\n"]+")?|\n|[^<\n]+|<(?![a-z]+:[^\s]*>)', para):
  54                                 if component == "\n":
  55                                         paraElt.appendChild(doc.createElement("br"))
  56                                 elif re.fullmatch(r'<[a-z]+:[^\s]*>(?:="[^\n"]+")?', component):
  57                                         href = component.split(">", maxsplit=1)[0][1:]
  58                                         anchorElt = paraElt.appendChild(doc.createElement("a"))
  59                                         anchorElt.setAttribute("href", href)
  60                                         anchorElt.setAttribute("rel", "noreferrer")
  61                                         anchorElt.appendChild(doc.createTextNode(component if len(href) == len(component) - 2 else component[len(href)+4:-1]))
  62                                 else:
  63                                         paraElt.appendChild(doc.createTextNode(component))
  64                 return articleElt.toxml()
  65
  66         # Map status paths to status objects, or None if there is an error.
  67         #
  68         # The provided path must be to a `text` object.
  69         def statusmap (topic, path):
  70                 status = { "@type": "MicroblogPost" }
  71                 version_path = next(path.parent.glob("0=*"))
  72                 if version_path and version_path.name != "0=x_status_git_1.0":
  73                         warn(f"Unrecognized version for {path}; skipping.")
  74                         return None
  75                 if topic:
  76                         status["subject"] = topic
  77                 author_path = next(path.parent.glob("1=*"))
  78                 if author_path:
  79                         status["author"] = { "name": author_path.name[2:] }
  80                         with author_path.open("r", encoding="utf-8") as text:
  81                                 status["author"]["@id"] = text.read().strip()
  82                 title_path = next(path.parent.glob("2=*"))
  83                 if title_path:
  84                         with title_path.open("r", encoding="utf-8") as text:
  85                                 title = text.read().strip()
  86                                 status["title"] = title
  87                 date_path = next(path.parent.glob("3=*"))
  88                 datetime = ""
  89                 if date_path:
  90                         with date_path.open("r", encoding="utf-8") as text:
  91                                 datetime = text.read().strip()
  92                                 status["created"] = datetime
  93                 else:
  94                         warn(f"Missing date for {path}; skipping.")
  95                         return None
  96                 identifier_path = next(path.parent.glob("4=*"))
  97                 identifier = ""
  98                 if identifier_path:
  99                         identifier = identifier_path.name[2:]
 100                         status["@id"] = f"{PUBLIC_URL}/topics/{topic}/{identifier}" if topic else f"{PUBLIC_URL}/statuses/{datetime[0:7]}/{identifier}"
 101                         with identifier_path.open("r", encoding="utf-8") as text:
 102                                 status["identifier"] = text.read().strip()
 103                         irimap[status["identifier"]] = status["@id"]
 104                 else:
 105                         warn(f"Missing identifier for {path}; skipping.")
 106                         return None
 107                 with path.open("r", encoding="utf-8") as text:
 108                         status["content"] = statusxml(text.read().strip())
 109                 return (datetime, identifier, status)
 110
 111         def atomForLD (ld):
 112                 doc = getDOMImplementation().createDocument(None, "feed", None)
 113                 atomElt = doc.documentElement
 114                 atomElt.setAttribute("xmlns", ATOM_NAMESPACE)
 115                 atomElt.setAttribute("xml:lang", LANG)
 116                 subject = ld["subject"] if "subject" in ld else "Statuses"
 117                 titleElt = atomElt.appendChild(doc.createElement("title"))
 118                 titleElt.appendChild(doc.createTextNode(f"{subject} @ {PUBLIC_URL}"))
 119                 updatedElt = atomElt.appendChild(doc.createElement("updated"))
 120                 updatedElt.appendChild(doc.createTextNode(CURRENT_DATETIME))
 121                 generatorElt = atomElt.appendChild(doc.createElement("generator"))
 122                 generatorElt.appendChild(doc.createTextNode("x_status_git"))
 123                 generatorElt.setAttribute("uri", "https://git.ladys.computer/x_status_git")
 124                 atomLinks = {}
 125                 if "OrderedCollectionPage" in ld["@type"]:
 126                         idElt = atomElt.appendChild(doc.createElement("id"))
 127                         idElt.appendChild(doc.createTextNode(f"{PUBLIC_URL}/statuses"))
 128                         atomLinks["alternate"] = f"{PUBLIC_URL}/statuses"
 129                         atomLinks["current"] = f"{PUBLIC_URL}/statuses.atom"
 130                         atomLinks["self"] = atomLinks["current"] if ld["@id"] == ld["current"] else f"{ld['@id']}.atom"
 131                         if "prev" in ld:
 132                                 atomLinks["prev-archive"] = f"{ld['prev']}.atom"
 133                         if "next" in ld and ld["next"] != ld["current"]:
 134                                 atomLinks["next-archive"] = f"{ld['next']}.atom"
 135                 else:
 136                         idElt = atomElt.appendChild(doc.createElement("id"))
 137                         idElt.appendChild(doc.createTextNode(ld["@id"]))
 138                         atomLinks["alternate"] = ld["@id"]
 139                         atomLinks["self"] = f"{ld['@id']}.atom"
 140                 for (rel, href) in atomLinks.items():
 141                         linkElt = atomElt.appendChild(doc.createElement("link"))
 142                         linkElt.setAttribute("rel", rel)
 143                         linkElt.setAttribute("href", href)
 144                 for item in ld["items"]:
 145                         entryElt = atomElt.appendChild(doc.createElement("entry"))
 146                         title = item["title"] if "title" in item else item["content"][0:27] + "…"
 147                         titleElt = entryElt.appendChild(doc.createElement("title"))
 148                         titleElt.appendChild(doc.createTextNode(title))
 149                         idElt = entryElt.appendChild(doc.createElement("id"))
 150                         idElt.appendChild(doc.createTextNode(item["@id"]))
 151                         updatedElt = entryElt.appendChild(doc.createElement("updated"))
 152                         updatedElt.appendChild(doc.createTextNode(CURRENT_DATETIME))
 153                         if "created" in item:
 154                                 publishedElt = entryElt.appendChild(doc.createElement("published"))
 155                                 publishedElt.appendChild(doc.createTextNode(item["created"]))
 156                         authorElt = entryElt.appendChild(doc.createElement("author"))
 157                         if "author" in item:
 158                                 nameElt = authorElt.appendChild(doc.createElement("name"))
 159                                 nameElt.appendChild(doc.createTextNode(item["author"]["name"]))
 160                                 uriElt = authorElt.appendChild(doc.createElement("uri"))
 161                                 uriElt.appendChild(doc.createTextNode(item["author"]["@id"]))
 162                         else:
 163                                 nameElt = authorElt.appendChild(doc.createElement("name"))
 164                                 nameElt.appendChild(doc.createTextNode("Anonymous"))
 165                         contentElt = entryElt.appendChild(doc.createElement("content"))
 166                         contentElt.setAttribute("type", "xhtml")
 167                         contentDiv = contentElt.appendChild(doc.createElement("div"))
 168                         contentDiv.setAttribute("xmlns", XHTML_NAMESPACE)
 169                         contentDiv.setAttribute("lang", LANG)
 170                         for child in list(parseString(item["content"]).documentElement.childNodes):
 171                                 contentDiv.appendChild(child)
 172                 return (atomLinks["self"], atomElt.toxml())
 173
 174         # Get status paths.
 175         status_paths = []
 176         for yearpath in Path(f"{BUILD_DIRECTORY}/").glob("[0-9][0-9][0-9][0-9]"):
 177                 for monthpath in yearpath.glob("[0-9][0-9]"):
 178                         for daypath in monthpath.glob("[0-9][0-9]"):
 179                                 for statuspath in daypath.glob("*/text"):
 180                                         status_paths.append((None, statuspath))
 181         for topicpath in Path(f"{BUILD_DIRECTORY}/").glob("topic/*"):
 182                 for hash0path in topicpath.glob("[0-9a-f]"):
 183                         for hash1path in hash0path.glob("[0-9a-f]"):
 184                                 for hash2path in hash1path.glob("[0-9a-f]"):
 185                                         for hash3path in hash2path.glob("[0-9a-f]"):
 186                                                 for statuspath in hash3path.glob("*/text"):
 187                                                         status_paths.append((topicpath.name, statuspath))
 188
 189         # Build status objects and listings.
 190         for (datetime, identifier, status) in sorted(filter(None, starmap(statusmap, status_paths))):
 191                 if "subject" in status:
 192                         topic = status["subject"]
 193                         if topic not in topics:
 194                                 topics[topic] = { "@context": { "@language": LANG, "activity": "https://www.w3.org/ns/activitystreams#", "dct": "http://purl.org/dc/terms/", "foaf": "http://xmlns.com/foaf/0.1/", "sioc": "http://rdfs.org/sioc/ns#", "sioct": "http://rdfs.org/sioc/types#", "OrderedCollection": "activity:OrderedCollection", "Thread": "sioc:Thread", "MicroblogPost": "sioct:MicroblogPost", "items": { "@id": "activity:items", "@type": "@id", "@container": "@list" }, "created": { "@id": "dct:created", "@type": "http://www.w3.org/2001/XMLSchema#dateTime" }, "creator": { "@id": "dct:creator", "@type": "@id" }, "identifier": { "@id":  "dct:identifier", "@type": "http://www.w3.org/2001/XMLSchema#anyURI" }, "subject": "dct:subject", "name": "foaf:name", "title": "dct:title", "content": { "@id": "sioc:content", "@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral" } }, "@id": f"{PUBLIC_URL}/topics/{topic}", "@type": ["OrderedCollection", "Thread"], "items": [], "subject": topic }
 195                         topics[topic]["items"].append(status)
 196                 else:
 197                         yyyy_mm = datetime[0:7]
 198                         if yyyy_mm not in months:
 199                                 months[yyyy_mm] = { "@context": { "@language": LANG, "activity": "https://www.w3.org/ns/activitystreams#", "dct": "http://purl.org/dc/terms/", "foaf": "http://xmlns.com/foaf/0.1/", "sioc": "http://rdfs.org/sioc/ns#", "sioct": "http://rdfs.org/sioc/types#", "OrderedCollectionPage": "activity:OrderedCollectionPage", "Thread": "sioc:Thread", "MicroblogPost": "sioct:MicroblogPost", "current": { "@id": "activity:current", "@type": "@id" }, "first": { "@id": "activity:first", "@type": "@id" }, "items": { "@id": "activity:items", "@type": "@id", "@container": "@list" }, "partOf": { "@id": "activity:partOf", "@type": "@id" }, "prev": { "@id": "activity:prev", "@type": "@id" }, "next": { "@id": "activity:next", "@type": "@id" }, "created": { "@id": "dct:created", "@type": "http://www.w3.org/2001/XMLSchema#dateTime" }, "creator": { "@id": "dct:creator", "@type": "@id" }, "identifier": { "@id":  "dct:identifier", "@type": "http://www.w3.org/2001/XMLSchema#anyURI" }, "name": "foaf:name", "title": "dct:title", "content": { "@id": "sioc:content", "@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral" } }, "@id": f"{PUBLIC_URL}/statuses/{yyyy_mm}", "@type": ["OrderedCollectionPage", "Thread"], "items": [], "partOf": f"{PUBLIC_URL}/statuses" }
 200                         months[yyyy_mm]["items"].append(status)
 201
 202         # Set up the public directory.
 203         if exists(PUBLIC_DIRECTORY):
 204                 rmtree(PUBLIC_DIRECTORY)
 205         mkdir(PUBLIC_DIRECTORY)
 206
 207         # Copy H·T·M·L files to their expected locations.
 208         copy2(f"{BUILD_DIRECTORY}/index.html", f"{PUBLIC_DIRECTORY}/index.html")
 209         copy2(f"{BUILD_DIRECTORY}/about.html", f"{PUBLIC_DIRECTORY}/.about.html")
 210         copy2(f"{BUILD_DIRECTORY}/status.html", f"{PUBLIC_DIRECTORY}/.status.html")
 211         copy2(f"{BUILD_DIRECTORY}/statuses.html", f"{PUBLIC_DIRECTORY}/.statuses.html")
 212         copy2(f"{BUILD_DIRECTORY}/topic.html", f"{PUBLIC_DIRECTORY}/.topic.html")
 213         copy2(f"{BUILD_DIRECTORY}/topics.html", f"{PUBLIC_DIRECTORY}/.topics.html")
 214
 215         # Output “about” metadata
 216         if not exists(f"{PUBLIC_DIRECTORY}/about"):
 217                 mkdir(f"{PUBLIC_DIRECTORY}/about")
 218         with open(f"{PUBLIC_DIRECTORY}/about/index.jsonld", "w", encoding="utf-8") as f:
 219                 json.dump({ "@context": { "@language": LANG, "activity": "https://www.w3.org/ns/activitystreams#", "sioc": "http://rdfs.org/sioc/ns#", "sioct": "http://rdfs.org/sioc/types#", "Forum": "sioc:Forum", "Thread": "sioc:Thread", "Microblog": "sioct:Microblog", "streams": { "@id": "activity:streams", "@type": "@id" } }, "@id": f"{PUBLIC_URL}", "@type": "Microblog", "streams": [{ "@id": f"{PUBLIC_URL}/statuses", "@type": "Thread" }, { "@id": f"{PUBLIC_URL}/topics", "@type": "Forum" }] }, f, ensure_ascii=False, allow_nan=False)
 220
 221         # Output month‐based listings and the non‐topic index
 222         if not exists(f"{PUBLIC_DIRECTORY}/statuses"):
 223                 mkdir(f"{PUBLIC_DIRECTORY}/statuses")
 224         statuspairs = list(enumerate(months.items()))
 225         for (index, (yyyy_mm, ld)) in statuspairs:
 226                 if not exists(f"{PUBLIC_DIRECTORY}/statuses/{yyyy_mm}"):
 227                         mkdir(f"{PUBLIC_DIRECTORY}/statuses/{yyyy_mm}")
 228                 ld["first"] = f"{PUBLIC_URL}/statuses/{statuspairs[0][1][0]}"
 229                 ld["current"] = f"{PUBLIC_URL}/statuses/{statuspairs[-1][1][0]}"
 230                 if index > 0:
 231                         ld["prev"] = f"{PUBLIC_URL}/statuses/{statuspairs[index - 1][1][0]}"
 232                 if index < len(statuspairs) - 1:
 233                         ld["next"] = f"{PUBLIC_URL}/statuses/{statuspairs[index + 1][1][0]}"
 234                 with open(f"{PUBLIC_DIRECTORY}/statuses/{yyyy_mm}/index.jsonld", "w", encoding="utf-8") as f:
 235                         json.dump(ld, f, ensure_ascii=False, allow_nan=False)
 236                 atomlink, atomxml = atomForLD(ld)
 237                 with open(f"{PUBLIC_DIRECTORY}/{atomlink[len(PUBLIC_URL):-5]}/index.atom", "w", encoding="utf-8") as f:
 238                         f.write(atomxml)
 239         with open(f"{PUBLIC_DIRECTORY}/statuses/index.jsonld", "w", encoding="utf-8") as f:
 240                 json.dump({ "@context": { "@language": LANG, "activity": "https://www.w3.org/ns/activitystreams#", "sioc": "http://rdfs.org/sioc/ns#", "OrderedCollection": "activity:OrderedCollection", "Thread": "sioc:Thread", "current": { "@id": "activity:current", "@type": "@id" }, "first": { "@id": "activity:first", "@type": "@id" }, "has_parent": { "@id": "sioc:has_parent", "@type": "id" } }, "@id": f"{PUBLIC_URL}/statuses", "@type": ["OrderedCollection", "Thread"], "first": f"{PUBLIC_URL}/statuses/{statuspairs[0][1][0]}", "current": f"{PUBLIC_URL}/statuses/{statuspairs[-1][1][0]}", "has_parent": f"{PUBLIC_URL}" }, f, ensure_ascii=False, allow_nan=False)
 241
 242         # Output topic‐based listings and the topic index
 243         if not exists(f"{PUBLIC_DIRECTORY}/topics"):
 244                 mkdir(f"{PUBLIC_DIRECTORY}/topics")
 245         for (topic, ld) in topics.items():
 246                 if not exists(f"{PUBLIC_DIRECTORY}/topics/{topic}"):
 247                         mkdir(f"{PUBLIC_DIRECTORY}/topics/{topic}")
 248                 with open(f"{PUBLIC_DIRECTORY}/topics/{topic}/index.jsonld", "w", encoding="utf-8") as f:
 249                         json.dump(ld, f, ensure_ascii=False, allow_nan=False)
 250                 atomlink, atomxml = atomForLD(ld)
 251                 with open(f"{PUBLIC_DIRECTORY}/{atomlink[len(PUBLIC_URL):-5]}/index.atom", "w", encoding="utf-8") as f:
 252                         f.write(atomxml)
 253         with open(f"{PUBLIC_DIRECTORY}/topics/index.jsonld", "w", encoding="utf-8") as f:
 254                 json.dump({ "@context": { "@language": LANG, "activity": "https://www.w3.org/ns/activitystreams#", "dct": "http://purl.org/dc/terms/", "sioc": "http://rdfs.org/sioc/ns#", "Collection": "activity:Collection", "Forum": "sioc:Forum", "items": { "@id": "activity:items", "@type": "@id" }, "has_parent": { "@id": "sioc:has_parent", "@type": "id" }, "subject": "dct:subject" }, "@id": f"{PUBLIC_URL}/topics", "@type": ["Collection", "Forum"], "items": list(map(lambda a: { "@id": a["@id"], "subject": a["subject"] }, topics.values())), "has_parent": f"{PUBLIC_URL}" }, f, ensure_ascii=False, allow_nan=False)
 255
 256         # Output the I·R·I redirection page
 257         with open(f"{PUBLIC_DIRECTORY}/.lookup.xhtml", "w", encoding="utf-8") as f:
 258                 doc = getDOMImplementation().createDocument(None, "xml", None)
 259                 htmlElt = doc.documentElement
 260                 htmlElt.setAttribute("xmlns", XHTML_NAMESPACE)
 261                 htmlElt.setAttribute("lang", LANG)
 262                 headElt = htmlElt.appendChild(doc.createElement("head"))
 263                 titleElt = headElt.appendChild(doc.createElement("title"))
 264                 titleElt.appendChild(doc.createTextNode("Redirecting…"))
 265                 scriptElt = headElt.appendChild(doc.createElement("script"))
 266                 scriptElt.setAttribute("type", "text/javascript")
 267                 scriptElt.appendChild(doc.createTextNode(f"location={json.dumps(irimap)}[location.pathname.substring(1)]??`/`"))
 268                 bodyElt = htmlElt.appendChild(doc.createElement("body"))
 269                 bodyElt.appendChild(doc.createTextNode("Attempting to redirect to the proper page… (Requires Javascript.)"))
 270                 f.write(doc.toxml())
 271
 272         # Remove the build directory.
 273         rmtree(BUILD_DIRECTORY)