Refactor initial chunking to be line‐based

The old parsing mechanism operated primarily on large string chunks,
which were re‐parsed into lines potentially multiple times. This
refactor changes the parsing to use lines and ranges, which is a little
more verbose/complicated from an X·Path perspective (it requires a lot
of `generate-id()` comparisons) but hopefully, on the whole, better.

diff --git a/parser.xslt b/parser.xslt
index 86c28ea1a31bade3aeefb20e4cd3c9779c3e1dba..f1ade04f126864d162b63879b59716c699134d2b 100644 (file)
--- a/parser.xslt
+++ b/parser.xslt
@@ -121,24 +121,15 @@ If a copy of the M·P·L was not distributed with this file, You can obtain one
                </choose>
        </template>
        <template name="LesML:parse">
-               <param name="source"/>
+               <param name="lines" select="/.."/>
                <param name="parent-params" select="/.."/>
+               <variable name="first-line" select="$lines[1]"/>
                <variable name="shebang">
-                       <if test="starts-with($source, '#!lesml')">
-                               <value-of select="substring-before($source, '&#xA;')"/>
+                       <if test="starts-with($first-line, '#!lesml')">
+                               <value-of select="$first-line"/>
                        </if>
                </variable>
-               <variable name="noshebang">
-                       <text>&#xA;</text>
-                       <choose>
-                               <when test="starts-with($source, '#!')">
-                                       <value-of select="substring-after($source, '&#xA;')"/>
-                               </when>
-                               <otherwise>
-                                       <value-of select="$source"/>
-                               </otherwise>
-                       </choose>
-               </variable>
+               <variable name="noshebang" select="$lines[position()>1 or not(starts-with(., '#!lesml') or starts-with(., '##'))]"/>
                <variable name="params-string">
                        <choose>
                                <when test="starts-with($shebang, '#!lesml@')">
@@ -186,13 +177,7 @@ If a copy of the M·P·L was not distributed with this file, You can obtain one
                        </html:dl>
                </variable>
                <variable name="params" select="exsl:node-set($params-fragment)/*"/>
-               <variable name="records">
-                       <call-template name="LesML:split">
-                               <with-param name="source" select="$noshebang"/>
-                               <with-param name="separator" select="'&#xA;%%'"/>
-                       </call-template>
-               </variable>
-               <variable name="record-nodes" select="exsl:node-set($records)/*"/>
+               <variable name="record-separators" select="$noshebang[starts-with(., '%%')]"/>
                <html:article>
                        <for-each select="$params/html:div/html:dt[string()=' LANG ']">
                                <attribute name="lang">
@@ -207,90 +192,84 @@ If a copy of the M·P·L was not distributed with this file, You can obtain one
                                        <value-of select="following-sibling::html:dd"/>
                                </attribute>
                        </for-each>
-                       <if test="count($record-nodes)>1">
+                       <if test="count($record-separators)>1">
                                <html:footer class="head">
-                                       <for-each select="$record-nodes[not(position()=last() or normalize-space(.)='')]">
-                                               <variable name="fields" select="exslstr:tokenize(., '&#xA;')"/>
-                                               <html:dl>
-                                                       <for-each select="$fields">
-                                                               <choose>
-                                                                       <when test="starts-with(., ' ') and $fields[generate-id()=generate-id(current()/preceding-sibling::*[1])]"/>
-                                                                       <otherwise>
-                                                                               <variable name="next" select="following-sibling::*[not(starts-with(., ' '))]"/>
-                                                                               <html:div>
-                                                                                       <html:dt>
-                                                                                               <value-of select="normalize-space(substring-before(., ':'))"/>
-                                                                                       </html:dt>
-                                                                                       <html:dd>
-                                                                                               <variable name="firstline">
-                                                                                                       <choose>
-                                                                                                               <when test="contains(., ':')">
-                                                                                                                       <value-of select="normalize-space(substring-after(., ':'))"/>
-                                                                                                               </when>
-                                                                                                               <otherwise>
-                                                                                                                       <value-of select="normalize-space(.)"/>
-                                                                                                               </otherwise>
-                                                                                                       </choose>
-                                                                                               </variable>
-                                                                                               <choose>
-                                                                                                       <when test="substring($firstline, string-length($firstline))='\' and following-sibling::*[position()=1 and starts-with(., ' ')]">
-                                                                                                               <value-of select="substring($firstline, 1, string-length($firstline)-1)"/>
-                                                                                                       </when>
-                                                                                                       <otherwise>
-                                                                                                               <value-of select="$firstline"/>
-                                                                                                       </otherwise>
-                                                                                               </choose>
-                                                                                               <for-each select="following-sibling::*[starts-with(., ' ') and not(preceding-sibling::*[generate-id()=generate-id($next)])]">
-                                                                                                       <variable name="nextline" select="normalize-space(.)"/>
+                                       <for-each select="$record-separators">
+                                               <variable name="position" select="position()"/>
+                                               <variable name="prev-separator" select="$record-separators[($position)-1]"/>
+                                               <variable name="fields" select="$noshebang[following-sibling::*[generate-id()=generate-id(current())] and (not($prev-separator) or preceding-sibling::*[generate-id()=generate-id($prev-separator)])]"/>
+                                               <if test="$fields">
+                                                       <html:dl>
+                                                               <for-each select="$fields">
+                                                                       <choose>
+                                                                               <when test="starts-with(., ' ') and $fields[generate-id()=generate-id(current()/preceding-sibling::*[1])]"/>
+                                                                               <otherwise>
+                                                                                       <variable name="next" select="following-sibling::*[not(starts-with(., ' '))]"/>
+                                                                                       <html:div>
+                                                                                               <html:dt>
+                                                                                                       <value-of select="normalize-space(substring-before(., ':'))"/>
+                                                                                               </html:dt>
+                                                                                               <html:dd>
+                                                                                                       <variable name="firstline">
+                                                                                                               <choose>
+                                                                                                                       <when test="contains(., ':')">
+                                                                                                                               <value-of select="normalize-space(substring-after(., ':'))"/>
+                                                                                                                       </when>
+                                                                                                                       <otherwise>
+                                                                                                                               <value-of select="normalize-space(.)"/>
+                                                                                                                       </otherwise>
+                                                                                                               </choose>
+                                                                                                       </variable>
                                                                                                        <choose>
-                                                                                                               <when test="substring($nextline, string-length($nextline))='\' and following-sibling::*[position()=1 and starts-with(., ' ')]">
-                                                                                                                       <value-of select="substring($nextline, 1, string-length($nextline)-1)"/>
+                                                                                                               <when test="substring($firstline, string-length($firstline))='\' and following-sibling::*[position()=1 and starts-with(., ' ')]">
+                                                                                                                       <value-of select="substring($firstline, 1, string-length($firstline)-1)"/>
                                                                                                                </when>
                                                                                                                <otherwise>
-                                                                                                                       <value-of select="$nextline"/>
+                                                                                                                       <value-of select="$firstline"/>
                                                                                                                </otherwise>
                                                                                                        </choose>
-                                                                                               </for-each>
-                                                                                       </html:dd>
-                                                                               </html:div>
-                                                                       </otherwise>
-                                                               </choose>
-                                                       </for-each>
-                                               </html:dl>
+                                                                                                       <for-each select="following-sibling::*[starts-with(., ' ') and not(preceding-sibling::*[generate-id()=generate-id($next)])]">
+                                                                                                               <variable name="nextline" select="normalize-space(.)"/>
+                                                                                                               <choose>
+                                                                                                                       <when test="substring($nextline, string-length($nextline))='\' and following-sibling::*[position()=1 and starts-with(., ' ')]">
+                                                                                                                               <value-of select="substring($nextline, 1, string-length($nextline)-1)"/>
+                                                                                                                       </when>
+                                                                                                                       <otherwise>
+                                                                                                                               <value-of select="$nextline"/>
+                                                                                                                       </otherwise>
+                                                                                                               </choose>
+                                                                                                       </for-each>
+                                                                                               </html:dd>
+                                                                                       </html:div>
+                                                                               </otherwise>
+                                                                       </choose>
+                                                               </for-each>
+                                                       </html:dl>
+                                               </if>
                                        </for-each>
                                </html:footer>
                        </if>
                        <html:div class="body">
                                <call-template name="LesML:paragraphize">
-                                       <with-param name="source" select="string($record-nodes[last()])"/>
+                                       <with-param name="lines" select="$noshebang[not($record-separators) or preceding-sibling::*[generate-id()=generate-id($record-separators[last()])]]"/>
                                </call-template>
                        </html:div>
                </html:article>
        </template>
        <template name="LesML:paragraphize">
-               <param name="source"/>
-               <variable name="paragraphs">
-                       <call-template name="LesML:split">
-                               <with-param name="source" select="$source"/>
-                               <with-param name="separator" select="'&#xA;&#xA;'"/>
-                       </call-template>
-               </variable>
+               <param name="lines" select="/.."/>
+               <variable name="last-lines" select="$lines[normalize-space()!='' and normalize-space(following-sibling::*[1])='']|$lines[last()]"/>
                <variable name="blocked">
-                       <for-each select="exsl:node-set($paragraphs)/*">
-                               <variable name="lines">
-                                       <call-template name="LesML:split">
-                                               <with-param name="source" select="string()"/>
-                                       </call-template>
-                               </variable>
-                               <variable name="linespans" select="exsl:node-set($lines)/*"/>
+                       <for-each select="$last-lines">
+                               <variable name="position" select="position()"/>
+                               <variable name="prev-last" select="$last-lines[($position)-1]"/>
+                               <variable name="linespans" select="$lines[following-sibling::*[generate-id()=generate-id(current())] and (not($prev-last) or preceding-sibling::*[generate-id()=generate-id($prev-last)]) and normalize-space()!='']|."/>
                                <variable name="quoted" select="not($linespans[not(starts-with(., ' ') or starts-with(., '&#x9;'))])"/>
                                <variable name="text">
                                        <for-each select="$linespans">
-                                               <if test="normalize-space()!=''">
-                                                       <value-of select="normalize-space()"/>
-                                                       <if test="following-sibling::*[normalize-space()!='']">
-                                                               <text> </text>
-                                                       </if>
+                                               <value-of select="normalize-space()"/>
+                                               <if test="position()!=count($linespans)">
+                                                       <text> </text>
                                                </if>
                                        </for-each>
                                </variable>
@@ -482,13 +461,17 @@ If a copy of the M·P·L was not distributed with this file, You can obtain one
                <apply-templates select="exsl:node-set($inlined)/node()" mode="LesML:finalize-tree"/>
        </template>
        <template match="html:script[@type='text/lesml']">
-               <variable name="source">
-                       <for-each select=".//text()">
-                               <value-of select="."/>
-                       </for-each>
+               <variable name="lines-fragment">
+                       <call-template name="LesML:split">
+                               <with-param name="source">
+                                       <for-each select=".//text()">
+                                               <value-of select="."/>
+                                       </for-each>
+                               </with-param>
+                       </call-template>
                </variable>
                <call-template name="LesML:parse">
-                       <with-param name="source" select="string($source)"/>
+                       <with-param name="lines" select="exsl:node-set($lines-fragment)/*"/>
                </call-template>
        </template>
        <template match="@*|node()" mode="LesML:finalize-tree" priority="-1">