special comment formatting for hex dumps of very long strings

this will help with the boot-source

there's a lot of nuanced decisions that go into this; the comments explain in depth

Change-Id: I391c937a04398115849f4308940d3b4df519c9ab
Force-Push: yes

2 files changed, 314 insertions, 18 deletions

diff --git a/compilation/execution.e b/compilation/execution.e
index b827a51..01f3b68 100644
--- a/compilation/execution.e
+++ b/compilation/execution.e
@@ -702,8 +702,8 @@
   ~ ; codeword. Then it has text, with a null terminator and alignment
   ~ : padding at the end.
   ~ :
-  ~ :   Unfortunately, this winds up being somewhat hard to read in the hex
-  ~ : dump. Sorry about that.
+  ~ :   Making this readable in the hex dump is no small task, but has been
+  ~ : valiantly attempted.
   ~ : blank-line
   ~ : indent
   3unroll pack64 ~ : 8 #-- (previous entry pointer)
diff --git a/compilation/transform.e b/compilation/transform.e
index 4895827..fae9105 100644
--- a/compilation/transform.e
+++ b/compilation/transform.e
@@ -3304,8 +3304,11 @@ allocate-transformation-state s" transformation-state" variable
 ~ immediately, on the assumption they'll be overwritten. The -keyword variant
 ~ expects to find a keyword (a word which, when executed, pushes its own
 ~ execution token onto the stack); the command extracts the name of the
-~ keyword from the dictionary entry header. All three of these save pointer to
-~ their respective strings in a push-subitem-string metadata entry.
+~ keyword from the dictionary entry header. All three of these save pointers
+~ to their respective strings in a push-subitem-string metadata entry. They
+~ also save the lengths of their strings to the subitem entry's length field;
+~ this is in contrast to the non-string provide-* commands, which use the
+~ command's parameter as their length values.
 ~
 ~   Admit it, you thought writing out "metadata entry" so many times instead
 ~ of just "entry" was silly because there was no other kind of entry in
@@ -4295,8 +4298,16 @@ allocate-transformation-state s" transformation-state" variable
   dup hex-output-metadata-entry-type @
   hex-output-metadata-entry-type-push-subitem-string = {
     dup hex-output-metadata-entry-content @
-    dup stringlen advance-current-column
-    emitstring
+    over hex-output-metadata-entry-data-length @
+    ~ (subitem entry, string point, n bytes remaining)
+    { dup } {
+      swap unpack8
+      value@ emitstring
+      drop swap
+      1-
+      1 advance-current-column
+    } while
+    drop drop
   } if ;
 
 
@@ -4339,8 +4350,12 @@ allocate-transformation-state s" transformation-state" variable
       swap dup indent advance-current-column 0 swap
 
       0x22 value@ emitstring drop
+      1 advance-current-column
+
       hex-emit-template-subitem
+
       0x22 value@ emitstring drop
+      1 advance-current-column
     } if
 
     drop drop 0 exit
@@ -4786,8 +4801,9 @@ allocate-transformation-state s" transformation-state" variable
   add-hex-output-metadata-entry
   ~ (label entry pointer, entry data start, entry data length)
 
-  2dup hex-output-metadata-entry-type-push-subitem-string
-  6 roll entry-to-name
+  over hex-output-metadata-entry-type-push-subitem-string
+  5 roll entry-to-name
+  dup stringlen 3unroll
 
   swap-transform-variables here @ swap-transform-variables dup 3unroll
   ~ (copy, original, copy)
@@ -4810,13 +4826,92 @@ allocate-transformation-state s" transformation-state" variable
     hex-output-metadata-latest-output-point !
   } { drop } if-else ;
 
-~ (output point, string pointer --)
-: hex-packstring-trace
-  dup stringlen 1+
-  ~ (output point, string pointer, string length)
-  2 pick over hex-pack-trace
-  swap
-  ~ (output point, string length, string pointer)
+
+~   Conveniently, this is both the maximum normal length of a line of text in
+~ Evocation's source per our informal style rules, and an exact multiple of
+~ the number of bytes we wrap to in the hex part of the output. That's
+~ sixteen, and 16 * 5 is 80.
+: long-string-threshold 80 ;
+
+~   Nothing particularly poetic about this number. It must be smaller than
+~ long-string-threshold by enough that there's room for indentation, quotation
+~ marks, and other such niceties. That's the only formal constraint.
+~
+~   However, it's not aesthetically pleasing to make the thresholds too close
+~ together, because that results in a lot of lines being wrapped into one
+~ which is quite long and one which is quite short.
+~
+~   So, this smaller value was chosen by trial and error to not lead the eye
+~ around too much.
+: long-string-comment-wrap-threshold 60 ;
+
+
+~   Choose where to line-wrap a string. This is for the outer loop, as
+~ described in hex-packstring-trace, below. If the string contains any
+~ newlines, this breaks immediately after the first. Otherwise, it doesn't
+~ break.
+~
+~   Note that our caller has already bitten off a maximum length that does not
+~ exceed long-string-threshold, so if this word chooses to not wrap, there's
+~ still fallback wrapping in place.
+~
+~ (string pointer, maximum length -- chosen length)
+: choose-string-outer-break-point
+  ~ If the maximum length is zero, do nothing.
+  dup {
+    ~ Scan forward from the start for a newline.
+    0 { 2dup 1+ < } {
+      ~ (string pointer, maximum length, offset)
+      2 pick over + 8@ 0x0a = {
+        3unroll 2drop 1+ exit
+      } if
+
+      ~ (string pointer, maximum length, offset)
+      1+
+    } while drop
+  } if
+
+  ~ No case matched, so choose the maximum length.
+  swap drop ;
+
+
+~   Choose where to line-wrap a string. This is for the inner loop, as
+~ described in hex-packstring-trace, below. If the string is shorter than
+~ long-string-comment-wrap-threshold, it is returned without wrapping.
+~ Otherwise, if it contains any spaces prior to that threshold, it is broken
+~ immediately before the last. If neither condition applies, the string is
+~ broken at the threshold despite this being notionally mid-word.
+~
+~ (string pointer, maximum length -- chosen length)
+: choose-string-inner-break-point
+  ~ If the maximum length is less than the threshold, do nothing.
+  dup long-string-comment-wrap-threshold <= {
+    long-string-comment-wrap-threshold 1- { dup 1 < } {
+      ~ (string pointer, maximum length, offset)
+      2 pick over + 8@ 0x20 = {
+        3unroll 2drop exit
+      } if
+
+      ~ (string pointer, maximum length, offset)
+      1-
+    } while drop
+  } if
+
+  ~ No case matched, so fall back to hard-wrapping at the threshold.
+  swap drop long-string-comment-wrap-threshold min ;
+
+
+~   The code in hex-packstring-trace, below, is broken into two cases, one for
+~ ordinary strings and one for very long strings that need special formatting.
+~ Most of its complexity pertains to iteration structure; the logic of
+~ actually creating metadata entries has been deferred to helpers. This helper
+~ handles the short-string case.
+~
+~   There is only one call to this helper per string, since that's all that's
+~ needed.
+~
+~ (output point, string length, string pointer --)
+: add-short-string-entry-pair
   drop 2dup over
   ~ (output point, string length, output point, string length, output point)
   hex-output-metadata-entry-type-push-subitem-string swap
@@ -4826,6 +4921,204 @@ allocate-transformation-state s" transformation-state" variable
   s" '#'   (string literal with null)"
   add-hex-output-metadata-entry ;
 
+
+~   The code in hex-packstring-trace, below, is broken into two cases, one for
+~ ordinary strings and one for very long strings that need special formatting.
+~ The long-string case begins by outputting a fixed set of entries that
+~ describe, to the reader, what's about to happen. This helper produces those
+~ entries.
+~
+~ (output point--)
+: add-long-string-header-entries
+  dup 0 hex-output-metadata-entry-type-blank-line 0
+  add-hex-output-metadata-entry
+
+  dup 0 hex-output-metadata-entry-type-line-comment
+  s"   What follows is a long string literal. For convenience, its contents"
+  add-hex-output-metadata-entry
+
+  dup 0 hex-output-metadata-entry-type-line-comment
+  s" are given in short runs, but it's all a single string. When possible,"
+  add-hex-output-metadata-entry
+
+  dup 0 hex-output-metadata-entry-type-line-comment
+  s" runs are broken at line endings or word breaks. Trailing line endings"
+  add-hex-output-metadata-entry
+
+  ~ Note the lack of a "dup" this time.
+  0 hex-output-metadata-entry-type-line-comment
+  s" are not shown in the comments; look for them in the hex."
+  add-hex-output-metadata-entry ;
+
+
+~   The code in hex-packstring-trace, below, is broken into two cases, one for
+~ ordinary strings and one for very long strings that need special formatting.
+~ The long-string case ends by outputting a fixed set of entries that
+~ inform the reader the special handling is complete. This helper produces
+~ those entries.
+~
+~ (output point--)
+: add-long-string-footer-entries
+  dup 0 hex-output-metadata-entry-type-blank-line 0
+  add-hex-output-metadata-entry
+
+  dup 0 hex-output-metadata-entry-type-line-comment
+  s"   This is the end of the long string literal. We now return you to the"
+  add-hex-output-metadata-entry
+
+  dup 0 hex-output-metadata-entry-type-line-comment
+  s" normal hex-dump stuff."
+  add-hex-output-metadata-entry
+
+  ~ Note the lack of a "dup" this time.
+  0 hex-output-metadata-entry-type-blank-line 0
+  add-hex-output-metadata-entry ;
+
+
+~   The code in hex-packstring-trace, below, is broken into two cases, one for
+~ ordinary strings and one for very long strings that need special formatting.
+~ The long-string case has two layers of loop, and the outer layer needs to
+~ output a fixed set of entries at the start of each of its iterations. This
+~ helper produces those entries.
+~
+~ (output point--)
+: add-long-string-outer-iteration-header-entries
+  0 hex-output-metadata-entry-type-blank-line 0
+  add-hex-output-metadata-entry ;
+
+
+~   The code in hex-packstring-trace, below, is broken into two cases, one for
+~ ordinary strings and one for very long strings that need special formatting.
+~ Most of its complexity pertains to iteration structure; the logic of
+~ actually creating metadata entries has been deferred to helpers. This helper
+~ handles the entries for a single iteration of the inner loop in the
+~ long-string case.
+~
+~   There are potentially quite a lot of calls to this helper, exactly one per
+~ inner iteration. See the full explanation in hex-packstring-trace about the
+~ meaning of the inner and outer layers of iteration.
+~
+~ (output point, string length, string pointer --)
+: add-long-string-entry-pair
+  ~   If the final character would be a newline, exclude that from the comment
+  ~ body.
+  ~
+  ~   Please notice that we do this check on every comment, though we only
+  ~ care about it for the final one in the block. Because of the way
+  ~ line-wrapping will always choose the first newline if there is one, we
+  ~ don't need to worry about this check firing when it's unwanted.
+  ~
+  ~   Letting that happen here means we have less trouble writing the
+  ~ termination check for the inner loop.
+  2dup + 1- 8@ 0x0a = { swap 1- swap } if
+  ~ (output point, string length, string pointer)
+
+  2 pick 2 pick 2 pick
+  hex-output-metadata-entry-type-push-subitem-string swap
+  add-hex-output-metadata-entry
+  ~ (output point, string length, string pointer)
+
+  drop
+  hex-output-metadata-entry-type-line-comment s" '#'"
+  add-hex-output-metadata-entry ;
+
+
+~ (output point, string pointer --)
+: hex-packstring-trace
+  dup stringlen 1+
+  ~ (output point, string pointer, string length)
+  2 pick over hex-pack-trace
+  swap
+  ~ (output point, string length, string pointer)
+  over long-string-threshold >= {
+    add-short-string-entry-pair
+  } {
+    2 pick add-long-string-header-entries
+
+    ~   It's important not to accidentally confuse the string point with the
+    ~ output point here. Because of the multi-pass nature of the label loop,
+    ~ there's a real chance the contents of both will be identical, making it
+    ~ very easy to use the wrong one since it will seem to work. There is no
+    ~ guarantee that all long strings in the final pass will be at the same
+    ~ locations they were in the penultimate pass, or that nothing has
+    ~ scribbled over them, so don't get it wrong!
+    ~
+    ~   Okay, got that firmly in mind? Good. Now, what we're doing is a
+    ~ two-layer iteration. That may be surprising complexity. To motivate it,
+    ~ contemplate how the text inside the string is likely wrapped to 80
+    ~ columns wide, and the formatted hex dump we're producing is likely ALSO
+    ~ wrapped to 80 columns wide. Our choice to displaying everything in
+    ~ hexadecimal, with comment syntax, and so on, results in us not being
+    ~ able to give the inner text the 80 columns it wants, and we have to get
+    ~ creative.
+    ~
+    ~   We resolve this by having an outer loop which breaks the string into
+    ~ lines (preferring to do so at newlines when possible), and renders each
+    ~ of these as a consecutive block of hex, nothing interrupting it. Then
+    ~ we also have an inner loop which breaks the comments preceding the hex
+    ~ (preferring to do so at spaces when possible), to make a block of
+    ~ comments that correspond to the hex and all come at the start of it.
+    ~
+    ~   In the event that lines are very long, fallback behavior may break
+    ~ them somewhere other than a newline, but conceptually it's easiest if
+    ~ you think of each hex "paragraph" as representing a single line of text
+    ~ from the inside-the-string perspective.
+    ~
+    ~   If that sounds abstract, it's a good idea to go look at an example of
+    ~ the rendered output at this point. Look in "evoke.hex" for the label
+    ~ "boot-source" and skim its contents; that's all rendered by this code.
+    ~
+    ~   Anyway, this "while" block is the outer loop.
+    swap { dup } {
+      ~ (output point, outer string point, remaining length)
+      2 pick add-long-string-outer-iteration-header-entries
+
+      dup long-string-threshold min
+      ~ (output point, outer string point, remaining length,
+      ~  max length this outer iteration)
+      2 pick swap choose-string-outer-break-point
+      ~ (output point, outer string point, remaining length,
+      ~  outer iteration length)
+
+      2 pick 0
+      ~ (output point, outer string point, remaining length,
+      ~  outer iteration length, inner string point, inner iteration offset)
+
+      { 2 pick over < } {
+        ~ (output point, outer string point, remaining length,
+        ~  outer iteration length, inner string point, inner iteration offset)
+        2 pick over -
+        ~ (output point, outer string point, remaining length,
+        ~  outer iteration length, inner string point, inner iteration offset,
+        ~  inner iteration length)
+        2 pick swap choose-string-inner-break-point
+
+        ~   It's worth noticing that we use the same output point for all of
+        ~ these, which is what makes them appear at the top of the hex block
+        ~ rather than interspersed throughout it.
+        ~
+        ~   We leave the newline-ignoring logic to add-long-string-entry-pair,
+        ~ as described further therein.
+        6 pick over 4 pick add-long-string-entry-pair
+
+        ~ (output point, outer string point, remaining length,
+        ~  outer iteration length, inner string point, inner iteration offset,
+        ~  inner iteration length)
+        dup 4 roll + 3unroll
+        dup 3roll + swap
+        drop
+      } while 2drop
+      ~ (output point, outer string point, remaining length,
+      ~  outer iteration length)
+
+      dup 5 roll + 4 unroll
+      dup 4 roll + 3unroll
+      -
+    } while
+
+    2drop add-long-string-footer-entries
+  } if-else ;
+
 ~   The purpose of this word is to make sure that the memory-sliding
 ~ techniques used by the high-level flow control words are appropriately
 ~ reflected in the output metadata. This is accomplished by finding all
@@ -5777,13 +6070,15 @@ allocate-transformation-state s" transformation-state" variable
   } if
   dup s" provide-string" stringcmp 0 = {
     drop drop
-    2 pick hex-output-metadata-entry-type-push-subitem-string swap
+    drop over dup stringlen swap
+    hex-output-metadata-entry-type-push-subitem-string swap
     add-hex-output-metadata-entry
     exit
   } if
   dup s" provide-string-copy" stringcmp 0 = {
     drop drop
-    2 pick hex-output-metadata-entry-type-push-subitem-string swap
+    drop over dup stringlen swap
+    hex-output-metadata-entry-type-push-subitem-string swap
     swap-transform-variables here @ swap-transform-variables dup 3unroll
     ~ (copy, original, copy)
     over stringlen 1+ memcopy
@@ -5794,7 +6089,8 @@ allocate-transformation-state s" transformation-state" variable
   dup s" provide-keyword" stringcmp 0 = {
     ~ Create a new "push subitem string" entry given a keyword pointer.
     drop drop
-    2 pick execution-token-to-entry entry-to-name
+    drop over execution-token-to-entry entry-to-name
+    dup stringlen swap
     hex-output-metadata-entry-type-push-subitem-string swap
     add-hex-output-metadata-entry
     exit