diff options
Diffstat (limited to 'compilation')
| -rw-r--r-- | compilation/execution.e | 4 | ||||
| -rw-r--r-- | compilation/transform.e | 328 |
2 files changed, 314 insertions, 18 deletions
diff --git a/compilation/execution.e b/compilation/execution.e index b827a51..01f3b68 100644 --- a/compilation/execution.e +++ b/compilation/execution.e @@ -702,8 +702,8 @@ ~ ; codeword. Then it has text, with a null terminator and alignment ~ : padding at the end. ~ : - ~ : Unfortunately, this winds up being somewhat hard to read in the hex - ~ : dump. Sorry about that. + ~ : Making this readable in the hex dump is no small task, but has been + ~ : valiantly attempted. ~ : blank-line ~ : indent 3unroll pack64 ~ : 8 #-- (previous entry pointer) diff --git a/compilation/transform.e b/compilation/transform.e index 4895827..fae9105 100644 --- a/compilation/transform.e +++ b/compilation/transform.e @@ -3304,8 +3304,11 @@ allocate-transformation-state s" transformation-state" variable ~ immediately, on the assumption they'll be overwritten. The -keyword variant ~ expects to find a keyword (a word which, when executed, pushes its own ~ execution token onto the stack); the command extracts the name of the -~ keyword from the dictionary entry header. All three of these save pointer to -~ their respective strings in a push-subitem-string metadata entry. +~ keyword from the dictionary entry header. All three of these save pointers +~ to their respective strings in a push-subitem-string metadata entry. They +~ also save the lengths of their strings to the subitem entry's length field; +~ this is in contrast to the non-string provide-* commands, which use the +~ command's parameter as their length values. ~ ~ Admit it, you thought writing out "metadata entry" so many times instead ~ of just "entry" was silly because there was no other kind of entry in @@ -4295,8 +4298,16 @@ allocate-transformation-state s" transformation-state" variable dup hex-output-metadata-entry-type @ hex-output-metadata-entry-type-push-subitem-string = { dup hex-output-metadata-entry-content @ - dup stringlen advance-current-column - emitstring + over hex-output-metadata-entry-data-length @ + ~ (subitem entry, string point, n bytes remaining) + { dup } { + swap unpack8 + value@ emitstring + drop swap + 1- + 1 advance-current-column + } while + drop drop } if ; @@ -4339,8 +4350,12 @@ allocate-transformation-state s" transformation-state" variable swap dup indent advance-current-column 0 swap 0x22 value@ emitstring drop + 1 advance-current-column + hex-emit-template-subitem + 0x22 value@ emitstring drop + 1 advance-current-column } if drop drop 0 exit @@ -4786,8 +4801,9 @@ allocate-transformation-state s" transformation-state" variable add-hex-output-metadata-entry ~ (label entry pointer, entry data start, entry data length) - 2dup hex-output-metadata-entry-type-push-subitem-string - 6 roll entry-to-name + over hex-output-metadata-entry-type-push-subitem-string + 5 roll entry-to-name + dup stringlen 3unroll swap-transform-variables here @ swap-transform-variables dup 3unroll ~ (copy, original, copy) @@ -4810,13 +4826,92 @@ allocate-transformation-state s" transformation-state" variable hex-output-metadata-latest-output-point ! } { drop } if-else ; -~ (output point, string pointer --) -: hex-packstring-trace - dup stringlen 1+ - ~ (output point, string pointer, string length) - 2 pick over hex-pack-trace - swap - ~ (output point, string length, string pointer) + +~ Conveniently, this is both the maximum normal length of a line of text in +~ Evocation's source per our informal style rules, and an exact multiple of +~ the number of bytes we wrap to in the hex part of the output. That's +~ sixteen, and 16 * 5 is 80. +: long-string-threshold 80 ; + +~ Nothing particularly poetic about this number. It must be smaller than +~ long-string-threshold by enough that there's room for indentation, quotation +~ marks, and other such niceties. That's the only formal constraint. +~ +~ However, it's not aesthetically pleasing to make the thresholds too close +~ together, because that results in a lot of lines being wrapped into one +~ which is quite long and one which is quite short. +~ +~ So, this smaller value was chosen by trial and error to not lead the eye +~ around too much. +: long-string-comment-wrap-threshold 60 ; + + +~ Choose where to line-wrap a string. This is for the outer loop, as +~ described in hex-packstring-trace, below. If the string contains any +~ newlines, this breaks immediately after the first. Otherwise, it doesn't +~ break. +~ +~ Note that our caller has already bitten off a maximum length that does not +~ exceed long-string-threshold, so if this word chooses to not wrap, there's +~ still fallback wrapping in place. +~ +~ (string pointer, maximum length -- chosen length) +: choose-string-outer-break-point + ~ If the maximum length is zero, do nothing. + dup { + ~ Scan forward from the start for a newline. + 0 { 2dup 1+ < } { + ~ (string pointer, maximum length, offset) + 2 pick over + 8@ 0x0a = { + 3unroll 2drop 1+ exit + } if + + ~ (string pointer, maximum length, offset) + 1+ + } while drop + } if + + ~ No case matched, so choose the maximum length. + swap drop ; + + +~ Choose where to line-wrap a string. This is for the inner loop, as +~ described in hex-packstring-trace, below. If the string is shorter than +~ long-string-comment-wrap-threshold, it is returned without wrapping. +~ Otherwise, if it contains any spaces prior to that threshold, it is broken +~ immediately before the last. If neither condition applies, the string is +~ broken at the threshold despite this being notionally mid-word. +~ +~ (string pointer, maximum length -- chosen length) +: choose-string-inner-break-point + ~ If the maximum length is less than the threshold, do nothing. + dup long-string-comment-wrap-threshold <= { + long-string-comment-wrap-threshold 1- { dup 1 < } { + ~ (string pointer, maximum length, offset) + 2 pick over + 8@ 0x20 = { + 3unroll 2drop exit + } if + + ~ (string pointer, maximum length, offset) + 1- + } while drop + } if + + ~ No case matched, so fall back to hard-wrapping at the threshold. + swap drop long-string-comment-wrap-threshold min ; + + +~ The code in hex-packstring-trace, below, is broken into two cases, one for +~ ordinary strings and one for very long strings that need special formatting. +~ Most of its complexity pertains to iteration structure; the logic of +~ actually creating metadata entries has been deferred to helpers. This helper +~ handles the short-string case. +~ +~ There is only one call to this helper per string, since that's all that's +~ needed. +~ +~ (output point, string length, string pointer --) +: add-short-string-entry-pair drop 2dup over ~ (output point, string length, output point, string length, output point) hex-output-metadata-entry-type-push-subitem-string swap @@ -4826,6 +4921,204 @@ allocate-transformation-state s" transformation-state" variable s" '#' (string literal with null)" add-hex-output-metadata-entry ; + +~ The code in hex-packstring-trace, below, is broken into two cases, one for +~ ordinary strings and one for very long strings that need special formatting. +~ The long-string case begins by outputting a fixed set of entries that +~ describe, to the reader, what's about to happen. This helper produces those +~ entries. +~ +~ (output point--) +: add-long-string-header-entries + dup 0 hex-output-metadata-entry-type-blank-line 0 + add-hex-output-metadata-entry + + dup 0 hex-output-metadata-entry-type-line-comment + s" What follows is a long string literal. For convenience, its contents" + add-hex-output-metadata-entry + + dup 0 hex-output-metadata-entry-type-line-comment + s" are given in short runs, but it's all a single string. When possible," + add-hex-output-metadata-entry + + dup 0 hex-output-metadata-entry-type-line-comment + s" runs are broken at line endings or word breaks. Trailing line endings" + add-hex-output-metadata-entry + + ~ Note the lack of a "dup" this time. + 0 hex-output-metadata-entry-type-line-comment + s" are not shown in the comments; look for them in the hex." + add-hex-output-metadata-entry ; + + +~ The code in hex-packstring-trace, below, is broken into two cases, one for +~ ordinary strings and one for very long strings that need special formatting. +~ The long-string case ends by outputting a fixed set of entries that +~ inform the reader the special handling is complete. This helper produces +~ those entries. +~ +~ (output point--) +: add-long-string-footer-entries + dup 0 hex-output-metadata-entry-type-blank-line 0 + add-hex-output-metadata-entry + + dup 0 hex-output-metadata-entry-type-line-comment + s" This is the end of the long string literal. We now return you to the" + add-hex-output-metadata-entry + + dup 0 hex-output-metadata-entry-type-line-comment + s" normal hex-dump stuff." + add-hex-output-metadata-entry + + ~ Note the lack of a "dup" this time. + 0 hex-output-metadata-entry-type-blank-line 0 + add-hex-output-metadata-entry ; + + +~ The code in hex-packstring-trace, below, is broken into two cases, one for +~ ordinary strings and one for very long strings that need special formatting. +~ The long-string case has two layers of loop, and the outer layer needs to +~ output a fixed set of entries at the start of each of its iterations. This +~ helper produces those entries. +~ +~ (output point--) +: add-long-string-outer-iteration-header-entries + 0 hex-output-metadata-entry-type-blank-line 0 + add-hex-output-metadata-entry ; + + +~ The code in hex-packstring-trace, below, is broken into two cases, one for +~ ordinary strings and one for very long strings that need special formatting. +~ Most of its complexity pertains to iteration structure; the logic of +~ actually creating metadata entries has been deferred to helpers. This helper +~ handles the entries for a single iteration of the inner loop in the +~ long-string case. +~ +~ There are potentially quite a lot of calls to this helper, exactly one per +~ inner iteration. See the full explanation in hex-packstring-trace about the +~ meaning of the inner and outer layers of iteration. +~ +~ (output point, string length, string pointer --) +: add-long-string-entry-pair + ~ If the final character would be a newline, exclude that from the comment + ~ body. + ~ + ~ Please notice that we do this check on every comment, though we only + ~ care about it for the final one in the block. Because of the way + ~ line-wrapping will always choose the first newline if there is one, we + ~ don't need to worry about this check firing when it's unwanted. + ~ + ~ Letting that happen here means we have less trouble writing the + ~ termination check for the inner loop. + 2dup + 1- 8@ 0x0a = { swap 1- swap } if + ~ (output point, string length, string pointer) + + 2 pick 2 pick 2 pick + hex-output-metadata-entry-type-push-subitem-string swap + add-hex-output-metadata-entry + ~ (output point, string length, string pointer) + + drop + hex-output-metadata-entry-type-line-comment s" '#'" + add-hex-output-metadata-entry ; + + +~ (output point, string pointer --) +: hex-packstring-trace + dup stringlen 1+ + ~ (output point, string pointer, string length) + 2 pick over hex-pack-trace + swap + ~ (output point, string length, string pointer) + over long-string-threshold >= { + add-short-string-entry-pair + } { + 2 pick add-long-string-header-entries + + ~ It's important not to accidentally confuse the string point with the + ~ output point here. Because of the multi-pass nature of the label loop, + ~ there's a real chance the contents of both will be identical, making it + ~ very easy to use the wrong one since it will seem to work. There is no + ~ guarantee that all long strings in the final pass will be at the same + ~ locations they were in the penultimate pass, or that nothing has + ~ scribbled over them, so don't get it wrong! + ~ + ~ Okay, got that firmly in mind? Good. Now, what we're doing is a + ~ two-layer iteration. That may be surprising complexity. To motivate it, + ~ contemplate how the text inside the string is likely wrapped to 80 + ~ columns wide, and the formatted hex dump we're producing is likely ALSO + ~ wrapped to 80 columns wide. Our choice to displaying everything in + ~ hexadecimal, with comment syntax, and so on, results in us not being + ~ able to give the inner text the 80 columns it wants, and we have to get + ~ creative. + ~ + ~ We resolve this by having an outer loop which breaks the string into + ~ lines (preferring to do so at newlines when possible), and renders each + ~ of these as a consecutive block of hex, nothing interrupting it. Then + ~ we also have an inner loop which breaks the comments preceding the hex + ~ (preferring to do so at spaces when possible), to make a block of + ~ comments that correspond to the hex and all come at the start of it. + ~ + ~ In the event that lines are very long, fallback behavior may break + ~ them somewhere other than a newline, but conceptually it's easiest if + ~ you think of each hex "paragraph" as representing a single line of text + ~ from the inside-the-string perspective. + ~ + ~ If that sounds abstract, it's a good idea to go look at an example of + ~ the rendered output at this point. Look in "evoke.hex" for the label + ~ "boot-source" and skim its contents; that's all rendered by this code. + ~ + ~ Anyway, this "while" block is the outer loop. + swap { dup } { + ~ (output point, outer string point, remaining length) + 2 pick add-long-string-outer-iteration-header-entries + + dup long-string-threshold min + ~ (output point, outer string point, remaining length, + ~ max length this outer iteration) + 2 pick swap choose-string-outer-break-point + ~ (output point, outer string point, remaining length, + ~ outer iteration length) + + 2 pick 0 + ~ (output point, outer string point, remaining length, + ~ outer iteration length, inner string point, inner iteration offset) + + { 2 pick over < } { + ~ (output point, outer string point, remaining length, + ~ outer iteration length, inner string point, inner iteration offset) + 2 pick over - + ~ (output point, outer string point, remaining length, + ~ outer iteration length, inner string point, inner iteration offset, + ~ inner iteration length) + 2 pick swap choose-string-inner-break-point + + ~ It's worth noticing that we use the same output point for all of + ~ these, which is what makes them appear at the top of the hex block + ~ rather than interspersed throughout it. + ~ + ~ We leave the newline-ignoring logic to add-long-string-entry-pair, + ~ as described further therein. + 6 pick over 4 pick add-long-string-entry-pair + + ~ (output point, outer string point, remaining length, + ~ outer iteration length, inner string point, inner iteration offset, + ~ inner iteration length) + dup 4 roll + 3unroll + dup 3roll + swap + drop + } while 2drop + ~ (output point, outer string point, remaining length, + ~ outer iteration length) + + dup 5 roll + 4 unroll + dup 4 roll + 3unroll + - + } while + + 2drop add-long-string-footer-entries + } if-else ; + ~ The purpose of this word is to make sure that the memory-sliding ~ techniques used by the high-level flow control words are appropriately ~ reflected in the output metadata. This is accomplished by finding all @@ -5777,13 +6070,15 @@ allocate-transformation-state s" transformation-state" variable } if dup s" provide-string" stringcmp 0 = { drop drop - 2 pick hex-output-metadata-entry-type-push-subitem-string swap + drop over dup stringlen swap + hex-output-metadata-entry-type-push-subitem-string swap add-hex-output-metadata-entry exit } if dup s" provide-string-copy" stringcmp 0 = { drop drop - 2 pick hex-output-metadata-entry-type-push-subitem-string swap + drop over dup stringlen swap + hex-output-metadata-entry-type-push-subitem-string swap swap-transform-variables here @ swap-transform-variables dup 3unroll ~ (copy, original, copy) over stringlen 1+ memcopy @@ -5794,7 +6089,8 @@ allocate-transformation-state s" transformation-state" variable dup s" provide-keyword" stringcmp 0 = { ~ Create a new "push subitem string" entry given a keyword pointer. drop drop - 2 pick execution-token-to-entry entry-to-name + drop over execution-token-to-entry entry-to-name + dup stringlen swap hex-output-metadata-entry-type-push-subitem-string swap add-hex-output-metadata-entry exit |