add an implicit way to define suffix comments

and make the explicit way more versatile (allowing zero length)

this required hooking the lexer at a pretty low level, but it worked out fine

Force-Push: yes
Change-Id: Icdbe233fbf9d4cf07cb8d03975eed3b4f7b620aa

2 files changed, 104 insertions, 49 deletions

diff --git a/hex.e b/hex.e
index c9de9da..25e6893 100644
--- a/hex.e
+++ b/hex.e
@@ -46,12 +46,12 @@
   ~ : blank-line
 
   ~ : If it's space or linefeed, skip it (go back to the loop start).
-  0x20 :rax cmp-reg64-imm8                  ~ ASCII space
+  0x20 :rax cmp-reg64-imm8                  ~ : ASCII space
   L@' input-loop-start :cc-equal jmp-cc-rel-imm8-from-here
-  0x0a :rax cmp-reg64-imm8                  ~ ASCII linefeed
+  0x0a :rax cmp-reg64-imm8                  ~ : ASCII linefeed
   L@' input-loop-start :cc-equal jmp-cc-rel-imm8-from-here
   ~ : If it's a comment, skip the whole thing.
-  0x7e :rax cmp-reg64-imm8                  ~ ASCII tilde
+  0x7e :rax cmp-reg64-imm8                  ~ : ASCII tilde
   L@' skip-comment :cc-equal jmp-cc-rel-imm8-from-here
   ~ : blank-line
 
@@ -88,10 +88,10 @@
 
   ~ : Output the byte. We reuse the buffer as a place to store it.
   :rbp :rbx mov-indirect-reg64-reg64
-  :rbx :rsi mov-reg64-reg64                 ~ buffer pointer
-  1 :rdx mov-reg64-imm32                    ~ buffer length
-  1 :rax mov-reg64-imm32                    ~ syscall number for sys-write
-  1 :rdi mov-reg64-imm32                    ~ file descriptor 1 is stdout
+  :rbx :rsi mov-reg64-reg64                 ~ : buffer pointer
+  1 :rdx mov-reg64-imm32                    ~ : buffer length
+  1 :rax mov-reg64-imm32                    ~ : syscall number for sys-write
+  1 :rdi mov-reg64-imm32                    ~ : file descriptor 1 is stdout
   syscall
   ~ : blank-line
 
@@ -116,7 +116,7 @@
   ~ : blank-line
 
   ~ : If it's linefeed, the comment is over.
-  0x0a :rax cmp-reg64-imm8                  ~ ASCII linefeed
+  0x0a :rax cmp-reg64-imm8                  ~ : ASCII linefeed
   L@' input-loop-start :cc-equal jmp-cc-rel-imm8-from-here
   ~ : blank-line
 
@@ -138,11 +138,11 @@
   ~ : indent
   current-offset L!' read-byte
   ~ : We use self-xor as a concise way to set registers to zero.
-  :rax :rax xor-reg64-reg64                 ~ syscall number for sys-read
-  :rdi :rdi xor-reg64-reg64                 ~ file descriptor 0 is stdin
-  :rbx :rsi mov-reg64-reg64                 ~ buffer pointer
+  :rax :rax xor-reg64-reg64                 ~ : syscall number for sys-read
+  :rdi :rdi xor-reg64-reg64                 ~ : file descriptor 0 is stdin
+  :rbx :rsi mov-reg64-reg64                 ~ : buffer pointer
   ~ : We read one byte at a time, because it makes the loop structure simple.
-  1 :rdx mov-reg64-imm32                    ~ buffer length
+  1 :rdx mov-reg64-imm32                    ~ : buffer length
   syscall
   ret
   ~ : deindent
@@ -162,13 +162,13 @@
   ~ : indent
   current-offset L!' decode-nibble
 
-  0x30 :rax sub-reg64-imm8                  ~ ASCII zero
+  0x30 :rax sub-reg64-imm8                  ~ : ASCII zero
   ~ If it's negative, jump to the error path.
   L@' invalid-byte :cc-less jmp-cc-rel-imm8-from-here
   10 :rax cmp-reg64-imm8
   ~ This is an unsigned comparison.
   L@' got-nibble :cc-below jmp-cc-rel-imm8-from-here
-  0x41 0x30 - :rax sub-reg64-imm8           ~ ASCII capital A
+  0x41 0x30 - :rax sub-reg64-imm8           ~ : ASCII capital A
   ~ If it's negative, jump to the error path.
   L@' invalid-byte :cc-less jmp-cc-rel-imm8-from-here
   ~ To simplify the range adjustment, we do it unconditionally, before
@@ -177,7 +177,7 @@
   16 :rax cmp-reg64-imm8
   ~ This is an unsigned comparison.
   L@' got-nibble :cc-below jmp-cc-rel-imm8-from-here
-  0x61 0x41 - 10 + :rax sub-reg64-imm8      ~ ASCII lowercase a
+  0x61 0x41 - 10 + :rax sub-reg64-imm8      ~ : ASCII lowercase a
   ~ If it's negative, jump to the error path.
   L@' invalid-byte :cc-less jmp-cc-rel-imm8-from-here
   ~ Again, we adjust the range unconditionally, then check the upper bound.
@@ -201,8 +201,8 @@
   ~ : This is the routine named "exit".
   ~ : indent
   current-offset L!' exit
-  60 :rax mov-reg64-imm32                   ~ syscall number for sys-exit
-  0 :rdi mov-reg64-imm32                    ~ exit code
+  60 :rax mov-reg64-imm32                   ~ : syscall number for sys-exit
+  0 :rdi mov-reg64-imm32                    ~ : exit code
   syscall
   ~ : deindent
   ;
@@ -234,12 +234,12 @@
   ~ Fall through.
 
   current-offset L!' exit-with-error
-  1 :rax mov-reg64-imm32                    ~ syscall number for sys-write
-  2 :rdi mov-reg64-imm32                    ~ file descriptor 2 is stderr
+  1 :rax mov-reg64-imm32                    ~ : syscall number for sys-write
+  2 :rdi mov-reg64-imm32                    ~ : file descriptor 2 is stderr
   syscall
 
-  60 :rax mov-reg64-imm32                   ~ syscall number for sys-exit
-  1 :rdi mov-reg64-imm32                    ~ exit code
+  60 :rax mov-reg64-imm32                   ~ : syscall number for sys-exit
+  1 :rdi mov-reg64-imm32                    ~ : exit code
   syscall
   ~ : deindent
   ;
diff --git a/transform.e b/transform.e
index 124eea9..84d43d0 100644
--- a/transform.e
+++ b/transform.e
@@ -2848,7 +2848,7 @@ allocate-transformation-state s" transformation-state" variable
 ~ can be positive or negative, and can be written in any base Evocation
 ~ normally supports.
 ~
-~   When a parameter is given, and it's not zero, and the comment doesn't
+~   When a parameter is given (even if it's zero), and the comment doesn't
 ~ match some other more-specific syntax, it's treated as a "suffix comment".
 ~ Much like a line comment, when a suffix comment is executed, it creates a
 ~ metadata entry at the end, of -entry-type-suffix-comment, attached to the
@@ -2862,6 +2862,11 @@ allocate-transformation-state s" transformation-state" variable
 ~ instructions should almost always be suffix comments, because this makes it
 ~ easy for a human reader to see which bytes of machine code go with them.
 ~
+~   There's also an implicit way to create a suffix comment, which is to have
+~ non-blank words of any kind on the line prior to the magic comment. In this
+~ case, the default is to be a suffix comment rather than a line comment.
+~ Suffix comments created this way will have zero length.
+~
 ~   Before we move on from magic comments that produce descriptive text, also
 ~ look at the third example above. This is another suffix comment, but it's
 ~ got a negative number as its parameter. It's got a bit of code on the line
@@ -3161,15 +3166,16 @@ allocate-transformation-state s" transformation-state" variable
 : hex-output-metadata-label-loop-buffer-start ;
 : hex-output-metadata-label-loop-buffer-length 8 + ;
 : hex-output-metadata-latest-output-point 2 8 * + ;
-: hex-output-metadata-suppression-count 3 8 * + ;
-: hex-output-metadata-is-fresh-line 4 8 * + ;
-: hex-output-metadata-indentation-depth 5 8 * + ;
-: hex-output-metadata-current-column 6 8 * + ;
-: hex-output-metadata-substring-entry-stack-depth 7 8 * + ;
-: hex-output-metadata-substring-entry-stack-zero 8 8 * + ;
+: hex-output-metadata-has-non-space-this-input-line 3 8 * + ;
+: hex-output-metadata-suppression-count 4 8 * + ;
+: hex-output-metadata-is-fresh-line 5 8 * + ;
+: hex-output-metadata-indentation-depth 6 8 * + ;
+: hex-output-metadata-current-column 7 8 * + ;
+: hex-output-metadata-substring-entry-stack-depth 8 8 * + ;
+: hex-output-metadata-substring-entry-stack-zero 9 8 * + ;
 : hex-output-metadata-substring-entry-stack-capacity 3 ;
 : hex-output-metadata-first-entry
-  8 hex-output-metadata-substring-entry-stack-capacity + 8 * + ;
+  9 hex-output-metadata-substring-entry-stack-capacity + 8 * + ;
 : hex-output-metadata-entry-data-start ;
 : hex-output-metadata-entry-data-length 8 + ;
 : hex-output-metadata-entry-type 2 8 * + ;
@@ -3202,6 +3208,7 @@ allocate-transformation-state s" transformation-state" variable
   dup hex-output-metadata-label-loop-buffer-start 0 swap !
   dup hex-output-metadata-label-loop-buffer-length 0 swap !
   dup hex-output-metadata-latest-output-point 0 swap !
+  dup hex-output-metadata-has-non-space-this-input-line 0 swap !
   dup hex-output-metadata-suppression-count 0 swap !
   dup hex-output-metadata-is-fresh-line 1 swap !
   dup hex-output-metadata-indentation-depth 0 swap !
@@ -3221,6 +3228,14 @@ allocate-transformation-state s" transformation-state" variable
   swap hex-output-metadata-label-loop-buffer-length @ +
   > && ;
 
+: has-non-space-this-input-line@
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-has-non-space-this-input-line @ ;
+
+: has-non-space-this-input-line!
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-has-non-space-this-input-line ! ;
+
 : is-comment-entry
   hex-output-metadata-entry-type @
   dup hex-output-metadata-entry-type-line-comment =
@@ -3419,6 +3434,7 @@ allocate-transformation-state s" transformation-state" variable
   over
   transformation-state transformation-state-output-metadata @
   hex-output-metadata-latest-output-point !
+  0 has-non-space-this-input-line!
 
   over 0 hex-output-metadata-entry-type-line-comment
   s"   This file is a commented hexadecimal representation of a compiled "
@@ -4647,12 +4663,12 @@ allocate-transformation-state s" transformation-state" variable
 ~ substitution occurs before the flow control words start sliding code around,
 ~ and it updates "here" appropriately.
 ~
-~ (string pointer)
+~ (has non-space this line, string pointer --)
 : hex-tilde-replacement
   transformation-state transformation-state-output-metadata @
   hex-output-metadata-latest-output-point @
   swap
-  ~ (data start, string pointer)
+  ~ (has non-space this line, data start, string pointer)
 
   ~   If the output wouldn't go in the label-loop buffer, don't do any of
   ~ this.
@@ -4661,7 +4677,7 @@ allocate-transformation-state s" transformation-state" variable
   ~ are used for compiling the final binary and defining words within the
   ~ compiler. In particular, "variable" will hit this, and there are surely
   ~ others.
-  over is-in-label-loop-buffer not { drop drop exit } if
+  over is-in-label-loop-buffer not { 3 ndrop exit } if
 
   ~   Copy the first space-separated word of the comment body into scratch
   ~ space to see if it's a special instruction. To minimize interference with
@@ -4679,54 +4695,63 @@ allocate-transformation-state s" transformation-state" variable
   here @
   swap-transform-variables
 
-  ~ (data start, string pointer, first word string pointer)
+  ~ (has non-space this line, data start, string pointer,
+  ~  first word string pointer)
   dup read-integer 0 = {
-    ~ (data start, string pointer, first word string pointer, data length)
+    ~ (has non-space, data start, string pointer, first word string pointer,
+    ~  data length)
     3unroll stringlen 1+ +
+    ~ (has non-space, data start, data length, adjusted string pointer)
+    4 roll drop
+    1 swap
+    ~ (data start, data length, is suffix, adjusted string pointer)
   } {
     drop 0 swap
+    ~ (has non-space, data start, data length, string pointer)
+    4 roll swap
+    ~ (data start, data length, is suffix, string pointer)
   } if-else
-  ~ (data start, data length, adjusted string pointer)
+  ~ (data start, data length, is suffix, adjusted string pointer)
 
   ~   Now look for instruction keywords. These keywords are expected to be the
   ~ entire line, except that a length field can come before them.
   dup s" adjust-length" stringcmp 0 = {
     ~ Modify the most recent entry by adjusting its length.
-    drop swap drop adjust-latest-hex-output-metadata-entry-length
+    drop drop swap drop adjust-latest-hex-output-metadata-entry-length
     exit
   } if
 
   dup s" suppress" stringcmp 0 = {
     ~   Suppress the next N entries, or adjust the remaining count of entries
     ~ to suppress.
-    drop swap drop adjust-hex-output-suppression-count
+    drop drop swap drop adjust-hex-output-suppression-count
     exit
   } if
 
   dup s" fresh-line" stringcmp 0 = {
     ~ Create a new fresh-line entry.
-    drop hex-output-metadata-entry-type-fresh-line 0
+    drop drop hex-output-metadata-entry-type-fresh-line 0
     add-hex-output-metadata-entry
     exit
   } if
 
   dup s" blank-line" stringcmp 0 = {
     ~ Create a new blank-line entry.
-    drop hex-output-metadata-entry-type-blank-line 0
+    drop drop hex-output-metadata-entry-type-blank-line 0
     add-hex-output-metadata-entry
     exit
   } if
 
   dup s" indent" stringcmp 0 = {
     ~ Create a new indent entry with a positive indentation delta.
-    drop hex-output-metadata-entry-type-indent 2
+    drop drop hex-output-metadata-entry-type-indent 2
     add-hex-output-metadata-entry
     exit
   } if
 
   dup s" deindent" stringcmp 0 = {
     ~ Create a new indent entry with a negative indentation delta.
-    drop hex-output-metadata-entry-type-indent -2
+    drop drop hex-output-metadata-entry-type-indent -2
     add-hex-output-metadata-entry
     exit
   } if
@@ -4737,44 +4762,51 @@ allocate-transformation-state s" transformation-state" variable
   ~ magic comment is embedded in.
   dup s" provide-decimal" stringcmp 0 = {
     ~ Create a new "push substring decimal" entry.
-    drop 2 pick hex-output-metadata-entry-type-push-substring-decimal swap
+    drop drop
+    2 pick hex-output-metadata-entry-type-push-substring-decimal swap
     add-hex-output-metadata-entry
     exit
   } if
   dup s" provide-hex8" stringcmp 0 = {
     ~ Create a new "push substring hex8" entry.
-    drop 2 pick hex-output-metadata-entry-type-push-substring-hex8 swap
+    drop drop
+    2 pick hex-output-metadata-entry-type-push-substring-hex8 swap
     add-hex-output-metadata-entry
     exit
   } if
   dup s" provide-hex16" stringcmp 0 = {
     ~ Create a new "push substring hex16" entry.
-    drop 2 pick hex-output-metadata-entry-type-push-substring-hex16 swap
+    drop drop
+    2 pick hex-output-metadata-entry-type-push-substring-hex16 swap
     add-hex-output-metadata-entry
     exit
   } if
   dup s" provide-hex32" stringcmp 0 = {
     ~ Create a new "push substring hex32" entry.
-    drop 2 pick hex-output-metadata-entry-type-push-substring-hex32 swap
+    drop drop
+    2 pick hex-output-metadata-entry-type-push-substring-hex32 swap
     add-hex-output-metadata-entry
     exit
   } if
   dup s" provide-hex64" stringcmp 0 = {
     ~ Create a new "push substring hex64" entry.
-    drop 2 pick hex-output-metadata-entry-type-push-substring-hex64 swap
+    drop drop
+    2 pick hex-output-metadata-entry-type-push-substring-hex64 swap
     add-hex-output-metadata-entry
     exit
   } if
   dup s" provide-keyword" stringcmp 0 = {
     ~ Create a new "push substring string" entry given a keyword pointer.
-    drop 2 pick execution-token-to-entry entry-to-name
+    drop drop
+    2 pick execution-token-to-entry entry-to-name
     hex-output-metadata-entry-type-push-substring-string swap
     add-hex-output-metadata-entry
     exit
   } if
 
-  ~ If it has a nonzero length, create a new suffix-comment entry.
-  over 0 != {
+  ~   If it's flagged as a suffix by the logic above, create a new
+  ~ suffix-comment entry.
+  swap {
     hex-output-metadata-entry-type-suffix-comment swap
     add-hex-output-metadata-entry
     exit
@@ -4839,6 +4871,8 @@ allocate-transformation-state s" transformation-state" variable
               ~   Fortunately, we can pack the string directly into the output
               ~ buffer, so we don't need to mess around with
               ~ accumulate-string.
+              s" lit" find entry-to-execution-token ,
+              has-non-space-this-input-line@ ,
               s" litstring" find entry-to-execution-token ,
               here @
               key { dup dup 0x0a != && } {
@@ -4847,6 +4881,9 @@ allocate-transformation-state s" transformation-state" variable
               0 pack8
               8 packalign here !
               ' hex-tilde-replacement entry-to-execution-token ,
+
+              ~ We just consumed a newline, so make note of the fact.
+              0 has-non-space-this-input-line!
             } {
               ~   The third byte was not 0, 0x0a, or 0x20. So our special
               ~ test failed, but the exit condition isn't met. So we're done
@@ -4876,6 +4913,20 @@ allocate-transformation-state s" transformation-state" variable
   } if-else ; make-immediate
 
 
+~   This input helper is a very unusual thing: It's used by hex-transform-one
+~ to skip the space between words. That's necessary because we need to keep
+~ track of whether we've had any non-blank stuff on each line yet, so that the
+~ tilde replacement can figure out certain cases where a magic comment needs
+~ to be a suffix comment rather than a line comment.
+: hex-skip-space
+  { peek dup is-space }
+  { consume
+    ~   This will set the flag once for each newline, if we have multiple
+    ~ consecutive ones or something like that, but that's harmless.
+    0x0a = { 0 has-non-space-this-input-line! } if
+    } while drop ;
+
+
 ~   This implements the hex transform for a single word. It is directly
 ~ analogous to "interpret", and reading interpret.e may help in understanding
 ~ it, though it's meant to still make sense on its own.
@@ -4910,6 +4961,7 @@ allocate-transformation-state s" transformation-state" variable
 ~
 ~ (-- done)
 : hex-transform-one
+  hex-skip-space
   word
 
   ~ If no word was returned, exit.
@@ -4924,6 +4976,9 @@ allocate-transformation-state s" transformation-state" variable
   dup transformation-state transformation-state-delimiter @ stringcmp 0 =
   { drop dropstring 1 exit } if
 
+  ~ If it's anything but tilde, set the input state flag accordingly.
+  dup s" ~" stringcmp 0 != { 1 has-non-space-this-input-line! } if
+
   ~   Check whether it's one of the words we have alternates for, and look up
   ~ the alternate if so.
   0 swap