add keyword support to magic comment templates

additionally, handle many (not yet all) of the keywords used by amd64 words

this exposed an existing flaw in the metadata entry processing, which is now fixed as well. to wit: it's common for provide-substring entries to come after the comment entries they go with in the entry array, but this is valid because adjust-length commands can edit the length field of the comment entry afterwards, and the comment entry will be moved to after the provide-substring entry based on its final length, but this can't happen if the adjust-length array isn't able to find the comment entry it goes with, which may not be the most recent entry. so, the adjust-length command needs to scan backwards to find the most recent comment entry, not just the most recent entry.

confused? don't worry... this CL includes documentation. wow. that was a lot.

Change-Id: I01c574fca020fd2e3adaf335267264fdbc8b87f1
Force-Push: yes please

2 files changed, 595 insertions, 130 deletions

diff --git a/amd64.e b/amd64.e
index e11ec2e..1449bfd 100644
--- a/amd64.e
+++ b/amd64.e
@@ -190,6 +190,7 @@ s" :cc-greater" keyword
 ~
 ~ (register -- 3-bit encoded value for register)
 : reg64
+  ~ : provide-keyword
   dup :rax = { drop 0 exit } if
   dup :rcx = { drop 1 exit } if
   dup :rdx = { drop 2 exit } if
@@ -202,6 +203,7 @@ s" :cc-greater" keyword
 
 ~ (register -- 3-bit encoded value for register)
 : extrareg64
+  ~ : provide-keyword
   dup :r8 = { drop 0 exit } if
   dup :r9 = { drop 1 exit } if
   dup :r10 = { drop 2 exit } if
@@ -214,6 +216,7 @@ s" :cc-greater" keyword
 
 ~ (register -- 3-bit encoded value for register)
 : reg32
+  ~ : provide-keyword
   dup :eax = { drop 0 exit } if
   dup :ecx = { drop 1 exit } if
   dup :edx = { drop 2 exit } if
@@ -226,6 +229,7 @@ s" :cc-greater" keyword
 
 ~ (register -- 3-bit encoded value for register)
 : reg16
+  ~ : provide-keyword
   dup :ax = { drop 0 exit } if
   dup :cx = { drop 1 exit } if
   dup :dx = { drop 2 exit } if
@@ -238,6 +242,7 @@ s" :cc-greater" keyword
 
 ~ (register -- 3-bit encoded value for register)
 : reg8
+  ~ : provide-keyword
   dup :al = { drop 0 exit } if
   dup :cl = { drop 1 exit } if
   dup :dl = { drop 2 exit } if
@@ -643,7 +648,7 @@ s" :cc-greater" keyword
 
 ~ (output point, immediate value, register -- output point)
 : mov-reg64-imm32
-  ~ : 5 # mov-reg64-imm32
+  ~ : 5 # # mov-reg64-imm32
   3roll
   rex-w 0xC7 pack8 swap
   0 swap addressing-reg64
@@ -651,12 +656,12 @@ s" :cc-greater" keyword
   ~ : provide-hex32
   pack32 ;
 : mov-reg64-imm64
-  ~ : 8 # mov-reg64-imm64
+  ~ : 8 # # mov-reg64-imm64
   3roll rex-w swap reg64 0xB8 opcodereg swap
   ~ : provide-hex64
   pack64 ;
 : mov-extrareg64-imm64
-  ~ : 8 # mov-extrareg64-imm64
+  ~ : 8 # # mov-extrareg64-imm64
   ~   Note the use of the B rex bit here; this instruction puts the register
   ~ number in the opcode field, so it uses Table 3-1.
   3roll rex-wb swap extrareg64 0xB8 opcodereg swap
@@ -665,31 +670,31 @@ s" :cc-greater" keyword
 
 ~ (output point, source register, target register -- output point)
 : mov-reg64-reg64
-  ~ : 1 mov-reg64-reg64
+  ~ : 1 # # mov-reg64-reg64
   3roll rex-w 0x89 pack8 3unroll
   swap reg64 swap addressing-reg64 ;
 : mov-indirect-reg64-reg64
-  ~ : 1 mov-indirect-reg64-reg64
+  ~ : 1 # # mov-indirect-reg64-reg64
   3roll rex-w 0x89 pack8 3unroll
   swap reg64 swap addressing-indirect-reg64 ;
 
 ~ (output point, source register, target register, target displacement value
 ~  -- output point)
 : mov-disp8-reg64-reg64
-  ~ : 1 mov-disp8-reg64-reg64
+  ~ : 1 # # mov-disp8-reg64-reg64
   4 roll rex-w 0x89 pack8 4 unroll
   3roll reg64 3unroll addressing-disp8-reg64 ;
 
 ~ (output point, source register, target register -- output point)
 : mov-reg64-indirect-reg64
-  ~ : 1 mov-reg64-indirect-reg64
+  ~ : 1 # # mov-reg64-indirect-reg64
   3roll rex-w 0x8B pack8 3unroll
   reg64 swap addressing-indirect-reg64 ;
 
 ~ (output point, source register, source displacement value, target register
 ~  -- output point)
 : mov-reg64-disp8-reg64
-  ~ : 1 mov-reg64-disp8-reg64
+  ~ : 1 # # mov-reg64-disp8-reg64
   4 roll rex-w 0x8B pack8 4 unroll
   reg64 3unroll addressing-disp8-reg64 ;
 : mov-reg64-disp32-reg64
@@ -1000,25 +1005,25 @@ s" :cc-greater" keyword
 
 ~ (output point, source register, target register -- output point)
 : add-reg64-reg64
-  ~ : 1 add-reg64-reg64
+  ~ : 1 # # add-reg64-reg64
   3roll rex-w 0x01 pack8 3unroll
   swap reg64 swap addressing-reg64 ;
 
 ~ (output point, source register, target register -- output point)
 : add-indirect-reg64-reg64
-  ~ : 1 add-indirect-reg64-reg64
+  ~ : 1 # # add-indirect-reg64-reg64
   3roll rex-w 0x01 pack8 3unroll
   swap reg64 swap addressing-indirect-reg64 ;
 
 ~ (output point, source register, target register -- output point)
 : add-reg64-indirect-reg64
-  ~ : 1 add-reg64-indirect-reg64
+  ~ : 1 # # add-reg64-indirect-reg64
   3roll rex-w 0x03 pack8 3unroll
   reg64 swap addressing-indirect-reg64 ;
 
 ~ (output point, immediate value, target register -- output point)
 : add-reg64-imm8
-  ~ : 2 # add-reg64-imm8
+  ~ : 2 # # add-reg64-imm8
   3roll rex-w 0x83 pack8 swap 0 swap addressing-reg64
   swap
   ~ : provide-hex8
@@ -1028,13 +1033,13 @@ s" :cc-greater" keyword
 ~
 ~ (output point, source register, target register -- output point)
 : sub-reg64-reg64
-  ~ : 1 sub-reg64-reg64
+  ~ : 1 # # sub-reg64-reg64
   3roll rex-w 0x2B pack8 3unroll
   reg64 swap addressing-reg64 ;
 
 ~ (output point, source register, target register -- output point)
 : sub-indirect-reg64-reg64
-  ~ : 1 sub-indirect-reg64-reg64
+  ~ : 1 # # sub-indirect-reg64-reg64
   3roll rex-w 0x2B pack8 3unroll
   swap reg64 swap addressing-indirect-reg64 ;
 
@@ -1042,7 +1047,7 @@ s" :cc-greater" keyword
 ~
 ~ (output point, immediate value, target register -- output point)
 : sub-reg64-imm8
-  ~ : 2 # sub-reg64-imm8
+  ~ : 2 # # sub-reg64-imm8
   3roll rex-w 0x83 pack8 swap 5 swap addressing-reg64
   swap
   ~ : provide-hex8
@@ -1052,7 +1057,7 @@ s" :cc-greater" keyword
 ~
 ~ (output point, source register, target register -- output point)
 : sbb-reg64-imm8
-  ~ : 2 # sbb-reg64-imm8
+  ~ : 2 # # sbb-reg64-imm8
   3roll rex-w 0x83 pack8 swap 3 swap addressing-reg64
   swap
   ~ : provide-hex8
@@ -1062,7 +1067,7 @@ s" :cc-greater" keyword
 ~
 ~ (output point, source register -- output point)
 : mul-reg64
-  ~ : 1 mul-reg64
+  ~ : 1 # mul-reg64
   swap rex-w 0xF7 pack8 swap
   4 swap addressing-reg64 ;
 
@@ -1075,7 +1080,7 @@ s" :cc-greater" keyword
 ~
 ~ (output point, divisor register -- output point)
 : divmod-reg64
-  ~ : 1 divmod-reg64
+  ~ : 1 # divmod-reg64
   swap rex-w 0xF7 pack8 swap
   6 swap addressing-reg64 ;
 
@@ -1083,29 +1088,29 @@ s" :cc-greater" keyword
 ~
 ~ (output point, divisor register -- output point)
 : idivmod-reg64
-  ~ : 1 idivmod-reg64
+  ~ : 1 # idivmod-reg64
   swap rex-w 0xF7 pack8 swap
   7 swap addressing-reg64 ;
 
 ~ (output point, target register -- output point)
 : inc-reg64
-  ~ : 1 inc-reg64
+  ~ : 1 # inc-reg64
   swap rex-w 0xFF pack8 swap 0 swap addressing-reg64 ;
 
 ~ (output point, target register -- output point)
 : dec-reg64
-  ~ : 1 dec-reg64
+  ~ : 1 # dec-reg64
   swap rex-w 0xFF pack8 swap 1 swap addressing-reg64 ;
 
 ~ (output point, source register, target register -- output point)
 : and-reg64-reg64
-  ~ : 1 and-reg64-reg64
+  ~ : 1 # # and-reg64-reg64
   3roll rex-w 0x23 pack8 3unroll
   reg64 swap addressing-reg64 ;
 
 ~ (output point, source value, target register -- output point)
 : and-reg64-imm8
-  ~ : 2 # and-reg64-imm8
+  ~ : 2 # # and-reg64-imm8
   3roll rex-w 0x83 pack8 swap
   4 swap addressing-reg64
   swap
@@ -1114,13 +1119,13 @@ s" :cc-greater" keyword
 
 ~ (output point, source register, target register -- output point)
 : or-reg64-reg64
-  ~ : 1 or-reg64-reg64
+  ~ : 1 # # or-reg64-reg64
   3roll rex-w 0x0B pack8 3unroll
   reg64 swap addressing-reg64 ;
 
 ~ (output point, source value, target register -- output point)
 : or-reg64-imm8
-  ~ : 2 # or-reg64-imm8
+  ~ : 2 # # or-reg64-imm8
   3roll rex-w 0x83 pack8 swap
   1 swap addressing-reg64
   swap
@@ -1129,19 +1134,19 @@ s" :cc-greater" keyword
 
 ~ (output point, source register, target register -- output point)
 : xor-reg64-reg64
-  ~ : 1 xor-reg64-reg64
+  ~ : 1 # # xor-reg64-reg64
   3roll rex-w 0x33 pack8 3unroll
   reg64 swap addressing-reg64 ;
 
 ~ (output point, target register -- output point)
 : not-reg64
-  ~ : 1 not-reg64
+  ~ : 1 # not-reg64
   swap rex-w 0xF7 pack8
   swap 2 swap addressing-reg64 ;
 
 ~ (output point, bit count, target register -- output point)
 : rol-reg64-imm8
-  ~ : 2 # rol-reg64-imm8
+  ~ : 2 # # rol-reg64-imm8
   3roll rex-w 0xC1 pack8 swap
   0 swap addressing-reg64
   swap
@@ -1150,7 +1155,7 @@ s" :cc-greater" keyword
 
 ~ (output point, bit count, target register -- output point)
 : rol-reg8-imm8
-  ~ : 2 # rol-reg8-imm8
+  ~ : 2 # # rol-reg8-imm8
   3roll 0xC0 pack8 swap
   0 swap addressing-reg8
   swap
@@ -1159,7 +1164,7 @@ s" :cc-greater" keyword
 
 ~ (output point, bit count, target register -- output point)
 : ror-reg64-imm8
-  ~ : 1 # ror-reg64-imm8
+  ~ : 1 # # ror-reg64-imm8
   3roll rex-w 0xC1 swap
   1 swap addressing-reg64
   swap
@@ -1168,7 +1173,7 @@ s" :cc-greater" keyword
 
 ~ (output point, bit count, target register -- output point)
 : ror-reg8-imm8
-  ~ : 2 # ror-reg8-imm8
+  ~ : 2 # # ror-reg8-imm8
   3roll 0xC0 pack8 swap
   1 swap addressing-reg64
   swap
@@ -1188,7 +1193,7 @@ s" :cc-greater" keyword
 ~
 ~ (output point, left register, right register -- output point)
 : cmp-reg64-reg64
-  ~ : 1 cmp-reg64-reg64
+  ~ : 1 # # cmp-reg64-reg64
   3roll rex-w 0x3B pack8 3unroll
   reg64 swap addressing-reg64 ;
 
@@ -1196,7 +1201,7 @@ s" :cc-greater" keyword
 ~
 ~ (output point, immediate value, target register -- output point)
 : cmp-reg64-imm8
-  ~ : 2 # cmp-reg64-imm8
+  ~ : 2 # # cmp-reg64-imm8
   3roll rex-w 0x83 pack8
   ~ (immediate value, register, output point)
   swap 7 swap addressing-reg64
@@ -1214,13 +1219,13 @@ s" :cc-greater" keyword
 ~
 ~ (output point, left register, right register -- output point)
 : test-reg64-reg64
-  ~ : 1 test-reg64-reg64
+  ~ : 1 # # test-reg64-reg64
   3roll rex-w 0x85 pack8 3unroll
   swap reg64 swap addressing-reg64 ;
 
 ~ (output point, condition code, target register -- output point)
 : set-reg8-cc
-  ~ : 1 set-reg8-cc
+  ~ : 1 # set-reg8-cc
   3roll 0x0F pack8
   3roll condition-code 0x90 opcodecc
   swap reg8 3 0 3roll modrm ;
@@ -1244,7 +1249,7 @@ s" :cc-greater" keyword
 
 ~ (output point, register -- output point)
 : jmp-abs-indirect-reg64
-  ~ : 1 jmp-abs-indirect-reg64
+  ~ : 1 # jmp-abs-indirect-reg64
   swap 0xFF pack8 swap
   4 swap addressing-indirect-reg64 ;
 
diff --git a/transform.e b/transform.e
index 7d5d5bd..54fd9e6 100644
--- a/transform.e
+++ b/transform.e
@@ -194,10 +194,17 @@
 ~
 ~   The hex transform keeps some large, complex state as it runs, unlike the
 ~ label and log-load transforms, which only require very simple state. This
-~ state is called the "output metadata". It's quite an intense topic, and to
-~ use the hex transform you only need to know that it exists, not how it's
-~ structured. So, details on the output metadata are described below, under
-~ "hex transform implementation", rather than here.
+~ state is called the "output metadata". Entries in the output metadata are
+~ created and modified as the code under transform runs, by "magic comments"
+~ embedded within it. There is a complex command language for these comments,
+~ which must be understood to write code designed to make good use of the hex
+~ transform. Fortunately, that's really just assembly-language
+~ implementations.
+~
+~   It's quite an intense topic, and many readers may wish only to know that
+~ the hex transform exists, not how it's structured. So, details on the output
+~ metadata and the magic comments are described below, under "hex transform
+~ implementation", rather than here.
 ~
 ~   The hex transform DOES NOT WORK yet. It's still in development.
 ~ TODO update this note when it does work
@@ -2712,7 +2719,430 @@ allocate-transformation-state s" transformation-state" variable
 ~     ... + 0x10 - ... + 0x17               String pointer
 ~ (entry fields repeat at successive offsets, until...)
 ~     ... + 0x00 - ... + 0x07 (end)         Zero, used as a delimiter
-
+~
+~
+~ Magic comment syntax
+~ ~~~~~~~~~~~~~~~~~~~~
+~
+~   How, the sufficiently determined reader may ask, does all this output
+~ metadata get created? Surely, the reader may rightly think, creating
+~ metadata entries which fully describe a compilation process must entail
+~ gathering and synthesizing information from disparate stages of that
+~ process, which seems quite involved.
+~
+~   That is correct: It is quite involved. It would be nice if there were a
+~ simple trick, but this appears to be essential complexity. Several
+~ approaches were considered. In principle it would be possible to have a
+~ specific trap defined here in this file for every word that generates
+~ nontrivial output, and keep knowledge of what information to gather from
+~ those words in the trap definitions. However, that would bind the important
+~ semantics quite closely to the already-overwhelming topic of the
+~ transformation data structures, and make it all but impossible to port
+~ Evocation to a new architecture. The knowledge we wish to capture is really
+~ knowledge about an instruction set architecture, not about data structures.
+~
+~   Instead, following the principle that different kinds of knowledge on the
+~ same topic should live close together, we've defined a magic-comment syntax
+~ which allows the assembly-language words in amd64.e to describe their own
+~ metadata, and is versatile enough to do the same for future architectures.
+~
+~   When the compiler operates normally, without the hex transform, these
+~ magic comments are simply regular comments, and are discarded as the
+~ interpreter finds them. When the compiler runs under the hex transform, they
+~ are instead recognized as commands to be processed by the hex transform.
+~ Magic comments can only appear in the body of a word being compiled, not at
+~ the top level. The hex transform intervenes in the compilation process to
+~ insert calls to its own internal words, which are executed, not when the
+~ comment is read, but later, when the word it was compiled into is called.
+~ When the word in question is itself part of the compiler, the magic comments
+~ will be able to see and work with all the state that exists as it runs, and
+~ gather it as they wish.
+~
+~   There are several kinds of magic comment. All of them begin with the
+~ characters : and space as the first two characters of the comment body (and
+~ don't forget that there's a space after ~, as well). The syntax on the rest
+~ of the line determines what type of command it is.
+~
+~   Here are some examples:
+~
+~   ~ : Intriguing commentary goes here.
+~   ~ : 5 This comment describes the next five bytes of output.
+~   777 pack32 ~ : -4 This comment describes the previous four bytes.
+~
+~   ~ : indent
+~   ~ : deindent
+~   ~ : fresh-line
+~   ~ : blank-line
+~
+~   ~ : 1 adjust-length
+~   ~ : 1 suppress
+~
+~   ~ : Do a thing with # and # ?
+~   ~ : provide-hex8
+~   ~ : provide-hex16
+~   ~ : provide-hex32
+~   ~ : provide-hex64
+~   ~ : provide-keyword
+~
+~   By the way, the reason it's possible to write these examples is that
+~ they're all embedded within an outer comment, the text you're reading, so
+~ they don't look like real magic comments to the hex transform. Hopefully
+~ that's not too confusing; there isn't really another way.
+~
+~   So. You'll have noticed that some magic comments seem to be descriptive
+~ text, and others seem to be commands. Every magic comment, in one way or
+~ another, creates or manipulates the metadata entries, which are stored in
+~ an array inside the output metadata as global mutable state. There are
+~ several phases of processing, and if you've gotten this far, you likely need
+~ to understand what they are and what order they happen in.
+~
+~   The simplest kind of comment is the "line comment", the first example
+~ above. Any comment that doesn't match some more-specific syntax is treated
+~ as a line comment. It has not escaped Irenes' notice that this means your
+~ syntax errors will seem to half-work; sorry about that. When a line comment
+~ is executed, it creates a metadata entry at the end of the array of entries,
+~ of -entry-type-line-comment, attached to the latest output point.
+~
+~   This is the general pattern: Any magic comment that creates a metadata
+~ entry will always create it at the end of the entry array, and will always
+~ attach it to the latest output point as of the time the comment executes.
+~ Notice that, although the entry array will eventually be sorted by the
+~ addresses entries are attached to, at the time entries are created they're
+~ simply appended. This is rarely an important distinction, but when it is, it
+~ is.
+~
+~   What is the latest output point? Well, the hex transform has trapped all
+~ the words from core that compilation processes use to directly append bytes
+~ to the label-loop buffer, including pack8, packstring, and the variants
+~ thereof. The traps are able to tell when the destination is in the
+~ label-loop buffer, and they ignore any writes that aren't. The traps keep
+~ track of the next address AFTER whatever was just written, and store that
+~ in the output metadata header so magic comments can make use of it.
+~
+~   It's necessary to take this sneaky approach, because the compiler is free
+~ to write things in any order it wants; the label loop goes back to the start
+~ every so often, and the high-level flow-control words even slide things
+~ around after the fact. The compiler is also free to pass around addresses
+~ however it wants within itself, and without restricting that, there's no way
+~ for the hex transform to know which value represents the place stuff MIGHT
+~ be written to.
+~
+~   The semantics of the latest output point are simple enough that they're
+~ easy to keep in mind while writing magic comments. In practice, the output
+~ point is almost never the cause of confusion about the behavior of magic
+~ comments, but it's important to know about since it's the input the whole
+~ thing starts with.
+~
+~   Now, one thing to keep in mind is that every metadata entry has both an
+~ address in the label-loop buffer that it attaches to, and a length field.
+~ Line comments insert themselves in between lines of output in the hex dump,
+~ and don't otherwise affect the surrounding formatting, so they don't need to
+~ make use of the length and it's always set to zero.
+~
+~   Now the second example above, which starts with the number 5 before its
+~ descriptive text. When the first space-separated word of a magic comment's
+~ body is a valid integer literal, this is taken as a parameter; different
+~ types of magic comment use that parameter in different ways. The parameter
+~ can be positive or negative, and can be written in any base Evocation
+~ normally supports.
+~
+~   When a parameter is given, and it's not zero, and the comment doesn't
+~ match some other more-specific syntax, it's treated as a "suffix comment".
+~ Much like a line comment, when a suffix comment is executed, it creates a
+~ metadata entry at the end, of -entry-type-suffix-comment, attached to the
+~ latest output point. The command uses the parameter as the initial value of
+~ the entry's length field, but it's worth being aware that other commands may
+~ change it later.
+~
+~   It's called a suffix comment because, when all the passes described below
+~ have run and the formatting happens, it will appear at the end of a line of
+~ text, after the byte values it's attached to. Things like assembly-language
+~ instructions should almost always be suffix comments, because this makes it
+~ easy for a human reader to see which bytes of machine code go with them.
+~
+~   Before we move on from magic comments that produce descriptive text, also
+~ look at the third example above. This is another suffix comment, but it's
+~ got a negative number as its parameter. It's got a bit of code on the line
+~ before it, to illustrate the typical way these negative parameters are used.
+~ Just as a suffix comment with a positive length describes a span of bytes
+~ with its start anchored at the address the comment is attached to, so when
+~ the length is negative, the span of bytes has its END anchored at the
+~ address.
+~
+~   You can see examples of how this syntax is useful in elf.e, where it
+~ describes fields of various header structures. When it fits the situation,
+~ this notation is concise and easy to visually skim both in the actual source
+~ code, and in the generated hex dump.
+~
+~   Now let's look at some magic comments that do more interesting things.
+~ The two commands "indent" and "deindent" both create an entry of
+~ -entry-type-indent. Neither takes a parameter as part of the comment syntax,
+~ but the entry structure has an internal value which is set to 2 for "indent"
+~ and -2 for "deindent". When output is eventually formatted, these entries
+~ will be used to adjust the amount of space at the beginning of each line,
+~ which can be very useful to visually group code together. The formatting
+~ process has some mutable state of its own, which doesn't exist until this
+~ very late stage (long after magic comments execute); among other things,
+~ there's a notion of current indentation level, which is preserved until
+~ these entries adjust it.
+~
+~   The commands "fresh-line" and "blank-line" create entries of
+~ -entry-type-fresh-line and -entry-type-blank-line, respectively. Just as the
+~ output formatting knows how deeply indented it is, it also knows whether
+~ it's at the start of a line. A fresh-line entry will insert a line break,
+~ but only if we're somewhere deep in a line; if we're already at the start,
+~ it does nothing. A blank-line entry will do the same thing as fresh-line,
+~ then it will add another line break, so that the result is to output a fully
+~ blank line in the hex dump. This can be useful to break things up visually.
+~
+~   Thus far, though we've taken pains to allude to the fact that there are
+~ many stages of processing of metadata entries after they're created, we
+~ haven't introduced any interesting features that rely on that. That changes
+~ with the "adjust-line" command. When this command executes, it doesn't
+~ create a new metadata entry. Rather, it scans backwards from the end of the
+~ entry array, to find the most recently-created entry which is a line or
+~ suffix comment. Then it adjusts the length of that entry, in-place! The
+~ parameter of adjust-line may be positive or negative, and either way it's
+~ added to the pre-existing length of the entry it modifies.
+~
+~   If that doesn't sound confusing, you may not have fully appreciated that
+~ adjust-line commands don't have to come in the same word as the command that
+~ created the entry they're modifying. For example, in amd64.e, rex-w is a
+~ common helper word called as part of many assembly instructions. It always
+~ outputs a single byte. Commentary in that file describes an accounting
+~ convention whereby keeping track of this byte is rex-w's job, so rex-w also
+~ has a magic comment with an adjust-length command. Whatever word calls rex-w
+~ will have already created a suffix comment giving the name of the
+~ instruction it's implementing, by the time this is called, and rex-w's
+~ call to adjust-length will find that comment's entry and modify it.
+~
+~   The use of this may not seem like much for a word like rex-w, which always
+~ outputs the same number of bytes, but now look at the definition of
+~ addressing-indirect-reg64. This is a common addressing mode shared by many
+~ instructions, and called as a helper from various instruction words. Notice
+~ how it has to generate a different encoding when it's given the register
+~ :rsp, which has an extra SIB byte that's not needed for any other register.
+~ There's no magic comments in addressing-indirect-reg64 itself, but the word
+~ "sib" is another helper which does an appropriate adjust-length, if and only
+~ if sib is actually called.
+~
+~   You can see how it might become quite confusing trying to decide which
+~ word should take responsibility for a particular byte, when there are
+~ multiple layers of abstraction like this. It's especially confusing because
+~ this is an extra form of side-effect which acts on global mutable state, and
+~ is out-of-band compared to the normal Forth value stack. When you're
+~ implementing an instruction set architecture, you should strongly consider
+~ defining an accounting convention which makes that clear and unambiguous.
+~ Feel free to consult the convention described at the top of amd64.e for
+~ inspiration.
+~
+~   We will revisit the topic of adjust-length later, when we describe the
+~ comment template feature, but for now we'll leave it there.
+~
+~   Direct your attention to the "suppress" command. Sometimes, you intend to
+~ call a helper word which is going to create its own entries, but you know
+~ that those entries don't suit your purpose, and you want to ignore them.
+~ For example, you may be calling a word which outputs a binary string and
+~ creates entries describing the string, but you're doing something more
+~ specific and would prefer to create entries that describe your special case,
+~ instead.
+~
+~   When you need this, you can call "suppress" with an integer parameter. If
+~ the parameter is 1, the next time a metadata entry would have been created,
+~ it won't; then things go back to normal. If you want to suppress more than
+~ one entry in a row, you can pass it a larger number. If you change your
+~ mind, you can pass it a negative number. At the time magic comments are
+~ executing, there's a running count of how many entries should be suppressed,
+~ which is decremented any time it's greater than zero and an entry would be
+~ created, and adjusted every time a call to "suppress" is executed.
+~
+~   Take a quick look at the remaining examples. These are part of the comment
+~ template feature. We'll need to describe the other phases of processing
+~ before we can talk about how they work, but notice that one of them has the
+~ space-separated word # inside it, while the others are commands whose names
+~ start with provide-. Any line or suffix comment can have # in it, which will
+~ consume a substring from somewhere else, replacing the # like it's a
+~ template. The various provide- commands are the origins of those substrings.
+~ Which # goes with which provide- command? Thereby hangs a tale.
+~
+~   So. Let's pretend for the moment that we've finished talking about things
+~ that happen when magic comments execute. We haven't, quite, but to finish it
+~ out you'll need the full picture in your head.
+~
+~   All this time, the hex transform has been executing the compiler, which
+~ is running its guts inside a utility called the label loop (see labels.e).
+~ The compiler guts have been calling their own various internal words and
+~ accumulating their binary output in the label-loop buffer. While this is
+~ happening, the hex transform's various traps have been tracking the latest
+~ output point, executing magic comments, and doing other vital but simpler
+~ tasks.
+~
+~   Eventually the label-loop will reach the end of its iteration, and then it
+~ will either conclude that all the labels in the program have been resolved
+~ successfully, or it will notice that some of them were given addresses that
+~ don't work. If the labels aren't all resolved, the label loop will start
+~ over from the beginning, running all the same compiler guts again. The hex
+~ transform will notice that this happened, clear out all the metadata
+~ entries, and restart its own activities as well.
+~
+~   When the label loop finally declares success, and exits, the compiler will
+~ want to actually produce its binary output. The hex transform has trapped
+~ this, too! Now there's a postprocessing phase which cleans up all the
+~ metadata entries, and then finally there's a formatting phase where the
+~ entries are "run", as if they're a program, and the final hex dump with all
+~ its comments is output.
+~
+~   As a side note, the hex transform doesn't stop the compiler short; it lets
+~ the compiler decide which bytes will be output and in what order. In
+~ practice, Evocation and the various supplemental programs around it only
+~ ever write their output in a single giant chunk, but the formatting phase
+~ actually runs every single time the compiler calls sys-write, and it only
+~ handles the portion of the label-loop buffer which the compiler passed to
+~ sys-write.
+~
+~   The postprocessing phase is idempotent - doing it twice is the same as
+~ doing it once - so it's simply called every time sys-write is, before the
+~ formatting phase starts. Postprocessing considers every metadata entry,
+~ every time it runs, even when sys-write is called on something less than
+~ the whole buffer.
+~
+~   If you add postprocessing features, make sure to keep them idempotent.
+~ While it's possible to imagine reasons one might want to have back-and-forth
+~ interplay between successive formatting and postprocessing phases, Evocation
+~ goes to some trouble to make sure there is none. In Irenes' opinion, this
+~ is already quite enough complexity and is nearing the limit of what humans
+~ will find worthwhile. If you break idempotence, you are hereby warned that
+~ you are at risk of falling into the abyss[1].
+~
+~   Postprocessing re-organizes entries, without changing their meaning, but
+~ in a way that will make them easier to format. There are two things it does,
+~ and they happen simultaneously: The whole array is bubble-sorted by the
+~ address it's attached to; and every suffix comment entry with a length
+~ that's not zero is is split apart into a fresh-line entry at the start of
+~ the span it attaches to, and a zero-length suffix comment entry at the end
+~ of the span. The bubble sort is a stable sort; any entries that are already
+~ attached to the same address will stay in whatever order they're in to begin
+~ with.
+~
+~   Please notice that adjust-length commands happen when magic comments are
+~ executed, which was long, long ago by now. By the time the postprocessing
+~ happens, every entry's length has been as adjusted as it's ever going to be.
+~ This is important, because the order of entries is going to matter during
+~ the formatting phase.
+~
+~   Notice also that the splitting of suffix comment entries changes what
+~ address the comment is attached to (unless it's a negative length, which is
+~ already at the end of its span). Any entries that were attached to output
+~ points somewhere inside the span will now definitely be before the suffix
+~ comment's entry in the entry array, whereas up till now they were likely
+~ somewhere after it, depending on the exact order everything was created in.
+~
+~   The postprocessing phase is conceptually simple - unlike the magic-comment
+~ execution phase before it, and unlike the formatting phase after it, there
+~ isn't any mutable state inside the phase that would make it matter what
+~ order things happen in. Enjoy this, because ordering is about to become
+~ super confusing.
+~
+~   Now let's consider the formatting phase. We've already mentioned that
+~ there is some global state which is tracked throughout the phase, such as
+~ the current indentation level and whether the output is at the start of a
+~ line. The formatting phase is given an overall block of bytes in the
+~ label-loop buffer, which it iterates through in order from beginning to end.
+~ Any time the formatting phase reaches an address which has metadata entries
+~ attached to it, all the entries at that address are executed, in the order
+~ they exist in the entry array.
+~
+~   Yes, that's right, executing an entry is different from executing the
+~ magic comment that created the entry. That's important.
+~
+~   The line-comment and suffix-comment entries do the obvious things: They
+~ print out the appropriate comment text. Suffix comments are also all
+~ horizontally aligned with each other, to make them easier to visually skim.
+~ The two kinds of comment entries also do a less-obvious thing, which is fill
+~ in any # template items that are part of the comment text, in order from
+~ left to right.
+~
+~   Where do those items come from? Well, there's a separate stack, the
+~ substring entry stack, which is kept in the hex output metadata header. The
+~ substring entry stack has an extremely small, fixed maximum depth. Unusually
+~ for Evocation, if you underflow or overflow it, you'll get a friendly error
+~ message rather than a segfault - how nice! Of course, the error message will
+~ appear in the middle of your regular output and corrupt it, but you can't
+~ have everything...
+~
+~   Each time a comment template is being filled and encounters a #
+~ placeholder, an item is popped off the substring entry stack and formatted
+~ according to what type of entry it is. It's called an "entry stack" for a
+~ reason: it's a stack of pointers to metadata entries, whose backing store
+~ is in this very same entry array that we're executing. So, all the various
+~ internal fields of an entry are available to provide rich behavior which
+~ allows different kinds of values to be formatted in different ways.
+~
+~   For those used to languages with automatic memory management, keep in mind
+~ that there's no language-provided guarantee that the thing a pointer points
+~ to still exists, and hasn't been replaced with some other
+~ misleadingly-similar thing. The separation of all this stuff into phases
+~ provides that guarantee, because there's nothing in the formatting phase
+~ that destroys or moves metadata entries in the array, but if that separation
+~ were to break down, astonishingly complex wrong behavior would be entirely
+~ likely. If you're reading all this, it's probably to touch that code, so
+~ please keep those consequences in mind.
+~
+~   How do entry pointers get pushed onto the substring entry stack, and where
+~ do the values inside them come from? Okay, now this is the payoff, the part
+~ where you need to understand all these phases at once. Cast your mind all
+~ the way back to the magic-comment execution, and take another look at the
+~ provide-* examples above. Recall that magic comment commands execute when
+~ the word they're contained in is running. For the other comment commands,
+~ this is mostly a curiosity; we talked about adjust-length, above, but that's
+~ still just recording the fact that the word did or didn't run.
+~
+~   The provide-* commands are deeper than that. They peek at the top item on
+~ the stack OF THE COMPILER, the code being transformed. The -hex* variants
+~ create a corresponding provide-substring-hex* metadata entry and copy the
+~ numeric value into it. The -keyword variant expects to find a keyword (a
+~ word which, when executed, pushes its own execution token onto the stack);
+~ the command extracts the name of the keyword from the dictionary entry
+~ header, and saves a pointer to the name in a provide-substring-string
+~ metadata entry.
+~
+~   Admit it, you thought writing out "metadata entry" so many times instead
+~ of just "entry" was silly because there was no other kind of entry in
+~ consideration, right up until you got to "dictionary entry". If so, hey
+~ look, neat, right? ;)
+~
+~   Anyway, notice that when the various provide-* commands execute, all
+~ they're doing is capturing values and creating metadata entries. Like all
+~ metadata entries, these are attached to the latest output point as of the
+~ time the magic comment executes. In particular, the substring entry stack
+~ isn't touched until all the way at the end, in the formatting phase. It's
+~ only when the formatting reaches the address the entry is attached to that
+~ the entry is executed, which has the effect of pushing a pointer to the
+~ entry onto the substring entry stack.
+~
+~   It's been said already, but just to bring it all full circle, the life
+~ cycle of a provide-* entry ends later in the formatting phase, when a
+~ comment entry pops it off the substring entry stack and pretty-prints the
+~ value it holds.
+~
+~   It's probably clear from the extreme emphasis this documentation has taken
+~ on describing the order things happen in, but it is quite common to define
+~ a suffix comment at the start of an assembly-instruction word; adjust its
+~ length as other words are called; accumulate provide-* entries; then rely
+~ on the address sorting to make sure the provide-* entries are executed
+~ before the suffix comment entry, even though the suffix-comment command
+~ executed before the others, and even though the adjust-length commands may
+~ have come before or after the provide-* commands that go with the same
+~ suffix comment.
+~
+~   This is the kind of forwards-and-backwards thinking which humans are
+~ surprisingly good at when reading and writing software, but which it's
+~ quite difficult to formalize in a way that lets a computer share that
+~ understanding. If you've read this far, Irenes would absolutely adore
+~ hearing about how you found the explanation and whether you did anything
+~ with it; don't be shy!
+~
+~ [1] https://katalepsis.net/
 
 ~   We have a bunch of accessors and traversers for the output metadata, which
 ~ are all functions from pointers to pointers. Don't confuse these with the
@@ -2734,6 +3164,7 @@ allocate-transformation-state s" transformation-state" variable
 : hex-output-metadata-entry-type 2 8 * + ;
 : hex-output-metadata-entry-string 3 8 * + ;
 : hex-output-metadata-next-entry 4 8 * + ;
+: hex-output-metadata-previous-entry 4 8 * - ;
 : hex-output-metadata-entry-type-line-comment 0 ;
 : hex-output-metadata-entry-type-suffix-comment 1 ;
 : hex-output-metadata-entry-type-fresh-line 2 ;
@@ -2746,6 +3177,7 @@ allocate-transformation-state s" transformation-state" variable
 : hex-output-metadata-entry-type-provide-substring-hex16 9 ;
 : hex-output-metadata-entry-type-provide-substring-hex32 10 ;
 : hex-output-metadata-entry-type-provide-substring-hex64 11 ;
+: hex-output-metadata-entry-type-provide-substring-string 12 ;
 
 ~   Initialize the contents of the output metadata to all zeroes. This is
 ~ called from hex-transform at its top level, at the very start, to make sure
@@ -2777,6 +3209,95 @@ allocate-transformation-state s" transformation-state" variable
   swap hex-output-metadata-label-loop-buffer-length @ +
   > && ;
 
+: is-comment-entry
+  hex-output-metadata-entry-type @
+  dup hex-output-metadata-entry-type-line-comment =
+  swap hex-output-metadata-entry-type-suffix-comment =
+  || ;
+
+: is-provide-substring-entry
+  hex-output-metadata-entry-type @
+  dup hex-output-metadata-entry-type-provide-substring-hex8 = swap
+  dup hex-output-metadata-entry-type-provide-substring-hex16 = 3roll || swap
+  dup hex-output-metadata-entry-type-provide-substring-hex32 = 3roll || swap
+  dup hex-output-metadata-entry-type-provide-substring-hex64 = 3roll || swap
+  hex-output-metadata-entry-type-provide-substring-string = || ;
+
+: is-fresh-line@
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-is-fresh-line @ ;
+
+: is-fresh-line!
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-is-fresh-line ! ;
+
+: fresh-line is-fresh-line@ not { newline } if ;
+
+: indentation-depth@
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-indentation-depth @ ;
+
+: indentation-depth!
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-indentation-depth ! ;
+
+: current-column@
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-current-column @ ;
+
+: current-column!
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-current-column ! ;
+
+: advance-current-column current-column@ + current-column! ;
+
+: describe-substring-entry-stack
+  transformation-state transformation-state-output-metadata @
+  ."    depth "
+  dup hex-output-metadata-substring-entry-stack-depth @ .hex64 newline
+  ." capacity "
+  hex-output-metadata-substring-entry-stack-capacity .hex64 newline
+  hex-output-metadata-substring-entry-stack-zero
+  hex-output-metadata-substring-entry-stack-capacity 8 * hexdump-from ;
+
+: substring-entry-stack-depth@
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-substring-entry-stack-depth @ ;
+
+: substring-entry-stack-depth!
+  transformation-state transformation-state-output-metadata @
+  hex-output-metadata-substring-entry-stack-depth ! ;
+
+: is-substring-entry-stack-empty
+  substring-entry-stack-depth@ 0 = ;
+
+: is-substring-entry-stack-full
+  substring-entry-stack-depth@
+  hex-output-metadata-substring-entry-stack-capacity = ;
+
+: push-substring-entry-stack
+  is-substring-entry-stack-full not {
+    transformation-state transformation-state-output-metadata @
+    hex-output-metadata-substring-entry-stack-zero
+    substring-entry-stack-depth@ 8 * + !
+
+    substring-entry-stack-depth@ 1+ substring-entry-stack-depth!
+  } {
+    drop
+    ." Substring entry stack overflow." newline
+  } if-else ;
+
+: pop-substring-entry-stack
+  is-substring-entry-stack-empty not {
+    substring-entry-stack-depth@ 1- substring-entry-stack-depth!
+
+    transformation-state transformation-state-output-metadata @
+    hex-output-metadata-substring-entry-stack-zero
+    substring-entry-stack-depth@ 8 * + @
+  } {
+    ." Substring entry stack underflow." newline
+  } if-else ;
+
 ~ (data start, data length, entry type, string pointer, entry pointer --)
 : hex-output-metadata-entry!
   dup hex-output-metadata-entry-string 3roll swap !
@@ -2816,12 +3337,28 @@ allocate-transformation-state s" transformation-state" variable
 
 ~ (length adjustment --)
 : adjust-latest-hex-output-metadata-entry-length
+  ~   First, loop forward from the beginning of the entry array to find the
+  ~ last actual entry.
   transformation-state transformation-state-output-metadata @
   hex-output-metadata-first-entry
   dup @ {
     { dup hex-output-metadata-next-entry @ }
     { hex-output-metadata-next-entry } while
+  } if
+  ~ (entry pointer or 0)
 
+  ~   If and only if we actually found one, now loop backwards to find the
+  ~ most recent one that's a comment entry. All length adjustments apply to
+  ~ comment entries, anything else needs to be skipped for this purpose.
+  { dup { dup is-comment-entry not } { 0 } if-else }
+  { transformation-state transformation-state-output-metadata @
+    hex-output-metadata-first-entry
+    over > {
+      hex-output-metadata-previous-entry } { drop 0 } if-else } while
+  ~ (entry pointer or 0)
+
+  ~ We've found the entry we want to act on, if it exists, so do it.
+  dup {
     hex-output-metadata-entry-data-length dup @ 3roll + swap !
   } {
     drop
@@ -2991,95 +3528,6 @@ allocate-transformation-state s" transformation-state" variable
 
   s" swap" find entry-to-execution-token , ;
 
-: is-comment-entry
-  hex-output-metadata-entry-type @
-  dup hex-output-metadata-entry-type-line-comment =
-  swap hex-output-metadata-entry-type-suffix-comment =
-  || ;
-
-: is-provide-substring-entry
-  hex-output-metadata-entry-type @
-  dup hex-output-metadata-entry-type-provide-substring-hex8 = swap
-  dup hex-output-metadata-entry-type-provide-substring-hex16 = swap
-  dup hex-output-metadata-entry-type-provide-substring-hex32 = swap
-  hex-output-metadata-entry-type-provide-substring-hex64 = swap
-  || || || ;
-
-: is-fresh-line@
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-is-fresh-line @ ;
-
-: is-fresh-line!
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-is-fresh-line ! ;
-
-: fresh-line is-fresh-line@ not { newline } if ;
-
-: indentation-depth@
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-indentation-depth @ ;
-
-: indentation-depth!
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-indentation-depth ! ;
-
-: current-column@
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-current-column @ ;
-
-: current-column!
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-current-column ! ;
-
-: advance-current-column current-column@ + current-column! ;
-
-: describe-substring-entry-stack
-  transformation-state transformation-state-output-metadata @
-  ."    depth "
-  dup hex-output-metadata-substring-entry-stack-depth @ .hex64 newline
-  ." capacity "
-  hex-output-metadata-substring-entry-stack-capacity .hex64 newline
-  hex-output-metadata-substring-entry-stack-zero
-  hex-output-metadata-substring-entry-stack-capacity 8 * hexdump-from ;
-
-: substring-entry-stack-depth@
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-substring-entry-stack-depth @ ;
-
-: substring-entry-stack-depth!
-  transformation-state transformation-state-output-metadata @
-  hex-output-metadata-substring-entry-stack-depth ! ;
-
-: is-substring-entry-stack-empty
-  substring-entry-stack-depth@ 0 = ;
-
-: is-substring-entry-stack-full
-  substring-entry-stack-depth@
-  hex-output-metadata-substring-entry-stack-capacity = ;
-
-: push-substring-entry-stack
-  is-substring-entry-stack-full not {
-    transformation-state transformation-state-output-metadata @
-    hex-output-metadata-substring-entry-stack-zero
-    substring-entry-stack-depth@ 8 * + !
-
-    substring-entry-stack-depth@ 1+ substring-entry-stack-depth!
-  } {
-    drop
-    ." Substring entry stack overflow." newline
-  } if-else ;
-
-: pop-substring-entry-stack
-  is-substring-entry-stack-empty not {
-    substring-entry-stack-depth@ 1- substring-entry-stack-depth!
-
-    transformation-state transformation-state-output-metadata @
-    hex-output-metadata-substring-entry-stack-zero
-    substring-entry-stack-depth@ 8 * + @
-  } {
-    ." Substring entry stack underflow." newline
-  } if-else ;
-
 ~   Metadata entries are post-processed before they're output, in order to
 ~ make the structure of the output code simpler. The post-processing won't
 ~ happen until output actually occurs, which is important for operations that
@@ -3275,6 +3723,11 @@ allocate-transformation-state s" transformation-state" variable
           dup hex-output-metadata-entry-string @ .hex64
         } if
 
+        dup hex-output-metadata-entry-type @
+        hex-output-metadata-entry-type-provide-substring-string = {
+          dup hex-output-metadata-entry-string @ emitstring
+        } if
+
         drop
       } {
         emitstring
@@ -4264,6 +4717,13 @@ allocate-transformation-state s" transformation-state" variable
     add-hex-output-metadata-entry
     exit
   } if
+  dup s" provide-keyword" stringcmp 0 = {
+    ~ Create a new "provide substring string" entry given a keyword pointer.
+    drop 2 pick execution-token-to-entry entry-to-name
+    hex-output-metadata-entry-type-provide-substring-string swap
+    add-hex-output-metadata-entry
+    exit
+  } if
 
   ~ If it has a nonzero length, create a new suffix-comment entry.
   over 0 != {