diff options
| -rw-r--r-- | amd64.e | 77 | ||||
| -rw-r--r-- | transform.e | 648 |
2 files changed, 595 insertions, 130 deletions
diff --git a/amd64.e b/amd64.e index e11ec2e..1449bfd 100644 --- a/amd64.e +++ b/amd64.e @@ -190,6 +190,7 @@ s" :cc-greater" keyword ~ ~ (register -- 3-bit encoded value for register) : reg64 + ~ : provide-keyword dup :rax = { drop 0 exit } if dup :rcx = { drop 1 exit } if dup :rdx = { drop 2 exit } if @@ -202,6 +203,7 @@ s" :cc-greater" keyword ~ (register -- 3-bit encoded value for register) : extrareg64 + ~ : provide-keyword dup :r8 = { drop 0 exit } if dup :r9 = { drop 1 exit } if dup :r10 = { drop 2 exit } if @@ -214,6 +216,7 @@ s" :cc-greater" keyword ~ (register -- 3-bit encoded value for register) : reg32 + ~ : provide-keyword dup :eax = { drop 0 exit } if dup :ecx = { drop 1 exit } if dup :edx = { drop 2 exit } if @@ -226,6 +229,7 @@ s" :cc-greater" keyword ~ (register -- 3-bit encoded value for register) : reg16 + ~ : provide-keyword dup :ax = { drop 0 exit } if dup :cx = { drop 1 exit } if dup :dx = { drop 2 exit } if @@ -238,6 +242,7 @@ s" :cc-greater" keyword ~ (register -- 3-bit encoded value for register) : reg8 + ~ : provide-keyword dup :al = { drop 0 exit } if dup :cl = { drop 1 exit } if dup :dl = { drop 2 exit } if @@ -643,7 +648,7 @@ s" :cc-greater" keyword ~ (output point, immediate value, register -- output point) : mov-reg64-imm32 - ~ : 5 # mov-reg64-imm32 + ~ : 5 # # mov-reg64-imm32 3roll rex-w 0xC7 pack8 swap 0 swap addressing-reg64 @@ -651,12 +656,12 @@ s" :cc-greater" keyword ~ : provide-hex32 pack32 ; : mov-reg64-imm64 - ~ : 8 # mov-reg64-imm64 + ~ : 8 # # mov-reg64-imm64 3roll rex-w swap reg64 0xB8 opcodereg swap ~ : provide-hex64 pack64 ; : mov-extrareg64-imm64 - ~ : 8 # mov-extrareg64-imm64 + ~ : 8 # # mov-extrareg64-imm64 ~ Note the use of the B rex bit here; this instruction puts the register ~ number in the opcode field, so it uses Table 3-1. 3roll rex-wb swap extrareg64 0xB8 opcodereg swap @@ -665,31 +670,31 @@ s" :cc-greater" keyword ~ (output point, source register, target register -- output point) : mov-reg64-reg64 - ~ : 1 mov-reg64-reg64 + ~ : 1 # # mov-reg64-reg64 3roll rex-w 0x89 pack8 3unroll swap reg64 swap addressing-reg64 ; : mov-indirect-reg64-reg64 - ~ : 1 mov-indirect-reg64-reg64 + ~ : 1 # # mov-indirect-reg64-reg64 3roll rex-w 0x89 pack8 3unroll swap reg64 swap addressing-indirect-reg64 ; ~ (output point, source register, target register, target displacement value ~ -- output point) : mov-disp8-reg64-reg64 - ~ : 1 mov-disp8-reg64-reg64 + ~ : 1 # # mov-disp8-reg64-reg64 4 roll rex-w 0x89 pack8 4 unroll 3roll reg64 3unroll addressing-disp8-reg64 ; ~ (output point, source register, target register -- output point) : mov-reg64-indirect-reg64 - ~ : 1 mov-reg64-indirect-reg64 + ~ : 1 # # mov-reg64-indirect-reg64 3roll rex-w 0x8B pack8 3unroll reg64 swap addressing-indirect-reg64 ; ~ (output point, source register, source displacement value, target register ~ -- output point) : mov-reg64-disp8-reg64 - ~ : 1 mov-reg64-disp8-reg64 + ~ : 1 # # mov-reg64-disp8-reg64 4 roll rex-w 0x8B pack8 4 unroll reg64 3unroll addressing-disp8-reg64 ; : mov-reg64-disp32-reg64 @@ -1000,25 +1005,25 @@ s" :cc-greater" keyword ~ (output point, source register, target register -- output point) : add-reg64-reg64 - ~ : 1 add-reg64-reg64 + ~ : 1 # # add-reg64-reg64 3roll rex-w 0x01 pack8 3unroll swap reg64 swap addressing-reg64 ; ~ (output point, source register, target register -- output point) : add-indirect-reg64-reg64 - ~ : 1 add-indirect-reg64-reg64 + ~ : 1 # # add-indirect-reg64-reg64 3roll rex-w 0x01 pack8 3unroll swap reg64 swap addressing-indirect-reg64 ; ~ (output point, source register, target register -- output point) : add-reg64-indirect-reg64 - ~ : 1 add-reg64-indirect-reg64 + ~ : 1 # # add-reg64-indirect-reg64 3roll rex-w 0x03 pack8 3unroll reg64 swap addressing-indirect-reg64 ; ~ (output point, immediate value, target register -- output point) : add-reg64-imm8 - ~ : 2 # add-reg64-imm8 + ~ : 2 # # add-reg64-imm8 3roll rex-w 0x83 pack8 swap 0 swap addressing-reg64 swap ~ : provide-hex8 @@ -1028,13 +1033,13 @@ s" :cc-greater" keyword ~ ~ (output point, source register, target register -- output point) : sub-reg64-reg64 - ~ : 1 sub-reg64-reg64 + ~ : 1 # # sub-reg64-reg64 3roll rex-w 0x2B pack8 3unroll reg64 swap addressing-reg64 ; ~ (output point, source register, target register -- output point) : sub-indirect-reg64-reg64 - ~ : 1 sub-indirect-reg64-reg64 + ~ : 1 # # sub-indirect-reg64-reg64 3roll rex-w 0x2B pack8 3unroll swap reg64 swap addressing-indirect-reg64 ; @@ -1042,7 +1047,7 @@ s" :cc-greater" keyword ~ ~ (output point, immediate value, target register -- output point) : sub-reg64-imm8 - ~ : 2 # sub-reg64-imm8 + ~ : 2 # # sub-reg64-imm8 3roll rex-w 0x83 pack8 swap 5 swap addressing-reg64 swap ~ : provide-hex8 @@ -1052,7 +1057,7 @@ s" :cc-greater" keyword ~ ~ (output point, source register, target register -- output point) : sbb-reg64-imm8 - ~ : 2 # sbb-reg64-imm8 + ~ : 2 # # sbb-reg64-imm8 3roll rex-w 0x83 pack8 swap 3 swap addressing-reg64 swap ~ : provide-hex8 @@ -1062,7 +1067,7 @@ s" :cc-greater" keyword ~ ~ (output point, source register -- output point) : mul-reg64 - ~ : 1 mul-reg64 + ~ : 1 # mul-reg64 swap rex-w 0xF7 pack8 swap 4 swap addressing-reg64 ; @@ -1075,7 +1080,7 @@ s" :cc-greater" keyword ~ ~ (output point, divisor register -- output point) : divmod-reg64 - ~ : 1 divmod-reg64 + ~ : 1 # divmod-reg64 swap rex-w 0xF7 pack8 swap 6 swap addressing-reg64 ; @@ -1083,29 +1088,29 @@ s" :cc-greater" keyword ~ ~ (output point, divisor register -- output point) : idivmod-reg64 - ~ : 1 idivmod-reg64 + ~ : 1 # idivmod-reg64 swap rex-w 0xF7 pack8 swap 7 swap addressing-reg64 ; ~ (output point, target register -- output point) : inc-reg64 - ~ : 1 inc-reg64 + ~ : 1 # inc-reg64 swap rex-w 0xFF pack8 swap 0 swap addressing-reg64 ; ~ (output point, target register -- output point) : dec-reg64 - ~ : 1 dec-reg64 + ~ : 1 # dec-reg64 swap rex-w 0xFF pack8 swap 1 swap addressing-reg64 ; ~ (output point, source register, target register -- output point) : and-reg64-reg64 - ~ : 1 and-reg64-reg64 + ~ : 1 # # and-reg64-reg64 3roll rex-w 0x23 pack8 3unroll reg64 swap addressing-reg64 ; ~ (output point, source value, target register -- output point) : and-reg64-imm8 - ~ : 2 # and-reg64-imm8 + ~ : 2 # # and-reg64-imm8 3roll rex-w 0x83 pack8 swap 4 swap addressing-reg64 swap @@ -1114,13 +1119,13 @@ s" :cc-greater" keyword ~ (output point, source register, target register -- output point) : or-reg64-reg64 - ~ : 1 or-reg64-reg64 + ~ : 1 # # or-reg64-reg64 3roll rex-w 0x0B pack8 3unroll reg64 swap addressing-reg64 ; ~ (output point, source value, target register -- output point) : or-reg64-imm8 - ~ : 2 # or-reg64-imm8 + ~ : 2 # # or-reg64-imm8 3roll rex-w 0x83 pack8 swap 1 swap addressing-reg64 swap @@ -1129,19 +1134,19 @@ s" :cc-greater" keyword ~ (output point, source register, target register -- output point) : xor-reg64-reg64 - ~ : 1 xor-reg64-reg64 + ~ : 1 # # xor-reg64-reg64 3roll rex-w 0x33 pack8 3unroll reg64 swap addressing-reg64 ; ~ (output point, target register -- output point) : not-reg64 - ~ : 1 not-reg64 + ~ : 1 # not-reg64 swap rex-w 0xF7 pack8 swap 2 swap addressing-reg64 ; ~ (output point, bit count, target register -- output point) : rol-reg64-imm8 - ~ : 2 # rol-reg64-imm8 + ~ : 2 # # rol-reg64-imm8 3roll rex-w 0xC1 pack8 swap 0 swap addressing-reg64 swap @@ -1150,7 +1155,7 @@ s" :cc-greater" keyword ~ (output point, bit count, target register -- output point) : rol-reg8-imm8 - ~ : 2 # rol-reg8-imm8 + ~ : 2 # # rol-reg8-imm8 3roll 0xC0 pack8 swap 0 swap addressing-reg8 swap @@ -1159,7 +1164,7 @@ s" :cc-greater" keyword ~ (output point, bit count, target register -- output point) : ror-reg64-imm8 - ~ : 1 # ror-reg64-imm8 + ~ : 1 # # ror-reg64-imm8 3roll rex-w 0xC1 swap 1 swap addressing-reg64 swap @@ -1168,7 +1173,7 @@ s" :cc-greater" keyword ~ (output point, bit count, target register -- output point) : ror-reg8-imm8 - ~ : 2 # ror-reg8-imm8 + ~ : 2 # # ror-reg8-imm8 3roll 0xC0 pack8 swap 1 swap addressing-reg64 swap @@ -1188,7 +1193,7 @@ s" :cc-greater" keyword ~ ~ (output point, left register, right register -- output point) : cmp-reg64-reg64 - ~ : 1 cmp-reg64-reg64 + ~ : 1 # # cmp-reg64-reg64 3roll rex-w 0x3B pack8 3unroll reg64 swap addressing-reg64 ; @@ -1196,7 +1201,7 @@ s" :cc-greater" keyword ~ ~ (output point, immediate value, target register -- output point) : cmp-reg64-imm8 - ~ : 2 # cmp-reg64-imm8 + ~ : 2 # # cmp-reg64-imm8 3roll rex-w 0x83 pack8 ~ (immediate value, register, output point) swap 7 swap addressing-reg64 @@ -1214,13 +1219,13 @@ s" :cc-greater" keyword ~ ~ (output point, left register, right register -- output point) : test-reg64-reg64 - ~ : 1 test-reg64-reg64 + ~ : 1 # # test-reg64-reg64 3roll rex-w 0x85 pack8 3unroll swap reg64 swap addressing-reg64 ; ~ (output point, condition code, target register -- output point) : set-reg8-cc - ~ : 1 set-reg8-cc + ~ : 1 # set-reg8-cc 3roll 0x0F pack8 3roll condition-code 0x90 opcodecc swap reg8 3 0 3roll modrm ; @@ -1244,7 +1249,7 @@ s" :cc-greater" keyword ~ (output point, register -- output point) : jmp-abs-indirect-reg64 - ~ : 1 jmp-abs-indirect-reg64 + ~ : 1 # jmp-abs-indirect-reg64 swap 0xFF pack8 swap 4 swap addressing-indirect-reg64 ; diff --git a/transform.e b/transform.e index 7d5d5bd..54fd9e6 100644 --- a/transform.e +++ b/transform.e @@ -194,10 +194,17 @@ ~ ~ The hex transform keeps some large, complex state as it runs, unlike the ~ label and log-load transforms, which only require very simple state. This -~ state is called the "output metadata". It's quite an intense topic, and to -~ use the hex transform you only need to know that it exists, not how it's -~ structured. So, details on the output metadata are described below, under -~ "hex transform implementation", rather than here. +~ state is called the "output metadata". Entries in the output metadata are +~ created and modified as the code under transform runs, by "magic comments" +~ embedded within it. There is a complex command language for these comments, +~ which must be understood to write code designed to make good use of the hex +~ transform. Fortunately, that's really just assembly-language +~ implementations. +~ +~ It's quite an intense topic, and many readers may wish only to know that +~ the hex transform exists, not how it's structured. So, details on the output +~ metadata and the magic comments are described below, under "hex transform +~ implementation", rather than here. ~ ~ The hex transform DOES NOT WORK yet. It's still in development. ~ TODO update this note when it does work @@ -2712,7 +2719,430 @@ allocate-transformation-state s" transformation-state" variable ~ ... + 0x10 - ... + 0x17 String pointer ~ (entry fields repeat at successive offsets, until...) ~ ... + 0x00 - ... + 0x07 (end) Zero, used as a delimiter - +~ +~ +~ Magic comment syntax +~ ~~~~~~~~~~~~~~~~~~~~ +~ +~ How, the sufficiently determined reader may ask, does all this output +~ metadata get created? Surely, the reader may rightly think, creating +~ metadata entries which fully describe a compilation process must entail +~ gathering and synthesizing information from disparate stages of that +~ process, which seems quite involved. +~ +~ That is correct: It is quite involved. It would be nice if there were a +~ simple trick, but this appears to be essential complexity. Several +~ approaches were considered. In principle it would be possible to have a +~ specific trap defined here in this file for every word that generates +~ nontrivial output, and keep knowledge of what information to gather from +~ those words in the trap definitions. However, that would bind the important +~ semantics quite closely to the already-overwhelming topic of the +~ transformation data structures, and make it all but impossible to port +~ Evocation to a new architecture. The knowledge we wish to capture is really +~ knowledge about an instruction set architecture, not about data structures. +~ +~ Instead, following the principle that different kinds of knowledge on the +~ same topic should live close together, we've defined a magic-comment syntax +~ which allows the assembly-language words in amd64.e to describe their own +~ metadata, and is versatile enough to do the same for future architectures. +~ +~ When the compiler operates normally, without the hex transform, these +~ magic comments are simply regular comments, and are discarded as the +~ interpreter finds them. When the compiler runs under the hex transform, they +~ are instead recognized as commands to be processed by the hex transform. +~ Magic comments can only appear in the body of a word being compiled, not at +~ the top level. The hex transform intervenes in the compilation process to +~ insert calls to its own internal words, which are executed, not when the +~ comment is read, but later, when the word it was compiled into is called. +~ When the word in question is itself part of the compiler, the magic comments +~ will be able to see and work with all the state that exists as it runs, and +~ gather it as they wish. +~ +~ There are several kinds of magic comment. All of them begin with the +~ characters : and space as the first two characters of the comment body (and +~ don't forget that there's a space after ~, as well). The syntax on the rest +~ of the line determines what type of command it is. +~ +~ Here are some examples: +~ +~ ~ : Intriguing commentary goes here. +~ ~ : 5 This comment describes the next five bytes of output. +~ 777 pack32 ~ : -4 This comment describes the previous four bytes. +~ +~ ~ : indent +~ ~ : deindent +~ ~ : fresh-line +~ ~ : blank-line +~ +~ ~ : 1 adjust-length +~ ~ : 1 suppress +~ +~ ~ : Do a thing with # and # ? +~ ~ : provide-hex8 +~ ~ : provide-hex16 +~ ~ : provide-hex32 +~ ~ : provide-hex64 +~ ~ : provide-keyword +~ +~ By the way, the reason it's possible to write these examples is that +~ they're all embedded within an outer comment, the text you're reading, so +~ they don't look like real magic comments to the hex transform. Hopefully +~ that's not too confusing; there isn't really another way. +~ +~ So. You'll have noticed that some magic comments seem to be descriptive +~ text, and others seem to be commands. Every magic comment, in one way or +~ another, creates or manipulates the metadata entries, which are stored in +~ an array inside the output metadata as global mutable state. There are +~ several phases of processing, and if you've gotten this far, you likely need +~ to understand what they are and what order they happen in. +~ +~ The simplest kind of comment is the "line comment", the first example +~ above. Any comment that doesn't match some more-specific syntax is treated +~ as a line comment. It has not escaped Irenes' notice that this means your +~ syntax errors will seem to half-work; sorry about that. When a line comment +~ is executed, it creates a metadata entry at the end of the array of entries, +~ of -entry-type-line-comment, attached to the latest output point. +~ +~ This is the general pattern: Any magic comment that creates a metadata +~ entry will always create it at the end of the entry array, and will always +~ attach it to the latest output point as of the time the comment executes. +~ Notice that, although the entry array will eventually be sorted by the +~ addresses entries are attached to, at the time entries are created they're +~ simply appended. This is rarely an important distinction, but when it is, it +~ is. +~ +~ What is the latest output point? Well, the hex transform has trapped all +~ the words from core that compilation processes use to directly append bytes +~ to the label-loop buffer, including pack8, packstring, and the variants +~ thereof. The traps are able to tell when the destination is in the +~ label-loop buffer, and they ignore any writes that aren't. The traps keep +~ track of the next address AFTER whatever was just written, and store that +~ in the output metadata header so magic comments can make use of it. +~ +~ It's necessary to take this sneaky approach, because the compiler is free +~ to write things in any order it wants; the label loop goes back to the start +~ every so often, and the high-level flow-control words even slide things +~ around after the fact. The compiler is also free to pass around addresses +~ however it wants within itself, and without restricting that, there's no way +~ for the hex transform to know which value represents the place stuff MIGHT +~ be written to. +~ +~ The semantics of the latest output point are simple enough that they're +~ easy to keep in mind while writing magic comments. In practice, the output +~ point is almost never the cause of confusion about the behavior of magic +~ comments, but it's important to know about since it's the input the whole +~ thing starts with. +~ +~ Now, one thing to keep in mind is that every metadata entry has both an +~ address in the label-loop buffer that it attaches to, and a length field. +~ Line comments insert themselves in between lines of output in the hex dump, +~ and don't otherwise affect the surrounding formatting, so they don't need to +~ make use of the length and it's always set to zero. +~ +~ Now the second example above, which starts with the number 5 before its +~ descriptive text. When the first space-separated word of a magic comment's +~ body is a valid integer literal, this is taken as a parameter; different +~ types of magic comment use that parameter in different ways. The parameter +~ can be positive or negative, and can be written in any base Evocation +~ normally supports. +~ +~ When a parameter is given, and it's not zero, and the comment doesn't +~ match some other more-specific syntax, it's treated as a "suffix comment". +~ Much like a line comment, when a suffix comment is executed, it creates a +~ metadata entry at the end, of -entry-type-suffix-comment, attached to the +~ latest output point. The command uses the parameter as the initial value of +~ the entry's length field, but it's worth being aware that other commands may +~ change it later. +~ +~ It's called a suffix comment because, when all the passes described below +~ have run and the formatting happens, it will appear at the end of a line of +~ text, after the byte values it's attached to. Things like assembly-language +~ instructions should almost always be suffix comments, because this makes it +~ easy for a human reader to see which bytes of machine code go with them. +~ +~ Before we move on from magic comments that produce descriptive text, also +~ look at the third example above. This is another suffix comment, but it's +~ got a negative number as its parameter. It's got a bit of code on the line +~ before it, to illustrate the typical way these negative parameters are used. +~ Just as a suffix comment with a positive length describes a span of bytes +~ with its start anchored at the address the comment is attached to, so when +~ the length is negative, the span of bytes has its END anchored at the +~ address. +~ +~ You can see examples of how this syntax is useful in elf.e, where it +~ describes fields of various header structures. When it fits the situation, +~ this notation is concise and easy to visually skim both in the actual source +~ code, and in the generated hex dump. +~ +~ Now let's look at some magic comments that do more interesting things. +~ The two commands "indent" and "deindent" both create an entry of +~ -entry-type-indent. Neither takes a parameter as part of the comment syntax, +~ but the entry structure has an internal value which is set to 2 for "indent" +~ and -2 for "deindent". When output is eventually formatted, these entries +~ will be used to adjust the amount of space at the beginning of each line, +~ which can be very useful to visually group code together. The formatting +~ process has some mutable state of its own, which doesn't exist until this +~ very late stage (long after magic comments execute); among other things, +~ there's a notion of current indentation level, which is preserved until +~ these entries adjust it. +~ +~ The commands "fresh-line" and "blank-line" create entries of +~ -entry-type-fresh-line and -entry-type-blank-line, respectively. Just as the +~ output formatting knows how deeply indented it is, it also knows whether +~ it's at the start of a line. A fresh-line entry will insert a line break, +~ but only if we're somewhere deep in a line; if we're already at the start, +~ it does nothing. A blank-line entry will do the same thing as fresh-line, +~ then it will add another line break, so that the result is to output a fully +~ blank line in the hex dump. This can be useful to break things up visually. +~ +~ Thus far, though we've taken pains to allude to the fact that there are +~ many stages of processing of metadata entries after they're created, we +~ haven't introduced any interesting features that rely on that. That changes +~ with the "adjust-line" command. When this command executes, it doesn't +~ create a new metadata entry. Rather, it scans backwards from the end of the +~ entry array, to find the most recently-created entry which is a line or +~ suffix comment. Then it adjusts the length of that entry, in-place! The +~ parameter of adjust-line may be positive or negative, and either way it's +~ added to the pre-existing length of the entry it modifies. +~ +~ If that doesn't sound confusing, you may not have fully appreciated that +~ adjust-line commands don't have to come in the same word as the command that +~ created the entry they're modifying. For example, in amd64.e, rex-w is a +~ common helper word called as part of many assembly instructions. It always +~ outputs a single byte. Commentary in that file describes an accounting +~ convention whereby keeping track of this byte is rex-w's job, so rex-w also +~ has a magic comment with an adjust-length command. Whatever word calls rex-w +~ will have already created a suffix comment giving the name of the +~ instruction it's implementing, by the time this is called, and rex-w's +~ call to adjust-length will find that comment's entry and modify it. +~ +~ The use of this may not seem like much for a word like rex-w, which always +~ outputs the same number of bytes, but now look at the definition of +~ addressing-indirect-reg64. This is a common addressing mode shared by many +~ instructions, and called as a helper from various instruction words. Notice +~ how it has to generate a different encoding when it's given the register +~ :rsp, which has an extra SIB byte that's not needed for any other register. +~ There's no magic comments in addressing-indirect-reg64 itself, but the word +~ "sib" is another helper which does an appropriate adjust-length, if and only +~ if sib is actually called. +~ +~ You can see how it might become quite confusing trying to decide which +~ word should take responsibility for a particular byte, when there are +~ multiple layers of abstraction like this. It's especially confusing because +~ this is an extra form of side-effect which acts on global mutable state, and +~ is out-of-band compared to the normal Forth value stack. When you're +~ implementing an instruction set architecture, you should strongly consider +~ defining an accounting convention which makes that clear and unambiguous. +~ Feel free to consult the convention described at the top of amd64.e for +~ inspiration. +~ +~ We will revisit the topic of adjust-length later, when we describe the +~ comment template feature, but for now we'll leave it there. +~ +~ Direct your attention to the "suppress" command. Sometimes, you intend to +~ call a helper word which is going to create its own entries, but you know +~ that those entries don't suit your purpose, and you want to ignore them. +~ For example, you may be calling a word which outputs a binary string and +~ creates entries describing the string, but you're doing something more +~ specific and would prefer to create entries that describe your special case, +~ instead. +~ +~ When you need this, you can call "suppress" with an integer parameter. If +~ the parameter is 1, the next time a metadata entry would have been created, +~ it won't; then things go back to normal. If you want to suppress more than +~ one entry in a row, you can pass it a larger number. If you change your +~ mind, you can pass it a negative number. At the time magic comments are +~ executing, there's a running count of how many entries should be suppressed, +~ which is decremented any time it's greater than zero and an entry would be +~ created, and adjusted every time a call to "suppress" is executed. +~ +~ Take a quick look at the remaining examples. These are part of the comment +~ template feature. We'll need to describe the other phases of processing +~ before we can talk about how they work, but notice that one of them has the +~ space-separated word # inside it, while the others are commands whose names +~ start with provide-. Any line or suffix comment can have # in it, which will +~ consume a substring from somewhere else, replacing the # like it's a +~ template. The various provide- commands are the origins of those substrings. +~ Which # goes with which provide- command? Thereby hangs a tale. +~ +~ So. Let's pretend for the moment that we've finished talking about things +~ that happen when magic comments execute. We haven't, quite, but to finish it +~ out you'll need the full picture in your head. +~ +~ All this time, the hex transform has been executing the compiler, which +~ is running its guts inside a utility called the label loop (see labels.e). +~ The compiler guts have been calling their own various internal words and +~ accumulating their binary output in the label-loop buffer. While this is +~ happening, the hex transform's various traps have been tracking the latest +~ output point, executing magic comments, and doing other vital but simpler +~ tasks. +~ +~ Eventually the label-loop will reach the end of its iteration, and then it +~ will either conclude that all the labels in the program have been resolved +~ successfully, or it will notice that some of them were given addresses that +~ don't work. If the labels aren't all resolved, the label loop will start +~ over from the beginning, running all the same compiler guts again. The hex +~ transform will notice that this happened, clear out all the metadata +~ entries, and restart its own activities as well. +~ +~ When the label loop finally declares success, and exits, the compiler will +~ want to actually produce its binary output. The hex transform has trapped +~ this, too! Now there's a postprocessing phase which cleans up all the +~ metadata entries, and then finally there's a formatting phase where the +~ entries are "run", as if they're a program, and the final hex dump with all +~ its comments is output. +~ +~ As a side note, the hex transform doesn't stop the compiler short; it lets +~ the compiler decide which bytes will be output and in what order. In +~ practice, Evocation and the various supplemental programs around it only +~ ever write their output in a single giant chunk, but the formatting phase +~ actually runs every single time the compiler calls sys-write, and it only +~ handles the portion of the label-loop buffer which the compiler passed to +~ sys-write. +~ +~ The postprocessing phase is idempotent - doing it twice is the same as +~ doing it once - so it's simply called every time sys-write is, before the +~ formatting phase starts. Postprocessing considers every metadata entry, +~ every time it runs, even when sys-write is called on something less than +~ the whole buffer. +~ +~ If you add postprocessing features, make sure to keep them idempotent. +~ While it's possible to imagine reasons one might want to have back-and-forth +~ interplay between successive formatting and postprocessing phases, Evocation +~ goes to some trouble to make sure there is none. In Irenes' opinion, this +~ is already quite enough complexity and is nearing the limit of what humans +~ will find worthwhile. If you break idempotence, you are hereby warned that +~ you are at risk of falling into the abyss[1]. +~ +~ Postprocessing re-organizes entries, without changing their meaning, but +~ in a way that will make them easier to format. There are two things it does, +~ and they happen simultaneously: The whole array is bubble-sorted by the +~ address it's attached to; and every suffix comment entry with a length +~ that's not zero is is split apart into a fresh-line entry at the start of +~ the span it attaches to, and a zero-length suffix comment entry at the end +~ of the span. The bubble sort is a stable sort; any entries that are already +~ attached to the same address will stay in whatever order they're in to begin +~ with. +~ +~ Please notice that adjust-length commands happen when magic comments are +~ executed, which was long, long ago by now. By the time the postprocessing +~ happens, every entry's length has been as adjusted as it's ever going to be. +~ This is important, because the order of entries is going to matter during +~ the formatting phase. +~ +~ Notice also that the splitting of suffix comment entries changes what +~ address the comment is attached to (unless it's a negative length, which is +~ already at the end of its span). Any entries that were attached to output +~ points somewhere inside the span will now definitely be before the suffix +~ comment's entry in the entry array, whereas up till now they were likely +~ somewhere after it, depending on the exact order everything was created in. +~ +~ The postprocessing phase is conceptually simple - unlike the magic-comment +~ execution phase before it, and unlike the formatting phase after it, there +~ isn't any mutable state inside the phase that would make it matter what +~ order things happen in. Enjoy this, because ordering is about to become +~ super confusing. +~ +~ Now let's consider the formatting phase. We've already mentioned that +~ there is some global state which is tracked throughout the phase, such as +~ the current indentation level and whether the output is at the start of a +~ line. The formatting phase is given an overall block of bytes in the +~ label-loop buffer, which it iterates through in order from beginning to end. +~ Any time the formatting phase reaches an address which has metadata entries +~ attached to it, all the entries at that address are executed, in the order +~ they exist in the entry array. +~ +~ Yes, that's right, executing an entry is different from executing the +~ magic comment that created the entry. That's important. +~ +~ The line-comment and suffix-comment entries do the obvious things: They +~ print out the appropriate comment text. Suffix comments are also all +~ horizontally aligned with each other, to make them easier to visually skim. +~ The two kinds of comment entries also do a less-obvious thing, which is fill +~ in any # template items that are part of the comment text, in order from +~ left to right. +~ +~ Where do those items come from? Well, there's a separate stack, the +~ substring entry stack, which is kept in the hex output metadata header. The +~ substring entry stack has an extremely small, fixed maximum depth. Unusually +~ for Evocation, if you underflow or overflow it, you'll get a friendly error +~ message rather than a segfault - how nice! Of course, the error message will +~ appear in the middle of your regular output and corrupt it, but you can't +~ have everything... +~ +~ Each time a comment template is being filled and encounters a # +~ placeholder, an item is popped off the substring entry stack and formatted +~ according to what type of entry it is. It's called an "entry stack" for a +~ reason: it's a stack of pointers to metadata entries, whose backing store +~ is in this very same entry array that we're executing. So, all the various +~ internal fields of an entry are available to provide rich behavior which +~ allows different kinds of values to be formatted in different ways. +~ +~ For those used to languages with automatic memory management, keep in mind +~ that there's no language-provided guarantee that the thing a pointer points +~ to still exists, and hasn't been replaced with some other +~ misleadingly-similar thing. The separation of all this stuff into phases +~ provides that guarantee, because there's nothing in the formatting phase +~ that destroys or moves metadata entries in the array, but if that separation +~ were to break down, astonishingly complex wrong behavior would be entirely +~ likely. If you're reading all this, it's probably to touch that code, so +~ please keep those consequences in mind. +~ +~ How do entry pointers get pushed onto the substring entry stack, and where +~ do the values inside them come from? Okay, now this is the payoff, the part +~ where you need to understand all these phases at once. Cast your mind all +~ the way back to the magic-comment execution, and take another look at the +~ provide-* examples above. Recall that magic comment commands execute when +~ the word they're contained in is running. For the other comment commands, +~ this is mostly a curiosity; we talked about adjust-length, above, but that's +~ still just recording the fact that the word did or didn't run. +~ +~ The provide-* commands are deeper than that. They peek at the top item on +~ the stack OF THE COMPILER, the code being transformed. The -hex* variants +~ create a corresponding provide-substring-hex* metadata entry and copy the +~ numeric value into it. The -keyword variant expects to find a keyword (a +~ word which, when executed, pushes its own execution token onto the stack); +~ the command extracts the name of the keyword from the dictionary entry +~ header, and saves a pointer to the name in a provide-substring-string +~ metadata entry. +~ +~ Admit it, you thought writing out "metadata entry" so many times instead +~ of just "entry" was silly because there was no other kind of entry in +~ consideration, right up until you got to "dictionary entry". If so, hey +~ look, neat, right? ;) +~ +~ Anyway, notice that when the various provide-* commands execute, all +~ they're doing is capturing values and creating metadata entries. Like all +~ metadata entries, these are attached to the latest output point as of the +~ time the magic comment executes. In particular, the substring entry stack +~ isn't touched until all the way at the end, in the formatting phase. It's +~ only when the formatting reaches the address the entry is attached to that +~ the entry is executed, which has the effect of pushing a pointer to the +~ entry onto the substring entry stack. +~ +~ It's been said already, but just to bring it all full circle, the life +~ cycle of a provide-* entry ends later in the formatting phase, when a +~ comment entry pops it off the substring entry stack and pretty-prints the +~ value it holds. +~ +~ It's probably clear from the extreme emphasis this documentation has taken +~ on describing the order things happen in, but it is quite common to define +~ a suffix comment at the start of an assembly-instruction word; adjust its +~ length as other words are called; accumulate provide-* entries; then rely +~ on the address sorting to make sure the provide-* entries are executed +~ before the suffix comment entry, even though the suffix-comment command +~ executed before the others, and even though the adjust-length commands may +~ have come before or after the provide-* commands that go with the same +~ suffix comment. +~ +~ This is the kind of forwards-and-backwards thinking which humans are +~ surprisingly good at when reading and writing software, but which it's +~ quite difficult to formalize in a way that lets a computer share that +~ understanding. If you've read this far, Irenes would absolutely adore +~ hearing about how you found the explanation and whether you did anything +~ with it; don't be shy! +~ +~ [1] https://katalepsis.net/ ~ We have a bunch of accessors and traversers for the output metadata, which ~ are all functions from pointers to pointers. Don't confuse these with the @@ -2734,6 +3164,7 @@ allocate-transformation-state s" transformation-state" variable : hex-output-metadata-entry-type 2 8 * + ; : hex-output-metadata-entry-string 3 8 * + ; : hex-output-metadata-next-entry 4 8 * + ; +: hex-output-metadata-previous-entry 4 8 * - ; : hex-output-metadata-entry-type-line-comment 0 ; : hex-output-metadata-entry-type-suffix-comment 1 ; : hex-output-metadata-entry-type-fresh-line 2 ; @@ -2746,6 +3177,7 @@ allocate-transformation-state s" transformation-state" variable : hex-output-metadata-entry-type-provide-substring-hex16 9 ; : hex-output-metadata-entry-type-provide-substring-hex32 10 ; : hex-output-metadata-entry-type-provide-substring-hex64 11 ; +: hex-output-metadata-entry-type-provide-substring-string 12 ; ~ Initialize the contents of the output metadata to all zeroes. This is ~ called from hex-transform at its top level, at the very start, to make sure @@ -2777,6 +3209,95 @@ allocate-transformation-state s" transformation-state" variable swap hex-output-metadata-label-loop-buffer-length @ + > && ; +: is-comment-entry + hex-output-metadata-entry-type @ + dup hex-output-metadata-entry-type-line-comment = + swap hex-output-metadata-entry-type-suffix-comment = + || ; + +: is-provide-substring-entry + hex-output-metadata-entry-type @ + dup hex-output-metadata-entry-type-provide-substring-hex8 = swap + dup hex-output-metadata-entry-type-provide-substring-hex16 = 3roll || swap + dup hex-output-metadata-entry-type-provide-substring-hex32 = 3roll || swap + dup hex-output-metadata-entry-type-provide-substring-hex64 = 3roll || swap + hex-output-metadata-entry-type-provide-substring-string = || ; + +: is-fresh-line@ + transformation-state transformation-state-output-metadata @ + hex-output-metadata-is-fresh-line @ ; + +: is-fresh-line! + transformation-state transformation-state-output-metadata @ + hex-output-metadata-is-fresh-line ! ; + +: fresh-line is-fresh-line@ not { newline } if ; + +: indentation-depth@ + transformation-state transformation-state-output-metadata @ + hex-output-metadata-indentation-depth @ ; + +: indentation-depth! + transformation-state transformation-state-output-metadata @ + hex-output-metadata-indentation-depth ! ; + +: current-column@ + transformation-state transformation-state-output-metadata @ + hex-output-metadata-current-column @ ; + +: current-column! + transformation-state transformation-state-output-metadata @ + hex-output-metadata-current-column ! ; + +: advance-current-column current-column@ + current-column! ; + +: describe-substring-entry-stack + transformation-state transformation-state-output-metadata @ + ." depth " + dup hex-output-metadata-substring-entry-stack-depth @ .hex64 newline + ." capacity " + hex-output-metadata-substring-entry-stack-capacity .hex64 newline + hex-output-metadata-substring-entry-stack-zero + hex-output-metadata-substring-entry-stack-capacity 8 * hexdump-from ; + +: substring-entry-stack-depth@ + transformation-state transformation-state-output-metadata @ + hex-output-metadata-substring-entry-stack-depth @ ; + +: substring-entry-stack-depth! + transformation-state transformation-state-output-metadata @ + hex-output-metadata-substring-entry-stack-depth ! ; + +: is-substring-entry-stack-empty + substring-entry-stack-depth@ 0 = ; + +: is-substring-entry-stack-full + substring-entry-stack-depth@ + hex-output-metadata-substring-entry-stack-capacity = ; + +: push-substring-entry-stack + is-substring-entry-stack-full not { + transformation-state transformation-state-output-metadata @ + hex-output-metadata-substring-entry-stack-zero + substring-entry-stack-depth@ 8 * + ! + + substring-entry-stack-depth@ 1+ substring-entry-stack-depth! + } { + drop + ." Substring entry stack overflow." newline + } if-else ; + +: pop-substring-entry-stack + is-substring-entry-stack-empty not { + substring-entry-stack-depth@ 1- substring-entry-stack-depth! + + transformation-state transformation-state-output-metadata @ + hex-output-metadata-substring-entry-stack-zero + substring-entry-stack-depth@ 8 * + @ + } { + ." Substring entry stack underflow." newline + } if-else ; + ~ (data start, data length, entry type, string pointer, entry pointer --) : hex-output-metadata-entry! dup hex-output-metadata-entry-string 3roll swap ! @@ -2816,12 +3337,28 @@ allocate-transformation-state s" transformation-state" variable ~ (length adjustment --) : adjust-latest-hex-output-metadata-entry-length + ~ First, loop forward from the beginning of the entry array to find the + ~ last actual entry. transformation-state transformation-state-output-metadata @ hex-output-metadata-first-entry dup @ { { dup hex-output-metadata-next-entry @ } { hex-output-metadata-next-entry } while + } if + ~ (entry pointer or 0) + ~ If and only if we actually found one, now loop backwards to find the + ~ most recent one that's a comment entry. All length adjustments apply to + ~ comment entries, anything else needs to be skipped for this purpose. + { dup { dup is-comment-entry not } { 0 } if-else } + { transformation-state transformation-state-output-metadata @ + hex-output-metadata-first-entry + over > { + hex-output-metadata-previous-entry } { drop 0 } if-else } while + ~ (entry pointer or 0) + + ~ We've found the entry we want to act on, if it exists, so do it. + dup { hex-output-metadata-entry-data-length dup @ 3roll + swap ! } { drop @@ -2991,95 +3528,6 @@ allocate-transformation-state s" transformation-state" variable s" swap" find entry-to-execution-token , ; -: is-comment-entry - hex-output-metadata-entry-type @ - dup hex-output-metadata-entry-type-line-comment = - swap hex-output-metadata-entry-type-suffix-comment = - || ; - -: is-provide-substring-entry - hex-output-metadata-entry-type @ - dup hex-output-metadata-entry-type-provide-substring-hex8 = swap - dup hex-output-metadata-entry-type-provide-substring-hex16 = swap - dup hex-output-metadata-entry-type-provide-substring-hex32 = swap - hex-output-metadata-entry-type-provide-substring-hex64 = swap - || || || ; - -: is-fresh-line@ - transformation-state transformation-state-output-metadata @ - hex-output-metadata-is-fresh-line @ ; - -: is-fresh-line! - transformation-state transformation-state-output-metadata @ - hex-output-metadata-is-fresh-line ! ; - -: fresh-line is-fresh-line@ not { newline } if ; - -: indentation-depth@ - transformation-state transformation-state-output-metadata @ - hex-output-metadata-indentation-depth @ ; - -: indentation-depth! - transformation-state transformation-state-output-metadata @ - hex-output-metadata-indentation-depth ! ; - -: current-column@ - transformation-state transformation-state-output-metadata @ - hex-output-metadata-current-column @ ; - -: current-column! - transformation-state transformation-state-output-metadata @ - hex-output-metadata-current-column ! ; - -: advance-current-column current-column@ + current-column! ; - -: describe-substring-entry-stack - transformation-state transformation-state-output-metadata @ - ." depth " - dup hex-output-metadata-substring-entry-stack-depth @ .hex64 newline - ." capacity " - hex-output-metadata-substring-entry-stack-capacity .hex64 newline - hex-output-metadata-substring-entry-stack-zero - hex-output-metadata-substring-entry-stack-capacity 8 * hexdump-from ; - -: substring-entry-stack-depth@ - transformation-state transformation-state-output-metadata @ - hex-output-metadata-substring-entry-stack-depth @ ; - -: substring-entry-stack-depth! - transformation-state transformation-state-output-metadata @ - hex-output-metadata-substring-entry-stack-depth ! ; - -: is-substring-entry-stack-empty - substring-entry-stack-depth@ 0 = ; - -: is-substring-entry-stack-full - substring-entry-stack-depth@ - hex-output-metadata-substring-entry-stack-capacity = ; - -: push-substring-entry-stack - is-substring-entry-stack-full not { - transformation-state transformation-state-output-metadata @ - hex-output-metadata-substring-entry-stack-zero - substring-entry-stack-depth@ 8 * + ! - - substring-entry-stack-depth@ 1+ substring-entry-stack-depth! - } { - drop - ." Substring entry stack overflow." newline - } if-else ; - -: pop-substring-entry-stack - is-substring-entry-stack-empty not { - substring-entry-stack-depth@ 1- substring-entry-stack-depth! - - transformation-state transformation-state-output-metadata @ - hex-output-metadata-substring-entry-stack-zero - substring-entry-stack-depth@ 8 * + @ - } { - ." Substring entry stack underflow." newline - } if-else ; - ~ Metadata entries are post-processed before they're output, in order to ~ make the structure of the output code simpler. The post-processing won't ~ happen until output actually occurs, which is important for operations that @@ -3275,6 +3723,11 @@ allocate-transformation-state s" transformation-state" variable dup hex-output-metadata-entry-string @ .hex64 } if + dup hex-output-metadata-entry-type @ + hex-output-metadata-entry-type-provide-substring-string = { + dup hex-output-metadata-entry-string @ emitstring + } if + drop } { emitstring @@ -4264,6 +4717,13 @@ allocate-transformation-state s" transformation-state" variable add-hex-output-metadata-entry exit } if + dup s" provide-keyword" stringcmp 0 = { + ~ Create a new "provide substring string" entry given a keyword pointer. + drop 2 pick execution-token-to-entry entry-to-name + hex-output-metadata-entry-type-provide-substring-string swap + add-hex-output-metadata-entry + exit + } if ~ If it has a nonzero length, create a new suffix-comment entry. over 0 != { |