diff options
| -rw-r--r-- | core-plus.e | 33 | ||||
| -rw-r--r-- | core.e | 141 | ||||
| -rw-r--r-- | evoke.e | 6 | ||||
| -rw-r--r-- | labels.e | 4 | ||||
| -rw-r--r-- | log-load.e | 42 | ||||
| -rw-r--r-- | to-consider.e | 38 | ||||
| -rw-r--r-- | transform.e | 211 | ||||
| -rw-r--r-- | vim/syntax/evocation.vim | 1 |
8 files changed, 390 insertions, 86 deletions
diff --git a/core-plus.e b/core-plus.e new file mode 100644 index 0000000..d323816 --- /dev/null +++ b/core-plus.e @@ -0,0 +1,33 @@ +~ This is a temporary holding place for stuff that probably should be in +~ core someday, but only after carefully considering whether to rewrite it in +~ assembly. Most of it originally came from boot_source in quine.asm. + + +~ Now some fancier stack combinators. +~ +~ While it might be nice, for performance reasons, to do these in +~ assembler, for now it's more important to have them at all. +: 1- 1 - ; +: 1+ 1 + ; +: max 2dup >= { swap drop } { drop } if-else ; +: min 2dup <= { swap drop } { drop } if-else ; + +: over swap dup 3unroll ; +: pick 2 + dup roll dup 3roll unroll ; + +~ Standard Forth doesn't have equivalents of our ndrop and ndup. The HP +~ calls them DROPN and DUPN but that doesn't go well with ie. 2dup or 3roll, +~ so we do it like this. +: ndrop { dup } { swap drop 1- } while drop ; +: ndup dup 1+ swap { dup } + { swap dup pick 3unroll swap 1- } while 2drop ; +: 3drop drop drop drop ; +: 3dup 2 pick 2 pick 2 pick ; + +: && 0 != swap 0 != * ; +: || | 0 != ; +: not 0 = ; +: negate -1 * ; + +: align-floor dup 3unroll /% swap drop * ; + diff --git a/core.e b/core.e index 9393a04..b2062d1 100644 --- a/core.e +++ b/core.e @@ -1,3 +1,15 @@ +~ ~~~~~~~~~~~~~~~~~~~~~~~~~ +~ ~~ Core Forth features ~~ +~ ~~~~~~~~~~~~~~~~~~~~~~~~~ +~ +~ This file provides extremely fundamental functionality which is a +~ necessary component of any Forth dialect, including Evocation. It is +~ included statically as part of any generated executable, and a second copy +~ of it is later copied into the log when that executable runs. Therefore, it +~ is written to obey the constraints of both the label transform, and the +~ log-load transform; see transform.e for more details on that. +~ +~ ~ Stack manipulation routines ~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~ @@ -955,3 +967,132 @@ :rax jmp-abs-indirect-reg64 here ! ] ;asm + +~ Dictionary entries +~ ~~~~~~~~~~~~~~~~~~ +~ +~ Now, we have a bunch of words that are used for traversing the Forth +~ core data structures that describe words. First, we have a couple that +~ relate to individual words and their pieces... +~ +~ The log-load transform produces code that requires +~ entry-to-execution-token, which means it's needed statically. So this stuff +~ to deal with word entry headers might as well go in core, since it has no +~ dependencies to speak of. +~ +~ These are the first words in core that are implemented in Forth rather +~ than assembler. That's not as big a deal as it may seem; the Forth execution +~ model has been ready-to-go ever since we implemented docol and exit, and +~ at this point we have enough basics to do useful things with it. + +~ Jonesforth calls this "TFCA" and ">CFA"; its author speculates that the +~ original meaning is "code field address". +~ +~ (entry pointer -- execution token) +: entry-to-execution-token + ~ Skip next-entry pointer, flag byte, and start terminator. + 10 + + ~ Skip string contents. + dup stringlen + + ~ Skip one for the null terminator, seven more for alignment. + 8 + + ~ Zero the low bits and now it's aligned. + 7 invert & ; + +~ Jonesforth calls this "CFA>". Jonesforth's implementation searches the +~ entire dictionary, since its word header format isn't designed to be +~ traversed in reverse, but ours is, so it should be fast. +~ +~ (execution token -- entry pointer) +: execution-token-to-entry + 1 - + dup reverse-padding-len - + dup reverse-stringlen - + 9 - ; + +~ (entry pointer -- flags byte) +: entry-flags@ + 8 + @ 0xFF & ; + +~ TODO these parameters are in a counterintuitive order, swap them +~ (entry pointer, new flags byte --) +: entry-flags! + swap + 8 + + dup @ 3roll + 0xFF & + swap 0xFFFFFFFFFFFFFF00 & | + swap ! + ; + +~ (entry pointer -- name string pointer) +: entry-to-name 10 + ; + + +~ Binary packing +~ ~~~~~~~~~~~~~~ +~ +~ These routines are for building up data structures in-memory. Sometimes +~ they're used for structures that are meant to stay in memory; other times +~ it's a buffer that will become output. +~ +~ The general pattern is that each routine takes an output address and +~ some specific datum, and returns the output address adjusted to point +~ after the new datum. That makes them easy to chain together. We call this +~ address the "output point", to capture the idea that it's a running total +~ which gets updated by each new datum as it's packed. + +~ (output point, value -- output point) +: pack64 swap dup 3unroll ! 8 + ; +: pack32 swap dup 3unroll 32! 4 + ; +: pack16 swap dup 3unroll 16! 2 + ; +: pack8 swap dup 3unroll 8! 1 + ; + +~ This works on C-style strings, which are characters followed by a null +~ terminator. The packed data includes the null terminator. +~ +~ (output point, string pointer -- output point) +: packstring + dup stringlen 1 + dup + ~ (output point, source, length, length) + 4 roll dup 5 unroll + ~ (destination, source, length, length, output point) + + 4 unroll + ~ (output point, destination, source, length) + memcopy ; + +~ (output point, alignment byte count -- output point) +: packalign + { 2dup /% drop { drop exit } unless + swap 0 pack8 swap } forever ; + + +~ Binary unpacking +~ ~~~~~~~~~~~~~~~~ +~ +~ These routines are for examining data structures in-memory. +~ +~ Similarly to the output routines, each routine takes an input address, +~ which it updates to point after the data item being read. We call this the +~ "input point". Since this is input, the routines return data items rather +~ than accepting them. + +~ (input point -- input point, value) +: unpack64 dup @ swap 8 + swap ; +: unpack32 dup 32@ swap 4 + swap ; +: unpack16 dup 16@ swap 2 + swap ; +: unpack8 dup 8@ swap 1 + swap ; + +~ TODO does this need to have a separate name? +~ (proposed size, alignment byte count -- adjusted size) +: align-size + dup 3unroll dup 3unroll + ~ (alignment, alignment, proposed size, alignment) + 1 - + swap /% swap drop * ; + +~ You might think this would be identical to packalign, but packalign has +~ side effects. +~ +~ (input point, alignment byte count -- input point) +: unpackalign align-size ; + diff --git a/evoke.e b/evoke.e index 9209efc..607242d 100644 --- a/evoke.e +++ b/evoke.e @@ -1,6 +1,6 @@ -~ (cat labels.e elf.e execution.e transform.e; \ +~ (cat labels.e elf.e transform.e execution.e \ ~ echo 65536 read-to-buffer; \ -~ cat core.e; \ +~ cat core.e core-plus.e log-load.e; \ ~ echo pyrzqxgl; \ ~ cat evoke.e) \ ~ | ./quine > evoke && chmod 755 evoke && ./evoke @@ -8,7 +8,7 @@ s" source-to-precompile" variable 1024 read-to-buffer -: foo 4 . ; +: fooze 4 . ; pyrzqxgl s" source-to-copy-to-log" variable diff --git a/labels.e b/labels.e index 4e315f7..8716364 100644 --- a/labels.e +++ b/labels.e @@ -286,7 +286,9 @@ ~ (execution token -- output start, output length) : label-loop 0 swap - 0x1000 allocate dup + ~ TODO every time you double this to fix a crash, you must publicly + ~ apologize for deferring a real fix. those are the rules + 0x2000 allocate dup ~ (iteration count, execution token, output start, output point) { 3 pick 100 > } { 2 pick execute 4 roll 1+ 4 unroll diff --git a/log-load.e b/log-load.e index 029b9da..8053291 100644 --- a/log-load.e +++ b/log-load.e @@ -176,3 +176,45 @@ : log-load-find swap log-load-latest @ swap 3unroll swap find-in ; +~ In the code generated by the log-load transform, it's convenient to have +~ only a single step needed to look up a word's execution token. This helper +~ does log-load-find, then gets the execution token if an entry is found. +~ +~ (log address, string pointer -- log address, execution token or 0) +: log-load-find-execution-token + log-load-find dup { entry-to-execution-token } if ; + + +~ This is the same as "create", from interpret.e, except that it takes the +~ log's address as a parameter rather than hardcoding it, so that it can be +~ used in situations where the normal compilation process isn't yet available. +~ +~ TODO: rework things so there's a common backend, rather than two +~ implementations +~ +~ (log address, string pointer -- log address) +: log-load-create + dup stringlen 1 + dup 3unroll + ~ (log address, name field length, string pointer, name field length) + + 3 pick log-load-here swap drop @ 10 + 3unroll memmove + ~ (log address, name field length) + + over log-load-here swap drop @ + ~ (log address, name field length, output point) + + 2 pick log-load-latest swap drop @ pack64 + ~ (log address, name field length, output point) + 0 pack8 + 0 pack8 + + + ~ (log address, output point) + 8 packalign + ~ (log address, output point) + + over log-load-here swap drop @ + ~ (log address, output point, old here value) + 2 pick log-load-latest swap drop ! + ~ (log address, output point) + over log-load-here swap drop ! ; + diff --git a/to-consider.e b/to-consider.e new file mode 100644 index 0000000..109464b --- /dev/null +++ b/to-consider.e @@ -0,0 +1,38 @@ +~ This file is for stuff that should probably be deleted after flatassembler +~ is removed, but is being kept here temporarily so that those decisions can +~ be made in an organized fashion. + + +~ In the interests of reducing our executable's size, since a lot of it goes +~ to pack* invocations, we define words that combine lit with pack*. This +~ shaves roughly 700 bytes as of when it was added. +~ +~ TODO are these really used anymore? also, they're the only thing beforenext +~ is used for. note that they can't go in core.e because ' is not compatible +~ with the label transform. +~ +~ (output point, value -- output point) +: litpack64 + [ here @ + lods64 + :rax push-reg64 + ' pack64 entry-to-execution-token pack-beforenext + here ! ] ;asm +: litpack32 + [ here @ + lods64 + :rax push-reg64 + ' pack32 entry-to-execution-token pack-beforenext + here ! ] ;asm +: litpack16 + [ here @ + lods64 + :rax push-reg64 + ' pack16 entry-to-execution-token pack-beforenext + here ! ] ;asm +: litpack8 + [ here @ + lods64 + :rax push-reg64 + ' pack8 entry-to-execution-token pack-beforenext + here ! ] ;asm diff --git a/transform.e b/transform.e index 5ddac78..42901f4 100644 --- a/transform.e +++ b/transform.e @@ -16,36 +16,51 @@ ~ specific way. The transforms rely on the label facility provided by ~ labels.e, and expect to run from within label-loop. ~ -~ The label transform operates on code that compiles itself, and ensures -~ that the result of the compilation is suitable to be included in an -~ executable binary as words that are statically referenced by their -~ addresses. To achieve this, it causes each newly-defined word to have a -~ corresponding label whose value is the offset of its codeword, and it causes -~ all compiled invocations of other words to be resolved by using these labels. -~ The label transform is suitable for code that must be directly invoked by -~ the warm-start routine provided by execution.e. -~ -~ The log-load transform also operates on code that compiles itself; it -~ produces a compiled routine which, when run, appends the original code to -~ the log. As the routine is run, each reference to another word is resolved -~ by looking up the name of the target word in the log. Furthermore, these -~ lookups are done using log-load-find, defined in log-load.e, which accepts -~ a pointer to the log's base address as a parameter. See that file for more -~ explanation of what the log is and why it's important. Thus, unlike normal -~ accesses to the log, this routine doesn't rely on already having the log's -~ base address hardcoded into it at the time of its own compilation. The -~ log-load transform is suitable for implementing the core responsibilities of -~ the warm-start routine provided by execution.e, relying on only a few -~ specific words that it statically references via labels. +~ The label transform produces code that uses one label per word it defines, +~ to statically reference everything. Thus, when output to an executable +~ binary, this code will function without external dependencies. The tradeoff +~ is that it has no way to reference data that exists only at runtime. +~ +~ The log-load transform relies on labels, but doesn't add any of its own. +~ It produces a compiled routine which, when run, dynamically looks up all the +~ references in the log, and appends the original code to the log. This adds +~ work that must be done when the runtime starts up, but the benefit is that +~ it can reference data that doesn't exist at compile-time. Most crucially, +~ it can reference the "here" and "latest" pointers in the log, which are +~ required for all the usual word-definition stuff to work. ~ ~ The log-load transform may also be useful for experimental tasks such as ~ creating additional, independent logs, or injecting Evocation into another ~ process's address space. ~ +~ Please notice that both these transforms, in different ways, navigate the +~ same underlying design tension: The Forth compilation model hardcodes +~ references at the time compilation happens, and Evocation makes the choice +~ to not decide the address of the log until runtime. Thus the label transform +~ can't be sufficient on its own. Other Forths avoid this problem by +~ hardcoding an address for the log, or by using OS-provided load-time +~ symbol relocation. Evocation, however, does it on hard mode, mostly for fun. +~ +~ Because it was clear from early on that the label transform couldn't stand +~ alone, and that another one would be necessary, we've refrained from adding +~ too many features to it. Since we have multiple transforms, they should each +~ be kept simple and well-defined, so that they can be composed in creative +~ new ways down the line. When adding additional behavior, always give thought +~ to whether it belongs in an existing transform or a new one. +~ ~ ~ About the label transform ~ ~~~~~~~~~~~~~~~~~~~~~~~~~ ~ +~ The label transform operates on code that compiles itself, and ensures +~ that the result of the compilation is suitable to be included in an +~ executable binary as words that are statically referenced by their +~ addresses. To achieve this, it causes each newly-defined word to have a +~ corresponding label whose value is the offset of its codeword, and it causes +~ all compiled invocations of other words to be resolved by using these labels. +~ The label transform is suitable for code that must be directly invoked by +~ the warm-start routine provided by execution.e. +~ ~ The most fundamental technique the label transform performs is to separate ~ words that run in compile mode from words that run immediately. There is no ~ distinction made between words running in immediate mode, and words declared @@ -105,6 +120,19 @@ ~ About the log-load transform ~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~ +~ The log-load transform also operates on code that compiles itself; it +~ produces a compiled routine which, when run, appends the original code to +~ the log. As the routine is run, each reference to another word is resolved +~ by looking up the name of the target word in the log. Furthermore, these +~ lookups are done using log-load-find, defined in log-load.e, which accepts +~ a pointer to the log's base address as a parameter. See that file for more +~ explanation of what the log is and why it's important. Thus, unlike normal +~ accesses to the log, this routine doesn't rely on already having the log's +~ base address hardcoded into it at the time of its own compilation. The +~ log-load transform is suitable for implementing the core responsibilities of +~ the warm-start routine provided by execution.e, relying on only a few +~ specific words that it statically references via labels. +~ ~ Much like the label transform, the log-load transform provides alternate ~ versions of certain immediate words used in word definition. Also like the ~ label transform, it provides its own copies of "here" and "latest". @@ -223,7 +251,7 @@ allocate-transform-state s" transform-state" variable ~ When calling the label facility during a transformation, it's necessary -~ to use the real, non-wrapped "heap" and "latest". +~ to use the real, non-wrapped "here" and "latest". : swap-transform-variables here @ transform-state transform-state-saved-here @ here ! transform-state transform-state-saved-here ! @@ -458,11 +486,11 @@ allocate-transform-state s" transform-state" variable ~ which is already what we want to output. ~ ~ An important caveat: Though it would require something weird to be - ~ happening, such as a forced forward reference, the label may be zero! - ~ We need to allow for that possibility by not examining the contents of - ~ a nonexistent entry. + ~ happening, such as a forced forward reference, the label may be + ~ zero! We need to allow for that possibility by not examining the + ~ contents of a nonexistent entry. ~ - ~ Fortunately we don't have to look at it, just append it to the heap + ~ Fortunately we don't have to look at it, just append it to the log ~ and clean up. offset-to-target-address-space , drop dropstring 0 exit } if @@ -489,7 +517,7 @@ allocate-transform-state s" transform-state" variable ~ It's a number. interpreter-flags @ 0x01 & { ~ We're in compile mode; append first "lit", then the number, to the - ~ heap. The version of "lit" we use is found by label, so it'll be the + ~ log. The version of "lit" we use is found by label, so it'll be the ~ one that exists when this code is ultimately run. dropstring-with-result @@ -572,30 +600,32 @@ allocate-transform-state s" transform-state" variable ~ below. It is likely to be extremely useful to read and understand "create" ~ in interpret.e before attempting to understand log-load-create. : log-load-create - dup stringlen 1 + dup 3unroll - here @ 10 + 3unroll memmove - here @ - - ~ This value of "latest" is going into the generated output, so we need - ~ to map it to the target address space. It's stored in the host address - ~ space to make immediate words work as expected, so the appropriate - ~ conversion is host-address-space-to-target. - latest @ host-address-space-to-target pack64 - 0 pack8 - 0 pack8 - + - 8 packalign - here @ latest ! - - ~ Now we're immediately after the word header, which is where the codeword - ~ will be. This is the value the label should taken on, so we set it. - dup host-address-space-to-offset - here @ 10 + + ~ dup stringlen 1 + dup 3unroll + ~ here @ 10 + 3unroll memmove + ~ here @ + + ~ ~ This value of "latest" is going into the generated output, so we need + ~ ~ to map it to the target address space. It's stored in the host address + ~ ~ space to make immediate words work as expected, so the appropriate + ~ ~ conversion is host-address-space-to-target. + ~ latest @ host-address-space-to-target pack64 + ~ 0 pack8 + ~ 0 pack8 + ~ + + ~ 8 packalign + ~ here @ latest ! + + ~ ~ Now we're immediately after the word header, which is where the codeword + ~ ~ will be. This is the value the label should taken on, so we set it. + ~ dup host-address-space-to-offset + ~ here @ 10 + + 0 swap ~ DO NOT SUBMIT swap-transform-variables intern-label set-label swap-transform-variables - here ! ; + ~ here ! + ; ~ This is the alternate version of ":" for use with the log-load transform. @@ -603,17 +633,19 @@ allocate-transform-state s" transform-state" variable ~ likely to be extremely useful to read and understand ":" in interpret.e ~ before attempting to understand "log-load:". : log-load: - ~ This calls "log-load-create" instead of "create". + ~ ~ This calls "log-load-create" instead of "create". word value@ log-load-create dropstring ~ This looks up "docol" by label. - swap-transform-variables - L@' docol - L@' origin - swap-transform-variables - + , + ~ swap-transform-variables + ~ L@' docol + ~ L@' origin + ~ swap-transform-variables + ~ + , - latest @ hide-entry ] ; + ~ TODO note no hiding the entry + ] + ; ~ This is the alternate version of ";" for use with the log-load transform. @@ -621,16 +653,16 @@ allocate-transform-state s" transform-state" variable ~ likely to be extremely useful to read and understand ";" in interpret.e ~ before attempting to understand "log-load;". : log-load; - ~ This looks up "exit" by label. - swap-transform-variables - L@' exit - swap-transform-variables - offset-to-target-address-space , + ~ ~ This looks up "exit" by label. + ~ swap-transform-variables + ~ L@' exit + ~ swap-transform-variables + ~ offset-to-target-address-space , - latest @ unhide-entry + ~ latest @ unhide-entry - ~ Since [ is an immediate word, we have to go to extra trouble to compile - ~ it as part of ;. + ~ ~ Since [ is an immediate word, we have to go to extra trouble to compile + ~ ~ it as part of ;. [ ' [ entry-to-execution-token , ] ; make-immediate @@ -640,15 +672,15 @@ allocate-transform-state s" transform-state" variable ~ below. It is likely to be extremely useful to read and understand ";asm" in ~ interpret.e before attempting to understand "log-load;asm". : log-load;asm - here @ pack-next 8 packalign here ! - latest @ dup unhide-entry entry-to-execution-token - ~ The codeword needs to be transformed to the target address space. - dup 8 + host-address-space-to-target - swap ! - - ~ Since [ is an immediate word, we have to go to extra trouble to compile - ~ it as part of ;asm. - [ ' [ entry-to-execution-token , ] + ~ here @ pack-next 8 packalign here ! + ~ latest @ dup unhide-entry entry-to-execution-token + ~ ~ The codeword needs to be transformed to the target address space. + ~ dup 8 + host-address-space-to-target + ~ swap ! + + ~ ~ Since [ is an immediate word, we have to go to extra trouble to compile + ~ ~ it as part of ;asm. + ~ [ ' [ entry-to-execution-token , ] ; make-immediate ~ This implements the log-load transform for a single word. It is directly @@ -669,8 +701,6 @@ allocate-transform-state s" transform-state" variable ~ (string) value@ - dup emitstring newline - ~ If it's the magic word, end the transformation. dup s" pyrzqxgl" stringcmp 0 = { drop dropstring 1 exit } if @@ -695,6 +725,7 @@ allocate-transform-state s" transform-state" variable dropstring-with-result entry-to-execution-token execute 0 exit } if + drop ~ (name as stack string) ~ Now we might have a compiled word, an immediate word, or an integer @@ -711,6 +742,8 @@ allocate-transform-state s" transform-state" variable ~ It's a number. dropstring-with-result + drop ~ TODO placeholder + interpreter-flags @ 0x01 & { ~ We're in compile mode, so we want to generate code which will compile ~ the number. @@ -721,24 +754,40 @@ allocate-transform-state s" transform-state" variable ~ We're in interpret mode, so we want to generate code which will push the ~ number to the stack. ~ TODO + swap-transform-variables L@' lit swap-transform-variables + offset-to-target-address-space , , 0 exit } if - drop ~ (name as stack string) ~ We know it's a regular word, and we're assuming it will exist at ~ runtime. We of course have no way to check what flags it will have, which ~ means immediate words don't work with this transform. We still treat it ~ differently based on whether we're in compile mode. - interpreter-flags @ 0x01 & { - ~ We're in compile mode. We compile code that compiles the word. - ~ TODO - dropstring 0 exit - } if + ~ interpreter-flags @ 0x01 & { + ~ ~ We're in compile mode. We compile code that compiles the word. + ~ ~ TODO + ~ dropstring 0 exit + ~ } if ~ (name as stack string) - ~ We're in immediate mode. We compile code that runs the word immediately. + ~ We're in immediate mode. We compile code that runs the word immediately. + ~ We check whether there's a label for the word; if there is, we output + ~ that. Otherwise we output code that looks it up and runs it. ~ TODO + value@ + swap-transform-variables + ~ Looking these up in reverse order saves us some stack juggling. Does + ~ help readability, or hurt it? Who can say... + L@' execute + L@' log-load-find-execution-token + L@' litstring + swap-transform-variables + offset-to-target-address-space , ~ litstring + 3roll here @ swap packstring 8 packalign here ! + offset-to-target-address-space , ~ log-load-find-execution-token + offset-to-target-address-space , ~ execute + ~ There's no such thing as not finding the word, with this transform. So ~ we just exit. @@ -749,8 +798,6 @@ allocate-transform-state s" transform-state" variable ~ an input string. It is directly analogous to "quit", in interpret.e, but is ~ far more complex. ~ -~ TODO TODO TODO this is just a stub, right now it's just a copy of the label -~ transform ~ (output buffer start, output point, input string pointer ~ -- output buffer start, output point) : log-load-transform diff --git a/vim/syntax/evocation.vim b/vim/syntax/evocation.vim index de3bd66..62042ae 100644 --- a/vim/syntax/evocation.vim +++ b/vim/syntax/evocation.vim @@ -21,6 +21,7 @@ syn match evocationNumber "\(^\|\s\)\zs-\?0o[0-7]\+\ze\($\|\s\)" syn match evocationNumber "\(^\|\s\)\zs-\?0x[0-9a-fA-F]\+\ze\($\|\s\)" syn match evocationOperator "\(^\|\s\)\zs[.,&|@!]\ze\($\|\s\)" +syn match evocationOperator "\(^\|\s\)\zs\(8\|16\|32\)[@!]\ze\($\|\s\)" syn match evocationLogic "\(^\|\s\)\zs&&\ze\($\|\s\)" syn match evocationLogic "\(^\|\s\)\zs||\ze\($\|\s\)" |