From 181f1fdac38277dca65e6e0d067d000f8d647636 Mon Sep 17 00:00:00 2001 From: Saagar Jha Date: Sat, 1 Jun 2024 02:53:31 -0700 Subject: [PATCH] Rename the JIT --- .github/workflows/ci.yml | 2 +- README.md | 4 +- asbestos/asbestos.c | 289 ++++++++++++++++++++ jit/jit.h => asbestos/asbestos.h | 36 +-- asbestos/frame.h | 15 + {jit => asbestos}/gadgets-aarch64/bits.S | 0 {jit => asbestos}/gadgets-aarch64/control.S | 22 +- {jit => asbestos}/gadgets-aarch64/entry.S | 30 +- {jit => asbestos}/gadgets-aarch64/gadgets.h | 2 +- {jit => asbestos}/gadgets-aarch64/math.S | 0 {jit => asbestos}/gadgets-aarch64/math.h | 0 {jit => asbestos}/gadgets-aarch64/memory.S | 2 +- {jit => asbestos}/gadgets-aarch64/misc.S | 0 {jit => asbestos}/gadgets-aarch64/string.S | 0 {jit => asbestos}/gadgets-generic.h | 0 {jit => asbestos}/gadgets-x86_64/bits.S | 0 {jit => asbestos}/gadgets-x86_64/control.S | 22 +- {jit => asbestos}/gadgets-x86_64/entry.S | 30 +- {jit => asbestos}/gadgets-x86_64/gadgets.h | 2 +- {jit => asbestos}/gadgets-x86_64/math.S | 0 {jit => asbestos}/gadgets-x86_64/memory.S | 2 +- {jit => asbestos}/gadgets-x86_64/misc.S | 0 {jit => asbestos}/gadgets-x86_64/string.S | 0 {jit => asbestos}/gen.c | 14 +- {jit => asbestos}/gen.h | 4 +- {jit => asbestos}/helpers.c | 0 {jit => asbestos}/offsets.c | 18 +- deps/linux | 2 +- emu/fpu.c | 2 +- emu/memory.c | 10 +- emu/memory.h | 4 +- emu/mmu.h | 2 +- iSH.xcodeproj/project.pbxproj | 50 ++-- jit/frame.h | 15 - jit/jit.c | 289 -------------------- meson.build | 12 +- meson_options.txt | 2 +- tools/staticdefine.sh | 2 +- 38 files changed, 435 insertions(+), 449 deletions(-) create mode 100644 asbestos/asbestos.c rename jit/jit.h => asbestos/asbestos.h (58%) create mode 100644 asbestos/frame.h rename {jit => asbestos}/gadgets-aarch64/bits.S (100%) rename {jit => asbestos}/gadgets-aarch64/control.S (94%) rename {jit => asbestos}/gadgets-aarch64/entry.S (73%) rename {jit => asbestos}/gadgets-aarch64/gadgets.h (99%) rename {jit => asbestos}/gadgets-aarch64/math.S (100%) rename {jit => asbestos}/gadgets-aarch64/math.h (100%) rename {jit => asbestos}/gadgets-aarch64/memory.S (99%) rename {jit => asbestos}/gadgets-aarch64/misc.S (100%) rename {jit => asbestos}/gadgets-aarch64/string.S (100%) rename {jit => asbestos}/gadgets-generic.h (100%) rename {jit => asbestos}/gadgets-x86_64/bits.S (100%) rename {jit => asbestos}/gadgets-x86_64/control.S (93%) rename {jit => asbestos}/gadgets-x86_64/entry.S (72%) rename {jit => asbestos}/gadgets-x86_64/gadgets.h (99%) rename {jit => asbestos}/gadgets-x86_64/math.S (100%) rename {jit => asbestos}/gadgets-x86_64/memory.S (99%) rename {jit => asbestos}/gadgets-x86_64/misc.S (100%) rename {jit => asbestos}/gadgets-x86_64/string.S (100%) rename {jit => asbestos}/gen.c (97%) rename {jit => asbestos}/gen.h (88%) rename {jit => asbestos}/helpers.c (100%) rename {jit => asbestos}/offsets.c (82%) delete mode 100644 jit/frame.h delete mode 100644 jit/jit.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99637cba13..22bfbf1222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: run: deps/clone-linux.sh - name: Build run: | - meson build -Dengine=jit -Dkernel=${{matrix.kernel}} + meson build -Dengine=asbestos -Dkernel=${{matrix.kernel}} ninja -C build env: CC: ${{matrix.cc}} diff --git a/README.md b/README.md index 3d39691140..da12503ae5 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,9 @@ Available channels: - `verbose`: Debug logs that don't fit into another category. - Grep for `DEFAULT_CHANNEL` to see if more log channels have been added since this list was updated. -# A note on the JIT +# A note on the interpreter -Possibly the most interesting thing I wrote as part of iSH is the JIT. It's not actually a JIT since it doesn't target machine code. Instead it generates an array of pointers to functions called gadgets, and each gadget ends with a tailcall to the next function; like the threaded code technique used by some Forth interpreters. The result is a speedup of roughly 3-5x compared to pure emulation. +Possibly the most interesting thing I wrote as part of iSH is the interpreter. It's not quite a JIT since it doesn't target machine code. Instead it generates an array of pointers to functions called gadgets, and each gadget ends with a tailcall to the next function; like the threaded code technique used by some Forth interpreters. The result is a speedup of roughly 3-5x compared to emulation using a simpler switch dispatch. Unfortunately, I made the decision to write nearly all of the gadgets in assembly language. This was probably a good decision with regards to performance (though I'll never know for sure), but a horrible decision with regards to readability, maintainability, and my sanity. The amount of bullshit I've had to put up with from the compiler/assembler/linker is insane. It's like there's a demon in there that makes sure my code is sufficiently deformed, and if not, makes up stupid reasons why it shouldn't compile. In order to stay sane while writing this code, I've had to ignore best practices in code structure and naming. You'll find macros and variables with such descriptive names as `ss` and `s` and `a`. Assembler macros nested beyond belief. And to top it off, there are almost no comments. diff --git a/asbestos/asbestos.c b/asbestos/asbestos.c new file mode 100644 index 0000000000..b7dc87a733 --- /dev/null +++ b/asbestos/asbestos.c @@ -0,0 +1,289 @@ +#define DEFAULT_CHANNEL instr +#include "debug.h" +#include "asbestos/asbestos.h" +#include "asbestos/gen.h" +#include "asbestos/frame.h" +#include "emu/cpu.h" +#include "emu/memory.h" +#include "emu/interrupt.h" +#include "util/list.h" + +extern int current_pid(void); + +static void fiber_block_disconnect(struct asbestos *asbestos, struct fiber_block *block); +static void fiber_block_free(struct asbestos *asbestos, struct fiber_block *block); +static void fiber_free_jetsam(struct asbestos *asbestos); +static void fiber_resize_hash(struct asbestos *asbestos, size_t new_size); + +struct asbestos *asbestos_new(struct mmu *mmu) { + struct asbestos *asbestos = calloc(1, sizeof(struct asbestos)); + asbestos->mmu = mmu; + fiber_resize_hash(asbestos, FIBER_INITIAL_HASH_SIZE); + asbestos->page_hash = calloc(FIBER_PAGE_HASH_SIZE, sizeof(*asbestos->page_hash)); + list_init(&asbestos->jetsam); + lock_init(&asbestos->lock); + wrlock_init(&asbestos->jetsam_lock); + return asbestos; +} + +void asbestos_free(struct asbestos *asbestos) { + for (size_t i = 0; i < asbestos->hash_size; i++) { + struct fiber_block *block, *tmp; + if (list_null(&asbestos->hash[i])) + continue; + list_for_each_entry_safe(&asbestos->hash[i], block, tmp, chain) { + fiber_block_free(asbestos, block); + } + } + fiber_free_jetsam(asbestos); + free(asbestos->page_hash); + free(asbestos->hash); + free(asbestos); +} + +static inline struct list *blocks_list(struct asbestos *asbestos, page_t page, int i) { + // TODO is this a good hash function? + return &asbestos->page_hash[page % FIBER_PAGE_HASH_SIZE].blocks[i]; +} + +void asbestos_invalidate_range(struct asbestos *absestos, page_t start, page_t end) { + lock(&absestos->lock); + struct fiber_block *block, *tmp; + for (page_t page = start; page < end; page++) { + for (int i = 0; i <= 1; i++) { + struct list *blocks = blocks_list(absestos, page, i); + if (list_null(blocks)) + continue; + list_for_each_entry_safe(blocks, block, tmp, page[i]) { + fiber_block_disconnect(absestos, block); + block->is_jetsam = true; + list_add(&absestos->jetsam, &block->jetsam); + } + } + } + unlock(&absestos->lock); +} + +void asbestos_invalidate_page(struct asbestos *asbestos, page_t page) { + asbestos_invalidate_range(asbestos, page, page + 1); +} +void asbestos_invalidate_all(struct asbestos *asbestos) { + asbestos_invalidate_range(asbestos, 0, MEM_PAGES); +} + +static void fiber_resize_hash(struct asbestos *asbestos, size_t new_size) { + TRACE_(verbose, "%d resizing hash to %lu, using %lu bytes for gadgets\n", current_pid(), new_size, asbestos->mem_used); + struct list *new_hash = calloc(new_size, sizeof(struct list)); + for (size_t i = 0; i < asbestos->hash_size; i++) { + if (list_null(&asbestos->hash[i])) + continue; + struct fiber_block *block, *tmp; + list_for_each_entry_safe(&asbestos->hash[i], block, tmp, chain) { + list_remove(&block->chain); + list_init_add(&new_hash[block->addr % new_size], &block->chain); + } + } + free(asbestos->hash); + asbestos->hash = new_hash; + asbestos->hash_size = new_size; +} + +static void fiber_insert(struct asbestos *asbestos, struct fiber_block *block) { + asbestos->mem_used += block->used; + asbestos->num_blocks++; + // target an average hash chain length of 1-2 + if (asbestos->num_blocks >= asbestos->hash_size * 2) + fiber_resize_hash(asbestos, asbestos->hash_size * 2); + + list_init_add(&asbestos->hash[block->addr % asbestos->hash_size], &block->chain); + list_init_add(blocks_list(asbestos, PAGE(block->addr), 0), &block->page[0]); + if (PAGE(block->addr) != PAGE(block->end_addr)) + list_init_add(blocks_list(asbestos, PAGE(block->end_addr), 1), &block->page[1]); +} + +static struct fiber_block *fiber_lookup(struct asbestos *asbestos, addr_t addr) { + struct list *bucket = &asbestos->hash[addr % asbestos->hash_size]; + if (list_null(bucket)) + return NULL; + struct fiber_block *block; + list_for_each_entry(bucket, block, chain) { + if (block->addr == addr) + return block; + } + return NULL; +} + +static struct fiber_block *fiber_block_compile(addr_t ip, struct tlb *tlb) { + struct gen_state state; + TRACE("%d %08x --- compiling:\n", current_pid(), ip); + gen_start(ip, &state); + while (true) { + if (!gen_step(&state, tlb)) + break; + // no block should span more than 2 pages + // guarantee this by limiting total block size to 1 page + // guarantee that by stopping as soon as there's less space left than + // the maximum length of an x86 instruction + // TODO refuse to decode instructions longer than 15 bytes + if (state.ip - ip >= PAGE_SIZE - 15) { + gen_exit(&state); + break; + } + } + gen_end(&state); + assert(state.ip - ip <= PAGE_SIZE); + state.block->used = state.capacity; + return state.block; +} + +// Remove all pointers to the block. It can't be freed yet because another +// thread may be executing it. +static void fiber_block_disconnect(struct asbestos *asbestos, struct fiber_block *block) { + if (asbestos != NULL) { + asbestos->mem_used -= block->used; + asbestos->num_blocks--; + } + list_remove(&block->chain); + for (int i = 0; i <= 1; i++) { + list_remove(&block->page[i]); + list_remove_safe(&block->jumps_from_links[i]); + + struct fiber_block *prev_block, *tmp; + list_for_each_entry_safe(&block->jumps_from[i], prev_block, tmp, jumps_from_links[i]) { + if (prev_block->jump_ip[i] != NULL) + *prev_block->jump_ip[i] = prev_block->old_jump_ip[i]; + list_remove(&prev_block->jumps_from_links[i]); + } + } +} + +static void fiber_block_free(struct asbestos *asbestos, struct fiber_block *block) { + fiber_block_disconnect(asbestos, block); + free(block); +} + +static void fiber_free_jetsam(struct asbestos *asbestos) { + struct fiber_block *block, *tmp; + list_for_each_entry_safe(&asbestos->jetsam, block, tmp, jetsam) { + list_remove(&block->jetsam); + free(block); + } +} + +int fiber_enter(struct fiber_block *block, struct fiber_frame *frame, struct tlb *tlb); + +static inline size_t fiber_cache_hash(addr_t ip) { + return (ip ^ (ip >> 12)) % FIBER_CACHE_SIZE; +} + +static int cpu_step_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { + struct asbestos *asbestos = cpu->mmu->asbestos; + read_wrlock(&asbestos->jetsam_lock); + + struct fiber_block **cache = calloc(FIBER_CACHE_SIZE, sizeof(*cache)); + struct fiber_frame *frame = malloc(sizeof(struct fiber_frame)); + memset(frame, 0, sizeof(*frame)); + frame->cpu = *cpu; + assert(asbestos->mmu == cpu->mmu); + + int interrupt = INT_NONE; + while (interrupt == INT_NONE) { + addr_t ip = frame->cpu.eip; + size_t cache_index = fiber_cache_hash(ip); + struct fiber_block *block = cache[cache_index]; + if (block == NULL || block->addr != ip) { + lock(&asbestos->lock); + block = fiber_lookup(asbestos, ip); + if (block == NULL) { + block = fiber_block_compile(ip, tlb); + fiber_insert(asbestos, block); + } else { + TRACE("%d %08x --- missed cache\n", current_pid(), ip); + } + cache[cache_index] = block; + unlock(&asbestos->lock); + } + struct fiber_block *last_block = frame->last_block; + if (last_block != NULL && + (last_block->jump_ip[0] != NULL || + last_block->jump_ip[1] != NULL)) { + lock(&asbestos->lock); + // can't mint new pointers to a block that has been marked jetsam + // and is thus assumed to have no pointers left + if (!last_block->is_jetsam && !block->is_jetsam) { + for (int i = 0; i <= 1; i++) { + if (last_block->jump_ip[i] != NULL && + (*last_block->jump_ip[i] & 0xffffffff) == block->addr) { + *last_block->jump_ip[i] = (unsigned long) block->code; + list_add(&block->jumps_from[i], &last_block->jumps_from_links[i]); + } + } + } + + unlock(&asbestos->lock); + } + frame->last_block = block; + + // block may be jetsam, but that's ok, because it can't be freed until + // every thread on this asbestos is not executing anything + + TRACE("%d %08x --- cycle %ld\n", current_pid(), ip, frame->cpu.cycle); + + interrupt = fiber_enter(block, frame, tlb); + if (interrupt == INT_NONE && __atomic_exchange_n(cpu->poked_ptr, false, __ATOMIC_SEQ_CST)) + interrupt = INT_TIMER; + if (interrupt == INT_NONE && ++frame->cpu.cycle % (1 << 10) == 0) + interrupt = INT_TIMER; + *cpu = frame->cpu; + } + + free(frame); + free(cache); + read_wrunlock(&asbestos->jetsam_lock); + return interrupt; +} + +static int cpu_single_step(struct cpu_state *cpu, struct tlb *tlb) { + struct gen_state state; + gen_start(cpu->eip, &state); + gen_step(&state, tlb); + gen_exit(&state); + gen_end(&state); + + struct fiber_block *block = state.block; + struct fiber_frame frame = {.cpu = *cpu}; + int interrupt = fiber_enter(block, &frame, tlb); + *cpu = frame.cpu; + fiber_block_free(NULL, block); + if (interrupt == INT_NONE) + interrupt = INT_DEBUG; + return interrupt; +} + +int cpu_run_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { + if (cpu->poked_ptr == NULL) + cpu->poked_ptr = &cpu->_poked; + tlb_refresh(tlb, cpu->mmu); + int interrupt = (cpu->tf ? cpu_single_step : cpu_step_to_interrupt)(cpu, tlb); + cpu->trapno = interrupt; + + struct asbestos *asbestos = cpu->mmu->asbestos; + lock(&asbestos->lock); + if (!list_empty(&asbestos->jetsam)) { + // write-lock the jetsam_lock to wait until other asbestos threads get + // to this point, so they will all clear out their block pointers + // TODO: use RCU for better performance + unlock(&asbestos->lock); + write_wrlock(&asbestos->jetsam_lock); + lock(&asbestos->lock); + fiber_free_jetsam(asbestos); + write_wrunlock(&asbestos->jetsam_lock); + } + unlock(&asbestos->lock); + + return interrupt; +} + +void cpu_poke(struct cpu_state *cpu) { + __atomic_store_n(cpu->poked_ptr, true, __ATOMIC_SEQ_CST); +} diff --git a/jit/jit.h b/asbestos/asbestos.h similarity index 58% rename from jit/jit.h rename to asbestos/asbestos.h index 1e6d323de0..f62330f334 100644 --- a/jit/jit.h +++ b/asbestos/asbestos.h @@ -1,16 +1,16 @@ -#ifndef JIT_H -#define JIT_H +#ifndef ASBESTOS_H +#define ASBESTOS_H #include "misc.h" #include "emu/mmu.h" #include "util/list.h" #include "util/sync.h" -#define JIT_INITIAL_HASH_SIZE (1 << 10) -#define JIT_CACHE_SIZE (1 << 10) -#define JIT_PAGE_HASH_SIZE (1 << 10) +#define FIBER_INITIAL_HASH_SIZE (1 << 10) +#define FIBER_CACHE_SIZE (1 << 10) +#define FIBER_PAGE_HASH_SIZE (1 << 10) -struct jit { - // there is one jit per address space +struct asbestos { + // there is one asbestos per address space struct mmu *mmu; size_t mem_used; size_t num_blocks; @@ -18,7 +18,7 @@ struct jit { struct list *hash; size_t hash_size; - // list of jit_blocks that should be freed soon (at the next RCU grace + // list of fiber_blocks that should be freed soon (at the next RCU grace // period, if we had such a thing) struct list jetsam; @@ -33,9 +33,9 @@ struct jit { // this is roughly the average number of instructions in a basic block according to anonymous sources // times 4, roughly the average number of gadgets/parameters in an instruction, according to anonymous sources -#define JIT_BLOCK_INITIAL_CAPACITY 16 +#define FIBER_BLOCK_INITIAL_CAPACITY 16 -struct jit_block { +struct fiber_block { addr_t addr; addr_t end_addr; size_t used; @@ -60,15 +60,15 @@ struct jit_block { unsigned long code[]; }; -// Create a new jit -struct jit *jit_new(struct mmu *mmu); -void jit_free(struct jit *jit); +// Create a new asbestos +struct asbestos *asbestos_new(struct mmu *mmu); +void asbestos_free(struct asbestos *asbestos); -// Invalidate all jit blocks in pages start (inclusive) to end (exclusive). -// Locks the jit. Should only be called by memory.c in conjunction with +// Invalidate all fiber blocks in pages start (inclusive) to end (exclusive). +// Locks the asbestos. Should only be called by memory.c in conjunction with // mem_changed. -void jit_invalidate_range(struct jit *jit, page_t start, page_t end); -void jit_invalidate_page(struct jit *jit, page_t page); -void jit_invalidate_all(struct jit *jit); +void asbestos_invalidate_range(struct asbestos *asbestos, page_t start, page_t end); +void asbestos_invalidate_page(struct asbestos *asbestos, page_t page); +void asbestos_invalidate_all(struct asbestos *asbestos); #endif diff --git a/asbestos/frame.h b/asbestos/frame.h new file mode 100644 index 0000000000..a19aef7ff7 --- /dev/null +++ b/asbestos/frame.h @@ -0,0 +1,15 @@ +#include +#include "emu/cpu.h" + +// keep in sync with asm +#define FIBER_RETURN_CACHE_SIZE 4096 +#define FIBER_RETURN_CACHE_HASH(x) ((x) & 0xFFF0) >> 4) + +struct fiber_frame { + struct cpu_state cpu; + void *bp; + addr_t value_addr; + uint64_t value[2]; // buffer for crosspage crap + struct fiber_block *last_block; + long ret_cache[FIBER_RETURN_CACHE_SIZE]; // a map of ip to pointer-to-call-gadget-arguments +}; diff --git a/jit/gadgets-aarch64/bits.S b/asbestos/gadgets-aarch64/bits.S similarity index 100% rename from jit/gadgets-aarch64/bits.S rename to asbestos/gadgets-aarch64/bits.S diff --git a/jit/gadgets-aarch64/control.S b/asbestos/gadgets-aarch64/control.S similarity index 94% rename from jit/gadgets-aarch64/control.S rename to asbestos/gadgets-aarch64/control.S index 4092c4c731..6fad18aeef 100644 --- a/jit/gadgets-aarch64/control.S +++ b/asbestos/gadgets-aarch64/control.S @@ -15,7 +15,7 @@ str _ip, [x13, x12, lsl 3] // jump to target ldr _ip, [_ip, 32] - b jit_ret_chain + b fiber_ret_chain write_bullshit 32, call .gadget call_indir @@ -33,7 +33,7 @@ str _ip, [x13, x12, lsl 3] // jump to target mov eip, _tmp - b jit_ret + b fiber_ret write_bullshit 32, call_indir .gadget ret @@ -55,11 +55,11 @@ ldr x8, [_ip, 8] cmp _tmp, w9 b.ne 1f - // good, now do return chaining, the logic is similar to `jit_ret_chain` + // good, now do return chaining, the logic is similar to `fiber_ret_chain` ldr _ip, [_ip, 24] cmp _ip, 0 b.lt 1f - sub x8, _ip, JIT_BLOCK_code + sub x8, _ip, FIBER_BLOCK_code str x8, [_cpu, LOCAL_last_block] gret 1: @@ -67,22 +67,22 @@ // fallthrough 2: mov eip, _tmp - b jit_ret + b fiber_ret read_bullshit 32, ret .gadget jmp_indir mov eip, _tmp - b jit_ret + b fiber_ret .gadget jmp ldr _ip, [_ip] - b jit_ret_chain + b fiber_ret_chain .gadget jcxz cbnz ecx, 1f ldr _ip, [_ip] - b jit_ret_chain + b fiber_ret_chain 1: ldr _ip, [_ip, 8] - b jit_ret_chain + b fiber_ret_chain #define COND_LIST o,c,z,cz,s,p,sxo,sxoz @@ -161,9 +161,9 @@ .gadget jmp_\cond do_jump \cond, 1f ldr _ip, [_ip, 8] - b jit_ret_chain + b fiber_ret_chain 1: ldr _ip, [_ip] - b jit_ret_chain + b fiber_ret_chain .gadget set_\cond do_jump \cond, 1f diff --git a/jit/gadgets-aarch64/entry.S b/asbestos/gadgets-aarch64/entry.S similarity index 73% rename from jit/gadgets-aarch64/entry.S rename to asbestos/gadgets-aarch64/entry.S index 2170f1f026..0637c6055c 100644 --- a/jit/gadgets-aarch64/entry.S +++ b/asbestos/gadgets-aarch64/entry.S @@ -1,9 +1,9 @@ #include "emu/interrupt.h" #include "gadgets.h" -.global NAME(jit_enter) -.type_compat jit_enter,function -NAME(jit_enter): +.global NAME(fiber_enter) +.type_compat fiber_enter,function +NAME(fiber_enter): stp x18, x19, [sp, -0x70]! stp x20, x21, [sp, 0x10] stp x22, x23, [sp, 0x20] @@ -11,36 +11,36 @@ NAME(jit_enter): stp x26, x27, [sp, 0x40] stp x28, x29, [sp, 0x50] str lr, [sp, 0x60] - add _ip, x0, JIT_BLOCK_code + add _ip, x0, FIBER_BLOCK_code # cpu is already x1 add _tlb, x2, TLB_entries load_regs gret -.global jit_ret_chain -jit_ret_chain: +.global fiber_ret_chain +fiber_ret_chain: cmp _ip, 0 - b.lt jit_ret + b.lt fiber_ret ldr x8, [_cpu, CPU_poked_ptr] ldrb w8, [x8] cmp w8, 0 b.ne poke - sub x8, _ip, JIT_BLOCK_code + sub x8, _ip, FIBER_BLOCK_code str x8, [_cpu, LOCAL_last_block] gret poke: - ldr eip, [_ip, -JIT_BLOCK_code+JIT_BLOCK_addr] + ldr eip, [_ip, -FIBER_BLOCK_code+FIBER_BLOCK_addr] # fallthrough -.global jit_ret -jit_ret: +.global fiber_ret +fiber_ret: # load -1 mvn _tmp, wzr # fallthrough -.global jit_exit -jit_exit: +.global fiber_exit +fiber_exit: save_regs ldr lr, [sp, 0x60] ldp x28, x29, [sp, 0x50] @@ -58,8 +58,8 @@ jit_exit: str w8, [_cpu, CPU_segfault_addr] ldr eip, [_ip, 8] strb wzr, [_cpu, CPU_segfault_was_write] - b jit_exit + b fiber_exit .gadget exit ldr eip, [_ip] - b jit_ret + b fiber_ret diff --git a/jit/gadgets-aarch64/gadgets.h b/asbestos/gadgets-aarch64/gadgets.h similarity index 99% rename from jit/gadgets-aarch64/gadgets.h rename to asbestos/gadgets-aarch64/gadgets.h index 67ab5e875a..fa5799d490 100644 --- a/jit/gadgets-aarch64/gadgets.h +++ b/asbestos/gadgets-aarch64/gadgets.h @@ -22,7 +22,7 @@ _tlb .req x2 _addr .req w3 _xaddr .req x3 -.extern jit_exit +.extern fiber_exit .macro .gadget name .global NAME(gadget_\()\name) diff --git a/jit/gadgets-aarch64/math.S b/asbestos/gadgets-aarch64/math.S similarity index 100% rename from jit/gadgets-aarch64/math.S rename to asbestos/gadgets-aarch64/math.S diff --git a/jit/gadgets-aarch64/math.h b/asbestos/gadgets-aarch64/math.h similarity index 100% rename from jit/gadgets-aarch64/math.h rename to asbestos/gadgets-aarch64/math.h diff --git a/jit/gadgets-aarch64/memory.S b/asbestos/gadgets-aarch64/memory.S similarity index 99% rename from jit/gadgets-aarch64/memory.S rename to asbestos/gadgets-aarch64/memory.S index 7e8f2c2be6..a35207c61c 100644 --- a/jit/gadgets-aarch64/memory.S +++ b/asbestos/gadgets-aarch64/memory.S @@ -99,7 +99,7 @@ segfault_\type: tst x0, 1 << 62 csel esp, esp, w8, eq mov x0, INT_GPF - b jit_exit + b fiber_exit .endr diff --git a/jit/gadgets-aarch64/misc.S b/asbestos/gadgets-aarch64/misc.S similarity index 100% rename from jit/gadgets-aarch64/misc.S rename to asbestos/gadgets-aarch64/misc.S diff --git a/jit/gadgets-aarch64/string.S b/asbestos/gadgets-aarch64/string.S similarity index 100% rename from jit/gadgets-aarch64/string.S rename to asbestos/gadgets-aarch64/string.S diff --git a/jit/gadgets-generic.h b/asbestos/gadgets-generic.h similarity index 100% rename from jit/gadgets-generic.h rename to asbestos/gadgets-generic.h diff --git a/jit/gadgets-x86_64/bits.S b/asbestos/gadgets-x86_64/bits.S similarity index 100% rename from jit/gadgets-x86_64/bits.S rename to asbestos/gadgets-x86_64/bits.S diff --git a/jit/gadgets-x86_64/control.S b/asbestos/gadgets-x86_64/control.S similarity index 93% rename from jit/gadgets-x86_64/control.S rename to asbestos/gadgets-x86_64/control.S index 29fa42701d..4da8810cc4 100644 --- a/jit/gadgets-x86_64/control.S +++ b/asbestos/gadgets-x86_64/control.S @@ -15,7 +15,7 @@ write_done 32, call // clobbers r14 // jump to target movq 32(%_ip), %_ip - jmp jit_ret_chain + jmp fiber_ret_chain .gadget call_indir // save return address @@ -32,7 +32,7 @@ write_done 32, call_indir // clobbers r14 // jump to target movl %_tmp, %_eip - jmp jit_ret + jmp fiber_ret .gadget ret movl %_esp, %_addr @@ -54,11 +54,11 @@ movq 8(%_ip), %r15 cmpl %r14d, %tmpd jnz 1f - // good, now do return chaining, the logic is similar to `jit_ret_chain` + // good, now do return chaining, the logic is similar to `fiber_ret_chain` movq 24(%_ip), %_ip btq $63, %_ip jc 1f - leaq -JIT_BLOCK_code(%_ip), %r15 + leaq -FIBER_BLOCK_code(%_ip), %r15 movq %r15, LOCAL_last_block(%_cpu) gret 1: @@ -66,22 +66,22 @@ // fallthrough 2: movl %tmpd, %_eip - jmp jit_ret + jmp fiber_ret .gadget jmp_indir movl %_tmp, %_eip - jmp jit_ret + jmp fiber_ret .gadget jmp movq (%_ip), %_ip - jmp jit_ret_chain + jmp fiber_ret_chain .gadget jcxz cmpl $0, %ecx jne 1f movq (%_ip), %_ip - jmp jit_ret_chain + jmp fiber_ret_chain 1: movq 8(%_ip), %_ip - jmp jit_ret_chain + jmp fiber_ret_chain #define COND_LIST o,c,z,cz,s,p,sxo,sxoz @@ -140,10 +140,10 @@ .gadget jmp_\cond do_jump \cond, 1f movq 8(%_ip), %_ip - jmp jit_ret_chain + jmp fiber_ret_chain 1: movq (%_ip), %_ip - jmp jit_ret_chain + jmp fiber_ret_chain .gadget set_\cond do_jump \cond, 1f diff --git a/jit/gadgets-x86_64/entry.S b/asbestos/gadgets-x86_64/entry.S similarity index 72% rename from jit/gadgets-x86_64/entry.S rename to asbestos/gadgets-x86_64/entry.S index c1397507ce..0aa61cfb32 100644 --- a/jit/gadgets-x86_64/entry.S +++ b/asbestos/gadgets-x86_64/entry.S @@ -1,9 +1,9 @@ #include "emu/interrupt.h" #include "gadgets.h" -.global NAME(jit_enter) -.type_compat NAME(jit_enter),function -NAME(jit_enter): +.global NAME(fiber_enter) +.type_compat NAME(fiber_enter),function +NAME(fiber_enter): push %rbp push %rbx push %r12 @@ -13,35 +13,35 @@ NAME(jit_enter): # make stack 16 byte aligned # this is preserved elsewhere by only doing even numbers of pushes sub $0x8, %rsp - leaq JIT_BLOCK_code(%rdi), %_ip + leaq THEADED_BLOCK_code(%rdi), %_ip movq %rsi, %_cpu movq %rsp, LOCAL_bp(%_cpu) leaq TLB_entries(%rdx), %_tlb load_regs gret -.global jit_ret_chain -jit_ret_chain: +.global fiber_ret_chain +fiber_ret_chain: btq $63, %_ip - jc jit_ret + jc fiber_ret mov CPU_poked_ptr(%_cpu), %r10 cmpb $0, (%r10) jnz poke - leaq -JIT_BLOCK_code(%_ip), %r10 + leaq -THEADED_BLOCK_code(%_ip), %r10 mov %r10, LOCAL_last_block(%_cpu) gret poke: - mov -JIT_BLOCK_code+JIT_BLOCK_addr(%_ip), %_eip + mov -THEADED_BLOCK_code+THEADED_BLOCK_addr(%_ip), %_eip # fallthrough -.global jit_ret -jit_ret: +.global fiber_ret +fiber_ret: movl $-1, %_tmp # fallthrough -.global jit_exit -jit_exit: +.global fiber_exit +fiber_exit: save_regs movl %_eip, CPU_eip(%_cpu) movq LOCAL_bp(%_cpu), %rsp @@ -61,8 +61,8 @@ jit_exit: movl %r14d, CPU_segfault_addr(%_cpu) movl 8(%_ip), %_eip movb $0, CPU_segfault_was_write(%_cpu) - jmp jit_exit + jmp fiber_exit .gadget exit movl (%_ip), %_eip - jmp jit_ret + jmp fiber_ret diff --git a/jit/gadgets-x86_64/gadgets.h b/asbestos/gadgets-x86_64/gadgets.h similarity index 99% rename from jit/gadgets-x86_64/gadgets.h rename to asbestos/gadgets-x86_64/gadgets.h index 2e6c251668..1a0238eae5 100644 --- a/jit/gadgets-x86_64/gadgets.h +++ b/asbestos/gadgets-x86_64/gadgets.h @@ -15,7 +15,7 @@ #define _addr r13d #define _addrq r13 -.extern jit_exit +.extern fiber_exit .macro .gadget name .global.name gadget_\()\name diff --git a/jit/gadgets-x86_64/math.S b/asbestos/gadgets-x86_64/math.S similarity index 100% rename from jit/gadgets-x86_64/math.S rename to asbestos/gadgets-x86_64/math.S diff --git a/jit/gadgets-x86_64/memory.S b/asbestos/gadgets-x86_64/memory.S similarity index 99% rename from jit/gadgets-x86_64/memory.S rename to asbestos/gadgets-x86_64/memory.S index 50a76a3eff..23a8f53d85 100644 --- a/jit/gadgets-x86_64/memory.S +++ b/asbestos/gadgets-x86_64/memory.S @@ -92,7 +92,7 @@ segfault_\type: cmovcl %r14d, %_esp movl %_eip, %_eip movl $INT_GPF, %_tmp - jmp jit_exit + jmp fiber_exit .endr diff --git a/jit/gadgets-x86_64/misc.S b/asbestos/gadgets-x86_64/misc.S similarity index 100% rename from jit/gadgets-x86_64/misc.S rename to asbestos/gadgets-x86_64/misc.S diff --git a/jit/gadgets-x86_64/string.S b/asbestos/gadgets-x86_64/string.S similarity index 100% rename from jit/gadgets-x86_64/string.S rename to asbestos/gadgets-x86_64/string.S diff --git a/jit/gen.c b/asbestos/gen.c similarity index 97% rename from jit/gen.c rename to asbestos/gen.c index 5d33a2ad22..56d2e934df 100644 --- a/jit/gen.c +++ b/asbestos/gen.c @@ -1,6 +1,6 @@ #include #include -#include "jit/gen.h" +#include "asbestos/gen.h" #include "emu/modrm.h" #include "emu/cpuid.h" #include "emu/fpu.h" @@ -20,10 +20,10 @@ static void gen(struct gen_state *state, unsigned long thing) { assert(state->size <= state->capacity); if (state->size >= state->capacity) { state->capacity *= 2; - struct jit_block *bigger_block = realloc(state->block, - sizeof(struct jit_block) + state->capacity * sizeof(unsigned long)); + struct fiber_block *bigger_block = realloc(state->block, + sizeof(struct fiber_block) + state->capacity * sizeof(unsigned long)); if (bigger_block == NULL) { - die("out of memory while jitting"); + die("out of memory while carcinizing"); } state->block = bigger_block; } @@ -32,7 +32,7 @@ static void gen(struct gen_state *state, unsigned long thing) { } void gen_start(addr_t addr, struct gen_state *state) { - state->capacity = JIT_BLOCK_INITIAL_CAPACITY; + state->capacity = FIBER_BLOCK_INITIAL_CAPACITY; state->size = 0; state->ip = addr; for (int i = 0; i <= 1; i++) { @@ -40,13 +40,13 @@ void gen_start(addr_t addr, struct gen_state *state) { } state->block_patch_ip = 0; - struct jit_block *block = malloc(sizeof(struct jit_block) + state->capacity * sizeof(unsigned long)); + struct fiber_block *block = malloc(sizeof(struct fiber_block) + state->capacity * sizeof(unsigned long)); state->block = block; block->addr = addr; } void gen_end(struct gen_state *state) { - struct jit_block *block = state->block; + struct fiber_block *block = state->block; for (int i = 0; i <= 1; i++) { if (state->jump_ip[i] != 0) { block->jump_ip[i] = &block->code[state->jump_ip[i]]; diff --git a/jit/gen.h b/asbestos/gen.h similarity index 88% rename from jit/gen.h rename to asbestos/gen.h index 3f7c299141..2d8d56de08 100644 --- a/jit/gen.h +++ b/asbestos/gen.h @@ -1,14 +1,14 @@ #ifndef EMU_GEN_H #define EMU_GEN_H -#include "jit/jit.h" +#include "asbestos/asbestos.h" #include "emu/tlb.h" struct gen_state { addr_t ip; addr_t orig_ip; unsigned long orig_ip_extra; - struct jit_block *block; + struct fiber_block *block; unsigned size; unsigned capacity; unsigned jump_ip[2]; diff --git a/jit/helpers.c b/asbestos/helpers.c similarity index 100% rename from jit/helpers.c rename to asbestos/helpers.c diff --git a/jit/offsets.c b/asbestos/offsets.c similarity index 82% rename from jit/offsets.c rename to asbestos/offsets.c index 4283ec05a9..f3a1c56d79 100644 --- a/jit/offsets.c +++ b/asbestos/offsets.c @@ -1,5 +1,5 @@ -#include "jit/jit.h" -#include "jit/frame.h" +#include "asbestos/asbestos.h" +#include "asbestos/frame.h" #include "emu/cpu.h" #include "emu/tlb.h" @@ -44,19 +44,19 @@ void cpu() { MACRO(SF_FLAG); MACRO(DF_FLAG); - OFFSET(LOCAL, jit_frame, bp); - OFFSET(LOCAL, jit_frame, value); - OFFSET(LOCAL, jit_frame, value_addr); - OFFSET(LOCAL, jit_frame, last_block); - OFFSET(LOCAL, jit_frame, ret_cache); + OFFSET(LOCAL, fiber_frame, bp); + OFFSET(LOCAL, fiber_frame, value); + OFFSET(LOCAL, fiber_frame, value_addr); + OFFSET(LOCAL, fiber_frame, last_block); + OFFSET(LOCAL, fiber_frame, ret_cache); OFFSET(CPU, cpu_state, segfault_addr); OFFSET(CPU, cpu_state, segfault_was_write); OFFSET(CPU, cpu_state, poked_ptr); MACRO(MEM_READ); MACRO(MEM_WRITE); - OFFSET(JIT_BLOCK, jit_block, addr); - OFFSET(JIT_BLOCK, jit_block, code); + OFFSET(FIBER_BLOCK, fiber_block, addr); + OFFSET(FIBER_BLOCK, fiber_block, code); OFFSET(TLB, tlb, entries); OFFSET(TLB, tlb, dirty_page); diff --git a/deps/linux b/deps/linux index 04a1ed7f85..31bf37ae3b 160000 --- a/deps/linux +++ b/deps/linux @@ -1 +1 @@ -Subproject commit 04a1ed7f85299e568cf152103f4bdc5050855e46 +Subproject commit 31bf37ae3ba5b31ef5f7d277edabcd1fa88ad309 diff --git a/emu/fpu.c b/emu/fpu.c index a58e88486b..d523e30fd0 100644 --- a/emu/fpu.c +++ b/emu/fpu.c @@ -1,4 +1,4 @@ -// I don't remember if the interpreter was supposed to use this in addition to the jit +// I don't remember if the interpreter was supposed to use this in addition to asbestos #include #include #include "emu/cpu.h" diff --git a/emu/memory.c b/emu/memory.c index ec3fcd161b..7626795511 100644 --- a/emu/memory.c +++ b/emu/memory.c @@ -11,7 +11,7 @@ #include "kernel/errno.h" #include "kernel/signal.h" #include "emu/memory.h" -#include "jit/jit.h" +#include "asbestos/asbestos.h" #include "kernel/vdso.h" #include "kernel/task.h" #include "fs/fd.h" @@ -24,7 +24,7 @@ void mem_init(struct mem *mem) { mem->pgdir = calloc(MEM_PGDIR_SIZE, sizeof(struct pt_entry *)); mem->pgdir_used = 0; mem->mmu.ops = &mem_mmu_ops; - mem->mmu.jit = jit_new(&mem->mmu); + mem->mmu.asbestos = asbestos_new(&mem->mmu); mem->mmu.changes = 0; wrlock_init(&mem->lock); } @@ -32,7 +32,7 @@ void mem_init(struct mem *mem) { void mem_destroy(struct mem *mem) { write_wrlock(&mem->lock); pt_unmap_always(mem, 0, MEM_PAGES); - jit_free(mem->mmu.jit); + asbestos_free(mem->mmu.asbestos); for (int i = 0; i < MEM_PGDIR_SIZE; i++) { if (mem->pgdir[i] != NULL) free(mem->pgdir[i]); @@ -147,7 +147,7 @@ int pt_unmap_always(struct mem *mem, page_t start, pages_t pages) { struct pt_entry *pt = mem_pt(mem, page); if (pt == NULL) continue; - jit_invalidate_page(mem->mmu.jit, page); + asbestos_invalidate_page(mem->mmu.asbestos, page); struct data *data = pt->data; mem_pt_del(mem, page); if (--data->refcount == 0) { @@ -273,7 +273,7 @@ void *mem_ptr(struct mem *mem, addr_t addr, int type) { entry->flags |= P_WRITE | P_COW; } // get rid of any compiled blocks in this page - jit_invalidate_page(mem->mmu.jit, page); + asbestos_invalidate_page(mem->mmu.asbestos, page); // if page is cow, ~~milk~~ copy it if (entry->flags & P_COW) { void *data = (char *) entry->data->data + entry->offset; diff --git a/emu/memory.h b/emu/memory.h index ba2a56030f..561036708c 100644 --- a/emu/memory.h +++ b/emu/memory.h @@ -8,13 +8,13 @@ #include "util/list.h" #include "util/sync.h" #include "misc.h" -struct jit; +struct asbestos; struct mem { struct pt_entry **pgdir; int pgdir_used; - struct jit *jit; + struct asbestos *asbestos; struct mmu mmu; wrlock_t lock; diff --git a/emu/mmu.h b/emu/mmu.h index 2f9a2c7cff..e0b27215a6 100644 --- a/emu/mmu.h +++ b/emu/mmu.h @@ -20,7 +20,7 @@ typedef dword_t pages_t; struct mmu { struct mmu_ops *ops; - struct jit *jit; + struct asbestos *asbestos; uint64_t changes; }; diff --git a/iSH.xcodeproj/project.pbxproj b/iSH.xcodeproj/project.pbxproj index 7878089beb..cb188d40b3 100644 --- a/iSH.xcodeproj/project.pbxproj +++ b/iSH.xcodeproj/project.pbxproj @@ -32,7 +32,6 @@ 497F6CF5254E5EA500C82F46 /* float80-test.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C5A254E5C7E00C82F46 /* float80-test.c */; }; 497F6CF6254E5EA500C82F46 /* float80.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C66254E5C7F00C82F46 /* float80.c */; }; 497F6CF7254E5EA500C82F46 /* fpu.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C5D254E5C7E00C82F46 /* fpu.c */; }; - 497F6CF8254E5EA500C82F46 /* interp.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C67254E5C7F00C82F46 /* interp.c */; }; 497F6CF9254E5EA500C82F46 /* memory.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C60254E5C7F00C82F46 /* memory.c */; }; 497F6CFA254E5EA500C82F46 /* tlb.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C59254E5C7E00C82F46 /* tlb.c */; }; 497F6CFB254E5EA500C82F46 /* vec.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C64254E5C7F00C82F46 /* vec.c */; }; @@ -66,7 +65,7 @@ 497F6D17254E5EA600C82F46 /* tty.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6BE2254E5C0D00C82F46 /* tty.c */; }; 497F6D18254E5EA600C82F46 /* gen.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C3D254E5C4F00C82F46 /* gen.c */; }; 497F6D19254E5EA600C82F46 /* helpers.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C3B254E5C4F00C82F46 /* helpers.c */; }; - 497F6D1A254E5EA600C82F46 /* jit.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C40254E5C4F00C82F46 /* jit.c */; }; + 497F6D1A254E5EA600C82F46 /* asbestos.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C40254E5C4F00C82F46 /* asbestos.c */; }; 497F6D1B254E5EA600C82F46 /* offsets.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C3C254E5C4F00C82F46 /* offsets.c */; }; 497F6D1C254E5EA600C82F46 /* calls.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C9F254E5C9800C82F46 /* calls.c */; }; 497F6D1D254E5EA600C82F46 /* epoll.c in Sources */ = {isa = PBXBuildFile; fileRef = 497F6C7D254E5C9700C82F46 /* epoll.c */; }; @@ -501,8 +500,8 @@ 497F6C3D254E5C4F00C82F46 /* gen.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = gen.c; sourceTree = ""; }; 497F6C3E254E5C4F00C82F46 /* gen.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = gen.h; sourceTree = ""; }; 497F6C3F254E5C4F00C82F46 /* frame.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = frame.h; sourceTree = ""; }; - 497F6C40254E5C4F00C82F46 /* jit.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = jit.c; sourceTree = ""; }; - 497F6C41254E5C4F00C82F46 /* jit.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = jit.h; sourceTree = ""; }; + 497F6C40254E5C4F00C82F46 /* asbestos.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = asbestos.c; sourceTree = ""; }; + 497F6C41254E5C4F00C82F46 /* asbestos.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = asbestos.h; sourceTree = ""; }; 497F6C58254E5C7E00C82F46 /* cpuid.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = cpuid.h; sourceTree = ""; }; 497F6C59254E5C7E00C82F46 /* tlb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = tlb.c; sourceTree = ""; }; 497F6C5A254E5C7E00C82F46 /* float80-test.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "float80-test.c"; sourceTree = ""; }; @@ -518,8 +517,6 @@ 497F6C64254E5C7F00C82F46 /* vec.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = vec.c; sourceTree = ""; }; 497F6C65254E5C7F00C82F46 /* regid.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = regid.h; sourceTree = ""; }; 497F6C66254E5C7F00C82F46 /* float80.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = float80.c; sourceTree = ""; }; - 497F6C67254E5C7F00C82F46 /* interp.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = interp.c; sourceTree = ""; }; - 497F6C69254E5C7F00C82F46 /* fpu.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fpu.h; sourceTree = ""; }; 497F6C6A254E5C7F00C82F46 /* modrm.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = modrm.h; sourceTree = ""; }; 497F6C6B254E5C7F00C82F46 /* tlb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = tlb.h; sourceTree = ""; }; 497F6C77254E5C9700C82F46 /* mm.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = mm.h; sourceTree = ""; }; @@ -847,14 +844,6 @@ path = "gadgets-aarch64"; sourceTree = ""; }; - 497F6C68254E5C7F00C82F46 /* interp */ = { - isa = PBXGroup; - children = ( - 497F6C69254E5C7F00C82F46 /* fpu.h */, - ); - path = interp; - sourceTree = ""; - }; BB0B848D2586A4B100208600 /* Configs */ = { isa = PBXGroup; children = ( @@ -980,16 +969,16 @@ BB4A53931FAA393B00A72ACE /* most of the code */ = { isa = PBXGroup; children = ( + BB7D93822087C2890008DA78 /* main.c */, + BB7D93812087C2890008DA78 /* debug.h */, + BB7D933B2087C2880008DA78 /* misc.h */, + BB7D93352087C2880008DA78 /* xX_main_Xx.h */, + BB88F4712152F75A00A341FD /* asbestos */, BB8636F82167F4F200E7ADC0 /* emu */, BB7D936A2087C2890008DA78 /* fs */, - BB88F4712152F75A00A341FD /* jit */, BB7D933C2087C2880008DA78 /* kernel */, BB2D71072354244700A10D1E /* platform */, BB7D93362087C2880008DA78 /* util */, - BB7D93822087C2890008DA78 /* main.c */, - BB7D933B2087C2880008DA78 /* misc.h */, - BB7D93812087C2890008DA78 /* debug.h */, - BB7D93352087C2880008DA78 /* xX_main_Xx.h */, ); name = "most of the code"; sourceTree = ""; @@ -1219,8 +1208,6 @@ 497F6C5B254E5C7E00C82F46 /* float80.h */, 497F6C5D254E5C7E00C82F46 /* fpu.c */, 497F6C63254E5C7F00C82F46 /* fpu.h */, - 497F6C68254E5C7F00C82F46 /* interp */, - 497F6C67254E5C7F00C82F46 /* interp.c */, 497F6C5C254E5C7E00C82F46 /* interrupt.h */, 497F6C60254E5C7F00C82F46 /* memory.c */, 497F6C61254E5C7F00C82F46 /* memory.h */, @@ -1234,21 +1221,21 @@ path = emu; sourceTree = ""; }; - BB88F4712152F75A00A341FD /* jit */ = { + BB88F4712152F75A00A341FD /* asbestos */ = { isa = PBXGroup; children = ( - 497F6C3F254E5C4F00C82F46 /* frame.h */, - 497F6C31254E5C4F00C82F46 /* gadgets-aarch64 */, - 497F6C30254E5C4F00C82F46 /* gadgets-generic.h */, - 497F6C27254E5C4F00C82F46 /* gadgets-x86_64 */, + 497F6C40254E5C4F00C82F46 /* asbestos.c */, 497F6C3D254E5C4F00C82F46 /* gen.c */, - 497F6C3E254E5C4F00C82F46 /* gen.h */, 497F6C3B254E5C4F00C82F46 /* helpers.c */, - 497F6C40254E5C4F00C82F46 /* jit.c */, - 497F6C41254E5C4F00C82F46 /* jit.h */, 497F6C3C254E5C4F00C82F46 /* offsets.c */, + 497F6C41254E5C4F00C82F46 /* asbestos.h */, + 497F6C3F254E5C4F00C82F46 /* frame.h */, + 497F6C30254E5C4F00C82F46 /* gadgets-generic.h */, + 497F6C3E254E5C4F00C82F46 /* gen.h */, + 497F6C31254E5C4F00C82F46 /* gadgets-aarch64 */, + 497F6C27254E5C4F00C82F46 /* gadgets-x86_64 */, ); - path = jit; + path = asbestos; sourceTree = ""; }; BB88F4912154760800A341FD /* FileProvider */ = { @@ -2008,7 +1995,6 @@ 497F6CF5254E5EA500C82F46 /* float80-test.c in Sources */, 497F6CF6254E5EA500C82F46 /* float80.c in Sources */, 497F6CF7254E5EA500C82F46 /* fpu.c in Sources */, - 497F6CF8254E5EA500C82F46 /* interp.c in Sources */, 497F6CF9254E5EA500C82F46 /* memory.c in Sources */, 497F6CFA254E5EA500C82F46 /* tlb.c in Sources */, 497F6CFB254E5EA500C82F46 /* vec.c in Sources */, @@ -2042,7 +2028,7 @@ 497F6D17254E5EA600C82F46 /* tty.c in Sources */, 497F6D18254E5EA600C82F46 /* gen.c in Sources */, 497F6D19254E5EA600C82F46 /* helpers.c in Sources */, - 497F6D1A254E5EA600C82F46 /* jit.c in Sources */, + 497F6D1A254E5EA600C82F46 /* asbestos.c in Sources */, 497F6D1B254E5EA600C82F46 /* offsets.c in Sources */, 497F6D1C254E5EA600C82F46 /* calls.c in Sources */, 497F6D1D254E5EA600C82F46 /* epoll.c in Sources */, diff --git a/jit/frame.h b/jit/frame.h deleted file mode 100644 index f37ebfe2bb..0000000000 --- a/jit/frame.h +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include "emu/cpu.h" - -// keep in sync with asm -#define JIT_RETURN_CACHE_SIZE 4096 -#define JIT_RETURN_CACHE_HASH(x) ((x) & 0xFFF0) >> 4) - -struct jit_frame { - struct cpu_state cpu; - void *bp; - addr_t value_addr; - uint64_t value[2]; // buffer for crosspage crap - struct jit_block *last_block; - long ret_cache[JIT_RETURN_CACHE_SIZE]; // a map of ip to pointer-to-call-gadget-arguments -}; diff --git a/jit/jit.c b/jit/jit.c deleted file mode 100644 index 907ef5c883..0000000000 --- a/jit/jit.c +++ /dev/null @@ -1,289 +0,0 @@ -#define DEFAULT_CHANNEL instr -#include "debug.h" -#include "jit/jit.h" -#include "jit/gen.h" -#include "jit/frame.h" -#include "emu/cpu.h" -#include "emu/memory.h" -#include "emu/interrupt.h" -#include "util/list.h" - -extern int current_pid(void); - -static void jit_block_disconnect(struct jit *jit, struct jit_block *block); -static void jit_block_free(struct jit *jit, struct jit_block *block); -static void jit_free_jetsam(struct jit *jit); -static void jit_resize_hash(struct jit *jit, size_t new_size); - -struct jit *jit_new(struct mmu *mmu) { - struct jit *jit = calloc(1, sizeof(struct jit)); - jit->mmu = mmu; - jit_resize_hash(jit, JIT_INITIAL_HASH_SIZE); - jit->page_hash = calloc(JIT_PAGE_HASH_SIZE, sizeof(*jit->page_hash)); - list_init(&jit->jetsam); - lock_init(&jit->lock); - wrlock_init(&jit->jetsam_lock); - return jit; -} - -void jit_free(struct jit *jit) { - for (size_t i = 0; i < jit->hash_size; i++) { - struct jit_block *block, *tmp; - if (list_null(&jit->hash[i])) - continue; - list_for_each_entry_safe(&jit->hash[i], block, tmp, chain) { - jit_block_free(jit, block); - } - } - jit_free_jetsam(jit); - free(jit->page_hash); - free(jit->hash); - free(jit); -} - -static inline struct list *blocks_list(struct jit *jit, page_t page, int i) { - // TODO is this a good hash function? - return &jit->page_hash[page % JIT_PAGE_HASH_SIZE].blocks[i]; -} - -void jit_invalidate_range(struct jit *jit, page_t start, page_t end) { - lock(&jit->lock); - struct jit_block *block, *tmp; - for (page_t page = start; page < end; page++) { - for (int i = 0; i <= 1; i++) { - struct list *blocks = blocks_list(jit, page, i); - if (list_null(blocks)) - continue; - list_for_each_entry_safe(blocks, block, tmp, page[i]) { - jit_block_disconnect(jit, block); - block->is_jetsam = true; - list_add(&jit->jetsam, &block->jetsam); - } - } - } - unlock(&jit->lock); -} - -void jit_invalidate_page(struct jit *jit, page_t page) { - jit_invalidate_range(jit, page, page + 1); -} -void jit_invalidate_all(struct jit *jit) { - jit_invalidate_range(jit, 0, MEM_PAGES); -} - -static void jit_resize_hash(struct jit *jit, size_t new_size) { - TRACE_(verbose, "%d resizing hash to %lu, using %lu bytes for gadgets\n", current_pid(), new_size, jit->mem_used); - struct list *new_hash = calloc(new_size, sizeof(struct list)); - for (size_t i = 0; i < jit->hash_size; i++) { - if (list_null(&jit->hash[i])) - continue; - struct jit_block *block, *tmp; - list_for_each_entry_safe(&jit->hash[i], block, tmp, chain) { - list_remove(&block->chain); - list_init_add(&new_hash[block->addr % new_size], &block->chain); - } - } - free(jit->hash); - jit->hash = new_hash; - jit->hash_size = new_size; -} - -static void jit_insert(struct jit *jit, struct jit_block *block) { - jit->mem_used += block->used; - jit->num_blocks++; - // target an average hash chain length of 1-2 - if (jit->num_blocks >= jit->hash_size * 2) - jit_resize_hash(jit, jit->hash_size * 2); - - list_init_add(&jit->hash[block->addr % jit->hash_size], &block->chain); - list_init_add(blocks_list(jit, PAGE(block->addr), 0), &block->page[0]); - if (PAGE(block->addr) != PAGE(block->end_addr)) - list_init_add(blocks_list(jit, PAGE(block->end_addr), 1), &block->page[1]); -} - -static struct jit_block *jit_lookup(struct jit *jit, addr_t addr) { - struct list *bucket = &jit->hash[addr % jit->hash_size]; - if (list_null(bucket)) - return NULL; - struct jit_block *block; - list_for_each_entry(bucket, block, chain) { - if (block->addr == addr) - return block; - } - return NULL; -} - -static struct jit_block *jit_block_compile(addr_t ip, struct tlb *tlb) { - struct gen_state state; - TRACE("%d %08x --- compiling:\n", current_pid(), ip); - gen_start(ip, &state); - while (true) { - if (!gen_step(&state, tlb)) - break; - // no block should span more than 2 pages - // guarantee this by limiting total block size to 1 page - // guarantee that by stopping as soon as there's less space left than - // the maximum length of an x86 instruction - // TODO refuse to decode instructions longer than 15 bytes - if (state.ip - ip >= PAGE_SIZE - 15) { - gen_exit(&state); - break; - } - } - gen_end(&state); - assert(state.ip - ip <= PAGE_SIZE); - state.block->used = state.capacity; - return state.block; -} - -// Remove all pointers to the block. It can't be freed yet because another -// thread may be executing it. -static void jit_block_disconnect(struct jit *jit, struct jit_block *block) { - if (jit != NULL) { - jit->mem_used -= block->used; - jit->num_blocks--; - } - list_remove(&block->chain); - for (int i = 0; i <= 1; i++) { - list_remove(&block->page[i]); - list_remove_safe(&block->jumps_from_links[i]); - - struct jit_block *prev_block, *tmp; - list_for_each_entry_safe(&block->jumps_from[i], prev_block, tmp, jumps_from_links[i]) { - if (prev_block->jump_ip[i] != NULL) - *prev_block->jump_ip[i] = prev_block->old_jump_ip[i]; - list_remove(&prev_block->jumps_from_links[i]); - } - } -} - -static void jit_block_free(struct jit *jit, struct jit_block *block) { - jit_block_disconnect(jit, block); - free(block); -} - -static void jit_free_jetsam(struct jit *jit) { - struct jit_block *block, *tmp; - list_for_each_entry_safe(&jit->jetsam, block, tmp, jetsam) { - list_remove(&block->jetsam); - free(block); - } -} - -int jit_enter(struct jit_block *block, struct jit_frame *frame, struct tlb *tlb); - -static inline size_t jit_cache_hash(addr_t ip) { - return (ip ^ (ip >> 12)) % JIT_CACHE_SIZE; -} - -static int cpu_step_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { - struct jit *jit = cpu->mmu->jit; - read_wrlock(&jit->jetsam_lock); - - struct jit_block **cache = calloc(JIT_CACHE_SIZE, sizeof(*cache)); - struct jit_frame *frame = malloc(sizeof(struct jit_frame)); - memset(frame, 0, sizeof(*frame)); - frame->cpu = *cpu; - assert(jit->mmu == cpu->mmu); - - int interrupt = INT_NONE; - while (interrupt == INT_NONE) { - addr_t ip = frame->cpu.eip; - size_t cache_index = jit_cache_hash(ip); - struct jit_block *block = cache[cache_index]; - if (block == NULL || block->addr != ip) { - lock(&jit->lock); - block = jit_lookup(jit, ip); - if (block == NULL) { - block = jit_block_compile(ip, tlb); - jit_insert(jit, block); - } else { - TRACE("%d %08x --- missed cache\n", current_pid(), ip); - } - cache[cache_index] = block; - unlock(&jit->lock); - } - struct jit_block *last_block = frame->last_block; - if (last_block != NULL && - (last_block->jump_ip[0] != NULL || - last_block->jump_ip[1] != NULL)) { - lock(&jit->lock); - // can't mint new pointers to a block that has been marked jetsam - // and is thus assumed to have no pointers left - if (!last_block->is_jetsam && !block->is_jetsam) { - for (int i = 0; i <= 1; i++) { - if (last_block->jump_ip[i] != NULL && - (*last_block->jump_ip[i] & 0xffffffff) == block->addr) { - *last_block->jump_ip[i] = (unsigned long) block->code; - list_add(&block->jumps_from[i], &last_block->jumps_from_links[i]); - } - } - } - - unlock(&jit->lock); - } - frame->last_block = block; - - // block may be jetsam, but that's ok, because it can't be freed until - // every thread on this jit is not executing anything - - TRACE("%d %08x --- cycle %ld\n", current_pid(), ip, frame->cpu.cycle); - - interrupt = jit_enter(block, frame, tlb); - if (interrupt == INT_NONE && __atomic_exchange_n(cpu->poked_ptr, false, __ATOMIC_SEQ_CST)) - interrupt = INT_TIMER; - if (interrupt == INT_NONE && ++frame->cpu.cycle % (1 << 10) == 0) - interrupt = INT_TIMER; - *cpu = frame->cpu; - } - - free(frame); - free(cache); - read_wrunlock(&jit->jetsam_lock); - return interrupt; -} - -static int cpu_single_step(struct cpu_state *cpu, struct tlb *tlb) { - struct gen_state state; - gen_start(cpu->eip, &state); - gen_step(&state, tlb); - gen_exit(&state); - gen_end(&state); - - struct jit_block *block = state.block; - struct jit_frame frame = {.cpu = *cpu}; - int interrupt = jit_enter(block, &frame, tlb); - *cpu = frame.cpu; - jit_block_free(NULL, block); - if (interrupt == INT_NONE) - interrupt = INT_DEBUG; - return interrupt; -} - -int cpu_run_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { - if (cpu->poked_ptr == NULL) - cpu->poked_ptr = &cpu->_poked; - tlb_refresh(tlb, cpu->mmu); - int interrupt = (cpu->tf ? cpu_single_step : cpu_step_to_interrupt)(cpu, tlb); - cpu->trapno = interrupt; - - struct jit *jit = cpu->mmu->jit; - lock(&jit->lock); - if (!list_empty(&jit->jetsam)) { - // write-lock the jetsam_lock to wait until other jit threads get to - // this point, so they will all clear out their block pointers - // TODO: use RCU for better performance - unlock(&jit->lock); - write_wrlock(&jit->jetsam_lock); - lock(&jit->lock); - jit_free_jetsam(jit); - write_wrunlock(&jit->jetsam_lock); - } - unlock(&jit->lock); - - return interrupt; -} - -void cpu_poke(struct cpu_state *cpu) { - __atomic_store_n(cpu->poked_ptr, true, __ATOMIC_SEQ_CST); -} diff --git a/meson.build b/meson.build index 2a4a1038cf..4a8ea6df48 100644 --- a/meson.build +++ b/meson.build @@ -40,7 +40,7 @@ dependencies = [librt, libm, libdl, threads, sqlite3] subdir('vdso') # ish depends on the vdso offsets = custom_target('offsets', - output: 'cpu-offsets.h', input: 'jit/offsets.c', depfile: 'cpu-offsets.h.d', + output: 'cpu-offsets.h', input: 'asbestos/offsets.c', depfile: 'cpu-offsets.h.d', command: [find_program('tools/staticdefine.sh'), '@OUTDIR@/compile_commands.json', '@INPUT@', '@OUTPUT@', '@DEPFILE@']) emu_src = [ @@ -51,12 +51,12 @@ emu_src = [ 'emu/float80.c', ] -if get_option('engine') == 'jit' - gadgets = 'jit/gadgets-' + host_machine.cpu_family() +if get_option('engine') == 'asbestos' + gadgets = 'asbestos/gadgets-' + host_machine.cpu_family() emu_src += [ - 'jit/jit.c', - 'jit/gen.c', - 'jit/helpers.c', + 'asbestos/asbestos.c', + 'asbestos/gen.c', + 'asbestos/helpers.c', gadgets+'/entry.S', gadgets+'/memory.S', gadgets+'/control.S', diff --git a/meson_options.txt b/meson_options.txt index 3a04ade546..fdba8e20b9 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,7 +2,7 @@ option('log', type: 'string', value: '') option('nolog', type: 'string', value: '') option('log_handler', type: 'string', value: 'dprintf') -option('engine', type: 'combo', choices: ['jit'], value: 'jit') +option('engine', type: 'combo', choices: ['asbestos'], value: 'asbestos') option('kernel', type: 'combo', choices: ['ish', 'linux'], value: 'ish') option('kconfig', type: 'array', value: []) diff --git a/tools/staticdefine.sh b/tools/staticdefine.sh index a7819e6588..08f589048b 100755 --- a/tools/staticdefine.sh +++ b/tools/staticdefine.sh @@ -8,7 +8,7 @@ import json with open('$compile_commands') as f: commands = json.load(f) for command in commands: - if command['file'].endswith('jit/jit.c'): + if command['file'].endswith('asbestos/asbestos.c'): break command = command['command'] command = command.split()[:-9] + ['-MD', '-MQ', '$output', '-MF', '$dep']