diff --git a/.gitignore b/.gitignore index 1e7f32abc5..c18d86027e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ fastlane/screenshots fastlane/test_output e2e_out/ +tests/aarch64/hello +tests/aarch64/exit42 diff --git a/README.md b/README.md index cba05d541c..5809aac9c5 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

-A project to get a Linux shell running on iOS, using usermode x86 emulation and syscall translation. +A project to get a Linux shell running on iOS, using usermode AArch64 (default) or x86 emulation and syscall translation. For the current status of the project, check the issues tab, and the commit logs. @@ -42,6 +42,20 @@ Open the project in Xcode, open iSH.xcconfig, and change `ROOT_BUNDLE_IDENTIFIER To set up your environment, cd to the project and run `meson build` to create a build directory in `build`. Then cd to the build directory and run `ninja`. +By default the guest architecture is AArch64. Build in Xcode with `ISH_GUEST_ARCH = aarch64` (set in `app/iSH.xcconfig`). + +For a runnable shell you need an **Alpine aarch64** fakefs root (the bundled App Store rootfs is i386): + +```bash +curl -fL -o alpine-aarch64.tar.gz \ + https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/aarch64/alpine-minirootfs-3.21.7-aarch64.tar.gz +./build/tools/fakefsify alpine-aarch64.tar.gz alpine-aarch64 +``` + +Then import `alpine-aarch64` in the iSH app or run `./build/ish -f alpine-aarch64 /bin/sh` from the CLI build. + +Legacy i386 guest: `-Dguest_arch=i386`. + To set up a self-contained Alpine linux filesystem, download the Alpine minirootfs tarball for i386 from the [Alpine website](https://alpinelinux.org/downloads/) and run `./tools/fakefsify`, with the minirootfs tarball as the first argument and the name of the output directory as the second argument. Then you can run things inside the Alpine filesystem with `./ish -f alpine /bin/sh`, assuming the output directory is called `alpine`. If `tools/fakefsify` doesn't exist for you in your build directory, that might be because it couldn't find libarchive on your system (see above for ways to install it.) You can replace `ish` with `tools/ptraceomatic` to run the program in a real process and single step and compare the registers at each step. I use it for debugging. Requires 64-bit Linux 4.11 or later. @@ -64,6 +78,8 @@ Available channels: Possibly the most interesting thing I wrote as part of iSH is the interpreter. It's not quite a JIT since it doesn't target machine code. Instead it generates an array of pointers to functions called gadgets, and each gadget ends with a tailcall to the next function; like the threaded code technique used by some Forth interpreters. The result is a speedup of roughly 3-5x compared to emulation using a simpler switch dispatch. +The default guest architecture is now AArch64. Guest gadgets live under `asbestos/gadgets-guest-aarch64/` (assembly on AArch64 hosts, C dispatch on x86_64 hosts for CI). Legacy i386 guests still use `asbestos/gadgets-x86_64/` or `asbestos/gadgets-aarch64/` depending on host CPU. + Unfortunately, I made the decision to write nearly all of the gadgets in assembly language. This was probably a good decision with regards to performance (though I'll never know for sure), but a horrible decision with regards to readability, maintainability, and my sanity. The amount of bullshit I've had to put up with from the compiler/assembler/linker is insane. It's like there's a demon in there that makes sure my code is sufficiently deformed, and if not, makes up stupid reasons why it shouldn't compile. In order to stay sane while writing this code, I've had to ignore best practices in code structure and naming. You'll find macros and variables with such descriptive names as `ss` and `s` and `a`. Assembler macros nested beyond belief. And to top it off, there are almost no comments. So a warning: Long-term exposure to this code may cause loss of sanity, nightmares about GAS macros and linker errors, or any number of other debilitating side effects. This code is known to the State of California to cause cancer, birth defects, and reproductive harm. diff --git a/app/iSH.xcconfig b/app/iSH.xcconfig index 7c3538f20a..bc2b297d8a 100644 --- a/app/iSH.xcconfig +++ b/app/iSH.xcconfig @@ -3,6 +3,9 @@ ROOT_BUNDLE_IDENTIFIER = app.ish.iSH // It's easiest to specify your development team ID in the project build settings, but you can alternatively put it here to reduce merge conflicts DEVELOPMENT_TEAM = +// Guest CPU architecture for meson (-Dguest_arch). Default: aarch64. +ISH_GUEST_ARCH = aarch64 + // Choose logging channels to enable. Separate by spaces. Try "verbose strace". ISH_LOG = ISH_LOGGER = $(ISH_LOGGER_$(PLATFORM_NAME)) @@ -10,4 +13,6 @@ ISH_LOGGER_iphoneos = nslog ISH_LOGGER_iphonesimulator = nslog ISH_LOGGER_macosx = dprintf +// AArch64 rootfs: use Alpine aarch64 minirootfs via fakefsify (see README). +// The bundled appstore APK below is i386-only; replace after converting rootfs. ROOTFS_URL = github.com/ish-app/roots/releases/download/g00712ff0a54b2839c5aa1a8ed758003ca65357dc/appstore-apk.tar.gz diff --git a/app/xcode-meson.sh b/app/xcode-meson.sh index d39d318b72..77822ad92c 100755 --- a/app/xcode-meson.sh +++ b/app/xcode-meson.sh @@ -56,7 +56,8 @@ if [[ -n "$ISH_KERNEL" ]]; then kernel=$ISH_KERNEL fi kconfig="" -for var in buildtype log b_ndebug b_sanitize log_handler kernel kconfig; do +guest_arch=${ISH_GUEST_ARCH:-aarch64} +for var in buildtype log b_ndebug b_sanitize log_handler kernel kconfig guest_arch; do old_value=$(python3 -c "import sys, json; v = next(x['value'] for x in json.load(sys.stdin) if x['name'] == '$var'); print(str(v).lower() if isinstance(v, bool) else ','.join(v) if isinstance(v, list) else v)" <<< $config) new_value=${!var} if [[ $old_value != $new_value ]]; then diff --git a/asbestos/asbestos.c b/asbestos/asbestos.c index 52e9b413bc..db5e1515df 100644 --- a/asbestos/asbestos.c +++ b/asbestos/asbestos.c @@ -4,9 +4,20 @@ #include "asbestos/gen.h" #include "asbestos/frame.h" #include "emu/cpu.h" -#include "emu/interrupt.h" +#include "guest/interrupt.h" +#include "guest/guest-config.h" #include "util/list.h" +#if GUEST_AARCH64 +#define guest_ip(cpu) ((cpu)->pc) +#define set_guest_ip(cpu, val) ((cpu)->pc = (val)) +#define GUEST_BLOCK_LIMIT (PAGE_SIZE - 4) +#else +#define guest_ip(cpu) ((cpu)->eip) +#define set_guest_ip(cpu, val) ((cpu)->eip = (val)) +#define GUEST_BLOCK_LIMIT (PAGE_SIZE - 15) +#endif + extern int current_pid(void); static void fiber_block_disconnect(struct asbestos *asbestos, struct fiber_block *block); @@ -124,7 +135,7 @@ static struct fiber_block *fiber_block_compile(addr_t ip, struct tlb *tlb) { // guarantee that by stopping as soon as there's less space left than // the maximum length of an x86 instruction // TODO refuse to decode instructions longer than 15 bytes - if (state.ip - ip >= PAGE_SIZE - 15) { + if (state.ip - ip >= GUEST_BLOCK_LIMIT) { gen_exit(&state); break; } @@ -187,7 +198,7 @@ static int cpu_step_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { int interrupt = INT_NONE; while (interrupt == INT_NONE) { - addr_t ip = frame->cpu.eip; + addr_t ip = guest_ip(&frame->cpu); size_t cache_index = fiber_cache_hash(ip); struct fiber_block *block = cache[cache_index]; if (block == NULL || block->addr != ip) { @@ -244,7 +255,7 @@ static int cpu_step_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { static int cpu_single_step(struct cpu_state *cpu, struct tlb *tlb) { struct gen_state state; - gen_start(cpu->eip, &state); + gen_start(guest_ip(cpu), &state); gen_step(&state, tlb); gen_exit(&state); gen_end(&state); @@ -263,7 +274,11 @@ int cpu_run_to_interrupt(struct cpu_state *cpu, struct tlb *tlb) { if (cpu->poked_ptr == NULL) cpu->poked_ptr = &cpu->_poked; tlb_refresh(tlb, cpu->mmu); +#if GUEST_AARCH64 + int interrupt = cpu_step_to_interrupt(cpu, tlb); +#else int interrupt = (cpu->tf ? cpu_single_step : cpu_step_to_interrupt)(cpu, tlb); +#endif cpu->trapno = interrupt; struct asbestos *asbestos = cpu->mmu->asbestos; diff --git a/asbestos/frame.h b/asbestos/frame.h index a19aef7ff7..e8df6ae535 100644 --- a/asbestos/frame.h +++ b/asbestos/frame.h @@ -1,3 +1,5 @@ +#ifndef ASBESTOS_FRAME_H +#define ASBESTOS_FRAME_H #include #include "emu/cpu.h" @@ -13,3 +15,5 @@ struct fiber_frame { struct fiber_block *last_block; long ret_cache[FIBER_RETURN_CACHE_SIZE]; // a map of ip to pointer-to-call-gadget-arguments }; + +#endif diff --git a/asbestos/gadgets-generic.h b/asbestos/gadgets-generic.h index 11b52a18ec..c5cfbc267c 100644 --- a/asbestos/gadgets-generic.h +++ b/asbestos/gadgets-generic.h @@ -1,3 +1,6 @@ +#ifndef GADGETS_GENERIC_H +#define GADGETS_GENERIC_H + #include "cpu-offsets.h" #define ifin(thing, ...) _ifin(thing, __COUNTER__, __VA_ARGS__) @@ -90,4 +93,6 @@ #define N ; #endif +#endif + # vim: ft=gas diff --git a/asbestos/gadgets-guest-aarch64/control.S b/asbestos/gadgets-guest-aarch64/control.S new file mode 100644 index 0000000000..619ae585b1 --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/control.S @@ -0,0 +1,183 @@ +#include "gadgets-asm.h" +#include "guest/interrupt.h" + +.gadget exit + ldr x10, [_ip], #8 + str x10, [_cpu, CPU_pc] + b fiber_ret + +.gadget interrupt + ldr w8, [_ip], #8 + ldr x10, [_ip], #8 + str x10, [_cpu, CPU_pc] + ldr x11, [_ip], #8 + str x11, [_cpu, CPU_segfault_addr] + strb wzr, [_cpu, CPU_segfault_was_write] + mov _tmp, w8 + b fiber_exit + +.gadget nop + ldr x10, [_cpu, CPU_pc] + add x10, x10, #4 + str x10, [_cpu, CPU_pc] + gret + +.gadget svc + ldr x10, [_cpu, CPU_pc] + add x10, x10, #4 + str x10, [_cpu, CPU_pc] + mov _tmp, #INT_SYSCALL + b fiber_exit + +.gadget brk + ldr x10, [_cpu, CPU_pc] + add x10, x10, #4 + str x10, [_cpu, CPU_pc] + mov _tmp, #INT_BREAKPOINT + b fiber_exit + +.gadget ret + ldr w10, [_ip], #8 + read_xn x11, w10 + str x11, [_cpu, CPU_pc] + b fiber_ret + +.gadget branch + ldr x10, [_ip], #8 + ldr w11, [_ip], #8 + ldr x12, [_cpu, CPU_pc] + add x12, x12, x10 + cbnz w11, 1f + b 2f +1: + add x13, x12, #4 + str x13, [_cpu, CPU_x + 30*8] +2: + str x12, [_cpu, CPU_pc] + b fiber_ret + +.gadget mov_wide + ldr w8, [_ip], #8 + ldr w9, [_ip], #8 + ldr w10, [_ip], #8 + ldr w11, [_ip], #8 + lsl w9, w9, #4 + lsl w12, w11, w9 + uxtw x12, w12 + cmp w8, #0 + b.eq 1f + cmp w8, #2 + b.eq 2f + cmp w8, #3 + b.ne 3f + read_xn x13, w10 + mov x14, #0xffff + uxtw x9, w9 + lsl x14, x14, x9 + mvn x14, x14 + and x13, x13, x14 + orr x12, x13, x12 + b 3f +1: + mvn x12, x12 + b 3f +2: + nop +3: + write_xn w10, x12 + ldr x10, [_cpu, CPU_pc] + add x10, x10, #4 + str x10, [_cpu, CPU_pc] + gret + +.gadget branch_cond + ldr w8, [_ip], #8 + ldr x10, [_ip], #8 + stp x29, x30, [sp, -0x10]! + mov x0, _cpu + mov w1, w8 + bl helper_a64_cond_true + ldp x29, x30, [sp], 0x10 + cbz w0, cond_false + ldr x14, [_cpu, CPU_pc] + add x14, x14, x10 + str x14, [_cpu, CPU_pc] + b fiber_ret +cond_false: + ldr x14, [_cpu, CPU_pc] + add x14, x14, #4 + str x14, [_cpu, CPU_pc] + gret + +.gadget cbz + // args: sf, op (0=cbz/1=cbnz), rt, imm + ldr w8, [_ip], #8 + ldr w9, [_ip], #8 + ldr w10, [_ip], #8 + ldr x11, [_ip], #8 + read_xn x12, w10 + cbnz w8, 1f + // sf=0: compare as 32-bit (W-form CBZ/CBNZ) + uxtw x12, w12 +1: + // take branch when (val==0) == (op==0), i.e. is_zero != op + cmp x12, #0 + cset w12, eq + cmp w12, w9 + b.eq 2f + // take branch + ldr x13, [_cpu, CPU_pc] + add x13, x13, x11 + str x13, [_cpu, CPU_pc] + b fiber_ret +2: + ldr x13, [_cpu, CPU_pc] + add x13, x13, #4 + str x13, [_cpu, CPU_pc] + gret + +.gadget tbz + ldr w8, [_ip], #8 + ldr w9, [_ip], #8 + ldr w10, [_ip], #8 + ldr x11, [_ip], #8 + read_xn x12, w9 + uxtw x10, w10 + lsr x12, x12, x10 + and x12, x12, #1 + cmp w12, w8 + b.ne 1f + ldr x13, [_cpu, CPU_pc] + add x13, x13, x11 + str x13, [_cpu, CPU_pc] + b fiber_ret +1: + ldr x13, [_cpu, CPU_pc] + add x13, x13, #4 + str x13, [_cpu, CPU_pc] + gret + +.gadget adr + ldr w9, [_ip], #8 + ldr x10, [_ip], #8 + ldr x11, [_cpu, CPU_pc] + add x11, x11, x10 + write_xn w9, x11 + ldr x10, [_cpu, CPU_pc] + add x10, x10, #4 + str x10, [_cpu, CPU_pc] + gret + +.gadget adrp + ldr w9, [_ip], #8 + ldr x10, [_ip], #8 + ldr x11, [_cpu, CPU_pc] + bic x11, x11, #0xfff + add x11, x11, x10, lsl #12 + write_xn w9, x11 + ldr x10, [_cpu, CPU_pc] + add x10, x10, #4 + str x10, [_cpu, CPU_pc] + gret + +# vim: ft=gas diff --git a/asbestos/gadgets-guest-aarch64/entry-host.c b/asbestos/gadgets-guest-aarch64/entry-host.c new file mode 100644 index 0000000000..793360ba75 --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/entry-host.c @@ -0,0 +1,16 @@ +#include "asbestos/asbestos.h" +#include "asbestos/gadgets-guest-aarch64/gadgets.h" +#include "guest/interrupt.h" + +int fiber_enter(struct fiber_block *block, struct fiber_frame *frame, struct tlb *tlb) { + unsigned long *ip = block->code; + while (true) { + a64_gadget_fn gadget = (a64_gadget_fn) *ip++; + int result = gadget(frame, tlb, &ip); + int kind = result & 0xff; + if (kind == A64_GADGET_INTERRUPT) + return result >> 8; + if (kind == A64_GADGET_END_BLOCK) + return INT_NONE; + } +} diff --git a/asbestos/gadgets-guest-aarch64/entry.S b/asbestos/gadgets-guest-aarch64/entry.S new file mode 100644 index 0000000000..d3da918771 --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/entry.S @@ -0,0 +1,49 @@ +#include "gadgets-asm.h" +#include "guest/interrupt.h" + +.global NAME(fiber_enter) +.type_compat fiber_enter,function +NAME(fiber_enter): + stp x19, x20, [sp, -0x50]! + stp x21, x22, [sp, 0x10] + stp x23, x24, [sp, 0x20] + str x25, [sp, 0x30] + str lr, [sp, 0x38] + + add _ip, x0, FIBER_BLOCK_code + mov _cpu, x1 + add _tlb, x2, TLB_entries + gret + +.global fiber_ret_chain +fiber_ret_chain: + tbz _ip, #63, 1f + b fiber_ret +1: + ldr x8, [_cpu, CPU_poked_ptr] + ldrb w8, [x8] + cbnz w8, poke + sub x8, _ip, FIBER_BLOCK_code + str x8, [_cpu, LOCAL_last_block] + gret + +poke: + ldr _tmp, [_ip, -FIBER_BLOCK_code+FIBER_BLOCK_addr] + str _tmp, [_cpu, CPU_pc] + b fiber_ret + +.global fiber_ret +fiber_ret: + mov _tmp, #-1 + b fiber_exit + +.global fiber_exit +fiber_exit: + ldr lr, [sp, 0x38] + ldr x25, [sp, 0x30] + ldp x23, x24, [sp, 0x20] + ldp x21, x22, [sp, 0x10] + ldp x19, x20, [sp], 0x50 + ret + +# vim: ft=gas diff --git a/asbestos/gadgets-guest-aarch64/gadgets-asm.h b/asbestos/gadgets-guest-aarch64/gadgets-asm.h new file mode 100644 index 0000000000..b1941af8f0 --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/gadgets-asm.h @@ -0,0 +1,69 @@ +#include "../gadgets-generic.h" + +_cpu .req x20 +_tlb .req x21 +_ip .req x22 +_tmp .req w0 +_xtmp .req x0 + +.extern fiber_exit +.extern fiber_ret +.extern fiber_interrupt +.extern helper_a64_run_insn_word +.extern helper_a64_cond_true + +.macro .gadget name + .global NAME(gadget_a64_\()\name) + .align 4 + NAME(gadget_a64_\()\name) : +.endm + +.macro gret pop=0 + ldr x8, [_ip, \pop*8]! + br x8 +.endm + +// Runtime register index in \reg (wN). x16 is IP0 and safe to clobber. +.macro read_xn dst, reg + cmp \reg, #31 + b.eq 1f + add x16, _cpu, CPU_x + ldr \dst, [x16, \reg, uxtw #3] + b 2f +1: ldr \dst, [_cpu, CPU_sp] +2: +.endm + +.macro write_xn reg, src + cmp \reg, #31 + b.eq 1f + add x16, _cpu, CPU_x + str \src, [x16, \reg, uxtw #3] + b 2f +1: str \src, [_cpu, CPU_sp] +2: +.endm + +.macro insn_gadget name +.gadget \name + ldr w19, [_ip], #8 + stp x19, x20, [sp, -0x20]! + stp x29, x30, [sp, 0x10] + mov x0, _cpu + sub x1, _tlb, TLB_entries + mov w2, w19 + bl helper_a64_run_insn_word + mov w23, w0 + ldp x29, x30, [sp, 0x10] + ldp x19, x20, [sp], 0x20 + and w8, w23, #0xff + cmp w8, #1 + b.ne 1f + b fiber_ret +1: cmp w8, #2 + b.ne 2f + b fiber_interrupt +2: gret +.endm + +# vim: ft=gas diff --git a/asbestos/gadgets-guest-aarch64/gadgets-c-hot.c b/asbestos/gadgets-guest-aarch64/gadgets-c-hot.c new file mode 100644 index 0000000000..3cf4bc7deb --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/gadgets-c-hot.c @@ -0,0 +1,102 @@ +#include +#include "asbestos/gadgets-guest-aarch64/gadgets.h" +#include "emu/aarch64-exec.h" +#include "guest/interrupt.h" + +#define GADGET_CONTINUE A64_GADGET_CONTINUE +#define GADGET_END_BLOCK A64_GADGET_END_BLOCK +#define GADGET_INTERRUPT A64_GADGET_INTERRUPT + +static int gadget_interrupt_emit(struct fiber_frame *frame, int interrupt, addr_t pc, addr_t addr) { + frame->cpu.pc = pc; + frame->cpu.segfault_addr = addr; + frame->cpu.segfault_was_write = false; + return GADGET_INTERRUPT | (interrupt << 8); +} + +static inline qword_t read_x(struct cpu_state *cpu, unsigned reg) { + if (reg == 31) + return cpu->sp; + return cpu->x[reg]; +} + +static inline void write_x(struct cpu_state *cpu, unsigned reg, qword_t val) { + if (reg == 31) + cpu->sp = val; + else + cpu->x[reg] = val; +} + +int gadget_a64_exit(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + frame->cpu.pc = (addr_t) (*ip)[0]; + (*ip)++; + return GADGET_END_BLOCK; +} + +int gadget_a64_interrupt(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + int interrupt = (int) (*ip)[0]; + addr_t pc = (addr_t) (*ip)[1]; + addr_t addr = (addr_t) (*ip)[2]; + (*ip) += 3; + return gadget_interrupt_emit(frame, interrupt, pc, addr); +} + +int gadget_a64_nop(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; (void) ip; + frame->cpu.pc += 4; + return GADGET_CONTINUE; +} + +int gadget_a64_svc(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; (void) ip; + return gadget_interrupt_emit(frame, INT_SYSCALL, frame->cpu.pc + 4, 0); +} + +int gadget_a64_brk(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; (void) ip; + return gadget_interrupt_emit(frame, INT_BREAKPOINT, frame->cpu.pc + 4, 0); +} + +int gadget_a64_ret(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned rn = (unsigned) (*ip)[0]; + (*ip)++; + frame->cpu.pc = read_x(&frame->cpu, rn); + return GADGET_END_BLOCK; +} + +int gadget_a64_branch(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + int64_t offset = (int64_t) (intptr_t) (*ip)[0]; + unsigned link = (unsigned) (*ip)[1]; + (*ip) += 2; + addr_t pc = frame->cpu.pc; + if (link) + write_x(&frame->cpu, 30, pc + 4); + frame->cpu.pc = pc + offset; + return GADGET_END_BLOCK; +} + +int gadget_a64_mov_wide(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned opc = (unsigned) (*ip)[0]; + unsigned hw = (unsigned) (*ip)[1]; + unsigned rd = (unsigned) (*ip)[2]; + uint16_t imm = (uint16_t) (*ip)[3]; + (*ip) += 4; + qword_t shift = (qword_t) imm << (hw * 16); + qword_t val; + if (opc == 0) + val = ~shift; + else if (opc == 2) + val = shift; + else if (opc == 3) + val = (read_x(&frame->cpu, rd) & ~(0xffffull << (hw * 16))) | shift; + else + val = read_x(&frame->cpu, rd); + write_x(&frame->cpu, rd, val); + frame->cpu.pc += 4; + return GADGET_CONTINUE; +} diff --git a/asbestos/gadgets-guest-aarch64/gadgets-common.c b/asbestos/gadgets-guest-aarch64/gadgets-common.c new file mode 100644 index 0000000000..714764c16c --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/gadgets-common.c @@ -0,0 +1,168 @@ +#include +#include "asbestos/gadgets-guest-aarch64/gadgets.h" +#include "asbestos/asbestos.h" +#include "emu/aarch64-exec.h" +#include "guest/interrupt.h" + +extern int helper_a64_run_insn_word(struct fiber_frame *frame, struct tlb *tlb, uint32_t insn); + +#define GADGET_CONTINUE A64_GADGET_CONTINUE +#define GADGET_END_BLOCK A64_GADGET_END_BLOCK +#define GADGET_INTERRUPT A64_GADGET_INTERRUPT + +static int gadget_interrupt_emit(struct fiber_frame *frame, int interrupt, addr_t pc, addr_t addr) { + frame->cpu.pc = pc; + frame->cpu.segfault_addr = addr; + frame->cpu.segfault_was_write = false; + return GADGET_INTERRUPT | (interrupt << 8); +} + +static inline qword_t read_x(struct cpu_state *cpu, unsigned reg) { + if (reg == 31) + return cpu->sp; + return cpu->x[reg]; +} + +static inline void write_x(struct cpu_state *cpu, unsigned reg, qword_t val) { + if (reg == 31) + cpu->sp = val; + else + cpu->x[reg] = val; +} + +static inline bool cond_true(struct cpu_state *cpu, unsigned cond) { + bool n = cpu_flag_n(cpu); + bool z = cpu_flag_z(cpu); + bool c = cpu_flag_c(cpu); + bool v = cpu_flag_v(cpu); + switch (cond) { + case 0x0: return z; + case 0x1: return !z; + case 0x2: return c; + case 0x3: return !c; + case 0x4: return n; + case 0x5: return !n; + case 0x6: return v; + case 0x7: return !v; + case 0x8: return c && !z; + case 0x9: return !c || z; + case 0xa: return n == v; + case 0xb: return n != v; + case 0xc: return !z && (n == v); + case 0xd: return z || (n != v); + case 0xe: return true; + case 0xf: return false; + default: return false; + } +} + +int gadget_a64_branch_cond(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned cond = (unsigned) (*ip)[0]; + int64_t offset = (int64_t) (intptr_t) (*ip)[1]; + (*ip) += 2; + addr_t pc = frame->cpu.pc; + if (cond_true(&frame->cpu, cond)) + frame->cpu.pc = pc + offset; + else + frame->cpu.pc = pc + 4; + return frame->cpu.pc == pc + 4 ? GADGET_CONTINUE : GADGET_END_BLOCK; +} + +int gadget_a64_cbz(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned sf = (unsigned) (*ip)[0]; + unsigned op = (unsigned) (*ip)[1]; + unsigned rt = (unsigned) (*ip)[2]; + int64_t offset = (int64_t) (intptr_t) (*ip)[3]; + (*ip) += 4; + qword_t val = read_x(&frame->cpu, rt); + if (!sf) + val = (uint32_t) val; + bool is_zero = val == 0; + addr_t pc = frame->cpu.pc; + if (is_zero == (op == 0)) + frame->cpu.pc = pc + offset; + else + frame->cpu.pc = pc + 4; + return frame->cpu.pc == pc + 4 ? GADGET_CONTINUE : GADGET_END_BLOCK; +} + +int gadget_a64_tbz(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned op = (unsigned) (*ip)[0]; + unsigned rt = (unsigned) (*ip)[1]; + unsigned bit = (unsigned) (*ip)[2]; + int64_t offset = (int64_t) (intptr_t) (*ip)[3]; + (*ip) += 4; + bool bit_set = (read_x(&frame->cpu, rt) >> bit) & 1; + addr_t pc = frame->cpu.pc; + if (bit_set == op) + frame->cpu.pc = pc + offset; + else + frame->cpu.pc = pc + 4; + return frame->cpu.pc == pc + 4 ? GADGET_CONTINUE : GADGET_END_BLOCK; +} + +int gadget_a64_adr(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned rd = (unsigned) (*ip)[0]; + int64_t offset = (int64_t) (intptr_t) (*ip)[1]; + (*ip) += 2; + write_x(&frame->cpu, rd, frame->cpu.pc + offset); + frame->cpu.pc += 4; + return GADGET_CONTINUE; +} + +int gadget_a64_adrp(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + (void) tlb; + unsigned rd = (unsigned) (*ip)[0]; + int64_t offset = (int64_t) (intptr_t) (*ip)[1]; + (*ip) += 2; + addr_t page = (frame->cpu.pc & ~0xfffull) + (offset << 12); + write_x(&frame->cpu, rd, page); + frame->cpu.pc += 4; + return GADGET_CONTINUE; +} + +int gadget_a64_insn(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + uint32_t insn = (uint32_t) (*ip)[0]; + (*ip)++; + return helper_a64_run_insn_word(frame, tlb, insn); +} + +int gadget_a64_ldst(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_ldst_reg(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_pair(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_exclusive(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_add_sub_imm(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_logical_imm(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_add_sub_reg(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_logical_reg(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} + +int gadget_a64_system(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip) { + return gadget_a64_insn(frame, tlb, ip); +} diff --git a/asbestos/gadgets-guest-aarch64/gadgets.h b/asbestos/gadgets-guest-aarch64/gadgets.h new file mode 100644 index 0000000000..d7ca02c073 --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/gadgets.h @@ -0,0 +1,17 @@ +#ifndef GADGETS_GUEST_AARCH64_H +#define GADGETS_GUEST_AARCH64_H + +#include "asbestos/frame.h" +#include "emu/tlb.h" + +enum a64_gadget_result { + A64_GADGET_CONTINUE = 0, + A64_GADGET_END_BLOCK = 1, + A64_GADGET_INTERRUPT = 2, +}; + +typedef int (*a64_gadget_fn)(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); + +int fiber_enter(struct fiber_block *block, struct fiber_frame *frame, struct tlb *tlb); + +#endif diff --git a/asbestos/gadgets-guest-aarch64/math.S b/asbestos/gadgets-guest-aarch64/math.S new file mode 100644 index 0000000000..66f3a22a0a --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/math.S @@ -0,0 +1,8 @@ +#include "gadgets-asm.h" + +insn_gadget add_sub_imm +insn_gadget logical_imm +insn_gadget add_sub_reg +insn_gadget logical_reg + +# vim: ft=gas diff --git a/asbestos/gadgets-guest-aarch64/memory.S b/asbestos/gadgets-guest-aarch64/memory.S new file mode 100644 index 0000000000..789c4b024d --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/memory.S @@ -0,0 +1,8 @@ +#include "gadgets-asm.h" + +insn_gadget ldst +insn_gadget ldst_reg +insn_gadget pair +insn_gadget exclusive + +# vim: ft=gas diff --git a/asbestos/gadgets-guest-aarch64/misc.S b/asbestos/gadgets-guest-aarch64/misc.S new file mode 100644 index 0000000000..e5608ecbb3 --- /dev/null +++ b/asbestos/gadgets-guest-aarch64/misc.S @@ -0,0 +1,12 @@ +#include "gadgets-asm.h" + +insn_gadget system +insn_gadget insn + +.global fiber_interrupt +fiber_interrupt: + // insn_gadget leaves packed result in w23: kind | (interrupt << 8) + lsr _tmp, w23, #8 + b fiber_exit + +# vim: ft=gas diff --git a/asbestos/gen-aarch64.c b/asbestos/gen-aarch64.c new file mode 100644 index 0000000000..3ddcb6e15b --- /dev/null +++ b/asbestos/gen-aarch64.c @@ -0,0 +1,184 @@ +#include +#include +#include +#include "asbestos/gen.h" +#include "emu/aarch64-exec.h" +#include "guest/interrupt.h" + +typedef int (*a64_gadget_fn)(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); + +extern int gadget_a64_exit(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_interrupt(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_nop(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_svc(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_brk(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_ret(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_branch(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_branch_cond(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_cbz(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_tbz(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_adr(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_adrp(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_mov_wide(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_ldst(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_ldst_reg(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_pair(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_exclusive(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_add_sub_imm(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_logical_imm(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_add_sub_reg(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_logical_reg(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_system(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); +extern int gadget_a64_insn(struct fiber_frame *frame, struct tlb *tlb, unsigned long **ip); + +static inline uint32_t bits32(uint32_t insn, int hi, int lo) { + return (insn >> lo) & ((1u << (hi - lo + 1)) - 1); +} + +static inline int64_t sign_extend(uint64_t val, int bits) { + uint64_t shift = 64 - bits; + return (int64_t) (val << shift) >> shift; +} + +static void gen(struct gen_state *state, unsigned long thing) { + assert(state->size <= state->capacity); + if (state->size >= state->capacity) { + state->capacity *= 2; + struct fiber_block *bigger_block = realloc(state->block, + sizeof(struct fiber_block) + state->capacity * sizeof(unsigned long)); + if (bigger_block == NULL) + die("out of memory while carcinizing"); + state->block = bigger_block; + } + state->block->code[state->size++] = thing; +} + +#define GEN(thing) gen(state, (unsigned long) (thing)) +#define g(gadget) GEN(gadget) +#define gg(gadget, a) do { g(gadget); GEN(a); } while (0) +#define ggg(gadget, a, b) do { g(gadget); GEN(a); GEN(b); } while (0) +#define gggg(gadget, a, b, c) do { g(gadget); GEN(a); GEN(b); GEN(c); } while (0) +#define ggggg(gadget, a, b, c, d) do { g(gadget); GEN(a); GEN(b); GEN(c); GEN(d); } while (0) +#define INTERRUPT(code) do { gggg(gadget_a64_interrupt, code, state->orig_ip, 0); return 0; } while (0) +#define SEGFAULT do { gggg(gadget_a64_interrupt, INT_GPF, state->orig_ip, tlb->segfault_addr); return 0; } while (0) + +void gen_start(addr_t addr, struct gen_state *state) { + state->capacity = FIBER_BLOCK_INITIAL_CAPACITY; + state->size = 0; + state->ip = addr; + for (int i = 0; i <= 1; i++) + state->jump_ip[i] = 0; + state->block_patch_ip = 0; + struct fiber_block *block = malloc(sizeof(struct fiber_block) + state->capacity * sizeof(unsigned long)); + state->block = block; + block->addr = addr; +} + +void gen_end(struct gen_state *state) { + struct fiber_block *block = state->block; + for (int i = 0; i <= 1; i++) { + if (state->jump_ip[i] != 0) { + block->jump_ip[i] = &block->code[state->jump_ip[i]]; + block->old_jump_ip[i] = *block->jump_ip[i]; + } else { + block->jump_ip[i] = NULL; + } + list_init(&block->jumps_from[i]); + list_init(&block->jumps_from_links[i]); + } + if (state->block_patch_ip != 0) + block->code[state->block_patch_ip] = (unsigned long) block; + if (block->addr != state->ip) + block->end_addr = state->ip - 1; + else + block->end_addr = block->addr; + list_init(&block->chain); + block->is_jetsam = false; + for (int i = 0; i <= 1; i++) + list_init(&block->page[i]); +} + +void gen_exit(struct gen_state *state) { + g(gadget_a64_exit); + GEN(state->ip); +} + +int gen_step(struct gen_state *state, struct tlb *tlb) { + state->orig_ip = state->ip; + uint32_t insn; + if (!tlb_read(tlb, state->ip, &insn, 4)) + SEGFAULT; + + bool end_block = false; + + if ((insn & 0xffe0001fu) == 0xd4000001u) { + g(gadget_a64_svc); + end_block = true; + } else if ((insn & 0xfffffc1fu) == 0xd61f0000u) { + g(gadget_a64_ret); + GEN(bits32(insn, 9, 5)); + end_block = true; + } else if (bits32(insn, 31, 26) == 0x25) { + int64_t imm = sign_extend(bits32(insn, 25, 0) << 2, 28); + ggg(gadget_a64_branch, (unsigned long) imm, insn >> 31); + end_block = true; + } else if ((insn & 0xff000010u) == 0x54000000u) { + int64_t imm = sign_extend(bits32(insn, 23, 5) << 2, 21); + ggg(gadget_a64_branch_cond, bits32(insn, 3, 0), (unsigned long) imm); + end_block = true; + } else if (bits32(insn, 31, 24) == 0xb4 || bits32(insn, 31, 24) == 0xb5) { + int64_t imm = sign_extend(bits32(insn, 23, 5) << 2, 21); + ggggg(gadget_a64_cbz, bits32(insn, 31, 31), bits32(insn, 24, 24), + bits32(insn, 4, 0), (unsigned long) imm); + end_block = true; + } else if (bits32(insn, 31, 24) == 0xb6 || bits32(insn, 31, 24) == 0xb7) { + int64_t imm = sign_extend(bits32(insn, 18, 5) << 2, 16); + ggggg(gadget_a64_tbz, bits32(insn, 24, 24), bits32(insn, 4, 0), + bits32(insn, 23, 19), (unsigned long) imm); + end_block = true; + } else if (bits32(insn, 31, 29) == 0) { + int64_t immlo = bits32(insn, 30, 29); + int64_t immhi = bits32(insn, 23, 5); + int64_t imm = sign_extend((immhi << 2) | immlo, 21); + unsigned op = bits32(insn, 31, 31); + if (op) + ggg(gadget_a64_adrp, bits32(insn, 4, 0), (unsigned long) imm); + else + ggg(gadget_a64_adr, bits32(insn, 4, 0), (unsigned long) imm); + } else if ((insn & 0x1f800000u) == 0x12800000u) { + ggggg(gadget_a64_mov_wide, bits32(insn, 30, 29), bits32(insn, 22, 21), + bits32(insn, 4, 0), bits32(insn, 20, 5)); + } else if (bits32(insn, 28, 22) == 0x39) { + gg(gadget_a64_ldst, insn); + } else if (bits32(insn, 28, 21) == 0xf1 || bits32(insn, 28, 21) == 0xf3) { + gg(gadget_a64_ldst_reg, insn); + } else if (bits32(insn, 28, 25) == 0xa) { + gg(gadget_a64_pair, insn); + } else if (bits32(insn, 28, 21) == 0x30) { + gg(gadget_a64_exclusive, insn); + } else if ((insn & 0x1f000000u) == 0x11000000u) { + gg(gadget_a64_add_sub_imm, insn); + } else if (bits32(insn, 28, 23) == 0x24) { + gg(gadget_a64_logical_imm, insn); + } else if (bits32(insn, 28, 24) == 0x0b || bits32(insn, 28, 24) == 0x1b) { + gg(gadget_a64_add_sub_reg, insn); + } else if (bits32(insn, 28, 24) == 0x0a) { + gg(gadget_a64_logical_reg, insn); + } else if (bits32(insn, 28, 21) == 0xd5) { + gg(gadget_a64_system, insn); + } else if (insn == 0xd503201fu) { + g(gadget_a64_nop); + } else if ((insn & 0xffe00000u) == 0xd4200000u) { + g(gadget_a64_brk); + end_block = true; + } else { + gg(gadget_a64_insn, insn); + if (aarch64_insn_ends_block(insn)) + end_block = true; + } + + state->ip += 4; + if (end_block) + return 0; + return 1; +} diff --git a/asbestos/helpers-aarch64.c b/asbestos/helpers-aarch64.c new file mode 100644 index 0000000000..01c629152f --- /dev/null +++ b/asbestos/helpers-aarch64.c @@ -0,0 +1,69 @@ +#include +#include "asbestos/frame.h" +#include "asbestos/gadgets-guest-aarch64/gadgets.h" +#include "emu/aarch64-exec.h" +#include "guest/interrupt.h" + +uint64_t helper_a64_decode_logical_imm(uint32_t insn) { + unsigned n = (insn >> 22) & 1; + unsigned immr = (insn >> 16) & 0x3f; + unsigned imms = (insn >> 10) & 0x3f; + unsigned size = 0; + while (size < 6 && !((n >> size) & 1)) + size++; + if (size >= 6) + return 0; + unsigned len = 64 - size; + unsigned levels = size - 1; + unsigned S = imms & levels; + unsigned R = immr & levels; + unsigned diff = S - R; + uint64_t t = 0; + for (unsigned i = 0; i < len; i++) { + if ((diff & levels) == (i & levels)) + t |= 1ull << i; + } + uint64_t wmask = t & ((1ull << (S + 1)) - 1); + uint64_t res = 0; + for (unsigned i = 0; i < 64; i += len) + res |= wmask << i; + return res >> R | res << (len - R); +} + +int helper_a64_run_insn_word(struct fiber_frame *frame, struct tlb *tlb, uint32_t insn) { + addr_t pc = frame->cpu.pc; + int interrupt = aarch64_exec_insn(&frame->cpu, tlb, pc, insn); + if (interrupt != INT_NONE) { + // Preserve segfault_was_write set by aarch64_exec_insn on store faults. + return A64_GADGET_INTERRUPT | (interrupt << 8); + } + if (frame->cpu.pc != pc + 4) + return A64_GADGET_END_BLOCK; + return A64_GADGET_CONTINUE; +} + +int helper_a64_cond_true(struct cpu_state *cpu, unsigned cond) { + bool n = cpu_flag_n(cpu); + bool z = cpu_flag_z(cpu); + bool c = cpu_flag_c(cpu); + bool v = cpu_flag_v(cpu); + switch (cond) { + case 0x0: return z; + case 0x1: return !z; + case 0x2: return c; + case 0x3: return !c; + case 0x4: return n; + case 0x5: return !n; + case 0x6: return v; + case 0x7: return !v; + case 0x8: return c && !z; + case 0x9: return !c || z; + case 0xa: return n == v; + case 0xb: return n != v; + case 0xc: return !z && (n == v); + case 0xd: return z || (n != v); + case 0xe: return true; + case 0xf: return false; + default: return false; + } +} diff --git a/asbestos/offsets-aarch64.c b/asbestos/offsets-aarch64.c new file mode 100644 index 0000000000..0642e129a6 --- /dev/null +++ b/asbestos/offsets-aarch64.c @@ -0,0 +1,38 @@ +#include "asbestos/asbestos.h" +#include "asbestos/frame.h" +#include "emu/cpu-aarch64.h" +#include "emu/tlb.h" + +void cpu(void) { + OFFSET(CPU, cpu_state, x); + OFFSET(CPU, cpu_state, sp); + OFFSET(CPU, cpu_state, pc); + OFFSET(CPU, cpu_state, pstate); + OFFSET(CPU, cpu_state, tls_ptr); + OFFSET(CPU, cpu_state, segfault_addr); + OFFSET(CPU, cpu_state, segfault_was_write); + OFFSET(CPU, cpu_state, poked_ptr); + OFFSET(CPU, cpu_state, exclusive_addr); + OFFSET(CPU, cpu_state, exclusive_valid); + + MACRO(PSTATE_N); + MACRO(PSTATE_Z); + MACRO(PSTATE_C); + MACRO(PSTATE_V); + + OFFSET(LOCAL, fiber_frame, bp); + OFFSET(LOCAL, fiber_frame, value); + OFFSET(LOCAL, fiber_frame, value_addr); + OFFSET(LOCAL, fiber_frame, last_block); + OFFSET(LOCAL, fiber_frame, ret_cache); + + OFFSET(FIBER_BLOCK, fiber_block, addr); + OFFSET(FIBER_BLOCK, fiber_block, code); + + OFFSET(TLB, tlb, entries); + OFFSET(TLB, tlb, dirty_page); + OFFSET(TLB, tlb, segfault_addr); + OFFSET(TLB_ENTRY, tlb_entry, page); + OFFSET(TLB_ENTRY, tlb_entry, page_if_writable); + OFFSET(TLB_ENTRY, tlb_entry, data_minus_addr); +} diff --git a/emu/aarch64-exec.c b/emu/aarch64-exec.c new file mode 100644 index 0000000000..8f793df753 --- /dev/null +++ b/emu/aarch64-exec.c @@ -0,0 +1,546 @@ +#include +#include +#include "emu/aarch64-exec.h" +#include "guest/interrupt.h" +#include "debug.h" + +uint64_t helper_a64_decode_logical_imm(uint32_t insn); + +#define DEFAULT_CHANNEL instr + +static inline uint32_t bits32(uint32_t insn, int hi, int lo) { + return (insn >> lo) & ((1u << (hi - lo + 1)) - 1); +} + +static inline int64_t sign_extend(uint64_t val, int bits) { + uint64_t shift = 64 - bits; + return (int64_t) (val << shift) >> shift; +} + +static bool mem_read(struct cpu_state *cpu, struct tlb *tlb, addr_t addr, void *buf, size_t size) { + if (!tlb_read(tlb, addr, buf, size)) { + cpu->segfault_addr = tlb->segfault_addr; + cpu->segfault_was_write = false; + return false; + } + return true; +} + +static bool mem_write(struct cpu_state *cpu, struct tlb *tlb, addr_t addr, const void *buf, size_t size) { + if (!tlb_write(tlb, addr, buf, size)) { + cpu->segfault_addr = tlb->segfault_addr; + cpu->segfault_was_write = true; + return false; + } + return true; +} + +static qword_t read_reg(struct cpu_state *cpu, unsigned reg) { + if (reg == 31) + return cpu->sp; + return cpu->x[reg]; +} + +static void write_reg(struct cpu_state *cpu, unsigned reg, qword_t val) { + if (reg == 31) + cpu->sp = val; + else + cpu->x[reg] = val; +} + +static void set_nzcv(struct cpu_state *cpu, bool n, bool z, bool c, bool v) { + cpu->pstate &= ~(PSTATE_N | PSTATE_Z | PSTATE_C | PSTATE_V); + if (n) cpu->pstate |= PSTATE_N; + if (z) cpu->pstate |= PSTATE_Z; + if (c) cpu->pstate |= PSTATE_C; + if (v) cpu->pstate |= PSTATE_V; +} + +static void add_with_carry(struct cpu_state *cpu, qword_t op1, qword_t op2, bool carry_in, qword_t *res_out) { + uint64_t res = (uint64_t) op1 + (uint64_t) op2 + (carry_in ? 1 : 0); + bool carry = (uint64_t) op1 + (uint64_t) op2 < (uint64_t) op1 || + (carry_in && res == (uint64_t) op1); + bool overflow = ((~((uint64_t) op1 ^ (uint64_t) op2) & ((uint64_t) op1 ^ res)) >> 63) & 1; + *res_out = res; + set_nzcv(cpu, (int64_t) res < 0, res == 0, carry, overflow); +} + +static void sub_with_carry(struct cpu_state *cpu, qword_t op1, qword_t op2, bool carry_in, qword_t *res_out) { + add_with_carry(cpu, op1, ~op2, carry_in, res_out); +} + +static bool cond_true(struct cpu_state *cpu, unsigned cond) { + bool n = cpu_flag_n(cpu); + bool z = cpu_flag_z(cpu); + bool c = cpu_flag_c(cpu); + bool v = cpu_flag_v(cpu); + switch (cond) { + case 0x0: return z; + case 0x1: return !z; + case 0x2: return c; + case 0x3: return !c; + case 0x4: return n; + case 0x5: return !n; + case 0x6: return v; + case 0x7: return !v; + case 0x8: return c && !z; + case 0x9: return !c || z; + case 0xa: return n == v; + case 0xb: return n != v; + case 0xc: return !z && (n == v); + case 0xd: return z || (n != v); + case 0xe: return true; + case 0xf: return false; + default: return false; + } +} + +static int handle_load_store(struct cpu_state *cpu, struct tlb *tlb, uint32_t insn) { + unsigned size = bits32(insn, 31, 30); + bool v = insn & (1u << 26); + unsigned opc = bits32(insn, 23, 22); + bool is_load = (opc & 1) == 1; + unsigned rt = bits32(insn, 4, 0); + unsigned rn = bits32(insn, 9, 5); + int64_t offset = bits32(insn, 21, 10) << size; + addr_t addr = read_reg(cpu, rn) + offset; + unsigned width = 1u << size; + + if (v) { + if (is_load) { + union vreg zero = {0}; + cpu->v[rt] = zero; + } + return INT_NONE; + } + + if (is_load) { + qword_t val = 0; + if (!mem_read(cpu, tlb, addr, &val, width)) + return INT_GPF; + if (size == 0 && (opc >> 1)) + val = sign_extend(val, 8 << (opc >> 1)); + else if (size == 1 && opc == 3) + val = sign_extend(val, 16); + else if (size == 2 && opc == 3) + val = sign_extend(val, 32); + write_reg(cpu, rt, val); + } else { + qword_t val = read_reg(cpu, rt); + if (width < 8) + val &= ((1ull << (width * 8)) - 1); + if (!mem_write(cpu, tlb, addr, &val, width)) + return INT_GPF; + } + return INT_NONE; +} + +static int handle_load_store_reg(struct cpu_state *cpu, struct tlb *tlb, uint32_t insn) { + unsigned size = bits32(insn, 31, 30); + unsigned opc = bits32(insn, 23, 22); + bool is_load = (opc & 1) == 1; + unsigned rt = bits32(insn, 4, 0); + unsigned rn = bits32(insn, 9, 5); + unsigned rm = bits32(insn, 20, 16); + unsigned extend = bits32(insn, 15, 13); + bool shifted = insn & (1u << 12); + qword_t offset = read_reg(cpu, rm); + if (shifted) + offset <<= size; + if (extend == 2 || extend == 6) + offset = sign_extend(offset, 32); + else if (extend == 3 || extend == 7) + offset = (uint32_t) offset; + addr_t addr = read_reg(cpu, rn) + offset; + unsigned width = 1u << size; + + if (insn & (1u << 26)) { + if (is_load) { + union vreg zero = {0}; + cpu->v[rt] = zero; + } + return INT_NONE; + } + + if (is_load) { + qword_t val = 0; + if (!mem_read(cpu, tlb, addr, &val, width)) + return INT_GPF; + if (size == 0 && opc >= 2) + val = sign_extend(val, 8); + else if (size == 1 && opc == 3) + val = sign_extend(val, 16); + else if (size == 2 && opc == 3) + val = sign_extend(val, 32); + write_reg(cpu, rt, val); + } else { + qword_t val = read_reg(cpu, rt); + if (!mem_write(cpu, tlb, addr, &val, width)) + return INT_GPF; + } + return INT_NONE; +} + +static int handle_pair(struct cpu_state *cpu, struct tlb *tlb, uint32_t insn) { + bool pre = insn & (1u << 24); + bool load = insn & (1u << 22); + bool writeback = insn & (1u << 23); + int64_t imm = sign_extend(bits32(insn, 21, 15), 7) * 8; + unsigned rt = bits32(insn, 4, 0); + unsigned rt2 = bits32(insn, 14, 10); + unsigned rn = bits32(insn, 9, 5); + addr_t base = read_reg(cpu, rn); + addr_t addr = pre ? base + imm : base; + qword_t v1 = 0, v2 = 0; + + if (load) { + if (!mem_read(cpu, tlb, addr, &v1, 8) || !mem_read(cpu, tlb, addr + 8, &v2, 8)) + return INT_GPF; + write_reg(cpu, rt, v1); + write_reg(cpu, rt2, v2); + } else { + v1 = read_reg(cpu, rt); + v2 = read_reg(cpu, rt2); + if (!mem_write(cpu, tlb, addr, &v1, 8) || !mem_write(cpu, tlb, addr + 8, &v2, 8)) + return INT_GPF; + } + if (writeback) + write_reg(cpu, rn, pre ? addr : base + imm); + return INT_NONE; +} + +static int handle_exclusive(struct cpu_state *cpu, struct tlb *tlb, uint32_t insn) { + unsigned rt = bits32(insn, 4, 0); + unsigned rn = bits32(insn, 9, 5); + unsigned rs = bits32(insn, 20, 16); + addr_t addr = read_reg(cpu, rn); + unsigned size = bits32(insn, 31, 30); + unsigned width = 1u << size; + bool is_store = bits32(insn, 23, 21) == 0; + + if (!is_store) { + qword_t val = 0; + if (!mem_read(cpu, tlb, addr, &val, width)) + return INT_GPF; + write_reg(cpu, rt, val); + cpu->exclusive_valid = true; + cpu->exclusive_addr = addr; + return INT_NONE; + } + + qword_t expected = read_reg(cpu, rs); + qword_t newval = read_reg(cpu, rt); + qword_t current = 0; + if (!mem_read(cpu, tlb, addr, ¤t, width)) + return INT_GPF; + if (cpu->exclusive_valid && cpu->exclusive_addr == addr && current == expected) { + if (!mem_write(cpu, tlb, addr, &newval, width)) + return INT_GPF; + write_reg(cpu, rs, 0); + } else { + write_reg(cpu, rs, 1); + } + cpu->exclusive_valid = false; + return INT_NONE; +} + +bool aarch64_insn_ends_block(uint32_t insn) { + if ((insn & 0xffe0001fu) == 0xd4000001u) + return true; + if ((insn & 0xfffffc1fu) == 0xd61f0000u) + return true; + if (bits32(insn, 31, 26) == 0x25) + return true; + if ((insn & 0xff000010u) == 0x54000000u) + return false; + if (bits32(insn, 31, 24) == 0xb4 || bits32(insn, 31, 24) == 0xb5) + return false; + if (bits32(insn, 31, 24) == 0xb6 || bits32(insn, 31, 24) == 0xb7) + return false; + return false; +} + +int aarch64_exec_insn(struct cpu_state *cpu, struct tlb *tlb, addr_t pc, uint32_t insn) { + TRACE("aarch64 %016llx: %08x\n", (unsigned long long) pc, insn); + + if ((insn & 0xffe0001fu) == 0xd4000001u) { + cpu->pc = pc + 4; + return INT_SYSCALL; + } + + if ((insn & 0xfffffc1fu) == 0xd61f0000u) { + unsigned rn = bits32(insn, 9, 5); + cpu->pc = read_reg(cpu, rn); + return INT_NONE; + } + + if (bits32(insn, 31, 26) == 0x25) { + int64_t imm = sign_extend(bits32(insn, 25, 0) << 2, 28); + if (insn & (1u << 31)) { + write_reg(cpu, 30, pc + 4); + cpu->pc = pc + imm; + } else { + cpu->pc = pc + imm; + } + return INT_NONE; + } + + if ((insn & 0xff000010u) == 0x54000000u) { + int64_t imm = sign_extend(bits32(insn, 23, 5) << 2, 21); + unsigned cond = bits32(insn, 3, 0); + if (cond_true(cpu, cond)) + cpu->pc = pc + imm; + else + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 31, 24) == 0xb4 || bits32(insn, 31, 24) == 0xb5) { + unsigned sf = bits32(insn, 31, 31); + unsigned op = bits32(insn, 24, 24); + int64_t imm = sign_extend(bits32(insn, 23, 5) << 2, 21); + unsigned rt = bits32(insn, 4, 0); + qword_t val = read_reg(cpu, rt); + if (!sf) + val = (uint32_t) val; + bool is_zero = val == 0; + if (is_zero == (op == 0)) + cpu->pc = pc + imm; + else + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 31, 24) == 0xb6 || bits32(insn, 31, 24) == 0xb7) { + unsigned op = bits32(insn, 24, 24); + int64_t imm = sign_extend(bits32(insn, 18, 5) << 2, 16); + unsigned bit = bits32(insn, 23, 19); + unsigned rt = bits32(insn, 4, 0); + bool bit_set = (read_reg(cpu, rt) >> bit) & 1; + if (bit_set == op) + cpu->pc = pc + imm; + else + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 31, 29) == 0) { + unsigned op = bits32(insn, 31, 31); + int64_t immlo = bits32(insn, 30, 29); + int64_t immhi = bits32(insn, 23, 5); + int64_t imm = sign_extend((immhi << 2) | immlo, 21); + unsigned rd = bits32(insn, 4, 0); + if (op) { + addr_t page = (pc & ~0xfffull) + (imm << 12); + write_reg(cpu, rd, page); + } else { + write_reg(cpu, rd, pc + imm); + } + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 28, 22) == 0x39) { + cpu->pc = pc + 4; + return handle_load_store(cpu, tlb, insn); + } + + if (bits32(insn, 28, 21) == 0xf1 || bits32(insn, 28, 21) == 0xf3) { + cpu->pc = pc + 4; + return handle_load_store_reg(cpu, tlb, insn); + } + + if (bits32(insn, 28, 25) == 0xa) { + cpu->pc = pc + 4; + return handle_pair(cpu, tlb, insn); + } + + if (bits32(insn, 28, 21) == 0x30) { + cpu->pc = pc + 4; + return handle_exclusive(cpu, tlb, insn); + } + + if ((insn & 0x1f800000u) == 0x12800000u) { + unsigned opc = bits32(insn, 30, 29); + unsigned hw = bits32(insn, 22, 21); + unsigned rd = bits32(insn, 4, 0); + uint16_t imm = bits32(insn, 20, 5); + qword_t val; + qword_t shift = (qword_t) imm << (hw * 16); + if (opc == 0) + val = ~shift; + else if (opc == 2) + val = shift; + else if (opc == 3) + val = (read_reg(cpu, rd) & ~(0xffffull << (hw * 16))) | shift; + else + val = read_reg(cpu, rd); + write_reg(cpu, rd, val); + cpu->pc = pc + 4; + return INT_NONE; + } + + if ((insn & 0x1f000000u) == 0x11000000u) { + unsigned sf = bits32(insn, 31, 31); + unsigned op = bits32(insn, 30, 30); + unsigned sh = bits32(insn, 23, 22); + uint32_t imm = bits32(insn, 21, 10); + if (sh) + imm <<= 12; + unsigned rn = bits32(insn, 9, 5); + unsigned rd = bits32(insn, 4, 0); + bool setflags = bits32(insn, 29, 29); + qword_t op1 = read_reg(cpu, rn); + qword_t res; + if (op) + sub_with_carry(cpu, op1, imm, !setflags, &res); + else + add_with_carry(cpu, op1, imm, false, &res); + if (!sf) + res = (uint32_t) res; + if (setflags && !sf) + set_nzcv(cpu, (int32_t) res < 0, (uint32_t) res == 0, cpu_flag_c(cpu), cpu_flag_v(cpu)); + if (!setflags || rd != 31) + write_reg(cpu, rd, res); + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 28, 23) == 0x24) { + unsigned sf = bits32(insn, 31, 31); + unsigned opc = bits32(insn, 30, 29); + unsigned rd = bits32(insn, 4, 0); + unsigned rn = bits32(insn, 9, 5); + qword_t imm = helper_a64_decode_logical_imm(insn); + qword_t val = read_reg(cpu, rn); + qword_t res = 0; + switch (opc) { + case 0: res = val & imm; break; + case 1: res = val | imm; break; + case 2: res = val ^ imm; break; + case 3: res = val & ~imm; break; + } + if (!sf) + res = (uint32_t) res; + cpu_set_nz(cpu, res); + write_reg(cpu, rd, res); + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 28, 24) == 0x0b || bits32(insn, 28, 24) == 0x1b) { + unsigned sf = bits32(insn, 31, 31); + unsigned opc = bits32(insn, 30, 29); + unsigned shift = bits32(insn, 23, 22); + unsigned rm = bits32(insn, 20, 16); + unsigned imm6 = bits32(insn, 15, 10); + unsigned rn = bits32(insn, 9, 5); + unsigned rd = bits32(insn, 4, 0); + qword_t val1 = read_reg(cpu, rn); + qword_t val2 = read_reg(cpu, rm); + unsigned shamt = imm6 & (sf ? 63 : 31); + if (shift == 0) + val2 <<= shamt; + else if (shift == 1) + val2 = sf ? ((uint64_t) val2 >> shamt) : ((uint32_t) val2 >> shamt); + else if (shift == 2) + val2 = (sqword_t) (sf ? val2 : (int32_t) val2) >> shamt; + qword_t res = 0; + bool setflags = opc >= 2; + switch (opc) { + case 0: add_with_carry(cpu, val1, val2, false, &res); break; + case 1: add_with_carry(cpu, val1, val2, true, &res); break; + case 2: sub_with_carry(cpu, val1, val2, false, &res); break; + case 3: sub_with_carry(cpu, val1, val2, true, &res); break; + } + if (!sf) + res = (uint32_t) res; + if (!setflags || rd != 31) + write_reg(cpu, rd, res); + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 28, 24) == 0x0a) { + unsigned sf = bits32(insn, 31, 31); + unsigned opc = bits32(insn, 30, 29); + unsigned shift = bits32(insn, 23, 22); + unsigned rm = bits32(insn, 20, 16); + unsigned imm6 = bits32(insn, 15, 10); + unsigned rn = bits32(insn, 9, 5); + unsigned rd = bits32(insn, 4, 0); + qword_t val1 = read_reg(cpu, rn); + qword_t val2 = read_reg(cpu, rm); + unsigned shamt = imm6 & (sf ? 63 : 31); + if (shift == 0) + val2 <<= shamt; + else if (shift == 1) + val2 = sf ? ((uint64_t) val2 >> shamt) : ((uint32_t) val2 >> shamt); + else if (shift == 2) + val2 = (sqword_t) (sf ? val2 : (int32_t) val2) >> shamt; + qword_t res = 0; + switch (opc) { + case 0: res = val1 & val2; break; + case 1: res = val1 | val2; break; + case 2: res = val1 ^ val2; break; + case 3: res = val1 & ~val2; break; + } + if (!sf) + res = (uint32_t) res; + cpu_set_nz(cpu, res); + write_reg(cpu, rd, res); + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 28, 23) == 0x25 && (insn & 0x1f800000u) != 0x12800000u) { + unsigned opc = bits32(insn, 30, 29); + unsigned hw = bits32(insn, 22, 21); + unsigned rd = bits32(insn, 4, 0); + uint16_t imm = bits32(insn, 20, 5); + qword_t val = 0; + if (opc == 0) + val = 0; + else + val = read_reg(cpu, rd); + if (opc == 0 || opc == 2) + val |= (qword_t) imm << (hw * 16); + else if (opc == 3) + val = ~((qword_t) imm << (hw * 16)); + write_reg(cpu, rd, val); + cpu->pc = pc + 4; + return INT_NONE; + } + + if (bits32(insn, 28, 21) == 0xd5) { + unsigned op0 = bits32(insn, 20, 19); + unsigned op1 = bits32(insn, 18, 16); + unsigned crn = bits32(insn, 15, 12); + unsigned crm = bits32(insn, 11, 8); + unsigned op2 = bits32(insn, 7, 5); + unsigned rt = bits32(insn, 4, 0); + bool is_msr = insn & (1u << 21); + if (op0 == 3 && op1 == 3 && crn == 13 && crm == 0 && op2 == 2) { + if (is_msr) + cpu->tls_ptr = read_reg(cpu, rt); + else + write_reg(cpu, rt, cpu->tls_ptr); + cpu->pc = pc + 4; + return INT_NONE; + } + } + + if (insn == 0xd503201fu) { + cpu->pc = pc + 4; + return INT_NONE; + } + + if ((insn & 0xffe00000u) == 0xd4200000u) { + cpu->pc = pc + 4; + return INT_BREAKPOINT; + } + + printk("unhandled aarch64 insn 0x%08x at 0x%llx\n", insn, (unsigned long long) pc); + cpu->pc = pc + 4; + return INT_UNDEFINED; +} diff --git a/emu/aarch64-exec.h b/emu/aarch64-exec.h new file mode 100644 index 0000000000..ff7fa3a4e5 --- /dev/null +++ b/emu/aarch64-exec.h @@ -0,0 +1,13 @@ +#ifndef AARCH64_EXEC_H +#define AARCH64_EXEC_H + +#include "emu/cpu.h" +#include "emu/tlb.h" + +/* Execute one AArch64 instruction at pc. Updates cpu->pc unless returning fault. */ +int aarch64_exec_insn(struct cpu_state *cpu, struct tlb *tlb, addr_t pc, uint32_t insn); + +/* True if this instruction ends a basic block at compile time. */ +bool aarch64_insn_ends_block(uint32_t insn); + +#endif diff --git a/emu/cpu-aarch64.h b/emu/cpu-aarch64.h new file mode 100644 index 0000000000..94c13bd139 --- /dev/null +++ b/emu/cpu-aarch64.h @@ -0,0 +1,115 @@ +#ifndef CPU_AARCH64_H +#define CPU_AARCH64_H + +#include "misc.h" +#include "emu/mmu.h" + +#ifdef __KERNEL__ +#include +#else +#include +#endif + +struct cpu_state; +struct tlb; +int cpu_run_to_interrupt(struct cpu_state *cpu, struct tlb *tlb); +void cpu_poke(struct cpu_state *cpu); + +#define AARCH64_REG_COUNT 31 + +/* AArch64 PSTATE flag bits (NZCV in bits 31-28 of pstate). */ +#define PSTATE_N (1u << 31) +#define PSTATE_Z (1u << 30) +#define PSTATE_C (1u << 29) +#define PSTATE_V (1u << 28) + +union vreg { + unsigned __int128 u128; + qword_t qw[2]; + uint32_t u32[4]; + uint16_t u16[8]; + uint8_t u8[16]; + double f64[2]; + float f32[4]; +}; +static_assert(sizeof(union vreg) == 16, "vreg size"); + +struct cpu_state { + struct mmu *mmu; + long cycle; + + qword_t x[AARCH64_REG_COUNT]; /* x0-x30; sp is x[31] alias via accessor */ + qword_t sp; + qword_t pc; + + /* NZCV + exception level / mode bits we care about */ + dword_t pstate; + + /* FP/SIMD */ + union vreg v[32]; + dword_t fpcr; + dword_t fpsr; + + /* TLS (TPIDR_EL0) */ + addr_t tls_ptr; + + /* page fault info */ + addr_t segfault_addr; + bool segfault_was_write; + + int trapno; + + bool *poked_ptr; + bool _poked; + + /* exclusive monitor for LDXR/STXR */ + addr_t exclusive_addr; + bool exclusive_valid; +}; + +#define CPU_OFFSET(field) offsetof(struct cpu_state, field) + +static inline qword_t *cpu_reg(struct cpu_state *cpu, unsigned reg) { + if (reg == 31) + return &cpu->sp; + return &cpu->x[reg]; +} + +static inline bool cpu_flag_n(struct cpu_state *cpu) { return cpu->pstate & PSTATE_N; } +static inline bool cpu_flag_z(struct cpu_state *cpu) { return cpu->pstate & PSTATE_Z; } +static inline bool cpu_flag_c(struct cpu_state *cpu) { return cpu->pstate & PSTATE_C; } +static inline bool cpu_flag_v(struct cpu_state *cpu) { return cpu->pstate & PSTATE_V; } + +static inline void cpu_set_nz(struct cpu_state *cpu, qword_t res) { + cpu->pstate &= ~(PSTATE_N | PSTATE_Z); + if (res == 0) + cpu->pstate |= PSTATE_Z; + if ((sqword_t) res < 0) + cpu->pstate |= PSTATE_N; +} + +static inline void cpu_set_nzc(struct cpu_state *cpu, qword_t res, bool carry) { + cpu_set_nz(cpu, res); + if (carry) + cpu->pstate |= PSTATE_C; + else + cpu->pstate &= ~PSTATE_C; +} + +static inline const char *aarch64_reg_name(unsigned reg) { + if (reg == 31) + return "sp"; + static char buf[4]; + buf[0] = 'x'; + if (reg < 10) { + buf[1] = '0' + reg; + buf[2] = '\0'; + } else { + buf[1] = '0' + reg / 10; + buf[2] = '0' + reg % 10; + buf[3] = '\0'; + } + return buf; +} + +#endif diff --git a/emu/cpu.h b/emu/cpu.h index 184ad15517..dced88e8e7 100644 --- a/emu/cpu.h +++ b/emu/cpu.h @@ -1,6 +1,12 @@ #ifndef EMU_H #define EMU_H +#include "guest/guest-config.h" + +#if GUEST_AARCH64 +#include "emu/cpu-aarch64.h" +#else + #include "misc.h" #include "emu/mmu.h" #include "emu/float80.h" @@ -232,4 +238,6 @@ static inline const char *reg32_name(enum reg32 reg) { } } +#endif /* GUEST_AARCH64 */ + #endif diff --git a/emu/mmu.h b/emu/mmu.h index 42093304a7..08bed8b6e6 100644 --- a/emu/mmu.h +++ b/emu/mmu.h @@ -3,9 +3,15 @@ #include "misc.h" -// top 20 bits of an address, i.e. address >> 12 +// top bits of an address, i.e. address >> 12 +#if GUEST_AARCH64 +typedef qword_t page_t; +#define MEM_PAGES (1ULL << 20) /* 4 GiB guest VA */ +#else typedef dword_t page_t; -#define BAD_PAGE 0x10000 +#define MEM_PAGES (1 << 20) +#endif +#define BAD_PAGE ((page_t) -1) #ifndef __KERNEL__ #define PAGE_BITS 12 @@ -13,10 +19,12 @@ typedef dword_t page_t; #define PAGE_SIZE (1 << PAGE_BITS) #define PAGE(addr) ((addr) >> PAGE_BITS) #define PGOFFSET(addr) ((addr) & (PAGE_SIZE - 1)) -typedef dword_t pages_t; -// bytes MUST be unsigned if you would like this to overflow to zero #define PAGE_ROUND_UP(bytes) (PAGE((bytes) + PAGE_SIZE - 1)) -#define MEM_PAGES (1 << 20) // at least on 32-bit +#if GUEST_AARCH64 +typedef qword_t pages_t; +#else +typedef dword_t pages_t; +#endif #endif struct mmu { diff --git a/emu/tlb.h b/emu/tlb.h index f2af97ed76..ec55ecfba5 100644 --- a/emu/tlb.h +++ b/emu/tlb.h @@ -23,7 +23,7 @@ struct tlb { }; #define TLB_INDEX(addr) (((addr >> PAGE_BITS) & (TLB_SIZE - 1)) ^ (addr >> (PAGE_BITS + TLB_BITS))) -#define TLB_PAGE(addr) (addr & 0xfffff000) +#define TLB_PAGE(addr) ((addr) & ~((addr_t) PAGE_SIZE - 1)) #define TLB_PAGE_EMPTY 1 void tlb_refresh(struct tlb *tlb, struct mmu *mmu); void tlb_free(struct tlb *tlb); diff --git a/guest/guest-config.h b/guest/guest-config.h new file mode 100644 index 0000000000..2f800f8f47 --- /dev/null +++ b/guest/guest-config.h @@ -0,0 +1,16 @@ +#ifndef GUEST_CONFIG_H +#define GUEST_CONFIG_H + +/* Guest ISA is selected at compile time via meson guest_arch (default: aarch64). */ +#if defined(GUEST_ARCH_AARCH64) +#define GUEST_AARCH64 1 +#define GUEST_I386 0 +#elif defined(GUEST_ARCH_I386) +#define GUEST_AARCH64 0 +#define GUEST_I386 1 +#else +#define GUEST_I386 1 +#define GUEST_AARCH64 0 +#endif + +#endif diff --git a/guest/interrupt.h b/guest/interrupt.h new file mode 100644 index 0000000000..ba722e097d --- /dev/null +++ b/guest/interrupt.h @@ -0,0 +1,22 @@ +#ifndef GUEST_INTERRUPT_H +#define GUEST_INTERRUPT_H + +#include "guest/guest-config.h" + +#if GUEST_AARCH64 + +#define INT_NONE -1 +#define INT_UNDEFINED 6 +#define INT_BREAKPOINT 3 +#define INT_DEBUG 1 +#define INT_GPF 13 +#define INT_TIMER 32 +#define INT_SYSCALL 0x100 + +#else + +#include "emu/interrupt.h" + +#endif + +#endif diff --git a/kernel/calls.c b/kernel/calls.c index 23e3306b87..2fe361defa 100644 --- a/kernel/calls.c +++ b/kernel/calls.c @@ -1,11 +1,422 @@ #include #include "debug.h" #include "kernel/calls.h" -#include "emu/interrupt.h" +#include "guest/guest-config.h" +#include "guest/interrupt.h" #include "kernel/memory.h" #include "kernel/signal.h" #include "kernel/task.h" +#if GUEST_AARCH64 + +typedef qword_t (*syscall_aarch64_t)(qword_t, qword_t, qword_t, qword_t, qword_t, qword_t); + +#define SC(fn) (syscall_aarch64_t)(fn) + +static qword_t sc_write(qword_t fd, qword_t buf, qword_t size, qword_t a4, qword_t a5, qword_t a6) { + return sys_write((fd_t) fd, (addr_t) buf, (dword_t) size); +} +static qword_t sc_read(qword_t fd, qword_t buf, qword_t size, qword_t a4, qword_t a5, qword_t a6) { + return sys_read((fd_t) fd, (addr_t) buf, (dword_t) size); +} +static qword_t sc_openat(qword_t dfd, qword_t path, qword_t flags, qword_t mode, qword_t a5, qword_t a6) { + return sys_openat((fd_t) dfd, (addr_t) path, (dword_t) flags, (mode_t_) mode); +} +static qword_t sc_close(qword_t fd, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_close((fd_t) fd); +} +static qword_t sc_exit(qword_t status, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_exit((dword_t) status); +} +static qword_t sc_exit_group(qword_t status, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_exit_group((dword_t) status); +} +static qword_t sc_brk(qword_t addr, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_brk((addr_t) addr); +} +static qword_t sc_getpid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_getpid(); +} +static qword_t sc_gettid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_gettid(); +} +static qword_t sc_getuid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_getuid32(); +} +static qword_t sc_getgid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_getgid32(); +} +static qword_t sc_mmap(qword_t addr, qword_t len, qword_t prot, qword_t flags, qword_t fd, qword_t offset) { + return sys_mmap2((addr_t) addr, (dword_t) len, (dword_t) prot, (dword_t) flags, (fd_t) fd, (dword_t) (offset >> 12)); +} +static qword_t sc_munmap(qword_t addr, qword_t len, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_munmap((addr_t) addr, (uint_t) len); +} +static qword_t sc_mprotect(qword_t addr, qword_t len, qword_t prot, qword_t a4, qword_t a5, qword_t a6) { + return sys_mprotect((addr_t) addr, (uint_t) len, (int_t) prot); +} +static qword_t sc_execve(qword_t file, qword_t argv, qword_t envp, qword_t a4, qword_t a5, qword_t a6) { + return sys_execve((addr_t) file, (addr_t) argv, (addr_t) envp); +} +static qword_t sc_clone(qword_t flags, qword_t stack, qword_t ptid, qword_t tls, qword_t ctid, qword_t a6) { + return sys_clone((dword_t) flags, (addr_t) stack, (addr_t) ptid, (addr_t) tls, (addr_t) ctid); +} +static qword_t sc_uname(qword_t buf, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_uname((addr_t) buf); +} +static qword_t sc_fcntl(qword_t fd, qword_t cmd, qword_t arg, qword_t a4, qword_t a5, qword_t a6) { + return sys_fcntl32((fd_t) fd, (dword_t) cmd, (dword_t) arg); +} +static qword_t sc_ioctl(qword_t fd, qword_t cmd, qword_t arg, qword_t a4, qword_t a5, qword_t a6) { + return sys_ioctl((fd_t) fd, (dword_t) cmd, (dword_t) arg); +} +static qword_t sc_futex(qword_t uaddr, qword_t op, qword_t val, qword_t timeout, qword_t uaddr2, qword_t val3) { + return sys_futex((addr_t) uaddr, (dword_t) op, (dword_t) val, (addr_t) timeout, (addr_t) uaddr2, (dword_t) val3); +} +static qword_t sc_set_tid_address(qword_t tid, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_set_tid_address((addr_t) tid); +} +static qword_t sc_set_robust_list(qword_t head, qword_t len, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_set_robust_list((addr_t) head, (dword_t) len); +} +static qword_t sc_rt_sigaction(qword_t sig, qword_t act, qword_t oldact, qword_t sigsetsize, qword_t a5, qword_t a6) { + return sys_rt_sigaction((int) sig, (addr_t) act, (addr_t) oldact, (dword_t) sigsetsize); +} +static qword_t sc_rt_sigprocmask(qword_t how, qword_t set, qword_t oldset, qword_t sigsetsize, qword_t a5, qword_t a6) { + return sys_rt_sigprocmask((int) how, (addr_t) set, (addr_t) oldset, (dword_t) sigsetsize); +} +static qword_t sc_rt_sigreturn(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_rt_sigreturn(); +} +static qword_t sc_clock_gettime(qword_t clk, qword_t tp, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_clock_gettime((clockid_t) clk, (addr_t) tp); +} +static qword_t sc_gettimeofday(qword_t tv, qword_t tz, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_gettimeofday((addr_t) tv, (addr_t) tz); +} +static qword_t sc_getrandom(qword_t buf, qword_t len, qword_t flags, qword_t a4, qword_t a5, qword_t a6) { + return sys_getrandom((addr_t) buf, (dword_t) len, (dword_t) flags); +} +static qword_t sc_wait4(qword_t pid, qword_t status, qword_t options, qword_t rusage, qword_t a5, qword_t a6) { + return sys_wait4((pid_t_) pid, (addr_t) status, (dword_t) options, (addr_t) rusage); +} +static qword_t sc_getcwd(qword_t buf, qword_t size, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_getcwd((addr_t) buf, (dword_t) size); +} +static qword_t sc_chdir(qword_t path, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_chdir((addr_t) path); +} +static qword_t sc_dup(qword_t fd, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_dup((fd_t) fd); +} +static qword_t sc_dup3(qword_t fd, qword_t newfd, qword_t flags, qword_t a4, qword_t a5, qword_t a6) { + return sys_dup3((fd_t) fd, (fd_t) newfd, (int_t) flags); +} +static qword_t sc_pipe2(qword_t pipefd, qword_t flags, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_pipe2((addr_t) pipefd, (int_t) flags); +} +static qword_t sc_prlimit64(qword_t pid, qword_t resource, qword_t new_limit, qword_t old_limit, qword_t a5, qword_t a6) { + return sys_prlimit64((pid_t_) pid, (dword_t) resource, (addr_t) new_limit, (addr_t) old_limit); +} +static qword_t sc_nanosleep(qword_t req, qword_t rem, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_nanosleep((addr_t) req, (addr_t) rem); +} +static qword_t sc_sched_yield(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_sched_yield(); +} +static qword_t sc_statx(qword_t dfd, qword_t path, qword_t flags, qword_t mask, qword_t buf, qword_t a6) { + return sys_statx((fd_t) dfd, (addr_t) path, (int_t) flags, (uint_t) mask, (addr_t) buf); +} +static qword_t sc_lseek(qword_t fd, qword_t offset, qword_t whence, qword_t a4, qword_t a5, qword_t a6) { + return sys_lseek((fd_t) fd, (dword_t) offset, (dword_t) whence); +} +static qword_t sc_getdents64(qword_t fd, qword_t dirp, qword_t count, qword_t a4, qword_t a5, qword_t a6) { + return sys_getdents64((fd_t) fd, (addr_t) dirp, (dword_t) count); +} +static qword_t sc_faccessat(qword_t dfd, qword_t path, qword_t mode, qword_t flags, qword_t a5, qword_t a6) { + return sys_faccessat((fd_t) dfd, (addr_t) path, (mode_t_) mode, (dword_t) flags); +} +static qword_t sc_fstatat(qword_t dfd, qword_t path, qword_t buf, qword_t flags, qword_t a5, qword_t a6) { + return sys_fstatat64((fd_t) dfd, (addr_t) path, (addr_t) buf, (dword_t) flags); +} +static qword_t sc_readlinkat(qword_t dfd, qword_t path, qword_t buf, qword_t bufsize, qword_t a5, qword_t a6) { + return sys_readlinkat((fd_t) dfd, (addr_t) path, (addr_t) buf, (dword_t) bufsize); +} +static qword_t sc_ppoll(qword_t fds, qword_t nfds, qword_t timeout, qword_t sigmask, qword_t sigsetsize, qword_t a6) { + return sys_ppoll((addr_t) fds, (dword_t) nfds, (addr_t) timeout, (addr_t) sigmask, (dword_t) sigsetsize); +} +static qword_t sc_epoll_create1(qword_t flags, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_epoll_create((int_t) flags); +} +static qword_t sc_epoll_ctl(qword_t epfd, qword_t op, qword_t fd, qword_t event, qword_t a5, qword_t a6) { + return sys_epoll_ctl((fd_t) epfd, (int_t) op, (fd_t) fd, (addr_t) event); +} +static qword_t sc_epoll_pwait(qword_t epfd, qword_t events, qword_t maxevents, qword_t timeout, qword_t sigmask, qword_t sigsetsize) { + return sys_epoll_pwait((fd_t) epfd, (addr_t) events, (int_t) maxevents, (int_t) timeout, (addr_t) sigmask, (dword_t) sigsetsize); +} +static qword_t sc_geteuid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_geteuid32(); +} +static qword_t sc_getegid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_getegid32(); +} +static qword_t sc_prctl(qword_t option, qword_t arg2, qword_t arg3, qword_t arg4, qword_t arg5, qword_t a6) { + return sys_prctl((dword_t) option, (uint_t) arg2, (uint_t) arg3, (uint_t) arg4, (uint_t) arg5); +} +static qword_t sc_tkill(qword_t tid, qword_t sig, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_tkill((pid_t_) tid, (dword_t) sig); +} +static qword_t sc_tgkill(qword_t tgid, qword_t tid, qword_t sig, qword_t a4, qword_t a5, qword_t a6) { + return sys_tgkill((pid_t_) tgid, (pid_t_) tid, (int) sig); +} +static qword_t sc_setsid(qword_t a1, qword_t a2, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_setsid(); +} +static qword_t sc_renameat(qword_t olddfd, qword_t oldpath, qword_t newdfd, qword_t newpath, qword_t a5, qword_t a6) { + return sys_renameat((fd_t) olddfd, (addr_t) oldpath, (fd_t) newdfd, (addr_t) newpath); +} +static qword_t sc_readv(qword_t fd, qword_t iov, qword_t count, qword_t a4, qword_t a5, qword_t a6) { + return sys_readv((fd_t) fd, (addr_t) iov, (dword_t) count); +} +static qword_t sc_writev(qword_t fd, qword_t iov, qword_t count, qword_t a4, qword_t a5, qword_t a6) { + return sys_writev((fd_t) fd, (addr_t) iov, (dword_t) count); +} +static qword_t sc_fstat(qword_t fd, qword_t buf, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_fstat64((fd_t) fd, (addr_t) buf); +} +static qword_t sc_clock_nanosleep(qword_t clk, qword_t flags, qword_t req, qword_t rem, qword_t a5, qword_t a6) { + (void) clk; (void) flags; + // Best-effort: ignore clock id/flags and sleep like nanosleep. + return sys_nanosleep((addr_t) req, (addr_t) rem); +} +static qword_t sc_socket(qword_t domain, qword_t type, qword_t protocol, qword_t a4, qword_t a5, qword_t a6) { + return sys_socket((dword_t) domain, (dword_t) type, (dword_t) protocol); +} +static qword_t sc_bind(qword_t fd, qword_t addr, qword_t addrlen, qword_t a4, qword_t a5, qword_t a6) { + return sys_bind((fd_t) fd, (addr_t) addr, (dword_t) addrlen); +} +static qword_t sc_listen(qword_t fd, qword_t backlog, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_listen((fd_t) fd, (dword_t) backlog); +} +static qword_t sc_accept(qword_t fd, qword_t addr, qword_t addrlen, qword_t a4, qword_t a5, qword_t a6) { + return sys_accept((fd_t) fd, (addr_t) addr, (addr_t) addrlen); +} +static qword_t sc_accept4(qword_t fd, qword_t addr, qword_t addrlen, qword_t flags, qword_t a5, qword_t a6) { + (void) flags; + return sys_accept((fd_t) fd, (addr_t) addr, (addr_t) addrlen); +} +static qword_t sc_socketpair(qword_t domain, qword_t type, qword_t protocol, qword_t sockets, qword_t a5, qword_t a6) { + return sys_socketpair((dword_t) domain, (dword_t) type, (dword_t) protocol, (addr_t) sockets); +} +static qword_t sc_connect(qword_t fd, qword_t addr, qword_t addrlen, qword_t a4, qword_t a5, qword_t a6) { + return sys_connect((fd_t) fd, (addr_t) addr, (dword_t) addrlen); +} +static qword_t sc_getsockname(qword_t fd, qword_t addr, qword_t addrlen, qword_t a4, qword_t a5, qword_t a6) { + return sys_getsockname((fd_t) fd, (addr_t) addr, (addr_t) addrlen); +} +static qword_t sc_getpeername(qword_t fd, qword_t addr, qword_t addrlen, qword_t a4, qword_t a5, qword_t a6) { + return sys_getpeername((fd_t) fd, (addr_t) addr, (addr_t) addrlen); +} +static qword_t sc_sendto(qword_t fd, qword_t buf, qword_t len, qword_t flags, qword_t dest, qword_t addrlen) { + return sys_sendto((fd_t) fd, (addr_t) buf, (dword_t) len, (dword_t) flags, (addr_t) dest, (dword_t) addrlen); +} +static qword_t sc_recvfrom(qword_t fd, qword_t buf, qword_t len, qword_t flags, qword_t src, qword_t addrlen) { + return sys_recvfrom((fd_t) fd, (addr_t) buf, (dword_t) len, (dword_t) flags, (addr_t) src, (addr_t) addrlen); +} +static qword_t sc_setsockopt(qword_t fd, qword_t level, qword_t optname, qword_t optval, qword_t optlen, qword_t a6) { + return sys_setsockopt((fd_t) fd, (dword_t) level, (dword_t) optname, (addr_t) optval, (dword_t) optlen); +} +static qword_t sc_getsockopt(qword_t fd, qword_t level, qword_t optname, qword_t optval, qword_t optlen, qword_t a6) { + return sys_getsockopt((fd_t) fd, (dword_t) level, (dword_t) optname, (addr_t) optval, (addr_t) optlen); +} +static qword_t sc_shutdown(qword_t fd, qword_t how, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_shutdown((fd_t) fd, (dword_t) how); +} +static qword_t sc_eventfd2(qword_t initval, qword_t flags, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_eventfd2((uint_t) initval, (int_t) flags); +} +static qword_t sc_mkdirat(qword_t dfd, qword_t path, qword_t mode, qword_t a4, qword_t a5, qword_t a6) { + return sys_mkdirat((fd_t) dfd, (addr_t) path, (mode_t_) mode); +} +static qword_t sc_unlinkat(qword_t dfd, qword_t path, qword_t flags, qword_t a4, qword_t a5, qword_t a6) { + return sys_unlinkat((fd_t) dfd, (addr_t) path, (int_t) flags); +} +static qword_t sc_renameat2(qword_t olddfd, qword_t oldpath, qword_t newdfd, qword_t newpath, qword_t flags, qword_t a6) { + return sys_renameat2((fd_t) olddfd, (addr_t) oldpath, (fd_t) newdfd, (addr_t) newpath, (int_t) flags); +} +static qword_t sc_clock_getres(qword_t clk, qword_t res, qword_t a3, qword_t a4, qword_t a5, qword_t a6) { + return sys_clock_getres((clockid_t) clk, (addr_t) res); +} + +static syscall_aarch64_t syscall_table_aarch64[450]; + +static void init_syscall_table(void) { + // Numbers from Linux asm-generic/unistd.h (aarch64). + memset(syscall_table_aarch64, 0, sizeof(syscall_table_aarch64)); + syscall_table_aarch64[17] = sc_getcwd; + syscall_table_aarch64[19] = sc_eventfd2; + syscall_table_aarch64[20] = sc_epoll_create1; + syscall_table_aarch64[21] = sc_epoll_ctl; + syscall_table_aarch64[22] = sc_epoll_pwait; + syscall_table_aarch64[23] = sc_dup; + syscall_table_aarch64[24] = sc_dup3; + syscall_table_aarch64[25] = sc_fcntl; + syscall_table_aarch64[29] = sc_ioctl; + syscall_table_aarch64[34] = sc_mkdirat; + syscall_table_aarch64[35] = sc_unlinkat; + syscall_table_aarch64[38] = sc_renameat; + syscall_table_aarch64[48] = sc_faccessat; + syscall_table_aarch64[49] = sc_chdir; + syscall_table_aarch64[56] = sc_openat; + syscall_table_aarch64[57] = sc_close; + syscall_table_aarch64[59] = sc_pipe2; + syscall_table_aarch64[61] = sc_getdents64; + syscall_table_aarch64[62] = sc_lseek; + syscall_table_aarch64[63] = sc_read; + syscall_table_aarch64[64] = sc_write; + syscall_table_aarch64[65] = sc_readv; + syscall_table_aarch64[66] = sc_writev; + syscall_table_aarch64[73] = sc_ppoll; + syscall_table_aarch64[78] = sc_readlinkat; + syscall_table_aarch64[79] = sc_fstatat; + syscall_table_aarch64[80] = sc_fstat; + syscall_table_aarch64[93] = sc_exit; + syscall_table_aarch64[94] = sc_exit_group; + syscall_table_aarch64[96] = sc_set_tid_address; + syscall_table_aarch64[98] = sc_futex; + syscall_table_aarch64[99] = sc_set_robust_list; + syscall_table_aarch64[101] = sc_nanosleep; + syscall_table_aarch64[113] = sc_clock_gettime; + syscall_table_aarch64[114] = sc_clock_getres; + syscall_table_aarch64[115] = sc_clock_nanosleep; + syscall_table_aarch64[124] = sc_sched_yield; + syscall_table_aarch64[130] = sc_tkill; + syscall_table_aarch64[131] = sc_tgkill; + syscall_table_aarch64[134] = sc_rt_sigaction; + syscall_table_aarch64[135] = sc_rt_sigprocmask; + syscall_table_aarch64[139] = sc_rt_sigreturn; + syscall_table_aarch64[157] = sc_setsid; + syscall_table_aarch64[160] = sc_uname; + syscall_table_aarch64[167] = sc_prctl; + syscall_table_aarch64[169] = sc_gettimeofday; + syscall_table_aarch64[172] = sc_getpid; + syscall_table_aarch64[174] = sc_getuid; + syscall_table_aarch64[175] = sc_geteuid; + syscall_table_aarch64[176] = sc_getgid; + syscall_table_aarch64[177] = sc_getegid; + syscall_table_aarch64[178] = sc_gettid; + syscall_table_aarch64[198] = sc_socket; + syscall_table_aarch64[199] = sc_socketpair; + syscall_table_aarch64[200] = sc_bind; + syscall_table_aarch64[201] = sc_listen; + syscall_table_aarch64[202] = sc_accept; + syscall_table_aarch64[203] = sc_connect; + syscall_table_aarch64[204] = sc_getsockname; + syscall_table_aarch64[205] = sc_getpeername; + syscall_table_aarch64[206] = sc_sendto; + syscall_table_aarch64[207] = sc_recvfrom; + syscall_table_aarch64[208] = sc_setsockopt; + syscall_table_aarch64[209] = sc_getsockopt; + syscall_table_aarch64[210] = sc_shutdown; + syscall_table_aarch64[214] = sc_brk; + syscall_table_aarch64[215] = sc_munmap; + syscall_table_aarch64[220] = sc_clone; + syscall_table_aarch64[221] = sc_execve; + syscall_table_aarch64[222] = sc_mmap; + syscall_table_aarch64[226] = sc_mprotect; + syscall_table_aarch64[242] = sc_accept4; + syscall_table_aarch64[260] = sc_wait4; + syscall_table_aarch64[261] = sc_prlimit64; + syscall_table_aarch64[276] = sc_renameat2; + syscall_table_aarch64[278] = sc_getrandom; + syscall_table_aarch64[291] = sc_statx; +} + +void dump_stack(int lines); + +void handle_interrupt(int interrupt) { + static bool initialized; + if (!initialized) { + init_syscall_table(); + initialized = true; + } + + struct cpu_state *cpu = ¤t->cpu; + if (interrupt == INT_SYSCALL) { + unsigned syscall_num = (unsigned) cpu->x[8]; + qword_t result; + if (syscall_num >= array_size(syscall_table_aarch64) || syscall_table_aarch64[syscall_num] == NULL) { + printk("%d(%s) missing aarch64 syscall %u\n", current->pid, current->comm, syscall_num); + result = _ENOSYS; + } else { + STRACE("%d call %-3u ", current->pid, syscall_num); + result = syscall_table_aarch64[syscall_num]( + cpu->x[0], cpu->x[1], cpu->x[2], cpu->x[3], cpu->x[4], cpu->x[5]); + STRACE(" = 0x%llx\n", (unsigned long long) result); + } + if ((sqword_t) result < 0) { + cpu->x[0] = -result; + cpu->pstate |= PSTATE_C; + } else { + cpu->x[0] = result; + cpu->pstate &= ~PSTATE_C; + } + } else if (interrupt == INT_GPF) { + read_wrlock(¤t->mem->lock); + void *ptr = mem_ptr(current->mem, cpu->segfault_addr, cpu->segfault_was_write ? MEM_WRITE : MEM_READ); + read_wrunlock(¤t->mem->lock); + if (ptr == NULL) { + printk("%d page fault on 0x%llx at 0x%llx\n", current->pid, + (unsigned long long) cpu->segfault_addr, (unsigned long long) cpu->pc); + struct siginfo_ info = { + .code = mem_segv_reason(current->mem, cpu->segfault_addr), + .fault.addr = cpu->segfault_addr, + }; + dump_stack(8); + deliver_signal(current, SIGSEGV_, info); + } + } else if (interrupt == INT_UNDEFINED) { + printk("%d illegal instruction at 0x%llx\n", current->pid, (unsigned long long) cpu->pc); + dump_stack(8); + deliver_signal(current, SIGILL_, (struct siginfo_) { + .code = SI_KERNEL_, + .fault.addr = cpu->pc, + }); + } else if (interrupt == INT_BREAKPOINT) { + lock(&pids_lock); + send_signal(current, SIGTRAP_, (struct siginfo_) { + .sig = SIGTRAP_, + .code = SI_KERNEL_, + }); + unlock(&pids_lock); + } else if (interrupt == INT_DEBUG) { + lock(&pids_lock); + send_signal(current, SIGTRAP_, (struct siginfo_) { + .sig = SIGTRAP_, + .code = TRAP_TRACE_, + }); + unlock(&pids_lock); + } else if (interrupt != INT_TIMER) { + printk("%d unhandled interrupt %d\n", current->pid, interrupt); + sys_exit(interrupt); + } + + receive_signals(); + struct tgroup *group = current->group; + lock(&group->lock); + while (group->stopped) + wait_for_ignore_signals(&group->stopped_cond, &group->lock, NULL); + unlock(&group->lock); +} + +void dump_stack(int lines) { + printk("stack at %llx, pc at %llx\n", (unsigned long long) current->cpu.sp, (unsigned long long) current->cpu.pc); +} + +#else /* GUEST_I386 */ + +#include "emu/interrupt.h" + dword_t syscall_stub(void) { return _ENOSYS; } @@ -256,6 +667,7 @@ syscall_t syscall_table[] = { void dump_stack(int lines); +#if !GUEST_AARCH64 void handle_interrupt(int interrupt) { struct cpu_state *cpu = ¤t->cpu; if (interrupt == INT_SYSCALL) { @@ -327,6 +739,7 @@ void handle_interrupt(int interrupt) { wait_for_ignore_signals(&group->stopped_cond, &group->lock, NULL); unlock(&group->lock); } +#endif void dump_maps(void) { extern void proc_maps_dump(struct task *task, struct proc_data *buf); @@ -360,10 +773,14 @@ void dump_mem(addr_t start, uint_t len) { } } +#if !GUEST_AARCH64 void dump_stack(int lines) { printk("stack at %x, base at %x, ip at %x\n", current->cpu.esp, current->cpu.ebp, current->cpu.eip); dump_mem(current->cpu.esp, lines * sizeof(dword_t) * 8); } +#endif + +#endif /* GUEST_AARCH64 */ // TODO find a home for this #ifdef LOG_OVERRIDE diff --git a/kernel/elf64.h b/kernel/elf64.h new file mode 100644 index 0000000000..cffb88bc08 --- /dev/null +++ b/kernel/elf64.h @@ -0,0 +1,68 @@ +#ifndef ELF64_H +#define ELF64_H + +#include "misc.h" + +#define ELF_AARCH64 183 + +struct elf64_header { + uint32_t magic; + byte_t bitness; + byte_t endian; + byte_t elfversion1; + byte_t abi; + byte_t abi_version; + byte_t padding[7]; + uint16_t type; + uint16_t machine; + uint32_t elfversion2; + qword_t entry_point; + qword_t prghead_off; + qword_t secthead_off; + uint32_t flags; + uint16_t header_size; + uint16_t phent_size; + uint16_t phent_count; + uint16_t shent_size; + uint16_t shent_count; + uint16_t sectname_index; +}; + +struct prg64_header { + uint32_t type; + uint32_t flags; + qword_t offset; + qword_t vaddr; + qword_t paddr; + qword_t filesize; + qword_t memsize; + qword_t alignment; +}; + +struct aux64_ent { + qword_t type; + qword_t value; +}; + +#define AX64_PHDR 3 +#define AX64_PHENT 4 +#define AX64_PHNUM 5 +#define AX64_PAGESZ 6 +#define AX64_BASE 7 +#define AX64_FLAGS 8 +#define AX64_ENTRY 9 +#define AX64_UID 11 +#define AX64_EUID 12 +#define AX64_GID 13 +#define AX64_EGID 14 +#define AX64_PLATFORM 15 +#define AX64_HWCAP 16 +#define AX64_CLKTCK 17 +#define AX64_SECURE 23 +#define AX64_RANDOM 25 +#define AX64_HWCAP2 26 +#define AX64_EXECFN 31 +#define AX64_SYSINFO 32 +#define AX64_SYSINFO_EHDR 33 + +#endif diff --git a/kernel/exec-aarch64.c b/kernel/exec-aarch64.c new file mode 100644 index 0000000000..c334e36785 --- /dev/null +++ b/kernel/exec-aarch64.c @@ -0,0 +1,416 @@ +#include "kernel/signal.h" +#include "task.h" +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include "misc.h" +#include "kernel/calls.h" +#include "kernel/random.h" +#include "kernel/errno.h" +#include "fs/fd.h" +#include "kernel/elf.h" +#include "kernel/elf64.h" +#include "kernel/vdso.h" +#include "tools/ptraceomatic-config.h" + +#define ARGV_MAX 32 * PAGE_SIZE + +struct exec_args { + size_t count; + const char *args; +}; + +static inline qword_t align_stack(qword_t sp) { + return sp & ~0xf; +} + +static inline int user_memset64(addr_t start, byte_t val, qword_t len) { + for (qword_t i = 0; i < len; i++) { + if (user_put(start + i, val)) + return 1; + } + return 0; +} + +static inline qword_t copy_string64(qword_t sp, const char *string) { + size_t len = strlen(string) + 1; + sp -= len; + if (user_write(sp, string, len)) + return 0; + return sp; +} + +static inline qword_t args_copy64(qword_t sp, struct exec_args args) { + const char *p = args.args; + for (size_t i = 0; i < args.count; i++) { + sp = copy_string64(sp, p); + if (sp == 0) + return 0; + p += strlen(p) + 1; + } + return sp; +} + +static size_t args_size64(struct exec_args args) { + size_t size = 0; + const char *p = args.args; + for (size_t i = 0; i < args.count; i++) { + size += strlen(p) + 1; + p += strlen(p) + 1; + } + return size; +} + +static int read_header64(struct fd *fd, struct elf64_header *header) { + if (fd->ops->lseek(fd, 0, SEEK_SET)) + return _EIO; + if (fd->ops->read(fd, header, sizeof(*header)) != sizeof(*header)) + return _ENOEXEC; + if (memcmp(&header->magic, ELF_MAGIC, 4) != 0 + || (header->type != ELF_EXECUTABLE && header->type != ELF_DYNAMIC) + || header->bitness != ELF_64BIT + || header->endian != ELF_LITTLEENDIAN + || header->elfversion1 != 1 + || header->machine != ELF_AARCH64) + return _ENOEXEC; + return 0; +} + +static int read_prg_headers64(struct fd *fd, struct elf64_header header, struct prg64_header **ph_out) { + ssize_t ph_size = sizeof(struct prg64_header) * header.phent_count; + struct prg64_header *ph = malloc(ph_size); + if (ph == NULL) + return _ENOMEM; + if (fd->ops->lseek(fd, header.prghead_off, SEEK_SET) < 0) { + free(ph); + return _EIO; + } + if (fd->ops->read(fd, ph, ph_size) != ph_size) { + free(ph); + return _ENOEXEC; + } + *ph_out = ph; + return 0; +} + +static int load_entry64(struct prg64_header ph, addr_t bias, struct fd *fd) { + addr_t addr = ph.vaddr + bias; + addr_t offset = ph.offset; + qword_t memsize = ph.memsize; + qword_t filesize = ph.filesize; + + int flags = P_READ; + if (ph.flags & PH_W) flags |= P_WRITE; + + int err = fd->ops->mmap(fd, current->mem, PAGE(addr), + PAGE_ROUND_UP(filesize + PGOFFSET(addr)), + offset - PGOFFSET(addr), flags, MMAP_PRIVATE); + if (err < 0) + return err; + + mem_pt(current->mem, PAGE(addr))->data->fd = fd_retain(fd); + mem_pt(current->mem, PAGE(addr))->data->file_offset = offset - PGOFFSET(addr); + + if (memsize > filesize) { + qword_t bss_size = memsize - filesize; + addr_t file_end = addr + filesize; + qword_t tail_size = PAGE_SIZE - PGOFFSET(file_end); + if (tail_size == PAGE_SIZE) + tail_size = 0; + if (tail_size != 0) { + write_wrunlock(¤t->mem->lock); + user_memset64(file_end, 0, tail_size); + write_wrlock(¤t->mem->lock); + } + if (tail_size > bss_size) + tail_size = bss_size; + if (bss_size - tail_size != 0) { + err = pt_map_nothing(current->mem, PAGE_ROUND_UP(addr + filesize), + PAGE_ROUND_UP(bss_size - tail_size), flags); + if (err < 0) + return err; + } + } + return 0; +} + +static addr_t find_hole_for_elf64(struct elf64_header *header, struct prg64_header *ph) { + struct prg64_header *first = NULL, *last = NULL; + for (int i = 0; i < header->phent_count; i++) { + if (ph[i].type == PT_LOAD) { + if (first == NULL) + first = &ph[i]; + last = &ph[i]; + } + } + pages_t size = 0; + if (first != NULL) { + pages_t a = PAGE_ROUND_UP(last->vaddr + last->memsize); + pages_t b = PAGE(first->vaddr); + size = a - b; + } + page_t hole = pt_find_hole(current->mem, size); + if (hole == BAD_PAGE) + return 0; + return (addr_t) hole << PAGE_BITS; +} + +int elf_exec_aarch64(struct fd *fd, const char *file, struct exec_args argv, struct exec_args envp) { + int err = 0; + struct elf64_header header; + if ((err = read_header64(fd, &header)) < 0) + return err; + struct prg64_header *ph; + if ((err = read_prg_headers64(fd, header, &ph)) < 0) + return err; + + char *interp_name = NULL; + struct fd *interp_fd = NULL; + struct elf64_header interp_header; + struct prg64_header *interp_ph = NULL; + for (unsigned i = 0; i < header.phent_count; i++) { + if (ph[i].type != PT_INTERP) + continue; + if (interp_name) { + err = _EINVAL; + goto out_free_interp; + } + interp_name = malloc(ph[i].filesize); + if (interp_name == NULL) { + err = _ENOMEM; + goto out_free_ph; + } + if (fd->ops->lseek(fd, ph[i].offset, SEEK_SET) < 0) { + err = _EIO; + goto out_free_interp; + } + if (fd->ops->read(fd, interp_name, ph[i].filesize) != (ssize_t) ph[i].filesize) { + err = _ENOEXEC; + goto out_free_interp; + } + interp_fd = generic_open(interp_name, O_RDONLY, 0); + if (IS_ERR(interp_fd)) { + err = PTR_ERR(interp_fd); + interp_fd = NULL; + goto out_free_interp; + } + if ((err = read_header64(interp_fd, &interp_header)) < 0) { + if (err == _ENOEXEC) + err = _ELIBBAD; + goto out_free_interp; + } + if ((err = read_prg_headers64(interp_fd, interp_header, &interp_ph)) < 0) { + if (err == _ENOEXEC) + err = _ELIBBAD; + goto out_free_interp; + } + } + + lock(¤t->general_lock); + mm_release(current->mm); + task_set_mm(current, mm_new()); + unlock(¤t->general_lock); + write_wrlock(¤t->mem->lock); + + current->mm->exefile = fd_retain(fd); + + addr_t load_addr = 0; + bool load_addr_set = false; + addr_t bias = 0; + + for (unsigned i = 0; i < header.phent_count; i++) { + if (ph[i].type != PT_LOAD) + continue; + if (!load_addr_set && header.type == ELF_DYNAMIC) { + if (interp_name) + bias = 0x56555000; + else + bias = find_hole_for_elf64(&header, ph); + } + if ((err = load_entry64(ph[i], bias, fd)) < 0) + goto beyond_hope; + if (!load_addr_set) { + load_addr = bias + ph[i].vaddr - ph[i].offset; + load_addr_set = true; + } + addr_t brk = bias + ph[i].vaddr + ph[i].memsize; + if (brk > current->mm->start_brk) + current->mm->start_brk = current->mm->brk = BYTES_ROUND_UP(brk); + } + + addr_t entry = bias + header.entry_point; + addr_t interp_base = 0; + + if (interp_name) { + interp_base = find_hole_for_elf64(&interp_header, interp_ph); + for (int i = interp_header.phent_count - 1; i >= 0; i--) { + if (interp_ph[i].type != PT_LOAD) + continue; + if ((err = load_entry64(interp_ph[i], interp_base, interp_fd)) < 0) + goto beyond_hope; + } + entry = interp_base + interp_header.entry_point; + } + + pages_t vdso_pages = sizeof(vdso_data) >> PAGE_BITS; + page_t vdso_page = pt_find_hole(current->mem, vdso_pages + 1); + if (vdso_page == BAD_PAGE) { + err = _ENOMEM; + goto beyond_hope; + } + vdso_page += 1; + if ((err = pt_map(current->mem, vdso_page, vdso_pages, (void *) vdso_data, 0, P_READ | P_EXEC)) < 0) + goto beyond_hope; + mem_pt(current->mem, vdso_page)->data->name = "[vdso]"; + current->mm->vdso = (addr_t) vdso_page << PAGE_BITS; + addr_t vdso_entry = current->mm->vdso + ((struct elf64_header *) vdso_data)->entry_point; + + page_t vvar_page = pt_find_hole(current->mem, VVAR_PAGES); + if (vvar_page == BAD_PAGE) { + err = _ENOMEM; + goto beyond_hope; + } + if ((err = pt_map_nothing(current->mem, vvar_page, VVAR_PAGES, 0)) < 0) + goto beyond_hope; + mem_pt(current->mem, vvar_page)->data->name = "[vvar]"; + + /* AArch64 stack near top of user VA */ + page_t stack_page = 0xfffff; + if ((err = pt_map_nothing(current->mem, stack_page, 1, P_WRITE | P_GROWSDOWN)) < 0) + goto beyond_hope; + write_wrunlock(¤t->mem->lock); + + qword_t sp = ((qword_t) stack_page << PAGE_BITS) + PAGE_SIZE; + sp -= sizeof(qword_t); + + addr_t file_addr = sp = copy_string64(sp, file); + if (sp == 0) goto beyond_hope; + addr_t envp_addr = sp = args_copy64(sp, envp); + if (sp == 0) goto beyond_hope; + current->mm->argv_end = sp; + addr_t argv_addr = sp = args_copy64(sp, argv); + if (sp == 0) goto beyond_hope; + current->mm->argv_start = sp; + sp = align_stack(sp); + + addr_t platform_addr = sp = copy_string64(sp, "aarch64"); + if (sp == 0) goto beyond_hope; + + char random[16] = {}; + get_random(random, sizeof(random)); + addr_t random_addr = sp -= sizeof(random); + if (user_put(sp, random)) + goto beyond_hope; + + size_t argc = argv.count; + size_t envc = envp.count; + struct aux64_ent aux[] = { + {AX64_SYSINFO, vdso_entry}, + {AX64_SYSINFO_EHDR, current->mm->vdso}, + {AX64_HWCAP, 0}, + {AX64_PAGESZ, PAGE_SIZE}, + {AX64_CLKTCK, 0x64}, + {AX64_PHDR, load_addr + header.prghead_off}, + {AX64_PHENT, sizeof(struct prg64_header)}, + {AX64_PHNUM, header.phent_count}, + {AX64_BASE, interp_base}, + {AX64_FLAGS, 0}, + {AX64_ENTRY, bias + header.entry_point}, + {AX64_UID, 0}, + {AX64_EUID, 0}, + {AX64_GID, 0}, + {AX64_EGID, 0}, + {AX64_SECURE, 0}, + {AX64_RANDOM, random_addr}, + {AX64_HWCAP2, 0}, + {AX64_EXECFN, file_addr}, + {AX64_PLATFORM, platform_addr}, + {0, 0}, + }; + + size_t aux_count = 0; + while (aux[aux_count].type != 0) + aux_count++; + + size_t stack_words = argc + 1 + envc + 1 + 1 + aux_count * 2 + 2; + qword_t stack_data_size = stack_words * sizeof(qword_t); + sp = align_stack(sp - stack_data_size); + + qword_t *stack = calloc(1, stack_data_size); + if (stack == NULL) { + err = _ENOMEM; + goto beyond_hope; + } + + size_t idx = 0; + stack[idx++] = argc; + for (size_t i = 0; i < argc; i++) { + const char *p = argv.args; + for (size_t j = 0; j < i; j++) + p += strlen(p) + 1; + stack[idx++] = argv_addr + (p - argv.args); + } + stack[idx++] = 0; + for (size_t i = 0; i < envc; i++) { + const char *p = envp.args; + for (size_t j = 0; j < i; j++) + p += strlen(p) + 1; + stack[idx++] = envp_addr + (p - envp.args); + } + stack[idx++] = 0; + for (size_t i = 0; i < aux_count; i++) { + stack[idx++] = aux[i].type; + stack[idx++] = aux[i].value; + } + stack[idx++] = 0; + stack[idx++] = 0; + + if (user_write(sp, stack, idx * sizeof(qword_t))) { + free(stack); + err = _EFAULT; + goto beyond_hope; + } + free(stack); + + current->cpu.pc = entry; + current->cpu.sp = sp; + current->cpu.x[0] = sp; + memset(¤t->cpu.x[1], 0, sizeof(current->cpu.x) - sizeof(current->cpu.x[0])); + current->cpu.pstate = 0; + + free(ph); + if (interp_name) + free(interp_name); + if (interp_fd) + fd_close(interp_fd); + if (interp_ph) + free(interp_ph); + return 0; + +out_free_interp: + if (interp_name) + free(interp_name); + if (interp_fd) + fd_close(interp_fd); + if (interp_ph) + free(interp_ph); +out_free_ph: + free(ph); + return err; + +beyond_hope: + write_wrunlock(¤t->mem->lock); + if (interp_name) + free(interp_name); + if (interp_fd) + fd_close(interp_fd); + if (interp_ph) + free(interp_ph); + free(ph); + return err; +} diff --git a/kernel/exec.c b/kernel/exec.c index 520693ea6c..ce40f788a9 100644 --- a/kernel/exec.c +++ b/kernel/exec.c @@ -10,6 +10,7 @@ #include #include "misc.h" +#include "guest/guest-config.h" #include "kernel/calls.h" #include "kernel/random.h" #include "kernel/errno.h" @@ -21,12 +22,14 @@ #define ARGV_MAX 32 * PAGE_SIZE struct exec_args { - // number of arguments size_t count; - // series of count null-terminated strings, plus an extra null for good measure const char *args; }; +#if GUEST_AARCH64 +int elf_exec_aarch64(struct fd *fd, const char *file, struct exec_args argv, struct exec_args envp); +#else + static inline dword_t align_stack(dword_t sp); static inline ssize_t user_strlen(dword_t p); static inline int user_memset(addr_t start, byte_t val, dword_t len); @@ -424,6 +427,8 @@ static int elf_exec(struct fd *fd, const char *file, struct exec_args argv, stru goto out_free_interp; } +#endif /* !GUEST_AARCH64 */ + static size_t args_size(struct exec_args args) { const char *args_end = args.args; for (size_t i = 0; i < args.count; i++) { @@ -473,7 +478,11 @@ static inline int user_memset(addr_t start, byte_t val, dword_t len) { } static int format_exec(struct fd *fd, const char *file, struct exec_args argv, struct exec_args envp) { +#if GUEST_AARCH64 + int err = elf_exec_aarch64(fd, file, argv, envp); +#else int err = elf_exec(fd, file, argv, envp); +#endif if (err != _ENOEXEC) return err; // other formats would go here diff --git a/kernel/fork.c b/kernel/fork.c index 82b7694285..86c713b590 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2,6 +2,7 @@ #include "kernel/task.h" #include "fs/fd.h" #include "kernel/calls.h" +#include "guest/guest-config.h" #include "fs/tty.h" #include "kernel/mm.h" #include "kernel/ptrace.h" @@ -55,8 +56,13 @@ static struct tgroup *tgroup_copy(struct tgroup *old_group) { static int copy_task(struct task *task, dword_t flags, addr_t stack, addr_t ptid_addr, addr_t tls_addr, addr_t ctid_addr) { task->vfork = NULL; - if (stack != 0) + if (stack != 0) { +#if GUEST_AARCH64 + task->cpu.sp = stack; +#else task->cpu.esp = stack; +#endif + } int err; struct mm *mm = task->mm; @@ -106,9 +112,13 @@ static int copy_task(struct task *task, dword_t flags, addr_t stack, addr_t ptid unlock(&pids_lock); if (flags & CLONE_SETTLS_) { +#if GUEST_AARCH64 + task->cpu.tls_ptr = tls_addr; +#else err = task_set_thread_area(task, tls_addr); if (err < 0) goto fail_free_sighand; +#endif } err = _EFAULT; @@ -161,7 +171,11 @@ dword_t sys_clone(dword_t flags, addr_t stack, addr_t ptid, addr_t tls, addr_t c unlock(&pids_lock); return err; } +#if GUEST_AARCH64 + task->cpu.x[0] = 0; +#else task->cpu.eax = 0; +#endif struct vfork_info vfork; if (flags & CLONE_VFORK_) { diff --git a/kernel/memory.c b/kernel/memory.c index f022f96565..9d98b711b2 100644 --- a/kernel/memory.c +++ b/kernel/memory.c @@ -14,6 +14,7 @@ #include "asbestos/asbestos.h" #include "kernel/vdso.h" #include "kernel/task.h" +#include "guest/guest-config.h" #include "fs/fd.h" // increment the change count @@ -79,9 +80,13 @@ void mem_next_page(struct mem *mem, page_t *page) { } page_t pt_find_hole(struct mem *mem, pages_t size) { - page_t hole_end = 0; // this can never be used before initializing but gcc doesn't realize + page_t hole_end = 0; bool in_hole = false; +#if GUEST_AARCH64 + for (page_t page = 0xfffff; page > 0x10000; page--) { +#else for (page_t page = 0xf7ffd; page > 0x40000; page--) { +#endif // I don't know how this works but it does if (!in_hole && mem_pt(mem, page) == NULL) { in_hole = true; diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 40d55cc589..bccdf0ecae 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -3,8 +3,18 @@ #include "kernel/errno.h" #include "kernel/signal.h" #include "task.h" +#include "guest/guest-config.h" #include +#if GUEST_AARCH64 + +dword_t sys_ptrace(dword_t request, dword_t pid, addr_t addr, dword_t data) { + (void) request; (void) pid; (void) addr; (void) data; + return _ENOSYS; +} + +#else + // Returns stopped child with the given pid, locked with the ptrace lock static struct task *find_child(pid_t_ pid) { struct task *child = NULL; @@ -253,3 +263,5 @@ dword_t sys_ptrace(dword_t request, dword_t pid, addr_t addr, dword_t data) { return _EPERM; } } + +#endif /* !GUEST_AARCH64 */ diff --git a/kernel/signal-aarch64.c b/kernel/signal-aarch64.c new file mode 100644 index 0000000000..ab878140cf --- /dev/null +++ b/kernel/signal-aarch64.c @@ -0,0 +1,22 @@ +#include +#include "kernel/signal-aarch64.h" +#include "guest/interrupt.h" + +void aarch64_setup_sigcontext(struct sigcontext_aarch64 *sc, struct cpu_state *cpu) { + memset(sc, 0, sizeof(*sc)); + for (int i = 0; i < 31; i++) + sc->regs[i] = cpu->x[i]; + sc->sp = cpu->sp; + sc->pc = cpu->pc; + sc->pstate = cpu->pstate; + if (cpu->trapno == INT_GPF) + sc->fault_address = cpu->segfault_addr; +} + +void aarch64_restore_sigcontext(struct sigcontext_aarch64 *context, struct cpu_state *cpu) { + for (int i = 0; i < 31; i++) + cpu->x[i] = context->regs[i]; + cpu->sp = context->sp; + cpu->pc = context->pc; + cpu->pstate = context->pstate; +} diff --git a/kernel/signal-aarch64.h b/kernel/signal-aarch64.h new file mode 100644 index 0000000000..1e3da27bd0 --- /dev/null +++ b/kernel/signal-aarch64.h @@ -0,0 +1,42 @@ +#ifndef SIGNAL_AARCH64_H +#define SIGNAL_AARCH64_H + +#include "kernel/signal.h" +#include "emu/cpu.h" + +struct sigcontext_aarch64 { + qword_t fault_address; + qword_t regs[31]; + qword_t sp; + qword_t pc; + qword_t pstate; + union { + byte_t __reserved[512]; + }; +}; + +struct ucontext_aarch64 { + qword_t flags; + qword_t link; + struct stack_t_ stack; + struct sigcontext_aarch64 mcontext; + sigset_t_ sigmask; +} __attribute__((packed)); + +struct rt_sigframe_aarch64 { + qword_t restorer; + int_t sig; + qword_t pinfo; + qword_t puc; + union { + struct siginfo_ info; + char __pad[128]; + }; + struct ucontext_aarch64 uc; + char retcode[8]; +}; + +void aarch64_setup_sigcontext(struct sigcontext_aarch64 *sc, struct cpu_state *cpu); +void aarch64_restore_sigcontext(struct sigcontext_aarch64 *context, struct cpu_state *cpu); + +#endif diff --git a/kernel/signal.c b/kernel/signal.c index 9aeea190f9..b3559e3d6c 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -7,6 +7,10 @@ #include "kernel/vdso.h" #include "emu/interrupt.h" +#if GUEST_AARCH64 +#include "kernel/signal-aarch64.h" +#endif + #if is_gcc(9) #pragma GCC diagnostic ignored "-Waddress-of-packed-member" #endif @@ -15,7 +19,7 @@ int xsave_extra = 0; int fxsave_extra = 0; static void sigmask_set(sigset_t_ set); static void altstack_to_user(struct sighand *sighand, struct stack_t_ *user_stack); -static bool is_on_altstack(dword_t sp, struct sighand *sighand); +static bool is_on_altstack(addr_t sp, struct sighand *sighand); static int signal_is_blockable(int sig) { return sig != SIGKILL_ && sig != SIGSTOP_; @@ -156,6 +160,9 @@ static addr_t sigreturn_trampoline(const char *name) { } static void setup_sigcontext(struct sigcontext_ *sc, struct cpu_state *cpu) { +#if GUEST_AARCH64 + (void) sc; (void) cpu; +#else sc->ax = cpu->eax; sc->bx = cpu->ebx; sc->cx = cpu->ecx; @@ -172,6 +179,7 @@ static void setup_sigcontext(struct sigcontext_ *sc, struct cpu_state *cpu) { sc->cr2 = cpu->segfault_addr; // TODO more shit sc->oldmask = current->blocked & 0xffffffff; +#endif } static void setup_sigframe(struct siginfo_ *info, struct sigframe_ *frame) { @@ -193,6 +201,17 @@ static void setup_sigframe(struct siginfo_ *info, struct sigframe_ *frame) { } static void setup_rt_sigframe(struct siginfo_ *info, struct rt_sigframe_ *frame) { +#if GUEST_AARCH64 + struct rt_sigframe_aarch64 *aframe = (struct rt_sigframe_aarch64 *) frame; + aframe->restorer = sigreturn_trampoline("__kernel_rt_sigreturn"); + aframe->sig = info->sig; + aframe->info = *info; + aframe->uc.flags = 0; + aframe->uc.link = 0; + altstack_to_user(current->sighand, &aframe->uc.stack); + aarch64_setup_sigcontext(&aframe->uc.mcontext, ¤t->cpu); + aframe->uc.sigmask = current->blocked; +#else frame->restorer = sigreturn_trampoline("__kernel_rt_sigreturn"); frame->sig = info->sig; frame->info = *info; @@ -213,6 +232,7 @@ static void setup_rt_sigframe(struct siginfo_ *info, struct rt_sigframe_ *frame) .int80 = 0x80cd, }; memcpy(frame->retcode, &rt_retcode, sizeof(rt_retcode)); +#endif } static void receive_signal(struct sighand *sighand, struct siginfo_ *info) { @@ -238,7 +258,12 @@ static void receive_signal(struct sighand *sighand, struct siginfo_ *info) { struct sigaction_ *action = &sighand->action[info->sig]; bool need_siginfo = action->flags & SA_SIGINFO_; - // setup the frame +#if GUEST_AARCH64 + struct rt_sigframe_aarch64 rt_frame = {}; + setup_rt_sigframe(info, (struct rt_sigframe_ *) &rt_frame); + void *frame_ptr = &rt_frame; + size_t frame_size = sizeof(rt_frame); +#else union { struct sigframe_ sigframe; struct rt_sigframe_ rt_sigframe; @@ -251,12 +276,19 @@ static void receive_signal(struct sighand *sighand, struct siginfo_ *info) { setup_sigframe(info, &frame.sigframe); frame_size = sizeof(frame.sigframe); } + void *frame_ptr = &frame; +#endif // set up registers for signal handler +#if GUEST_AARCH64 + current->cpu.x[0] = info->sig; + current->cpu.pc = sighand->action[info->sig].handler; + addr_t sp = current->cpu.sp; +#else current->cpu.eax = info->sig; current->cpu.eip = sighand->action[info->sig].handler; - dword_t sp = current->cpu.esp; +#endif if (sighand->altstack && !is_on_altstack(sp, sighand)) { sp = sighand->altstack + sighand->altstack_size; } @@ -269,9 +301,15 @@ static void receive_signal(struct sighand *sighand, struct siginfo_ *info) { sp -= fxsave_extra; } sp -= frame_size; +#if GUEST_AARCH64 + // AArch64 ABI: SP must be 16-byte aligned at a public interface. + sp &= ~0xfull; + current->cpu.sp = sp; +#else // align sp + 4 on a 16-byte boundary because that's what the abi says sp = ((sp + 4) & ~0xf) - 4; current->cpu.esp = sp; +#endif // Update the mask. By default the signal will be blocked while in the // handler, but sigaction is allowed to customize this. @@ -281,14 +319,23 @@ static void receive_signal(struct sighand *sighand, struct siginfo_ *info) { // these have to be filled in after the location of the frame is known if (need_siginfo) { - frame.rt_sigframe.pinfo = sp + offsetof(struct rt_sigframe_, info); - frame.rt_sigframe.puc = sp + offsetof(struct rt_sigframe_, uc); - current->cpu.edx = frame.rt_sigframe.pinfo; - current->cpu.ecx = frame.rt_sigframe.puc; +#if GUEST_AARCH64 + struct rt_sigframe_aarch64 *aframe = frame_ptr; + aframe->pinfo = sp + offsetof(struct rt_sigframe_aarch64, info); + aframe->puc = sp + offsetof(struct rt_sigframe_aarch64, uc); + current->cpu.x[1] = aframe->pinfo; + current->cpu.x[2] = aframe->puc; +#else + struct rt_sigframe_ *frame = frame_ptr; + frame->pinfo = sp + offsetof(struct rt_sigframe_, info); + frame->puc = sp + offsetof(struct rt_sigframe_, uc); + current->cpu.edx = frame->pinfo; + current->cpu.ecx = frame->puc; +#endif } // install frame - if (user_write(sp, &frame, frame_size)) { + if (user_write(sp, frame_ptr, frame_size)) { printk("failed to install frame for %d at %#x\n", info->sig, sp); deliver_signal(current, SIGSEGV_, SIGINFO_NIL); } @@ -380,6 +427,7 @@ void receive_signals(void) { } } +#if !GUEST_AARCH64 static void restore_sigcontext(struct sigcontext_ *context, struct cpu_state *cpu) { cpu->eax = context->ax; cpu->ebx = context->bx; @@ -396,8 +444,27 @@ static void restore_sigcontext(struct sigcontext_ *context, struct cpu_state *cp #define USE_FLAGS 0b1010000110111010101 cpu->eflags = (context->flags & USE_FLAGS) | (cpu->eflags & ~USE_FLAGS); } +#endif dword_t sys_rt_sigreturn(void) { +#if GUEST_AARCH64 + struct cpu_state *cpu = ¤t->cpu; + struct rt_sigframe_aarch64 frame; + if (user_get(cpu->sp - offsetof(struct rt_sigframe_aarch64, sig), frame)) { + deliver_signal(current, SIGSEGV_, SIGINFO_NIL); + return _EFAULT; + } + aarch64_restore_sigcontext(&frame.uc.mcontext, cpu); + lock(¤t->sighand->lock); + if (!is_on_altstack(cpu->sp, current->sighand) && + frame.uc.stack.size >= MINSIGSTKSZ_) { + current->sighand->altstack = frame.uc.stack.stack; + current->sighand->altstack_size = frame.uc.stack.size; + } + sigmask_set(frame.uc.sigmask); + unlock(¤t->sighand->lock); + return 0; +#else struct cpu_state *cpu = ¤t->cpu; struct rt_sigframe_ frame; // esp points past the first field of the frame @@ -417,8 +484,10 @@ dword_t sys_rt_sigreturn(void) { sigmask_set(frame.uc.sigmask); unlock(¤t->sighand->lock); return cpu->eax; +#endif } +#if !GUEST_AARCH64 dword_t sys_sigreturn(void) { struct cpu_state *cpu = ¤t->cpu; struct sigframe_ frame; @@ -435,6 +504,7 @@ dword_t sys_sigreturn(void) { unlock(¤t->sighand->lock); return cpu->eax; } +#endif struct sighand *sighand_new(void) { struct sighand *sighand = malloc(sizeof(struct sighand)); @@ -568,7 +638,7 @@ int_t sys_rt_sigpending(addr_t set_addr) { return 0; } -static bool is_on_altstack(dword_t sp, struct sighand *sighand) { +static bool is_on_altstack(addr_t sp, struct sighand *sighand) { return sp > sighand->altstack && sp <= sighand->altstack + sighand->altstack_size; } @@ -578,7 +648,11 @@ static void altstack_to_user(struct sighand *sighand, struct stack_t_ *user_stac user_stack->flags = 0; if (sighand->altstack == 0) user_stack->flags |= SS_DISABLE_; +#if GUEST_AARCH64 + if (is_on_altstack(current->cpu.sp, sighand)) +#else if (is_on_altstack(current->cpu.esp, sighand)) +#endif user_stack->flags |= SS_ONSTACK_; } @@ -595,7 +669,12 @@ dword_t sys_sigaltstack(addr_t ss_addr, addr_t old_ss_addr) { } } if (ss_addr != 0) { - if (is_on_altstack(current->cpu.esp, sighand)) { +#if GUEST_AARCH64 + if (is_on_altstack(current->cpu.sp, sighand)) +#else + if (is_on_altstack(current->cpu.esp, sighand)) +#endif + { unlock(&sighand->lock); return _EPERM; } diff --git a/kernel/uname.c b/kernel/uname.c index 94c4dd4c50..4e3a5f415c 100644 --- a/kernel/uname.c +++ b/kernel/uname.c @@ -1,6 +1,7 @@ #include #include #include "kernel/calls.h" +#include "guest/guest-config.h" #include "platform/platform.h" #if __APPLE__ @@ -24,7 +25,11 @@ void do_uname(struct uname *uts) { strcpy(uts->hostname, hostname); strcpy(uts->release, "4.20.69-ish"); snprintf(uts->version, sizeof(uts->version), "%s %s %s", uname_version, __DATE__, __TIME__); +#if GUEST_AARCH64 + strcpy(uts->arch, "aarch64"); +#else strcpy(uts->arch, "i686"); +#endif strcpy(uts->domain, "(none)"); } diff --git a/kernel/vdso.c b/kernel/vdso.c index 463aa385f7..dd8d5ad3a7 100644 --- a/kernel/vdso.c +++ b/kernel/vdso.c @@ -2,6 +2,10 @@ #include #include #include "kernel/elf.h" +#include "guest/guest-config.h" +#if GUEST_AARCH64 +#include "kernel/elf64.h" +#endif #include "kernel/vdso.h" __asm__(".data\n" @@ -11,10 +15,22 @@ __asm__(".data\n" ".skip "str(VDSO_PAGES)" * (1 << 12) - (. - vdso_data)\n"); int vdso_symbol(const char *name) { +#if GUEST_AARCH64 + struct elf64_header *header = (void *) vdso_data; + struct prg64_header *ph = (void *) ((char *) header + header->prghead_off); + struct dyn_ent { + qword_t tag; + qword_t val; + } *dyn = NULL; + for (int i = 0; i < header->phent_count; i++) { + if (ph[i].type == PT_DYNAMIC) { + dyn = (void *) ((char *) header + ph[i].offset); + break; + } + } +#else struct elf_header *header = (void *) vdso_data; struct prg_header *ph = (void *) ((char *) header + header->prghead_off); - - // find the PT_DYNAMIC section struct dyn_ent *dyn = NULL; for (int i = 0; i < header->phent_count; i++) { if (ph[i].type == PT_DYNAMIC) { @@ -22,10 +38,10 @@ int vdso_symbol(const char *name) { break; } } +#endif if (dyn == NULL) goto fail; - // grab pointers to the symbols and the strings char *strings = NULL; struct elf_sym *syms = NULL; uint32_t *hash = NULL; @@ -41,15 +57,13 @@ int vdso_symbol(const char *name) { if (strings == NULL || syms == NULL || hash == NULL) goto fail; - // conveniently enough, the hashtable includes the number of symbols, which doesn't seeem to be anywhere else - // https://flapenguin.me/2017/04/24/elf-lookup-dt-hash/ int num_syms = hash[1]; for (int i = 0; i < num_syms; i++) { char *sym_name = strings + syms[i].name; if (strcmp(name, sym_name) == 0) return syms[i].value; } - return 0; // symbol not found + return 0; fail: // It shouldn't be possible to actually end up with an invalid vsdo compiled in diff --git a/meson.build b/meson.build index 69545530c2..52f925b5d4 100644 --- a/meson.build +++ b/meson.build @@ -22,6 +22,13 @@ endforeach add_project_arguments('-DLOG_HANDLER_' + get_option('log_handler').to_upper() + '=1', language: 'c') add_project_arguments('-DENGINE_' + get_option('engine').to_upper() + '=1', language: 'c') +guest_arch = get_option('guest_arch') +if guest_arch == 'aarch64' + add_project_arguments('-DGUEST_ARCH_AARCH64', language: 'c') +else + add_project_arguments('-DGUEST_ARCH_I386', language: 'c') +endif + if get_option('no_crlf') add_project_arguments('-DNO_CRLF', language: 'c') endif @@ -39,32 +46,59 @@ dependencies = [librt, libm, libdl, threads, sqlite3] subdir('vdso') # ish depends on the vdso -offsets = custom_target('offsets', - output: 'cpu-offsets.h', input: 'asbestos/offsets.c', depfile: 'cpu-offsets.h.d', - command: [find_program('tools/staticdefine.sh'), '@OUTDIR@/compile_commands.json', '@INPUT@', '@OUTPUT@', '@DEPFILE@']) - -emu_src = [ - 'emu/tlb.c', - 'emu/fpu.c', - 'emu/vec.c', - 'emu/mmx.c', - 'emu/float80.c', - -] -gadgets = 'asbestos/gadgets-' + host_machine.cpu_family() -emu_src += [ - 'asbestos/asbestos.c', - 'asbestos/gen.c', - 'asbestos/helpers.c', - gadgets+'/entry.S', - gadgets+'/memory.S', - gadgets+'/control.S', - gadgets+'/math.S', - gadgets+'/bits.S', - gadgets+'/string.S', - gadgets+'/misc.S', - offsets, -] +emu_src = [] +if guest_arch == 'aarch64' + offsets = custom_target('offsets', + output: 'cpu-offsets.h', input: 'asbestos/offsets-aarch64.c', depfile: 'cpu-offsets.h.d', + command: [find_program('tools/staticdefine.sh'), '@OUTDIR@/compile_commands.json', '@INPUT@', '@OUTPUT@', '@DEPFILE@']) + guest_gadgets = 'asbestos/gadgets-guest-aarch64' + emu_src += [ + 'emu/tlb.c', + 'emu/aarch64-exec.c', + 'asbestos/asbestos.c', + 'asbestos/gen-aarch64.c', + 'asbestos/helpers-aarch64.c', + offsets, + ] + if host_machine.cpu_family() == 'aarch64' + emu_src += [ + guest_gadgets + '/entry.S', + guest_gadgets + '/control.S', + guest_gadgets + '/memory.S', + guest_gadgets + '/math.S', + guest_gadgets + '/misc.S', + ] + else + emu_src += [ + guest_gadgets + '/entry-host.c', + guest_gadgets + '/gadgets-c-hot.c', + guest_gadgets + '/gadgets-common.c', + ] + endif +else + offsets = custom_target('offsets', + output: 'cpu-offsets.h', input: 'asbestos/offsets.c', depfile: 'cpu-offsets.h.d', + command: [find_program('tools/staticdefine.sh'), '@OUTDIR@/compile_commands.json', '@INPUT@', '@OUTPUT@', '@DEPFILE@']) + gadgets = 'asbestos/gadgets-' + host_machine.cpu_family() + emu_src += [ + 'emu/tlb.c', + 'emu/fpu.c', + 'emu/vec.c', + 'emu/mmx.c', + 'emu/float80.c', + 'asbestos/asbestos.c', + 'asbestos/gen.c', + 'asbestos/helpers.c', + gadgets+'/entry.S', + gadgets+'/memory.S', + gadgets+'/control.S', + gadgets+'/math.S', + gadgets+'/bits.S', + gadgets+'/string.S', + gadgets+'/misc.S', + offsets, + ] +endif libish_emu = library('ish_emu', emu_src, include_directories: includes) @@ -145,7 +179,14 @@ if get_option('kernel') == 'ish' 'fs/poll.c', 'kernel/poll.c', 'kernel/epoll.c', - + ] + if guest_arch == 'aarch64' + src += [ + 'kernel/exec-aarch64.c', + 'kernel/signal-aarch64.c', + ] + endif + src += [ 'util/timer.c', 'util/sync.c', 'util/fifo.c', @@ -213,7 +254,14 @@ if not meson.is_cross_build() # test for floating point library float80_test = executable('float80_test', ['emu/float80.c', 'emu/float80-test.c'], dependencies: [libm]) test('float80', float80_test) + + if guest_arch == 'aarch64' + aarch64_test = find_program('tests/aarch64/run-test.sh') + test('aarch64-hello', aarch64_test, timeout: 60) + endif endif -e2e_test = find_program('tests/e2e/e2e.bash') -test('e2e', e2e_test, args: ['-y'], timeout: 180) +if guest_arch == 'i386' + e2e_test = find_program('tests/e2e/e2e.bash') + test('e2e', e2e_test, args: ['-y'], timeout: 180) +endif diff --git a/meson_options.txt b/meson_options.txt index 4ff0b3ecfc..d70e5c6bc5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,6 +2,8 @@ option('log', type: 'string', value: '') option('nolog', type: 'string', value: '') option('log_handler', type: 'string', value: 'dprintf') +option('guest_arch', type: 'combo', choices: ['i386', 'aarch64'], value: 'aarch64', + description: 'Guest CPU architecture to emulate (i386 is legacy test-only)') option('engine', type: 'combo', choices: ['asbestos', 'unicorn'], value: 'asbestos') option('kernel', type: 'combo', choices: ['ish', 'linux'], value: 'ish') option('kconfig', type: 'array', value: []) diff --git a/misc.h b/misc.h index bf3ad090ca..81caedc330 100644 --- a/misc.h +++ b/misc.h @@ -1,6 +1,8 @@ #ifndef MISC_H #define MISC_H +#include "guest/guest-config.h" + #ifdef __KERNEL__ #include #else @@ -110,7 +112,11 @@ typedef int32_t sdword_t; typedef uint16_t word_t; typedef uint8_t byte_t; +#if GUEST_AARCH64 +typedef qword_t addr_t; +#else typedef dword_t addr_t; +#endif typedef dword_t uint_t; typedef sdword_t int_t; diff --git a/tests/aarch64/exit42.S b/tests/aarch64/exit42.S new file mode 100644 index 0000000000..212dd5d039 --- /dev/null +++ b/tests/aarch64/exit42.S @@ -0,0 +1,6 @@ + .global _start + .section .text +_start: + movz x8, #93, lsl #0 + movz x0, #42, lsl #0 + svc #0 diff --git a/tests/aarch64/hello.S b/tests/aarch64/hello.S new file mode 100644 index 0000000000..0235a3aabd --- /dev/null +++ b/tests/aarch64/hello.S @@ -0,0 +1,13 @@ + .global _start + .section .text +_start: + movz x8, #64, lsl #0 + movz x0, #1, lsl #0 + adr x1, msg + movz x2, #6, lsl #0 + svc #0 + movz x8, #93, lsl #0 + movz x0, #0, lsl #0 + svc #0 +msg: + .ascii "hello\n" diff --git a/tests/aarch64/run-test.sh b/tests/aarch64/run-test.sh new file mode 100755 index 0000000000..c66c7e4de5 --- /dev/null +++ b/tests/aarch64/run-test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +ISH="${ISH:-$ROOT/build/ish}" +HELLO="$ROOT/tests/aarch64/hello" + +if [[ ! -x "$ISH" ]]; then + echo "aarch64 ish binary not found at $ISH" >&2 + exit 1 +fi + +if [[ ! -f "$HELLO" ]]; then + clang -target aarch64-linux -fuse-ld=lld -nostdlib -static \ + -o "$HELLO" "$ROOT/tests/aarch64/hello.S" +fi + +EXIT42="$ROOT/tests/aarch64/exit42" +if [[ ! -f "$EXIT42" ]]; then + clang -target aarch64-linux -fuse-ld=lld -nostdlib -static \ + -o "$EXIT42" "$ROOT/tests/aarch64/exit42.S" +fi + +out="$("$ISH" "$HELLO" 2>/dev/null)" +test "$out" = "hello" + +"$ISH" "$EXIT42" 2>/dev/null || rc=$? +test "${rc:-0}" = "42" diff --git a/tools/meson.build b/tools/meson.build index fc06c10fa4..ccc5326dbc 100644 --- a/tools/meson.build +++ b/tools/meson.build @@ -1,6 +1,6 @@ if get_option('kernel') == 'ish' - # these tools are specific to x86_64 linux - if host_machine.system() == 'linux' and host_machine.cpu() == 'x86_64' + # these tools are specific to x86_64 linux with i386 guests + if host_machine.system() == 'linux' and host_machine.cpu() == 'x86_64' and get_option('guest_arch') == 'i386' transplant_src = [ 'vdso-transplant.c', 'ptutil.c', @@ -15,7 +15,7 @@ if get_option('kernel') == 'ish' endif unicorn = cc.find_library('unicorn', required: false) - if unicorn.found() + if unicorn.found() and get_option('guest_arch') == 'i386' executable('unicornomatic', ['unicornomatic.c', 'undefined-flags.c'], dependencies: [ish, unicorn]) configure_file(input: 'ptraceomatic-gdb.gdb', output: 'unicornomatic-gdb.gdb', copy: true) endif diff --git a/vdso/meson.build b/vdso/meson.build index 4be7b952d3..11cb4bb803 100644 --- a/vdso/meson.build +++ b/vdso/meson.build @@ -1,11 +1,5 @@ -# The VDSO gets inserted into the address space of emulated processes, so it -# needs to be compiled as an i386 ELF shared library. This requires a cross -# compiler and linker on most of the platforms that matter (Mac, 64-bit Linux, -# Raspberry Pi). Clang and LLD are easy to install on all of these platforms, -# so that's currently the only supported option (though if you have some other -# compatible toolchain you can edit this file to point to it instead.) +# VDSO build for guest architecture (i386 or aarch64). -# Default install paths for Homebrew on Intel, Homebrew on Apple silicon, and MacPorts, respectively. clang = find_program('/usr/local/opt/llvm/bin/clang', '/opt/homebrew/opt/llvm/bin/clang', '/opt/local/bin/clang', 'clang') check_cc = find_program('check-cc.sh') result = run_command(check_cc, clang, check: false) @@ -18,9 +12,23 @@ if result.returncode() != 0 endif error('Could not find usable VDSO compiler. ' + install_msg) endif -vdso_compiler = [clang, '-target', 'i386-linux', '-fuse-ld=lld'] -vdso = custom_target('vdso', input: ['vdso.S', 'vdso.c', 'vdso.lds'], output: 'libvdso.so.elf', +guest_arch = get_option('guest_arch') +if guest_arch == 'aarch64' + vdso_target = 'aarch64-linux' + vdso_asm = 'vdso-aarch64.S' + vdso_c = 'vdso-aarch64.c' + vdso_lds = 'vdso-aarch64.lds' +else + vdso_target = 'i386-linux' + vdso_asm = 'vdso.S' + vdso_c = 'vdso.c' + vdso_lds = 'vdso.lds' +endif + +vdso_compiler = [clang, '-target', vdso_target, '-fuse-ld=lld'] + +vdso = custom_target('vdso', input: [vdso_asm, vdso_c, vdso_lds], output: 'libvdso.so.elf', command: vdso_compiler + ['-o', '@OUTPUT@', '@INPUT0@', '@INPUT1@', '-nostdlib', '-Wl,-T,@INPUT2@', '-Wl,--hash-style,sysv', '-shared', '-fPIC'] + get_option('vdso_c_args').split()) diff --git a/vdso/vdso-aarch64.S b/vdso/vdso-aarch64.S new file mode 100644 index 0000000000..5e34cc573d --- /dev/null +++ b/vdso/vdso-aarch64.S @@ -0,0 +1,15 @@ + .text + .global __kernel_rt_sigreturn + .type __kernel_rt_sigreturn,@function +__kernel_rt_sigreturn: + mov x8, #139 + svc #0 + nop + + .global __kernel_sigreturn + .type __kernel_sigreturn,@function +__kernel_sigreturn: + // aarch64 only has rt_sigreturn (NR 139); keep alias for ABI completeness + mov x8, #139 + svc #0 + nop diff --git a/vdso/vdso-aarch64.c b/vdso/vdso-aarch64.c new file mode 100644 index 0000000000..89b8d4bebb --- /dev/null +++ b/vdso/vdso-aarch64.c @@ -0,0 +1,30 @@ +#if !defined(__aarch64__) || !defined(__ELF__) +#error "VDSO must be built for aarch64 elf" +#endif + +typedef long time_t; +typedef int clockid_t; + +time_t __vdso_time(time_t *t) { + time_t result; + register long x0 __asm__("x0") = 201; + register time_t *x1 __asm__("x1") = t; + __asm__ volatile("svc #0" : "+r"(x0) : "r"(x1) : "memory", "x8"); + return x0; +} + +int __vdso_gettimeofday(void *timeval, void *timezone) { + register long x0 __asm__("x0") = 169; + register void *x1 __asm__("x1") = timeval; + register void *x2 __asm__("x2") = timezone; + __asm__ volatile("mov x8, %1\n\t svc #0" : "+r"(x0) : "r"(x0), "r"(x1), "r"(x2) : "x8", "memory"); + return (int) x0; +} + +int __vdso_clock_gettime(clockid_t clock, void *timespec) { + register long x0 __asm__("x0") = 113; + register clockid_t x1 __asm__("x1") = clock; + register void *x2 __asm__("x2") = timespec; + __asm__ volatile("mov x8, %1\n\t svc #0" : "+r"(x0) : "r"(x0), "r"(x1), "r"(x2) : "x8", "memory"); + return (int) x0; +} diff --git a/vdso/vdso-aarch64.lds b/vdso/vdso-aarch64.lds new file mode 100644 index 0000000000..718472c8ad --- /dev/null +++ b/vdso/vdso-aarch64.lds @@ -0,0 +1,59 @@ +ENTRY(__kernel_rt_sigreturn); + +VERSION { + LINUX_2.5 { + global: + __kernel_sigreturn; + __kernel_rt_sigreturn; + local: *; + }; + + LINUX_2.6 { + global: + __vdso_clock_gettime; + __vdso_gettimeofday; + __vdso_time; + }; +} + +SECTIONS { + . = SIZEOF_HEADERS; + + .hash : {*(.hash)} :text + .dynsym : {*(.dynsym)} + .dynstr : {*(.dynstr)} + .gnu.version : {*(.gnu.version)} + .gnu.version_d : {*(.gnu.version_d)} + .gnu.version_r : {*(.gnu.version_r)} + + .dynamic : {*(.dynamic)} :text :dynamic + + .rodata : { + *(.rodata*) + *(.data*) + *(.sdata*) + *(.got.plt) *(.got) + *(.gnu.linkonce.d.*) + *(.bss*) + *(.dynbss*) + *(.gnu.linkonce.b.*) + } :text + .note : {*(.note.*)} :text :note + + .eh_frame_hdr : {*(.eh_frame_hdr)} :text :eh_frame_hdr + .eh_frame : {KEEP (*(.eh_frame))} :text + + .text : {*(.text*)} :text + + /DISCARD/ : { + *(.debug*) + *(.comment) + } +} + +PHDRS { + text PT_LOAD FLAGS(5) FILEHDR PHDRS; + dynamic PT_DYNAMIC FLAGS(4); + note PT_NOTE FLAGS(4); + eh_frame_hdr PT_GNU_EH_FRAME; +}