/*
 * Copyright (c) 2015,2016 ETH Zurich.
 * All rights reserved.
 *
 * This file is distributed under the terms in the attached LICENSE file.
 * If you do not find this file, copies can be found by writing to:
 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
 */

#include <asmoffsets.h>
#include <offsets.h>
#include <exceptions.h>

.global do_resume
.global vectors

.macro invalid_exception vector
    /* Just stick the trap frame on the kernel stack - we're about to panic
     * anyway.  */

    add sp, sp, #-(34 * 8)

    /* Spill the GPRs */
    stp  x0,  x1, [sp]
    stp  x2,  x3, [sp, #( 2 * 8)]
    stp  x4,  x5, [sp, #( 4 * 8)]
    stp  x6,  x7, [sp, #( 6 * 8)]
    stp  x8,  x9, [sp, #( 8 * 8)]
    stp x10, x11, [sp, #(10 * 8)]
    stp x12, x13, [sp, #(12 * 8)]
    stp x14, x15, [sp, #(14 * 8)]
    stp x16, x17, [sp, #(16 * 8)]
    stp x18, x19, [sp, #(18 * 8)]
    stp x20, x21, [sp, #(20 * 8)]
    stp x22, x23, [sp, #(22 * 8)]
    stp x24, x25, [sp, #(24 * 8)]
    stp x26, x27, [sp, #(26 * 8)]
    stp x28, x29, [sp, #(28 * 8)]

    /* Stack pointer */
    mrs x0, sp_el0
    stp x30, x0, [sp, #(30 * 8)]

    mrs x0, elr_el1
    mrs x1, spsr_el1
    stp x0, x1, [sp, #(32 * 8)]

    /* Exception Syndrome Register */
    mrs x2, esr_el1

    /* Exception vector */
    mov x3, \vector

    /* Base of the register save area. */
    mov x4, sp

    /* Arguments: x0 = EPC, x1 = SPSR, x2 = ESR, x3 = vector, x4 = save area. */
    b fatal_kernel_fault
.endm

/**********************************/
/*** Start of exception vectors ***/
/**********************************/

/* The AArch64 exception vectors occupy 2kiB */
.align 11
vectors:

/* Offset 0x000 */
/* Exceptions from the current EL, on the EL0 stack.  We never do this. */
/* Each table entry occupies 128B, which lets us put up to 32 instructions
 * here before we branch. */
.align 7 /* 0x000 */
el1_sp_el0_sync:
    invalid_exception AARCH64_EVECTOR_EL0_SYNC
.align 7 /* 0x080 */
el1_sp_el0_irq:
    invalid_exception AARCH64_EVECTOR_EL0_IRQ 
.align 7 /* 0x100 */
el1_sp_el0_fiq:
    invalid_exception AARCH64_EVECTOR_EL0_FIQ
.align 7 /* 0x180 */
el1_sp_el0_serror:
    invalid_exception AARCH64_EVECTOR_EL0_SERROR

/* Offset 0x200 */
/* Exceptions from the kernel itself, at EL1. */
.align 7 /* 0x200 */
el1_sync:
    invalid_exception AARCH64_EVECTOR_EL1_SYNC
.align 7 /* 0x280 */
el1_irq:
    /* This happens only in case the kernel runs out of work and
     * calls wait_for_interrupt. To make sure we come from this function
     * we store a magic value in a register */
    mov w1, #WAIT_FOR_INTERRUPT_MAGIC
    cmp w0,w1
    b.ne el1_irq_failure
    b nosave_handle_irq
el1_irq_failure:
    invalid_exception AARCH64_EVECTOR_EL1_IRQ
.align 7 /* 0x300 */
el1_fiq:
    invalid_exception AARCH64_EVECTOR_EL1_FIQ
.align 7 /* 0x380 */
el1_serror:
    invalid_exception AARCH64_EVECTOR_EL1_SERROR

/* Offset 0x400 */
/* Exceptions from user level, EL0, executing AArch64.  For any of these four
 * exceptions, the stack pointer is SP_EL1, which is left at the top of
 * 'kernel_stack'. */
 .align 7 /* 0x400 */
/*
 * Synchronous exceptions from a lower execution level using AArch64: SVC
 * (syscall), data abort, prefetch abort and undefined instruction.
 *
 * Assumption:
 * when coming from a syscall (SVC) the arguments are in registers x0-x11
 */
el0_aarch64_sync:
    /* Reenable breakpoints and aborts.  Interrupts remain disabled. */
    msr daifset, #3 /* IRQ and FIQ masked, Debug and Abort enabled. */

    /* Spill a few working registers.
     * Registers x0-x11 contain the syscall arguments. We use x12-, as these won't
     * need to be restored if we're doing a syscall; they're caller-saved.  We
     * preserve x0-x6 in registers unless we branch to the abort path, so that they're
     * immediately available to the syscall handler, sys_syscall.
     */
    stp x14, x15, [sp, #-(2 * 8)]!
    stp x12, x13, [sp, #-(2 * 8)]!

    /* The EL1 thread ID register holds the address of the currently-running
     * dispatcher's shared control block. */
    mrs x13, tpidr_el1

    /* x13 = dispatcher_shared_aarch64 */

    /* Exception PC */
    mrs x12, elr_el1

    /* x12 = EPC, x13 = dispatcher_shared_aarch64 */

    /* Check whether the current dispatcher is disabled.  See el0_aarch64_irq
     * for a description of this test. */
    ldp x14, x15, [x13, #OFFSETOF_DISP_CRIT_PC_LOW]
    cmp x14, x12
    ccmp x15, x12, #0, ls
    ldr w14, [x13, #OFFSETOF_DISP_DISABLED]
    ccmp w14, wzr, #0, ls
    /* NE <-> (low <= PC && PC < high) || disabled != 0 */

    /* Figure out what sort of exception we've got.  All paths need this. */
    mrs x14, esr_el1  /* Exception Syndrome Register */
    lsr x14, x14, #26 /* Exception Class field is bits [31:26] */

    /* x12 = EPC, x14 = EC, x13 = dispatcher_shared_aarch64 */

    /* Faults while disabled should be rare, if the critical section is short,
     * and will always be within the dispatcher code.  Therefore we branch out
     * to the slowpath here. */
    b.ne el0_sync_disabled

    /* 13 instructions to here. */

    /* All exceptions use the 'enabled' area if the dispatcher is enabled. */
    add x13, x13, #OFFSETOF_DISP_ENABLED_AREA

    /* x12 = EPC, x13 = base of save area, x14 = EC */

    /* now we branch off as we are running out of space */
    b save_syscall_context

    /* 15 instructions to here. */

.align 7 /* 0x480 */
/* An interrupt at user level */
el0_aarch64_irq:
    /* Reenable breakpoints and aborts.  Interrupts remain disabled. */
    msr daifset, #3 /* IRQ and FIQ masked, Debug and Abort enabled. */

    /* Free scratch registers. */
    stp x2, x3, [sp, #-(2 * 8)]!
    stp x0, x1, [sp, #-(2 * 8)]!

    /* Find the dispatcher. */
    mrs x3, tpidr_el1

    /* Get the exception address (EPC) */
    mrs x1, elr_el1

    /* x0 = crit_pc_low, x1 = EPC,
       x2 = crit_pc_high, x3 = dispatcher_shared_aarch64 */

    /* Dispatcher disabled? */
    ldp x0, x2, [x3, #OFFSETOF_DISP_CRIT_PC_LOW]

    /* Condition LS holds iff low <= PC. */
    cmp x0, x1
    /* Short-circuit computation of P /\ Q: If the success condition for P
     * (low <= PC) holds, here LS (Less or Same), then test Q (PC < high),
     * giving either HI (true) or LS (false).  Otherwise, set the condition
     * flags to a failing state for Q (LS), as the conjunction cannot hold if
     * the P does not. */
    ccmp x2, x1, #0, ls
    /* If HI (C=1, Z=0) holds, then the test executed and passed, which means
     * that P held previously (low <= PC), and Q holds now (PC < high).
     * Otherwise LS holds, and thus either HI held before this test, and thus
     * PC < low, or the test executed and failed, in which case high <= PC.
     * Thus condition HI holds iff low <= PC /\ PC < high. */
    ldr w0, [x3, #OFFSETOF_DISP_DISABLED]
    /* We do the same to calculate (P /\ Q) \/ R: If P /\ Q doesn't hold, we
     * need to test R (is the disabled flag 0), giving either EQ or NE.  If
     * P /\ Q holds, we skip the test, as it doesn't affect the result, and
     * instead set the condition code directly NE, i.e. disabled=1. */
    ccmp w0, wzr, #0, ls
    /* NE <-> (low <= PC && PC < high) || disabled != 0 */

    /* x1 = EPC, x3 = dispatcher_shared_aarch64 */

    /* Choose the right save area, using the condition flags. */
    add x0, x3, #OFFSETOF_DISP_ENABLED_AREA
    add x2, x3, #OFFSETOF_DISP_DISABLED_AREA
    csel x0, x2, x0, ne

    /* x0 = save area, x1 = EPC */

    /* 13 instructions. */

    /* Save the register context, starting from x4.  x0-x3, the scratch
     * registers, can be copied to the trap frame from within the C handler,
     * as can the user stack pointer and SPSR, which are sitting in their own
     * system registers. */
    stp  x4,  x5, [x0, #( 4 * 8)]
    stp  x6,  x7, [x0, #( 6 * 8)]
    stp  x8,  x9, [x0, #( 8 * 8)]
    stp x10, x11, [x0, #(10 * 8)]
    stp x12, x13, [x0, #(12 * 8)]
    stp x14, x15, [x0, #(14 * 8)]
    stp x16, x17, [x0, #(16 * 8)]
    stp x18, x19, [x0, #(18 * 8)]
    stp x20, x21, [x0, #(20 * 8)]
    stp x22, x23, [x0, #(22 * 8)]
    stp x24, x25, [x0, #(24 * 8)]
    stp x26, x27, [x0, #(26 * 8)]
    stp x28, x29, [x0, #(28 * 8)]
    str x30,      [x0, #(30 * 8)]

    /* 27 instructions. */

    /* Load the saved scratch registers, and pass them as arguments to the
     * handler.  We can't save them ourselves as we've run out of
     * instructions.  We need to do at least this, to clear our stack frame.
     * */
    ldp x2, x3, [sp], #16 /* x0, x1 */
    ldp x4, x5, [sp], #16 /* x2, x3 */

    /* x0 = save area, x1 = EPC,
     * x2 = user x0, x3 = user x1,
     * x4 = user x2, x5 = user x3 */
    b save_handle_irq

    /* 30 instructions. */

.align 7 /* 0x500 */
/* We don't implement fast IRQs */
el0_aarch64_fiq:
    invalid_exception AARCH64_EVECTOR_EL0_FIQ

.align 7 /* 0x580 */
/* A delayed abort.  We don't handle this. */
el0_aarch64_serror:
    invalid_exception AARCH64_EVECTOR_EL0_SERROR

/* Offset 0x600 */
/* Exceptions from user level, EL0, executing AArch32.  This is currently
 * unimplemented. */
.align 7 /* 0x600 */
el0_aarch32_sync:
    invalid_exception AARCH32_EVECTOR_EL0_SYNC
.align 7 /* 0x680 */
el0_aarch32_irq:
    invalid_exception AARCH32_EVECTOR_EL0_IRQ
.align 7 /* 0x700 */
el0_aarch32_fiq:
    invalid_exception AARCH32_EVECTOR_EL0_FIQ
.align 7 /* 0x780 */
el0_aarch32_serror:
    invalid_exception AARCH32_EVECTOR_EL0_SERROR

.align 11
/********************************/
/*** End of exception vectors ***/
/********************************/

/* The tail of the user syscall handler doesn't fit in the table. */
save_syscall_context:
    /* x12 = EPC, x13 = base of save area, x14 = EC */

    /*
     * We need to save callee save registers r19-30 no matter what, so get on with it.
     */

    /* Callee-saved registers */
    stp x19, x20, [x13, #(19 * 8)]
    stp x21, x22, [x13, #(21 * 8)]
    stp x23, x24, [x13, #(23 * 8)]
    stp x25, x26, [x13, #(25 * 8)]
    stp x27, x28, [x13, #(27 * 8)]
    stp x29, x30, [x13, #(29 * 8)] /* FP & LR */

    stp q0, q1, [x13, #(34 * 8)]
    stp q2, q3, [x13, #(38 * 8)]
    stp q4, q5, [x13, #(42 * 8)]
    stp q6, q7, [x13, #(46 * 8)]
    stp q8, q9, [x13, #(50 * 8)]
    stp q10, q11, [x13, #(54 * 8)]
    stp q12, q13, [x13, #(58 * 8)]
    stp q14, q15, [x13, #(62 * 8)]
    stp q16, q17, [x13, #(66 * 8)]
    stp q18, q19, [x13, #(70 * 8)]
    stp q20, q21, [x13, #(74 * 8)]
    stp q22, q23, [x13, #(78 * 8)]
    stp q24, q25, [x13, #(82 * 8)]
    stp q26, q27, [x13, #(86 * 8)]
    stp q28, q29, [x13, #(90 * 8)]
    stp q30, q31, [x13, #(94 * 8)]

    /* High registers are now available. */

    /* User SP and PC */
    mrs x20, sp_el0
    stp x20, x12, [x13, #(31 * 8)]

    /* SPSR */
    mrs x19, spsr_el1
    str x19, [x13, #(33 * 8)]

    /* Is this a syscall? */
    cmp x14, #0x15 /* SVC or HVC from AArch64 EL0 */
    b.ne el0_abort_enabled

    /*
     * we need to save r7-r11 as those are the syscall arguments which are passed
     * on the stack on a function call, so we need to move them to the trap frame
     */
    stp x7, x8,     [x13, #(7 * 8)]
    stp x9, x10,    [x13, #(9 * 8)]
    str x11,        [x13, #(11 * 8)]

    /* If we're here, this is a syscall and we don't need to restore the
     * scratch registers. Just throw the stack frame away. */
    add sp, sp, #(4 * 8)

    /* Pass the address of the trap frame as argument 8. */
    mov x7, x13

    /* Jump to the common (C) syscall handler. */
    b sys_syscall


/* The tail of the user abort handler doesn't fit in the table. */
el0_abort_enabled:
    /* x12 = EPC, x13 = base of save area, x14 = EC */

    /*
     * we need to save the caller-saved registers. As it is an
     * abort, those weren't saved by prior to the call.
     */


    /* We saved x12-x15 in our stack frame. We load them from
     * memory so we can store them we use x20-23 as they where
     * saved in the common path */
    ldp x20,  x21,  [sp], #16 /* x12, x13 */
    ldp x22,  x23,  [sp], #16 /* x14, x15 */

    /* now we can store the caller-saved registers in the trap frame */
    stp x0,   x1,  [x13]
    stp x2,   x3,  [x13, #(  2 * 8)]
    stp x4,   x5,  [x13, #(  4 * 8)]
    stp x6,   x7,  [x13, #(  6 * 8)]
    stp x8,   x9,  [x13, #(  8 * 8)]
    stp x10, x11,  [x13, #( 10 * 8)]
    stp x20, x21,  [x13, #(12 * 8)]
    stp x22, x23,  [x13, #(14 * 8)]
    stp x16, x17,  [x13, #(16 * 8)]
    str x18,       [x13, #(18 * 8)]

    /* x19-x30, SP, SPSR and ELR were saved in the common path. */

    /* x12 = EPC, x13 = base of save area, x14 = EC */

    /* All registers are now available. */

    /* Pass the EPC and the save area address to the handler. */
    mrs x0, far_el1
    mov x1, x14
    mov x2, x13

    /* Now we can jump to the handler. */
    b handle_user_fault

/* A synchronous exception while the dispatcher is disabled.  This is the
 * slowpath.  In fact, libbarrelfish currently (2015) just refuses to continue
 * if this ever happens. */
el0_sync_disabled:
    /* x12 = EPC, x14 = EC, x13 = dispatcher_shared_aarch64 */

    /* Filter out aborts. */
    cmp x14, #0x15 /* SVC or HVC from AArch64 EL0 */
    b.ne el0_abort_disabled

    /* Use the 'disabled' area. */
    add x13, x13, #OFFSETOF_DISP_DISABLED_AREA

    /* Jump back into the syscall path. */
    b save_syscall_context

/* This is the *really* unexpected path: a page fault in the dispatcher
 * critical section.  It's (relatively) slow, and libbarrelfish doesn't
 * actually handle it at present (2015). */
el0_abort_disabled:

    /* x12 = EPC, x13 = base of save area, x14 = EC */

    /* x11 = EC,
       x13 = dispatcher_shared_aarch64 */

    /* Use the 'trap' area. */
    add x13, x13, #OFFSETOF_DISP_TRAP_AREA

    /* Save the reduced context. */
    /* Callee-saved registers */
    stp x19, x20, [x13, #(19 * 8)]
    stp x21, x22, [x13, #(21 * 8)]
    stp x23, x24, [x13, #(23 * 8)]
    stp x25, x26, [x13, #(25 * 8)]
    stp x27, x28, [x13, #(27 * 8)]
    stp x29, x30, [x13, #(29 * 8)] /* FP & LR */

    /* SPSR */
    mrs x19, spsr_el1
    str x19, [x13, #(31 * 8)]

    /* User PC and SP */
    mrs x20, sp_el0
    stp x20, x12, [x13, #(31 * 8)]

    /* x12 = EPC, x13 = base of save area, x14 = EC */

    /* Now reuse the 'enabled' abort handler. */
    b el0_abort_enabled

/* Restore the dispatcher's execution context.  x0 holds the base of the
 * appropriate save area. */
do_resume:
    /* Skip to the end... */
    add x0, x0, #((34 + 30 * 2) * 8)

    /* Reset the kernel stack pointer. */
    ldr x1, =kernel_stack   // load the address of the kernel stack
    ldr x2,  [x1]           // read the entry of the kernel stack
    mov sp, x2

    /* Flush the TLB - XXX kill this with fire. */
    dsb sy
    tlbi vmalle1
    dsb sy
    isb

    /* Restore the NEON registers. */
    ldp q30, q31, [x0], #-(2 * 16)
    ldp q28, q29, [x0], #-(2 * 16)
    ldp q26, q27, [x0], #-(2 * 16)
    ldp q24, q25, [x0], #-(2 * 16)
    ldp q22, q23, [x0], #-(2 * 16)
    ldp q20, q21, [x0], #-(2 * 16)
    ldp q18, q19, [x0], #-(2 * 16)
    ldp q16, q17, [x0], #-(2 * 16)
    ldp q14, q15, [x0], #-(2 * 16)
    ldp q12, q13, [x0], #-(2 * 16)
    ldp q10, q11, [x0], #-(2 * 16)
    ldp q8,   q9, [x0], #-(2 * 16)
    ldp q6,   q7, [x0], #-(2 * 16)
    ldp q4,   q5, [x0], #-(2 * 16)
    ldp q2,   q3, [x0], #-(2 * 16)
    ldp q0,   q1, [x0], #-(2 *  8)

    /* Restore SPSR, PC (ELR) and SP, which need temporary registers, before
     * we restore those. */
    ldp x2, x3, [x0], #-8 /* pc, spsr */
    ldr x1, [x0], #-16 /* stack */
    msr spsr_el1, x3
    msr elr_el1,  x2
    msr sp_el0,   x1

    /* Restore the general-purpose registers. */
    ldp x29, x30, [x0], #-(2 * 8)
    ldp x27, x28, [x0], #-(2 * 8)
    ldp x25, x26, [x0], #-(2 * 8)
    ldp x23, x24, [x0], #-(2 * 8)
    ldp x21, x22, [x0], #-(2 * 8)
    ldp x19, x20, [x0], #-(2 * 8)
    ldp x17, x18, [x0], #-(2 * 8)
    ldp x15, x16, [x0], #-(2 * 8)
    ldp x13, x14, [x0], #-(2 * 8)
    ldp x11, x12, [x0], #-(2 * 8)
    ldp x9,  x10, [x0], #-(2 * 8)
    ldp x7,  x8,  [x0], #-(2 * 8)
    ldp x5,  x6,  [x0], #-(2 * 8)
    ldp x3,  x4,  [x0], #-(2 * 8)
    ldp x1,  x2,  [x0], #-(1 * 8)
    ldr x0,       [x0]

    /* Return from exception.  This clears the load exclusive monitor. */
    eret
