/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2020-2023 Loongson Technology Corporation Limited */ #include #include #include #include #include #include #define HGPR_OFFSET(x) (PT_R0 + 8*x) #define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x) .macro kvm_save_host_gpr base .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 st.d $r\n, \base, HGPR_OFFSET(\n) .endr .endm .macro kvm_restore_host_gpr base .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 ld.d $r\n, \base, HGPR_OFFSET(\n) .endr .endm /* * Save and restore all GPRs except base register, * and default value of base register is a2. */ .macro kvm_save_guest_gprs base .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 st.d $r\n, \base, GGPR_OFFSET(\n) .endr .endm .macro kvm_restore_guest_gprs base .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 ld.d $r\n, \base, GGPR_OFFSET(\n) .endr .endm /* * Prepare switch to guest, save host regs and restore guest regs. * a2: kvm_vcpu_arch, don't touch it until 'ertn' * t0, t1: temp register */ .macro kvm_switch_to_guest /* Set host ECFG.VS=0, all exceptions share one exception entry */ csrrd t0, LOONGARCH_CSR_ECFG bstrins.w t0, zero, CSR_ECFG_VS_SHIFT_END, CSR_ECFG_VS_SHIFT csrwr t0, LOONGARCH_CSR_ECFG /* Load up the new EENTRY */ ld.d t0, a2, KVM_ARCH_GEENTRY csrwr t0, LOONGARCH_CSR_EENTRY /* Set Guest ERA */ ld.d t0, a2, KVM_ARCH_GPC csrwr t0, LOONGARCH_CSR_ERA /* Save host PGDL */ csrrd t0, LOONGARCH_CSR_PGDL st.d t0, a2, KVM_ARCH_HPGD /* Switch to kvm */ ld.d t1, a2, KVM_VCPU_KVM - KVM_VCPU_ARCH /* Load guest PGDL */ li.w t0, KVM_GPGD ldx.d t0, t1, t0 csrwr t0, LOONGARCH_CSR_PGDL /* Mix GID and RID */ csrrd t1, LOONGARCH_CSR_GSTAT bstrpick.w t1, t1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT csrrd t0, LOONGARCH_CSR_GTLBC bstrins.w t0, t1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT csrwr t0, LOONGARCH_CSR_GTLBC /* * Enable intr in root mode with future ertn so that host interrupt * can be responsed during VM runs * Guest CRMD comes from separate GCSR_CRMD register */ ori t0, zero, CSR_PRMD_PIE csrxchg t0, t0, LOONGARCH_CSR_PRMD /* Set PVM bit to setup ertn to guest context */ ori t0, zero, CSR_GSTAT_PVM csrxchg t0, t0, LOONGARCH_CSR_GSTAT /* Load Guest GPRs */ kvm_restore_guest_gprs a2 /* Load KVM_ARCH register */ ld.d a2, a2, (KVM_ARCH_GGPR + 8 * REG_A2) ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0 */ .endm /* * Exception entry for general exception from guest mode * - IRQ is disabled * - kernel privilege in root mode * - page mode keep unchanged from previous PRMD in root mode * - Fixme: tlb exception cannot happen since registers relative with TLB * - is still in guest mode, such as pgd table/vmid registers etc, * - will fix with hw page walk enabled in future * load kvm_vcpu from reserved CSR KVM_VCPU_KS, and save a2 to KVM_TEMP_KS */ .text .cfi_sections .debug_frame SYM_CODE_START(kvm_exc_entry) UNWIND_HINT_UNDEFINED csrwr a2, KVM_TEMP_KS csrrd a2, KVM_VCPU_KS addi.d a2, a2, KVM_VCPU_ARCH /* After save GPRs, free to use any GPR */ kvm_save_guest_gprs a2 /* Save guest A2 */ csrrd t0, KVM_TEMP_KS st.d t0, a2, (KVM_ARCH_GGPR + 8 * REG_A2) /* A2 is kvm_vcpu_arch, A1 is free to use */ csrrd s1, KVM_VCPU_KS ld.d s0, s1, KVM_VCPU_RUN csrrd t0, LOONGARCH_CSR_ESTAT st.d t0, a2, KVM_ARCH_HESTAT csrrd t0, LOONGARCH_CSR_ERA st.d t0, a2, KVM_ARCH_GPC csrrd t0, LOONGARCH_CSR_BADV st.d t0, a2, KVM_ARCH_HBADV csrrd t0, LOONGARCH_CSR_BADI st.d t0, a2, KVM_ARCH_HBADI /* Restore host ECFG.VS */ csrrd t0, LOONGARCH_CSR_ECFG ld.d t1, a2, KVM_ARCH_HECFG or t0, t0, t1 csrwr t0, LOONGARCH_CSR_ECFG /* Restore host EENTRY */ ld.d t0, a2, KVM_ARCH_HEENTRY csrwr t0, LOONGARCH_CSR_EENTRY /* Restore host pgd table */ ld.d t0, a2, KVM_ARCH_HPGD csrwr t0, LOONGARCH_CSR_PGDL /* * Disable PGM bit to enter root mode by default with next ertn */ ori t0, zero, CSR_GSTAT_PVM csrxchg zero, t0, LOONGARCH_CSR_GSTAT /* * Clear GTLBC.TGID field * 0: for root tlb update in future tlb instr * others: for guest tlb update like gpa to hpa in future tlb instr */ csrrd t0, LOONGARCH_CSR_GTLBC bstrins.w t0, zero, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT csrwr t0, LOONGARCH_CSR_GTLBC ld.d tp, a2, KVM_ARCH_HTP ld.d sp, a2, KVM_ARCH_HSP /* restore per cpu register */ ld.d u0, a2, KVM_ARCH_HPERCPU addi.d sp, sp, -PT_SIZE /* Prepare handle exception */ or a0, s0, zero or a1, s1, zero ld.d t8, a2, KVM_ARCH_HANDLE_EXIT jirl ra, t8, 0 or a2, s1, zero addi.d a2, a2, KVM_VCPU_ARCH /* Resume host when ret <= 0 */ blez a0, ret_to_host /* * Return to guest * Save per cpu register again, maybe switched to another cpu */ st.d u0, a2, KVM_ARCH_HPERCPU /* Save kvm_vcpu to kscratch */ csrwr s1, KVM_VCPU_KS kvm_switch_to_guest ret_to_host: ld.d a2, a2, KVM_ARCH_HSP addi.d a2, a2, -PT_SIZE kvm_restore_host_gpr a2 jr ra SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL) SYM_CODE_END(kvm_exc_entry) /* * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu) * * @register_param: * a0: kvm_run* run * a1: kvm_vcpu* vcpu */ SYM_FUNC_START(kvm_enter_guest) /* Allocate space in stack bottom */ addi.d a2, sp, -PT_SIZE /* Save host GPRs */ kvm_save_host_gpr a2 addi.d a2, a1, KVM_VCPU_ARCH st.d sp, a2, KVM_ARCH_HSP st.d tp, a2, KVM_ARCH_HTP /* Save per cpu register */ st.d u0, a2, KVM_ARCH_HPERCPU /* Save kvm_vcpu to kscratch */ csrwr a1, KVM_VCPU_KS kvm_switch_to_guest SYM_INNER_LABEL(kvm_enter_guest_end, SYM_L_LOCAL) SYM_FUNC_END(kvm_enter_guest) SYM_FUNC_START(kvm_save_fpu) fpu_save_csr a0 t1 fpu_save_double a0 t1 fpu_save_cc a0 t1 t2 jr ra SYM_FUNC_END(kvm_save_fpu) SYM_FUNC_START(kvm_restore_fpu) fpu_restore_double a0 t1 fpu_restore_csr a0 t1 t2 fpu_restore_cc a0 t1 t2 jr ra SYM_FUNC_END(kvm_restore_fpu) #ifdef CONFIG_CPU_HAS_LSX SYM_FUNC_START(kvm_save_lsx) fpu_save_csr a0 t1 fpu_save_cc a0 t1 t2 lsx_save_data a0 t1 jr ra SYM_FUNC_END(kvm_save_lsx) SYM_FUNC_START(kvm_restore_lsx) lsx_restore_data a0 t1 fpu_restore_cc a0 t1 t2 fpu_restore_csr a0 t1 t2 jr ra SYM_FUNC_END(kvm_restore_lsx) #endif #ifdef CONFIG_CPU_HAS_LASX SYM_FUNC_START(kvm_save_lasx) fpu_save_csr a0 t1 fpu_save_cc a0 t1 t2 lasx_save_data a0 t1 jr ra SYM_FUNC_END(kvm_save_lasx) SYM_FUNC_START(kvm_restore_lasx) lasx_restore_data a0 t1 fpu_restore_cc a0 t1 t2 fpu_restore_csr a0 t1 t2 jr ra SYM_FUNC_END(kvm_restore_lasx) #endif .section ".rodata" SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry) SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) #ifdef CONFIG_CPU_HAS_LBT STACK_FRAME_NON_STANDARD kvm_restore_fpu STACK_FRAME_NON_STANDARD kvm_restore_lsx STACK_FRAME_NON_STANDARD kvm_restore_lasx #endif