2013-07-18 20:14:28 +02:00
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
#
|
|
|
|
# Copyright (c) 2008 - 2009, Apple Inc. All rights reserved.<BR>
|
2016-07-30 01:06:32 +02:00
|
|
|
# Copyright (c) 2011 - 2016, ARM Limited. All rights reserved.
|
2016-08-10 14:35:01 +02:00
|
|
|
# Copyright (c) 2016, Linaro Limited. All rights reserved.
|
2013-07-18 20:14:28 +02:00
|
|
|
#
|
2019-04-04 01:03:18 +02:00
|
|
|
# SPDX-License-Identifier: BSD-2-Clause-Patent
|
2013-07-18 20:14:28 +02:00
|
|
|
#
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
#include <AsmMacroIoLibV8.h>
|
|
|
|
|
2016-02-23 00:08:27 +01:00
|
|
|
.set DAIF_RD_FIQ_BIT, (1 << 6)
|
|
|
|
.set DAIF_RD_IRQ_BIT, (1 << 7)
|
2013-07-18 20:14:28 +02:00
|
|
|
|
ArmPkg/ArmMmuLib AARCH64: cache-invalidate initial page table entries
In the AARCH64 version of ArmMmuLib, we are currently relying on
set/way invalidation to ensure that the caches are in a consistent
state with respect to main memory once we turn the MMU on. Even if
set/way operations were the appropriate method to achieve this, doing
an invalidate-all first and then populating the page table entries
creates a window where page table entries could be loaded speculatively
into the caches before we modify them, and shadow the new values that
we write there.
So let's get rid of the blanket clean/invalidate operations, and
instead, update ArmUpdateTranslationTableEntry () to invalidate each
page table entry *after* it is written if the MMU is still disabled
at this point.
On ARMv8, it is guaranteed that memory accesses done by the page table
walker are cache coherent, and so we can ignore the case where the
MMU is on.
Since the MMU and D-cache are already off when we reach this point, we
can drop the MMU and D-cache disables as well. Maintenance of the I-cache
is unnecessary, since we are not modifying any code, and the installed
mapping is guaranteed to be 1:1. This means we can also leave it enabled
while the page table population code is running.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif@nuviainc.com>
2020-02-26 09:40:33 +01:00
|
|
|
.set SCTLR_ELx_M_BIT_POS, (0)
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmReadMidr)
|
2013-07-18 20:14:28 +02:00
|
|
|
mrs x0, midr_el1 // Read from Main ID Register (MIDR)
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmCacheInfo)
|
2013-07-18 20:14:28 +02:00
|
|
|
mrs x0, ctr_el0 // Read from Cache Type Regiter (CTR)
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmGetInterruptState)
|
2013-07-18 20:14:28 +02:00
|
|
|
mrs x0, daif
|
2016-02-23 00:08:27 +01:00
|
|
|
tst w0, #DAIF_RD_IRQ_BIT // Check if IRQ is enabled. Enabled if 0 (Z=1)
|
|
|
|
cset w0, eq // if Z=1 return 1, else 0
|
2013-07-18 20:14:28 +02:00
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmGetFiqState)
|
2013-07-18 20:14:28 +02:00
|
|
|
mrs x0, daif
|
2016-02-23 00:08:27 +01:00
|
|
|
tst w0, #DAIF_RD_FIQ_BIT // Check if FIQ is enabled. Enabled if 0 (Z=1)
|
|
|
|
cset w0, eq // if Z=1 return 1, else 0
|
2013-07-18 20:14:28 +02:00
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmWriteCpacr)
|
2013-07-18 20:14:28 +02:00
|
|
|
msr cpacr_el1, x0 // Coprocessor Access Control Reg (CPACR)
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmWriteAuxCr)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2(x1)
|
|
|
|
1:msr actlr_el1, x0 // Aux Control Reg (ACTLR) at EL1. Also available in EL2 and EL3
|
2014-03-01 12:01:00 +01:00
|
|
|
ret
|
2013-07-18 20:14:28 +02:00
|
|
|
2:msr actlr_el2, x0 // Aux Control Reg (ACTLR) at EL1. Also available in EL2 and EL3
|
2014-03-01 12:01:00 +01:00
|
|
|
ret
|
2013-07-18 20:14:28 +02:00
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmReadAuxCr)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2(x1)
|
|
|
|
1:mrs x0, actlr_el1 // Aux Control Reg (ACTLR) at EL1. Also available in EL2 and EL3
|
2014-03-01 12:01:00 +01:00
|
|
|
ret
|
2013-07-18 20:14:28 +02:00
|
|
|
2:mrs x0, actlr_el2 // Aux Control Reg (ACTLR) at EL1. Also available in EL2 and EL3
|
2014-03-01 12:01:00 +01:00
|
|
|
ret
|
2013-07-18 20:14:28 +02:00
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmSetTTBR0)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:msr ttbr0_el1, x0 // Translation Table Base Reg 0 (TTBR0)
|
|
|
|
b 4f
|
|
|
|
2:msr ttbr0_el2, x0 // Translation Table Base Reg 0 (TTBR0)
|
|
|
|
b 4f
|
|
|
|
3:msr ttbr0_el3, x0 // Translation Table Base Reg 0 (TTBR0)
|
|
|
|
4:isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmGetTTBR0BaseAddress)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2(x1)
|
|
|
|
1:mrs x0, ttbr0_el1
|
|
|
|
b 3f
|
|
|
|
2:mrs x0, ttbr0_el2
|
2016-08-10 14:35:01 +02:00
|
|
|
3:and x0, x0, 0xFFFFFFFFFFFF /* Look at bottom 48 bits */
|
2013-07-18 20:14:28 +02:00
|
|
|
isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmGetTCR)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:mrs x0, tcr_el1
|
|
|
|
b 4f
|
|
|
|
2:mrs x0, tcr_el2
|
|
|
|
b 4f
|
|
|
|
3:mrs x0, tcr_el3
|
|
|
|
4:isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmSetTCR)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:msr tcr_el1, x0
|
|
|
|
b 4f
|
|
|
|
2:msr tcr_el2, x0
|
|
|
|
b 4f
|
|
|
|
3:msr tcr_el3, x0
|
|
|
|
4:isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmGetMAIR)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:mrs x0, mair_el1
|
|
|
|
b 4f
|
|
|
|
2:mrs x0, mair_el2
|
|
|
|
b 4f
|
|
|
|
3:mrs x0, mair_el3
|
|
|
|
4:isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmSetMAIR)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:msr mair_el1, x0
|
|
|
|
b 4f
|
|
|
|
2:msr mair_el2, x0
|
|
|
|
b 4f
|
|
|
|
3:msr mair_el3, x0
|
|
|
|
4:isb
|
|
|
|
ret
|
|
|
|
|
|
|
|
|
|
|
|
//
|
|
|
|
//VOID
|
|
|
|
//ArmUpdateTranslationTableEntry (
|
|
|
|
// IN VOID *TranslationTableEntry // X0
|
|
|
|
// IN VOID *MVA // X1
|
|
|
|
// );
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmUpdateTranslationTableEntry)
|
2019-01-07 08:15:01 +01:00
|
|
|
dsb nshst
|
|
|
|
lsr x1, x1, #12
|
2020-03-19 17:37:05 +01:00
|
|
|
EL1_OR_EL2_OR_EL3(x2)
|
2013-07-18 20:14:28 +02:00
|
|
|
1: tlbi vaae1, x1 // TLB Invalidate VA , EL1
|
ArmPkg/ArmMmuLib AARCH64: cache-invalidate initial page table entries
In the AARCH64 version of ArmMmuLib, we are currently relying on
set/way invalidation to ensure that the caches are in a consistent
state with respect to main memory once we turn the MMU on. Even if
set/way operations were the appropriate method to achieve this, doing
an invalidate-all first and then populating the page table entries
creates a window where page table entries could be loaded speculatively
into the caches before we modify them, and shadow the new values that
we write there.
So let's get rid of the blanket clean/invalidate operations, and
instead, update ArmUpdateTranslationTableEntry () to invalidate each
page table entry *after* it is written if the MMU is still disabled
at this point.
On ARMv8, it is guaranteed that memory accesses done by the page table
walker are cache coherent, and so we can ignore the case where the
MMU is on.
Since the MMU and D-cache are already off when we reach this point, we
can drop the MMU and D-cache disables as well. Maintenance of the I-cache
is unnecessary, since we are not modifying any code, and the installed
mapping is guaranteed to be 1:1. This means we can also leave it enabled
while the page table population code is running.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif@nuviainc.com>
2020-02-26 09:40:33 +01:00
|
|
|
mrs x2, sctlr_el1
|
2013-07-18 20:14:28 +02:00
|
|
|
b 4f
|
|
|
|
2: tlbi vae2, x1 // TLB Invalidate VA , EL2
|
ArmPkg/ArmMmuLib AARCH64: cache-invalidate initial page table entries
In the AARCH64 version of ArmMmuLib, we are currently relying on
set/way invalidation to ensure that the caches are in a consistent
state with respect to main memory once we turn the MMU on. Even if
set/way operations were the appropriate method to achieve this, doing
an invalidate-all first and then populating the page table entries
creates a window where page table entries could be loaded speculatively
into the caches before we modify them, and shadow the new values that
we write there.
So let's get rid of the blanket clean/invalidate operations, and
instead, update ArmUpdateTranslationTableEntry () to invalidate each
page table entry *after* it is written if the MMU is still disabled
at this point.
On ARMv8, it is guaranteed that memory accesses done by the page table
walker are cache coherent, and so we can ignore the case where the
MMU is on.
Since the MMU and D-cache are already off when we reach this point, we
can drop the MMU and D-cache disables as well. Maintenance of the I-cache
is unnecessary, since we are not modifying any code, and the installed
mapping is guaranteed to be 1:1. This means we can also leave it enabled
while the page table population code is running.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif@nuviainc.com>
2020-02-26 09:40:33 +01:00
|
|
|
mrs x2, sctlr_el2
|
2013-07-18 20:14:28 +02:00
|
|
|
b 4f
|
|
|
|
3: tlbi vae3, x1 // TLB Invalidate VA , EL3
|
ArmPkg/ArmMmuLib AARCH64: cache-invalidate initial page table entries
In the AARCH64 version of ArmMmuLib, we are currently relying on
set/way invalidation to ensure that the caches are in a consistent
state with respect to main memory once we turn the MMU on. Even if
set/way operations were the appropriate method to achieve this, doing
an invalidate-all first and then populating the page table entries
creates a window where page table entries could be loaded speculatively
into the caches before we modify them, and shadow the new values that
we write there.
So let's get rid of the blanket clean/invalidate operations, and
instead, update ArmUpdateTranslationTableEntry () to invalidate each
page table entry *after* it is written if the MMU is still disabled
at this point.
On ARMv8, it is guaranteed that memory accesses done by the page table
walker are cache coherent, and so we can ignore the case where the
MMU is on.
Since the MMU and D-cache are already off when we reach this point, we
can drop the MMU and D-cache disables as well. Maintenance of the I-cache
is unnecessary, since we are not modifying any code, and the installed
mapping is guaranteed to be 1:1. This means we can also leave it enabled
while the page table population code is running.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif@nuviainc.com>
2020-02-26 09:40:33 +01:00
|
|
|
mrs x2, sctlr_el3
|
|
|
|
4: tbnz x2, SCTLR_ELx_M_BIT_POS, 5f
|
|
|
|
dc ivac, x0 // invalidate in Dcache if MMU is still off
|
|
|
|
5: dsb nsh
|
2013-07-18 20:14:28 +02:00
|
|
|
isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmInvalidateTlb)
|
2013-07-18 20:14:28 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x0)
|
2013-07-26 19:14:07 +02:00
|
|
|
1: tlbi vmalle1
|
2013-07-18 20:14:28 +02:00
|
|
|
b 4f
|
|
|
|
2: tlbi alle2
|
|
|
|
b 4f
|
|
|
|
3: tlbi alle3
|
|
|
|
4: dsb sy
|
|
|
|
isb
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmWriteCptr)
|
2013-07-18 20:14:28 +02:00
|
|
|
msr cptr_el3, x0 // EL3 Coprocessor Trap Reg (CPTR)
|
2013-08-06 12:59:19 +02:00
|
|
|
ret
|
2013-07-18 20:14:28 +02:00
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmWriteScr)
|
2013-07-18 20:14:28 +02:00
|
|
|
msr scr_el3, x0 // Secure configuration register EL3
|
2016-02-03 18:07:47 +01:00
|
|
|
isb
|
2013-07-18 20:14:28 +02:00
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmWriteMVBar)
|
2014-03-01 12:01:00 +01:00
|
|
|
msr vbar_el3, x0 // Exception Vector Base address for Monitor on EL3
|
2013-07-18 20:14:28 +02:00
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmCallWFE)
|
2013-07-18 20:14:28 +02:00
|
|
|
wfe
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmCallSEV)
|
2013-07-18 20:14:28 +02:00
|
|
|
sev
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmReadCpuActlr)
|
2014-03-26 20:31:01 +01:00
|
|
|
mrs x0, S3_1_c15_c2_0
|
|
|
|
ret
|
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmWriteCpuActlr)
|
2014-03-26 20:31:01 +01:00
|
|
|
msr S3_1_c15_c2_0, x0
|
|
|
|
dsb sy
|
|
|
|
isb
|
|
|
|
ret
|
2013-07-18 20:14:28 +02:00
|
|
|
|
2016-08-10 14:35:01 +02:00
|
|
|
ASM_FUNC(ArmReadSctlr)
|
2016-07-30 01:06:32 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:mrs x0, sctlr_el1
|
|
|
|
ret
|
|
|
|
2:mrs x0, sctlr_el2
|
|
|
|
ret
|
|
|
|
3:mrs x0, sctlr_el3
|
|
|
|
4:ret
|
|
|
|
|
2018-01-13 07:52:45 +01:00
|
|
|
ASM_FUNC(ArmWriteSctlr)
|
|
|
|
EL1_OR_EL2_OR_EL3(x1)
|
|
|
|
1:msr sctlr_el1, x0
|
|
|
|
ret
|
|
|
|
2:msr sctlr_el2, x0
|
|
|
|
ret
|
|
|
|
3:msr sctlr_el3, x0
|
|
|
|
4:ret
|
|
|
|
|
2018-11-23 13:14:27 +01:00
|
|
|
ASM_FUNC(ArmGetPhysicalAddressBits)
|
|
|
|
mrs x0, id_aa64mmfr0_el1
|
|
|
|
adr x1, .LPARanges
|
|
|
|
and x0, x0, #0xf
|
|
|
|
ldrb w0, [x1, x0]
|
|
|
|
ret
|
|
|
|
|
|
|
|
//
|
|
|
|
// Bits 0..3 of the AA64MFR0_EL1 system register encode the size of the
|
|
|
|
// physical address space support on this CPU:
|
|
|
|
// 0 == 32 bits, 1 == 36 bits, etc etc
|
|
|
|
// 7 and up are reserved
|
|
|
|
//
|
|
|
|
.LPARanges:
|
|
|
|
.byte 32, 36, 40, 42, 44, 48, 52, 0
|
|
|
|
.byte 0, 0, 0, 0, 0, 0, 0, 0
|
|
|
|
|
2013-07-18 20:14:28 +02:00
|
|
|
ASM_FUNCTION_REMOVE_IF_UNREFERENCED
|