2016-06-15 18:49:09 +02:00
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
#
|
|
|
|
# Copyright (c) 2016, Linaro Limited. All rights reserved.
|
|
|
|
#
|
2019-04-04 01:03:18 +02:00
|
|
|
# SPDX-License-Identifier: BSD-2-Clause-Patent
|
2016-06-15 18:49:09 +02:00
|
|
|
#
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
#include <AsmMacroIoLibV8.h>
|
|
|
|
|
|
|
|
.set CTRL_M_BIT, (1 << 0)
|
|
|
|
|
|
|
|
.macro __replace_entry, el
|
|
|
|
|
2022-09-24 18:26:19 +02:00
|
|
|
// check whether we should disable the MMU
|
|
|
|
cbz x3, .L1_\@
|
|
|
|
|
|
|
|
// clean and invalidate first so that we don't clobber
|
|
|
|
// adjacent entries that are dirty in the caches
|
|
|
|
dc civac, x0
|
|
|
|
dsb nsh
|
|
|
|
|
2016-06-15 18:49:09 +02:00
|
|
|
// disable the MMU
|
|
|
|
mrs x8, sctlr_el\el
|
|
|
|
bic x9, x8, #CTRL_M_BIT
|
|
|
|
msr sctlr_el\el, x9
|
|
|
|
isb
|
|
|
|
|
|
|
|
// write updated entry
|
|
|
|
str x1, [x0]
|
|
|
|
|
|
|
|
// invalidate again to get rid of stale clean cachelines that may
|
|
|
|
// have been filled speculatively since the last invalidate
|
|
|
|
dmb sy
|
|
|
|
dc ivac, x0
|
|
|
|
|
2019-01-07 08:15:01 +01:00
|
|
|
// flush translations for the target address from the TLBs
|
|
|
|
lsr x2, x2, #12
|
2016-06-15 18:49:09 +02:00
|
|
|
.if \el == 1
|
2019-01-07 08:15:01 +01:00
|
|
|
tlbi vaae1, x2
|
2016-06-15 18:49:09 +02:00
|
|
|
.else
|
2019-01-07 08:15:01 +01:00
|
|
|
tlbi vae\el, x2
|
2016-06-15 18:49:09 +02:00
|
|
|
.endif
|
2019-01-07 08:15:01 +01:00
|
|
|
dsb nsh
|
2016-06-15 18:49:09 +02:00
|
|
|
|
|
|
|
// re-enable the MMU
|
|
|
|
msr sctlr_el\el, x8
|
|
|
|
isb
|
2022-09-24 18:26:19 +02:00
|
|
|
b .L2_\@
|
|
|
|
|
|
|
|
.L1_\@:
|
|
|
|
// write invalid entry
|
|
|
|
str xzr, [x0]
|
|
|
|
dsb nshst
|
|
|
|
|
|
|
|
// flush translations for the target address from the TLBs
|
|
|
|
lsr x2, x2, #12
|
|
|
|
.if \el == 1
|
|
|
|
tlbi vaae1, x2
|
|
|
|
.else
|
|
|
|
tlbi vae\el, x2
|
|
|
|
.endif
|
|
|
|
dsb nsh
|
|
|
|
|
|
|
|
// write updated entry
|
|
|
|
str x1, [x0]
|
|
|
|
dsb nshst
|
ArmPkg/ArmMmuLib AARCH64: Add missing ISB after page table update
The helper that updates live page table entries writes a zero entry,
invalidates the covered address range from the TLBs, and finally writes
the actual entry. This ensures that no TLB conflicts can occur.
Writing the final entry needs to complete before any translations can be
performed, as otherwise, the zero entry, which describes an invalid
translation, may be observed by the page table walker, resulting in a
translation fault. For this reason, the final write is followed by a DSB
barrier instruction.
However, this barrier will not stall the pipeline, and instruction
fetches may still hit this invalid translation, as has been observed and
reported by Oliver. To ensure that the new translation is fully active
before returning from this helper, we have to insert an ISB barrier as
well.
Reported-by: Oliver Steffen <osteffen@redhat.com>
Tested-by: Oliver Steffen <osteffen@redhat.com>
Reviewed-by: Leif Lindholm <quic_llindhol@quicinc.com>
Acked-by: Michael D Kinney <michael.d.kinney@intel.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
2023-05-19 23:44:12 +02:00
|
|
|
isb
|
2022-09-24 18:26:19 +02:00
|
|
|
|
|
|
|
.L2_\@:
|
2016-06-15 18:49:09 +02:00
|
|
|
.endm
|
|
|
|
|
|
|
|
//VOID
|
|
|
|
//ArmReplaceLiveTranslationEntry (
|
|
|
|
// IN UINT64 *Entry,
|
2019-01-07 08:15:01 +01:00
|
|
|
// IN UINT64 Value,
|
|
|
|
// IN UINT64 Address
|
2016-06-15 18:49:09 +02:00
|
|
|
// )
|
2023-04-20 17:24:15 +02:00
|
|
|
//
|
|
|
|
// Align this routine to a log2 upper bound of its size, so that it is
|
|
|
|
// guaranteed not to cross a page or block boundary.
|
|
|
|
ASM_FUNC_ALIGN(ArmReplaceLiveTranslationEntry, 0x200)
|
2016-06-15 18:49:09 +02:00
|
|
|
|
|
|
|
// disable interrupts
|
2019-01-07 08:15:01 +01:00
|
|
|
mrs x4, daif
|
2016-06-15 18:49:09 +02:00
|
|
|
msr daifset, #0xf
|
|
|
|
isb
|
|
|
|
|
2022-09-24 18:26:19 +02:00
|
|
|
EL1_OR_EL2_OR_EL3(x5)
|
2016-06-15 18:49:09 +02:00
|
|
|
1:__replace_entry 1
|
|
|
|
b 4f
|
|
|
|
2:__replace_entry 2
|
|
|
|
b 4f
|
|
|
|
3:__replace_entry 3
|
|
|
|
|
2019-01-07 08:15:01 +01:00
|
|
|
4:msr daif, x4
|
2016-06-15 18:49:09 +02:00
|
|
|
ret
|
|
|
|
|
2016-08-10 13:43:17 +02:00
|
|
|
ASM_GLOBAL ASM_PFX(ArmReplaceLiveTranslationEntrySize)
|
|
|
|
|
2016-06-15 18:49:09 +02:00
|
|
|
ASM_PFX(ArmReplaceLiveTranslationEntrySize):
|
|
|
|
.long . - ArmReplaceLiveTranslationEntry
|
2022-09-24 18:26:19 +02:00
|
|
|
|
2023-04-20 17:24:15 +02:00
|
|
|
// Double check that we did not overrun the assumed maximum size or cross a
|
|
|
|
// 0x200 boundary (and thus implicitly not any larger power of two, including
|
|
|
|
// the page size).
|
|
|
|
.balign 0x200
|
2022-09-24 18:26:19 +02:00
|
|
|
.org ArmReplaceLiveTranslationEntry + 0x200
|