ArmPkg/ArmLib: don't invalidate entire I-cache on range operation

Instead of cleaning the data cache to the PoU by virtual address and
subsequently invalidating the entire I-cache, invalidate only the
range that we just cleaned. This way, we don't invalidate other
cachelines unnecessarily.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif.lindholm@linaro.org>

ArmPkg/Include/Library/ArmLib.h

@@ -183,13 +183,19 @@ ArmInvalidateDataCacheEntryByMVA (
 
 VOID
 EFIAPI
-ArmCleanDataCacheEntryToPoUByMVA(
+ArmCleanDataCacheEntryToPoUByMVA (
   IN  UINTN   Address
   );
 
 VOID
 EFIAPI
-ArmCleanDataCacheEntryByMVA(
+ArmInvalidateInstructionCacheEntryToPoUByMVA (
+  IN  UINTN   Address
+  );
+
+VOID
+EFIAPI
+ArmCleanDataCacheEntryByMVA (
 IN  UINTN   Address
 );
 

ArmPkg/Library/ArmCacheMaintenanceLib/ArmCacheMaintenanceLib.c

@@ -17,15 +17,16 @@
 #include <Library/DebugLib.h>
 #include <Library/PcdLib.h>
 
+STATIC
 VOID
 CacheRangeOperation (
   IN  VOID            *Start,
   IN  UINTN           Length,
-  IN  LINE_OPERATION  LineOperation
+  IN  LINE_OPERATION  LineOperation,
+  IN  UINTN           LineLength
   )
 {
-  UINTN ArmCacheLineLength         = ArmDataCacheLineLength();
+  UINTN ArmCacheLineAlignmentMask  = LineLength - 1;
-  UINTN ArmCacheLineAlignmentMask  = ArmCacheLineLength - 1;
 
   // Align address (rounding down)
   UINTN AlignedAddress = (UINTN)Start - ((UINTN)Start & ArmCacheLineAlignmentMask);
@@ -34,7 +35,7 @@ CacheRangeOperation (
   // Perform the line operation on an address in each cache line
   while (AlignedAddress < EndAddress) {
     LineOperation(AlignedAddress);
-    AlignedAddress += ArmCacheLineLength;
+    AlignedAddress += LineLength;
   }
   ArmDataSynchronizationBarrier ();
 }
@@ -64,8 +65,14 @@ InvalidateInstructionCacheRange (
   IN      UINTN                     Length
   )
 {
-  CacheRangeOperation (Address, Length, ArmCleanDataCacheEntryToPoUByMVA);
+  CacheRangeOperation (Address, Length, ArmCleanDataCacheEntryToPoUByMVA,
-  ArmInvalidateInstructionCache ();
+    ArmDataCacheLineLength ());
+  CacheRangeOperation (Address, Length,
+    ArmInvalidateInstructionCacheEntryToPoUByMVA,
+    ArmInstructionCacheLineLength ());
+
+  ArmInstructionSynchronizationBarrier ();
+
   return Address;
 }
 
@@ -85,7 +92,8 @@ WriteBackInvalidateDataCacheRange (
   IN      UINTN                     Length
   )
 {
-  CacheRangeOperation(Address, Length, ArmCleanInvalidateDataCacheEntryByMVA);
+  CacheRangeOperation(Address, Length, ArmCleanInvalidateDataCacheEntryByMVA,
+    ArmDataCacheLineLength ());
   return Address;
 }
 
@@ -105,7 +113,8 @@ WriteBackDataCacheRange (
   IN      UINTN                     Length
   )
 {
-  CacheRangeOperation(Address, Length, ArmCleanDataCacheEntryByMVA);
+  CacheRangeOperation(Address, Length, ArmCleanDataCacheEntryByMVA,
+    ArmDataCacheLineLength ());
   return Address;
 }
 
@@ -116,6 +125,7 @@ InvalidateDataCacheRange (
   IN      UINTN                     Length
   )
 {
-  CacheRangeOperation(Address, Length, ArmInvalidateDataCacheEntryByMVA);
+  CacheRangeOperation(Address, Length, ArmInvalidateDataCacheEntryByMVA,
+    ArmDataCacheLineLength ());
   return Address;
 }

ArmPkg/Library/ArmLib/AArch64/AArch64Support.S

@@ -23,6 +23,7 @@ GCC_ASM_EXPORT (ArmInvalidateInstructionCache)
 GCC_ASM_EXPORT (ArmInvalidateDataCacheEntryByMVA)
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryByMVA)
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryToPoUByMVA)
+GCC_ASM_EXPORT (ArmInvalidateInstructionCacheEntryToPoUByMVA)
 GCC_ASM_EXPORT (ArmCleanInvalidateDataCacheEntryByMVA)
 GCC_ASM_EXPORT (ArmInvalidateDataCacheEntryBySetWay)
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryBySetWay)
@@ -80,6 +81,10 @@ ASM_PFX(ArmCleanDataCacheEntryToPoUByMVA):
   dc      cvau, x0    // Clean single data cache line to PoU
   ret
 
+ASM_PFX(ArmInvalidateInstructionCacheEntryToPoUByMVA):
+  ic      ivau, x0    // Invalidate single instruction cache line to PoU
+  ret
+
 
 ASM_PFX(ArmCleanInvalidateDataCacheEntryByMVA):
   dc      civac, x0   // Clean and invalidate single data cache line

ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.S

@@ -18,6 +18,7 @@
 
 GCC_ASM_EXPORT (ArmInvalidateInstructionCache)
 GCC_ASM_EXPORT (ArmInvalidateDataCacheEntryByMVA)
+GCC_ASM_EXPORT (ArmInvalidateInstructionCacheEntryToPoUByMVA)
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryByMVA)
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryToPoUByMVA)
 GCC_ASM_EXPORT (ArmCleanInvalidateDataCacheEntryByMVA)
@@ -74,6 +75,10 @@ ASM_PFX(ArmCleanDataCacheEntryToPoUByMVA):
   mcr     p15, 0, r0, c7, c11, 1  @clean single data cache line to PoU
   bx      lr
 
+ASM_PFX(ArmInvalidateInstructionCacheEntryToPoUByMVA):
+  mcr     p15, 0, r0, c7, c5, 1  @Invalidate single instruction cache line to PoU
+  mcr     p15, 0, r0, c7, c5, 7  @Invalidate branch predictor
+  bx      lr
 
 ASM_PFX(ArmCleanInvalidateDataCacheEntryByMVA):
   mcr     p15, 0, r0, c7, c14, 1  @clean and invalidate single data cache line

ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.asm

@@ -34,6 +34,12 @@ CTRL_I_BIT      EQU     (1 << 12)
   bx      lr
 
 
+ RVCT_ASM_EXPORT ArmInvalidateInstructionCacheEntryToPoUByMVA
+  mcr     p15, 0, r0, c7, c5, 1   ; invalidate single instruction cache line to PoU
+  mcr     p15, 0, r0, c7, c5, 7   ; invalidate branch predictor
+  bx      lr
+
+
  RVCT_ASM_EXPORT ArmCleanDataCacheEntryToPoUByMVA
   mcr     p15, 0, r0, c7, c11, 1  ; clean single data cache line to PoU
   bx      lr