Ring3: Added support for AARCH64 EL2&0 translation regime.

ArmPkg/Include/Chipset/AArch64.h

@@ -23,6 +23,7 @@
 
 // Coprocessor Trap Register (CPTR)
 #define AARCH64_CPTR_TFP  (1 << 10)
+#define AARCH64_CPTR_FPEN (3 << 20)
 
 // ID_AA64MMFR1 - AArch64 Memory Model Feature Register 0 definitions
 #define AARCH64_MMFR1_VH  (0xF << 8)
@@ -44,6 +45,9 @@
 #define SCR_FW   (1 << 4)
 #define SCR_AW   (1 << 5)
 
+// SCTLR - System Control Register definitions
+#define SCTLR_EPAN          BIT57
+
 // MIDR - Main ID Register definitions
 #define ARM_CPU_TYPE_SHIFT  4
 #define ARM_CPU_TYPE_MASK   0xFFF
@@ -65,6 +69,7 @@
 #define ARM_HCR_AMO  BIT5
 #define ARM_HCR_TSC  BIT19
 #define ARM_HCR_TGE  BIT27
+#define ARM_HCR_E2H  BIT34
 
 // Exception Syndrome Register
 #define AARCH64_ESR_EC(Ecr)   ((0x3F << 26) & (Ecr))

ArmPkg/Include/Library/ArmLib.h

@@ -551,7 +551,7 @@ ArmWriteMVBar (
   IN  UINT32  VectorMonitorBase
   );
 
-UINT32
+UINTN
 EFIAPI
 ArmReadSctlr (
   VOID
@@ -560,7 +560,7 @@ ArmReadSctlr (
 VOID
 EFIAPI
 ArmWriteSctlr (
-  IN  UINT32  Value
+  IN  UINTN  Value
   );
 
 UINTN

ArmPkg/Library/ArmLib/AArch64/ArmLibSupport.S

@@ -149,7 +149,12 @@ ASM_FUNC(ArmInvalidateTlb)
    ret
 
 ASM_FUNC(ArmWriteCptr)
-  msr     cptr_el3, x0           // EL3 Coprocessor Trap Reg (CPTR)
+  EL1_OR_EL2_OR_EL3(x1)
+1:ret
+2:msr     cptr_el2, x0
+  b       4f
+3:msr     cptr_el3, x0           // EL3 Coprocessor Trap Reg (CPTR)
+4:isb
   ret
 
 ASM_FUNC(ArmWriteScr)

ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c

@@ -39,13 +39,9 @@ ArmMemoryAttributeToPageAttribute (
 
     case ARM_MEMORY_REGION_ATTRIBUTE_WRITE_BACK_XP:
     case ARM_MEMORY_REGION_ATTRIBUTE_DEVICE:
-      if (ArmReadCurrentEL () == AARCH64_EL2) {
-        Permissions = TT_XN_MASK;
-      } else {
-        Permissions = TT_UXN_MASK | TT_PXN_MASK;
-      }
-
+      Permissions = TT_UXN_MASK | TT_PXN_MASK;
       break;
+
     default:
       Permissions = 0;
       break;
@@ -444,40 +440,31 @@ GcdAttributeToPageAttribute (
   if (((GcdAttributes & EFI_MEMORY_XP) != 0) ||
       ((GcdAttributes & EFI_MEMORY_CACHETYPE_MASK) == EFI_MEMORY_UC))
   {
-    if (ArmReadCurrentEL () == AARCH64_EL2) {
-      PageAttributes |= TT_XN_MASK;
-    } else {
-      PageAttributes |= TT_UXN_MASK | TT_PXN_MASK;
-    }
+    PageAttributes |= TT_UXN_MASK | TT_PXN_MASK;
   }
 
   if ((GcdAttributes & EFI_MEMORY_RP) == 0) {
     PageAttributes |= TT_AF;
   }
 
-  if ((GcdAttributes & EFI_MEMORY_USER) != 0) {
-    PageAttributes |= TT_PXN_MASK;
-
-    if ((GcdAttributes & EFI_MEMORY_RO) != 0) {
-      PageAttributes |= TT_AP_RO_RO;
-    } else {
-      PageAttributes |= TT_AP_RW_RW;
-    }
-  } else {
-    if (ArmReadCurrentEL () == AARCH64_EL1) {
-      //
-      // TODO: Add EL2&0 support.
-      //
-      PageAttributes |= TT_UXN_MASK;
-    }
-
-    if ((GcdAttributes & EFI_MEMORY_RO) != 0) {
-      PageAttributes |= TT_AP_NO_RO;
-    } else {
-      PageAttributes |= TT_AP_NO_RW;
-    }
-  }
-
+  if ((GcdAttributes & EFI_MEMORY_USER) != 0) {
+    PageAttributes |= TT_PXN_MASK;
+
+    if ((GcdAttributes & EFI_MEMORY_RO) != 0) {
+      PageAttributes |= TT_AP_RO_RO;
+    } else {
+      PageAttributes |= TT_AP_RW_RW;
+    }
+  } else {
+    PageAttributes |= TT_UXN_MASK;
+
+    if ((GcdAttributes & EFI_MEMORY_RO) != 0) {
+      PageAttributes |= TT_AP_NO_RO;
+    } else {
+      PageAttributes |= TT_AP_NO_RW;
+    }
+  }
+
   return PageAttributes;
 }
 
@@ -579,6 +566,7 @@ ArmConfigureMmu (
   UINTN       RootTableEntryCount;
   UINT64      TCR;
   EFI_STATUS  Status;
+  UINTN       Hcr;
 
   if (MemoryTable == NULL) {
     ASSERT (MemoryTable != NULL);
@@ -598,38 +586,25 @@ ArmConfigureMmu (
   T0SZ                = 64 - MaxAddressBits;
   RootTableEntryCount = GetRootTableEntryCount (T0SZ);
 
+  if (ArmReadCurrentEL () == AARCH64_EL2) {
+    //
+    // Switch to EL2&0 translation regime.
+    //
+    Hcr  = ArmReadHcr ();
+    Hcr |= ARM_HCR_E2H | ARM_HCR_TGE;
+    ArmWriteHcr (Hcr);
+    //
+    // Allow access to the Advanced SIMD and floating-point registers.
+    //
+    ArmWriteCptr (AARCH64_CPTR_FPEN);
+  }
+
   //
   // Set TCR that allows us to retrieve T0SZ in the subsequent functions
   //
   // Ideally we will be running at EL2, but should support EL1 as well.
   // UEFI should not run at EL3.
-  if (ArmReadCurrentEL () == AARCH64_EL2) {
-    // Note: Bits 23 and 31 are reserved(RES1) bits in TCR_EL2
-    TCR = T0SZ | (1UL << 31) | (1UL << 23) | TCR_TG0_4KB;
-
-    // Set the Physical Address Size using MaxAddress
-    if (MaxAddress < SIZE_4GB) {
-      TCR |= TCR_PS_4GB;
-    } else if (MaxAddress < SIZE_64GB) {
-      TCR |= TCR_PS_64GB;
-    } else if (MaxAddress < SIZE_1TB) {
-      TCR |= TCR_PS_1TB;
-    } else if (MaxAddress < SIZE_4TB) {
-      TCR |= TCR_PS_4TB;
-    } else if (MaxAddress < SIZE_16TB) {
-      TCR |= TCR_PS_16TB;
-    } else if (MaxAddress < SIZE_256TB) {
-      TCR |= TCR_PS_256TB;
-    } else {
-      DEBUG ((
-        DEBUG_ERROR,
-        "ArmConfigureMmu: The MaxAddress 0x%lX is not supported by this MMU configuration.\n",
-        MaxAddress
-        ));
-      ASSERT (0); // Bigger than 48-bit memory space are not supported
-      return EFI_UNSUPPORTED;
-    }
-  } else if (ArmReadCurrentEL () == AARCH64_EL1) {
+  if ((ArmReadCurrentEL () == AARCH64_EL1) || (ArmReadCurrentEL () == AARCH64_EL2)) {
     // Due to Cortex-A57 erratum #822227 we must set TG1[1] == 1, regardless of EPD1.
     TCR = T0SZ | TCR_TG0_4KB | TCR_TG1_4KB | TCR_EPD1;
 

ArmVirtPkg/ArmVirt.dsc.inc

@@ -106,7 +106,7 @@
   ArmGicArchLib|ArmVirtPkg/Library/ArmVirtGicArchLib/ArmVirtGicArchLib.inf
   ArmSmcLib|ArmPkg/Library/ArmSmcLib/ArmSmcLib.inf
   ArmHvcLib|ArmPkg/Library/ArmHvcLib/ArmHvcLib.inf
-  ArmGenericTimerCounterLib|ArmPkg/Library/ArmGenericTimerVirtCounterLib/ArmGenericTimerVirtCounterLib.inf
+  ArmGenericTimerCounterLib|ArmPkg/Library/ArmGenericTimerPhyCounterLib/ArmGenericTimerPhyCounterLib.inf
 
   PlatformPeiLib|ArmVirtPkg/Library/PlatformPeiLib/PlatformPeiLib.inf
   MemoryInitPeiLib|ArmVirtPkg/Library/ArmVirtMemoryInitPeiLib/ArmVirtMemoryInitPeiLib.inf

MdeModulePkg/Core/Dxe/SysCall/AARCH64/CoreBootServices.S

@@ -88,24 +88,23 @@ ASM_FUNC(ArmCallRing3)
   // Disable interrupts.
   msr  daifset, #0xf
   isb
+  // Copy PSTATE to SPSR.
+  mrs  x6, nzcv
+  mrs  x7, pan
+  orr  x6, x6, x7
   // Prepare Ring3 SP and EntryPoint.
   msr  sp_el0, x1
-  msr  elr_el1, x2
+  EL1_OR_EL2(x1)
+1:msr  elr_el1, x2
+  msr  spsr_el1, x6
+  b    3f
+2:msr  elr_el2, x2
+  msr  spsr_el2, x6
   // Save Core SP and switch to CoreSysCall Stack.
-  mov  x5, sp
+3:mov  x5, sp
   str  x5, [x4]
   mov  sp, x3
-  // Copy PSTATE to SPSR.
-  mrs  x1, nzcv
-  mrs  x2, pan
-  orr  x1, x1, x2
-  //
-  // M[3:0], bits [3:0] AArch64 Exception level and selected Stack Pointer.
-  // 0b0000 - EL0.
-  // 0b0100 - EL1 with SP_EL0 (ELt).
-  // 0b0101 - EL1 with SP_EL1 (EL1h).
-  //
-  msr  spsr_el1, x1
+
   isb
   eret
 

MdeModulePkg/Core/Dxe/SysCall/AARCH64/InitializeMsr.c

@@ -97,6 +97,7 @@ InitializeMsr (
   )
 {
   UINTN  Tcr;
+  UINTN  Sctlr;
   //
   // If HCR_EL2.NV is 1 and the current Exception level is EL1,
   // then EL1 read accesses to the CurrentEL register return a value of 0x2 in bits[3:2].
@@ -115,6 +116,10 @@ InitializeMsr (
     //
     // Enable Privileged Access Never feature.
     //
+    Sctlr  = ArmReadSctlr ();
+    Sctlr |= SCTLR_EPAN;
+    ArmWriteSctlr (Sctlr);
+
     ArmSetPan ();
   }