Ring3: Added support for AARCH64 User page table.

ArmPkg/Library/ArmExceptionLib/AArch64/ExceptionSupport.S

@@ -101,6 +101,8 @@ GCC_ASM_EXPORT(CommonCExceptionHandler)
 #define FP_CONTEXT_SIZE    (32 * 16)
 #define SYS_CONTEXT_SIZE   ( 6 *  8) // 5 SYS regs + Alignment requirement (ie: the stack must be aligned on 0x10)
 
+ASM_FUNC_ALIGN(ExceptionHandlerBase, 4096)
+
 //
 // There are two methods for installing AArch64 exception vectors:
 //  1. Install a copy of the vectors to a location specified by a PCD
@@ -264,6 +266,28 @@ ASM_PFX(CommonExceptionEntry):
   stp      q28, q29, [x28, #0x1c0]
   stp      q30, q31, [x28, #0x1e0]
 
+  ldr      x2, [x28, #(FP_CONTEXT_SIZE + 0x8)] // Saved Processor Status Register
+  and      x2, x2, #0xF
+  cmp      x2, 0           // Check whether EL0 process was interrupted
+  b.ne     NoTTBR0Switch
+  adrp     x1, ASM_PFX(CorePageTable)
+  EL1_OR_EL2(x3)
+1:mrs      x4, ttbr0_el1
+  str      x4, [x1, #0x8]  // UserPageTable
+  ldr      x4, [x1]        // CorePageTable
+  msr      ttbr0_el1, x4
+  tlbi     vmalle1
+  b        3f
+2:mrs      x4, ttbr0_el2
+  str      x4, [x1, #0x8]  // UserPageTable
+  ldr      x4, [x1]        // CorePageTable
+  msr      ttbr0_el2, x4
+  tlbi     alle2
+3:dsb      sy
+  isb
+
+NoTTBR0Switch:
+
   // x0 still holds the exception type.
   // Set x1 to point to the top of our struct on the Stack
   mov      x1, sp
@@ -279,6 +303,24 @@ ASM_PFX(CommonExceptionEntry):
   // We do not try to recover.
   bl       ASM_PFX(CommonCExceptionHandler) // Call exception handler
 
+  ldr      x2, [x28, #(FP_CONTEXT_SIZE + 0x8)] // Saved Processor Status Register
+  and      x2, x2, #0xF
+  cmp      x2, 0           // Check whether EL0 process was interrupted
+  b.ne     NoTTBR0Switch2
+  adrp     x1, ASM_PFX(CorePageTable)
+  EL1_OR_EL2(x3)
+1:ldr      x4, [x1, #0x8]  // UserPageTable
+  msr      ttbr0_el1, x4
+  tlbi     vmalle1
+  b        3f
+2:ldr      x4, [x1, #0x8]  // UserPageTable
+  msr      ttbr0_el2, x4
+  tlbi     alle2
+3:dsb      sy
+  isb
+
+NoTTBR0Switch2:
+
   // Pop as many GP regs as we can before entering the critical section below
   ldp      x2,  x3,  [sp, #0x10]
   ldp      x4,  x5,  [sp, #0x20]
@@ -356,6 +398,22 @@ continue:
 
   eret
 
+ASM_FUNC_ALIGN(ExceptionHandlerFinal, 4096)
+
 ASM_FUNC(RegisterEl0Stack)
   msr     sp_el0, x0
   ret
+
+.data
+
+.global ASM_PFX(CorePageTable)
+.balign 4096
+ASM_PFX(CorePageTable):
+.ds.d 1
+
+UserPageTable:
+.ds.d 1
+
+.balign 4096
+Padding:
+.ds.b 1

ArmPkg/Library/ArmExceptionLib/ArmExceptionLib.c

@@ -41,6 +41,20 @@ extern UINTN                   gMaxExceptionNumber;
 extern EFI_EXCEPTION_CALLBACK  gExceptionHandlers[];
 extern PHYSICAL_ADDRESS        gExceptionVectorAlignmentMask;
 
+EXCEPTION_ADDRESSES   mAddresses;
+
+VOID
+ExceptionHandlerBase (
+  VOID
+  );
+
+VOID
+ExceptionHandlerFinal (
+  VOID
+  );
+
+extern UINTN          CorePageTable;
+
 /**
 Initializes all CPU exceptions entries and provides the default exception handlers.
 
@@ -138,10 +152,6 @@ CommonCExceptionHandler (
   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
   )
 {
-  if (ArmHasPan ()) {
-    ArmClearPan ();
-  }
-
   if ((UINTN)ExceptionType <= gMaxExceptionNumber) {
     if (gExceptionHandlers[ExceptionType]) {
       gExceptionHandlers[ExceptionType](ExceptionType, SystemContext);
@@ -181,3 +191,28 @@ InitializeSeparateExceptionStacks (
 {
   return EFI_SUCCESS;
 }
+
+EXCEPTION_ADDRESSES *
+EFIAPI
+GetExceptionAddresses (
+  VOID
+  )
+{
+  return &mAddresses;
+}
+
+VOID
+EFIAPI
+SetExceptionAddresses (
+  IN VOID   *Buffer,
+  IN UINTN  BufferSize
+  )
+{
+  mAddresses.ExceptionStackBase   = (UINTN)Buffer;
+  mAddresses.ExceptionStackSize   = BufferSize;
+  mAddresses.ExceptionHandlerBase = (UINTN)ExceptionHandlerBase;
+  mAddresses.ExceptionHandlerSize = (UINTN)ExceptionHandlerFinal - mAddresses.ExceptionHandlerBase;
+  mAddresses.ExceptionDataBase    = (UINTN)&CorePageTable;
+
+  CorePageTable = (UINTN)ArmGetTTBR0BaseAddress ();
+}

ArmPkg/Library/DefaultExceptionHandlerLib/AArch64/DefaultExceptionHandler.c

@@ -232,6 +232,10 @@ DefaultExceptionHandler (
   // gets reused.
   UnicodeSPrintAsciiFormat (UnicodeBuffer, MAX_PRINT_CHARS, Buffer);
 
+  if (ArmHasPan ()) {
+    ArmClearPan ();
+  }
+
   DEBUG_CODE_BEGIN ();
   CONST CHAR8  *Pdb, *PrevPdb;
   UINTN        ImageBase;

MdeModulePkg/Core/Dxe/SysCall/AARCH64/CoreBootServices.S

@@ -55,6 +55,7 @@ call:
     add	 sp, sp, #0x10
     ret
 
+ASM_FUNC_ALIGN(SysCallBase, 4096)
 //------------------------------------------------------------------------------
 // EFI_STATUS
 // EFIAPI
@@ -67,6 +68,7 @@ call:
 //   (x2) gRing3EntryPoint
 //   (x3) gCoreSysCallStackTop
 //   (x4) &CoreSp
+//   (x5) gUserPageTable
 //------------------------------------------------------------------------------
 ASM_FUNC(ArmCallRing3)
   // Save registers.
@@ -88,6 +90,10 @@ ASM_FUNC(ArmCallRing3)
   // Disable interrupts.
   msr  daifset, #0xf
   isb
+  // Save Core SP and switch to CoreSysCall Stack.
+  mov  x6, sp
+  str  x6, [x4]
+  mov  sp, x3
   // Copy PSTATE to SPSR.
   mrs  x6, nzcv
   mrs  x7, pan
@@ -97,17 +103,18 @@ ASM_FUNC(ArmCallRing3)
   EL1_OR_EL2(x1)
 1:msr  elr_el1, x2
   msr  spsr_el1, x6
+  msr  ttbr0_el1, x5
+  tlbi vmalle1
   b    3f
 2:msr  elr_el2, x2
   msr  spsr_el2, x6
-  // Save Core SP and switch to CoreSysCall Stack.
+  msr  ttbr0_el2, x5
-3:mov  x5, sp
+  tlbi alle2
-  str  x5, [x4]
+3:dsb   sy
-  mov  sp, x3
-
   isb
   eret
 
+ASM_FUNC_ALIGN(SysCallEnd, 4096)
 //------------------------------------------------------------------------------
 // VOID
 // EFIAPI

MdeModulePkg/Core/Dxe/SysCall/AARCH64/InitializeAARCH64.c

@@ -13,7 +13,7 @@
 #include "DxeMain.h"
 
 STATIC UINTN  mCoreSp;
-extern UINTN  gUartBaseAddress;
+UINTN         gUserPageTable;
 
 EFI_STATUS
 EFIAPI
@@ -22,7 +22,8 @@ ArmCallRing3 (
   IN VOID            *StackPointer,
   IN VOID            *EntryPoint,
   IN VOID            *SysCallStack,
-  IN VOID            *CoreStack
+  IN VOID            *CoreStack,
+  IN UINTN           UserPageTable
   );
 
 VOID
@@ -65,12 +66,6 @@ SysCallBootService (
 
   AllowSupervisorAccessToUserMemory ();
   CopyMem ((VOID *)((UINTN)Physical + sizeof (UINTN)), (VOID *)UserRsp, 8 * sizeof (UINTN));
-
-  SetUefiImageMemoryAttributes (
-    gUartBaseAddress,
-    EFI_PAGE_SIZE,
-    EFI_MEMORY_XP
-    );
   ForbidSupervisorAccessToUserMemory ();
 
   Status = CallBootService (
@@ -81,12 +76,6 @@ SysCallBootService (
 
   CoreFreePages (Physical, EFI_SIZE_TO_PAGES (9 * sizeof (UINTN)));
 
-  SetUefiImageMemoryAttributes (
-    gUartBaseAddress,
-    EFI_PAGE_SIZE,
-    EFI_MEMORY_XP | EFI_MEMORY_USER
-    );
-
   return Status;
 }
 
@@ -128,13 +117,6 @@ InitializeMsr (
   UINTN  Tcr;
   UINTN  Sctlr;
   //
-  // If HCR_EL2.NV is 1 and the current Exception level is EL1,
-  // then EL1 read accesses to the CurrentEL register return a value of 0x2 in bits[3:2].
-  // CurrentEL == 1 -> HCR_EL2.NV == 0
-  //
-  // If stage 1 is enabled and stage 1 Base permissions use Direct permissions,
-  // then GCS access is not permitted and UnprivGCS and PrivGCS are not present.
-  //
   // Disable Hierarchical permissions just in case.
   //
   Tcr = ArmGetTCR ();
@@ -153,6 +135,7 @@ InitializeMsr (
   }
 
   InitializeSysCallHandler ((VOID *)SysCallBootService);
+  SetExceptionAddresses (NULL, 0);
 }
 
 VOID
@@ -183,5 +166,5 @@ CallRing3 (
   IN RING3_CALL_DATA *Data
   )
 {
-  return ArmCallRing3 (Data, gRing3CallStackTop, gRing3EntryPoint, gCoreSysCallStackTop, &mCoreSp);
+  return ArmCallRing3 (Data, gRing3CallStackTop, gRing3EntryPoint, gCoreSysCallStackTop, &mCoreSp, gUserPageTable);
 }

MdeModulePkg/Core/Dxe/SysCall/BootServices.c

@@ -22,7 +22,7 @@ CHAR8 *SysCallNames[] = {
   //
   // BootServices
   //
-  "SysCallReturnToCore",     // Must always be zero for CoreBootServices.nasm.
+  "SysCallReturnToCore",
   "SysCallLocateProtocol",
   "SysCallOpenProtocol",
   "SysCallInstallMultipleProtocolInterfaces",

MdeModulePkg/Core/Dxe/SysCall/Initialization.c

@@ -20,6 +20,8 @@ UINTN       gUartBaseAddress;
 
 UEFI_IMAGE_RECORD    *mDxeRing3;
 EXCEPTION_ADDRESSES  *mExceptionAddresses;
+UINTN                mConfigurationTable;
+UINTN                mConfigurationTableSize;
 
 extern UINTN         SysCallBase;
 extern UINTN         SysCallEnd;
@@ -79,10 +81,12 @@ InitializeRing3 (
     );
 
   if (PcdGetBool (PcdSerialUseMmio)) {
+    mConfigurationTableSize = (gRing3Data->SystemTable.NumberOfTableEntries + 1) * sizeof (EFI_CONFIGURATION_TABLE);
+
     Status = CoreAllocatePages (
                AllocateAnyPages,
                EfiRing3MemoryType,
-               EFI_SIZE_TO_PAGES ((gRing3Data->SystemTable.NumberOfTableEntries + 1) * sizeof (EFI_CONFIGURATION_TABLE)),
+               EFI_SIZE_TO_PAGES (mConfigurationTableSize),
                &Physical
                );
     if (EFI_ERROR (Status)) {
@@ -111,6 +115,7 @@ InitializeRing3 (
     DEBUG ((DEBUG_ERROR, "Core: gUartBaseAddress = 0x%p\n", gUartBaseAddress));
 
     gRing3Data->SystemTable.ConfigurationTable = (EFI_CONFIGURATION_TABLE *)(UINTN)Physical;
+    mConfigurationTable                        = (UINTN)Physical;
   }
   //
   // Initialize DxeRing3 with Supervisor privileges.
@@ -210,7 +215,6 @@ InitializeUserPageTable (
   UINTN                      SectionAddress;
   UINT32                     Index;
   UEFI_IMAGE_RECORD          *UserImageRecord;
-  IA32_DESCRIPTOR            IdtDescriptor;
 
   //
   // TODO: Remove ASSERTs, add proper checks and return status.
@@ -270,14 +274,6 @@ InitializeUserPageTable (
           EFI_MEMORY_RO
           );
 
-  gCpu->SetUserMemoryAttributes (
-          gCpu,
-          UserPageTable,
-          (UINTN)&gCorePageTable,
-          SIZE_4KB,
-          EFI_MEMORY_RO | EFI_MEMORY_XP
-          );
-
   gCpu->SetUserMemoryAttributes (
           gCpu,
           UserPageTable,
@@ -289,14 +285,6 @@ InitializeUserPageTable (
   //
   // Map ExceptionHandlers, ExceptionStacks, Idt
   //
-  gCpu->SetUserMemoryAttributes (
-          gCpu,
-          UserPageTable,
-          mExceptionAddresses->ExceptionStackBase,
-          mExceptionAddresses->ExceptionStackSize,
-          EFI_MEMORY_XP
-          );
-
   gCpu->SetUserMemoryAttributes (
           gCpu,
           UserPageTable,
@@ -313,6 +301,25 @@ InitializeUserPageTable (
           EFI_MEMORY_XP
           );
 
+#if defined (MDE_CPU_X64) || defined (MDE_CPU_IA32)
+  IA32_DESCRIPTOR  IdtDescriptor;
+
+  gCpu->SetUserMemoryAttributes (
+          gCpu,
+          UserPageTable,
+          (UINTN)&gCorePageTable,
+          SIZE_4KB,
+          EFI_MEMORY_RO | EFI_MEMORY_XP
+          );
+
+  gCpu->SetUserMemoryAttributes (
+          gCpu,
+          UserPageTable,
+          mExceptionAddresses->ExceptionStackBase,
+          mExceptionAddresses->ExceptionStackSize,
+          EFI_MEMORY_XP
+          );
+
   AsmReadIdtr (&IdtDescriptor);
   gCpu->SetUserMemoryAttributes (
           gCpu,
@@ -332,6 +339,25 @@ InitializeUserPageTable (
           FixedPcdGet32 (PcdOvmfWorkAreaSize),
           EFI_MEMORY_XP | EFI_MEMORY_USER
           );
+#elif defined (MDE_CPU_AARCH64) || defined (MDE_CPU_ARM)
+  gCpu->SetUserMemoryAttributes (
+          gCpu,
+          UserPageTable,
+          mConfigurationTable,
+          ALIGN_VALUE (mConfigurationTableSize, EFI_PAGE_SIZE),
+          EFI_MEMORY_XP | EFI_MEMORY_USER
+          );
+  //
+  // Necessary fix for DEBUG printings.
+  //
+  gCpu->SetUserMemoryAttributes (
+          gCpu,
+          UserPageTable,
+          gUartBaseAddress,
+          SIZE_4KB,
+          EFI_MEMORY_XP | EFI_MEMORY_USER
+          );
+#endif
 
   //
   // Map User Image

MdeModulePkg/Core/Dxe/SysCall/SupportedProtocols.c

@@ -13,10 +13,6 @@ EFI_FILE_PROTOCOL                mRing3FileProtocol;
 
 EFI_SIMPLE_FILE_SYSTEM_PROTOCOL  *mRing3SimpleFileSystemPointer;
 
-#if defined (MDE_CPU_AARCH64) || defined (MDE_CPU_ARM)
-extern UINTN  gUartBaseAddress;
-#endif
-
 typedef struct {
   EFI_FILE_PROTOCOL  Protocol;
   EFI_FILE_PROTOCOL  *Ring3File;
@@ -62,28 +58,8 @@ GoToRing3 (
   VA_END (Marker);
   ForbidSupervisorAccessToUserMemory ();
 
-#if defined (MDE_CPU_AARCH64) || defined (MDE_CPU_ARM)
-  //
-  // Necessary fix for DEBUG printings.
-  //
-  SetUefiImageMemoryAttributes (
-    gUartBaseAddress,
-    EFI_PAGE_SIZE,
-    EFI_MEMORY_XP | EFI_MEMORY_USER
-    );
-#endif
   Status = CallRing3 (Input);
 
-#if defined (MDE_CPU_AARCH64) || defined (MDE_CPU_ARM)
-  AllowSupervisorAccessToUserMemory ();
-  SetUefiImageMemoryAttributes (
-    gUartBaseAddress,
-    EFI_PAGE_SIZE,
-    EFI_MEMORY_XP
-    );
-  ForbidSupervisorAccessToUserMemory ();
-#endif
-
   CoreFreePages (Ring3Pages, PagesNumber);
 
   return Status;