UefiCpuPkg: Has APs in 64 bit long-mode before booting to OS.

During the finalization of Mp initialization before booting into the OS,
 depending on whether Mwait is supported or not, AsmRelocateApLoop
 places Aps in MWAIT-loop or HLT-loop.

Since paging is necessary for long mode, the original implementation of
moving APs to 32-bit was to disable paging to ensure that the booting
does not crash.

The current modification creates a page table in reserved memory,
avoiding switching modes and reclaiming memory by OS. This modification
is only for 64 bit mode.

More specifically, we keep the AMD logic as the original code flow,
extract and update the Intel-related code, where the APs would stay
in 64-bit, and run in a Mwait or Hlt loop until the OS wake them up.

Signed-off-by: Ray Ni <ray.ni@intel.com>
Signed-off-by: Yuanhao Xie <yuanhao.xie@intel.com>
Reviewed-by: Ray Ni <ray.ni@intel.com>

UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf

@@ -1,7 +1,7 @@
 ## @file
 #  MP Initialize Library instance for DXE driver.
 #
-#  Copyright (c) 2016 - 2021, Intel Corporation. All rights reserved.<BR>
+#  Copyright (c) 2016 - 2022, Intel Corporation. All rights reserved.<BR>
 #  SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 ##
@@ -24,9 +24,11 @@
 [Sources.IA32]
   Ia32/AmdSev.c
   Ia32/MpFuncs.nasm
+  Ia32/CreatePageTable.c
 
 [Sources.X64]
   X64/AmdSev.c
+  X64/CreatePageTable.c
   X64/MpFuncs.nasm
 
 [Sources.common]
@@ -57,6 +59,9 @@
   CcExitLib
   MicrocodeLib
 
+[LibraryClasses.X64]
+  CpuPageTableLib
+
 [Protocols]
   gEfiTimerArchProtocolGuid                     ## SOMETIMES_CONSUMES
 

UefiCpuPkg/Library/MpInitLib/DxeMpLib.c

@@ -28,6 +28,7 @@ volatile BOOLEAN  mStopCheckAllApsStatus       = TRUE;
 VOID              *mReservedApLoopFunc         = NULL;
 UINTN             mReservedTopOfApStack;
 volatile UINT32   mNumberToFinish = 0;
+UINTN             mApPageTable;
 
 //
 // Begin wakeup buffer allocation below 0x88000
@@ -407,12 +408,9 @@ RelocateApLoop (
     AsmRelocateApLoopFunc (
       MwaitSupport,
       CpuMpData->ApTargetCState,
-      CpuMpData->PmCodeSegment,
       StackStart - ProcessorNumber * AP_SAFE_STACK_SIZE,
       (UINTN)&mNumberToFinish,
-      CpuMpData->Pm16CodeSegment,
+      mApPageTable
-      CpuMpData->SevEsAPBuffer,
-      CpuMpData->WakeupBuffer
       );
   }
 
@@ -477,7 +475,6 @@ InitMpGlobalData (
   )
 {
   EFI_STATUS                       Status;
-  EFI_PHYSICAL_ADDRESS             Address;
   UINTN                            ApSafeBufferSize;
   UINTN                            Index;
   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  MemDesc;
@@ -545,60 +542,45 @@ InitMpGlobalData (
   // Allocating it in advance since memory services are not available in
   // Exit Boot Services callback function.
   //
-  ApSafeBufferSize = EFI_PAGES_TO_SIZE (
+  // +------------+
-                       EFI_SIZE_TO_PAGES (
+  // | Ap Loop    |
-                         CpuMpData->AddressMap.RelocateApLoopFuncSize
+  // +------------+
-                         )
+  // | Stack * N  |
-                       );
+  // +------------+ (low address)
-  Address = BASE_4GB - 1;
-  Status  = gBS->AllocatePages (
-                   AllocateMaxAddress,
-                   EfiReservedMemoryType,
-                   EFI_SIZE_TO_PAGES (ApSafeBufferSize),
-                   &Address
-                   );
-  ASSERT_EFI_ERROR (Status);
-
-  mReservedApLoopFunc = (VOID *)(UINTN)Address;
-  ASSERT (mReservedApLoopFunc != NULL);
-
   //
-  // Make sure that the buffer memory is executable if NX protection is enabled
-  // for EfiReservedMemoryType.
-  //
-  // TODO: Check EFI_MEMORY_XP bit set or not once it's available in DXE GCD
-  //       service.
-  //
-  Status = gDS->GetMemorySpaceDescriptor (Address, &MemDesc);
-  if (!EFI_ERROR (Status)) {
-    gDS->SetMemorySpaceAttributes (
-           Address,
-           ApSafeBufferSize,
-           MemDesc.Attributes & (~EFI_MEMORY_XP)
-           );
-  }
-
   ApSafeBufferSize = EFI_PAGES_TO_SIZE (
                        EFI_SIZE_TO_PAGES (
                          CpuMpData->CpuCount * AP_SAFE_STACK_SIZE
+                         + CpuMpData->AddressMap.RelocateApLoopFuncSize
                          )
                        );
-  Address = BASE_4GB - 1;
-  Status  = gBS->AllocatePages (
-                   AllocateMaxAddress,
-                   EfiReservedMemoryType,
-                   EFI_SIZE_TO_PAGES (ApSafeBufferSize),
-                   &Address
-                   );
-  ASSERT_EFI_ERROR (Status);
 
-  mReservedTopOfApStack = (UINTN)Address + ApSafeBufferSize;
+  mReservedTopOfApStack = (UINTN)AllocateReservedPages (EFI_SIZE_TO_PAGES (ApSafeBufferSize));
+  ASSERT (mReservedTopOfApStack != 0);
   ASSERT ((mReservedTopOfApStack & (UINTN)(CPU_STACK_ALIGNMENT - 1)) == 0);
-  CopyMem (
+  ASSERT ((AP_SAFE_STACK_SIZE & (CPU_STACK_ALIGNMENT - 1)) == 0);
-    mReservedApLoopFunc,
+
-    CpuMpData->AddressMap.RelocateApLoopFuncAddress,
+  mReservedApLoopFunc = (VOID *)(mReservedTopOfApStack + CpuMpData->CpuCount * AP_SAFE_STACK_SIZE);
-    CpuMpData->AddressMap.RelocateApLoopFuncSize
+  if (StandardSignatureIsAuthenticAMD ()) {
-    );
+    CopyMem (
+      mReservedApLoopFunc,
+      CpuMpData->AddressMap.RelocateApLoopFuncAddressAmd,
+      CpuMpData->AddressMap.RelocateApLoopFuncSizeAmd
+      );
+  } else {
+    CopyMem (
+      mReservedApLoopFunc,
+      CpuMpData->AddressMap.RelocateApLoopFuncAddress,
+      CpuMpData->AddressMap.RelocateApLoopFuncSize
+      );
+
+    mApPageTable = CreatePageTable (
+                     mReservedTopOfApStack,
+                     ApSafeBufferSize
+                     );
+  }
+
+  mReservedTopOfApStack += CpuMpData->CpuCount * AP_SAFE_STACK_SIZE;
 
   Status = gBS->CreateEvent (
                   EVT_TIMER | EVT_NOTIFY_SIGNAL,

UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c

@@ -0,0 +1,27 @@
+/** @file
+  Function to create page talbe.
+  Only create page table for x64, and leave the CreatePageTable empty for Ia32.
+
+  Copyright (c) 2022, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+
+#include <Base.h>
+
+/**
+  Only create page table for x64, and leave the CreatePageTable empty for Ia32.
+
+  @param[in]      LinearAddress  The start of the linear address range.
+  @param[in]      Length         The length of the linear address range.
+
+  @return The page table to be created.
+**/
+UINTN
+CreatePageTable (
+  IN UINTN  Address,
+  IN UINTN  Length
+  )
+{
+  return 0;
+}

UefiCpuPkg/Library/MpInitLib/MpLib.h

@@ -392,12 +392,9 @@ typedef
 (EFIAPI *ASM_RELOCATE_AP_LOOP)(
   IN BOOLEAN                 MwaitSupport,
   IN UINTN                   ApTargetCState,
-  IN UINTN                   PmCodeSegment,
   IN UINTN                   TopOfApStack,
   IN UINTN                   NumberToFinish,
-  IN UINTN                   Pm16CodeSegment,
+  IN UINTN                   Cr3
-  IN UINTN                   SevEsAPJumpTable,
-  IN UINTN                   WakeupBuffer
   );
 
 /**
@@ -512,6 +509,20 @@ WakeUpAP (
   IN BOOLEAN           WakeUpDisabledAps
   );
 
+/**
+  Create 1:1 mapping page table in reserved memory to map the specified address range.
+
+  @param[in]      LinearAddress  The start of the linear address range.
+  @param[in]      Length         The length of the linear address range.
+
+  @return The page table to be created.
+**/
+UINTN
+CreatePageTable (
+  IN UINTN  Address,
+  IN UINTN  Length
+  );
+
 /**
   Initialize global data for MP support.
 

UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c

@@ -0,0 +1,75 @@
+/** @file
+  Function to create page talbe.
+  Only create page table for x64, and leave the CreatePageTable empty for Ia32.
+
+  Copyright (c) 2022, Intel Corporation. All rights reserved.<BR>
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+
+**/
+#include <Library/CpuPageTableLib.h>
+#include <Library/MemoryAllocationLib.h>
+#include <Base.h>
+#include <Library/BaseMemoryLib.h>
+#include <Library/DebugLib.h>
+
+/**
+  Create 1:1 mapping page table in reserved memory to map the specified address range.
+
+  @param[in]      LinearAddress  The start of the linear address range.
+  @param[in]      Length         The length of the linear address range.
+
+  @return The page table to be created.
+**/
+UINTN
+CreatePageTable (
+  IN UINTN  Address,
+  IN UINTN  Length
+  )
+{
+  EFI_STATUS  Status;
+  VOID        *PageTableBuffer;
+  UINTN       PageTableBufferSize;
+  UINTN       PageTable;
+
+  IA32_MAP_ATTRIBUTE  MapAttribute;
+  IA32_MAP_ATTRIBUTE  MapMask;
+
+  MapAttribute.Uint64         = Address;
+  MapAttribute.Bits.Present   = 1;
+  MapAttribute.Bits.ReadWrite = 1;
+
+  MapMask.Bits.PageTableBaseAddress = 1;
+  MapMask.Bits.Present              = 1;
+  MapMask.Bits.ReadWrite            = 1;
+
+  PageTable           = 0;
+  PageTableBufferSize = 0;
+
+  Status = PageTableMap (
+             &PageTable,
+             Paging4Level,
+             NULL,
+             &PageTableBufferSize,
+             Address,
+             Length,
+             &MapAttribute,
+             &MapMask
+             );
+  ASSERT (Status == EFI_BUFFER_TOO_SMALL);
+  DEBUG ((DEBUG_INFO, "AP Page Table Buffer Size = %x\n", PageTableBufferSize));
+
+  PageTableBuffer = AllocateReservedPages (EFI_SIZE_TO_PAGES (PageTableBufferSize));
+  ASSERT (PageTableBuffer != NULL);
+  Status = PageTableMap (
+             &PageTable,
+             Paging4Level,
+             PageTableBuffer,
+             &PageTableBufferSize,
+             Address,
+             Length,
+             &MapAttribute,
+             &MapMask
+             );
+  ASSERT_EFI_ERROR (Status);
+  return PageTable;
+}

UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm

@@ -279,120 +279,42 @@ CProcedureInvoke:
 RendezvousFunnelProcEnd:
 
 ;-------------------------------------------------------------------------------------
-;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
+;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3);
+;  This function is called during the finalizaiton of Mp initialization before booting
+;  to OS, and aim to put Aps either in Mwait or HLT.
 ;-------------------------------------------------------------------------------------
+; +----------------+
+; | Cr3            |  rsp+40
+; +----------------+
+; | CountTofinish  |  r9
+; +----------------+
+; | TopOfApStack   |  r8
+; +----------------+
+; | ApTargetCState |  rdx
+; +----------------+
+; | MwaitSupport   |  rcx
+; +----------------+
+; | the return     |
+; +----------------+ low address
+
 AsmRelocateApLoopStart:
-BITS 64
+    mov        rax, r9           ; CountTofinish
-    cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable
-    je         NoSevEs
-
-    ;
-    ; Perform some SEV-ES related setup before leaving 64-bit mode
-    ;
-    push       rcx
-    push       rdx
-
-    ;
-    ; Get the RDX reset value using CPUID
-    ;
-    mov        rax, 1
-    cpuid
-    mov        rsi, rax          ; Save off the reset value for RDX
-
-    ;
-    ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
-    ;   - Must be done while in 64-bit long mode so that writes to
-    ;     the GHCB memory will be unencrypted.
-    ;   - No NAE events can be generated once this is set otherwise
-    ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.
-    ;
-    mov        rcx, 0xc0010130
-    rdmsr                        ; Retrieve current GHCB address
-    shl        rdx, 32
-    or         rdx, rax
-
-    mov        rdi, rdx
-    xor        rax, rax
-    mov        rcx, 0x800
-    shr        rcx, 3
-    rep stosq                    ; Clear the GHCB
-
-    mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD
-    mov        [rdx + 0x390], rax
-    mov        rax, 114          ; Set SwExitCode valid bit
-    bts        [rdx + 0x3f0], rax
-    inc        rax               ; Set SwExitInfo1 valid bit
-    bts        [rdx + 0x3f0], rax
-    inc        rax               ; Set SwExitInfo2 valid bit
-    bts        [rdx + 0x3f0], rax
-
-    pop        rdx
-    pop        rcx
-
-NoSevEs:
-    cli                          ; Disable interrupt before switching to 32-bit mode
-    mov        rax, [rsp + 40]   ; CountTofinish
     lock dec   dword [rax]       ; (*CountTofinish)--
 
-    mov        r10, [rsp + 48]   ; Pm16CodeSegment
+    mov        rax, [rsp + 40]    ; Cr3
-    mov        rax, [rsp + 56]   ; SevEsAPJumpTable
+    ; Do not push on old stack, since old stack is not mapped
-    mov        rbx, [rsp + 64]   ; WakeupBuffer
+    ; in the page table pointed by cr3
-    mov        rsp, r9           ; TopOfApStack
+    mov        cr3, rax
-
+    mov        rsp, r8            ; TopOfApStack
-    push       rax               ; Save SevEsAPJumpTable
-    push       rbx               ; Save WakeupBuffer
-    push       r10               ; Save Pm16CodeSegment
-    push       rcx               ; Save MwaitSupport
-    push       rdx               ; Save ApTargetCState
-
-    lea        rax, [PmEntry]    ; rax <- The start address of transition code
-
-    push       r8
-    push       rax
-
-    ;
-    ; Clear R8 - R15, for reset, before going into 32-bit mode
-    ;
-    xor        r8, r8
-    xor        r9, r9
-    xor        r10, r10
-    xor        r11, r11
-    xor        r12, r12
-    xor        r13, r13
-    xor        r14, r14
-    xor        r15, r15
-
-    ;
-    ; Far return into 32-bit mode
-    ;
-    retfq
-
-BITS 32
-PmEntry:
-    mov        eax, cr0
-    btr        eax, 31           ; Clear CR0.PG
-    mov        cr0, eax          ; Disable paging and caches
-
-    mov        ecx, 0xc0000080
-    rdmsr
-    and        ah, ~ 1           ; Clear LME
-    wrmsr
-    mov        eax, cr4
-    and        al, ~ (1 << 5)    ; Clear PAE
-    mov        cr4, eax
-
-    pop        edx
-    add        esp, 4
-    pop        ecx,
-    add        esp, 4
 
 MwaitCheck:
     cmp        cl, 1              ; Check mwait-monitor support
     jnz        HltLoop
-    mov        ebx, edx           ; Save C-State to ebx
+    mov        rbx, rdx           ; Save C-State to ebx
+
 MwaitLoop:
     cli
-    mov        eax, esp           ; Set Monitor Address
+    mov        rax, rsp           ; Set Monitor Address
     xor        ecx, ecx           ; ecx = 0
     xor        edx, edx           ; edx = 0
     monitor
@@ -402,49 +324,10 @@ MwaitLoop:
     jmp        MwaitLoop
 
 HltLoop:
-    pop        edx                ; PM16CodeSegment
-    add        esp, 4
-    pop        ebx                ; WakeupBuffer
-    add        esp, 4
-    pop        eax                ; SevEsAPJumpTable
-    add        esp, 4
-    cmp        eax, 0             ; Check for SEV-ES
-    je         DoHlt
-
-    cli
-    ;
-    ; SEV-ES is enabled, use VMGEXIT (GHCB information already
-    ; set by caller)
-    ;
-BITS 64
-    rep        vmmcall
-BITS 32
-
-    ;
-    ; Back from VMGEXIT AP_HLT_LOOP
-    ;   Push the FLAGS/CS/IP values to use
-    ;
-    push       word 0x0002        ; EFLAGS
-    xor        ecx, ecx
-    mov        cx, [eax + 2]      ; CS
-    push       cx
-    mov        cx, [eax]          ; IP
-    push       cx
-    push       word 0x0000        ; For alignment, will be discarded
-
-    push       edx
-    push       ebx
-
-    mov        edx, esi           ; Restore RDX reset value
-
-    retf
-
-DoHlt:
     cli
     hlt
-    jmp        DoHlt
+    jmp        HltLoop
 
-BITS 64
 AsmRelocateApLoopEnd:
 
 ;-------------------------------------------------------------------------------------

UefiCpuPkg/UefiCpuPkg.dsc

@@ -94,6 +94,7 @@
   MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf
   HobLib|MdePkg/Library/DxeHobLib/DxeHobLib.inf
   CpuExceptionHandlerLib|UefiCpuPkg/Library/CpuExceptionHandlerLib/DxeCpuExceptionHandlerLib.inf
+  CpuPageTableLib|UefiCpuPkg/Library/CpuPageTableLib/CpuPageTableLib.inf
   MpInitLib|UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
   RegisterCpuFeaturesLib|UefiCpuPkg/Library/RegisterCpuFeaturesLib/DxeRegisterCpuFeaturesLib.inf
   CpuCacheInfoLib|UefiCpuPkg/Library/CpuCacheInfoLib/DxeCpuCacheInfoLib.inf