diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf index 5832765956..1771575c69 100644 --- a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf +++ b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf @@ -52,6 +52,7 @@ DebugAgentLib SynchronizationLib PcdLib + VmgExitLib [Protocols] gEfiTimerArchProtocolGuid ## SOMETIMES_CONSUMES @@ -72,4 +73,6 @@ gUefiCpuPkgTokenSpaceGuid.PcdCpuApTargetCstate ## SOMETIMES_CONSUMES gUefiCpuPkgTokenSpaceGuid.PcdCpuApStatusCheckIntervalInMicroSeconds ## CONSUMES gUefiCpuPkgTokenSpaceGuid.PcdSevEsIsEnabled ## CONSUMES + gUefiCpuPkgTokenSpaceGuid.PcdSevEsWorkAreaBase ## SOMETIMES_CONSUMES gEfiMdeModulePkgTokenSpaceGuid.PcdCpuStackGuard ## CONSUMES + gEfiMdeModulePkgTokenSpaceGuid.PcdGhcbBase ## CONSUMES diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c index 8ccddf8e9f..9115ff9e3e 100644 --- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c +++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include @@ -144,6 +146,39 @@ GetModeTransitionBuffer ( return (UINTN)StartAddress; } +/** + Return the address of the SEV-ES AP jump table. + + This buffer is required in order for an SEV-ES guest to transition from + UEFI into an OS. + + @return Return SEV-ES AP jump table buffer +**/ +UINTN +GetSevEsAPMemory ( + VOID + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS StartAddress; + + // + // Allocate 1 page for AP jump table page + // + StartAddress = BASE_4GB - 1; + Status = gBS->AllocatePages ( + AllocateMaxAddress, + EfiReservedMemoryType, + 1, + &StartAddress + ); + ASSERT_EFI_ERROR (Status); + + DEBUG ((DEBUG_INFO, "Dxe: SevEsAPMemory = %lx\n", (UINTN) StartAddress)); + + return (UINTN) StartAddress; +} + /** Checks APs status and updates APs status if needed. @@ -218,6 +253,38 @@ CheckApsStatus ( } } +/** + Get Protected mode code segment with 16-bit default addressing + from current GDT table. + + @return Protected mode 16-bit code segment value. +**/ +UINT16 +GetProtectedMode16CS ( + VOID + ) +{ + IA32_DESCRIPTOR GdtrDesc; + IA32_SEGMENT_DESCRIPTOR *GdtEntry; + UINTN GdtEntryCount; + UINT16 Index; + + Index = (UINT16) -1; + AsmReadGdtr (&GdtrDesc); + GdtEntryCount = (GdtrDesc.Limit + 1) / sizeof (IA32_SEGMENT_DESCRIPTOR); + GdtEntry = (IA32_SEGMENT_DESCRIPTOR *) GdtrDesc.Base; + for (Index = 0; Index < GdtEntryCount; Index++) { + if (GdtEntry->Bits.L == 0) { + if (GdtEntry->Bits.Type > 8 && GdtEntry->Bits.DB == 0) { + break; + } + } + GdtEntry++; + } + ASSERT (Index != GdtEntryCount); + return Index * 8; +} + /** Get Protected mode code segment from current GDT table. @@ -238,7 +305,7 @@ GetProtectedModeCS ( GdtEntry = (IA32_SEGMENT_DESCRIPTOR *) GdtrDesc.Base; for (Index = 0; Index < GdtEntryCount; Index++) { if (GdtEntry->Bits.L == 0) { - if (GdtEntry->Bits.Type > 8 && GdtEntry->Bits.L == 0) { + if (GdtEntry->Bits.Type > 8 && GdtEntry->Bits.DB == 1) { break; } } @@ -300,6 +367,7 @@ MpInitChangeApLoopCallback ( CpuMpData = GetCpuMpData (); CpuMpData->PmCodeSegment = GetProtectedModeCS (); + CpuMpData->Pm16CodeSegment = GetProtectedMode16CS (); CpuMpData->ApLoopMode = PcdGet8 (PcdCpuApLoopMode); mNumberToFinish = CpuMpData->CpuCount - 1; WakeUpAP (CpuMpData, TRUE, 0, RelocateApLoop, NULL, TRUE); diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/MpEqu.inc b/UefiCpuPkg/Library/MpInitLib/Ia32/MpEqu.inc index efb1bc2bf7..4f5a7c859a 100644 --- a/UefiCpuPkg/Library/MpInitLib/Ia32/MpEqu.inc +++ b/UefiCpuPkg/Library/MpInitLib/Ia32/MpEqu.inc @@ -19,7 +19,7 @@ CPU_SWITCH_STATE_IDLE equ 0 CPU_SWITCH_STATE_STORED equ 1 CPU_SWITCH_STATE_LOADED equ 2 -LockLocation equ (RendezvousFunnelProcEnd - RendezvousFunnelProcStart) +LockLocation equ (SwitchToRealProcEnd - RendezvousFunnelProcStart) StackStartAddressLocation equ LockLocation + 04h StackSizeLocation equ LockLocation + 08h ApProcedureLocation equ LockLocation + 0Ch diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm b/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm index b74046b76a..309d53bf3b 100644 --- a/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm +++ b/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm @@ -215,6 +215,16 @@ CProcedureInvoke: jmp $ ; Never reach here RendezvousFunnelProcEnd: +;------------------------------------------------------------------------------------- +;SwitchToRealProc procedure follows. +;NOT USED IN 32 BIT MODE. +;------------------------------------------------------------------------------------- +global ASM_PFX(SwitchToRealProc) +ASM_PFX(SwitchToRealProc): +SwitchToRealProcStart: + jmp $ ; Never reach here +SwitchToRealProcEnd: + ;------------------------------------------------------------------------------------- ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish); ;------------------------------------------------------------------------------------- @@ -263,6 +273,11 @@ ASM_PFX(AsmGetAddressMap): mov dword [ebx + 0Ch], AsmRelocateApLoopStart mov dword [ebx + 10h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart mov dword [ebx + 14h], Flat32Start - RendezvousFunnelProcStart + mov dword [ebx + 18h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize + mov dword [ebx + 1Ch], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset + mov dword [ebx + 20h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset + mov dword [ebx + 24h], 0 ; SwitchToRealPM16ModeOffset + mov dword [ebx + 28h], 0 ; SwitchToRealPM16ModeSize popad ret diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.c b/UefiCpuPkg/Library/MpInitLib/MpLib.c index 2a3fbeef35..90416c81b6 100644 --- a/UefiCpuPkg/Library/MpInitLib/MpLib.c +++ b/UefiCpuPkg/Library/MpInitLib/MpLib.c @@ -9,6 +9,9 @@ **/ #include "MpLib.h" +#include +#include +#include EFI_GUID mCpuInitMpLibHobGuid = CPU_INIT_MP_LIB_HOB_GUID; @@ -291,6 +294,14 @@ GetApLoopMode ( // ApLoopMode = ApInHltLoop; } + + if (PcdGetBool (PcdSevEsIsEnabled)) { + // + // For SEV-ES, force AP in Hlt-loop mode in order to use the GHCB + // protocol for starting APs + // + ApLoopMode = ApInHltLoop; + } } if (ApLoopMode != ApInMwaitLoop) { @@ -587,6 +598,112 @@ InitializeApData ( SetApState (&CpuMpData->CpuData[ProcessorNumber], CpuStateIdle); } +/** + Get Protected mode code segment with 16-bit default addressing + from current GDT table. + + @return Protected mode 16-bit code segment value. +**/ +STATIC +UINT16 +GetProtectedMode16CS ( + VOID + ) +{ + IA32_DESCRIPTOR GdtrDesc; + IA32_SEGMENT_DESCRIPTOR *GdtEntry; + UINTN GdtEntryCount; + UINT16 Index; + + Index = (UINT16) -1; + AsmReadGdtr (&GdtrDesc); + GdtEntryCount = (GdtrDesc.Limit + 1) / sizeof (IA32_SEGMENT_DESCRIPTOR); + GdtEntry = (IA32_SEGMENT_DESCRIPTOR *) GdtrDesc.Base; + for (Index = 0; Index < GdtEntryCount; Index++) { + if (GdtEntry->Bits.L == 0 && + GdtEntry->Bits.DB == 0 && + GdtEntry->Bits.Type > 8) { + break; + } + GdtEntry++; + } + ASSERT (Index != GdtEntryCount); + return Index * 8; +} + +/** + Get Protected mode code segment with 32-bit default addressing + from current GDT table. + + @return Protected mode 32-bit code segment value. +**/ +STATIC +UINT16 +GetProtectedMode32CS ( + VOID + ) +{ + IA32_DESCRIPTOR GdtrDesc; + IA32_SEGMENT_DESCRIPTOR *GdtEntry; + UINTN GdtEntryCount; + UINT16 Index; + + Index = (UINT16) -1; + AsmReadGdtr (&GdtrDesc); + GdtEntryCount = (GdtrDesc.Limit + 1) / sizeof (IA32_SEGMENT_DESCRIPTOR); + GdtEntry = (IA32_SEGMENT_DESCRIPTOR *) GdtrDesc.Base; + for (Index = 0; Index < GdtEntryCount; Index++) { + if (GdtEntry->Bits.L == 0 && + GdtEntry->Bits.DB == 1 && + GdtEntry->Bits.Type > 8) { + break; + } + GdtEntry++; + } + ASSERT (Index != GdtEntryCount); + return Index * 8; +} + +/** + Reset an AP when in SEV-ES mode. + + If successful, this function never returns. + + @param[in] Ghcb Pointer to the GHCB + @param[in] CpuMpData Pointer to CPU MP Data + +**/ +STATIC +VOID +MpInitLibSevEsAPReset ( + IN GHCB *Ghcb, + IN CPU_MP_DATA *CpuMpData + ) +{ + UINT16 Code16, Code32; + AP_RESET *APResetFn; + UINTN BufferStart; + UINTN StackStart; + + Code16 = GetProtectedMode16CS (); + Code32 = GetProtectedMode32CS (); + + if (CpuMpData->WakeupBufferHigh != 0) { + APResetFn = (AP_RESET *) (CpuMpData->WakeupBufferHigh + CpuMpData->AddressMap.SwitchToRealNoNxOffset); + } else { + APResetFn = (AP_RESET *) (CpuMpData->MpCpuExchangeInfo->BufferStart + CpuMpData->AddressMap.SwitchToRealOffset); + } + + BufferStart = CpuMpData->MpCpuExchangeInfo->BufferStart; + StackStart = CpuMpData->SevEsAPResetStackStart - + (AP_RESET_STACK_SIZE * GetApicId ()); + + // + // This call never returns. + // + APResetFn (BufferStart, Code16, Code32, StackStart); +} + /** This function will be called from AP reset code if BSP uses WakeUpAP. @@ -648,7 +765,14 @@ ApWakeupFunction ( InitializeApData (CpuMpData, ProcessorNumber, BistData, ApTopOfStack); ApStartupSignalBuffer = CpuMpData->CpuData[ProcessorNumber].StartupApSignal; - InterlockedDecrement ((UINT32 *) &CpuMpData->MpCpuExchangeInfo->NumApsExecuting); + // + // Delay decrementing the APs executing count when SEV-ES is enabled + // to allow the APs to issue an AP_RESET_HOLD before the BSP possibly + // performs another INIT-SIPI-SIPI sequence. + // + if (!CpuMpData->SevEsIsEnabled) { + InterlockedDecrement ((UINT32 *) &CpuMpData->MpCpuExchangeInfo->NumApsExecuting); + } } else { // // Execute AP function if AP is ready @@ -755,7 +879,52 @@ ApWakeupFunction ( // while (TRUE) { DisableInterrupts (); - CpuSleep (); + if (CpuMpData->SevEsIsEnabled) { + MSR_SEV_ES_GHCB_REGISTER Msr; + GHCB *Ghcb; + UINT64 Status; + BOOLEAN DoDecrement; + + if (CpuMpData->InitFlag == ApInitConfig) { + DoDecrement = TRUE; + } + + while (TRUE) { + Msr.GhcbPhysicalAddress = AsmReadMsr64 (MSR_SEV_ES_GHCB); + Ghcb = Msr.Ghcb; + + VmgInit (Ghcb); + + if (DoDecrement) { + DoDecrement = FALSE; + + // + // Perform the delayed decrement just before issuing the first + // VMGEXIT with AP_RESET_HOLD. + // + InterlockedDecrement ((UINT32 *) &CpuMpData->MpCpuExchangeInfo->NumApsExecuting); + } + + Status = VmgExit (Ghcb, SVM_EXIT_AP_RESET_HOLD, 0, 0); + if ((Status == 0) && (Ghcb->SaveArea.SwExitInfo2 != 0)) { + VmgDone (Ghcb); + break; + } + + VmgDone (Ghcb); + } + + // + // Awakened in a new phase? Use the new CpuMpData + // + if (CpuMpData->NewCpuMpData != NULL) { + CpuMpData = CpuMpData->NewCpuMpData; + } + + MpInitLibSevEsAPReset (Ghcb, CpuMpData); + } else { + CpuSleep (); + } CpuPause (); } } @@ -868,6 +1037,9 @@ FillExchangeInfoData ( ExchangeInfo->Enable5LevelPaging = (BOOLEAN) (Cr4.Bits.LA57 == 1); DEBUG ((DEBUG_INFO, "%a: 5-Level Paging = %d\n", gEfiCallerBaseName, ExchangeInfo->Enable5LevelPaging)); + ExchangeInfo->SevEsIsEnabled = CpuMpData->SevEsIsEnabled; + ExchangeInfo->GhcbBase = (UINTN) CpuMpData->GhcbBase; + // // Get the BSP's data of GDT and IDT // @@ -894,8 +1066,9 @@ FillExchangeInfoData ( // EfiBootServicesCode to avoid page fault if NX memory protection is enabled. // if (CpuMpData->WakeupBufferHigh != 0) { - Size = CpuMpData->AddressMap.RendezvousFunnelSize - - CpuMpData->AddressMap.ModeTransitionOffset; + Size = CpuMpData->AddressMap.RendezvousFunnelSize + + CpuMpData->AddressMap.SwitchToRealSize - + CpuMpData->AddressMap.ModeTransitionOffset; CopyMem ( (VOID *)CpuMpData->WakeupBufferHigh, CpuMpData->AddressMap.RendezvousFunnelAddress + @@ -948,7 +1121,8 @@ BackupAndPrepareWakeupBuffer( CopyMem ( (VOID *) CpuMpData->WakeupBuffer, (VOID *) CpuMpData->AddressMap.RendezvousFunnelAddress, - CpuMpData->AddressMap.RendezvousFunnelSize + CpuMpData->AddressMap.RendezvousFunnelSize + + CpuMpData->AddressMap.SwitchToRealSize ); } @@ -969,6 +1143,44 @@ RestoreWakeupBuffer( ); } +/** + Calculate the size of the reset stack. + + @return Total amount of memory required for stacks +**/ +STATIC +UINTN +GetApResetStackSize ( + VOID + ) +{ + return AP_RESET_STACK_SIZE * PcdGet32(PcdCpuMaxLogicalProcessorNumber); +} + +/** + Calculate the size of the reset vector. + + @param[in] AddressMap The pointer to Address Map structure. + + @return Total amount of memory required for the AP reset area +**/ +STATIC +UINTN +GetApResetVectorSize ( + IN MP_ASSEMBLY_ADDRESS_MAP *AddressMap + ) +{ + UINTN Size; + + Size = ALIGN_VALUE (AddressMap->RendezvousFunnelSize + + AddressMap->SwitchToRealSize + + sizeof (MP_CPU_EXCHANGE_INFO), + CPU_STACK_ALIGNMENT); + Size += GetApResetStackSize (); + + return Size; +} + /** Allocate reset vector buffer. @@ -982,16 +1194,22 @@ AllocateResetVector ( UINTN ApResetVectorSize; if (CpuMpData->WakeupBuffer == (UINTN) -1) { - ApResetVectorSize = CpuMpData->AddressMap.RendezvousFunnelSize + - sizeof (MP_CPU_EXCHANGE_INFO); + ApResetVectorSize = GetApResetVectorSize (&CpuMpData->AddressMap); CpuMpData->WakeupBuffer = GetWakeupBuffer (ApResetVectorSize); CpuMpData->MpCpuExchangeInfo = (MP_CPU_EXCHANGE_INFO *) (UINTN) - (CpuMpData->WakeupBuffer + CpuMpData->AddressMap.RendezvousFunnelSize); + (CpuMpData->WakeupBuffer + + CpuMpData->AddressMap.RendezvousFunnelSize + + CpuMpData->AddressMap.SwitchToRealSize); CpuMpData->WakeupBufferHigh = GetModeTransitionBuffer ( - CpuMpData->AddressMap.RendezvousFunnelSize - + CpuMpData->AddressMap.RendezvousFunnelSize + + CpuMpData->AddressMap.SwitchToRealSize - CpuMpData->AddressMap.ModeTransitionOffset ); + // + // The reset stack starts at the end of the buffer. + // + CpuMpData->SevEsAPResetStackStart = CpuMpData->WakeupBuffer + ApResetVectorSize; } BackupAndPrepareWakeupBuffer (CpuMpData); } @@ -1006,7 +1224,80 @@ FreeResetVector ( IN CPU_MP_DATA *CpuMpData ) { - RestoreWakeupBuffer (CpuMpData); + // + // If SEV-ES is enabled, the reset area is needed for AP parking and + // and AP startup in the OS, so the reset area is reserved. Do not + // perform the restore as this will overwrite memory which has data + // needed by SEV-ES. + // + if (!CpuMpData->SevEsIsEnabled) { + RestoreWakeupBuffer (CpuMpData); + } +} + +/** + Allocate the SEV-ES AP jump table buffer. + + @param[in, out] CpuMpData The pointer to CPU MP Data structure. +**/ +VOID +AllocateSevEsAPMemory ( + IN OUT CPU_MP_DATA *CpuMpData + ) +{ + if (CpuMpData->SevEsAPBuffer == (UINTN) -1) { + CpuMpData->SevEsAPBuffer = + CpuMpData->SevEsIsEnabled ? GetSevEsAPMemory () : 0; + } +} + +/** + Program the SEV-ES AP jump table buffer. + + @param[in] SipiVector The SIPI vector used for the AP Reset +**/ +VOID +SetSevEsJumpTable ( + IN UINTN SipiVector + ) +{ + SEV_ES_AP_JMP_FAR *JmpFar; + UINT32 Offset, InsnByte; + UINT8 LoNib, HiNib; + + JmpFar = (SEV_ES_AP_JMP_FAR *) FixedPcdGet32 (PcdSevEsWorkAreaBase); + ASSERT (JmpFar != NULL); + + // + // Obtain the address of the Segment/Rip location in the workarea. + // This will be set to a value derived from the SIPI vector and will + // be the memory address used for the far jump below. + // + Offset = FixedPcdGet32 (PcdSevEsWorkAreaBase); + Offset += sizeof (JmpFar->InsnBuffer); + LoNib = (UINT8) Offset; + HiNib = (UINT8) (Offset >> 8); + + // + // Program the workarea (which is the initial AP boot address) with + // far jump to the SIPI vector (where XX and YY represent the + // address of where the SIPI vector is stored. + // + // JMP FAR [CS:XXYY] => 2E FF 2E YY XX + // + InsnByte = 0; + JmpFar->InsnBuffer[InsnByte++] = 0x2E; // CS override prefix + JmpFar->InsnBuffer[InsnByte++] = 0xFF; // JMP (FAR) + JmpFar->InsnBuffer[InsnByte++] = 0x2E; // ModRM (JMP memory location) + JmpFar->InsnBuffer[InsnByte++] = LoNib; // YY offset ... + JmpFar->InsnBuffer[InsnByte++] = HiNib; // XX offset ... + + // + // Program the Segment/Rip based on the SIPI vector (always at least + // 16-byte aligned, so Rip is set to 0). + // + JmpFar->Rip = 0; + JmpFar->Segment = (UINT16) (SipiVector >> 4); } /** @@ -1043,6 +1334,7 @@ WakeUpAP ( CpuMpData->InitFlag != ApInitDone) { ResetVectorRequired = TRUE; AllocateResetVector (CpuMpData); + AllocateSevEsAPMemory (CpuMpData); FillExchangeInfoData (CpuMpData); SaveLocalApicTimerSetting (CpuMpData); } @@ -1079,6 +1371,15 @@ WakeUpAP ( } } if (ResetVectorRequired) { + // + // For SEV-ES, the initial AP boot address will be defined by + // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address + // from the original INIT-SIPI-SIPI. + // + if (CpuMpData->SevEsIsEnabled) { + SetSevEsJumpTable (ExchangeInfo->BufferStart); + } + // // Wakeup all APs // @@ -1170,6 +1471,16 @@ WakeUpAP ( *(UINT32 *) CpuData->StartupApSignal = WAKEUP_AP_SIGNAL; if (ResetVectorRequired) { CpuInfoInHob = (CPU_INFO_IN_HOB *) (UINTN) CpuMpData->CpuInfoInHob; + + // + // For SEV-ES, the initial AP boot address will be defined by + // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address + // from the original INIT-SIPI-SIPI. + // + if (CpuMpData->SevEsIsEnabled) { + SetSevEsJumpTable (ExchangeInfo->BufferStart); + } + SendInitSipiSipi ( CpuInfoInHob[ProcessorNumber].ApicId, (UINT32) ExchangeInfo->BufferStart @@ -1646,7 +1957,7 @@ MpInitLibInitialize ( ASSERT (MaxLogicalProcessorNumber != 0); AsmGetAddressMap (&AddressMap); - ApResetVectorSize = AddressMap.RendezvousFunnelSize + sizeof (MP_CPU_EXCHANGE_INFO); + ApResetVectorSize = GetApResetVectorSize (&AddressMap); ApStackSize = PcdGet32(PcdCpuApStackSize); ApLoopMode = GetApLoopMode (&MonitorFilterSize); @@ -1705,6 +2016,8 @@ MpInitLibInitialize ( CpuMpData->CpuInfoInHob = (UINT64) (UINTN) (CpuMpData->CpuData + MaxLogicalProcessorNumber); InitializeSpinLock(&CpuMpData->MpLock); CpuMpData->SevEsIsEnabled = PcdGetBool (PcdSevEsIsEnabled); + CpuMpData->SevEsAPBuffer = (UINTN) -1; + CpuMpData->GhcbBase = PcdGet64 (PcdGhcbBase); // // Make sure no memory usage outside of the allocated buffer. @@ -1763,6 +2076,7 @@ MpInitLibInitialize ( // APs have been wakeup before, just get the CPU Information // from HOB // + OldCpuMpData->NewCpuMpData = CpuMpData; CpuMpData->CpuCount = OldCpuMpData->CpuCount; CpuMpData->BspNumber = OldCpuMpData->BspNumber; CpuMpData->CpuInfoInHob = OldCpuMpData->CpuInfoInHob; diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h b/UefiCpuPkg/Library/MpInitLib/MpLib.h index 5b46c295b6..b1a9d99cb3 100644 --- a/UefiCpuPkg/Library/MpInitLib/MpLib.h +++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h @@ -173,6 +173,11 @@ typedef struct { UINT8 *RelocateApLoopFuncAddress; UINTN RelocateApLoopFuncSize; UINTN ModeTransitionOffset; + UINTN SwitchToRealSize; + UINTN SwitchToRealOffset; + UINTN SwitchToRealNoNxOffset; + UINTN SwitchToRealPM16ModeOffset; + UINTN SwitchToRealPM16ModeSize; } MP_ASSEMBLY_ADDRESS_MAP; typedef struct _CPU_MP_DATA CPU_MP_DATA; @@ -211,6 +216,8 @@ typedef struct { // Enable5LevelPaging indicates whether 5-level paging is enabled in long mode. // BOOLEAN Enable5LevelPaging; + BOOLEAN SevEsIsEnabled; + UINTN GhcbBase; } MP_CPU_EXCHANGE_INFO; #pragma pack() @@ -257,6 +264,7 @@ struct _CPU_MP_DATA { UINT8 ApLoopMode; UINT8 ApTargetCState; UINT16 PmCodeSegment; + UINT16 Pm16CodeSegment; CPU_AP_DATA *CpuData; volatile MP_CPU_EXCHANGE_INFO *MpCpuExchangeInfo; @@ -278,8 +286,47 @@ struct _CPU_MP_DATA { BOOLEAN WakeUpByInitSipiSipi; BOOLEAN SevEsIsEnabled; + UINTN SevEsAPBuffer; + UINTN SevEsAPResetStackStart; + CPU_MP_DATA *NewCpuMpData; + + UINT64 GhcbBase; }; +#define AP_RESET_STACK_SIZE 64 + +#pragma pack(1) + +typedef struct { + UINT8 InsnBuffer[8]; + UINT16 Rip; + UINT16 Segment; +} SEV_ES_AP_JMP_FAR; + +#pragma pack() + +/** + Assembly code to move an AP from long mode to real mode. + + Move an AP from long mode to real mode in preparation to invoking + the reset vector. This is used for SEV-ES guests where a hypervisor + is not allowed to set the CS and RIP to point to the reset vector. + + @param[in] BufferStart The reset vector target. + @param[in] Code16 16-bit protected mode code segment value. + @param[in] Code32 32-bit protected mode code segment value. + @param[in] StackStart The start of a stack to be used for transitioning + from long mode to real mode. +**/ +typedef +VOID +(EFIAPI AP_RESET) ( + IN UINTN BufferStart, + IN UINT16 Code16, + IN UINT16 Code32, + IN UINTN StackStart + ); + extern EFI_GUID mCpuInitMpLibHobGuid; /** @@ -385,6 +432,19 @@ GetModeTransitionBuffer ( IN UINTN BufferSize ); +/** + Return the address of the SEV-ES AP jump table. + + This buffer is required in order for an SEV-ES guest to transition from + UEFI into an OS. + + @return Return SEV-ES AP jump table buffer +**/ +UINTN +GetSevEsAPMemory ( + VOID + ); + /** This function will be called by BSP to wakeup AP. diff --git a/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf b/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf index 4b3d39fbf3..34abf25d43 100644 --- a/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf +++ b/UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf @@ -51,6 +51,7 @@ SynchronizationLib PeiServicesLib PcdLib + VmgExitLib [Pcd] gUefiCpuPkgTokenSpaceGuid.PcdCpuMaxLogicalProcessorNumber ## CONSUMES @@ -62,6 +63,8 @@ gUefiCpuPkgTokenSpaceGuid.PcdCpuApLoopMode ## CONSUMES gUefiCpuPkgTokenSpaceGuid.PcdCpuApTargetCstate ## SOMETIMES_CONSUMES gUefiCpuPkgTokenSpaceGuid.PcdSevEsIsEnabled ## CONSUMES + gUefiCpuPkgTokenSpaceGuid.PcdSevEsWorkAreaBase ## SOMETIMES_CONSUMES + gEfiMdeModulePkgTokenSpaceGuid.PcdGhcbBase ## CONSUMES [Ppis] gEdkiiPeiShadowMicrocodePpiGuid ## SOMETIMES_CONSUMES diff --git a/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c b/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c index a548fed23f..3989bd6a7a 100644 --- a/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c +++ b/UefiCpuPkg/Library/MpInitLib/PeiMpLib.c @@ -280,6 +280,25 @@ GetModeTransitionBuffer ( return 0; } +/** + Return the address of the SEV-ES AP jump table. + + This buffer is required in order for an SEV-ES guest to transition from + UEFI into an OS. + + @return Return SEV-ES AP jump table buffer +**/ +UINTN +GetSevEsAPMemory ( + VOID + ) +{ + // + // PEI phase doesn't need to do such transition. So simply return 0. + // + return 0; +} + /** Checks APs status and updates APs status if needed. diff --git a/UefiCpuPkg/Library/MpInitLib/X64/MpEqu.inc b/UefiCpuPkg/Library/MpInitLib/X64/MpEqu.inc index 58ef369342..c92daaaffd 100644 --- a/UefiCpuPkg/Library/MpInitLib/X64/MpEqu.inc +++ b/UefiCpuPkg/Library/MpInitLib/X64/MpEqu.inc @@ -19,7 +19,7 @@ CPU_SWITCH_STATE_IDLE equ 0 CPU_SWITCH_STATE_STORED equ 1 CPU_SWITCH_STATE_LOADED equ 2 -LockLocation equ (RendezvousFunnelProcEnd - RendezvousFunnelProcStart) +LockLocation equ (SwitchToRealProcEnd - RendezvousFunnelProcStart) StackStartAddressLocation equ LockLocation + 08h StackSizeLocation equ LockLocation + 10h ApProcedureLocation equ LockLocation + 18h @@ -41,3 +41,5 @@ ModeTransitionSegmentLocation equ LockLocation + 98h ModeHighMemoryLocation equ LockLocation + 9Ah ModeHighSegmentLocation equ LockLocation + 9Eh Enable5LevelPagingLocation equ LockLocation + 0A0h +SevEsIsEnabledLocation equ LockLocation + 0A1h +GhcbBaseLocation equ LockLocation + 0A2h diff --git a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm index 87f2523e85..6956b408d0 100644 --- a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm +++ b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm @@ -184,9 +184,97 @@ Releaselock: add edi, StackStartAddressLocation add rax, qword [edi] mov rsp, rax + + lea edi, [esi + SevEsIsEnabledLocation] + cmp byte [edi], 1 ; SevEsIsEnabled + jne CProcedureInvoke + + ; + ; program GHCB + ; Each page after the GHCB is a per-CPU page, so the calculation programs + ; a GHCB to be every 8KB. + ; + mov eax, SIZE_4KB + shl eax, 1 ; EAX = SIZE_4K * 2 + mov ecx, ebx + mul ecx ; EAX = SIZE_4K * 2 * CpuNumber + mov edi, esi + add edi, GhcbBaseLocation + add rax, qword [edi] + mov rdx, rax + shr rdx, 32 + mov rcx, 0xc0010130 + wrmsr jmp CProcedureInvoke GetApicId: + lea edi, [esi + SevEsIsEnabledLocation] + cmp byte [edi], 1 ; SevEsIsEnabled + jne DoCpuid + + ; + ; Since we don't have a stack yet, we can't take a #VC + ; exception. Use the GHCB protocol to perform the CPUID + ; calls. + ; + mov rcx, 0xc0010130 + rdmsr + shl rdx, 32 + or rax, rdx + mov rdi, rax ; RDI now holds the original GHCB GPA + + mov rdx, 0 ; CPUID function 0 + mov rax, 0 ; RAX register requested + or rax, 4 + wrmsr + rep vmmcall + rdmsr + cmp edx, 0bh + jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY + + mov rdx, 0bh ; CPUID function 0x0b + mov rax, 040000000h ; RBX register requested + or rax, 4 + wrmsr + rep vmmcall + rdmsr + test edx, 0ffffh + jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero + + mov rdx, 0bh ; CPUID function 0x0b + mov rax, 0c0000000h ; RDX register requested + or rax, 4 + wrmsr + rep vmmcall + rdmsr + + ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX + jmp RestoreGhcb + +NoX2ApicSevEs: + ; Processor is not x2APIC capable, so get 8-bit APIC ID + mov rdx, 1 ; CPUID function 1 + mov rax, 040000000h ; RBX register requested + or rax, 4 + wrmsr + rep vmmcall + rdmsr + shr edx, 24 + +RestoreGhcb: + mov rbx, rdx ; Save x2APIC/APIC ID + + mov rdx, rdi ; RDI holds the saved GHCB GPA + shr rdx, 32 + mov eax, edi + wrmsr + + mov rdx, rbx + + ; x2APIC ID or APIC ID is in EDX + jmp GetProcessorNumber + +DoCpuid: mov eax, 0 cpuid cmp eax, 0bh @@ -253,12 +341,158 @@ CProcedureInvoke: RendezvousFunnelProcEnd: +;------------------------------------------------------------------------------------- +;SwitchToRealProc procedure follows. +;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC +;IS IN MACHINE CODE. +; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart) +; rcx - Buffer Start +; rdx - Code16 Selector Offset +; r8 - Code32 Selector Offset +; r9 - Stack Start +;------------------------------------------------------------------------------------- +global ASM_PFX(SwitchToRealProc) +ASM_PFX(SwitchToRealProc): +SwitchToRealProcStart: +BITS 64 + cli + + ; + ; Get RDX reset value before changing stacks since the + ; new stack won't be able to accomodate a #VC exception. + ; + push rax + push rbx + push rcx + push rdx + + mov rax, 1 + cpuid + mov rsi, rax ; Save off the reset value for RDX + + pop rdx + pop rcx + pop rbx + pop rax + + ; + ; Establish stack below 1MB + ; + mov rsp, r9 + + ; + ; Push ultimate Reset Vector onto the stack + ; + mov rax, rcx + shr rax, 4 + push word 0x0002 ; RFLAGS + push ax ; CS + push word 0x0000 ; RIP + push word 0x0000 ; For alignment, will be discarded + + ; + ; Get address of "16-bit operand size" label + ; + lea rbx, [PM16Mode] + + ; + ; Push addresses used to change to compatibility mode + ; + lea rax, [CompatMode] + push r8 + push rax + + ; + ; Clear R8 - R15, for reset, before going into 32-bit mode + ; + xor r8, r8 + xor r9, r9 + xor r10, r10 + xor r11, r11 + xor r12, r12 + xor r13, r13 + xor r14, r14 + xor r15, r15 + + ; + ; Far return into 32-bit mode + ; +o64 retf + +BITS 32 +CompatMode: + ; + ; Set up stack to prepare for exiting protected mode + ; + push edx ; Code16 CS + push ebx ; PM16Mode label address + + ; + ; Disable paging + ; + mov eax, cr0 ; Read CR0 + btr eax, 31 ; Set PG=0 + mov cr0, eax ; Write CR0 + + ; + ; Disable long mode + ; + mov ecx, 0c0000080h ; EFER MSR number + rdmsr ; Read EFER + btr eax, 8 ; Set LME=0 + wrmsr ; Write EFER + + ; + ; Disable PAE + ; + mov eax, cr4 ; Read CR4 + btr eax, 5 ; Set PAE=0 + mov cr4, eax ; Write CR4 + + mov edx, esi ; Restore RDX reset value + + ; + ; Switch to 16-bit operand size + ; + retf + +BITS 16 + ; + ; At entry to this label + ; - RDX will have its reset value + ; - On the top of the stack + ; - Alignment data (two bytes) to be discarded + ; - IP for Real Mode (two bytes) + ; - CS for Real Mode (two bytes) + ; +PM16Mode: + mov eax, cr0 ; Read CR0 + btr eax, 0 ; Set PE=0 + mov cr0, eax ; Write CR0 + + pop ax ; Discard alignment data + + ; + ; Clear registers (except RDX and RSP) before going into 16-bit mode + ; + xor eax, eax + xor ebx, ebx + xor ecx, ecx + xor esi, esi + xor edi, edi + xor ebp, ebp + + iret + +SwitchToRealProcEnd: + ;------------------------------------------------------------------------------------- ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish); ;------------------------------------------------------------------------------------- global ASM_PFX(AsmRelocateApLoop) ASM_PFX(AsmRelocateApLoop): AsmRelocateApLoopStart: +BITS 64 cli ; Disable interrupt before switching to 32-bit mode mov rax, [rsp + 40] ; CountTofinish lock dec dword [rax] ; (*CountTofinish)-- @@ -324,6 +558,11 @@ ASM_PFX(AsmGetAddressMap): mov qword [rcx + 18h], rax mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart + mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize + mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset + mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset + mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset + mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize ret ;------------------------------------------------------------------------------------- diff --git a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmFuncsArch.c b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmFuncsArch.c index 6298571e29..28f8e8e133 100644 --- a/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmFuncsArch.c +++ b/UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmFuncsArch.c @@ -121,7 +121,7 @@ GetProtectedModeCS ( GdtEntry = (IA32_SEGMENT_DESCRIPTOR *) GdtrDesc.Base; for (Index = 0; Index < GdtEntryCount; Index++) { if (GdtEntry->Bits.L == 0) { - if (GdtEntry->Bits.Type > 8 && GdtEntry->Bits.L == 0) { + if (GdtEntry->Bits.Type > 8 && GdtEntry->Bits.DB == 1) { break; } }