ArmVirtualizationPkg/PciHostBridgeDxe: MMIO aperture must not be uncached

Quite non-intuitively, we must allow guest-side writes to emulated PCI
MMIO regions to go through the CPU cache, otherwise QEMU, whose accesses
always go through the cache, may see stale data in the region.

This change makes no difference for QEMU/TCG, but it is important for
QEMU/KVM, at the moment.

Because gDS->SetMemorySpaceAttributes() is ultimately implemented by
EFI_CPU_ARCH_PROTOCOL.SetMemoryAttributes() -- see
"MdeModulePkg/Core/Dxe/Gcd/Gcd.c" and "ArmPkg/Drivers/CpuDxe/" -- we add
the CPU architectural protocol to the module's DepEx.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Olivier Martin <olivier.martin@arm.com>

git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@16904 6f19259b-4bc3-4df7-8a09-765794883524

ArmPlatformPkg/ArmVirtualizationPkg/ArmVirtualizationPkg.dec

@@ -56,3 +56,28 @@
 
   gArmVirtualizationTokenSpaceGuid.PcdFwCfgSelectorAddress|0x0|UINT64|0x00000004
   gArmVirtualizationTokenSpaceGuid.PcdFwCfgDataAddress|0x0|UINT64|0x00000005
+
+[PcdsFeatureFlag]
+  #
+  # "Map PCI MMIO as Cached"
+  #
+  # Due to the way Stage1 and Stage2 mappings are combined on Aarch64, and
+  # because KVM -- for the time being -- does not try to interfere with the
+  # Stage1 mappings, we must not set EFI_MEMORY_UC for emulated PCI MMIO
+  # regions.
+  #
+  # EFI_MEMORY_UC is mapped to Device-nGnRnE, and that Stage1 attribute would
+  # direct guest writes to host DRAM immediately, bypassing the cache
+  # regardless of Stage2 attributes. However, QEMU's reads of the same range
+  # can easily be served from the (stale) CPU cache.
+  #
+  # Setting this PCD to TRUE will use EFI_MEMORY_WB for mapping PCI MMIO
+  # regions, which ensures that guest writes to such regions go through the CPU
+  # cache. Strictly speaking this is wrong, but it is needed as a temporary
+  # workaround for emulated PCI devices. Setting the PCD to FALSE results in
+  # the theoretically correct EFI_MEMORY_UC mapping, and should be the long
+  # term choice, especially with assigned devices.
+  #
+  # The default is to turn off the kludge; DSC's can selectively enable it.
+  #
+  gArmVirtualizationTokenSpaceGuid.PcdKludgeMapPciMmioAsCached|FALSE|BOOLEAN|0x00000006

ArmPlatformPkg/ArmVirtualizationPkg/ArmVirtualizationQemu.dsc

@@ -86,6 +86,9 @@
   #  It could be set FALSE to save size.
   gEfiMdeModulePkgTokenSpaceGuid.PcdConOutGopSupport|FALSE
 
+  # Activate KVM workaround for now.
+  gArmVirtualizationTokenSpaceGuid.PcdKludgeMapPciMmioAsCached|TRUE
+
 [PcdsFixedAtBuild.common]
   gEfiMdePkgTokenSpaceGuid.PcdDebugPrintErrorLevel|0x8000004F
 

ArmPlatformPkg/ArmVirtualizationPkg/PciHostBridgeDxe/PciHostBridge.c

@@ -97,6 +97,7 @@ InitializePciHostBridge (
   IN EFI_SYSTEM_TABLE  *SystemTable
   )
 {
+  UINT64                      MmioAttributes;
   EFI_STATUS                  Status;
   UINTN                       Loop1;
   UINTN                       Loop2;
@@ -133,17 +134,31 @@ InitializePciHostBridge (
                   );
   ASSERT_EFI_ERROR (Status);
 
+  MmioAttributes = FeaturePcdGet (PcdKludgeMapPciMmioAsCached) ?
+                   EFI_MEMORY_WB : EFI_MEMORY_UC;
+
   Status = gDS->AddMemorySpace (
                   EfiGcdMemoryTypeMemoryMappedIo,
                   PcdGet32 (PcdPciMmio32Base),
                   PcdGet32 (PcdPciMmio32Size),
-                  EFI_MEMORY_UC
+                  MmioAttributes
                   );
   if (EFI_ERROR (Status)) {
     DEBUG ((EFI_D_ERROR, "%a: AddMemorySpace: %r\n", __FUNCTION__, Status));
     return Status;
   }
 
+  Status = gDS->SetMemorySpaceAttributes (
+                  PcdGet32 (PcdPciMmio32Base),
+                  PcdGet32 (PcdPciMmio32Size),
+                  MmioAttributes
+                  );
+  if (EFI_ERROR (Status)) {
+    DEBUG ((EFI_D_ERROR, "%a: SetMemorySpaceAttributes: %r\n", __FUNCTION__,
+      Status));
+    return Status;
+  }
+
   //
   // Create Host Bridge Device Handle
   //

ArmPlatformPkg/ArmVirtualizationPkg/PciHostBridgeDxe/PciHostBridgeDxe.inf

@@ -24,6 +24,7 @@
 [Packages]
   MdePkg/MdePkg.dec
   ArmPlatformPkg/ArmPlatformPkg.dec
+  ArmPlatformPkg/ArmVirtualizationPkg/ArmVirtualizationPkg.dec
 
 [LibraryClasses]
   UefiDriverEntryPoint
@@ -60,5 +61,9 @@
   gArmPlatformTokenSpaceGuid.PcdPciMmio32Size
   gEfiMdePkgTokenSpaceGuid.PcdPciExpressBaseAddress
 
+[FeaturePcd]
+  gArmVirtualizationTokenSpaceGuid.PcdKludgeMapPciMmioAsCached
+
 [depex]
-  gEfiMetronomeArchProtocolGuid
+  gEfiMetronomeArchProtocolGuid AND
+  gEfiCpuArchProtocolGuid