MdeModulePkg/Bus/Pci/PciBusDxe: Support PCIe Resizable BAR Capability

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=313

Add PcdPcieResizableBarSupport to enable/disable PCIe Resizable
BAR Capability fearture.
Program the Resizable BAR Register if the device suports PCIe Resizable
BAR Capability and PcdPcieResizableBarSupport is TRUE.

Cc: Ray Ni <ray.ni@intel.com>
Cc: Hao A Wu <hao.a.wu@intel.com>
Signed-off-by: Heng Luo <heng.luo@intel.com>
Reviewed-by: Ray Ni <ray.ni@intel.com>

MdeModulePkg/Bus/Pci/PciBusDxe/PciBus.h

@@ -1,7 +1,7 @@
 /** @file
   Header files and data structures needed by PCI Bus module.
 
-Copyright (c) 2006 - 2019, Intel Corporation. All rights reserved.<BR>
+Copyright (c) 2006 - 2021, Intel Corporation. All rights reserved.<BR>
 SPDX-License-Identifier: BSD-2-Clause-Patent
 
 **/
@@ -280,6 +280,8 @@ struct _PCI_IO_DEVICE {
   // This field is used to support this case.
   //
   UINT16                                    BridgeIoAlignment;
+  UINT32                                    ResizableBarOffset;
+  UINT32                                    ResizableBarNumber;
 };
 
 #define PCI_IO_DEVICE_FROM_PCI_IO_THIS(a) \

MdeModulePkg/Bus/Pci/PciBusDxe/PciBusDxe.inf

@@ -2,7 +2,7 @@
 #  The PCI bus driver will probe all PCI devices and allocate MMIO and IO space for these devices.
 #  Please use PCD feature flag PcdPciBusHotplugDeviceSupport to enable hot plug supporting.
 #
-#  Copyright (c) 2006 - 2019, Intel Corporation. All rights reserved.<BR>
+#  Copyright (c) 2006 - 2021, Intel Corporation. All rights reserved.<BR>
 #
 #  SPDX-License-Identifier: BSD-2-Clause-Patent
 #
@@ -106,6 +106,7 @@
   gEfiMdeModulePkgTokenSpaceGuid.PcdAriSupport                  ## CONSUMES
   gEfiMdeModulePkgTokenSpaceGuid.PcdMrIovSupport                ## CONSUMES
   gEfiMdeModulePkgTokenSpaceGuid.PcdPciDisableBusEnumeration    ## SOMETIMES_CONSUMES
+  gEfiMdeModulePkgTokenSpaceGuid.PcdPcieResizableBarSupport     ## CONSUMES
 
 [UserExtensions.TianoCore."ExtraFiles"]
   PciBusDxeExtra.uni

MdeModulePkg/Bus/Pci/PciBusDxe/PciEnumeratorSupport.c

@@ -1,7 +1,7 @@
 /** @file
   PCI emumeration support functions implementation for PCI Bus module.
 
-Copyright (c) 2006 - 2019, Intel Corporation. All rights reserved.<BR>
+Copyright (c) 2006 - 2021, Intel Corporation. All rights reserved.<BR>
 (C) Copyright 2015 Hewlett Packard Enterprise Development LP<BR>
 SPDX-License-Identifier: BSD-2-Clause-Patent
 
@@ -2426,6 +2426,31 @@ CreatePciIoDevice (
     }
   }
 
+  PciIoDevice->ResizableBarOffset = 0;
+  if (PcdGetBool (PcdPcieResizableBarSupport)) {
+    Status = LocatePciExpressCapabilityRegBlock (
+               PciIoDevice,
+               PCI_EXPRESS_EXTENDED_CAPABILITY_RESIZABLE_BAR_ID,
+               &PciIoDevice->ResizableBarOffset,
+               NULL
+               );
+    if (!EFI_ERROR (Status)) {
+      PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_CONTROL ResizableBarControl;
+      UINT32                                                  Offset;
+      Offset = PciIoDevice->ResizableBarOffset + sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_HEADER)
+                + sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_CAPABILITY),
+      PciIo->Pci.Read (
+              PciIo,
+              EfiPciIoWidthUint8,
+              Offset,
+              sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_CONTROL),
+              &ResizableBarControl
+              );
+      PciIoDevice->ResizableBarNumber = ResizableBarControl.Bits.ResizableBarNumber;
+      PciProgramResizableBar (PciIoDevice, PciResizableBarMax);
+    }
+  }
+
   //
   // Initialize the reserved resource list
   //

MdeModulePkg/Bus/Pci/PciBusDxe/PciEnumeratorSupport.h

@@ -1,7 +1,7 @@
 /** @file
   PCI enumeration support functions declaration for PCI Bus module.
 
-Copyright (c) 2006 - 2019, Intel Corporation. All rights reserved.<BR>
+Copyright (c) 2006 - 2021, Intel Corporation. All rights reserved.<BR>
 SPDX-License-Identifier: BSD-2-Clause-Patent
 
 **/
@@ -467,4 +467,14 @@ DumpPpbPaddingResource (
   IN PCI_BAR_TYPE                     ResourceType
   );
 
+/**
+  Dump the PCI BAR information.
+
+  @param PciIoDevice     PCI IO instance.
+**/
+VOID
+DumpPciBars (
+  IN PCI_IO_DEVICE                    *PciIoDevice
+  );
+
 #endif

MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.c

@@ -1,7 +1,7 @@
 /** @file
   Internal library implementation for PCI Bus module.
 
-Copyright (c) 2006 - 2019, Intel Corporation. All rights reserved.<BR>
+Copyright (c) 2006 - 2021, Intel Corporation. All rights reserved.<BR>
 (C) Copyright 2015 Hewlett Packard Enterprise Development LP<BR>
 SPDX-License-Identifier: BSD-2-Clause-Patent
 
@@ -377,6 +377,60 @@ DumpResourceMap (
   }
 }
 
+/**
+  Adjust the Devices' BAR size to minimum value if it support Resizeable BAR capability.
+
+  @param RootBridgeDev  Pointer to instance of PCI_IO_DEVICE..
+
+  @return TRUE if BAR size is adjusted.
+
+**/
+BOOLEAN
+AdjustPciDeviceBarSize (
+  IN PCI_IO_DEVICE *RootBridgeDev
+  )
+{
+  PCI_IO_DEVICE     *PciIoDevice;
+  LIST_ENTRY        *CurrentLink;
+  BOOLEAN           Adjusted;
+  UINTN             Offset;
+  UINTN             BarIndex;
+
+  Adjusted    = FALSE;
+  CurrentLink = RootBridgeDev->ChildList.ForwardLink;
+
+  while (CurrentLink != NULL && CurrentLink != &RootBridgeDev->ChildList) {
+    PciIoDevice = PCI_IO_DEVICE_FROM_LINK (CurrentLink);
+
+    if (IS_PCI_BRIDGE (&PciIoDevice->Pci)) {
+      if (AdjustPciDeviceBarSize (PciIoDevice)) {
+        Adjusted = TRUE;
+      }
+    } else {
+      if (PciIoDevice->ResizableBarOffset != 0) {
+        DEBUG ((
+          DEBUG_ERROR,
+          "PciBus: [%02x|%02x|%02x] Adjust Pci Device Bar Size\n",
+          PciIoDevice->BusNumber, PciIoDevice->DeviceNumber, PciIoDevice->FunctionNumber
+          ));
+        PciProgramResizableBar (PciIoDevice, PciResizableBarMin);
+        //
+        // Start to parse the bars
+        //
+        for (Offset = 0x10, BarIndex = 0; Offset <= 0x24 && BarIndex < PCI_MAX_BAR; BarIndex++) {
+          Offset = PciParseBar (PciIoDevice, Offset, BarIndex);
+        }
+        Adjusted = TRUE;
+        DEBUG_CODE (DumpPciBars (PciIoDevice););
+      }
+    }
+
+    CurrentLink = CurrentLink->ForwardLink;
+  }
+
+  return Adjusted;
+}
+
 /**
   Submits the I/O and memory resource requirements for the specified PCI Host Bridge.
 
@@ -422,6 +476,10 @@ PciHostBridgeResourceAllocator (
   PCI_RESOURCE_NODE                              PMem64Pool;
   EFI_DEVICE_HANDLE_EXTENDED_DATA_PAYLOAD        HandleExtendedData;
   EFI_RESOURCE_ALLOC_FAILURE_ERROR_DATA_PAYLOAD  AllocFailExtendedData;
+  BOOLEAN                                        ResizableBarNeedAdjust;
+  BOOLEAN                                        ResizableBarAdjusted;
+
+  ResizableBarNeedAdjust = PcdGetBool (PcdPcieResizableBarSupport);
 
   //
   // It may try several times if the resource allocation fails
@@ -703,19 +761,30 @@ PciHostBridgeResourceAllocator (
             sizeof (AllocFailExtendedData)
             );
 
-      Status = PciHostBridgeAdjustAllocation (
+     //
-                 &IoPool,
+     // When resource conflict happens, adjust the BAR size first.
-                 &Mem32Pool,
+     // Only when adjusting BAR size doesn't help or BAR size cannot be adjusted,
-                 &PMem32Pool,
+     // reject the device who requests largest resource that causes conflict.
-                 &Mem64Pool,
+     //
-                 &PMem64Pool,
+      ResizableBarAdjusted = FALSE;
-                 IoResStatus,
+      if (ResizableBarNeedAdjust) {
-                 Mem32ResStatus,
+        ResizableBarAdjusted = AdjustPciDeviceBarSize (RootBridgeDev);
-                 PMem32ResStatus,
+        ResizableBarNeedAdjust = FALSE;
-                 Mem64ResStatus,
+      }
-                 PMem64ResStatus
+      if (!ResizableBarAdjusted) {
-                 );
+        Status = PciHostBridgeAdjustAllocation (
-
+                  &IoPool,
+                  &Mem32Pool,
+                  &PMem32Pool,
+                  &Mem64Pool,
+                  &PMem64Pool,
+                  IoResStatus,
+                  Mem32ResStatus,
+                  PMem32ResStatus,
+                  Mem64ResStatus,
+                  PMem64ResStatus
+                  );
+      }
       //
       // Destroy all the resource tree
       //
@@ -1651,3 +1720,91 @@ PciHostBridgeEnumerator (
 
   return EFI_SUCCESS;
 }
+
+/**
+  This function is used to program the Resizable BAR Register.
+
+  @param PciIoDevice            A pointer to the PCI_IO_DEVICE.
+  @param ResizableBarOp         PciResizableBarMax: Set BAR to max size
+                                PciResizableBarMin: set BAR to min size.
+
+  @retval EFI_SUCCESS           Successfully enumerated the host bridge.
+  @retval other                 Some error occurred when enumerating the host bridge.
+
+**/
+EFI_STATUS
+PciProgramResizableBar (
+  IN PCI_IO_DEVICE                *PciIoDevice,
+  IN PCI_RESIZABLE_BAR_OPERATION  ResizableBarOp
+  )
+{
+  EFI_PCI_IO_PROTOCOL  *PciIo;
+  UINT64                Capabilities;
+  UINT32                Index;
+  UINT32                Offset;
+  INTN                  Bit;
+  UINTN                 ResizableBarNumber;
+  EFI_STATUS            Status;
+  PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_ENTRY   Entries[PCI_MAX_BAR];
+
+  ASSERT (PciIoDevice->ResizableBarOffset != 0);
+
+  DEBUG ((DEBUG_INFO, "   Programs Resizable BAR register, offset: 0x%08x, number: %d\n",
+        PciIoDevice->ResizableBarOffset, PciIoDevice->ResizableBarNumber));
+
+  ResizableBarNumber = MIN (PciIoDevice->ResizableBarNumber, PCI_MAX_BAR);
+  PciIo = &PciIoDevice->PciIo;
+  Status = PciIo->Pci.Read (
+          PciIo,
+          EfiPciIoWidthUint8,
+          PciIoDevice->ResizableBarOffset + sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_HEADER),
+          sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_ENTRY) * ResizableBarNumber,
+          (VOID *)(&Entries)
+          );
+  ASSERT_EFI_ERROR (Status);
+
+  for (Index = 0; Index < ResizableBarNumber; Index++) {
+
+    //
+    // When the bit of Capabilities Set, indicates that the Function supports
+    // operating with the BAR sized to (2^Bit) MB.
+    // Example:
+    // Bit 0 is set: supports operating with the BAR sized to 1 MB
+    // Bit 1 is set: supports operating with the BAR sized to 2 MB
+    // Bit n is set: supports operating with the BAR sized to (2^n) MB
+    //
+    Capabilities = LShiftU64(Entries[Index].ResizableBarControl.Bits.BarSizeCapability, 28)
+                  | Entries[Index].ResizableBarCapability.Bits.BarSizeCapability;
+
+    if (ResizableBarOp == PciResizableBarMax) {
+      Bit = HighBitSet64(Capabilities);
+    } else if (ResizableBarOp == PciResizableBarMin) {
+      Bit = LowBitSet64(Capabilities);
+    } else {
+      ASSERT ((ResizableBarOp == PciResizableBarMax) || (ResizableBarOp == PciResizableBarMin));
+    }
+
+    ASSERT (Bit >= 0);
+
+    Offset = PciIoDevice->ResizableBarOffset + sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_HEADER)
+            + Index * sizeof (PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_ENTRY)
+            + OFFSET_OF (PCI_EXPRESS_EXTENDED_CAPABILITIES_RESIZABLE_BAR_ENTRY, ResizableBarControl);
+
+    Entries[Index].ResizableBarControl.Bits.BarSize = (UINT32) Bit;
+    DEBUG ((
+      DEBUG_INFO,
+      "   Resizable Bar: Offset = 0x%x, Bar Size Capability = 0x%016lx, New Bar Size = 0x%lx\n",
+      OFFSET_OF (PCI_TYPE00, Device.Bar[Entries[Index].ResizableBarControl.Bits.BarIndex]),
+      Capabilities, LShiftU64 (SIZE_1MB, Bit)
+      ));
+    PciIo->Pci.Write (
+            PciIo,
+            EfiPciIoWidthUint32,
+            Offset,
+            1,
+            &Entries[Index].ResizableBarControl.Uint32
+            );
+  }
+
+  return EFI_SUCCESS;
+}

MdeModulePkg/Bus/Pci/PciBusDxe/PciLib.h

@@ -1,7 +1,7 @@
 /** @file
   Internal library declaration for PCI Bus module.
 
-Copyright (c) 2006 - 2011, Intel Corporation. All rights reserved.<BR>
+Copyright (c) 2006 - 2021, Intel Corporation. All rights reserved.<BR>
 SPDX-License-Identifier: BSD-2-Clause-Patent
 
 **/
@@ -24,6 +24,10 @@ typedef struct {
   UINT8                              *AllocRes;
 } EFI_RESOURCE_ALLOC_FAILURE_ERROR_DATA_PAYLOAD;
 
+typedef enum {
+  PciResizableBarMin = 0x00,
+  PciResizableBarMax = 0xFF
+} PCI_RESIZABLE_BAR_OPERATION;
 
 /**
   Retrieve the PCI Card device BAR information via PciIo interface.
@@ -156,4 +160,20 @@ PciHostBridgeEnumerator (
   IN EFI_PCI_HOST_BRIDGE_RESOURCE_ALLOCATION_PROTOCOL  *PciResAlloc
   );
 
+/**
+  This function is used to program the Resizable BAR Register.
+
+  @param PciIoDevice            A pointer to the PCI_IO_DEVICE.
+  @param ResizableBarOp         PciResizableBarMax: Set BAR to max size
+                                PciResizableBarMin: set BAR to min size.
+
+  @retval EFI_SUCCESS           Successfully enumerated the host bridge.
+  @retval other                 Some error occurred when enumerating the host bridge.
+
+**/
+EFI_STATUS
+PciProgramResizableBar (
+  IN PCI_IO_DEVICE                *PciIoDevice,
+  IN PCI_RESIZABLE_BAR_OPERATION  ResizableBarOp
+  );
 #endif

MdeModulePkg/MdeModulePkg.dec

@@ -4,7 +4,7 @@
 # and libraries instances, which are used for those modules.
 #
 # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
-# Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR>
 # Copyright (c) 2016, Linaro Ltd. All rights reserved.<BR>
 # (C) Copyright 2016 - 2019 Hewlett Packard Enterprise Development LP<BR>
 # Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
@@ -2043,6 +2043,12 @@
   # @Prompt Enable StatusCode via memory.
   gEfiMdeModulePkgTokenSpaceGuid.PcdStatusCodeUseMemory|FALSE|BOOLEAN|0x00010023
 
+  ## Indicates if the PCIe Resizable BAR Capability Supported.<BR><BR>
+  #   TRUE  - PCIe Resizable BAR Capability is supported.<BR>
+  #   FALSE - PCIe Resizable BAR Capability is not supported.<BR>
+  # @Prompt Enable PCIe Resizable BAR Capability support.
+  gEfiMdeModulePkgTokenSpaceGuid.PcdPcieResizableBarSupport|TRUE|BOOLEAN|0x10000024
+
 [PcdsPatchableInModule]
   ## Specify memory size with page number for PEI code when
   #  Loading Module at Fixed Address feature is enabled.