IOMMU
- https://luohao-brian.gitbooks.io/interrupt-virtualization/content/vt-d-interrupt-remapping-fen-xi.html
- https://xillybus.com/tutorials/iommu-swiotlb-linux
看看文档
- https://www.amd.com/content/dam/amd/en/documents/processor-tech-docs/specifications/48882_IOMMU.pdf
- https://events19.linuxfoundation.cn/wp-content/uploads/2017/11/Shared-Virtual-Addressing_Yisheng-Xie-_-Bob-Liu.pdf
- https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/virtualization_deployment_and_administration_guide/app-iommu
- redhat 的这个应该也是不错的
https://docs.kernel.org/arch/x86/iommu.html
Some ACPI Keywords:
DMAR - Intel DMA Remapping table DRHD - Intel DMA Remapping Hardware Unit Definition RMRR - Intel Reserved Memory Region Reporting Structure IVRS - AMD I/O Virtualization Reporting Structure IVDB - AMD I/O Virtualization Definition Block IVHD - AMD I/O Virtualization Hardware Definition
为什么会和 ACPI 关联起来。
[ ] 如果 iommu=on，那么是不是自动意味着 intel_iommu=on 啊
这个应该重新读读，最近更新了
https://docs.kernel.org/driver-api/vfio.html
vfio group 就是最小单元，但是通过 vfio container 可以让多个 vfio group 共享页表，例如一个 QEMU 持有多个设备。
这几个 config 逐个看下
# CONFIG_ALTERA_MBOX is not set
CONFIG_IOMMU_IOVA=y
CONFIG_IOMMU_API=y
CONFIG_IOMMUFD_DRIVER=y
CONFIG_IOMMU_SUPPORT=y
#
# Generic IOMMU Pagetable Support
#
CONFIG_IOMMU_IO_PGTABLE=y
# end of Generic IOMMU Pagetable Support
CONFIG_IOMMU_DEBUGFS=y
# CONFIG_IOMMU_DEFAULT_DMA_STRICT is not set
CONFIG_IOMMU_DEFAULT_DMA_LAZY=y
# CONFIG_IOMMU_DEFAULT_PASSTHROUGH is not set
CONFIG_IOMMU_DMA=y
CONFIG_IOMMU_SVA=y
CONFIG_AMD_IOMMU=y
CONFIG_AMD_IOMMU_DEBUGFS=y
CONFIG_DMAR_TABLE=y
CONFIG_DMAR_PERF=y
CONFIG_DMAR_DEBUG=y
CONFIG_INTEL_IOMMU=y
CONFIG_INTEL_IOMMU_DEBUGFS=y
CONFIG_INTEL_IOMMU_SVM=y
CONFIG_INTEL_IOMMU_DEFAULT_ON=y
CONFIG_INTEL_IOMMU_FLOPPY_WA=y
CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON=y
CONFIG_INTEL_IOMMU_PERF_EVENTS=y
CONFIG_IOMMUFD=y
# CONFIG_IOMMUFD_TEST is not set
CONFIG_IRQ_REMAP=y
CONFIG_VIRTIO_IOMMU=y
到底什么是 dmar ?
官方文档需要分析下
/home/martins3/core/linux/Documentation/arch/x86/iommu.rst
问题
- 有什么方法提前获取到 vfio cgroup 的数值，例如 /dev/vfio/12
- 必须深入理解 dma 的 coherent 的行为
- iommu group 的数量是什么决定的，是硬件吗?
- 总结下 intel 和 amd 对于 iommu 的态度的差别
- amd 中是默认打开 iommu=pt 的吗?
iommu=pt 对于 GPU 直通有影响吗?
看看 iommu 的用户态接口
/sys/kernel/iommu_groups
debugfs
[ ] sys
cd /sys 然后 fd iommu
intel 上观察到的
class/iommu/
class/misc/iommu
devices/pci0000:00/0000:00:00.0/iommu
devices/pci0000:00/0000:00:00.0/iommu_group
devices/pci0000:00/0000:00:01.0/0000:01:00.0/iommu
devices/pci0000:00/0000:00:01.0/0000:01:00.0/iommu_group
devices/pci0000:00/0000:00:01.0/0000:01:00.1/iommu
devices/pci0000:00/0000:00:01.0/0000:01:00.1/iommu_group
devices/pci0000:00/0000:00:01.0/iommu
devices/pci0000:00/0000:00:01.0/iommu_group
devices/pci0000:00/0000:00:02.0/iommu
devices/pci0000:00/0000:00:02.0/iommu_group
devices/pci0000:00/0000:00:06.0/0000:02:00.0/iommu
devices/pci0000:00/0000:00:06.0/0000:02:00.0/iommu_group
devices/pci0000:00/0000:00:06.0/iommu
devices/pci0000:00/0000:00:06.0/iommu_group
devices/pci0000:00/0000:00:0a.0/iommu
devices/pci0000:00/0000:00:0a.0/iommu_group
devices/pci0000:00/0000:00:0e.0/iommu
devices/pci0000:00/0000:00:0e.0/iommu_group
devices/pci0000:00/0000:00:14.0/iommu
devices/pci0000:00/0000:00:14.0/iommu_group
devices/pci0000:00/0000:00:14.2/iommu
devices/pci0000:00/0000:00:14.2/iommu_group
devices/pci0000:00/0000:00:14.3/iommu
devices/pci0000:00/0000:00:14.3/iommu_group
devices/pci0000:00/0000:00:15.0/iommu
devices/pci0000:00/0000:00:15.0/iommu_group
devices/pci0000:00/0000:00:15.1/iommu
devices/pci0000:00/0000:00:15.1/iommu_group
devices/pci0000:00/0000:00:15.2/iommu
devices/pci0000:00/0000:00:15.2/iommu_group
devices/pci0000:00/0000:00:16.0/iommu
devices/pci0000:00/0000:00:16.0/iommu_group
devices/pci0000:00/0000:00:17.0/iommu
devices/pci0000:00/0000:00:17.0/iommu_group
devices/pci0000:00/0000:00:1a.0/0000:03:00.0/iommu
devices/pci0000:00/0000:00:1a.0/0000:03:00.0/iommu_group
devices/pci0000:00/0000:00:1a.0/iommu
devices/pci0000:00/0000:00:1a.0/iommu_group
devices/pci0000:00/0000:00:1c.0/iommu
devices/pci0000:00/0000:00:1c.0/iommu_group
devices/pci0000:00/0000:00:1c.2/0000:05:00.0/iommu
devices/pci0000:00/0000:00:1c.2/0000:05:00.0/iommu_group
devices/pci0000:00/0000:00:1c.2/iommu
devices/pci0000:00/0000:00:1c.2/iommu_group
devices/pci0000:00/0000:00:1f.0/iommu
devices/pci0000:00/0000:00:1f.0/iommu_group
devices/pci0000:00/0000:00:1f.3/iommu
devices/pci0000:00/0000:00:1f.3/iommu_group
devices/pci0000:00/0000:00:1f.4/iommu
devices/pci0000:00/0000:00:1f.4/iommu_group
devices/pci0000:00/0000:00:1f.5/iommu
devices/pci0000:00/0000:00:1f.5/iommu_group
devices/virtual/iommu/
devices/virtual/iommu/dmar0/intel-iommu/
devices/virtual/iommu/dmar1/intel-iommu/
devices/virtual/misc/iommu/
kernel/btf/iommufd
kernel/btf/vfio_iommu_type1
kernel/iommu_groups/
kernel/slab/iommu_iova
module/iommufd/
module/vfio/holders/vfio_iommu_type1
module/vfio_iommu_type1/
00:00.0 Host bridge: Intel Corporation Device a700 (rev 01)
00:01.0 PCI bridge: Intel Corporation Device a70d (rev 01)
00:02.0 VGA compatible controller: Intel Corporation Raptor Lake-S GT1 [UHD Graphics 770] (rev 04)
00:06.0 PCI bridge: Intel Corporation Device a74d (rev 01)
00:0a.0 Signal processing controller: Intel Corporation Device a77d (rev 01)
00:0e.0 RAID bus controller: Intel Corporation Volume Management Device NVMe RAID Controller Intel Corporation
00:14.0 USB controller: Intel Corporation Alder Lake-S PCH USB 3.2 Gen 2x2 XHCI Controller (rev 11)
00:14.2 RAM memory: Intel Corporation Alder Lake-S PCH Shared SRAM (rev 11)
00:14.3 Network controller: Intel Corporation Alder Lake-S PCH CNVi WiFi (rev 11)
00:15.0 Serial bus controller: Intel Corporation Alder Lake-S PCH Serial IO I2C Controller #0 (rev 11)
00:15.1 Serial bus controller: Intel Corporation Alder Lake-S PCH Serial IO I2C Controller #1 (rev 11)
00:15.2 Serial bus controller: Intel Corporation Alder Lake-S PCH Serial IO I2C Controller #2 (rev 11)
00:16.0 Communication controller: Intel Corporation Alder Lake-S PCH HECI Controller #1 (rev 11)
00:17.0 SATA controller: Intel Corporation Alder Lake-S PCH SATA Controller [AHCI Mode] (rev 11)
00:1a.0 PCI bridge: Intel Corporation Alder Lake-S PCH PCI Express Root Port #25 (rev 11)
00:1c.0 PCI bridge: Intel Corporation Alder Lake-S PCH PCI Express Root Port #1 (rev 11)
00:1c.2 PCI bridge: Intel Corporation Device 7aba (rev 11)
00:1f.0 ISA bridge: Intel Corporation Device 7a86 (rev 11)
00:1f.3 Audio device: Intel Corporation Alder Lake-S HD Audio Controller (rev 11)
00:1f.4 SMBus: Intel Corporation Alder Lake-S PCH SMBus Controller (rev 11)
00:1f.5 Serial bus controller: Intel Corporation Alder Lake-S PCH SPI Controller (rev 11)
01:00.0 VGA compatible controller: NVIDIA Corporation GP106 [GeForce GTX 1060 3GB] (rev a1)
01:00.1 Audio device: NVIDIA Corporation GP106 High Definition Audio Controller (rev a1)
02:00.0 Non-Volatile memory controller: Yangtze Memory Technologies Co.,Ltd ZHITAI TiPro7000 (rev 01)
03:00.0 Non-Volatile memory controller: Yangtze Memory Technologies Co.,Ltd Device 0071 (rev 01)
05:00.0 Ethernet controller: Intel Corporation Ethernet Controller I225-V (rev 03)
amd 上观察到的:
bus/event_source/devices/amd_iommu_0
class/iommu/
class/misc/iommu
devices/amd_iommu_0/
devices/amd_iommu_0/events/mem_iommu_tlb_pde_hit
devices/amd_iommu_0/events/mem_iommu_tlb_pde_mis
devices/amd_iommu_0/events/mem_iommu_tlb_pte_hit
devices/amd_iommu_0/events/mem_iommu_tlb_pte_mis
devices/pci0000:00/0000:00:00.2/iommu/
devices/pci0000:00/0000:00:00.2/iommu/ivhd0/amd-iommu/
devices/pci0000:00/0000:00:01.0/iommu
devices/pci0000:00/0000:00:01.0/iommu_group
devices/pci0000:00/0000:00:01.1/0000:01:00.0/iommu
devices/pci0000:00/0000:00:01.1/0000:01:00.0/iommu_group
devices/pci0000:00/0000:00:01.1/0000:01:00.1/iommu
devices/pci0000:00/0000:00:01.1/0000:01:00.1/iommu_group
devices/pci0000:00/0000:00:01.1/iommu
devices/pci0000:00/0000:00:01.1/iommu_group
devices/pci0000:00/0000:00:01.2/0000:02:00.0/iommu
devices/pci0000:00/0000:00:01.2/0000:02:00.0/iommu_group
devices/pci0000:00/0000:00:01.2/iommu
devices/pci0000:00/0000:00:01.2/iommu_group
devices/pci0000:00/0000:00:02.0/iommu
devices/pci0000:00/0000:00:02.0/iommu_group
devices/pci0000:00/0000:00:02.1/iommu
devices/pci0000:00/0000:00:02.1/iommu_group
devices/pci0000:00/0000:00:02.2/0000:04:00.0/iommu
devices/pci0000:00/0000:00:02.2/0000:04:00.0/iommu_group
devices/pci0000:00/0000:00:02.2/iommu
devices/pci0000:00/0000:00:02.2/iommu_group
devices/pci0000:00/0000:00:03.0/iommu
devices/pci0000:00/0000:00:03.0/iommu_group
devices/pci0000:00/0000:00:03.1/iommu
devices/pci0000:00/0000:00:03.1/iommu_group
devices/pci0000:00/0000:00:03.2/iommu
devices/pci0000:00/0000:00:03.2/iommu_group
devices/pci0000:00/0000:00:03.3/0000:07:00.0/iommu
devices/pci0000:00/0000:00:03.3/0000:07:00.0/iommu_group
devices/pci0000:00/0000:00:03.3/iommu
devices/pci0000:00/0000:00:03.3/iommu_group
devices/pci0000:00/0000:00:04.0/iommu
devices/pci0000:00/0000:00:04.0/iommu_group
devices/pci0000:00/0000:00:08.0/iommu
devices/pci0000:00/0000:00:08.0/iommu_group
devices/pci0000:00/0000:00:08.1/0000:08:00.0/iommu
devices/pci0000:00/0000:00:08.1/0000:08:00.0/iommu_group
devices/pci0000:00/0000:00:08.1/0000:08:00.2/iommu
devices/pci0000:00/0000:00:08.1/0000:08:00.2/iommu_group
devices/pci0000:00/0000:00:08.1/0000:08:00.3/iommu
devices/pci0000:00/0000:00:08.1/0000:08:00.3/iommu_group
devices/pci0000:00/0000:00:08.1/0000:08:00.4/iommu
devices/pci0000:00/0000:00:08.1/0000:08:00.4/iommu_group
devices/pci0000:00/0000:00:08.1/0000:08:00.5/iommu
devices/pci0000:00/0000:00:08.1/0000:08:00.5/iommu_group
devices/pci0000:00/0000:00:08.1/0000:08:00.6/iommu
devices/pci0000:00/0000:00:08.1/0000:08:00.6/iommu_group
devices/pci0000:00/0000:00:08.1/iommu
devices/pci0000:00/0000:00:08.1/iommu_group
devices/pci0000:00/0000:00:08.3/0000:09:00.0/iommu
devices/pci0000:00/0000:00:08.3/0000:09:00.0/iommu_group
devices/pci0000:00/0000:00:08.3/iommu
devices/pci0000:00/0000:00:08.3/iommu_group
devices/pci0000:00/0000:00:14.0/iommu
devices/pci0000:00/0000:00:14.0/iommu_group
devices/pci0000:00/0000:00:14.3/iommu
devices/pci0000:00/0000:00:14.3/iommu_group
devices/pci0000:00/0000:00:18.0/iommu
devices/pci0000:00/0000:00:18.0/iommu_group
devices/pci0000:00/0000:00:18.1/iommu
devices/pci0000:00/0000:00:18.1/iommu_group
devices/pci0000:00/0000:00:18.2/iommu
devices/pci0000:00/0000:00:18.2/iommu_group
devices/pci0000:00/0000:00:18.3/iommu
devices/pci0000:00/0000:00:18.3/iommu_group
devices/pci0000:00/0000:00:18.4/iommu
devices/pci0000:00/0000:00:18.4/iommu_group
devices/pci0000:00/0000:00:18.5/iommu
devices/pci0000:00/0000:00:18.5/iommu_group
devices/pci0000:00/0000:00:18.6/iommu
devices/pci0000:00/0000:00:18.6/iommu_group
devices/pci0000:00/0000:00:18.7/iommu
devices/pci0000:00/0000:00:18.7/iommu_group
devices/virtual/misc/iommu/
kernel/btf/iommufd
kernel/btf/vfio_iommu_type1
kernel/iommu_groups/
kernel/slab/iommu_iova
module/iommufd/
module/vfio/holders/vfio_iommu_type1
module/vfio_iommu_type1/
00:00.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14d8
00:00.2 IOMMU: Advanced Micro Devices, Inc. [AMD] Device 14d9
00:01.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14da
00:01.1 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:01.2 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:02.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14da
00:02.1 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:02.2 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:03.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14da
00:03.1 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:03.2 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:03.3 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14db
00:04.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14da
00:08.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14da
00:08.1 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14dd
00:08.3 PCI bridge: Advanced Micro Devices, Inc. [AMD] Device 14dd
00:14.0 SMBus: Advanced Micro Devices, Inc. [AMD] FCH SMBus Controller (rev 71)
00:14.3 ISA bridge: Advanced Micro Devices, Inc. [AMD] FCH LPC Bridge (rev 51)
00:18.0 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e0
00:18.1 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e1
00:18.2 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e2
00:18.3 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e3
00:18.4 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e4
00:18.5 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e5
00:18.6 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e6
00:18.7 Host bridge: Advanced Micro Devices, Inc. [AMD] Device 14e7
01:00.0 VGA compatible controller: NVIDIA Corporation AD107M [GeForce RTX 4060 Max-Q / Mobile] (rev a1)
01:00.1 Audio device: NVIDIA Corporation Device 22be (rev a1)
02:00.0 Non-Volatile memory controller: Samsung Electronics Co Ltd NVMe SSD Controller PM9A1/PM9A3/980PRO
04:00.0 Network controller: MEDIATEK Corp. MT7922 802.11ax PCI Express Wireless Network Adapter
07:00.0 Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller (rev 15)
08:00.0 Non-Essential Instrumentation [1300]: Advanced Micro Devices, Inc. [AMD] Device 14de (rev d8)
08:00.2 Encryption controller: Advanced Micro Devices, Inc. [AMD] VanGogh PSP/CCP
08:00.3 USB controller: Advanced Micro Devices, Inc. [AMD] Device 15b6
08:00.4 USB controller: Advanced Micro Devices, Inc. [AMD] Device 15b7
08:00.5 Multimedia controller: Advanced Micro Devices, Inc. [AMD] ACP/ACP3X/ACP6x Audio Coprocessor (rev 62)
08:00.6 Audio device: Advanced Micro Devices, Inc. [AMD] Family 17h/19h HD Audio Controller
09:00.0 USB controller: Advanced Micro Devices, Inc. [AMD] Device 15b8
- PCI 和 Host Bridge 是什么关系?
[ ] iommu 的 kernel 参数理解
AMD 的参数
amd_iommu= [HW,X86-64]
Pass parameters to the AMD IOMMU driver in the system.
Possible values are:
fullflush - Deprecated, equivalent to iommu.strict=1
off - do not initialize any AMD IOMMU found in
the system
force_isolation - Force device isolation for all
devices. The IOMMU driver is not
allowed anymore to lift isolation
requirements as needed. This option
does not override iommu=pt
force_enable - Force enable the IOMMU on platforms known
to be buggy with IOMMU enabled. Use this
option with care.
pgtbl_v1 - Use v1 page table for DMA-API (Default).
pgtbl_v2 - Use v2 page table for DMA-API.
amd_iommu_dump= [HW,X86-64]
Enable AMD IOMMU driver option to dump the ACPI table
for AMD IOMMU. With this option enabled, AMD IOMMU
driver will print ACPI tables for AMD IOMMU during
IOMMU initialization.
amd_iommu_intr= [HW,X86-64]
Specifies one of the following AMD IOMMU interrupt
remapping modes:
legacy - Use legacy interrupt remapping mode.
vapic - Use virtual APIC mode, which allows IOMMU
to inject interrupts directly into guest.
This mode requires kvm-amd.avic=1.
(Default when IOMMU HW support is present.)
似乎当没有 amd_iommu=on 中启动的时候，存在如下的
qemu-system-x86_64: -device vfio-pci,host=09:00.0: vfio 0000:09:00.0: group 4 is not viable
Please ensure all devices within the iommu_group are bound to their vfio bus driver.
amd_iommu_intr= [HW,X86-64]
Specifies one of the following AMD IOMMU interrupt
remapping modes:
legacy - Use legacy interrupt remapping mode.
vapic - Use virtual APIC mode, which allows IOMMU
to inject interrupts directly into guest.
This mode requires kvm-amd.avic=1.
(Default when IOMMU HW support is present.)
看内核参数:
__setup("amd_iommu_dump", parse_amd_iommu_dump);
__setup("amd_iommu=", parse_amd_iommu_options);
__setup("amd_iommu_intr=", parse_amd_iommu_intr);
__setup("ivrs_ioapic", parse_ivrs_ioapic);
__setup("ivrs_hpet", parse_ivrs_hpet);
__setup("ivrs_acpihid", parse_ivrs_acpihid);
Intel 参数
intel_iommu= [DMAR] Intel IOMMU driver (DMAR) option
on
Enable intel iommu driver.
off
Disable intel iommu driver.
igfx_off [Default Off]
By default, gfx is mapped as normal device. If a gfx
device has a dedicated DMAR unit, the DMAR unit is
bypassed by not enabling DMAR with this option. In
this case, gfx device will use physical address for
DMA.
strict [Default Off]
Deprecated, equivalent to iommu.strict=1.
sp_off [Default Off]
By default, super page will be supported if Intel IOMMU
has the capability. With this option, super page will
not be supported.
sm_on
Enable the Intel IOMMU scalable mode if the hardware
advertises that it has support for the scalable mode
translation.
sm_off
Disallow use of the Intel IOMMU scalable mode.
tboot_noforce [Default Off]
Do not force the Intel IOMMU enabled under tboot.
By default, tboot will force Intel IOMMU on, which
could harm performance of some high-throughput
devices like 40GBit network cards, even if identity
mapping is enabled.
Note that using this option lowers the security
provided by tboot because it makes the system
vulnerable to DMA attacks.
intremap= [X86-64, Intel-IOMMU]
on enable Interrupt Remapping (default)
off disable Interrupt Remapping
nosid disable Source ID checking
no_x2apic_optout
BIOS x2APIC opt-out request will be ignored
nopost disable Interrupt Posting
才知道 intel 是默认不打开的 iommu 的(和 distrubution 有关?)，打开之后，可以搜索到如下的内容:
[ 0.678026] iommu: Default domain type: Translated
[ 0.678027] iommu: DMA domain TLB invalidation policy: lazy mode
[ 0.736751] pci 0000:00:02.0: Adding to iommu group 0
[ 0.737027] pci 0000:00:00.0: Adding to iommu group 1
[ 0.737031] pci 0000:00:01.0: Adding to iommu group 2
[ 0.737035] pci 0000:00:06.0: Adding to iommu group 3
[ 0.737038] pci 0000:00:0a.0: Adding to iommu group 4
[ 0.737042] pci 0000:00:0e.0: Adding to iommu group 5
[ 0.737048] pci 0000:00:14.0: Adding to iommu group 6
[ 0.737052] pci 0000:00:14.2: Adding to iommu group 6
[ 0.737055] pci 0000:00:14.3: Adding to iommu group 7
[ 0.737063] pci 0000:00:15.0: Adding to iommu group 8
[ 0.737066] pci 0000:00:15.1: Adding to iommu group 8
[ 0.737069] pci 0000:00:15.2: Adding to iommu group 8
[ 0.737073] pci 0000:00:16.0: Adding to iommu group 9
[ 0.737076] pci 0000:00:17.0: Adding to iommu group 10
[ 0.737089] pci 0000:00:1a.0: Adding to iommu group 11
[ 0.737095] pci 0000:00:1c.0: Adding to iommu group 12
[ 0.737100] pci 0000:00:1c.2: Adding to iommu group 13
[ 0.737109] pci 0000:00:1f.0: Adding to iommu group 14
[ 0.737113] pci 0000:00:1f.3: Adding to iommu group 14
[ 0.737116] pci 0000:00:1f.4: Adding to iommu group 14
[ 0.737120] pci 0000:00:1f.5: Adding to iommu group 14
[ 0.737127] pci 0000:01:00.0: Adding to iommu group 15
[ 0.737131] pci 0000:01:00.1: Adding to iommu group 15
[ 0.737135] pci 0000:02:00.0: Adding to iommu group 16
[ 0.737148] pci 0000:03:00.0: Adding to iommu group 17
[ 0.737152] pci 0000:05:00.0: Adding to iommu group 18
通用的参数
iommu= [X86]
off
force
noforce
biomerge
panic
nopanic
merge
nomerge
soft
pt [X86]
nopt [X86]
nobypass [PPC/POWERNV]
Disable IOMMU bypass, using IOMMU for PCI devices.
iommu.forcedac= [ARM64, X86] Control IOVA allocation for PCI devices.
Format: { "0" | "1" }
0 - Try to allocate a 32-bit DMA address first, before
falling back to the full range if needed.
1 - Allocate directly from the full usable range,
forcing Dual Address Cycle for PCI cards supporting
greater than 32-bit addressing.
iommu.strict= [ARM64, X86] Configure TLB invalidation behaviour
Format: { "0" | "1" }
0 - Lazy mode.
Request that DMA unmap operations use deferred
invalidation of hardware TLBs, for increased
throughput at the cost of reduced device isolation.
Will fall back to strict mode if not supported by
the relevant IOMMU driver.
1 - Strict mode.
DMA unmap operations invalidate IOMMU hardware TLBs
synchronously.
unset - Use value of CONFIG_IOMMU_DEFAULT_DMA_{LAZY,STRICT}.
Note: on x86, strict mode specified via one of the
legacy driver-specific options takes precedence.
iommu.passthrough=
[ARM64, X86] Configure DMA to bypass the IOMMU by default.
Format: { "0" | "1" }
0 - Use IOMMU translation for DMA.
1 - Bypass the IOMMU for DMA.
unset - Use value of CONFIG_IOMMU_DEFAULT_PASSTHROUGH.
iommu.passthrough
/*
* This are the possible domain-types
*
* IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate
* devices
* IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses
* IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used
* for VMs
* IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations.
* This flag allows IOMMU drivers to implement
* certain optimizations for these domains
* IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB
* invalidation.
* IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses
* represented by mm_struct's.
*/
- IOMMU_DOMAIN_SVA 这个是给 dpdk 使用吗?
phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
{
if (domain->type == IOMMU_DOMAIN_IDENTITY)
return iova;
if (domain->type == IOMMU_DOMAIN_BLOCKED)
return 0;
return domain->ops->iova_to_phys(domain, iova);
}
[x] iommu=pt 为什么还是可以使用 GPU 直通，无法理解啊
因为 iommu=pt 应该是只是管理 dma
想不到 iommu=pt 之后，根本没有人走 iommu_dma_unmap_page
nvme 也是走的这个路线:
dma_map_page_attrs+5
nvme_prep_rq.part.0+1460
nvme_queue_rq+123
__blk_mq_try_issue_directly+348
blk_mq_try_issue_directly+22
blk_mq_submit_bio+1199
submit_bio_noacct_nocheck+818
__blkdev_direct_IO_async+260
blkdev_read_iter+295
aio_read+306
io_submit_one+1451
__x64_sys_io_submit+173
do_syscall_64+59
entry_SYSCALL_64_after_hwframe+114
sudo bpftrace -e 'kfunc:dma_map_page_attrs { @[args->dev->dma_ops]=count() }'
得到：
@[0x0]: 562818
看来的确是所有的 guest 都没有的。
[x] iommu=pt 和 swiotlb 啥关系?
当使用 iommu=pt 的时候，自动将会采用 swiotlb 的如果 无法满足 4G 的分配需求的时候。
问题 && TODO
- drivers/iommu/hyperv-iommu.c 是个什么概念 ?
- 能不能 hacking 一个 minimal 的用户态 nvme 驱动，能够读取一个 block 上来的那种
- 在代码中找到 device page table 的内容，以及 IOMMU 防护恶意驱动
- 据说 IOMMU 对于性能会存在影响。
- IOMMU 能不能解决 cma 的问题，也就是，使用离散的物理内存提供给设备，让设备以为自己访问的是连续的物理内存
- https://wiki.qemu.org/Features/VT-d 分析了下为什么 guest 需要 vIOMMU
isca_iommu_tutorial
Extraneous IPI adds overheads => Each extra interrupt can add 5-10K cycles ==> Needs dynamic remapping of interrupts
似乎是 core 1 setup 了 io device 的中断，那么之后，io device 的中断到其他的 core 都需要额外的 ipi. 然后使用 iommu 之后，这个中断 remap 的事情不需要软件处理了
在异构计算中间，可以实现 GPU 共享 CPU 的 page table 之后，获取相同的虚拟地址空间。
IOMMU IS PART OF PROCESSOR COMPLEX
io device 经过各级 pci hub 到达 root complex, 进入 iommu 翻译，然后到达 mmu controller
Better solution: IOMMU remaps 32bit device physical address to system physical address beyond 32bit ‒ DMA goes directly into 64bit memory ‒ No CPU transfer ‒ More efficient
If access occurs, OS gets notified and can shut the device & driver down and notifies the user or administrator
Some I/O devices can issue DMA requests to system memory directly, without OS or Firmware intervention ‒ e.g.,1394/Firewire, network cards, as part of network boot ‒ That allows attacks to modify memory before even the OS has a chance to protect against the attacks
IOMMU redirects device physical address set up by Guest OS driver (= Guest Physical Addresses) to the actual Host System Physical Address (SPA)
Some memory copies are gone, because the same memory is accessed
‒ But the memory is not accessible concurrently, because of cache policies
Two memory pools remain (cache coherent + non-coherent memory regions)
Jobs are still queued through the OS driver chain and suffer from overhead
Still requires expert programmers to get performance
IOMMU Driver (running on CPU) issues commands to IOMMU ‒ e.g., Invalidate IOMMU TLB Entry, Invalidate IOTLB Entry ‒ e.g., Invalidate Device Table Entry ‒ e.g., Complete PPR, Completion Wait , etc.
Issued via Command Buffer ‒ Memory resident circular buffer ‒ MMIO registers: Base, Head, and Tail register
device remapping table
interrupt remapping table
https://blog.kernel.love/intel_iommu.html
- 解释了一下 intel iommu 启动的过程
AMD 手册
An Introduction to IOMMU Infrastructure in the Linux Kernel
- 主要是，这 DMA coherent 是什么关系哇
#define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
- 这是一个直接映射的驱动的样子:
#0 dma_map_page_attrs (dev=0xffff8881007ac8c8, page=0xffffea0004067580, offset=0, size=4096, dir=DMA_FROM_DEVICE, attrs=0) at kernel/dma/mapping.c:145
#1 0xffffffff819dde37 in nvme_setup_prp_simple (dev=0xffff8881019d3000, dev=0xffff8881019d3000, bv=<synthetic pointer>, cmnd=0xffff88814046c928, req=0xffff88814046c800) at include/linux/blk-mq.h:203
#2 nvme_map_data (dev=dev@entry=0xffff8881019d3000, req=req@entry=0xffff88814046c800, cmnd=cmnd@entry=0xffff88814046c928) at drivers/nvme/host/pci.c:840
#3 0xffffffff819de0b0 in nvme_prep_rq (req=<optimized out>, dev=0xffff8881019d3000) at drivers/nvme/host/pci.c:910
#4 nvme_prep_rq (req=<optimized out>, dev=0xffff8881019d3000) at drivers/nvme/host/pci.c:896
#5 nvme_queue_rq (hctx=<optimized out>, bd=0xffffc90000c73bb8) at drivers/nvme/host/pci.c:952
#6 0xffffffff8161412f in blk_mq_dispatch_rq_list (hctx=hctx@entry=0xffff88814053a600, list=list@entry=0xffffc90000c73c08, nr_budgets=nr_budgets@entry=0) at block/blk-mq.c:1902
#7 0xffffffff8161a423 in __blk_mq_sched_dispatch_requests (hctx=hctx@entry=0xffff88814053a600) at block/blk-mq-sched.c:306
#8 0xffffffff8161a500 in blk_mq_sched_dispatch_requests (hctx=hctx@entry=0xffff88814053a600) at block/blk-mq-sched.c:339
#9 0xffffffff81610f50 in __blk_mq_run_hw_queue (hctx=0xffff88814053a600) at block/blk-mq.c:2020
#10 0xffffffff81611200 in __blk_mq_delay_run_hw_queue (hctx=<optimized out>, async=<optimized out>, msecs=msecs@entry=0) at block/blk-mq.c:2096
#11 0xffffffff81611469 in blk_mq_run_hw_queue (hctx=<optimized out>, async=<optimized out>) at block/blk-mq.c:2144
#12 0xffffffff8161a6c8 in blk_mq_sched_insert_request (rq=rq@entry=0xffff88814046c800, at_head=<optimized out>, at_head@entry=false, run_queue=run_queue@entry=true, async=async@entry=false) at block/blk-mq-sched.c:458
#13 0xffffffff81611a55 in blk_execute_rq (rq=rq@entry=0xffff88814046c800, at_head=at_head@entry=false) at block/blk-mq.c:1278
#14 0xffffffff819cf8a8 in nvme_execute_rq (at_head=false, rq=0xffff88814046c800) at drivers/nvme/host/core.c:1005
#15 __nvme_submit_sync_cmd (q=<optimized out>, cmd=cmd@entry=0xffffc90000c73d08, result=result@entry=0x0 <fixed_percpu_data>, buffer=0xffff8881019d6000, bufflen=bufflen@entry=4096, qid=qid@entry=-1, at_head=0, flags=0) at drivers/nvme/host/core.c:1041
#16 0xffffffff819d2b74 in nvme_submit_sync_cmd (bufflen=4096, buffer=<optimized out>, cmd=<optimized out>, q=<optimized out>) at drivers/nvme/host/core.c:1053
#17 nvme_identify_ctrl (dev=dev@entry=0xffff8881019d3210, id=id@entry=0xffffc90000c73d80) at drivers/nvme/host/core.c:1295
#18 0xffffffff819d50ce in nvme_init_identify (ctrl=0xffff8881019d3210) at drivers/nvme/host/core.c:3081
#19 nvme_init_ctrl_finish (ctrl=ctrl@entry=0xffff8881019d3210) at drivers/nvme/host/core.c:3246
#20 0xffffffff819df784 in nvme_reset_work (work=0xffff8881019d3608) at drivers/nvme/host/pci.c:2870
#21 0xffffffff81123d37 in process_one_work (worker=worker@entry=0xffff88814049ca80, work=0xffff8881019d3608) at kernel/workqueue.c:2289
#22 0xffffffff811242c8 in worker_thread (__worker=0xffff88814049ca80) at kernel/workqueue.c:2436
#23 0xffffffff8112ac73 in kthread (_create=0xffff8881404a0d40) at kernel/kthread.c:376
#24 0xffffffff81001a72 in ret_from_fork () at arch/x86/entry/entry_64.S:306
#25 0x0000000000000000 in ?? ()
- 即使不是直通，那么存在保护的功能吗?
如果不是直接 iommu_dma_map_page 的，
iommu_dma_ops
static const struct dma_map_ops iommu_dma_ops = {
.flags = DMA_F_PCI_P2PDMA_SUPPORTED,
.alloc = iommu_dma_alloc,
.free = iommu_dma_free,
.alloc_pages = dma_common_alloc_pages,
.free_pages = dma_common_free_pages,
.alloc_noncontiguous = iommu_dma_alloc_noncontiguous,
.free_noncontiguous = iommu_dma_free_noncontiguous,
.mmap = iommu_dma_mmap,
.get_sgtable = iommu_dma_get_sgtable,
.map_page = iommu_dma_map_page,
.unmap_page = iommu_dma_unmap_page,
.map_sg = iommu_dma_map_sg,
.unmap_sg = iommu_dma_unmap_sg,
.sync_single_for_cpu = iommu_dma_sync_single_for_cpu,
.sync_single_for_device = iommu_dma_sync_single_for_device,
.sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu,
.sync_sg_for_device = iommu_dma_sync_sg_for_device,
.map_resource = iommu_dma_map_resource,
.unmap_resource = iommu_dma_unmap_resource,
.get_merge_boundary = iommu_dma_get_merge_boundary,
.opt_mapping_size = iommu_dma_opt_mapping_size,
};
如果是给 guest 映射的 DMA 空间，需要同步的修改 IOMMU 才对
iommu
-
iommu 是可以实现每一个 device 都存在各自的映射，而不是一个虚拟机使用一个映射的。
-
为什么 IOMMU 需要搞出来 iommu group，需要更加详细的解释。
[ ] IOMMU interrupt remapping 是如何实现的
参考 intel vt-d 标准吧！
my question
- mmio 可以 remap 吗 ?
- dma engine 是一个需要的硬件支持放在哪里了 ?
- 怎么知道一个设备在进行 dma ?
- 一个真正的物理设备，当需要发起 dma 的时候，进行的 IO 地址本来应该在 pa, 由于 vm 的存在，实际上是在 gpa 上，需要进行在 hpa 上
VT-d Posted Interrupts
- Motivation
- Interrupt virtualization efficiency
- Interrupt migration complexity
- Big requirement of host vector for different assigned devices
- migration ?
- host vector for different assigned devices ?
VT-d
How to enable IRQ_REMAP in make menuconfig:
Device Drivers ==> IOMMU Hareware Support ==> Support for Interrupt Remapping
intel_setup_irq_remapping ==> iommu_set_irq_remapping, setup Interrupt Remapping Table Address Register which hold address IRET locate [^3] 163,164
好家伙，才意识到 ITRE 其实存在两种格式，remapped interrupt 的格式下，其功能相当于 IO-APIC 的功能，作为设备和 CPU 之间的联系，而 Posted-interrupt 的格式下，就是我们熟悉的内容。 在 Posted-interrupt 格式下，IRET 中间没有目标 CPU 等字段，而是 posted-interrupt descriptor 的地址
Xen Implementation Details:
- Update IRET according to guest’s modification to the interrupt configuration (MSI address, data)
- Interrupt migration during VCPU scheduling
从 vfio 通往 iommu
禁用 iommu
网卡无法启动了。但是不是存在 swiotlb 吗？
[ 2.517363] ------------[ cut here ]------------
[ 2.517364] mt7921e 0000:04:00.0: DMA addr 0x0000000117eaf000+4096 overflow (mask ffffffff, bus limit 0).
[ 2.517367] WARNING: CPU: 19 PID: 1054 at kernel/dma/direct.h:103 dma_map_page_attrs+0x242/0x280
[ 2.517371] Modules linked in: ip6_tables xt_conntrack ip6t_rpfilter ipt_rpfilter xt_pkttype xt_LOG nf_log_syslog xt_tcpudp snd_sof_amd_rembrandt mt7921e(+) snd_sof_amd_renoir snd_sof_amd_acp snd_hda_codec_realtek mt7921_common snd_sof_pci snd_sof_xtensa_dsp mt76_connac_lib snd_hda_codec_generic mousedev joydev hid_multitouch nft_compat ledtrig_audio snd_hda_codec_hdmi snd_sof mt76 snd_hda_intel snd_sof_utils snd_intel_dspcfg snd_soc_core snd_intel_sdw_acpi uvcvideo mac80211 snd_hda_codec snd_compress videobuf2_vmalloc ac97_bus uvc videobuf2_memops snd_pcm_dmaengine videobuf2_v4l2 snd_pci_ps snd_rpl_pci_acp6x snd_hda_core snd_acp_pci videodev nf_tables snd_pci_acp6x edac_mce_amd snd_hwdep intel_rapl_msr snd_pci_acp5x edac_core nls_iso8859_1 snd_rn_pci_acp3x snd_pcm intel_rapl_common snd_acp_config nls_cp437 crc32_pclmul videobuf2_common snd_soc_acpi vfat polyval_clmulni sch_fq_codel nfnetlink polyval_generic fat cfg80211 mc hid_generic snd_timer snd_pci_acp3x gf128mul ghash_clmulni_intel btusb snd i2c_hid_acpi r8169
[ 2.517389] sha512_ssse3 sha512_generic sp5100_tco btrtl wdat_wdt ucsi_acpi aesni_intel btbcm typec_ucsi btintel realtek crypto_simd i2c_hid mdio_devres ideapad_laptop wmi_bmof cryptd btmtk rapl typec watchdog libphy k10temp sparse_keymap i2c_piix4 soundcore libarc4 nvidia_drm(PO) roles battery platform_profile bluetooth drm_kms_helper evdev tpm_crb input_leds tiny_power_button led_class tpm_tis tpm_tis_core syscopyarea mac_hid sysfillrect sysimgblt i2c_designware_platform acpi_cpufreq i2c_designware_core ac button usbhid serio_raw ecdh_generic nvidia_modeset(PO) hid rfkill ecc video libaes wmi nvidia_uvm(PO) nvidia(PO) ctr loop xt_nat br_netfilter veth tap macvlan bridge stp llc openvswitch nsh nf_conncount nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c tun kvm_amd ccp kvm drm fuse deflate backlight efi_pstore i2c_core configfs efivarfs tpm rng_core dmi_sysfs ip_tables x_tables autofs4 ext4 crc32c_generic crc16 mbcache jbd2 xhci_pci xhci_pci_renesas xhci_hcd atkbd nvme libps2 vivaldi_fmap usbcore
[ 2.517415] nvme_core t10_pi crc32c_intel crc64_rocksoft crc64 crc_t10dif usb_common crct10dif_generic crct10dif_pclmul i8042 crct10dif_common rtc_cmos serio dm_mod dax vfio_pci vfio_pci_core irqbypass vfio_iommu_type1 vfio iommufd
[ 2.517420] CPU: 19 PID: 1054 Comm: (udev-worker) Tainted: P O 6.3.5 #1-NixOS
[ 2.517422] Hardware name: LENOVO 82WM/LNVNB161216, BIOS LPCN41WW 05/24/2023
[ 2.517422] RIP: 0010:dma_map_page_attrs+0x242/0x280
[ 2.517424] Code: 8b 5d 00 48 89 ef e8 bd 68 63 00 4d 89 e9 4d 89 e0 48 89 da 41 56 48 89 c6 48 c7 c7 40 05 5c a5 48 8d 4c 24 10 e8 9e 8e f4 ff <0f> 0b 58 e9 7b ff ff ff 48 c7 44 24 08 ff ff ff ff 4d 85 c0 0f 84
[ 2.517425] RSP: 0018:ffff998b43b3ba08 EFLAGS: 00010282
[ 2.517426] RAX: 0000000000000000 RBX: ffff8b6784b5cae0 RCX: 0000000000000027
[ 2.517426] RDX: ffff8b766dae14c8 RSI: 0000000000000001 RDI: ffff8b766dae14c0
[ 2.517427] RBP: ffff8b6784c070c8 R08: 0000000000000000 R09: ffff998b43b3b8b0
[ 2.517427] R10: 0000000000000003 R11: ffffffffa5d38868 R12: 0000000000001000
[ 2.517428] R13: 00000000ffffffff R14: 0000000000000000 R15: ffff8b67888f3908
[ 2.517428] FS: 00007f14a4b49c40(0000) GS:ffff8b766dac0000(0000) knlGS:0000000000000000
[ 2.517429] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2.517429] CR2: 00007ff0bce47dfc CR3: 0000000106cf6000 CR4: 0000000000750ee0
[ 2.517430] PKRU: 55555554
[ 2.517430] Call Trace:
[ 2.517431] <TASK>
[ 2.517431] ? dma_map_page_attrs+0x242/0x280
[ 2.517433] ? __warn+0x81/0x130
[ 2.517436] ? dma_map_page_attrs+0x242/0x280
[ 2.517437] ? report_bug+0x171/0x1a0
[ 2.517439] ? handle_bug+0x41/0x70
[ 2.517441] ? exc_invalid_op+0x17/0x70
[ 2.517442] ? asm_exc_invalid_op+0x1a/0x20
[ 2.517445] ? dma_map_page_attrs+0x242/0x280
[ 2.517446] ? dma_map_page_attrs+0x242/0x280
[ 2.517447] page_pool_dma_map+0x30/0x70
[ 2.517449] __page_pool_alloc_pages_slow+0x133/0x3d0
[ 2.517451] ? sched_clock_cpu+0xf2/0x190
[ 2.517453] page_pool_alloc_frag+0x14c/0x1d0
[ 2.517455] mt76_dma_rx_fill.isra.0+0x132/0x390 [mt76]
[ 2.517460] ? __pfx_mt7921_poll_rx+0x10/0x10 [mt7921e]
[ 2.517463] ? napi_kthread_create+0x48/0x90
[ 2.517465] ? __pfx_mt7921_poll_rx+0x10/0x10 [mt7921e]
[ 2.517467] mt76_dma_init+0x110/0x140 [mt76]
[ 2.517471] ? __pfx_mt7921_rr+0x10/0x10 [mt7921e]
[ 2.517473] mt7921_dma_init+0x18b/0x1f0 [mt7921e]
[ 2.517475] mt7921_pci_probe+0x387/0x430 [mt7921e]
[ 2.517477] local_pci_probe+0x3f/0x90
[ 2.517480] pci_device_probe+0xc3/0x240
[ 2.517482] ? sysfs_do_create_link_sd+0x6e/0xe0
[ 2.517484] really_probe+0x19f/0x400
[ 2.517487] ? __pfx___driver_attach+0x10/0x10
[ 2.517488] __driver_probe_device+0x78/0x160
[ 2.517489] driver_probe_device+0x1f/0x90
[ 2.517490] __driver_attach+0xd2/0x1c0
[ 2.517491] bus_for_each_dev+0x85/0xd0
[ 2.517493] bus_add_driver+0x116/0x220
[ 2.517494] driver_register+0x59/0x100
[ 2.517495] ? __pfx_init_module+0x10/0x10 [mt7921e]
[ 2.517497] do_one_initcall+0x5a/0x240
[ 2.517500] do_init_module+0x4a/0x200
[ 2.517501] __do_sys_init_module+0x17f/0x1b0
[ 2.517503] do_syscall_64+0x3b/0x90
[ 2.517505] entry_SYSCALL_64_after_hwframe+0x72/0xdc
[ 2.517507] RIP: 0033:0x7f14a4722b5e
[ 2.517531] Code: 48 8b 0d bd e2 0c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa 49 89 ca b8 af 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 8a e2 0c 00 f7 d8 64 89 01 48
[ 2.517531] RSP: 002b:00007ffda4e74928 EFLAGS: 00000246 ORIG_RAX: 00000000000000af
[ 2.517532] RAX: ffffffffffffffda RBX: 00005616c16d0060 RCX: 00007f14a4722b5e
[ 2.517532] RDX: 00007f14a4cecb0d RSI: 0000000000019bc8 RDI: 00005616c1ee5290
[ 2.517533] RBP: 00007f14a4cecb0d R08: 0000000000000007 R09: 00005616c16ccd20
[ 2.517533] R10: 0000000000000005 R11: 0000000000000246 R12: 00005616c1ee5290
[ 2.517533] R13: 0000000000000000 R14: 00005616c16bcfe0 R15: 0000000000000000
[ 2.517534] </TASK>
[ 2.517534] ---[ end trace 0000000000000000 ]---
[ 2.517778] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 2.592359] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 2.667874] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 2.739891] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 2.782399] NET: Registered PF_ALG protocol family
[ 2.812364] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 2.887399] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 2.962403] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 3.037498] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 3.112444] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 3.152283] Generic FE-GE Realtek PHY r8169-0-700:00: attached PHY driver (mii_bus:phy_addr=r8169-0-700:00, irq=MAC)
[ 3.187426] mt7921e 0000:04:00.0: Failed to get patch semaphore
[ 3.262368] mt7921e 0000:04:00.0: hardware init failed
[ 3.308670] r8169 0000:07:00.0 enp7s0: No native access to PCI extended config space, falling back to CSI
[ 3.330433] memfd_create() without MFD_EXEC nor MFD_NOEXEC_SEAL, pid=1582 'systemd'
[ 3.331709] r8169 0000:07:00.0 enp7s0: Link is Down
[ 3.463048] systemd-journald[944]: /var/log/journal/c4d5dffd3fce45ff9046f3017148dd83/user-1000.journal: Monotonic clock jumped backwards relative to last journal entry, rotating.
[ 3.562042] ACPI Warning: \_SB.NPCF._DSM: Argument #4 type mismatch - Found [Buffer], ACPI requires [Package] (20221020/nsarguments-61)
[ 3.562087] ACPI Warning: \_SB.PCI0.GPP0.PEGP._DSM: Argument #4 type mismatch - Found [Buffer], ACPI requires [Package] (20221020/nsarguments-61)
[ ] 请问 iommu=off 和 amd_iommu=off 有啥区别?
只是关掉 amd_iommu 的时候:
[root@nixos:/sys/kernel/debug/swiotlb]# cat io_tlb_nslabs
32768
[root@nixos:/sys/kernel/debug/swiotlb]# cat io_tlb_used
2801
[root@nixos:/sys/kernel/debug/swiotlb]# cat /proc/cmdline
initrd=\efi\nixos\i7ijxg4186dp43vzkavh7gb1vq8dp916-initrd-linux-6.3.5-initrd.efi init=/nix/store/17yknrp3kdnzar0mxymvyca8ppjcgcd8-nixos-system-nixos-23.05
.563.70f7275b32f/init transparent_hugepage=always intel_iommu=on amd_iommu=off ftrace=function_graph ftrace_filter=iommu_setup_dma_ops fsck.mode=force fsc
k.repair=yes loglevel=4
应该是缺少 iommu 导致的
[ 613.669612] vfio-pci: probe of 0000:07:00.0 failed with error -22
[ ] 为什么要强调是通过 cmdline 设置的
[ 0.498658] iommu: Default domain type: Passthrough (set via kernel command line)
[ ] 当 amd_iommu=off 的时候，采用
设置为
[ 0.495931] iommu: Default domain type: Translated
[ 0.495931] iommu: DMA domain TLB invalidation policy: lazy mode
iommu=pt 导致可能使用上 swiotlb
[root@nixos:/sys/kernel/debug/swiotlb]# cat io_tlb_nslabs
32768
[root@nixos:/sys/kernel/debug/swiotlb]# cat io_tlb_used
2877
[root@nixos:/sys/kernel/debug/swiotlb]# cat /proc/cmdline
initrd=\efi\nixos\zkf5v40bsl6lzvgbi9a4mnf5wm55aqh6-initrd-linux-6.3.5-initrd.efi init=/nix/store/np8xhzj01vrhld37vhcd0zlgan1v9kwc-nixos-system-nixos-23.05.563.70f7275b32f/init
transparent_hugepage=always intel_iommu=on iommu=pt fsck.mode=force fsck.repair=yes loglevel=4
参数处理函数: iommu_setup
代码上的证据: iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY
- dma_map_page_attrs
#0 set_dma_ops (dma_ops=0x0 <fixed_percpu_data>, dev=0xffff888007ccf0c8) at ./include/linux/dma-map-ops.h:99
#1 viommu_probe_finalize (dev=0xffff888007ccf0c8) at drivers/iommu/virtio-iommu.c:984
#2 0xffffffff81955f1d in iommu_probe_device (dev=0xffff888007ccf0c8) at drivers/iommu/iommu.c:449
#3 iommu_probe_device (dev=dev@entry=0xffff888007ccf0c8) at drivers/iommu/iommu.c:404
#4 0xffffffff8187ddf6 in acpi_iommu_configure_id ( id_in=0x0 <fixed_percpu_data>, dev=0xffff888007ccf0c8) at drivers/acpi/scan.c:1583
没有 iommu=pt
[ 0.693169] virtio_iommu: [->:set_dma_ops:99] 0000000000000000
[ 0.694918] [->:iommu_setup_dma_ops:1597] 0000000000000000
存在 iommu=pt 的时候
[ 0.558555] virtio_iommu: [->:set_dma_ops:99] 0000000000000000
[ 0.559772] virtio_iommu: [->:set_dma_ops:99] 0000000000000000
其调用路径为:
#0 iommu_setup_dma_ops (dev=0xffff888007bba0c8, dma_base=0, dma_limit=18446744073709551615) at drivers/iommu/dma-iommu.c:1584
#1 0xffffffff81955f7d in iommu_probe_device ( dev=0xffff888007bba0c8) at drivers/iommu/iommu.c:449
#2 iommu_probe_device (dev=dev@entry=0xffff888007bba0c8) at drivers/iommu/iommu.c:404
#3 0xffffffff8187de0d in acpi_iommu_configure_id ( id_in=0x0 <fixed_percpu_data>, dev=0xffff888007bba0c8) at drivers/acpi/scan.c:1583
原来是在
static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
{
return domain->type & __IOMMU_DOMAIN_DMA_API;
}
void iommu_set_default_translated(bool cmd_line)
{
if (cmd_line)
iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
iommu_def_domain_type = IOMMU_DOMAIN_DMA;
}
[ ] iommu=pt 为什么还可以直通?
分析下 typo 结构
[ 0.497832] pci 0000:00:01.0: Adding to iommu group 0
[ 0.497836] pci 0000:00:01.1: Adding to iommu group 0
[ 0.497840] pci 0000:00:01.2: Adding to iommu group 0
[ 0.497848] pci 0000:00:02.0: Adding to iommu group 1
[ 0.497852] pci 0000:00:02.1: Adding to iommu group 1
[ 0.497856] pci 0000:00:02.2: Adding to iommu group 1
[ 0.497867] pci 0000:00:03.0: Adding to iommu group 2
[ 0.497871] pci 0000:00:03.1: Adding to iommu group 2
[ 0.497875] pci 0000:00:03.2: Adding to iommu group 2
[ 0.497879] pci 0000:00:03.3: Adding to iommu group 2
[ 0.497884] pci 0000:00:04.0: Adding to iommu group 3
[ 0.497893] pci 0000:00:08.0: Adding to iommu group 4
[ 0.497897] pci 0000:00:08.1: Adding to iommu group 4
[ 0.497901] pci 0000:00:08.3: Adding to iommu group 4
[ 0.497909] pci 0000:00:14.0: Adding to iommu group 5
[ 0.497914] pci 0000:00:14.3: Adding to iommu group 5
[ 0.497931] pci 0000:00:18.0: Adding to iommu group 6
[ 0.497935] pci 0000:00:18.1: Adding to iommu group 6
[ 0.497940] pci 0000:00:18.2: Adding to iommu group 6
[ 0.497944] pci 0000:00:18.3: Adding to iommu group 6
[ 0.497949] pci 0000:00:18.4: Adding to iommu group 6
[ 0.497953] pci 0000:00:18.5: Adding to iommu group 6
[ 0.497957] pci 0000:00:18.6: Adding to iommu group 6
[ 0.497961] pci 0000:00:18.7: Adding to iommu group 6
[ 0.497964] pci 0000:01:00.0: Adding to iommu group 0
[ 0.497965] pci 0000:01:00.1: Adding to iommu group 0
[ 0.497967] pci 0000:02:00.0: Adding to iommu group 0
[ 0.497969] pci 0000:03:00.0: Adding to iommu group 1
[ 0.497972] pci 0000:04:00.0: Adding to iommu group 1
[ 0.497975] pci 0000:07:00.0: Adding to iommu group 2
[ 0.497977] pci 0000:08:00.0: Adding to iommu group 4
[ 0.497980] pci 0000:08:00.2: Adding to iommu group 4
[ 0.497982] pci 0000:08:00.3: Adding to iommu group 4
[ 0.497984] pci 0000:08:00.4: Adding to iommu group 4
[ 0.497986] pci 0000:08:00.5: Adding to iommu group 4
[ 0.497988] pci 0000:08:00.6: Adding to iommu group 4
[ 0.497990] pci 0000:09:00.0: Adding to iommu group 4
IOMMU Group 0:
00:01.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14da]
00:01.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
00:01.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
01:00.0 VGA compatible controller [0300]: NVIDIA Corporation AD107M [GeForce RTX 4060 Max-Q / Mobile] [10de:28e0] (rev a1)
01:00.1 Audio device [0403]: NVIDIA Corporation Device [10de:22be] (rev a1)
02:00.0 Non-Volatile memory controller [0108]: Samsung Electronics Co Ltd NVMe SSD Controller PM9A1/PM9A3/980PRO [144d:a80a]
IOMMU Group 1:
00:02.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14da]
00:02.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
00:02.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
03:00.0 Non-Volatile memory controller [0108]: MAXIO Technology (Hangzhou) Ltd. Device [1e4b:1602] (rev 01)
04:00.0 Network controller [0280]: MEDIATEK Corp. MT7922 802.11ax PCI Express Wireless Network Adapter [14c3:0616]
IOMMU Group 2:
00:03.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14da]
00:03.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
00:03.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
00:03.3 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14db]
07:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller [10ec:8168] (rev 15)
IOMMU Group 3:
00:04.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14da]
IOMMU Group 4:
00:08.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14da]
00:08.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14dd]
00:08.3 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Device [1022:14dd]
08:00.0 Non-Essential Instrumentation [1300]: Advanced Micro Devices, Inc. [AMD] Device [1022:14de] (rev d8)
08:00.2 Encryption controller [1080]: Advanced Micro Devices, Inc. [AMD] VanGogh PSP/CCP [1022:1649]
08:00.3 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Device [1022:15b6]
08:00.4 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Device [1022:15b7]
08:00.5 Multimedia controller [0480]: Advanced Micro Devices, Inc. [AMD] ACP/ACP3X/ACP6x Audio Coprocessor [1022:15e2] (rev 62)
08:00.6 Audio device [0403]: Advanced Micro Devices, Inc. [AMD] Family 17h/19h HD Audio Controller [1022:15e3]
09:00.0 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Device [1022:15b8]
IOMMU Group 5:
00:14.0 SMBus [0c05]: Advanced Micro Devices, Inc. [AMD] FCH SMBus Controller [1022:790b] (rev 71)
00:14.3 ISA bridge [0601]: Advanced Micro Devices, Inc. [AMD] FCH LPC Bridge [1022:790e] (rev 51)
IOMMU Group 6:
00:18.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e0]
00:18.1 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e1]
00:18.2 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e2]
00:18.3 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e3]
00:18.4 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e4]
00:18.5 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e5]
00:18.6 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e6]
00:18.7 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:14e7]
- 在一个 group 上这种事情，是如何发现的, acpi 还是扫描 pci 的 typo 结构
- 如何查看 pci 的 typo 结构如何查看来着
intel dmar 有什么特殊之处吗?
drivers/iommu/intel/dmar.c
- /sys/devices/virtual/iommu/dmar0/intel-iommu/ecap
- /sys/devices/virtual/iommu/dmar1/intel-iommu/ecap
[ 0.181268] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.182790] DMAR-IR: Enabled IRQ remapping in x2apic mode
[ 0.182791] x2apic enabled
[ 0.182814] Switched APIC routing to cluster x2apic.
不知道为什么将 R9000P 的网卡直通之后，重启之后网卡无法使用，需要进入到 windows 去一下
记录下 posted-interrupt
没有
➜ ~ fio test.fio
trash: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=128
fio-3.29
Starting 1 process
^Cbs: 1 (f=1): [r(1)][3.3%][r=1525MiB/s][r=390k IOPS][eta 16m:07s]
fio: terminating on signal 2
trash: (groupid=0, jobs=1): err= 0: pid=1346: Fri Aug 4 19:09:55 2023
read: IOPS=389k, BW=1521MiB/s (1594MB/s)(49.4GiB/33298msec)
slat (nsec): min=963, max=209081, avg=1803.16, stdev=1376.88
clat (usec): min=11, max=6669, avg=326.71, stdev=67.77
lat (usec): min=12, max=6672, avg=328.57, stdev=67.79
clat percentiles (usec):
| 1.00th=[ 229], 5.00th=[ 281], 10.00th=[ 297], 20.00th=[ 306],
| 30.00th=[ 310], 40.00th=[ 314], 50.00th=[ 318], 60.00th=[ 322],
| 70.00th=[ 330], 80.00th=[ 334], 90.00th=[ 347], 95.00th=[ 383],
| 99.00th=[ 668], 99.50th=[ 775], 99.90th=[ 1004], 99.95th=[ 1139],
| 99.99th=[ 1450]
bw ( MiB/s): min= 1046, max= 1582, per=100.00%, avg=1520.94, stdev=80.78, samples=66
iops : min=267904, max=405102, avg=389361.88, stdev=20678.80, samples=66
lat (usec) : 20=0.01%, 50=0.01%, 100=0.08%, 250=1.41%, 500=96.81%
lat (usec) : 750=1.05%, 1000=0.54%
lat (msec) : 2=0.10%, 4=0.01%, 10=0.01%
cpu : usr=15.51%, sys=56.37%, ctx=43957, majf=0, minf=783
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued rwts: total=12961553,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=128
Run status group 0 (all jobs):
READ: bw=1521MiB/s (1594MB/s), 1521MiB/s-1521MiB/s (1594MB/s-1594MB/s), io=49.4GiB (53.1GB), run=33298-33298msec
Disk stats (read/write):
nvme0n1: ios=12901764/0, merge=0/0, ticks=810904/0, in_queue=810904, util=99.75%
有的设备不能正常工作了，我以为这个设备结束了，收回用于 dma 的空间
是不是可以通过 dma 来保证这个设备绝对不可能写内存。
iommu 没有 enable 会导致如下现象
这个命令会失败:
echo 0000:03:00.0 | sudo tee /sys/bus/pci/devices/0000:03:00.0/driver/unbind
echo 1e49 0071 | sudo tee /sys/bus/pci/drivers/vfio-pci/new_id
日志为:
[ 63.467515] VFIO - User Level meta-driver version: 0.3
[ 67.134125] vfio-pci: probe of 0000:03:00.0 failed with error -22
简单跟踪了下，发现在这里开始返回错误: vfio_pci_probe
执行 ls /sys/class/iommu 和 ls /sys/kernel/iommu_groups/ 没有任何内容
但是 dmesg 的结果为:
🤒 sudo dmesg | grep -e DMAR -e IOMMU
[sudo] password for martins3:
[ 0.003961] ACPI: DMAR 0x0000000074411000 000050 (v02 INTEL EDK2 00000002 01000013)
[ 0.003984] ACPI: Reserving DMAR table memory at [mem 0x74411000-0x7441104f]
[ 0.160801] DMAR: Host address width 39
[ 0.160801] DMAR: DRHD base: 0x000000fed91000 flags: 0x1
[ 0.160806] DMAR: dmar0: reg_base_addr fed91000 ver 5:0 cap d2008c40660462 ecap f050da
[ 0.160808] DMAR-IR: IOAPIC id 2 under DRHD base 0xfed91000 IOMMU 0
[ 0.160808] DMAR-IR: HPET id 0 under DRHD base 0xfed91000
[ 0.160809] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.162198] DMAR-IR: Enabled IRQ remapping in x2apic mode
[ 176.360470] AMD-Vi: AMD IOMMUv2 functionality not available on this system - This is not a bug.
当时 swiotlb 中的内容为；
[root@nixos:/sys/kernel/debug/swiotlb]# grep . *
io_tlb_nslabs:32768
io_tlb_used:0
io_tlb_used_hiwater:0
看来，之前一直存在一个错误的想法，那就是 intel 下永远没有 swiotlb ，但是实际上并不是。
[ ] 存在 iommu 的时候，普通的 pci 的探测过程是不是首先探测 iommu ，然后让 iommu 来探测设备
有证据显示 iommu 探测的更早，至少还是 do_initcalls
#0 iommu_group_alloc () at drivers/iommu/iommu.c:933
#1 0xffffffff81b47683 in pci_device_group (dev=<optimized out>) at drivers/iommu/iommu.c:1730
#2 0xffffffff81b4a5e4 in iommu_init_device (ops=0xffffffff826e9ae0 <intel_iommu_ops>, dev=0xffff8881024b40b8) at drivers/iommu/iommu.c:418
#3 __iommu_probe_device (dev=0xffff8881024b40b8, group_list=0xffffc90000017d50) at drivers/iommu/iommu.c:513
#4 0xffffffff81b4a9be in probe_iommu_group (dev=<optimized out>, data=<optimized out>) at drivers/iommu/iommu.c:1825
#5 0xffffffff81b5da95 in bus_for_each_dev (bus=<optimized out>, start=start@entry=0x0 <fixed_percpu_data>, data=data@entry=0xffffc90000017d50,
fn=fn@entry=0xffffffff81b4a9b0 <probe_iommu_group>) at drivers/base/bus.c:368
#6 0xffffffff81b4abb7 in bus_iommu_probe (bus=<optimized out>) at drivers/iommu/iommu.c:1969
#7 0xffffffff81b4ad8c in iommu_device_register (iommu=iommu@entry=0xffff88810034f130, ops=ops@entry=0xffffffff826e9ae0 <intel_iommu_ops>,
hwdev=hwdev@entry=0x0 <fixed_percpu_data>) at drivers/iommu/iommu.c:278
#8 0xffffffff83aa4cbf in intel_iommu_init () at drivers/iommu/intel/iommu.c:3878
#9 0xffffffff83a4a4c2 in pci_iommu_init () at arch/x86/kernel/pci-dma.c:178
#10 0xffffffff81001b78 in do_one_initcall (fn=0xffffffff83a4a4b0 <pci_iommu_init>) at init/main.c:1236
#11 0xffffffff83a3a744 in do_initcall_level (command_line=0xffff888100b5ec00 "root", level=5) at init/main.c:1298
#12 do_initcalls () at init/main.c:1314
#13 do_basic_setup () at init/main.c:1333
#14 kernel_init_freeable () at init/main.c:1551
#15 0xffffffff823a100a in kernel_init (unused=<optimized out>) at init/main.c:1441
#16 0xffffffff810e5b41 in ret_from_fork (prev=<optimized out>, regs=0xffffc90000017f58, fn=0xffffffff823a0ff0 <kernel_init>, fn_arg=0x0 <fixed_percpu_data>)
at arch/x86/kernel/process.c:147
#17 0xffffffff8100259b in ret_from_fork_asm () at arch/x86/entry/entry_64.S:242
#18 0x0000000000000000 in ?? ()
#0 __iommu_probe_device (dev=0xffff8881024b50b8, group_list=0xffffc90000017d50) at drivers/iommu/iommu.c:489
#1 0xffffffff81b4a9be in probe_iommu_group (dev=<optimized out>, data=<optimized out>) at drivers/iommu/iommu.c:1825
#2 0xffffffff81b5da95 in bus_for_each_dev (bus=<optimized out>, start=start@entry=0x0 <fixed_percpu_data>, data=data@entry=0xffffc90000017d50,
fn=fn@entry=0xffffffff81b4a9b0 <probe_iommu_group>) at drivers/base/bus.c:368
#3 0xffffffff81b4abb7 in bus_iommu_probe (bus=<optimized out>) at drivers/iommu/iommu.c:1969
#4 0xffffffff81b4ad8c in iommu_device_register (iommu=iommu@entry=0xffff88810034f130, ops=ops@entry=0xffffffff826e9ae0 <intel_iommu_ops>,
hwdev=hwdev@entry=0x0 <fixed_percpu_data>) at drivers/iommu/iommu.c:278
#5 0xffffffff83aa4cbf in intel_iommu_init () at drivers/iommu/intel/iommu.c:3878
#6 0xffffffff83a4a4c2 in pci_iommu_init () at arch/x86/kernel/pci-dma.c:178
#7 0xffffffff81001b78 in do_one_initcall (fn=0xffffffff83a4a4b0 <pci_iommu_init>) at init/main.c:1236
#8 0xffffffff83a3a744 in do_initcall_level (command_line=0xffff888100b5ec00 "root", level=5) at init/main.c:1298
#9 do_initcalls () at init/main.c:1314
#10 do_basic_setup () at init/main.c:1333
#11 kernel_init_freeable () at init/main.c:1551
#12 0xffffffff823a100a in kernel_init (unused=<optimized out>) at init/main.c:1441
#13 0xffffffff810e5b41 in ret_from_fork (prev=<optimized out>, regs=0xffffc90000017f58, fn=0xffffffff823a0ff0 <kernel_init>, fn_arg=0x0 <fixed_percpu_data>)
at arch/x86/kernel/process.c:147
#14 0xffffffff8100259b in ret_from_fork_asm () at arch/x86/entry/entry_64.S:242
#15 0x0000000000000000 in ?? ()
而 pci 的探测这是通过 thread fn 的:
#0 pci_device_probe (dev=0xffff8881025d90b8) at drivers/pci/pci-driver.c:445
#1 0xffffffff81b5fdbc in call_driver_probe (drv=0xffffffff83274ed8 <nvme_driver+120>, dev=0xffff8881025d90b8) at drivers/base/dd.c:579
#2 really_probe (dev=dev@entry=0xffff8881025d90b8, drv=drv@entry=0xffffffff83274ed8 <nvme_driver+120>) at drivers/base/dd.c:658
#3 0xffffffff81b60043 in __driver_probe_device (drv=0xffffffff83274ed8 <nvme_driver+120>, dev=dev@entry=0xffff8881025d90b8) at drivers/base/dd.c:800
#4 0xffffffff81b6011f in driver_probe_device (drv=<optimized out>, dev=dev@entry=0xffff8881025d90b8) at drivers/base/dd.c:830
#5 0xffffffff81b604f3 in __driver_attach_async_helper (_dev=0xffff8881025d90b8, cookie=<optimized out>) at drivers/base/dd.c:1148
#6 0xffffffff8117c1c1 in async_run_entry_fn (work=0xffff8881090ad5c0) at kernel/async.c:127
#7 0xffffffff81167718 in process_one_work (worker=worker@entry=0xffff888103ccdd80, work=0xffff8881090ad5c0) at kernel/workqueue.c:2630
#8 0xffffffff81167bd5 in process_scheduled_works (worker=<optimized out>) at kernel/workqueue.c:2703
#9 worker_thread (__worker=0xffff888103ccdd80) at kernel/workqueue.c:2784
#10 0xffffffff81171c33 in kthread (_create=0xffff8881025d7b00) at kernel/kthread.c:388
#11 0xffffffff810e5b41 in ret_from_fork (prev=<optimized out>, regs=0xffffc90000503f58, fn=0xffffffff81171b50 <kthread>, fn_arg=0xffff8881025d7b00)
at arch/x86/kernel/process.c:147
#12 0xffffffff8100259b in ret_from_fork_asm () at arch/x86/entry/entry_64.S:242
iommu_group 中的字段是什么含义?
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
struct xarray pasid_array;
struct mutex mutex;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *blocking_domain;
struct iommu_domain *domain;
struct list_head entry;
unsigned int owner_cnt;
void *owner;
};
才发现注意到内核文档中所说的，当创建一个软链接的时候，
a new VFIO group will appear for the group as /dev/vfio/$GROUP, where $GROUP is the IOMMU group number of which the device is a member.
/sys/kernel/iommu_groups ├── 16 │ ├── devices │ │ └── 0000:03:00.0 -> ../../../../devices/pci0000:00/0000:00:1a.0/0000:03:00.0 │ ├── reserved_regions │ └── type
🧀 ls /dev/vfio
16 vfio
从这个路径，我们可以看到 iommu group 分配 iommu domain
#0 __iommu_domain_alloc (ops=0xffffffff826e9ae0 <intel_iommu_ops>, dev=0xffff8881024ea0b8, type=3) at drivers/iommu/iommu.c:2084
#1 0xffffffff81b4a1f4 in __iommu_group_domain_alloc (group=0xffff88810277f800, type=<optimized out>) at drivers/iommu/iommu.c:2127
#2 __iommu_group_alloc_default_domain (req_type=<optimized out>, group=0xffff88810277f800) at drivers/iommu/iommu.c:1752
#3 iommu_group_alloc_default_domain (req_type=0, group=0xffff88810277f800) at drivers/iommu/iommu.c:1799
#4 iommu_setup_default_domain (group=group@entry=0xffff88810277f800, target_type=target_type@entry=0) at drivers/iommu/iommu.c:3043
#5 0xffffffff81b4ac1a in bus_iommu_probe (bus=<optimized out>) at drivers/iommu/iommu.c:1986
#6 0xffffffff81b4ad8c in iommu_device_register (iommu=iommu@entry=0xffff888100343130, ops=ops@entry=0xffffffff826e9ae0 <intel_iommu_ops>,
hwdev=hwdev@entry=0x0 <fixed_percpu_data>) at drivers/iommu/iommu.c:278
#7 0xffffffff83aa4cbf in intel_iommu_init () at drivers/iommu/intel/iommu.c:3878
#8 0xffffffff83a4a4c2 in pci_iommu_init () at arch/x86/kernel/pci-dma.c:178
#9 0xffffffff81001b78 in do_one_initcall (fn=0xffffffff83a4a4b0 <pci_iommu_init>) at init/main.c:1236
#10 0xffffffff83a3a744 in do_initcall_level (command_line=0xffff888100b52c00 "root", level=5) at init/main.c:1298
#11 do_initcalls () at init/main.c:1314
#12 do_basic_setup () at init/main.c:1333
#13 kernel_init_freeable () at init/main.c:1551
#14 0xffffffff823a100a in kernel_init (unused=<optimized out>) at init/main.c:1441
#15 0xffffffff810e5b41 in ret_from_fork (prev=<optimized out>, regs=0xffffc90000017f58, fn=0xffffffff823a0ff0 <kernel_init>, fn_arg=0x0 <fixed_percpu_data>)
at arch/x86/kernel/process.c:147
#16 0xffffffff8100259b in ret_from_fork_asm () at arch/x86/entry/entry_64.S:242
iommu domain 和 iommu group
为什么需要创建出来 iommu domain 的概念？
- vfio_iommu_type1_attach_group <———— 看看这里，在这里我们找到看到 domain 是现场制作的
- iommu_attach_group
__iommu_attach_group__iommu_group_set_domain__iommu_group_set_domain_internalfor_each_group_device(group, gdev) { ret = __iommu_device_set_domain(group, gdev->dev, new_domain, flags); // ... } group->domain = new_domain;
- iommu_attach_group
目前看，iommu domain 和 vfio container 是有关系的 iommu group 和 vfio group 是关联的
- 但是，看上去 vfio domain 和 iommu domain 是相关
- vfio_container vfio_domain vfio_iommu_group vfio_iommu 是啥关系
观察 iommu_group_store_type 的实现，居然所谓的设置 iommu_group 的 type 是设置 defautl domain
ret = iommu_setup_default_domain(group, req_type);
为什么 amd iommu 就是一个 PCI 设备，而 intel iommu lspci 则是看不到的
AMD 中的 IRQ_REMAP_X2APIC_MODE 和 IRQ_REMAP_XAPIC_MODE
static int iommu_init_irq(struct amd_iommu *iommu)
{
int ret;
if (iommu->int_enabled)
goto enable_faults;
if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
ret = iommu_setup_intcapxt(iommu);
else if (iommu->dev->msi_cap)
ret = iommu_setup_msi(iommu);
else
ret = -ENODEV;
if (ret)
return ret;
iommu->int_enabled = true;
enable_faults:
if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
return 0;
}
iommu 等于 pt 未必所有的 domain 都是 pt 的，那个只是默认数值
当时 kernel 的实现，bridge 下的需要
lrwxrwxrwx 1 root root 0 Nov 19 12:41 0000:00:04.0 -> ../../../../devices/pci0000:00/0000:00:04.0
lrwxrwxrwx 1 root root 0 Nov 19 12:41 0000:01:01.0 -> ../../../../devices/pci0000:00/0000:00:04.0/0000:01:01.0
[root@localhost 12:41:55 devices]$ lspci
00:00.0 Host bridge: Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
00:01.0 VGA compatible controller: Device 1234:1111 (rev 02)
00:02.0 SCSI storage controller: Virtio: Virtio block device
00:03.0 SCSI storage controller: Virtio: Virtio SCSI
00:04.0 PCI bridge: Red Hat, Inc. QEMU PCI-PCI bridge
00:05.0 Ethernet controller: Virtio: Virtio network device
00:06.0 Unclassified device [00ff]: Virtio: Virtio memory balloon
00:07.0 Unclassified device [0002]: Virtio: Virtio filesystem
00:08.0 SCSI storage controller: Virtio: Virtio SCSI
00:09.0 Non-Volatile memory controller: Red Hat, Inc. Device 0010 (rev 02)
00:0a.0 SCSI storage controller: Virtio: Virtio block device
00:0b.0 SCSI storage controller: Virtio: Virtio block device
00:1f.0 ISA bridge: Intel Corporation 82801IB (ICH9) LPC Interface Controller (rev 02)
00:1f.2 SATA controller: Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode] (rev 02)
00:1f.3 SMBus: Intel Corporation 82801I (ICH9 Family) SMBus Controller (rev 02)
01:01.0 Non-Volatile memory controller: Red Hat, Inc. Device 0010 (rev 02)
物理的机器上，存在类似的场景:
total 0
drwxr-xr-x 2 root root 0 Jun 17 22:06 .
drwxr-xr-x 3 root root 0 Jun 17 22:06 ..
lrwxrwxrwx 1 root root 0 Nov 19 12:43 0000:00:1a.0 -> ../../../../devices/pci0000:00/0000:00:1a.0
/sys/kernel/iommu_groups/40/devices /sys/kernel/iommu_groups
00:1a.0 USB controller: Intel Corporation C610/X99 series chipset USB Enhanced Host Controller #2 (rev 05)
/sys/kernel/iommu_groups
total 0
drwxr-xr-x 2 root root 0 Jun 17 22:06 .
drwxr-xr-x 3 root root 0 Jun 17 22:06 ..
lrwxrwxrwx 1 root root 0 Nov 19 12:47 0000:00:1d.0 -> ../../../../devices/pci0000:00/0000:00:1d.0
/sys/kernel/iommu_groups/43/devices /sys/kernel/iommu_groups
00:1d.0 USB controller: Intel Corporation C610/X99 series chipset USB Enhanced Host Controller #1 (rev 05)
/sys/kernel/iommu_groups
total 0
drwxr-xr-x 2 root root 0 Jun 17 22:06 .
drwxr-xr-x 3 root root 0 Jun 17 22:06 ..
lrwxrwxrwx 1 root root 0 Nov 19 12:47 0000:06:00.0 -> ../../../../devices/pci0000:00/0000:00:1c.2/0000:06:00.0
lrwxrwxrwx 1 root root 0 Nov 19 12:47 0000:07:00.0 -> ../../../../devices/pci0000:00/0000:00:1c.2/0000:06:00.0/0000:07:00.0
/sys/kernel/iommu_groups/49/devices /sys/kernel/iommu_groups
cd /sys/kernel/iommu_groups/
for i in */; do
if [[ $(cat "$i"/type) == DMA ]]; then
ls -la "$i"/devices
pushd "$i"/devices
for d in *; do
lspci -s $d
done
popd
fi
done
TODO
IOMMU_DEFAULT_DMA_LAZY
如果理解 CONFIG_VFIO_IOMMU_TYPE1
跟踪下 intel 和 amd 对应于 virtio_iommu_switch_address_space 的操作是什么含义?
[ ] 可能是多个 IOMMU 的问题吗?
https://stackoverflow.com/questions/60219639/kernel-error-irq-remapping-doesnt-support-x2apic-mode-disabled-x2apic
VF 直通到虚拟机的时候，是不是需要对应内核驱动在 host 的支持
iommu=pt 可以实现 DMA 保护吗？
hyperv-iommu 是啥？
Symbol: HYPERV_IOMMU [=n] │
Type : bool │
Defined at drivers/iommu/Kconfig:462 │
Prompt: Hyper-V IRQ Handling │
Depends on: IOMMU_SUPPORT [=y] && HYPERV [=n] && X86 │
Location: │
-> Device Drivers │
(5) -> IOMMU Hardware Support (IOMMU_SUPPORT [=y]) │
-> Hyper-V IRQ Handling (HYPERV_IOMMU [=n]) │
Selects: IOMMU_API [=y]
问题 iommu=pt ，swiotlb , iommu 那种模式可以检出来 dma 错误
virtio_iommu 居然是不支持 interrupt remapping 的，有点意思
IOMMU page fault 具体代码在哪里?
https://mysummary.readthedocs.io/zh/latest/%E8%BD%AF%E4%BB%B6%E6%9E%84%E6%9E%B6%E8%AE%BE%E8%AE%A1/%E5%9C%B0%E5%9D%80%E7%A9%BA%E9%97%B4%E7%9A%84%E6%95%85%E4%BA%8B.html#id7
config INTEL_IOMMU_SVM
bool "Support for Shared Virtual Memory with Intel IOMMU"
depends on X86_64
select MMU_NOTIFIER
select IOMMU_SVA
select IOMMU_IOPF
help
Shared Virtual Memory (SVM) provides a facility for devices
to access DMA resources through process address space by
means of a Process Address Space ID (PASID).
几个简单的问题，似乎还是无法肯定回答
iommu=pt 到底意味着啥来着?
为什么 iommu=pt 的时候，GPU 直通玩原神的有问题?
iommu=pt 是默认打开的吗? 如果是，GPU 直通为什么后来为什么会出现问题?
iommu=pt 之后，是不是会影响 swiotlb 呀?
iommu=pt 之后，iommu 硬件是不是完全不工作？
理解一下 vhost 中的 iommu 吧
例如在
static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,
u64 iova, u64 size,
u64 pa, u32 perm, void *opaque)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
int ret;
if (asid >= vdpasim->dev_attr.nas)
return -EINVAL;
spin_lock(&vdpasim->iommu_lock);
if (vdpasim->iommu_pt[asid]) {
vhost_iotlb_reset(&vdpasim->iommu[asid]);
vdpasim->iommu_pt[asid] = false;
}
ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,
iova + size - 1, pa, perm, opaque);
spin_unlock(&vdpasim->iommu_lock);
return ret;
}
在例如 drivers/vhost/vringh.c 中无数的 iotlb 。
各个 iommu 架构的实现入口
static const struct iommu_ops viommu_ops = {
.capable = viommu_capable,
.domain_alloc_identity = viommu_domain_alloc_identity,
.domain_alloc_paging = viommu_domain_alloc_paging,
.probe_device = viommu_probe_device,
.release_device = viommu_release_device,
.device_group = viommu_device_group,
.get_resv_regions = viommu_get_resv_regions,
.of_xlate = viommu_of_xlate,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = viommu_attach_dev,
.map_pages = viommu_map_pages,
.unmap_pages = viommu_unmap_pages,
.iova_to_phys = viommu_iova_to_phys,
.flush_iotlb_all = viommu_flush_iotlb_all,
.iotlb_sync = viommu_iotlb_sync,
.iotlb_sync_map = viommu_iotlb_sync_map,
.free = viommu_domain_free,
}
};
const struct iommu_ops intel_iommu_ops = {
.blocked_domain = &blocking_domain,
.release_domain = &blocking_domain,
.identity_domain = &identity_domain,
.capable = intel_iommu_capable,
.hw_info = intel_iommu_hw_info,
.domain_alloc_paging_flags = intel_iommu_domain_alloc_paging_flags,
.domain_alloc_sva = intel_svm_domain_alloc,
.domain_alloc_nested = intel_iommu_domain_alloc_nested,
.probe_device = intel_iommu_probe_device,
.probe_finalize = intel_iommu_probe_finalize,
.release_device = intel_iommu_release_device,
.get_resv_regions = intel_iommu_get_resv_regions,
.device_group = intel_iommu_device_group,
.is_attach_deferred = intel_iommu_is_attach_deferred,
.def_domain_type = device_def_domain_type,
.page_response = intel_iommu_page_response,
};
本站所有文章转发 CSDN 将按侵权追究法律责任，其它情况随意。