pci-dma.c 5.31 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
#include <linux/dma-direct.h>
3
#include <linux/dma-debug.h>
4
#include <linux/dmar.h>
5
#include <linux/export.h>
6
#include <linux/bootmem.h>
7
#include <linux/gfp.h>
8
#include <linux/pci.h>
9

10 11
#include <asm/proto.h>
#include <asm/dma.h>
12
#include <asm/iommu.h>
13
#include <asm/gart.h>
14
#include <asm/calgary.h>
15
#include <asm/x86_init.h>
16
#include <asm/iommu_table.h>
Glauber Costa's avatar
Glauber Costa committed
17

18 19
static int forbid_dac __read_mostly;

20
const struct dma_map_ops *dma_ops = &dma_direct_ops;
21 22
EXPORT_SYMBOL(dma_ops);

23
static int iommu_sac_force __read_mostly;
Glauber Costa's avatar
Glauber Costa committed
24

Glauber Costa's avatar
Glauber Costa committed
25 26 27 28 29 30 31 32
#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

33 34 35 36 37 38
int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

39 40
/*
 * This variable becomes 1 if iommu=pt is passed on the kernel command line.
41
 * If this variable is 1, IOMMU implementations do no DMA translation for
42
 * devices and allow every device to access to whole physical memory. This is
43
 * useful if a user wants to use an IOMMU only for KVM device assignment to
44 45 46
 * guests and not for driver dma translation.
 */
int iommu_pass_through __read_mostly;
47

48 49
extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];

50
/* Dummy device used for NULL arguments (normally ISA). */
51
struct device x86_dma_fallback_dev = {
52
	.init_name = "fallback device",
53
	.coherent_dma_mask = ISA_DMA_BIT_MASK,
54
	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
Glauber Costa's avatar
Glauber Costa committed
55
};
56
EXPORT_SYMBOL(x86_dma_fallback_dev);
Glauber Costa's avatar
Glauber Costa committed
57

58
/* Number of entries preallocated for DMA-API debugging */
59
#define PREALLOC_DMA_DEBUG_ENTRIES       65536
60

61 62
void __init pci_iommu_alloc(void)
{
63 64 65 66
	struct iommu_table_entry *p;

	sort_iommu_table(__iommu_table, __iommu_table_end);
	check_iommu_entries(__iommu_table, __iommu_table_end);
67

68 69 70 71 72 73 74 75 76
	for (p = __iommu_table; p < __iommu_table_end; p++) {
		if (p && p->detect && p->detect() > 0) {
			p->flags |= IOMMU_DETECTED;
			if (p->early_init)
				p->early_init();
			if (p->flags & IOMMU_FINISH_IF_DETECTED)
				break;
		}
	}
77
}
78

79
bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
80
{
81 82 83
	if (!*dev)
		*dev = &x86_dma_fallback_dev;

84 85 86
	if (!is_device_dma_capable(*dev))
		return false;
	return true;
87

88
}
89
EXPORT_SYMBOL(arch_dma_alloc_attrs);
90

91
/*
Paul Bolle's avatar
Paul Bolle committed
92 93
 * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
 * parameter documentation.
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
132
			forbid_dac = 1;
133 134 135 136 137 138 139
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
140
#endif
141
		if (!strncmp(p, "pt", 2))
Fenghua Yu's avatar
Fenghua Yu committed
142
			iommu_pass_through = 1;
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

159
int arch_dma_supported(struct device *dev, u64 mask)
Glauber Costa's avatar
Glauber Costa committed
160 161 162
{
#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
163
		dev_info(dev, "PCI: Disallowing DAC for device\n");
Glauber Costa's avatar
Glauber Costa committed
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
		return 0;
	}
#endif

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
180
	if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
181
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
Glauber Costa's avatar
Glauber Costa committed
182 183 184 185 186
		return 0;
	}

	return 1;
}
187 188
EXPORT_SYMBOL(arch_dma_supported);

189 190
static int __init pci_iommu_init(void)
{
191
	struct iommu_table_entry *p;
192 193
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

194 195 196
#ifdef CONFIG_PCI
	dma_debug_add_bus(&pci_bus_type);
#endif
197 198
	x86_init.iommu.iommu_init();

199 200 201 202
	for (p = __iommu_table; p < __iommu_table_end; p++) {
		if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
			p->late_init();
	}
203

204 205 206
	return 0;
}
/* Must execute after PCI subsystem */
207
rootfs_initcall(pci_iommu_init);
208 209 210 211

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

212
static void via_no_dac(struct pci_dev *dev)
213
{
214
	if (forbid_dac == 0) {
215
		dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
216 217 218
		forbid_dac = 1;
	}
}
219 220
DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID,
				PCI_CLASS_BRIDGE_PCI, 8, via_no_dac);
221
#endif