idt.c 9.47 KB
Newer Older
1
2
3
4
5
6
7
/*
 * Interrupt descriptor table related code
 *
 * This file is licensed under the GPL V2
 */
#include <linux/interrupt.h>

8
9
#include <asm/traps.h>
#include <asm/proto.h>
10
11
#include <asm/desc.h>

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
struct idt_data {
	unsigned int	vector;
	unsigned int	segment;
	struct idt_bits	bits;
	const void	*addr;
};

#define DPL0		0x0
#define DPL3		0x3

#define DEFAULT_STACK	0

#define G(_vector, _addr, _ist, _type, _dpl, _segment)	\
	{						\
		.vector		= _vector,		\
		.bits.ist	= _ist,			\
		.bits.type	= _type,		\
		.bits.dpl	= _dpl,			\
		.bits.p		= 1,			\
		.addr		= _addr,		\
		.segment	= _segment,		\
	}

/* Interrupt gate */
#define INTG(_vector, _addr)				\
	G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS)

/* System interrupt gate */
#define SYSG(_vector, _addr)				\
	G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)

/* Interrupt gate with interrupt stack */
#define ISTG(_vector, _addr, _ist)			\
	G(_vector, _addr, _ist, GATE_INTERRUPT, DPL0, __KERNEL_CS)

47
48
49
50
/* System interrupt gate with interrupt stack */
#define SISTG(_vector, _addr, _ist)			\
	G(_vector, _addr, _ist, GATE_INTERRUPT, DPL3, __KERNEL_CS)

51
52
53
54
/* Task gate */
#define TSKG(_vector, _gdt)				\
	G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)

55
56
57
58
59
60
61
62
63
64
65
66
/*
 * Early traps running on the DEFAULT_STACK because the other interrupt
 * stacks work only after cpu_init().
 */
static const __initdata struct idt_data early_idts[] = {
	INTG(X86_TRAP_DB,		debug),
	SYSG(X86_TRAP_BP,		int3),
#ifdef CONFIG_X86_32
	INTG(X86_TRAP_PF,		page_fault),
#endif
};

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
/*
 * The default IDT entries which are set up in trap_init() before
 * cpu_init() is invoked. Interrupt stacks cannot be used at that point and
 * the traps which use them are reinitialized with IST after cpu_init() has
 * set up TSS.
 */
static const __initdata struct idt_data def_idts[] = {
	INTG(X86_TRAP_DE,		divide_error),
	INTG(X86_TRAP_NMI,		nmi),
	INTG(X86_TRAP_BR,		bounds),
	INTG(X86_TRAP_UD,		invalid_op),
	INTG(X86_TRAP_NM,		device_not_available),
	INTG(X86_TRAP_OLD_MF,		coprocessor_segment_overrun),
	INTG(X86_TRAP_TS,		invalid_TSS),
	INTG(X86_TRAP_NP,		segment_not_present),
	INTG(X86_TRAP_SS,		stack_segment),
	INTG(X86_TRAP_GP,		general_protection),
	INTG(X86_TRAP_SPURIOUS,		spurious_interrupt_bug),
	INTG(X86_TRAP_MF,		coprocessor_error),
	INTG(X86_TRAP_AC,		alignment_check),
	INTG(X86_TRAP_XF,		simd_coprocessor_error),

#ifdef CONFIG_X86_32
	TSKG(X86_TRAP_DF,		GDT_ENTRY_DOUBLEFAULT_TSS),
#else
	INTG(X86_TRAP_DF,		double_fault),
#endif
	INTG(X86_TRAP_DB,		debug),

#ifdef CONFIG_X86_MCE
	INTG(X86_TRAP_MC,		&machine_check),
#endif

	SYSG(X86_TRAP_OF,		overflow),
#if defined(CONFIG_IA32_EMULATION)
	SYSG(IA32_SYSCALL_VECTOR,	entry_INT80_compat),
#elif defined(CONFIG_X86_32)
	SYSG(IA32_SYSCALL_VECTOR,	entry_INT80_32),
#endif
};

108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
/*
 * The APIC and SMP idt entries
 */
static const __initdata struct idt_data apic_idts[] = {
#ifdef CONFIG_SMP
	INTG(RESCHEDULE_VECTOR,		reschedule_interrupt),
	INTG(CALL_FUNCTION_VECTOR,	call_function_interrupt),
	INTG(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt),
	INTG(IRQ_MOVE_CLEANUP_VECTOR,	irq_move_cleanup_interrupt),
	INTG(REBOOT_VECTOR,		reboot_interrupt),
#endif

#ifdef CONFIG_X86_THERMAL_VECTOR
	INTG(THERMAL_APIC_VECTOR,	thermal_interrupt),
#endif

#ifdef CONFIG_X86_MCE_THRESHOLD
	INTG(THRESHOLD_APIC_VECTOR,	threshold_interrupt),
#endif

#ifdef CONFIG_X86_MCE_AMD
	INTG(DEFERRED_ERROR_VECTOR,	deferred_error_interrupt),
#endif

#ifdef CONFIG_X86_LOCAL_APIC
	INTG(LOCAL_TIMER_VECTOR,	apic_timer_interrupt),
	INTG(X86_PLATFORM_IPI_VECTOR,	x86_platform_ipi),
# ifdef CONFIG_HAVE_KVM
	INTG(POSTED_INTR_VECTOR,	kvm_posted_intr_ipi),
	INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi),
	INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi),
# endif
# ifdef CONFIG_IRQ_WORK
	INTG(IRQ_WORK_VECTOR,		irq_work_interrupt),
# endif
	INTG(SPURIOUS_APIC_VECTOR,	spurious_interrupt),
	INTG(ERROR_APIC_VECTOR,		error_interrupt),
#endif
};

148
149
150
151
152
153
154
155
#ifdef CONFIG_X86_64
/*
 * Early traps running on the DEFAULT_STACK because the other interrupt
 * stacks work only after cpu_init().
 */
static const __initdata struct idt_data early_pf_idts[] = {
	INTG(X86_TRAP_PF,		page_fault),
};
156
157
158
159
160
161
162
163
164

/*
 * Override for the debug_idt. Same as the default, but with interrupt
 * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
 */
static const __initdata struct idt_data dbg_idts[] = {
	INTG(X86_TRAP_DB,	debug),
	INTG(X86_TRAP_BP,	int3),
};
165
166
#endif

167
168
169
170
/* Must be page-aligned because the real IDT is used in a fixmap. */
gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;

struct desc_ptr idt_descr __ro_after_init = {
171
	.size		= (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1,
172
173
174
	.address	= (unsigned long) idt_table,
};

175
176
177
178
#ifdef CONFIG_X86_64
/* No need to be aligned, but done to keep all IDTs defined the same way. */
gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss;

179
180
181
182
183
184
185
/*
 * The exceptions which use Interrupt stacks. They are setup after
 * cpu_init() when the TSS has been initialized.
 */
static const __initdata struct idt_data ist_idts[] = {
	ISTG(X86_TRAP_DB,	debug,		DEBUG_STACK),
	ISTG(X86_TRAP_NMI,	nmi,		NMI_STACK),
186
	SISTG(X86_TRAP_BP,	int3,		DEBUG_STACK),
187
188
189
190
191
192
	ISTG(X86_TRAP_DF,	double_fault,	DOUBLEFAULT_STACK),
#ifdef CONFIG_X86_MCE
	ISTG(X86_TRAP_MC,	&machine_check,	MCE_STACK),
#endif
};

193
194
195
196
/*
 * Override for the debug_idt. Same as the default, but with interrupt
 * stack set to DEFAULT_STACK (0). Required for NMI trap handling.
 */
197
198
199
200
201
const struct desc_ptr debug_idt_descr = {
	.size		= IDT_ENTRIES * 16 - 1,
	.address	= (unsigned long) debug_idt_table,
};
#endif
202

203
204
205
206
207
208
209
210
211
212
213
214
215
216
static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
{
	unsigned long addr = (unsigned long) d->addr;

	gate->offset_low	= (u16) addr;
	gate->segment		= (u16) d->segment;
	gate->bits		= d->bits;
	gate->offset_middle	= (u16) (addr >> 16);
#ifdef CONFIG_X86_64
	gate->offset_high	= (u32) (addr >> 32);
	gate->reserved		= 0;
#endif
}

217
218
static void
idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys)
219
220
221
222
223
224
{
	gate_desc desc;

	for (; size > 0; t++, size--) {
		idt_init_desc(&desc, t);
		write_idt_entry(idt, t->vector, &desc);
225
226
		if (sys)
			set_bit(t->vector, used_vectors);
227
228
229
	}
}

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
static void set_intr_gate(unsigned int n, const void *addr)
{
	struct idt_data data;

	BUG_ON(n > 0xFF);

	memset(&data, 0, sizeof(data));
	data.vector	= n;
	data.addr	= addr;
	data.segment	= __KERNEL_CS;
	data.bits.type	= GATE_INTERRUPT;
	data.bits.p	= 1;

	idt_setup_from_table(idt_table, &data, 1, false);
}

246
247
248
249
250
251
252
253
254
/**
 * idt_setup_early_traps - Initialize the idt table with early traps
 *
 * On X8664 these traps do not use interrupt stacks as they can't work
 * before cpu_init() is invoked and sets up TSS. The IST variants are
 * installed after that.
 */
void __init idt_setup_early_traps(void)
{
255
256
	idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts),
			     true);
257
258
259
	load_idt(&idt_descr);
}

260
261
262
263
264
/**
 * idt_setup_traps - Initialize the idt table with default traps
 */
void __init idt_setup_traps(void)
{
265
	idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true);
266
267
}

268
269
270
271
272
273
274
275
276
277
278
279
280
281
#ifdef CONFIG_X86_64
/**
 * idt_setup_early_pf - Initialize the idt table with early pagefault handler
 *
 * On X8664 this does not use interrupt stacks as they can't work before
 * cpu_init() is invoked and sets up TSS. The IST variant is installed
 * after that.
 *
 * FIXME: Why is 32bit and 64bit installing the PF handler at different
 * places in the early setup code?
 */
void __init idt_setup_early_pf(void)
{
	idt_setup_from_table(idt_table, early_pf_idts,
282
			     ARRAY_SIZE(early_pf_idts), true);
283
}
284

285
286
287
288
289
/**
 * idt_setup_ist_traps - Initialize the idt table with traps using IST
 */
void __init idt_setup_ist_traps(void)
{
290
	idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
291
292
}

293
294
295
296
297
298
299
/**
 * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps
 */
void __init idt_setup_debugidt_traps(void)
{
	memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16);

300
	idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false);
301
}
302
303
#endif

304
305
306
307
308
/**
 * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
 */
void __init idt_setup_apic_and_irq_gates(void)
{
309
310
311
	int i = FIRST_EXTERNAL_VECTOR;
	void *entry;

312
	idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327

	for_each_clear_bit_from(i, used_vectors, FIRST_SYSTEM_VECTOR) {
		entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
		set_intr_gate(i, entry);
	}

	for_each_clear_bit_from(i, used_vectors, NR_VECTORS) {
#ifdef CONFIG_X86_LOCAL_APIC
		set_bit(i, used_vectors);
		set_intr_gate(i, spurious_interrupt);
#else
		entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
		set_intr_gate(i, entry);
#endif
	}
328
329
}

330
331
332
333
334
335
336
337
338
/**
 * idt_setup_early_handler - Initializes the idt table with early handlers
 */
void __init idt_setup_early_handler(void)
{
	int i;

	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
		set_intr_gate(i, early_idt_handler_array[i]);
339
340
341
342
#ifdef CONFIG_X86_32
	for ( ; i < NR_VECTORS; i++)
		set_intr_gate(i, early_ignore_irq);
#endif
343
344
345
	load_idt(&idt_descr);
}

346
347
348
349
350
351
352
353
354
355
/**
 * idt_invalidate - Invalidate interrupt descriptor table
 * @addr:	The virtual address of the 'invalid' IDT
 */
void idt_invalidate(void *addr)
{
	struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 };

	load_idt(&idt);
}
356

357
void __init update_intr_gate(unsigned int n, const void *addr)
358
{
359
360
361
	if (WARN_ON_ONCE(!test_bit(n, used_vectors)))
		return;
	set_intr_gate(n, addr);
362
363
364
365
}

void alloc_intr_gate(unsigned int n, const void *addr)
{
366
367
368
	BUG_ON(n < FIRST_SYSTEM_VECTOR);
	if (!test_and_set_bit(n, used_vectors))
		set_intr_gate(n, addr);
369
}