processor.h 24 KB
Newer Older
1 2
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
3

4 5
#include <asm/processor-flags.h>

6 7 8
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
9
struct vm86;
10

11 12 13
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
14
#include <uapi/asm/sigcontext.h>
15
#include <asm/current.h>
16
#include <asm/cpufeatures.h>
17
#include <asm/page.h>
18
#include <asm/pgtable_types.h>
19
#include <asm/percpu.h>
20 21
#include <asm/msr.h>
#include <asm/desc_defs.h>
22
#include <asm/nops.h>
23
#include <asm/special_insns.h>
24
#include <asm/fpu/types.h>
25
#include <asm/unwind_hints.h>
26

27
#include <linux/personality.h>
28
#include <linux/cache.h>
29
#include <linux/threads.h>
30
#include <linux/math64.h>
31
#include <linux/err.h>
32
#include <linux/irqflags.h>
33
#include <linux/mem_encrypt.h>
34 35 36 37 38 39 40 41

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN	0
42

43
#define HBP_NUM 4
44 45 46 47 48 49 50
/*
 * Default implementation of macro that returns current
 * instruction pointer ("program counter").
 */
static inline void *current_text_addr(void)
{
	void *pc;
51 52 53

	asm volatile("mov $1f, %0; 1:":"=r" (pc));

54 55 56
	return pc;
}

57 58 59 60 61
/*
 * These alignment constraints are for performance in the vSMP case,
 * but in the task_struct case we must also meet hardware imposed
 * alignment requirements of the FPU state:
 */
62
#ifdef CONFIG_X86_VSMP
63 64
# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
65
#else
66
# define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
67
# define ARCH_MIN_MMSTRUCT_ALIGN	0
68 69
#endif

70 71 72 73 74 75 76 77 78 79 80
enum tlb_infos {
	ENTRIES,
	NR_INFO
};

extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
81
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
82

83 84
/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
85
 *  Members of this structure are referenced in head_32.S, so think twice
86 87 88 89
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
90 91 92 93
	__u8			x86;		/* CPU family */
	__u8			x86_vendor;	/* CPU vendor */
	__u8			x86_model;
	__u8			x86_mask;
94
#ifdef CONFIG_X86_64
95
	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
96
	int			x86_tlbsize;
97
#endif
98 99 100 101
	__u8			x86_virt_bits;
	__u8			x86_phys_bits;
	/* CPUID returned core id bits: */
	__u8			x86_coreid_bits;
102
	__u8			cu_id;
103 104 105 106
	/* Max extended CPUID function supported: */
	__u32			extended_cpuid_level;
	/* Maximum supported CPUID level, -1=no CPUID: */
	int			cpuid_level;
107
	__u32			x86_capability[NCAPINTS + NBUGINTS];
108 109 110 111 112
	char			x86_vendor_id[16];
	char			x86_model_id[64];
	/* in KB - valid for CPUS which support this call: */
	int			x86_cache_size;
	int			x86_cache_alignment;	/* In bytes */
113 114 115
	/* Cache QoS architectural values: */
	int			x86_cache_max_rmid;	/* max index */
	int			x86_cache_occ_scale;	/* scale to bytes */
116 117 118 119 120
	int			x86_power;
	unsigned long		loops_per_jiffy;
	/* cpuid returned max cores value: */
	u16			 x86_max_cores;
	u16			apicid;
Yinghai Lu's avatar
Yinghai Lu committed
121
	u16			initial_apicid;
122 123 124 125 126
	u16			x86_clflush_size;
	/* number of cores as seen by the OS: */
	u16			booted_cores;
	/* Physical processor id: */
	u16			phys_proc_id;
127 128
	/* Logical processor id: */
	u16			logical_proc_id;
129 130 131 132
	/* Core id: */
	u16			cpu_core_id;
	/* Index into per_cpu list: */
	u16			cpu_index;
133
	u32			microcode;
134
} __randomize_layout;
135

136 137 138 139 140 141 142 143 144 145 146
struct cpuid_regs {
	u32 eax, ebx, ecx, edx;
};

enum cpuid_regs_idx {
	CPUID_EAX = 0,
	CPUID_EBX,
	CPUID_ECX,
	CPUID_EDX,
};

147 148 149 150 151 152 153 154 155 156
#define X86_VENDOR_INTEL	0
#define X86_VENDOR_CYRIX	1
#define X86_VENDOR_AMD		2
#define X86_VENDOR_UMC		3
#define X86_VENDOR_CENTAUR	5
#define X86_VENDOR_TRANSMETA	7
#define X86_VENDOR_NSC		8
#define X86_VENDOR_NUM		9

#define X86_VENDOR_UNKNOWN	0xff
157

158 159 160
/*
 * capabilities of CPUs
 */
161 162 163 164
extern struct cpuinfo_x86	boot_cpu_data;
extern struct cpuinfo_x86	new_cpu_data;

extern struct tss_struct	doublefault_tss;
165 166
extern __u32			cpu_caps_cleared[NCAPINTS];
extern __u32			cpu_caps_set[NCAPINTS];
167 168

#ifdef CONFIG_SMP
169
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
170 171
#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
#else
172
#define cpu_info		boot_cpu_data
173 174 175
#define cpu_data(cpu)		boot_cpu_data
#endif

176 177
extern const struct seq_operations cpuinfo_op;

178 179 180
#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);
181

182
extern void early_cpu_init(void);
183 184
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
185
extern void print_cpu_info(struct cpuinfo_x86 *);
186
void print_cpu_msr(struct cpuinfo_x86 *);
187
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
188 189 190
extern u32 get_scattered_cpuid_leaf(unsigned int level,
				    unsigned int sub_leaf,
				    enum cpuid_regs_idx reg);
191
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
192
extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
193

194
extern void detect_extended_topology(struct cpuinfo_x86 *c);
195 196
extern void detect_ht(struct cpuinfo_x86 *c);

197 198 199 200 201 202 203 204
#ifdef CONFIG_X86_32
extern int have_cpuid_p(void);
#else
static inline int have_cpuid_p(void)
{
	return 1;
}
#endif
205
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
206
				unsigned int *ecx, unsigned int *edx)
207 208
{
	/* ecx is often an input as well as an output. */
209
	asm volatile("cpuid"
210 211 212 213
	    : "=a" (*eax),
	      "=b" (*ebx),
	      "=c" (*ecx),
	      "=d" (*edx)
214 215
	    : "0" (*eax), "2" (*ecx)
	    : "memory");
216 217
}

218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
#define native_cpuid_reg(reg)					\
static inline unsigned int native_cpuid_##reg(unsigned int op)	\
{								\
	unsigned int eax = op, ebx, ecx = 0, edx;		\
								\
	native_cpuid(&eax, &ebx, &ecx, &edx);			\
								\
	return reg;						\
}

/*
 * Native CPUID functions returning a single datum.
 */
native_cpuid_reg(eax)
native_cpuid_reg(ebx)
native_cpuid_reg(ecx)
native_cpuid_reg(edx)

236 237 238 239 240 241 242 243
/*
 * Friendlier CR3 helpers.
 */
static inline unsigned long read_cr3_pa(void)
{
	return __read_cr3() & CR3_ADDR_MASK;
}

244 245 246 247 248
static inline unsigned long native_read_cr3_pa(void)
{
	return __native_read_cr3() & CR3_ADDR_MASK;
}

249 250
static inline void load_cr3(pgd_t *pgdir)
{
251
	write_cr3(__sme_pa(pgdir));
252
}
253

254 255 256
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
257 258 259
	unsigned short		back_link, __blh;
	unsigned long		sp0;
	unsigned short		ss0, __ss0h;
260
	unsigned long		sp1;
261 262

	/*
263 264 265 266 267 268
	 * We don't use ring 1, so ss1 is a convenient scratch space in
	 * the same cacheline as sp0.  We use ss1 to cache the value in
	 * MSR_IA32_SYSENTER_CS.  When we context switch
	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
	 * written matches ss1, and, if it's not, then we wrmsr the new
	 * value and update ss1.
269
	 *
270 271 272 273
	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
	 * that we set it to zero in vm86 tasks to avoid corrupting the
	 * stack if we were to go through the sysenter path from vm86
	 * mode.
274 275 276 277
	 */
	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */

	unsigned short		__ss1h;
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
	unsigned long		sp2;
	unsigned short		ss2, __ss2h;
	unsigned long		__cr3;
	unsigned long		ip;
	unsigned long		flags;
	unsigned long		ax;
	unsigned long		cx;
	unsigned long		dx;
	unsigned long		bx;
	unsigned long		sp;
	unsigned long		bp;
	unsigned long		si;
	unsigned long		di;
	unsigned short		es, __esh;
	unsigned short		cs, __csh;
	unsigned short		ss, __ssh;
	unsigned short		ds, __dsh;
	unsigned short		fs, __fsh;
	unsigned short		gs, __gsh;
	unsigned short		ldt, __ldth;
	unsigned short		trace;
	unsigned short		io_bitmap_base;

301 302 303
} __attribute__((packed));
#else
struct x86_hw_tss {
304 305 306 307 308 309 310 311 312 313 314
	u32			reserved1;
	u64			sp0;
	u64			sp1;
	u64			sp2;
	u64			reserved2;
	u64			ist[7];
	u32			reserved3;
	u32			reserved4;
	u16			reserved5;
	u16			io_bitmap_base;

315
} __attribute__((packed));
316 317 318
#endif

/*
319
 * IO-bitmap sizes:
320
 */
321 322 323 324 325
#define IO_BITMAP_BITS			65536
#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
#define IO_BITMAP_OFFSET		offsetof(struct tss_struct, io_bitmap)
#define INVALID_IO_BITMAP_OFFSET	0x8000
326 327

struct tss_struct {
328 329 330 331
	/*
	 * The hardware state:
	 */
	struct x86_hw_tss	x86_tss;
332 333 334 335 336 337 338

	/*
	 * The extra 1 is there because the CPU will access an
	 * additional byte beyond the end of the IO permission
	 * bitmap. The extra byte must be all 1 bits, and must
	 * be within the limit.
	 */
339 340
	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];

341
#ifdef CONFIG_X86_32
342
	/*
343
	 * Space for the temporary SYSENTER stack.
344
	 */
345
	unsigned long		SYSENTER_stack_canary;
346
	unsigned long		SYSENTER_stack[64];
347
#endif
348

349
} ____cacheline_aligned;
350

351
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss);
352

353 354 355 356 357 358 359 360 361 362
/*
 * sizeof(unsigned long) coming from an extra "long" at the end
 * of the iobitmap.
 *
 * -1? seg base+limit should be pointing to the address of the
 * last valid byte
 */
#define __KERNEL_TSS_LIMIT	\
	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)

363 364 365 366
#ifdef CONFIG_X86_32
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
#endif

367 368 369
/*
 * Save the original ist values for checking stack pointers during debugging
 */
370
struct orig_ist {
371
	unsigned long		ist[7];
372 373
};

374
#ifdef CONFIG_X86_64
375
DECLARE_PER_CPU(struct orig_ist, orig_ist);
376

377 378 379 380 381 382 383 384 385 386 387 388 389
union irq_stack_union {
	char irq_stack[IRQ_STACK_SIZE];
	/*
	 * GCC hardcodes the stack canary as %gs:40.  Since the
	 * irq_stack is the object at %gs:0, we reserve the bottom
	 * 48 bytes of the irq stack for the canary.
	 */
	struct {
		char gs_base[40];
		unsigned long stack_canary;
	};
};

390
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
391 392
DECLARE_INIT_PER_CPU(irq_stack_union);

393
DECLARE_PER_CPU(char *, irq_stack_ptr);
394 395
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
396 397
#else	/* X86_64 */
#ifdef CONFIG_CC_STACKPROTECTOR
398 399 400 401 402 403 404 405 406 407
/*
 * Make sure stack canary segment base is cached-aligned:
 *   "For Intel Atom processors, avoid non zero segment base address
 *    that is not aligned to cache line boundary at all cost."
 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
 */
struct stack_canary {
	char __pad[20];		/* canary at %gs:20 */
	unsigned long canary;
};
408
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
409
#endif
410 411 412 413 414 415 416 417 418
/*
 * per-CPU IRQ handling stacks
 */
struct irq_stack {
	u32                     stack[THREAD_SIZE/sizeof(u32)];
} __aligned(THREAD_SIZE);

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
419
#endif	/* X86_64 */
420

421
extern unsigned int fpu_kernel_xstate_size;
422
extern unsigned int fpu_user_xstate_size;
423

424 425
struct perf_event;

426 427 428 429
typedef struct {
	unsigned long		seg;
} mm_segment_t;

430
struct thread_struct {
431 432 433 434
	/* Cached TLS descriptors: */
	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
	unsigned long		sp0;
	unsigned long		sp;
435
#ifdef CONFIG_X86_32
436
	unsigned long		sysenter_cs;
437
#else
438 439 440 441
	unsigned short		es;
	unsigned short		ds;
	unsigned short		fsindex;
	unsigned short		gsindex;
442
#endif
443 444 445

	u32			status;		/* thread synchronous flags */

446
#ifdef CONFIG_X86_64
447 448 449 450 451 452 453 454 455
	unsigned long		fsbase;
	unsigned long		gsbase;
#else
	/*
	 * XXX: this could presumably be unsigned short.  Alternatively,
	 * 32-bit kernels could be taught to use fsindex instead.
	 */
	unsigned long fs;
	unsigned long gs;
456
#endif
457

458 459 460 461
	/* Save middle states of ptrace breakpoints */
	struct perf_event	*ptrace_bps[HBP_NUM];
	/* Debug status used for traps, single steps, etc... */
	unsigned long           debugreg6;
462 463
	/* Keep track of the exact dr7 value set by the user */
	unsigned long           ptrace_dr7;
464 465
	/* Fault info: */
	unsigned long		cr2;
466
	unsigned long		trap_nr;
467
	unsigned long		error_code;
468
#ifdef CONFIG_VM86
469
	/* Virtual 86 mode info */
470
	struct vm86		*vm86;
471
#endif
472 473 474 475 476
	/* IO permissions: */
	unsigned long		*io_bitmap_ptr;
	unsigned long		iopl;
	/* Max allowed port in the bitmap, in bytes: */
	unsigned		io_bitmap_max;
477

478 479
	mm_segment_t		addr_limit;

480
	unsigned int		sig_on_uaccess_err:1;
481 482
	unsigned int		uaccess_err:1;	/* uaccess failed */

483 484 485 486 487 488
	/* Floating point and extended processor state */
	struct fpu		fpu;
	/*
	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
	 * the end.
	 */
489 490
};

491 492 493 494 495 496 497 498 499
/*
 * Thread-synchronous status.
 *
 * This is different from the flags in that nobody else
 * ever touches our thread-synchronous status, so we don't
 * have to worry about atomic accesses.
 */
#define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/

500 501 502 503 504 505 506
/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
	unsigned int reg;
507

508 509 510 511 512 513 514 515
	asm volatile ("pushfl;"
		      "popl %0;"
		      "andl %1, %0;"
		      "orl %2, %0;"
		      "pushl %0;"
		      "popfl"
		      : "=&r" (reg)
		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
516 517 518
#endif
}

519 520
static inline void
native_load_sp0(struct tss_struct *tss, struct thread_struct *thread)
521 522 523
{
	tss->x86_tss.sp0 = thread->sp0;
#ifdef CONFIG_X86_32
524
	/* Only happens when SEP is enabled, no need to test "SEP"arately: */
525 526 527 528 529 530
	if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) {
		tss->x86_tss.ss1 = thread->sysenter_cs;
		wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
	}
#endif
}
531

532 533 534 535 536 537 538
static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
	asm volatile("swapgs" ::: "memory");
#endif
}

539
static inline unsigned long current_top_of_stack(void)
540
{
541
#ifdef CONFIG_X86_64
542
	return this_cpu_read_stable(cpu_tss.x86_tss.sp0);
543 544 545 546
#else
	/* sp0 on x86_32 is special in and around vm86 mode. */
	return this_cpu_read_stable(cpu_current_top_of_stack);
#endif
547 548
}

549 550 551
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
552
#define __cpuid			native_cpuid
553

554 555
static inline void load_sp0(struct tss_struct *tss,
			    struct thread_struct *thread)
556 557 558 559
{
	native_load_sp0(tss, thread);
}

560
#define set_iopl_mask native_set_iopl_mask
561 562
#endif /* CONFIG_PARAVIRT */

563 564 565 566
/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

unsigned long get_wchan(struct task_struct *p);
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
			 unsigned int *eax, unsigned int *ebx,
			 unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = 0;
	__cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
			       unsigned int *eax, unsigned int *ebx,
			       unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = count;
	__cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
600

601 602
	return eax;
}
603

604 605 606 607 608
static inline unsigned int cpuid_ebx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
609

610 611
	return ebx;
}
612

613 614 615 616 617
static inline unsigned int cpuid_ecx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
618

619 620
	return ecx;
}
621

622 623 624 625 626
static inline unsigned int cpuid_edx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
627

628 629 630
	return edx;
}

631
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
632
static __always_inline void rep_nop(void)
633
{
634
	asm volatile("rep; nop" ::: "memory");
635 636
}

637
static __always_inline void cpu_relax(void)
638 639 640 641
{
	rep_nop();
}

642 643 644 645 646 647 648 649 650 651 652 653 654 655
/*
 * This function forces the icache and prefetched instruction stream to
 * catch up with reality in two very specific cases:
 *
 *  a) Text was modified using one virtual address and is about to be executed
 *     from the same physical page at a different virtual address.
 *
 *  b) Text was modified on a different CPU, may subsequently be
 *     executed on this CPU, and you want to make sure the new version
 *     gets executed.  This generally means you're calling this in a IPI.
 *
 * If you're calling this for a different reason, you're probably doing
 * it wrong.
 */
656 657
static inline void sync_core(void)
{
658
	/*
659 660 661 662 663 664 665 666 667 668 669 670
	 * There are quite a few ways to do this.  IRET-to-self is nice
	 * because it works on every CPU, at any CPL (so it's compatible
	 * with paravirtualization), and it never exits to a hypervisor.
	 * The only down sides are that it's a bit slow (it seems to be
	 * a bit more than 2x slower than the fastest options) and that
	 * it unmasks NMIs.  The "push %cs" is needed because, in
	 * paravirtual environments, __KERNEL_CS may not be a valid CS
	 * value when we do IRET directly.
	 *
	 * In case NMI unmasking or performance ever becomes a problem,
	 * the next best option appears to be MOV-to-CR2 and an
	 * unconditional jump.  That sequence also works on all CPUs,
671
	 * but it will fault at CPL3 (i.e. Xen PV).
672 673 674 675 676 677 678
	 *
	 * CPUID is the conventional way, but it's nasty: it doesn't
	 * exist on some 486-like CPUs, and it usually exits to a
	 * hypervisor.
	 *
	 * Like all of Linux's memory ordering operations, this is a
	 * compiler barrier as well.
679
	 */
680 681 682 683 684 685 686
#ifdef CONFIG_X86_32
	asm volatile (
		"pushfl\n\t"
		"pushl %%cs\n\t"
		"pushl $1f\n\t"
		"iret\n\t"
		"1:"
687
		: ASM_CALL_CONSTRAINT : : "memory");
688
#else
689 690 691
	unsigned int tmp;

	asm volatile (
692
		UNWIND_HINT_SAVE
693 694 695 696 697 698 699 700 701
		"mov %%ss, %0\n\t"
		"pushq %q0\n\t"
		"pushq %%rsp\n\t"
		"addq $8, (%%rsp)\n\t"
		"pushfq\n\t"
		"mov %%cs, %0\n\t"
		"pushq %q0\n\t"
		"pushq $1f\n\t"
		"iretq\n\t"
702
		UNWIND_HINT_RESTORE
703
		"1:"
704
		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
705
#endif
706 707 708
}

extern void select_idle_routine(const struct cpuinfo_x86 *c);
709
extern void amd_e400_c1e_apic_setup(void);
710

711
extern unsigned long		boot_option_idle_override;
712

713
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
714
			 IDLE_POLL};
715

716 717 718
extern void enable_sep_cpu(void);
extern int sysenter_setup(void);

719
extern void early_trap_init(void);
720
void early_trap_pf_init(void);
721

722
/* Defined in head.S */
723
extern struct desc_ptr		early_gdt_descr;
724 725

extern void cpu_set_gdt(int);
726
extern void switch_to_new_gdt(int);
727
extern void load_direct_gdt(int);
728
extern void load_fixmap_gdt(int);
729
extern void load_percpu_segment(int);
730 731
extern void cpu_init(void);

732 733
static inline unsigned long get_debugctlmsr(void)
{
Peter Zijlstra's avatar
Peter Zijlstra committed
734
	unsigned long debugctlmsr = 0;
735 736 737 738 739 740 741

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

Peter Zijlstra's avatar
Peter Zijlstra committed
742
	return debugctlmsr;
743 744
}

745 746 747 748 749 750 751 752 753
static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

754 755
extern void set_task_blockstep(struct task_struct *task, bool on);

756 757
/* Boot loader type from the setup header: */
extern int			bootloader_type;
758
extern int			bootloader_version;
759

760
extern char			ignore_fpu_irq;
761 762 763 764 765

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

766
#ifdef CONFIG_X86_32
767
# define BASE_PREFETCH		""
768
# define ARCH_HAS_PREFETCH
769
#else
770
# define BASE_PREFETCH		"prefetcht0 %P1"
771 772
#endif

773 774 775 776 777 778
/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
779 780
static inline void prefetch(const void *x)
{
781
	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
782
			  X86_FEATURE_XMM,
783
			  "m" (*(const char *)x));
784 785
}

786 787 788 789 790
/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
791 792
static inline void prefetchw(const void *x)
{
793 794 795
	alternative_input(BASE_PREFETCH, "prefetchw %P1",
			  X86_FEATURE_3DNOWPREFETCH,
			  "m" (*(const char *)x));
796 797
}

798 799 800 801 802
static inline void spin_lock_prefetch(const void *x)
{
	prefetchw(x);
}

803 804 805
#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
			   TOP_OF_KERNEL_STACK_PADDING)

806 807 808 809
#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
810
#define IA32_PAGE_OFFSET	PAGE_OFFSET
811
#define TASK_SIZE		PAGE_OFFSET
812
#define TASK_SIZE_LOW		TASK_SIZE
813
#define TASK_SIZE_MAX		TASK_SIZE
814
#define DEFAULT_MAP_WINDOW	TASK_SIZE
815 816 817 818
#define STACK_TOP		TASK_SIZE
#define STACK_TOP_MAX		STACK_TOP

#define INIT_THREAD  {							  \
819
	.sp0			= TOP_OF_INIT_STACK,			  \
820 821
	.sysenter_cs		= __KERNEL_CS,				  \
	.io_bitmap_ptr		= NULL,					  \
822
	.addr_limit		= KERNEL_DS,				  \
823 824 825
}

/*
826
 * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack.
827
 * This is necessary to guarantee that the entire "struct pt_regs"
828
 * is accessible even if the CPU haven't stored the SS/ESP registers
829 830 831 832 833 834
 * on the stack (interrupt gate does not save these registers
 * when switching to the same priv ring).
 * Therefore beware: accessing the ss/esp fields of the
 * "struct pt_regs" is possible, but they may contain the
 * completely wrong values.
 */
835 836 837 838 839
#define task_pt_regs(task) \
({									\
	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
	((struct pt_regs *)__ptr) - 1;					\
840 841
})

842
#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
843 844 845

#else
/*
846 847 848 849 850 851 852
 * User space process size. 47bits minus one guard page.  The guard
 * page is necessary on Intel CPUs: if a SYSCALL instruction is at
 * the highest possible canonical userspace address, then that
 * syscall will enter the kernel with a non-canonical return
 * address, and SYSRET will explode dangerously.  We avoid this
 * particular problem by preventing anything from being mapped
 * at the maximum canonical address.
853
 */
854
#define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
855

856
#define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
857 858 859 860

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
861 862
#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
					0xc0000000 : 0xFFFFe000)
863

864 865
#define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \
					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
866
#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
867
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
868
#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
869
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
870

871
#define STACK_TOP		TASK_SIZE_LOW
872
#define STACK_TOP_MAX		TASK_SIZE_MAX
873

874 875 876
#define INIT_THREAD  {						\
	.sp0			= TOP_OF_INIT_STACK,		\
	.addr_limit		= KERNEL_DS,			\
877 878
}

879
#define task_pt_regs(tsk)	((struct pt_regs *)(tsk)->thread.sp0 - 1)
880
extern unsigned long KSTK_ESP(struct task_struct *task);
881

882 883
#endif /* CONFIG_X86_64 */

Ingo Molnar's avatar
Ingo Molnar committed
884 885 886
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
					       unsigned long new_sp);

887 888
/*
 * This decides where the kernel will search for a free chunk of vm
889 890
 * space during mmap's.
 */
891
#define __TASK_UNMAPPED_BASE(task_size)	(PAGE_ALIGN(task_size / 3))
892
#define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
893

894
#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
895

896 897 898 899 900 901 902
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
#define SET_TSC_CTL(val)	set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

903 904
DECLARE_PER_CPU(u64, msr_misc_features_shadow);

905
/* Register/unregister a process' MPX related resource */
906 907
#define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
#define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
908 909

#ifdef CONFIG_X86_INTEL_MPX
910 911
extern int mpx_enable_management(void);
extern int mpx_disable_management(void);
912
#else
913
static inline int mpx_enable_management(void)
914 915 916
{
	return -EINVAL;
}
917
static inline int mpx_disable_management(void)
918 919 920 921 922
{
	return -EINVAL;
}
#endif /* CONFIG_X86_INTEL_MPX */

923
#ifdef CONFIG_CPU_SUP_AMD
924
extern u16 amd_get_nb_id(int cpu);
925
extern u32 amd_get_nodes_per_socket(void);
926 927 928 929
#else
static inline u16 amd_get_nb_id(int cpu)		{ return 0; }
static inline u32 amd_get_nodes_per_socket(void)	{ return 0; }
#endif
930

931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
{
	uint32_t base, eax, signature[3];

	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);

		if (!memcmp(sig, signature, 12) &&
		    (leaves == 0 || ((eax - base) >= leaves)))
			return base;
	}

	return 0;
}

946 947 948 949
extern unsigned long arch_align_stack(unsigned long sp);
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);

void default_idle(void);
950 951 952 953 954
#ifdef	CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif
955 956

void stop_this_cpu(void *dummy);
957
void df_debug(struct pt_regs *regs, long error_code);
958
#endif /* _ASM_X86_PROCESSOR_H */