diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h
index 9b5a3469fed94fda814e64714cf196f9fbae76bf..5e97a43531470d13b63b7e4379a95d741363ebb7 100644
--- a/arch/s390/include/asm/futex.h
+++ b/arch/s390/include/asm/futex.h
@@ -26,9 +26,9 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
u32 __user *uaddr)
{
int oldval = 0, newval, ret;
+ mm_segment_t old_fs;
- load_kernel_asce();
-
+ old_fs = enable_sacf_uaccess();
pagefault_disable();
switch (op) {
case FUTEX_OP_SET:
@@ -55,6 +55,7 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
ret = -ENOSYS;
}
pagefault_enable();
+ disable_sacf_uaccess(old_fs);
if (!ret)
*oval = oldval;
@@ -65,9 +66,10 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
u32 oldval, u32 newval)
{
+ mm_segment_t old_fs;
int ret;
- load_kernel_asce();
+ old_fs = enable_sacf_uaccess();
asm volatile(
" sacf 256\n"
"0: cs %1,%4,0(%5)\n"
@@ -77,6 +79,7 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
: "=d" (ret), "+d" (oldval), "=m" (*uaddr)
: "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
: "cc", "memory");
+ disable_sacf_uaccess(old_fs);
*uval = oldval;
return ret;
}
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 9eb36a1592c797a7d1e70e86b2c905a8266d4760..2306fa17f6cdb6644fb951c96493cafd4f1e3c50 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -115,33 +115,28 @@ struct lowcore {
/* Address space pointer. */
__u64 kernel_asce; /* 0x0378 */
__u64 user_asce; /* 0x0380 */
+ __u64 vdso_asce; /* 0x0388 */
/*
* The lpp and current_pid fields form a
* 64-bit value that is set as program
* parameter with the LPP instruction.
*/
- __u32 lpp; /* 0x0388 */
- __u32 current_pid; /* 0x038c */
+ __u32 lpp; /* 0x0390 */
+ __u32 current_pid; /* 0x0394 */
/* SMP info area */
- __u32 cpu_nr; /* 0x0390 */
- __u32 softirq_pending; /* 0x0394 */
- __u64 percpu_offset; /* 0x0398 */
- __u64 vdso_per_cpu_data; /* 0x03a0 */
- __u64 machine_flags; /* 0x03a8 */
- __u32 preempt_count; /* 0x03b0 */
- __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */
- __u64 gmap; /* 0x03b8 */
- __u32 spinlock_lockval; /* 0x03c0 */
- __u32 spinlock_index; /* 0x03c4 */
- __u32 fpu_flags; /* 0x03c8 */
- __u8 pad_0x03cc[0x0400-0x03cc]; /* 0x03cc */
-
- /* Per cpu primary space access list */
- __u32 paste[16]; /* 0x0400 */
-
- __u8 pad_0x04c0[0x0e00-0x0440]; /* 0x0440 */
+ __u32 cpu_nr; /* 0x0398 */
+ __u32 softirq_pending; /* 0x039c */
+ __u32 preempt_count; /* 0x03a0 */
+ __u32 spinlock_lockval; /* 0x03a4 */
+ __u32 spinlock_index; /* 0x03a8 */
+ __u32 fpu_flags; /* 0x03ac */
+ __u64 percpu_offset; /* 0x03b0 */
+ __u64 vdso_per_cpu_data; /* 0x03b8 */
+ __u64 machine_flags; /* 0x03c0 */
+ __u64 gmap; /* 0x03c8 */
+ __u8 pad_0x03d0[0x0e00-0x03d0]; /* 0x03d0 */
/*
* 0xe00 contains the address of the IPL Parameter Information
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 43607bb12cc2b303e7e9f87d391f053ad0aae2d2..6133aa376b7c321a03a66ca78978bd5b32dbc9a7 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -71,41 +71,38 @@ static inline int init_new_context(struct task_struct *tsk,
static inline void set_user_asce(struct mm_struct *mm)
{
S390_lowcore.user_asce = mm->context.asce;
- if (current->thread.mm_segment.ar4)
- __ctl_load(S390_lowcore.user_asce, 7, 7);
- set_cpu_flag(CIF_ASCE_PRIMARY);
+ __ctl_load(S390_lowcore.user_asce, 1, 1);
+ clear_cpu_flag(CIF_ASCE_PRIMARY);
}
static inline void clear_user_asce(void)
{
S390_lowcore.user_asce = S390_lowcore.kernel_asce;
-
- __ctl_load(S390_lowcore.user_asce, 1, 1);
- __ctl_load(S390_lowcore.user_asce, 7, 7);
-}
-
-static inline void load_kernel_asce(void)
-{
- unsigned long asce;
-
- __ctl_store(asce, 1, 1);
- if (asce != S390_lowcore.kernel_asce)
- __ctl_load(S390_lowcore.kernel_asce, 1, 1);
+ __ctl_load(S390_lowcore.kernel_asce, 1, 1);
set_cpu_flag(CIF_ASCE_PRIMARY);
}
+mm_segment_t enable_sacf_uaccess(void);
+void disable_sacf_uaccess(mm_segment_t old_fs);
+
static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk)
{
int cpu = smp_processor_id();
- S390_lowcore.user_asce = next->context.asce;
if (prev == next)
return;
+ S390_lowcore.user_asce = next->context.asce;
cpumask_set_cpu(cpu, &next->context.cpu_attach_mask);
- /* Clear old ASCE by loading the kernel ASCE. */
- __ctl_load(S390_lowcore.kernel_asce, 1, 1);
- __ctl_load(S390_lowcore.kernel_asce, 7, 7);
+ /* Clear previous user-ASCE from CR1 and CR7 */
+ if (!test_cpu_flag(CIF_ASCE_PRIMARY)) {
+ __ctl_load(S390_lowcore.kernel_asce, 1, 1);
+ set_cpu_flag(CIF_ASCE_PRIMARY);
+ }
+ if (test_cpu_flag(CIF_ASCE_SECONDARY)) {
+ __ctl_load(S390_lowcore.vdso_asce, 7, 7);
+ clear_cpu_flag(CIF_ASCE_SECONDARY);
+ }
cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask);
}
@@ -115,7 +112,6 @@ static inline void finish_arch_post_lock_switch(void)
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
- load_kernel_asce();
if (mm) {
preempt_disable();
while (atomic_read(&mm->context.flush_count))
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index f25bfe888933fe92dda958003b3dcad3e19ac2fc..709351bce80ef4ddc7502461008241b8647cb3de 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -109,9 +109,7 @@ extern void execve_tail(void);
#define HAVE_ARCH_PICK_MMAP_LAYOUT
-typedef struct {
- __u32 ar4;
-} mm_segment_t;
+typedef unsigned int mm_segment_t;
/*
* Thread structure
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index cdd0f0d999e2617f0ec372828da9482d5fa6b021..ad6b91013a0525d82002090788038e0c9773b34a 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -16,7 +16,7 @@
#include <asm/processor.h>
#include <asm/ctl_reg.h>
#include <asm/extable.h>
-
+#include <asm/facility.h>
/*
* The fs value determines whether argument validity checking should be
@@ -26,27 +26,16 @@
* For historical reasons, these macros are grossly misnamed.
*/
-#define MAKE_MM_SEG(a) ((mm_segment_t) { (a) })
-
-
-#define KERNEL_DS MAKE_MM_SEG(0)
-#define USER_DS MAKE_MM_SEG(1)
+#define KERNEL_DS (0)
+#define KERNEL_DS_SACF (1)
+#define USER_DS (2)
+#define USER_DS_SACF (3)
#define get_ds() (KERNEL_DS)
#define get_fs() (current->thread.mm_segment)
-#define segment_eq(a,b) ((a).ar4 == (b).ar4)
+#define segment_eq(a,b) (((a) & 2) == ((b) & 2))
-static inline void set_fs(mm_segment_t fs)
-{
- current->thread.mm_segment = fs;
- if (uaccess_kernel()) {
- set_cpu_flag(CIF_ASCE_SECONDARY);
- __ctl_load(S390_lowcore.kernel_asce, 7, 7);
- } else {
- clear_cpu_flag(CIF_ASCE_SECONDARY);
- __ctl_load(S390_lowcore.user_asce, 7, 7);
- }
-}
+void set_fs(mm_segment_t fs);
static inline int __range_ok(unsigned long addr, unsigned long size)
{
@@ -95,7 +84,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n);
static inline int __put_user_fn(void *x, void __user *ptr, unsigned long size)
{
- unsigned long spec = 0x810000UL;
+ unsigned long spec = 0x010000UL;
int rc;
switch (size) {
@@ -125,7 +114,7 @@ static inline int __put_user_fn(void *x, void __user *ptr, unsigned long size)
static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size)
{
- unsigned long spec = 0x81UL;
+ unsigned long spec = 0x01UL;
int rc;
switch (size) {
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 33ec80df7ed443ffc161d99ee3fc591381cec0b8..587b195b588dd092ab5ff73fbccd31993ba895d7 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -171,6 +171,7 @@ int main(void)
OFFSET(__LC_RESTART_DATA, lowcore, restart_data);
OFFSET(__LC_RESTART_SOURCE, lowcore, restart_source);
OFFSET(__LC_USER_ASCE, lowcore, user_asce);
+ OFFSET(__LC_VDSO_ASCE, lowcore, vdso_asce);
OFFSET(__LC_LPP, lowcore, lpp);
OFFSET(__LC_CURRENT_PID, lowcore, current_pid);
OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset);
@@ -178,7 +179,6 @@ int main(void)
OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags);
OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count);
OFFSET(__LC_GMAP, lowcore, gmap);
- OFFSET(__LC_PASTE, lowcore, paste);
/* software defined ABI-relevant lowcore locations 0xe00 - 0xe20 */
OFFSET(__LC_DUMP_REIPL, lowcore, ipib);
/* hardware defined lowcore locations 0x1000 - 0x18ff */
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index ee53ac7b1ab8c9b11a083a28e4ed80ba41b99871..a316cd6999ad9712defdf46db85e16eb429aebcb 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -379,13 +379,21 @@ ENTRY(system_call)
jg s390_handle_mcck # TIF bit will be cleared by handler
#
-# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce
+# _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce
#
.Lsysc_asce:
+ ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY
+ lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce
+ TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY
+ jz .Lsysc_return
+#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
+ tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ?
+ jnz .Lsysc_set_fs_fixup
ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
- TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_SECONDARY
- jz .Lsysc_return
+ j .Lsysc_return
+.Lsysc_set_fs_fixup:
+#endif
larl %r14,.Lsysc_return
jg set_fs_fixup
@@ -741,10 +749,18 @@ ENTRY(io_int_handler)
# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce
#
.Lio_asce:
+ ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY
+ lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce
+ TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY
+ jz .Lio_return
+#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
+ tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ?
+ jnz .Lio_set_fs_fixup
ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
- TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_SECONDARY
- jz .Lio_return
+ j .Lio_return
+.Lio_set_fs_fixup:
+#endif
larl %r14,.Lio_return
jg set_fs_fixup
diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S
index 172002da70754eb31d3d512f75db46e304fc1eb7..38a973ccf50108b423683496ee4daa28ab2382d6 100644
--- a/arch/s390/kernel/head64.S
+++ b/arch/s390/kernel/head64.S
@@ -28,7 +28,7 @@ ENTRY(startup_continue)
lctlg %c0,%c15,.Lctl-.LPG1(%r13) # load control registers
lg %r12,.Lparmaddr-.LPG1(%r13) # pointer to parameter area
# move IPL device to lowcore
- lghi %r0,__LC_PASTE
+ larl %r0,boot_vdso_data
stg %r0,__LC_VDSO_PER_CPU
#
# Setup stack
diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c
index 0520854a4dab27f7ac0feef52311a765686aebf1..39a218703c50add3defe9aa2ce5a85f88188e776 100644
--- a/arch/s390/kernel/vdso.c
+++ b/arch/s390/kernel/vdso.c
@@ -158,16 +158,9 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore)
{
unsigned long segment_table, page_table, page_frame;
struct vdso_per_cpu_data *vd;
- u32 *psal, *aste;
- int i;
-
- lowcore->vdso_per_cpu_data = __LC_PASTE;
-
- if (!vdso_enabled)
- return 0;
segment_table = __get_free_pages(GFP_KERNEL, SEGMENT_ORDER);
- page_table = get_zeroed_page(GFP_KERNEL | GFP_DMA);
+ page_table = get_zeroed_page(GFP_KERNEL);
page_frame = get_zeroed_page(GFP_KERNEL);
if (!segment_table || !page_table || !page_frame)
goto out;
@@ -179,25 +172,15 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore)
vd->cpu_nr = lowcore->cpu_nr;
vd->node_id = cpu_to_node(vd->cpu_nr);
- /* Set up access register mode page table */
+ /* Set up page table for the vdso address space */
memset64((u64 *)segment_table, _SEGMENT_ENTRY_EMPTY, _CRST_ENTRIES);
memset64((u64 *)page_table, _PAGE_INVALID, PTRS_PER_PTE);
*(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table;
*(unsigned long *) page_table = _PAGE_PROTECT + page_frame;
- psal = (u32 *) (page_table + 256*sizeof(unsigned long));
- aste = psal + 32;
-
- for (i = 4; i < 32; i += 4)
- psal[i] = 0x80000000;
-
- lowcore->paste[4] = (u32)(addr_t) psal;
- psal[0] = 0x02000000;
- psal[2] = (u32)(addr_t) aste;
- *(unsigned long *) (aste + 2) = segment_table +
+ lowcore->vdso_asce = segment_table +
_ASCE_TABLE_LENGTH + _ASCE_USER_BITS + _ASCE_TYPE_SEGMENT;
- aste[4] = (u32)(addr_t) psal;
lowcore->vdso_per_cpu_data = page_frame;
return 0;
@@ -212,14 +195,8 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore)
void vdso_free_per_cpu(struct lowcore *lowcore)
{
unsigned long segment_table, page_table, page_frame;
- u32 *psal, *aste;
-
- if (!vdso_enabled)
- return;
- psal = (u32 *)(addr_t) lowcore->paste[4];
- aste = (u32 *)(addr_t) psal[2];
- segment_table = *(unsigned long *)(aste + 2) & PAGE_MASK;
+ segment_table = lowcore->vdso_asce & PAGE_MASK;
page_table = *(unsigned long *) segment_table;
page_frame = *(unsigned long *) page_table;
@@ -228,16 +205,6 @@ void vdso_free_per_cpu(struct lowcore *lowcore)
free_pages(segment_table, SEGMENT_ORDER);
}
-static void vdso_init_cr5(void)
-{
- unsigned long cr5;
-
- if (!vdso_enabled)
- return;
- cr5 = offsetof(struct lowcore, paste);
- __ctl_load(cr5, 5, 5);
-}
-
/*
* This is called from binfmt_elf, we create the special vma for the
* vDSO and insert it into the mm struct tree
@@ -314,8 +281,6 @@ static int __init vdso_init(void)
{
int i;
- if (!vdso_enabled)
- return 0;
vdso_init_data(vdso_data);
#ifdef CONFIG_COMPAT
/* Calculate the size of the 32 bit vDSO */
@@ -354,7 +319,6 @@ static int __init vdso_init(void)
vdso64_pagelist[vdso64_pages] = NULL;
if (vdso_alloc_per_cpu(&S390_lowcore))
BUG();
- vdso_init_cr5();
get_page(virt_to_page(vdso_data));
diff --git a/arch/s390/kernel/vdso32/getcpu.S b/arch/s390/kernel/vdso32/getcpu.S
index 6e30769dd017654550617efa39f02a07814dd5ca..5477a2c112fb800e3c1013cb17a03a1d6aa47228 100644
--- a/arch/s390/kernel/vdso32/getcpu.S
+++ b/arch/s390/kernel/vdso32/getcpu.S
@@ -15,23 +15,11 @@
.type __kernel_getcpu,@function
__kernel_getcpu:
.cfi_startproc
- ear %r1,%a4
- lhi %r4,1
- sll %r4,24
- sar %a4,%r4
la %r4,0
- epsw %r0,0
- sacf 512
+ sacf 256
l %r5,__VDSO_CPU_NR(%r4)
l %r4,__VDSO_NODE_ID(%r4)
- tml %r0,0x4000
- jo 1f
- tml %r0,0x8000
- jno 0f
- sacf 256
- j 1f
-0: sacf 0
-1: sar %a4,%r1
+ sacf 0
ltr %r2,%r2
jz 2f
st %r5,0(%r2)
diff --git a/arch/s390/kernel/vdso64/clock_gettime.S b/arch/s390/kernel/vdso64/clock_gettime.S
index 9c3b12626dbae06a77a12b4e186a4bb0eb253e09..5d7b56b49458d03ba885f105a9c07bcdecea019a 100644
--- a/arch/s390/kernel/vdso64/clock_gettime.S
+++ b/arch/s390/kernel/vdso64/clock_gettime.S
@@ -114,23 +114,12 @@ __kernel_clock_gettime:
br %r14
/* CPUCLOCK_VIRT for this thread */
-9: icm %r0,15,__VDSO_ECTG_OK(%r5)
+9: lghi %r4,0
+ icm %r0,15,__VDSO_ECTG_OK(%r5)
jz 12f
- ear %r2,%a4
- llilh %r4,0x0100
- sar %a4,%r4
- lghi %r4,0
- epsw %r5,0
- sacf 512 /* Magic ectg instruction */
+ sacf 256 /* Magic ectg instruction */
.insn ssf,0xc80100000000,__VDSO_ECTG_BASE(4),__VDSO_ECTG_USER(4),4
- tml %r5,0x4000
- jo 11f
- tml %r5,0x8000
- jno 10f
- sacf 256
- j 11f
-10: sacf 0
-11: sar %a4,%r2
+ sacf 0
algr %r1,%r0 /* r1 = cputime as TOD value */
mghi %r1,1000 /* convert to nanoseconds */
srlg %r1,%r1,12 /* r1 = cputime in nanosec */
diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S
index 43983764b959827951ddb121ff8158dabbfa1a12..e9c34364d97b1aafdef2b0fa24c3ece518e3fe53 100644
--- a/arch/s390/kernel/vdso64/getcpu.S
+++ b/arch/s390/kernel/vdso64/getcpu.S
@@ -15,22 +15,11 @@
.type __kernel_getcpu,@function
__kernel_getcpu:
.cfi_startproc
- ear %r1,%a4
- llilh %r4,0x0100
- sar %a4,%r4
la %r4,0
- epsw %r0,0
- sacf 512
+ sacf 256
l %r5,__VDSO_CPU_NR(%r4)
l %r4,__VDSO_NODE_ID(%r4)
- tml %r0,0x4000
- jo 1f
- tml %r0,0x8000
- jno 0f
- sacf 256
- j 1f
-0: sacf 0
-1: sar %a4,%r1
+ sacf 0
ltgr %r2,%r2
jz 2f
st %r5,0(%r2)
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index 802903c50de125f54f8b4f3713e243c0b85a5b6e..cae5a1e16cbd2d9ac5cc7b2fd1f67443919b8f80 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -40,10 +40,67 @@ static inline int copy_with_mvcos(void)
}
#endif
+void set_fs(mm_segment_t fs)
+{
+ current->thread.mm_segment = fs;
+ if (fs == USER_DS) {
+ __ctl_load(S390_lowcore.user_asce, 1, 1);
+ clear_cpu_flag(CIF_ASCE_PRIMARY);
+ } else {
+ __ctl_load(S390_lowcore.kernel_asce, 1, 1);
+ set_cpu_flag(CIF_ASCE_PRIMARY);
+ }
+ if (fs & 1) {
+ if (fs == USER_DS_SACF)
+ __ctl_load(S390_lowcore.user_asce, 7, 7);
+ else
+ __ctl_load(S390_lowcore.kernel_asce, 7, 7);
+ set_cpu_flag(CIF_ASCE_SECONDARY);
+ }
+}
+EXPORT_SYMBOL(set_fs);
+
+mm_segment_t enable_sacf_uaccess(void)
+{
+ mm_segment_t old_fs;
+ unsigned long asce, cr;
+
+ old_fs = current->thread.mm_segment;
+ if (old_fs & 1)
+ return old_fs;
+ current->thread.mm_segment |= 1;
+ asce = S390_lowcore.kernel_asce;
+ if (likely(old_fs == USER_DS)) {
+ __ctl_store(cr, 1, 1);
+ if (cr != S390_lowcore.kernel_asce) {
+ __ctl_load(S390_lowcore.kernel_asce, 1, 1);
+ set_cpu_flag(CIF_ASCE_PRIMARY);
+ }
+ asce = S390_lowcore.user_asce;
+ }
+ __ctl_store(cr, 7, 7);
+ if (cr != asce) {
+ __ctl_load(asce, 7, 7);
+ set_cpu_flag(CIF_ASCE_SECONDARY);
+ }
+ return old_fs;
+}
+EXPORT_SYMBOL(enable_sacf_uaccess);
+
+void disable_sacf_uaccess(mm_segment_t old_fs)
+{
+ if (old_fs == USER_DS && test_facility(27)) {
+ __ctl_load(S390_lowcore.user_asce, 1, 1);
+ clear_cpu_flag(CIF_ASCE_PRIMARY);
+ }
+ current->thread.mm_segment = old_fs;
+}
+EXPORT_SYMBOL(disable_sacf_uaccess);
+
static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
unsigned long size)
{
- register unsigned long reg0 asm("0") = 0x81UL;
+ register unsigned long reg0 asm("0") = 0x01UL;
unsigned long tmp1, tmp2;
tmp1 = -4096UL;
@@ -74,8 +131,9 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
unsigned long size)
{
unsigned long tmp1, tmp2;
+ mm_segment_t old_fs;
- load_kernel_asce();
+ old_fs = enable_sacf_uaccess();
tmp1 = -256UL;
asm volatile(
" sacf 0\n"
@@ -102,6 +160,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
: : "cc", "memory");
+ disable_sacf_uaccess(old_fs);
return size;
}
@@ -116,7 +175,7 @@ EXPORT_SYMBOL(raw_copy_from_user);
static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
unsigned long size)
{
- register unsigned long reg0 asm("0") = 0x810000UL;
+ register unsigned long reg0 asm("0") = 0x010000UL;
unsigned long tmp1, tmp2;
tmp1 = -4096UL;
@@ -147,8 +206,9 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
unsigned long size)
{
unsigned long tmp1, tmp2;
+ mm_segment_t old_fs;
- load_kernel_asce();
+ old_fs = enable_sacf_uaccess();
tmp1 = -256UL;
asm volatile(
" sacf 0\n"
@@ -175,6 +235,7 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
: : "cc", "memory");
+ disable_sacf_uaccess(old_fs);
return size;
}
@@ -189,7 +250,7 @@ EXPORT_SYMBOL(raw_copy_to_user);
static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
unsigned long size)
{
- register unsigned long reg0 asm("0") = 0x810081UL;
+ register unsigned long reg0 asm("0") = 0x010001UL;
unsigned long tmp1, tmp2;
tmp1 = -4096UL;
@@ -212,9 +273,10 @@ static inline unsigned long copy_in_user_mvcos(void __user *to, const void __use
static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from,
unsigned long size)
{
+ mm_segment_t old_fs;
unsigned long tmp1;
- load_kernel_asce();
+ old_fs = enable_sacf_uaccess();
asm volatile(
" sacf 256\n"
" aghi %0,-1\n"
@@ -238,6 +300,7 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user
EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
: "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
: : "cc", "memory");
+ disable_sacf_uaccess(old_fs);
return size;
}
@@ -251,7 +314,7 @@ EXPORT_SYMBOL(raw_copy_in_user);
static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
{
- register unsigned long reg0 asm("0") = 0x810000UL;
+ register unsigned long reg0 asm("0") = 0x010000UL;
unsigned long tmp1, tmp2;
tmp1 = -4096UL;
@@ -279,9 +342,10 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
{
+ mm_segment_t old_fs;
unsigned long tmp1, tmp2;
- load_kernel_asce();
+ old_fs = enable_sacf_uaccess();
asm volatile(
" sacf 256\n"
" aghi %0,-1\n"
@@ -310,6 +374,7 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
: "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
: : "cc", "memory");
+ disable_sacf_uaccess(old_fs);
return size;
}
@@ -345,10 +410,15 @@ static inline unsigned long strnlen_user_srst(const char __user *src,
unsigned long __strnlen_user(const char __user *src, unsigned long size)
{
+ mm_segment_t old_fs;
+ unsigned long len;
+
if (unlikely(!size))
return 0;
- load_kernel_asce();
- return strnlen_user_srst(src, size);
+ old_fs = enable_sacf_uaccess();
+ len = strnlen_user_srst(src, size);
+ disable_sacf_uaccess(old_fs);
+ return len;
}
EXPORT_SYMBOL(__strnlen_user);
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index be974b3eb7e45a1ae4fdbe8522eef5eb41f2626b..14654007dce4984ad47275d0d717dd034d9d4702 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -50,6 +50,13 @@
#define VM_FAULT_SIGNAL 0x080000
#define VM_FAULT_PFAULT 0x100000
+enum fault_type {
+ KERNEL_FAULT,
+ USER_FAULT,
+ VDSO_FAULT,
+ GMAP_FAULT,
+};
+
static unsigned long store_indication __read_mostly;
static int __init fault_init(void)
@@ -99,27 +106,34 @@ void bust_spinlocks(int yes)
}
/*
- * Returns the address space associated with the fault.
- * Returns 0 for kernel space and 1 for user space.
+ * Find out which address space caused the exception.
+ * Access register mode is impossible, ignore space == 3.
*/
-static inline int user_space_fault(struct pt_regs *regs)
+static inline enum fault_type get_fault_type(struct pt_regs *regs)
{
unsigned long trans_exc_code;
- /*
- * The lowest two bits of the translation exception
- * identification indicate which paging table was used.
- */
trans_exc_code = regs->int_parm_long & 3;
- if (trans_exc_code == 3) /* home space -> kernel */
- return 0;
- if (user_mode(regs))
- return 1;
- if (trans_exc_code == 2) /* secondary space -> set_fs */
- return current->thread.mm_segment.ar4;
- if (test_pt_regs_flag(regs, PIF_GUEST_FAULT))
- return 1;
- return 0;
+ if (likely(trans_exc_code == 0)) {
+ /* primary space exception */
+ if (IS_ENABLED(CONFIG_PGSTE) &&
+ test_pt_regs_flag(regs, PIF_GUEST_FAULT))
+ return GMAP_FAULT;
+ if (current->thread.mm_segment == USER_DS)
+ return USER_FAULT;
+ return KERNEL_FAULT;
+ }
+ if (trans_exc_code == 2) {
+ /* secondary space exception */
+ if (current->thread.mm_segment & 1) {
+ if (current->thread.mm_segment == USER_DS_SACF)
+ return USER_FAULT;
+ return KERNEL_FAULT;
+ }
+ return VDSO_FAULT;
+ }
+ /* home space exception -> access via kernel ASCE */
+ return KERNEL_FAULT;
}
static int bad_address(void *p)
@@ -204,20 +218,23 @@ static void dump_fault_info(struct pt_regs *regs)
break;
}
pr_cont("mode while using ");
- if (!user_space_fault(regs)) {
- asce = S390_lowcore.kernel_asce;
- pr_cont("kernel ");
- }
-#ifdef CONFIG_PGSTE
- else if (test_pt_regs_flag(regs, PIF_GUEST_FAULT)) {
- struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
- asce = gmap->asce;
- pr_cont("gmap ");
- }
-#endif
- else {
+ switch (get_fault_type(regs)) {
+ case USER_FAULT:
asce = S390_lowcore.user_asce;
pr_cont("user ");
+ break;
+ case VDSO_FAULT:
+ asce = S390_lowcore.vdso_asce;
+ pr_cont("vdso ");
+ break;
+ case GMAP_FAULT:
+ asce = ((struct gmap *) S390_lowcore.gmap)->asce;
+ pr_cont("gmap ");
+ break;
+ case KERNEL_FAULT:
+ asce = S390_lowcore.kernel_asce;
+ pr_cont("kernel ");
+ break;
}
pr_cont("ASCE.\n");
dump_pagetable(asce, regs->int_parm_long & __FAIL_ADDR_MASK);
@@ -273,7 +290,7 @@ static noinline void do_no_context(struct pt_regs *regs)
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
- if (!user_space_fault(regs))
+ if (get_fault_type(regs) == KERNEL_FAULT)
printk(KERN_ALERT "Unable to handle kernel pointer dereference"
" in virtual kernel address space\n");
else
@@ -395,12 +412,11 @@ static noinline void do_fault_error(struct pt_regs *regs, int access, int fault)
*/
static inline int do_exception(struct pt_regs *regs, int access)
{
-#ifdef CONFIG_PGSTE
struct gmap *gmap;
-#endif
struct task_struct *tsk;
struct mm_struct *mm;
struct vm_area_struct *vma;
+ enum fault_type type;
unsigned long trans_exc_code;
unsigned long address;
unsigned int flags;
@@ -425,8 +441,19 @@ static inline int do_exception(struct pt_regs *regs, int access)
* user context.
*/
fault = VM_FAULT_BADCONTEXT;
- if (unlikely(!user_space_fault(regs) || faulthandler_disabled() || !mm))
+ type = get_fault_type(regs);
+ switch (type) {
+ case KERNEL_FAULT:
+ goto out;
+ case VDSO_FAULT:
+ fault = VM_FAULT_BADMAP;
goto out;
+ case USER_FAULT:
+ case GMAP_FAULT:
+ if (faulthandler_disabled() || !mm)
+ goto out;
+ break;
+ }
address = trans_exc_code & __FAIL_ADDR_MASK;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
@@ -437,10 +464,9 @@ static inline int do_exception(struct pt_regs *regs, int access)
flags |= FAULT_FLAG_WRITE;
down_read(&mm->mmap_sem);
-#ifdef CONFIG_PGSTE
- gmap = test_pt_regs_flag(regs, PIF_GUEST_FAULT) ?
- (struct gmap *) S390_lowcore.gmap : NULL;
- if (gmap) {
+ gmap = NULL;
+ if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) {
+ gmap = (struct gmap *) S390_lowcore.gmap;
current->thread.gmap_addr = address;
current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE);
current->thread.gmap_int_code = regs->int_code & 0xffff;
@@ -452,7 +478,6 @@ static inline int do_exception(struct pt_regs *regs, int access)
if (gmap->pfault_enabled)
flags |= FAULT_FLAG_RETRY_NOWAIT;
}
-#endif
retry:
fault = VM_FAULT_BADMAP;
@@ -507,15 +532,14 @@ static inline int do_exception(struct pt_regs *regs, int access)
regs, address);
}
if (fault & VM_FAULT_RETRY) {
-#ifdef CONFIG_PGSTE
- if (gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) {
+ if (IS_ENABLED(CONFIG_PGSTE) && gmap &&
+ (flags & FAULT_FLAG_RETRY_NOWAIT)) {
/* FAULT_FLAG_RETRY_NOWAIT has been set,
* mmap_sem has not been released */
current->thread.gmap_pfault = 1;
fault = VM_FAULT_PFAULT;
goto out_up;
}
-#endif
/* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
* of starvation. */
flags &= ~(FAULT_FLAG_ALLOW_RETRY |
@@ -525,8 +549,7 @@ static inline int do_exception(struct pt_regs *regs, int access)
goto retry;
}
}
-#ifdef CONFIG_PGSTE
- if (gmap) {
+ if (IS_ENABLED(CONFIG_PGSTE) && gmap) {
address = __gmap_link(gmap, current->thread.gmap_addr,
address);
if (address == -EFAULT) {
@@ -538,7 +561,6 @@ static inline int do_exception(struct pt_regs *regs, int access)
goto out_up;
}
}
-#endif
fault = 0;
out_up:
up_read(&mm->mmap_sem);
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 817c9e16e83e5d3fe5e75b7e3b440cea5b23fe95..671535e64abab615afca53c4b57ce89863981f9a 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -95,6 +95,7 @@ void __init paging_init(void)
}
init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits;
S390_lowcore.kernel_asce = init_mm.context.asce;
+ S390_lowcore.user_asce = S390_lowcore.kernel_asce;
crst_table_init((unsigned long *) init_mm.pgd, pgd_type);
vmem_map_init();
diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
index 4ad4c4f77b4d9fa0362ed42a6e8474e443285e26..434a9564917beceadeffd0f34d041683b717af1c 100644
--- a/arch/s390/mm/pgalloc.c
+++ b/arch/s390/mm/pgalloc.c
@@ -71,10 +71,8 @@ static void __crst_table_upgrade(void *arg)
{
struct mm_struct *mm = arg;
- if (current->active_mm == mm) {
- clear_user_asce();
+ if (current->active_mm == mm)
set_user_asce(mm);
- }
__tlb_flush_local();
}