• Sergey Senozhatsky's avatar
    lib/ratelimit.c: use deferred printk() version · 656d61ce
    Sergey Senozhatsky authored
    printk_ratelimit() invokes ___ratelimit() which may invoke a normal
    printk() (pr_warn() in this particular case) to warn about suppressed
    output.  Given that printk_ratelimit() may be called from anywhere, that
    pr_warn() is dangerous - it may end up deadlocking the system.  Fix
    ___ratelimit() by using deferred printk().
    
    Sasha reported the following lockdep error:
    
     : Unregister pv shared memory for cpu 8
     : select_fallback_rq: 3 callbacks suppressed
     : process 8583 (trinity-c78) no longer affine to cpu8
     :
     : ======================================================
     : WARNING: possible circular locking dependency detected
     : 4.14.0-rc2-next-20170927+ #252 Not tainted
     : ------------------------------------------------------
     : migration/8/62 is trying to acquire lock:
     : (&port_lock_key){-.-.}, at: serial8250_console_write()
     :
     : but task is already holding lock:
     : (&rq->lock){-.-.}, at: sched_cpu_dying()
     :
     : which lock already depends on the new lock.
     :
     :
     : the existing dependency chain (in reverse order) is:
     :
     : -> #3 (&rq->lock){-.-.}:
     : __lock_acquire()
     : lock_acquire()
     : _raw_spin_lock()
     : task_fork_fair()
     : sched_fork()
     : copy_process.part.31()
     : _do_fork()
     : kernel_thread()
     : rest_init()
     : start_kernel()
     : x86_64_start_reservations()
     : x86_64_start_kernel()
     : verify_cpu()
     :
     : -> #2 (&p->pi_lock){-.-.}:
     : __lock_acquire()
     : lock_acquire()
     : _raw_spin_lock_irqsave()
     : try_to_wake_up()
     : default_wake_function()
     : woken_wake_function()
     : __wake_up_common()
     : __wake_up_common_lock()
     : __wake_up()
     : tty_wakeup()
     : tty_port_default_wakeup()
     : tty_port_tty_wakeup()
     : uart_write_wakeup()
     : serial8250_tx_chars()
     : serial8250_handle_irq.part.25()
     : serial8250_default_handle_irq()
     : serial8250_interrupt()
     : __handle_irq_event_percpu()
     : handle_irq_event_percpu()
     : handle_irq_event()
     : handle_level_irq()
     : handle_irq()
     : do_IRQ()
     : ret_from_intr()
     : native_safe_halt()
     : default_idle()
     : arch_cpu_idle()
     : default_idle_call()
     : do_idle()
     : cpu_startup_entry()
     : rest_init()
     : start_kernel()
     : x86_64_start_reservations()
     : x86_64_start_kernel()
     : verify_cpu()
     :
     : -> #1 (&tty->write_wait){-.-.}:
     : __lock_acquire()
     : lock_acquire()
     : _raw_spin_lock_irqsave()
     : __wake_up_common_lock()
     : __wake_up()
     : tty_wakeup()
     : tty_port_default_wakeup()
     : tty_port_tty_wakeup()
     : uart_write_wakeup()
     : serial8250_tx_chars()
     : serial8250_handle_irq.part.25()
     : serial8250_default_handle_irq()
     : serial8250_interrupt()
     : __handle_irq_event_percpu()
     : handle_irq_event_percpu()
     : handle_irq_event()
     : handle_level_irq()
     : handle_irq()
     : do_IRQ()
     : ret_from_intr()
     : native_safe_halt()
     : default_idle()
     : arch_cpu_idle()
     : default_idle_call()
     : do_idle()
     : cpu_startup_entry()
     : rest_init()
     : start_kernel()
     : x86_64_start_reservations()
     : x86_64_start_kernel()
     : verify_cpu()
     :
     : -> #0 (&port_lock_key){-.-.}:
     : check_prev_add()
     : __lock_acquire()
     : lock_acquire()
     : _raw_spin_lock_irqsave()
     : serial8250_console_write()
     : univ8250_console_write()
     : console_unlock()
     : vprintk_emit()
     : vprintk_default()
     : vprintk_func()
     : printk()
     : ___ratelimit()
     : __printk_ratelimit()
     : select_fallback_rq()
     : sched_cpu_dying()
     : cpuhp_invoke_callback()
     : take_cpu_down()
     : multi_cpu_stop()
     : cpu_stopper_thread()
     : smpboot_thread_fn()
     : kthread()
     : ret_from_fork()
     :
     : other info that might help us debug this:
     :
     : Chain exists of:
     :   &port_lock_key --> &p->pi_lock --> &rq->lock
     :
     :  Possible unsafe locking scenario:
     :
     :        CPU0                    CPU1
     :        ----                    ----
     :   lock(&rq->lock);
     :                                lock(&p->pi_lock);
     :                                lock(&rq->lock);
     :   lock(&port_lock_key);
     :
     :  *** DEADLOCK ***
     :
     : 4 locks held by migration/8/62:
     : #0: (&p->pi_lock){-.-.}, at: sched_cpu_dying()
     : #1: (&rq->lock){-.-.}, at: sched_cpu_dying()
     : #2: (printk_ratelimit_state.lock){....}, at: ___ratelimit()
     : #3: (console_lock){+.+.}, at: vprintk_emit()
     :
     : stack backtrace:
     : CPU: 8 PID: 62 Comm: migration/8 Not tainted 4.14.0-rc2-next-20170927+ #252
     : Call Trace:
     : dump_stack()
     : print_circular_bug()
     : check_prev_add()
     : ? add_lock_to_list.isra.26()
     : ? check_usage()
     : ? kvm_clock_read()
     : ? kvm_sched_clock_read()
     : ? sched_clock()
     : ? check_preemption_disabled()
     : __lock_acquire()
     : ? __lock_acquire()
     : ? add_lock_to_list.isra.26()
     : ? debug_check_no_locks_freed()
     : ? memcpy()
     : lock_acquire()
     : ? serial8250_console_write()
     : _raw_spin_lock_irqsave()
     : ? serial8250_console_write()
     : serial8250_console_write()
     : ? serial8250_start_tx()
     : ? lock_acquire()
     : ? memcpy()
     : univ8250_console_write()
     : console_unlock()
     : ? __down_trylock_console_sem()
     : vprintk_emit()
     : vprintk_default()
     : vprintk_func()
     : printk()
     : ? show_regs_print_info()
     : ? lock_acquire()
     : ___ratelimit()
     : __printk_ratelimit()
     : select_fallback_rq()
     : sched_cpu_dying()
     : ? sched_cpu_starting()
     : ? rcutree_dying_cpu()
     : ? sched_cpu_starting()
     : cpuhp_invoke_callback()
     : ? cpu_disable_common()
     : take_cpu_down()
     : ? trace_hardirqs_off_caller()
     : ? cpuhp_invoke_callback()
     : multi_cpu_stop()
     : ? __this_cpu_preempt_check()
     : ? cpu_stop_queue_work()
     : cpu_stopper_thread()
     : ? cpu_stop_create()
     : smpboot_thread_fn()
     : ? sort_range()
     : ? schedule()
     : ? __kthread_parkme()
     : kthread()
     : ? sort_range()
     : ? kthread_create_on_node()
     : ret_from_fork()
     : process 9121 (trinity-c78) no longer affine to cpu8
     : smpboot: CPU 8 is now offline
    
    Link: http://lkml.kernel.org/r/20170928120405.18273-1-sergey.senozhatsky@gmail.com
    Fixes: 6b1d174b ("ratelimit: extend to print suppressed messages on release")
    Signed-off-by: default avatarSergey Senozhatsky <sergey.senozhatsky@gmail.com>
    Reported-by: default avatarSasha Levin <levinsasha928@gmail.com>
    Reviewed-by: default avatarPetr Mladek <pmladek@suse.com>
    Cc: Peter Zijlstra <peterz@infradead.org>
    Cc: Thomas Gleixner <tglx@linutronix.de>
    Cc: Ingo Molnar <mingo@elte.hu>
    Cc: Borislav Petkov <bp@suse.de>
    Cc: Steven Rostedt <rostedt@goodmis.org>
    Cc: <stable@vger.kernel.org>
    Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
    Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
    656d61ce
Name
Last commit
Last update
..
842 Loading commit data...
fonts Loading commit data...
lz4 Loading commit data...
lzo Loading commit data...
mpi Loading commit data...
raid6 Loading commit data...
reed_solomon Loading commit data...
xz Loading commit data...
zlib_deflate Loading commit data...
zlib_inflate Loading commit data...
zstd Loading commit data...
.gitignore Loading commit data...
Kconfig Loading commit data...
Kconfig.debug Loading commit data...
Kconfig.kasan Loading commit data...
Kconfig.kgdb Loading commit data...
Kconfig.kmemcheck Loading commit data...
Kconfig.ubsan Loading commit data...
Makefile Loading commit data...
argv_split.c Loading commit data...
asn1_decoder.c Loading commit data...
assoc_array.c Loading commit data...
atomic64.c Loading commit data...
atomic64_test.c Loading commit data...
audit.c Loading commit data...
bcd.c Loading commit data...
bch.c Loading commit data...
bitmap.c Loading commit data...
bitrev.c Loading commit data...
bsearch.c Loading commit data...
btree.c Loading commit data...
bug.c Loading commit data...
build_OID_registry Loading commit data...
bust_spinlocks.c Loading commit data...
chacha20.c Loading commit data...
check_signature.c Loading commit data...
checksum.c Loading commit data...
clz_ctz.c Loading commit data...
clz_tab.c Loading commit data...
cmdline.c Loading commit data...
compat_audit.c Loading commit data...
cordic.c Loading commit data...
cpu_rmap.c Loading commit data...
cpumask.c Loading commit data...
crc-ccitt.c Loading commit data...
crc-itu-t.c Loading commit data...
crc-t10dif.c Loading commit data...
crc16.c Loading commit data...
crc32.c Loading commit data...
crc32defs.h Loading commit data...
crc32test.c Loading commit data...
crc4.c Loading commit data...
crc7.c Loading commit data...
crc8.c Loading commit data...
ctype.c Loading commit data...
debug_info.c Loading commit data...
debug_locks.c Loading commit data...
debugobjects.c Loading commit data...
dec_and_lock.c Loading commit data...
decompress.c Loading commit data...
decompress_bunzip2.c Loading commit data...
decompress_inflate.c Loading commit data...
decompress_unlz4.c Loading commit data...
decompress_unlzma.c Loading commit data...
decompress_unlzo.c Loading commit data...
decompress_unxz.c Loading commit data...
devres.c Loading commit data...
digsig.c Loading commit data...
div64.c Loading commit data...
dma-debug.c Loading commit data...
dma-noop.c Loading commit data...
dma-virt.c Loading commit data...
dump_stack.c Loading commit data...
dynamic_debug.c Loading commit data...
dynamic_queue_limits.c Loading commit data...
earlycpio.c Loading commit data...
errseq.c Loading commit data...
extable.c Loading commit data...
fault-inject.c Loading commit data...
fdt.c Loading commit data...
fdt_empty_tree.c Loading commit data...
fdt_ro.c Loading commit data...
fdt_rw.c Loading commit data...
fdt_strerror.c Loading commit data...
fdt_sw.c Loading commit data...
fdt_wip.c Loading commit data...
find_bit.c Loading commit data...
flex_array.c Loading commit data...
flex_proportions.c Loading commit data...
gcd.c Loading commit data...
gen_crc32table.c Loading commit data...
genalloc.c Loading commit data...
glob.c Loading commit data...
globtest.c Loading commit data...
hexdump.c Loading commit data...
hweight.c Loading commit data...
idr.c Loading commit data...
inflate.c Loading commit data...
int_sqrt.c Loading commit data...
interval_tree.c Loading commit data...
interval_tree_test.c Loading commit data...
iomap.c Loading commit data...
iomap_copy.c Loading commit data...
iommu-common.c Loading commit data...
iommu-helper.c Loading commit data...
ioremap.c Loading commit data...
iov_iter.c Loading commit data...
irq_poll.c Loading commit data...
irq_regs.c Loading commit data...
is_single_threaded.c Loading commit data...
jedec_ddr_data.c Loading commit data...
kasprintf.c Loading commit data...
kfifo.c Loading commit data...
klist.c Loading commit data...
kobject.c Loading commit data...
kobject_uevent.c Loading commit data...
kstrtox.c Loading commit data...
kstrtox.h Loading commit data...
lcm.c Loading commit data...
libcrc32c.c Loading commit data...
list_debug.c Loading commit data...
list_sort.c Loading commit data...
llist.c Loading commit data...
locking-selftest-hardirq.h Loading commit data...
locking-selftest-mutex.h Loading commit data...
locking-selftest-rlock-hardirq.h Loading commit data...
locking-selftest-rlock-softirq.h Loading commit data...
locking-selftest-rlock.h Loading commit data...
locking-selftest-rsem.h Loading commit data...
locking-selftest-rtmutex.h Loading commit data...
locking-selftest-softirq.h Loading commit data...
locking-selftest-spin-hardirq.h Loading commit data...
locking-selftest-spin-softirq.h Loading commit data...
locking-selftest-spin.h Loading commit data...
locking-selftest-wlock-hardirq.h Loading commit data...
locking-selftest-wlock-softirq.h Loading commit data...
locking-selftest-wlock.h Loading commit data...
locking-selftest-wsem.h Loading commit data...
locking-selftest.c Loading commit data...
lockref.c Loading commit data...
lru_cache.c Loading commit data...
memory-notifier-error-inject.c Loading commit data...
memweight.c Loading commit data...
net_utils.c Loading commit data...
netdev-notifier-error-inject.c Loading commit data...
nlattr.c Loading commit data...
nmi_backtrace.c Loading commit data...
nodemask.c Loading commit data...
notifier-error-inject.c Loading commit data...
notifier-error-inject.h Loading commit data...
of-reconfig-notifier-error-inject.c Loading commit data...
oid_registry.c Loading commit data...
once.c Loading commit data...
parman.c Loading commit data...
parser.c Loading commit data...
pci_iomap.c Loading commit data...
percpu-refcount.c Loading commit data...
percpu_counter.c Loading commit data...
percpu_ida.c Loading commit data...
percpu_test.c Loading commit data...
plist.c Loading commit data...
pm-notifier-error-inject.c Loading commit data...
prime_numbers.c Loading commit data...
radix-tree.c Loading commit data...
random32.c Loading commit data...
ratelimit.c Loading commit data...
rational.c Loading commit data...
rbtree.c Loading commit data...
rbtree_test.c Loading commit data...
reciprocal_div.c Loading commit data...
refcount.c Loading commit data...
rhashtable.c Loading commit data...
sbitmap.c Loading commit data...
scatterlist.c Loading commit data...
seq_buf.c Loading commit data...
sg_pool.c Loading commit data...
sg_split.c Loading commit data...
sha1.c Loading commit data...
show_mem.c Loading commit data...
siphash.c Loading commit data...
smp_processor_id.c Loading commit data...
sort.c Loading commit data...
stackdepot.c Loading commit data...
stmp_device.c Loading commit data...
string.c Loading commit data...
string_helpers.c Loading commit data...
strncpy_from_user.c Loading commit data...
strnlen_user.c Loading commit data...
swiotlb.c Loading commit data...
syscall.c Loading commit data...
test-kstrtox.c Loading commit data...
test-string_helpers.c Loading commit data...
test_bitmap.c Loading commit data...
test_bpf.c Loading commit data...
test_debug_virtual.c Loading commit data...
test_firmware.c Loading commit data...
test_hash.c Loading commit data...
test_hexdump.c Loading commit data...
test_kasan.c Loading commit data...
test_kmod.c Loading commit data...
test_list_sort.c Loading commit data...
test_module.c Loading commit data...
test_parman.c Loading commit data...
test_printf.c Loading commit data...
test_rhashtable.c Loading commit data...
test_siphash.c Loading commit data...
test_sort.c Loading commit data...
test_static_key_base.c Loading commit data...
test_static_keys.c Loading commit data...
test_sysctl.c Loading commit data...
test_user_copy.c Loading commit data...
test_uuid.c Loading commit data...
textsearch.c Loading commit data...
timerqueue.c Loading commit data...
ts_bm.c Loading commit data...
ts_fsm.c Loading commit data...
ts_kmp.c Loading commit data...
ubsan.c Loading commit data...
ubsan.h Loading commit data...
ucs2_string.c Loading commit data...
usercopy.c Loading commit data...
uuid.c Loading commit data...
vsprintf.c Loading commit data...
win_minmax.c Loading commit data...
xxhash.c Loading commit data...