From daedbaf406365be60e3948642208dc95271c2285 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Fri, 22 Sep 2023 17:35:04 +0000 Subject: [PATCH 149/184] printk: Add kthread for all legacy consoles The write callback of legacy consoles make use of spinlocks. This is not permitted with PREEMPT_RT in atomic contexts. Create a new kthread to handling printing of all the legacy consoles (and nbcon consoles if boot consoles are registered). Since the consoles are printing in a task context, it is no longer appropriate to support the legacy handover mechanism. These changes exist only for CONFIG_PREEMPT_RT. Signed-off-by: John Ogness Signed-off-by: Sebastian Andrzej Siewior --- kernel/printk/internal.h | 1 + kernel/printk/nbcon.c | 22 +++- kernel/printk/printk.c | 243 +++++++++++++++++++++++++++++++-------- 3 files changed, 219 insertions(+), 47 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index c709010a885a..a3ae0710346c 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -93,6 +93,7 @@ void nbcon_atomic_flush_all(void); bool nbcon_atomic_emit_next_record(struct console *con); void nbcon_kthread_create(struct console *con); void nbcon_wake_threads(void); +void nbcon_legacy_kthread_create(void); /* * Check if the given console is currently capable and allowed to print diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 427e26c89067..a9be219673db 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1279,8 +1279,21 @@ enum nbcon_prio nbcon_get_default_prio(void) */ bool nbcon_atomic_emit_next_record(struct console *con) { - enum nbcon_prio prio = nbcon_get_default_prio(); + struct uart_port *port = con->uart_port(con); + static DEFINE_SPINLOCK(shared_spinlock); bool progress = false; + enum nbcon_prio prio; + unsigned long flags; + + /* + * If there is no port lock available, fallback to a shared + * spinlock. This serves to provide the necessary type of + * migration/preemption disabling while printing. + */ + if (port) + spin_lock_irqsave(&port->lock, flags); + else + spin_lock_irqsave(&shared_spinlock, flags); /* * Do not emit for EMERGENCY priority. The console will be @@ -1297,6 +1310,11 @@ bool nbcon_atomic_emit_next_record(struct console *con) progress = nbcon_atomic_emit_one(&wctxt); } + if (port) + spin_unlock_irqrestore(&port->lock, flags); + else + spin_unlock_irqrestore(&shared_spinlock, flags); + return progress; } @@ -1485,6 +1503,8 @@ static int __init printk_setup_threads(void) printk_threads_enabled = true; for_each_console(con) nbcon_kthread_create(con); + if (IS_ENABLED(CONFIG_PREEMPT_RT) && printing_via_unlock) + nbcon_legacy_kthread_create(); console_list_unlock(); return 0; } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index d16d5905795e..cff34b06fabf 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -486,6 +486,9 @@ bool have_boot_console; static int __read_mostly suppress_panic_printk; DECLARE_WAIT_QUEUE_HEAD(log_wait); + +static DECLARE_WAIT_QUEUE_HEAD(legacy_wait); + /* All 3 protected by @syslog_lock. */ /* the next printk record to read by syslog(READ) or /proc/kmsg */ static u64 syslog_seq; @@ -2335,8 +2338,8 @@ asmlinkage int vprintk_emit(int facility, int level, { static atomic_t panic_noise_count = ATOMIC_INIT(0); + bool do_trylock_unlock = printing_via_unlock && !IS_ENABLED(CONFIG_PREEMPT_RT); int printed_len; - bool in_sched = false; /* Suppress unimportant messages after panic happens */ if (unlikely(suppress_printk)) @@ -2361,7 +2364,8 @@ asmlinkage int vprintk_emit(int facility, int level, if (level == LOGLEVEL_SCHED) { level = LOGLEVEL_DEFAULT; - in_sched = true; + /* If called from the scheduler, we can not call up(). */ + do_trylock_unlock = false; } printk_delay(level); @@ -2396,8 +2400,7 @@ asmlinkage int vprintk_emit(int facility, int level, nbcon_wake_threads(); - /* If called from the scheduler, we can not call up(). */ - if (printing_via_unlock && !in_sched) { + if (do_trylock_unlock) { /* * The caller may be holding system-critical or * timing-sensitive locks. Disable preemption during @@ -2418,12 +2421,11 @@ asmlinkage int vprintk_emit(int facility, int level, console_unlock(); } preempt_enable(); - } - if (in_sched) - defer_console_output(); - else wake_up_klogd(); + } else { + defer_console_output(); + } return printed_len; } @@ -2451,6 +2453,14 @@ EXPORT_SYMBOL(_printk); static bool pr_flush(int timeout_ms, bool reset_on_progress); static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress); +static struct task_struct *nbcon_legacy_kthread; + +static inline void wake_up_legacy_kthread(void) +{ + if (nbcon_legacy_kthread) + wake_up_interruptible(&legacy_wait); +} + #else /* CONFIG_PRINTK */ #define printk_time false @@ -2463,7 +2473,8 @@ static u64 syslog_seq; static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; } static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; } - +static inline void nbcon_legacy_kthread_create(void) { } +static inline void wake_up_legacy_kthread(void) { } #endif /* CONFIG_PRINTK */ #ifdef CONFIG_EARLY_PRINTK @@ -2702,6 +2713,8 @@ void resume_console(void) } console_srcu_read_unlock(cookie); + wake_up_legacy_kthread(); + pr_flush(1000, true); } @@ -2716,7 +2729,8 @@ void resume_console(void) */ static int console_cpu_notify(unsigned int cpu) { - if (!cpuhp_tasks_frozen && printing_via_unlock) { + if (!cpuhp_tasks_frozen && printing_via_unlock && + !IS_ENABLED(CONFIG_PREEMPT_RT)) { /* If trylock fails, someone else is doing the printing */ if (console_trylock()) console_unlock(); @@ -2941,31 +2955,38 @@ static bool console_emit_next_record(struct console *con, bool *handover, int co con->dropped = 0; } - /* - * While actively printing out messages, if another printk() - * were to occur on another CPU, it may wait for this one to - * finish. This task can not be preempted if there is a - * waiter waiting to take over. - * - * Interrupts are disabled because the hand over to a waiter - * must not be interrupted until the hand over is completed - * (@console_waiter is cleared). - */ - printk_safe_enter_irqsave(flags); - console_lock_spinning_enable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + /* + * While actively printing out messages, if another printk() + * were to occur on another CPU, it may wait for this one to + * finish. This task can not be preempted if there is a + * waiter waiting to take over. + * + * Interrupts are disabled because the hand over to a waiter + * must not be interrupted until the hand over is completed + * (@console_waiter is cleared). + */ + printk_safe_enter_irqsave(flags); + console_lock_spinning_enable(); - /* Do not trace print latency. */ - stop_critical_timings(); + /* Do not trace print latency. */ + stop_critical_timings(); + } /* Write everything out to the hardware. */ con->write(con, outbuf, pmsg.outbuf_len); - start_critical_timings(); - con->seq = pmsg.seq + 1; - *handover = console_lock_spinning_disable_and_check(cookie); - printk_safe_exit_irqrestore(flags); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + *handover = false; + } else { + start_critical_timings(); + + *handover = console_lock_spinning_disable_and_check(cookie); + + printk_safe_exit_irqrestore(flags); + } skip: return true; } @@ -3074,19 +3095,7 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove return false; } -/** - * console_unlock - unblock the console subsystem from printing - * - * Releases the console_lock which the caller holds to block printing of - * the console subsystem. - * - * While the console_lock was held, console output may have been buffered - * by printk(). If this is the case, console_unlock(); emits - * the output prior to releasing the lock. - * - * console_unlock(); may be called from any context. - */ -void console_unlock(void) +static u64 console_flush_and_unlock(void) { bool do_cond_resched; bool handover; @@ -3129,6 +3138,34 @@ void console_unlock(void) * fails, another context is already handling the printing. */ } while (prb_read_valid(prb, next_seq, NULL) && console_trylock()); + + return next_seq; +} + +/** + * console_unlock - unblock the console subsystem from printing + * + * Releases the console_lock which the caller holds to block printing of + * the console subsystem. + * + * While the console_lock was held, console output may have been buffered + * by printk(). If this is the case, console_unlock(); emits + * the output prior to releasing the lock. + * + * console_unlock(); may be called from any context. + */ +void console_unlock(void) +{ + /* + * PREEMPT_RT relies on kthread and atomic consoles for printing. + * It never attempts to print from console_unlock(). + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + __console_unlock(); + return; + } + + console_flush_and_unlock(); } EXPORT_SYMBOL(console_unlock); @@ -3340,11 +3377,106 @@ void console_start(struct console *console) if (flags & CON_NBCON) nbcon_kthread_wake(console); + else + wake_up_legacy_kthread(); __pr_flush(console, 1000, true); } EXPORT_SYMBOL(console_start); +#ifdef CONFIG_PRINTK +static bool printer_should_wake(u64 seq) +{ + bool available = false; + struct console *con; + int cookie; + + if (kthread_should_stop()) + return true; + + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + short flags = console_srcu_read_flags(con); + u64 printk_seq; + + /* + * The legacy printer thread is only for legacy consoles, + * unless the nbcon console has no kthread printer. + */ + if ((flags & CON_NBCON) && con->kthread) + continue; + + if (!console_is_usable(con, flags, true)) + continue; + + if (flags & CON_NBCON) { + printk_seq = nbcon_seq_read(con); + } else { + /* + * It is safe to read @seq because only this + * thread context updates @seq. + */ + printk_seq = con->seq; + } + + if (prb_read_valid(prb, printk_seq, NULL)) { + available = true; + break; + } + } + console_srcu_read_unlock(cookie); + + return available; +} + +static int nbcon_legacy_kthread_func(void *unused) +{ + u64 seq = 0; + int error; + + for (;;) { + error = wait_event_interruptible(legacy_wait, printer_should_wake(seq)); + + if (kthread_should_stop()) + break; + + if (error) + continue; + + console_lock(); + seq = console_flush_and_unlock(); + } + return 0; +} + +void nbcon_legacy_kthread_create(void) +{ + struct task_struct *kt; + + lockdep_assert_held(&console_mutex); + + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + return; + + if (!printk_threads_enabled || nbcon_legacy_kthread) + return; + + kt = kthread_run(nbcon_legacy_kthread_func, NULL, "pr/legacy"); + if (IS_ERR(kt)) { + pr_err("unable to start legacy printing thread\n"); + return; + } + + nbcon_legacy_kthread = kt; + + /* + * It is important that console printing threads are scheduled + * shortly after a printk call and with generous runtime budgets. + */ + sched_set_normal(nbcon_legacy_kthread, -20); +} +#endif /* CONFIG_PRINTK */ + static int __read_mostly keep_bootcon; static int __init keep_bootcon_setup(char *str) @@ -3601,6 +3733,7 @@ void register_console(struct console *newcon) nbcon_init(newcon); } else { have_legacy_console = true; + nbcon_legacy_kthread_create(); } if (newcon->flags & CON_BOOT) @@ -3737,6 +3870,13 @@ static int unregister_console_locked(struct console *console) nbcon_kthread_create(c); } +#ifdef CONFIG_PRINTK + if (!printing_via_unlock && nbcon_legacy_kthread) { + kthread_stop(nbcon_legacy_kthread); + nbcon_legacy_kthread = NULL; + } +#endif + return res; } @@ -3896,8 +4036,12 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre seq = prb_next_reserve_seq(prb); - /* Flush the consoles so that records up to @seq are printed. */ - if (printing_via_unlock) { + /* + * Flush the consoles so that records up to @seq are printed. + * Otherwise this function will just wait for the threaded printers + * to print up to @seq. + */ + if (printing_via_unlock && !IS_ENABLED(CONFIG_PREEMPT_RT)) { console_lock(); console_unlock(); } @@ -4005,9 +4149,16 @@ static void wake_up_klogd_work_func(struct irq_work *irq_work) int pending = this_cpu_xchg(printk_pending, 0); if (pending & PRINTK_PENDING_OUTPUT) { - /* If trylock fails, someone else is doing the printing */ - if (console_trylock()) - console_unlock(); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + wake_up_interruptible(&legacy_wait); + } else { + /* + * If trylock fails, some other context + * will do the printing. + */ + if (console_trylock()) + console_unlock(); + } } if (pending & PRINTK_PENDING_WAKEUP) -- 2.34.1