From 406e53807cace6715e47304b17ea6b159ac52d24 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 22 Sep 2023 14:12:21 +0000 Subject: [PATCH 138/184] printk: nbcon: Introduce printing kthreads Add the infrastructure to create a printer kthread per console along with the required kthread function, which is takeover/handover aware. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Signed-off-by: Sebastian Andrzej Siewior --- include/linux/console.h | 10 ++ kernel/printk/internal.h | 28 +++++ kernel/printk/nbcon.c | 233 ++++++++++++++++++++++++++++++++++++++- kernel/printk/printk.c | 31 ++++++ 4 files changed, 299 insertions(+), 3 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 3cfc0e7f11d6..8f535835f7ab 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -17,6 +17,7 @@ #include #include #include +#include #include struct vc_data; @@ -296,10 +297,14 @@ struct nbcon_write_context { * @node: hlist node for the console list * * @write_atomic: Write callback for atomic context + * @write_thread: Write callback for non-atomic context + * @uart_port: Callback to provide the associated uart port * @nbcon_state: State for nbcon consoles * @nbcon_seq: Sequence number of the next record for nbcon to print * @pbufs: Pointer to nbcon private buffer * @locked_port: True, if the port lock is locked by nbcon + * @kthread: Printer kthread for this console + * @rcuwait: RCU-safe wait object for @kthread waking */ struct console { char name[16]; @@ -323,10 +328,15 @@ struct console { /* nbcon console specific members */ bool (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt); + bool (*write_thread)(struct console *con, + struct nbcon_write_context *wctxt); + struct uart_port * (*uart_port)(struct console *con); atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; struct printk_buffers *pbufs; bool locked_port; + struct task_struct *kthread; + struct rcuwait rcuwait; }; #ifdef CONFIG_LOCKDEP diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 9b86e9d87a9c..8a3f4a0669f9 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -44,6 +44,7 @@ enum printk_info_flags { }; extern struct printk_ringbuffer *prb; +extern bool have_boot_console; __printf(4, 0) int vprintk_store(int facility, int level, @@ -80,6 +81,7 @@ void nbcon_free(struct console *con); enum nbcon_prio nbcon_get_default_prio(void); void nbcon_atomic_flush_all(void); bool nbcon_atomic_emit_next_record(struct console *con); +void nbcon_kthread_create(struct console *con); /* * Check if the given console is currently capable and allowed to print @@ -98,6 +100,8 @@ static inline bool console_is_usable(struct console *con, short flags) if (flags & CON_NBCON) { if (!con->write_atomic) return false; + if (!con->write_thread || !con->kthread) + return false; } else { if (!con->write) return false; @@ -114,12 +118,36 @@ static inline bool console_is_usable(struct console *con, short flags) return true; } +/** + * nbcon_kthread_wake - Wake up a printk thread + * @con: Console to operate on + */ +static inline void nbcon_kthread_wake(struct console *con) +{ + /* + * Guarantee any new records can be seen by tasks preparing to wait + * before this context checks if the rcuwait is empty. + * + * The full memory barrier in rcuwait_wake_up() pairs with the full + * memory barrier within set_current_state() of + * ___rcuwait_wait_event(), which is called after prepare_to_rcuwait() + * adds the waiter but before it has checked the wait condition. + * + * This pairs with nbcon_kthread_func:A. + */ + rcuwait_wake_up(&con->rcuwait); /* LMM(nbcon_kthread_wake:A) */ +} + #else #define PRINTK_PREFIX_MAX 0 #define PRINTK_MESSAGE_MAX 0 #define PRINTKRB_RECORD_MAX 0 +static inline void nbcon_kthread_wake(struct console *con) { } +static inline void nbcon_kthread_create(struct console *con) { } +#define printk_threads_enabled (false) + /* * In !PRINTK builds we still export console_sem * semaphore and some of console functions (console_unlock()/etc.), so diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 41654cb7ba85..dfc4ca06e5c5 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -5,8 +5,11 @@ #include #include #include +#include #include #include +#include +#include "printk_ringbuffer.h" #include "internal.h" /* * Printk console printing implementation for consoles which does not depend @@ -211,6 +214,11 @@ void nbcon_seq_force(struct console *con, u64 seq) con->seq = 0; } +static void nbcon_context_seq_set(struct nbcon_context *ctxt) +{ + ctxt->seq = nbcon_seq_read(ctxt->console); +} + /** * nbcon_seq_try_update - Try to update the console sequence number * @ctxt: Pointer to an acquire context that contains @@ -235,6 +243,8 @@ static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) } } +bool printk_threads_enabled __ro_after_init; + /** * nbcon_context_try_acquire_direct - Try to acquire directly * @ctxt: The context of the caller @@ -860,6 +870,7 @@ EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); /** * nbcon_emit_next_record - Emit a record in the acquired context * @wctxt: The write context that will be handed to the write function + * @use_atomic: True if the write_atomic callback is to be used * * Return: True if this context still owns the console. False if * ownership was handed over or taken. @@ -873,7 +884,7 @@ EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); * When true is returned, @wctxt->ctxt.backlog indicates whether there are * still records pending in the ringbuffer, */ -static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) +static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic) { struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); struct console *con = ctxt->console; @@ -923,8 +934,16 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) nbcon_state_read(con, &cur); wctxt->unsafe_takeover = cur.unsafe_takeover; - if (con->write_atomic) { + if (use_atomic && + con->write_atomic) { done = con->write_atomic(con, wctxt); + + } else if (!use_atomic && + con->write_thread && + con->kthread) { + WARN_ON_ONCE(con->kthread != current); + done = con->write_thread(con, wctxt); + } else { WARN_ON_ONCE(1); done = false; @@ -964,6 +983,141 @@ static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) return nbcon_context_exit_unsafe(ctxt); } +/** + * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup + * @con: Console to operate on + * @ctxt: The acquire context that contains the state + * at console_acquire() + * + * Return: True if the thread should shutdown or if the console is + * allowed to print and a record is available. False otherwise. + * + * After the thread wakes up, it must first check if it should shutdown before + * attempting any printing. + */ +static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt) +{ + struct nbcon_state cur; + bool is_usable; + short flags; + int cookie; + + do { + if (kthread_should_stop()) + return true; + + cookie = console_srcu_read_lock(); + flags = console_srcu_read_flags(con); + is_usable = console_is_usable(con, flags); + console_srcu_read_unlock(cookie); + + if (!is_usable) + return false; + + nbcon_state_read(con, &cur); + + /* + * Some other CPU is using the console. Patiently poll + * to see if it becomes available. This is more efficient + * than having every release trigger an irq_work to wake + * the kthread. + */ + msleep(1); + } while (cur.prio != NBCON_PRIO_NONE); + + /* Bring the sequence in @ctxt up to date */ + nbcon_context_seq_set(ctxt); + + return prb_read_valid(prb, ctxt->seq, NULL); +} + +/** + * nbcon_kthread_func - The printer thread function + * @__console: Console to operate on + */ +static int nbcon_kthread_func(void *__console) +{ + struct console *con = __console; + struct nbcon_write_context wctxt = { + .ctxt.console = con, + .ctxt.prio = NBCON_PRIO_NORMAL, + }; + struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt); + struct uart_port *port = NULL; + unsigned long flags; + short con_flags; + bool backlog; + int cookie; + int ret; + + if (con->uart_port) + port = con->uart_port(con); + +wait_for_event: + /* + * Guarantee this task is visible on the rcuwait before + * checking the wake condition. + * + * The full memory barrier within set_current_state() of + * ___rcuwait_wait_event() pairs with the full memory + * barrier within rcuwait_has_sleeper(). + * + * This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A. + */ + ret = rcuwait_wait_event(&con->rcuwait, + nbcon_kthread_should_wakeup(con, ctxt), + TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */ + + if (kthread_should_stop()) + return 0; + + /* Wait was interrupted by a spurious signal, go back to sleep. */ + if (ret) + goto wait_for_event; + + do { + backlog = false; + + cookie = console_srcu_read_lock(); + + con_flags = console_srcu_read_flags(con); + + if (console_is_usable(con, con_flags)) { + /* + * Ensure this stays on the CPU to make handover and + * takeover possible. + */ + if (port) + spin_lock_irqsave(&port->lock, flags); + else + migrate_disable(); + + if (nbcon_context_try_acquire(ctxt)) { + /* + * If the emit fails, this context is no + * longer the owner. + */ + if (nbcon_emit_next_record(&wctxt, false)) { + nbcon_context_release(ctxt); + backlog = ctxt->backlog; + } + } + + if (port) + spin_unlock_irqrestore(&port->lock, flags); + else + migrate_enable(); + } + + console_srcu_read_unlock(cookie); + + cond_resched(); + + } while (backlog); + + goto wait_for_event; +} + /* Track the nbcon emergency nesting per CPU. */ static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting); static unsigned int early_nbcon_pcpu_emergency_nesting __initdata; @@ -1011,7 +1165,7 @@ static bool nbcon_atomic_emit_one(struct nbcon_write_context *wctxt) * handed over or taken over. In both cases the context is no * longer valid. */ - if (!nbcon_emit_next_record(wctxt)) + if (!nbcon_emit_next_record(wctxt, true)) return false; nbcon_context_release(ctxt); @@ -1195,6 +1349,76 @@ void nbcon_cpu_emergency_exit(void) preempt_enable(); } +/** + * nbcon_kthread_stop - Stop a printer thread + * @con: Console to operate on + */ +static void nbcon_kthread_stop(struct console *con) +{ + lockdep_assert_console_list_lock_held(); + + if (!con->kthread) + return; + + kthread_stop(con->kthread); + con->kthread = NULL; +} + +/** + * nbcon_kthread_create - Create a printer thread + * @con: Console to operate on + * + * If it fails, let the console proceed. The atomic part might + * be usable and useful. + */ +void nbcon_kthread_create(struct console *con) +{ + struct task_struct *kt; + + lockdep_assert_console_list_lock_held(); + + if (!(con->flags & CON_NBCON) || !con->write_thread) + return; + + if (!printk_threads_enabled || con->kthread) + return; + + /* + * Printer threads cannot be started as long as any boot console is + * registered because there is no way to synchronize the hardware + * registers between boot console code and regular console code. + */ + if (have_boot_console) + return; + + kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index); + if (IS_ERR(kt)) { + con_printk(KERN_ERR, con, "failed to start printing thread\n"); + return; + } + + con->kthread = kt; + + /* + * It is important that console printing threads are scheduled + * shortly after a printk call and with generous runtime budgets. + */ + sched_set_normal(con->kthread, -20); +} + +static int __init printk_setup_threads(void) +{ + struct console *con; + + console_list_lock(); + printk_threads_enabled = true; + for_each_console(con) + nbcon_kthread_create(con); + console_list_unlock(); + return 0; +} +early_initcall(printk_setup_threads); + /** * nbcon_alloc - Allocate buffers needed by the nbcon console * @con: Console to allocate buffers for @@ -1241,8 +1465,10 @@ void nbcon_init(struct console *con) /* nbcon_alloc() must have been called and successful! */ BUG_ON(!con->pbufs); + rcuwait_init(&con->rcuwait); nbcon_seq_force(con, con->seq); nbcon_state_set(con, &state); + nbcon_kthread_create(con); } /** @@ -1253,6 +1479,7 @@ void nbcon_free(struct console *con) { struct nbcon_state state = { }; + nbcon_kthread_stop(con); nbcon_state_set(con, &state); /* Boot consoles share global printk buffers. */ diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 8aba9be0ee31..4771339a9413 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2636,6 +2636,8 @@ void suspend_console(void) void resume_console(void) { struct console *con; + short flags; + int cookie; if (!console_suspend_enabled) return; @@ -2652,6 +2654,14 @@ void resume_console(void) */ synchronize_srcu(&console_srcu); + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + flags = console_srcu_read_flags(con); + if (flags & CON_NBCON) + nbcon_kthread_wake(con); + } + console_srcu_read_unlock(cookie); + pr_flush(1000, true); } @@ -2972,6 +2982,13 @@ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handove u64 printk_seq; bool progress; + /* + * console_flush_all() is only for legacy consoles, + * unless the nbcon console has no kthread printer. + */ + if ((flags & CON_NBCON) && con->kthread) + continue; + if (!console_is_usable(con, flags)) continue; any_usable = true; @@ -3266,9 +3283,23 @@ EXPORT_SYMBOL(console_stop); void console_start(struct console *console) { + short flags; + console_list_lock(); console_srcu_write_flags(console, console->flags | CON_ENABLED); + flags = console->flags; console_list_unlock(); + + /* + * Ensure that all SRCU list walks have completed. The related + * printing context must be able to see it is enabled so that + * it is guaranteed to wake up and resume printing. + */ + synchronize_srcu(&console_srcu); + + if (flags & CON_NBCON) + nbcon_kthread_wake(console); + __pr_flush(console, 1000, true); } EXPORT_SYMBOL(console_start); -- 2.34.1