/*
 * Copyright (c) 2003-2023 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * Implements the architecture independant portion of the LWKT subsystem.
 */

#ifndef _SYS_THREAD_H_
#define	_SYS_THREAD_H_

#ifndef _SYS_PARAM_H_
#include <sys/param.h>		/* MAXCOMLEN */
#endif
#ifndef _SYS_QUEUE_H_
#include <sys/queue.h>		/* TAILQ_* macros */
#endif
#ifndef _SYS_MSGPORT_H_
#include <sys/msgport.h>	/* lwkt_port */
#endif
#ifndef _SYS_TIME_H_
#include <sys/time.h>   	/* struct timeval */
#endif
#ifndef _SYS_LOCK_H
#include <sys/lock.h>
#endif
#ifndef _SYS_SPINLOCK_H_
#include <sys/spinlock.h>
#endif
#ifndef _SYS_IOSCHED_H_
#include <sys/iosched.h>
#endif
#include <machine/thread.h>	/* md_thread */
#include <machine/stdint.h>
#include <machine/ucontext.h>

struct globaldata;
struct lwp;
struct proc;
struct thread;
struct lwkt_queue;
struct lwkt_token;
struct lwkt_tokref;
struct lwkt_ipiq;
#if 0
struct lwkt_cpu_msg;
struct lwkt_cpu_port;
#endif
struct lwkt_cpusync;
struct fdnode;
union sysunion;

struct sleepqueue_wchan;

typedef struct lwkt_queue	*lwkt_queue_t;
typedef struct lwkt_token	*lwkt_token_t;
typedef struct lwkt_tokref	*lwkt_tokref_t;
#if 0
typedef struct lwkt_cpu_msg	*lwkt_cpu_msg_t;
typedef struct lwkt_cpu_port	*lwkt_cpu_port_t;
#endif
typedef struct lwkt_ipiq	*lwkt_ipiq_t;
typedef struct lwkt_cpusync	*lwkt_cpusync_t;
typedef struct thread 		*thread_t;

typedef TAILQ_HEAD(lwkt_queue, thread) lwkt_queue;

/*
 * Differentiation between kernel threads and user threads.  Userland
 * programs which want to access to kernel structures have to define
 * _KERNEL_STRUCTURES.  This is a kinda safety valve to prevent badly
 * written user programs from getting an LWKT thread that is neither the
 * kernel nor the user version.
 */
#if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
#ifndef _SYS_CPUMASK_H_
#include <sys/cpumask.h>	/* cpumask_t */
#endif
#ifndef _CPU_FRAME_H_
#include <machine/frame.h>
#endif
#else
struct intrframe;
#endif

/*
 * Tokens are used to serialize access to information.  They are 'soft'
 * serialization entities that only stay in effect while a thread is
 * running.  If the thread blocks, other threads can run holding the same
 * token(s).  The tokens are reacquired when the original thread resumes.
 *
 * Tokens guarantee that no deadlock can happen regardless of type or
 * ordering.  However, obtaining the same token first shared, then
 * stacking exclusive, is not allowed and will panic.
 *
 * A thread can depend on its serialization remaining intact through a
 * preemption.  An interrupt which attempts to use the same token as the
 * thread being preempted will reschedule itself for non-preemptive
 * operation, so the new token code is capable of interlocking against
 * interrupts as well as other cpus.  This means that your token can only
 * be (temporarily) lost if you *explicitly* block.
 *
 * Tokens are managed through a helper reference structure, lwkt_tokref.  Each
 * thread has a stack of tokref's to keep track of acquired tokens.  Multiple
 * tokref's may reference the same token.
 *
 * EXCLUSIVE TOKENS
 *	Acquiring an exclusive token requires acquiring the EXCLUSIVE bit
 *	with count == 0.  If the exclusive bit cannot be acquired, EXCLREQ
 *	is set.  Once acquired, EXCLREQ is cleared (but could get set by
 *	another thread also trying for an exclusive lock at any time).
 *
 * SHARED TOKENS
 *	Acquiring a shared token requires waiting for the EXCLUSIVE bit
 *	to be cleared and then acquiring a count.  A shared lock request
 *	can temporarily acquire a count and then back it out if it is
 *	unable to obtain the EXCLUSIVE bit, allowing fetchadd to be used.
 *
 *	A thread attempting to get a single shared token will defer to
 *	pending exclusive requesters.  However, a thread already holding
 *	one or more tokens and trying to get an additional shared token
 *	cannot defer to exclusive requesters because doing so can lead
 *	to a deadlock.
 *
 * Multiple exclusive tokens are handled by treating the additional tokens
 * as a special case of the shared token, incrementing the count value.  This
 * reduces the complexity of the token release code.
 */

struct lwkt_token {
    long		t_count;	/* Shared/exclreq/exclusive access */
    struct lwkt_tokref	*t_ref;		/* Exclusive ref */
    long		t_collisions;	/* Collision counter */
    const char		*t_desc;	/* Descriptive name */
};

#define TOK_EXCLUSIVE	0x00000001	/* Exclusive lock held */
#define TOK_EXCLREQ	0x00000002	/* Exclusive request pending */
#define TOK_INCR	4		/* Shared count increment */
#define TOK_COUNTMASK	(~(long)(TOK_EXCLUSIVE|TOK_EXCLREQ))

/*
 * Static initialization for a lwkt_token.
 */
#define LWKT_TOKEN_INITIALIZER(name)	\
{					\
	.t_count = 0,			\
	.t_ref = NULL,			\
	.t_collisions = 0,		\
	.t_desc = #name			\
}

/*
 * Assert that a particular token is held
 */
#define LWKT_TOKEN_HELD_ANY(tok)	_lwkt_token_held_any(tok, curthread)
#define LWKT_TOKEN_HELD_EXCL(tok)	_lwkt_token_held_excl(tok, curthread)

#define ASSERT_LWKT_TOKEN_HELD(tok)		\
	KKASSERT(LWKT_TOKEN_HELD_ANY(tok))

#define ASSERT_LWKT_TOKEN_HELD_EXCL(tok)	\
	KKASSERT(LWKT_TOKEN_HELD_EXCL(tok))

#define ASSERT_NO_TOKENS_HELD(td)	\
	KKASSERT((td)->td_toks_stop == &td->td_toks_array[0])

struct lwkt_tokref {
    lwkt_token_t	tr_tok;		/* token in question */
    long		tr_count;	/* TOK_EXCLUSIVE|TOK_EXCLREQ or 0 */
    struct thread	*tr_owner;	/* me */
};

#define MAXCPUFIFO      256	/* power of 2 */
#define MAXCPUFIFO_MASK	(MAXCPUFIFO - 1)
#define LWKT_MAXTOKENS	32	/* max tokens beneficially held by thread */

#if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
/*
 * Always cast to ipifunc_t when registering an ipi.  The actual ipi function
 * is called with both the data and an interrupt frame, but the ipi function
 * that is registered might only declare a data argument.
 */
typedef void (*ipifunc1_t)(void *arg);
typedef void (*ipifunc2_t)(void *arg, int arg2);
typedef void (*ipifunc3_t)(void *arg, int arg2, struct intrframe *frame);

struct lwkt_ipiq {
    int		ip_rindex;      /* only written by target cpu */
    int		ip_xindex;      /* written by target, indicates completion */
    int		ip_windex;      /* only written by source cpu */
    int		ip_drain;	/* drain source limit */
    struct {
	ipifunc3_t	func;
	void		*arg1;
	int		arg2;
	char		filler[32 - sizeof(int) - sizeof(void *) * 2];
    } ip_info[MAXCPUFIFO];
};

/*
 * CPU Synchronization structure.  See lwkt_cpusync_init() and
 * lwkt_cpusync_interlock() for more information.
 */
typedef void (*cpusync_func_t)(void *arg);

struct lwkt_cpusync {
    cpumask_t	cs_mask;		/* cpus running the sync */
    cpumask_t	cs_mack;		/* mask acknowledge */
    cpusync_func_t cs_func;		/* function to execute */
    void	*cs_data;		/* function data */
};
#endif /* _KERNEL || _KERNEL_STRUCTURES */

/*
 * The standard message and queue structure used for communications between
 * cpus.  Messages are typically queued via a machine-specific non-linked
 * FIFO matrix allowing any cpu to send a message to any other cpu without
 * blocking.
 */
#if 0
typedef struct lwkt_cpu_msg {
    void	(*cm_func)(lwkt_cpu_msg_t msg);	/* primary dispatch function */
    int		cm_code;		/* request code if applicable */
    int		cm_cpu;			/* reply to cpu */
    thread_t	cm_originator;		/* originating thread for wakeup */
} lwkt_cpu_msg;
#endif

/*
 * per-thread file descriptor cache
 */
struct fdcache {
	int     fd;			/* descriptor being cached */
	int     locked;
	struct file *fp;		/* cached referenced fp */
	int	lru;
	int	unused[3];
} __cachealign;

#define NFDCACHE	4		/* max fd's cached by a thread */

/*
 * Thread structure.  Note that ownership of a thread structure is special
 * cased and there is no 'token'.  A thread is always owned by the cpu
 * represented by td_gd, any manipulation of the thread by some other cpu
 * must be done through cpu_*msg() functions.  e.g. you could request
 * ownership of a thread that way, or hand a thread off to another cpu.
 *
 * NOTE: td_ucred is synchronized from the p_ucred on user->kernel syscall,
 *	 trap, and AST/signal transitions to provide a stable ucred for
 *	 (primarily) system calls.  This field will be NULL for pure kernel
 *	 threads.
 */
struct md_intr_info;

struct thread {
    TAILQ_ENTRY(thread) td_threadq;
    TAILQ_ENTRY(thread) td_allq;
    TAILQ_ENTRY(thread) td_sleepq;
    lwkt_port	td_msgport;	/* built-in message port for replies */
    struct lwp	*td_lwp;	/* (optional) associated lwp */
    struct proc	*td_proc;	/* (optional) associated process */
    struct pcb	*td_pcb;	/* points to pcb and top of kstack */
    struct globaldata *td_gd;	/* associated with this cpu */
    const char	*td_wmesg;	/* string name for blockage */
    const volatile void	*td_wchan;	/* waiting on channel */
    int		td_pri;		/* 0-31, 31=highest priority (note 1) */
    int		td_critcount;	/* critical section priority */
    u_int	td_flags;	/* TDF flags */
    int		td_wdomain;	/* domain for wchan address (typ 0) */
    void	(*td_preemptable)(struct thread *td, int critcount);
    void	(*td_release)(struct thread *td);
    char	*td_kstack;	/* kernel stack */
    int		td_kstack_size;	/* size of kernel stack */
    char	*td_sp;		/* kernel stack pointer for LWKT restore */
    thread_t	(*td_switch)(struct thread *ntd);
    __uint64_t	td_uticks;	/* Statclock hits in user mode (uS) */
    __uint64_t	td_sticks;      /* Statclock hits in system mode (uS) */
    __uint64_t	td_iticks;	/* Statclock hits processing intr (uS) */
    int		td_locks;	/* lockmgr lock debugging */
    struct plimit *td_limit;	/* synchronized from proc->p_limit */
    int		td_refs;	/* hold position in gd_tdallq / hold free */
    int		td_nest_count;	/* prevent splz nesting */
    u_int	td_contended;	/* token contention count */
    u_int	td_mpflags;	/* flags can be set by foreign cpus */
    int		td_cscount;	/* cpu synchronization master */
    int		td_wakefromcpu;	/* who woke me up? */
    int		td_upri;	/* user priority (sub-priority under td_pri) */
    int		td_type;	/* thread type, TD_TYPE_ */
    int		td_tracker;	/* misc use (base value 0), recursion count */
    int		td_fdcache_lru;
    int		td_unused03[3];	/* for future fields */
    struct iosched_data td_iosdata;	/* Dynamic I/O scheduling data */
    struct timeval td_start;	/* start time for a thread/process */
    char	td_comm[MAXCOMLEN+1]; /* typ 16+1 bytes */
    struct thread *td_preempted; /* we preempted this thread */
    struct ucred *td_ucred;	/* synchronized from proc->p_ucred */
    mcontext_t	*td_kfpuctx;	/* kernel_fpu_begin()/kernel_fpu_end() */
    lwkt_tokref_t td_toks_have;		/* tokens we own */
    lwkt_tokref_t td_toks_stop;		/* tokens we want */
    struct lwkt_tokref td_toks_array[LWKT_MAXTOKENS];
    int		td_fairq_load;		/* fairq */
    int		td_fairq_count;		/* fairq */
    struct globaldata *td_migrate_gd;	/* target gd for thread migration */
    struct fdcache    td_fdcache[NFDCACHE];

    /*
     * Linux and FreeBSD compat fields
     */
    void	*td_linux_task;		/* drm/linux support */
    struct sleepqueue_wchan *td_sqwc;	/* freebsd sleepq*() API */
    sbintime_t	td_sqtimo;		/* freebsd sleepq*() API */
    int		td_sqqueue;		/* freebsd sleepq*() API */

    /*
     * Debugging
     */
#ifdef DEBUG_CRIT_SECTIONS
#define CRIT_DEBUG_ARRAY_SIZE   32
#define CRIT_DEBUG_ARRAY_MASK   (CRIT_DEBUG_ARRAY_SIZE - 1)
    const char	*td_crit_debug_array[CRIT_DEBUG_ARRAY_SIZE];
    int		td_crit_debug_index;
    int		td_in_crit_report;
#endif

    /*
     * machine-specific
     */
    struct md_thread td_mach;

    /*
     * Debugging
     */
#ifdef DEBUG_LOCKS
#define SPINLOCK_DEBUG_ARRAY_SIZE	32
   int 	td_spinlock_stack_id[SPINLOCK_DEBUG_ARRAY_SIZE];
   struct spinlock *td_spinlock_stack[SPINLOCK_DEBUG_ARRAY_SIZE];
   void 	*td_spinlock_caller_pc[SPINLOCK_DEBUG_ARRAY_SIZE];

    /*
     * Track lockmgr locks held; lk->lk_filename:lk->lk_lineno is the holder
     */
#define LOCKMGR_DEBUG_ARRAY_SIZE	8
    int		td_lockmgr_stack_id[LOCKMGR_DEBUG_ARRAY_SIZE];
    struct lock	*td_lockmgr_stack[LOCKMGR_DEBUG_ARRAY_SIZE];
#endif
};

#define td_toks_base		td_toks_array[0]
#define td_toks_end		td_toks_array[LWKT_MAXTOKENS]

#define TD_TOKS_HELD(td)	((td)->td_toks_stop != &(td)->td_toks_base)
#define TD_TOKS_NOT_HELD(td)	((td)->td_toks_stop == &(td)->td_toks_base)

/*
 * Thread flags.  Note that TDF_RUNNING is cleared on the old thread after
 * we switch to the new one, which is necessary because LWKTs don't need
 * to hold the BGL.  This flag is used by the exit code and the managed
 * thread migration code.  Note in addition that preemption will cause
 * TDF_RUNNING to be cleared temporarily, so any code checking TDF_RUNNING
 * must also check TDF_PREEMPT_LOCK.
 *
 * LWKT threads stay on their (per-cpu) run queue while running, not to
 * be confused with user processes which are removed from the user scheduling
 * run queue while actually running.
 *
 * td_threadq can represent the thread on one of three queues... the LWKT
 * run queue, a tsleep queue, or an lwkt blocking queue.  The LWKT subsystem
 * does not allow a thread to be scheduled if it already resides on some
 * queue.
 */
#define TDF_RUNNING		0x00000001	/* thread still active */
#define TDF_RUNQ		0x00000002	/* on an LWKT run queue */
#define TDF_PREEMPT_LOCK	0x00000004	/* I have been preempted */
#define TDF_PREEMPT_DONE	0x00000008	/* ac preemption complete */
#define TDF_NOSTART		0x00000010	/* do not schedule on create */
#define TDF_MIGRATING		0x00000020	/* thread is being migrated */
#define TDF_SINTR		0x00000040	/* interruptability for 'ps' */
#define TDF_TSLEEPQ		0x00000080	/* on a tsleep wait queue */

#define TDF_SYSTHREAD		0x00000100	/* reserve memory may be used */
#define TDF_ALLOCATED_THREAD	0x00000200	/* objcache allocated thread */
#define TDF_ALLOCATED_STACK	0x00000400	/* objcache allocated stack */
#define TDF_FPU_HEUR		0x00000800	/* active restore on switch */
#define TDF_DEADLKTREAT		0x00001000	/* special lockmgr treatment */
#define TDF_MARKER		0x00002000	/* tdallq list scan marker */
#define TDF_TIMEOUT_RUNNING	0x00004000	/* tsleep timeout race */
#define TDF_TIMEOUT		0x00008000	/* tsleep timeout */
#define TDF_INTTHREAD		0x00010000	/* interrupt thread */
#define TDF_TSLEEP_DESCHEDULED	0x00020000	/* tsleep core deschedule */
#define TDF_BLOCKED		0x00040000	/* Thread is blocked */
#define TDF_PANICWARN		0x00080000	/* panic warning in switch */
#define TDF_BLOCKQ		0x00100000	/* on block queue */
#define TDF_FORCE_SPINPORT	0x00200000
#define TDF_EXITING		0x00400000	/* thread exiting */
#define TDF_USINGFP		0x00800000	/* thread using fp coproc */
#define TDF_KERNELFP		0x01000000	/* kernel using fp coproc */
#define TDF_DELAYED_WAKEUP	0x02000000
#define TDF_FIXEDCPU		0x04000000	/* running cpu is fixed */
#define TDF_USERMODE		0x08000000	/* in or entering user mode */
#define TDF_NOFAULT		0x10000000	/* force onfault on fault */
#define TDF_CLKTHREAD		0x20000000	/* detect INTTHREAD clock */

#define TDF_MP_STOPREQ		0x00000001	/* suspend_kproc */
#define TDF_MP_WAKEREQ		0x00000002	/* resume_kproc */
#define TDF_MP_EXITWAIT		0x00000004	/* reaper, see lwp_wait() */
#define TDF_MP_EXITSIG		0x00000008	/* reaper, see lwp_wait() */
#define TDF_MP_BATCH_DEMARC	0x00000010	/* batch mode handling */
#define TDF_MP_DIDYIELD		0x00000020	/* effects scheduling */

#define TD_TYPE_GENERIC		0		/* generic thread */
#define TD_TYPE_CRYPTO		1		/* crypto thread */
#define TD_TYPE_NETISR		2		/* netisr thread */

/*
 * Thread priorities.  Typically only one thread from any given
 * user process scheduling queue is on the LWKT run queue at a time.
 * Remember that there is one LWKT run queue per cpu.
 *
 * Critical sections are handled by bumping td_pri above TDPRI_MAX, which
 * causes interrupts to be masked as they occur.  When this occurs a
 * rollup flag will be set in mycpu->gd_reqflags.
 */
#define TDPRI_IDLE_THREAD	0	/* the idle thread */
#define TDPRI_IDLE_WORK		1	/* idle work (page zero, etc) */
#define TDPRI_USER_SCHEDULER	2	/* user scheduler helper */
#define TDPRI_USER_IDLE		4	/* user scheduler idle */
#define TDPRI_USER_NORM		6	/* user scheduler normal */
#define TDPRI_USER_REAL		8	/* user scheduler real time */
#define TDPRI_KERN_LPSCHED	9	/* (comparison point only) */
#define TDPRI_KERN_USER		10	/* kernel / block in syscall */
#define TDPRI_KERN_DAEMON	12	/* kernel daemon (pageout, etc) */
#define TDPRI_SOFT_NORM		14	/* kernel / normal */
#define TDPRI_SOFT_TIMER	16	/* kernel / timer */
#define TDPRI_UNUSED19		19
#define TDPRI_INT_SUPPORT	20	/* kernel / high priority support */
#define TDPRI_INT_LOW		27	/* low priority interrupt */
#define TDPRI_INT_MED		28	/* medium priority interrupt */
#define TDPRI_INT_HIGH		29	/* high priority interrupt */
#define TDPRI_MAX		31

#define LWKT_THREAD_STACK	(UPAGES * PAGE_SIZE)

#define IN_CRITICAL_SECT(td)	((td)->td_critcount)

#ifdef _KERNEL

extern void (*linux_task_drop_callback)(struct thread *);
extern void (*linux_proc_drop_callback)(struct proc *);

/*
 * Global tokens
 */
extern struct lwkt_token mp_token;
extern struct lwkt_token pmap_token;
extern struct lwkt_token dev_token;
extern struct lwkt_token vm_token;
extern struct lwkt_token vmspace_token;
extern struct lwkt_token kvm_token;
extern struct lwkt_token sigio_token;
extern struct lwkt_token tty_token;
extern struct lwkt_token vnode_token;
extern struct lwkt_token revoke_token;
extern struct lwkt_token kbd_token;
extern struct lwkt_token vga_token;

/*
 * Procedures
 */
struct thread *lwkt_alloc_thread(struct thread *, int, int, int);
void lwkt_init_thread(struct thread *, void *, int, int, struct globaldata *);
void lwkt_set_interrupt_support_thread(void);
void lwkt_set_comm(thread_t, const char *, ...) __printflike(2, 3);
void lwkt_free_thread(struct thread *);
void lwkt_gdinit(struct globaldata *);
void lwkt_switch(void);
void lwkt_switch_return(struct thread *);
void lwkt_preempt(thread_t, int);
void lwkt_schedule(thread_t);
void lwkt_schedule_noresched(thread_t);
void lwkt_schedule_self(thread_t);
void lwkt_deschedule(thread_t);
void lwkt_deschedule_self(thread_t);
void lwkt_yield(void);
void lwkt_yield_quick(void);
void lwkt_user_yield(void);
void lwkt_hold(thread_t);
void lwkt_rele(thread_t);
void lwkt_passive_release(thread_t);
void lwkt_maybe_splz(thread_t);

void lwkt_gettoken(lwkt_token_t);
void lwkt_gettoken_shared(lwkt_token_t);
int  lwkt_trytoken(lwkt_token_t);
void lwkt_reltoken(lwkt_token_t);
int  lwkt_cnttoken(lwkt_token_t, thread_t);
int  lwkt_getalltokens(thread_t, int);
void lwkt_relalltokens(thread_t);
void lwkt_token_init(lwkt_token_t, const char *);
void lwkt_token_uninit(lwkt_token_t);

void lwkt_token_pool_init(void);
lwkt_token_t lwkt_token_pool_lookup(void *);
lwkt_token_t lwkt_getpooltoken(void *);
void lwkt_relpooltoken(void *);

void lwkt_token_swap(void);

void lwkt_setpri(thread_t, int);
void lwkt_setpri_initial(thread_t, int);
void lwkt_setpri_self(int);
void lwkt_schedulerclock(thread_t td);
void lwkt_setcpu_self(struct globaldata *);
void lwkt_migratecpu(int);

void lwkt_giveaway(struct thread *);
void lwkt_acquire(struct thread *);
int  lwkt_send_ipiq3(struct globaldata *, ipifunc3_t, void *, int);
int  lwkt_send_ipiq3_passive(struct globaldata *, ipifunc3_t, void *, int);
int  lwkt_send_ipiq3_bycpu(int, ipifunc3_t, void *, int);
int  lwkt_send_ipiq3_mask(cpumask_t, ipifunc3_t, void *, int);
void lwkt_wait_ipiq(struct globaldata *, int);
void lwkt_process_ipiq(void);
void lwkt_process_ipiq_frame(struct intrframe *);
void lwkt_smp_stopped(void);
void lwkt_synchronize_ipiqs(const char *);

/* lwkt_cpusync_init() - inline function in sys/thread2.h */
void lwkt_cpusync_simple(cpumask_t, cpusync_func_t, void *);
void lwkt_cpusync_interlock(lwkt_cpusync_t);
void lwkt_cpusync_deinterlock(lwkt_cpusync_t);
void lwkt_cpusync_quick(lwkt_cpusync_t);

void crit_panic(void) __dead2;
struct lwp *lwkt_preempted_proc(void);

int  lwkt_create(void (*)(void *), void *, struct thread **, struct thread *,
	int, int, const char *, ...) __printflike(7, 8);
void lwkt_exit(void) __dead2;
void lwkt_remove_tdallq(struct thread *);

#endif /* _KERNEL */

#endif /* !_SYS_THREAD_H_ */

