/*
 * SPDX-License-Identifier: GPL-2.0
 *
 * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
 */
#include <linux/timekeeper_internal.h>
#include <linux/sched/signal.h>
#include <linux/irq_pipeline.h>
#include <linux/dovetail.h>
#include <asm/unistd.h>
#include <asm/syscall.h>
#include <uapi/asm-generic/dovetail.h>

static bool dovetail_enabled;

void __weak arch_inband_task_init(struct task_struct *p)
{
}

void inband_task_init(struct task_struct *p)
{
	struct thread_info *ti = task_thread_info(p);

	clear_ti_local_flags(ti, _TLF_DOVETAIL|_TLF_OOB|_TLF_OFFSTAGE);
	arch_inband_task_init(p);
}

void dovetail_init_altsched(struct dovetail_altsched_context *p)
{
	struct task_struct *tsk = current;
	struct mm_struct *mm = tsk->mm;

	check_inband_stage();
	p->task = tsk;
	p->active_mm = mm;
	p->borrowed_mm = false;

	/*
	 * Make sure the current process will not share any private
	 * page with its child upon fork(), sparing it the random
	 * latency induced by COW. MMF_DOVETAILED is never cleared once
	 * set. We serialize with dup_mmap() which holds the mm write
	 * lock.
	 */
	if (!(tsk->flags & PF_KTHREAD) &&
		!test_bit(MMF_DOVETAILED, &mm->flags)) {
		mmap_write_lock(mm);
		__set_bit(MMF_DOVETAILED, &mm->flags);
		mmap_write_unlock(mm);
	}
}
EXPORT_SYMBOL_GPL(dovetail_init_altsched);

void dovetail_start_altsched(void)
{
	check_inband_stage();
	set_thread_local_flags(_TLF_DOVETAIL);
}
EXPORT_SYMBOL_GPL(dovetail_start_altsched);

void dovetail_stop_altsched(void)
{
	clear_thread_local_flags(_TLF_DOVETAIL);
	clear_thread_flag(TIF_MAYDAY);
}
EXPORT_SYMBOL_GPL(dovetail_stop_altsched);

int __weak handle_oob_syscall(struct pt_regs *regs)
{
	return 0;
}

int __weak handle_pipelined_syscall(struct irq_stage *stage,
				    struct pt_regs *regs)
{
	return 0;	/* i.e. propagate to in-band handler. */
}

void __weak handle_oob_mayday(struct pt_regs *regs)
{
}

static inline
void call_mayday(struct thread_info *ti, struct pt_regs *regs)
{
	clear_ti_thread_flag(ti, TIF_MAYDAY);
	handle_oob_mayday(regs);
}

void dovetail_call_mayday(struct pt_regs *regs)
{
	struct thread_info *ti = current_thread_info();
	unsigned long flags;

	flags = hard_local_irq_save();
	call_mayday(ti, regs);
	hard_local_irq_restore(flags);
}

void inband_retuser_notify(void)
{
	clear_thread_flag(TIF_RETUSER);
	inband_event_notify(INBAND_TASK_RETUSER, current);
	/* CAUTION: we might have switched out-of-band here. */
}

int __pipeline_syscall(struct pt_regs *regs)
{
	struct thread_info *ti = current_thread_info();
	struct irq_stage *caller_stage, *target_stage;
	struct irq_stage_data *p, *this_context;
	unsigned long flags;
	int ret = 0;

	/*
	 * We should definitely not pipeline a syscall through the
	 * slow path with IRQs off.
	 */
	WARN_ON_ONCE(dovetail_debug() && hard_irqs_disabled());

	if (!dovetail_enabled)
		return 0;

	flags = hard_local_irq_save();
	caller_stage = current_irq_stage;
	this_context = current_irq_staged;
	target_stage = &oob_stage;
next:
	p = this_staged(target_stage);
	set_current_irq_staged(p);
	hard_local_irq_restore(flags);
	ret = handle_pipelined_syscall(caller_stage, regs);
	flags = hard_local_irq_save();
	/*
	 * Be careful about stage switching _and_ CPU migration that
	 * might have happened as a result of handing over the syscall
	 * to the out-of-band handler.
	 *
	 * - if a stage migration is detected, fetch the new
	 * per-stage, per-CPU context pointer.
	 *
	 * - if no stage migration happened, switch back to the
	 * initial call stage, on a possibly different CPU though.
	 */
	if (current_irq_stage != target_stage) {
		this_context = current_irq_staged;
	} else {
		p = this_staged(this_context->stage);
		set_current_irq_staged(p);
	}

	if (this_context->stage == &inband_stage) {
		if (target_stage != &inband_stage && ret == 0) {
			target_stage = &inband_stage;
			goto next;
		}
		p = this_inband_staged();
		if (stage_irqs_pending(p))
			sync_current_irq_stage();
	} else {
		if (test_ti_thread_flag(ti, TIF_MAYDAY))
			call_mayday(ti, regs);
	}

	hard_local_irq_restore(flags);

	return ret;
}

static inline bool maybe_oob_syscall(unsigned int nr, struct pt_regs *regs)
{
	/*
	 * Check whether the companion core might be interested in the
	 * syscall call. If the old syscall form is handled, pass the
	 * request to the core if __OOB_SYSCALL_BIT is set in
	 * @nr. Otherwise, only check whether an oob syscall is folded
	 * into a prctl() request.
	 */
	if (IS_ENABLED(CONFIG_DOVETAIL_LEGACY_SYSCALL_RANGE)) {
		if (nr & __OOB_SYSCALL_BIT)
			return true;
	}

	return arch_dovetail_is_syscall(nr) && syscall_get_arg0(regs) & __OOB_SYSCALL_BIT;
}

int pipeline_syscall(unsigned int nr, struct pt_regs *regs)
{
	struct thread_info *ti = current_thread_info();
	unsigned long local_flags = READ_ONCE(ti_local_flags(ti));
	int ret;

	WARN_ON_ONCE(dovetail_debug() && hard_irqs_disabled());

	/*
	 * If the syscall signature belongs to the out-of-band syscall
	 * set and we are running out-of-band, pass the request
	 * directly to the companion core by calling the oob syscall
	 * handler.
	 *
	 * Otherwise, if this is an out-of-band syscall or alternate
	 * scheduling is enabled for the caller, propagate the syscall
	 * through the pipeline stages, so that:
	 *
	 * - the core can manipulate the current execution stage for
	 * handling the request, which includes switching the current
	 * thread back to the in-band context if the syscall is a
	 * native one, or promoting it to the oob stage if handling an
	 * oob syscall requires this.
	 *
	 * - the core can receive the initial oob syscall a thread
	 * might have to emit for enabling dovetailing from the
	 * in-band stage.
	 *
	 * Native syscalls from common (non-dovetailed) threads are
	 * not subject to pipelining, but flow down to the in-band
	 * system call handler directly.
	 *
	 * Sanity check: we bark on returning from a syscall on a
	 * stalled in-band stage, which combined with running with
	 * hard irqs on might cause interrupts to linger in the log
	 * after exiting to user.
	 */

	if ((local_flags & _TLF_OOB) && maybe_oob_syscall(nr, regs)) {
		ret = handle_oob_syscall(regs);
		if (!IS_ENABLED(CONFIG_DOVETAIL_LEGACY_SYSCALL_RANGE))
			WARN_ON_ONCE(dovetail_debug() && !ret);
		local_flags = READ_ONCE(ti_local_flags(ti));
		if (likely(ret)) {
			if (local_flags & _TLF_OOB) {
				if (test_ti_thread_flag(ti, TIF_MAYDAY))
					dovetail_call_mayday(regs);
				return 1; /* don't pass down, no tail work. */
			} else {
				WARN_ON_ONCE(dovetail_debug() && irqs_disabled());
				return -1; /* don't pass down, do tail work. */
			}
		}
	}

	if ((local_flags & _TLF_DOVETAIL) || maybe_oob_syscall(nr, regs)) {
		ret = __pipeline_syscall(regs);
		local_flags = READ_ONCE(ti_local_flags(ti));
		if (local_flags & _TLF_OOB)
			return 1; /* don't pass down, no tail work. */
		if (ret) {
			WARN_ON_ONCE(dovetail_debug() && irqs_disabled());
			return -1; /* don't pass down, do tail work. */
		}
	}

	return 0; /* pass syscall down to the in-band dispatcher. */
}

void __weak handle_oob_trap_entry(unsigned int trapnr, struct pt_regs *regs)
{
}

noinstr void __oob_trap_notify(unsigned int exception,
			       struct pt_regs *regs)
{
	unsigned long flags;

	/*
	 * We send a notification about exceptions raised over a
	 * registered oob stage only. The trap_entry handler expects
	 * hard irqs off on entry. It may demote the current context
	 * to the in-band stage, may return with hard irqs on.
	 */
	if (dovetail_enabled) {
		set_thread_local_flags(_TLF_OOBTRAP);
		flags = hard_local_irq_save();
		instrumentation_begin();
		handle_oob_trap_entry(exception, regs);
		instrumentation_end();
		hard_local_irq_restore(flags);
	}
}

void __weak handle_oob_trap_exit(unsigned int trapnr, struct pt_regs *regs)
{
}

noinstr void __oob_trap_unwind(unsigned int exception, struct pt_regs *regs)
{
	/*
	 * The trap_exit handler runs only if trap_entry was called
	 * for the same trap occurrence. It expects hard irqs off on
	 * entry, may switch the current context back to the oob
	 * stage. Must return with hard irqs off.
	 */
	hard_local_irq_disable();
	clear_thread_local_flags(_TLF_OOBTRAP);
	instrumentation_begin();
	handle_oob_trap_exit(exception, regs);
	instrumentation_end();
}

void __weak handle_inband_event(enum inband_event_type event, void *data)
{
}

void inband_event_notify(enum inband_event_type event, void *data)
{
	check_inband_stage();

	if (dovetail_enabled)
		handle_inband_event(event, data);
}

void __weak resume_oob_task(struct task_struct *p)
{
}

static void finalize_oob_transition(void) /* hard IRQs off */
{
	struct irq_pipeline_data *pd;
	struct irq_stage_data *p;
	struct thread_info *ti;
	struct task_struct *t;

	pd = raw_cpu_ptr(&irq_pipeline);
	t = pd->task_inflight;
	if (t == NULL)
		return;

	/*
	 * @t which is in flight to the oob stage might have received
	 * a signal while waiting in off-stage state to be actually
	 * scheduled out. We can't act upon that signal safely from
	 * here, we simply let the task complete the migration process
	 * to the oob stage. The pending signal will be handled when
	 * the task eventually exits the out-of-band context by the
	 * converse migration.
	 */
	pd->task_inflight = NULL;
	ti = task_thread_info(t);

	/*
	 * The transition handler in the companion core assumes the
	 * oob stage is stalled, fix this up.
	 */
	stall_oob();
	resume_oob_task(t);
	unstall_oob();
	p = this_oob_staged();
	if (stage_irqs_pending(p))
		/* Current stage (in-band) != p->stage (oob). */
		sync_irq_stage(p->stage);
}

void oob_trampoline(void)
{
	unsigned long flags;

	check_inband_stage();
	flags = hard_local_irq_save();
	finalize_oob_transition();
	hard_local_irq_restore(flags);
}

bool inband_switch_tail(void)
{
	bool oob;

	check_hard_irqs_disabled();

	/*
	 * We may run this code either over the inband or oob
	 * contexts. If inband, we may have a thread blocked in
	 * dovetail_leave_inband(), waiting for the companion core to
	 * schedule it back in over the oob context, in which case
	 * finalize_oob_transition() should take care of it. If oob,
	 * the core just switched us back, and we may update the
	 * context markers before returning to context_switch().
	 *
	 * Since the preemption count does not reflect the active
	 * stage yet upon inband -> oob transition, we figure out
	 * which one we are on by testing _TLF_OFFSTAGE. Having this
	 * bit set when running the inband switch tail code means that
	 * we are completing such transition for the current task,
	 * switched in by dovetail_context_switch() over the oob
	 * stage. If so, update the context markers appropriately.
	 */
	oob = test_thread_local_flags(_TLF_OFFSTAGE);
	if (oob) {
		/*
		 * The companion core assumes a stalled stage on exit
		 * from dovetail_leave_inband().
		 */
		stall_oob();
		set_thread_local_flags(_TLF_OOB);
		if (!IS_ENABLED(CONFIG_HAVE_PERCPU_PREEMPT_COUNT)) {
			WARN_ON_ONCE(dovetail_debug() &&
				(preempt_count() & STAGE_MASK));
			preempt_count_add(STAGE_OFFSET);
		}
	} else {
		finalize_oob_transition();
		hard_local_irq_enable();
	}

	return oob;
}

void __weak inband_clock_was_set(void)
{
}

void __weak install_inband_fd(unsigned int fd, struct file *file,
			      struct files_struct *files)
{
}

void __weak uninstall_inband_fd(unsigned int fd, struct file *file,
				struct files_struct *files)
{
}

void __weak replace_inband_fd(unsigned int fd, struct file *file,
			      struct files_struct *files)
{
}

int dovetail_start(void)
{
	check_inband_stage();

	if (dovetail_enabled)
		return -EBUSY;

	if (!oob_stage_present())
		return -EAGAIN;

	dovetail_enabled = true;
	smp_wmb();

	return 0;
}
EXPORT_SYMBOL_GPL(dovetail_start);

void dovetail_stop(void)
{
	check_inband_stage();

	dovetail_enabled = false;
	smp_wmb();
}
EXPORT_SYMBOL_GPL(dovetail_stop);