/*
 * SPDX-License-Identifier: GPL-2.0
 *
 * Copyright (C) 2017 Philippe Gerum  <rpm@xenomai.org>.
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/printk.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/err.h>
#include <linux/cpumask.h>
#include <linux/clockchips.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irq_pipeline.h>
#include <linux/stop_machine.h>
#include <linux/slab.h>
#include "tick-internal.h"

static unsigned int proxy_tick_irq;

static DEFINE_MUTEX(proxy_mutex);

static DEFINE_PER_CPU(struct clock_proxy_device, proxy_tick_device);

static inline struct clock_event_device *
get_real_tick_device(struct clock_event_device *proxy_dev)
{
	return container_of(proxy_dev, struct clock_proxy_device, proxy_device)->real_device;
}

static void proxy_event_handler(struct clock_event_device *real_dev)
{
	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
	struct clock_event_device *proxy_dev = &dev->proxy_device;

	proxy_dev->event_handler(proxy_dev);
}

static int proxy_set_state_oneshot(struct clock_event_device *dev)
{
	struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->set_state_oneshot(real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static int proxy_set_state_periodic(struct clock_event_device *dev)
{
	struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->set_state_periodic(real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static int proxy_set_state_oneshot_stopped(struct clock_event_device *dev)
{
        struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->set_state_oneshot_stopped(real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static int proxy_set_state_shutdown(struct clock_event_device *dev)
{
        struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->set_state_shutdown(real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static void proxy_suspend(struct clock_event_device *dev)
{
        struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;

	flags = hard_local_irq_save();
	real_dev->suspend(real_dev);
	hard_local_irq_restore(flags);
}

static void proxy_resume(struct clock_event_device *dev)
{
        struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;

	flags = hard_local_irq_save();
	real_dev->resume(real_dev);
	hard_local_irq_restore(flags);
}

static int proxy_tick_resume(struct clock_event_device *dev)
{
        struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->tick_resume(real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static void proxy_broadcast(const struct cpumask *mask)
{
	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
        struct clock_event_device *real_dev = dev->real_device;
	unsigned long flags;

	flags = hard_local_irq_save();
	real_dev->broadcast(mask);
	hard_local_irq_restore(flags);
}

static int proxy_set_next_event(unsigned long delay,
				struct clock_event_device *dev)
{
	struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->set_next_event(delay, real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static int proxy_set_next_ktime(ktime_t expires,
				struct clock_event_device *dev)
{
	struct clock_event_device *real_dev = get_real_tick_device(dev);
	unsigned long flags;
	int ret;

	flags = hard_local_irq_save();
	ret = real_dev->set_next_ktime(expires, real_dev);
	hard_local_irq_restore(flags);

	return ret;
}

static irqreturn_t proxy_irq_handler(int sirq, void *dev_id)
{
	struct clock_event_device *evt;

	/*
	 * Tricky: we may end up running this in-band IRQ handler
	 * because tick_notify_proxy() was posted either:
	 *
	 * - from the out-of-band stage via ->handle_oob_event() for
	 * emulating an in-band tick.  In this case, the active tick
	 * device for the in-band timing core is the proxy device,
	 * whose event handler is still the same than the real tick
	 * device's.
	 *
	 * - directly by the clock chip driver on the local CPU via
	 * clockevents_handle_event(), for propagating a tick to the
	 * in-band stage nobody from the out-of-band stage is
	 * interested on i.e. no proxy device was registered on the
	 * receiving CPU, which was excluded from @cpumask in the call
	 * to tick_install_proxy(). In this case, the active tick
	 * device for the in-band timing core is a real clock event
	 * device.
	 *
	 * In both cases, we are running on the in-band stage, and we
	 * should fire the event handler of the currently active tick
	 * device for the in-band timing core.
	 */
	evt = raw_cpu_ptr(&tick_cpu_device)->evtdev;
	evt->event_handler(evt);

	return IRQ_HANDLED;
}

#define interpose_proxy_handler(__proxy, __real, __h)		\
	do {							\
		if ((__real)->__h)				\
			(__proxy)->__h = proxy_ ## __h;		\
	} while (0)

/*
 * Setup a proxy which is about to override the tick device on the
 * current CPU. Called with clockevents_lock held and irqs off so that
 * the tick device does not change under our feet.
 */
int tick_setup_proxy(struct clock_proxy_device *dev)
{
	struct clock_event_device *proxy_dev, *real_dev;

	real_dev = raw_cpu_ptr(&tick_cpu_device)->evtdev;
	if ((real_dev->features &
			(CLOCK_EVT_FEAT_PIPELINE|CLOCK_EVT_FEAT_ONESHOT))
		!= (CLOCK_EVT_FEAT_PIPELINE|CLOCK_EVT_FEAT_ONESHOT)) {
		WARN(1, "cannot use clockevent device %s in proxy mode!",
			real_dev->name);
		return -ENODEV;
	}

 	/*
 	 * The assumption is that neither us nor clockevents_register_proxy()
	 * can fail afterwards, so this is ok to advertise the new proxy as
	 * built by setting dev->real_device early.
 	 */
	dev->real_device = real_dev;
	dev->__original_handler = real_dev->event_handler;

	/*
	 * Inherit the feature bits since the proxy device has the
	 * same capabilities than the real one we are overriding
	 * (including CLOCK_EVT_FEAT_C3STOP if present).
	 */
	proxy_dev = &dev->proxy_device;
	memset(proxy_dev, 0, sizeof(*proxy_dev));
	proxy_dev->features = real_dev->features |
		CLOCK_EVT_FEAT_PERCPU | CLOCK_EVT_FEAT_PROXY;
	proxy_dev->name = "proxy";
	proxy_dev->irq = real_dev->irq;
	proxy_dev->bound_on = -1;
	proxy_dev->cpumask = cpumask_of(smp_processor_id());
	proxy_dev->rating = real_dev->rating + 1;
	proxy_dev->mult = real_dev->mult;
	proxy_dev->shift = real_dev->shift;
	proxy_dev->max_delta_ticks = real_dev->max_delta_ticks;
	proxy_dev->min_delta_ticks = real_dev->min_delta_ticks;
	proxy_dev->max_delta_ns = real_dev->max_delta_ns;
	proxy_dev->min_delta_ns = real_dev->min_delta_ns;
	/*
	 * Interpose default handlers which are safe wrt preemption by
	 * the out-of-band stage.
	 */
	interpose_proxy_handler(proxy_dev, real_dev, set_state_oneshot);
	interpose_proxy_handler(proxy_dev, real_dev, set_state_oneshot_stopped);
	interpose_proxy_handler(proxy_dev, real_dev, set_state_periodic);
	interpose_proxy_handler(proxy_dev, real_dev, set_state_shutdown);
	interpose_proxy_handler(proxy_dev, real_dev, suspend);
	interpose_proxy_handler(proxy_dev, real_dev, resume);
	interpose_proxy_handler(proxy_dev, real_dev, tick_resume);
	interpose_proxy_handler(proxy_dev, real_dev, broadcast);
	interpose_proxy_handler(proxy_dev, real_dev, set_next_event);
	interpose_proxy_handler(proxy_dev, real_dev, set_next_ktime);

	dev->__setup_handler(dev);

	return 0;
}

static int enable_oob_timer(void *arg) /* hard_irqs_disabled() */
{
	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
	struct clock_event_device *real_dev;

	/*
	 * Install the out-of-band handler on this CPU's real clock
	 * device, then turn on out-of-band mode for the associated
	 * IRQ (duplicates are silently ignored if the IRQ is common
	 * to multiple CPUs).
	 */
	real_dev = dev->real_device;
	real_dev->event_handler = dev->handle_oob_event;
	real_dev->features |= CLOCK_EVT_FEAT_OOB;
	barrier();

	/*
	 * irq_switch_oob() grabs the IRQ descriptor lock which is
	 * hybrid, so that is fine to invoke this routine with hard
	 * IRQs off.
	 */
	irq_switch_oob(real_dev->irq, true);

	return 0;
}

struct proxy_install_arg {
	void (*setup_proxy)(struct clock_proxy_device *dev);
	int result;
};

static void register_proxy_device(void *arg) /* irqs_disabled() */
{
	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
	struct proxy_install_arg *req = arg;
	int ret;

	dev->__setup_handler = req->setup_proxy;
	ret = clockevents_register_proxy(dev);
	if (ret) {
		if (!req->result)
			req->result = ret;
	} else {
		dev->real_device->event_handler = proxy_event_handler;
	}
}

int tick_install_proxy(void (*setup_proxy)(struct clock_proxy_device *dev),
		const struct cpumask *cpumask)
{
	struct proxy_install_arg arg;
	int ret, sirq;

	mutex_lock(&proxy_mutex);

	ret = -EAGAIN;
	if (proxy_tick_irq)
		goto out;

	sirq = irq_create_direct_mapping(synthetic_irq_domain);
	if (WARN_ON(sirq == 0))
		goto out;

	ret = __request_percpu_irq(sirq, proxy_irq_handler,
				   IRQF_NO_THREAD, /* no IRQF_TIMER here. */
				   "proxy tick",
				   &proxy_tick_device);
	if (WARN_ON(ret)) {
		irq_dispose_mapping(sirq);
		goto out;
	}

	proxy_tick_irq = sirq;
	barrier();

	/*
	 * Install a proxy tick device on each CPU. As the proxy
	 * device is picked, the previous (real) tick device is
	 * switched to reserved state by the clockevent core.
	 * Immediately after, the proxy device starts controlling the
	 * real device under the hood to carry out the timing requests
	 * it receives.
	 *
	 * For a short period of time, after the proxy device is
	 * installed, and until the real device IRQ is switched to
	 * out-of-band mode, the flow is as follows:
	 *
	 *    [inband timing request]
	 *        proxy_dev->set_next_event(proxy_dev)
	 *            oob_program_event(proxy_dev)
	 *                real_dev->set_next_event(real_dev)
	 *        ...
	 *        <tick event>
	 *        original_timer_handler() [in-band stage]
	 *            clockevents_handle_event(real_dev)
	 *               proxy_event_handler(real_dev)
	 *                  inband_event_handler(proxy_dev)
	 *
	 * Eventually, we substitute the original (in-band) clock
	 * event handler with the out-of-band handler for the real
	 * clock event device, then turn on out-of-band mode for the
	 * timer IRQ associated to the latter. These two steps are
	 * performed over a stop_machine() context, so that no tick
	 * can race with this code while we swap handlers.
	 *
	 * Once the hand over is complete, the flow is as follows:
	 *
	 *    [inband timing request]
	 *        proxy_dev->set_next_event(proxy_dev)
	 *            oob_program_event(proxy_dev)
	 *                real_dev->set_next_event(real_dev)
	 *        ...
	 *        <tick event>
	 *        inband_event_handler() [out-of-band stage]
	 *            clockevents_handle_event(real_dev)
	 *                handle_oob_event(proxy_dev)
	 *                    ...(inband tick emulation)...
	 *                         tick_notify_proxy()
	 *        ...
	 *        proxy_irq_handler(proxy_dev) [in-band stage]
	 *            clockevents_handle_event(proxy_dev)
	 *                inband_event_handler(proxy_dev)
	 */
	arg.setup_proxy = setup_proxy;
	arg.result = 0;
	on_each_cpu_mask(cpumask, register_proxy_device, &arg, true);
	if (arg.result) {
		tick_uninstall_proxy(cpumask);
		return arg.result;
	}

	/*
	 * Start ticking from the out-of-band interrupt stage upon
	 * receipt of out-of-band timer events.
	 */
	stop_machine(enable_oob_timer, NULL, cpumask);
out:
	mutex_unlock(&proxy_mutex);

	return ret;
}
EXPORT_SYMBOL_GPL(tick_install_proxy);

static int disable_oob_timer(void *arg) /* hard_irqs_disabled() */
{
	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
	struct clock_event_device *real_dev;

	dev = raw_cpu_ptr(&proxy_tick_device);
	real_dev = dev->real_device;
	real_dev->event_handler = dev->__original_handler;
	real_dev->features &= ~CLOCK_EVT_FEAT_OOB;
	barrier();

	irq_switch_oob(real_dev->irq, false);

	return 0;
}

static void unregister_proxy_device(void *arg) /* irqs_disabled() */
{
	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);

	if (dev->real_device) {
		clockevents_unregister_proxy(dev);
		dev->real_device = NULL;
	}
}

void tick_uninstall_proxy(const struct cpumask *cpumask)
{
	/*
	 * Undo all we did in tick_install_proxy(), handing over
	 * control of the tick device back to the inband code.
	 */
	mutex_lock(&proxy_mutex);
	stop_machine(disable_oob_timer, NULL, cpu_online_mask);
	on_each_cpu_mask(cpumask, unregister_proxy_device, NULL, true);
	free_percpu_irq(proxy_tick_irq, &proxy_tick_device);
	irq_dispose_mapping(proxy_tick_irq);
	proxy_tick_irq = 0;
	mutex_unlock(&proxy_mutex);
}
EXPORT_SYMBOL_GPL(tick_uninstall_proxy);

void tick_notify_proxy(void)
{
	/*
	 * Schedule a tick on the proxy device to occur from the
	 * in-band stage, which will trigger proxy_irq_handler() at
	 * some point (i.e. when the in-band stage is back in control
	 * and not stalled). Note that we might be called from the
	 * in-band stage in some cases (see proxy_irq_handler()).
	 */
	irq_post_inband(proxy_tick_irq);
}
EXPORT_SYMBOL_GPL(tick_notify_proxy);