/*
 * SPDX-License-Identifier: GPL-2.0
 *
 * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
 */
#ifndef __LINUX_SPINLOCK_PIPELINE_H
#define __LINUX_SPINLOCK_PIPELINE_H

#ifndef __LINUX_SPINLOCK_H
# error "Please don't include this file directly. Use spinlock.h."
#endif

#include <dovetail/spinlock.h>

#define hard_spin_lock_irqsave(__rlock, __flags)		\
	do {							\
		(__flags) = __hard_spin_lock_irqsave(__rlock);	\
	} while (0)

#define hard_spin_trylock_irqsave(__rlock, __flags)			\
	({								\
		int __locked;						\
		(__flags) = __hard_spin_trylock_irqsave(__rlock, &__locked); \
		__locked;						\
	})

#define hybrid_spin_lock_init(__rlock)	hard_spin_lock_init(__rlock)

/*
 * CAUTION: We don't want the hand-coded irq-enable of
 * do_raw_spin_lock_flags(), hard locked sections assume that
 * interrupts are not re-enabled during lock-acquire.
 */
#define hard_lock_acquire(__rlock, __try, __ip)				\
	do {								\
		hard_spin_lock_prepare(__rlock);			\
		if (irq_pipeline_debug_locking()) {			\
			spin_acquire(&(__rlock)->dep_map, 0, __try, __ip); \
			LOCK_CONTENDED(__rlock, do_raw_spin_trylock, do_raw_spin_lock); \
		} else {						\
			do_raw_spin_lock(__rlock);			\
		}							\
	} while (0)

#define hard_lock_acquire_nested(__rlock, __subclass, __ip)		\
	do {								\
		hard_spin_lock_prepare(__rlock);			\
		if (irq_pipeline_debug_locking()) {			\
			spin_acquire(&(__rlock)->dep_map, __subclass, 0, __ip); \
			LOCK_CONTENDED(__rlock, do_raw_spin_trylock, do_raw_spin_lock); \
		} else {						\
			do_raw_spin_lock(__rlock);			\
		}							\
	} while (0)

#define hard_trylock_acquire(__rlock, __try, __ip)			\
	do {								\
		if (irq_pipeline_debug_locking())			\
			spin_acquire(&(__rlock)->dep_map, 0, __try, __ip); \
	} while (0)

#define hard_lock_release(__rlock, __ip)				\
	do {								\
		if (irq_pipeline_debug_locking())			\
			spin_release(&(__rlock)->dep_map, __ip);	\
		do_raw_spin_unlock(__rlock);				\
		hard_spin_unlock_finish(__rlock);			\
	} while (0)

#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)

#ifdef CONFIG_DEBUG_SPINLOCK
/*
 * Hard spinlocks are not checked for invalid wait context in-band
 * wise (LD_WAIT_INV). We could be smarter with handling a specific
 * wait type for them, so that we could detect hard_spin_lock ->
 * {raw_}spin_lock for instance, but we already have
 * check_inband_stage() calls all over the place in the latter API, so
 * that kind of misusage would be detected regardless.
 */
#define hard_spin_lock_init(__lock)				\
	do {							\
		static struct lock_class_key __key;		\
		__raw_spin_lock_init((raw_spinlock_t *)__lock, #__lock, &__key, LD_WAIT_INV); \
	} while (0)
#else
#define hard_spin_lock_init(__rlock)				\
	do { *(__rlock) = __RAW_SPIN_LOCK_UNLOCKED(__rlock); } while (0)
#endif

/*
 * XXX: no preempt_enable/disable when hard locking.
 */

static inline
void hard_spin_lock(struct raw_spinlock *rlock)
{
	hard_lock_acquire(rlock, 0, _THIS_IP_);
}

#ifdef CONFIG_DEBUG_LOCK_ALLOC
static inline
void hard_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
{
	hard_lock_acquire_nested(rlock, subclass, _THIS_IP_);
}
#else
static inline
void hard_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
{
	hard_spin_lock(rlock);
}
#endif

static inline
void hard_spin_unlock(struct raw_spinlock *rlock)
{
	hard_lock_release(rlock, _THIS_IP_);
}

static inline
void hard_spin_lock_irq(struct raw_spinlock *rlock)
{
	hard_local_irq_disable();
	hard_lock_acquire(rlock, 0, _THIS_IP_);
}

static inline
void hard_spin_unlock_irq(struct raw_spinlock *rlock)
{
	hard_lock_release(rlock, _THIS_IP_);
	hard_local_irq_enable();
}

static inline
void hard_spin_unlock_irqrestore(struct raw_spinlock *rlock,
				 unsigned long flags)
{
	hard_lock_release(rlock, _THIS_IP_);
	hard_local_irq_restore(flags);
}

static inline
unsigned long __hard_spin_lock_irqsave(struct raw_spinlock *rlock)
{
	unsigned long flags = hard_local_irq_save();

	hard_lock_acquire(rlock, 0, _THIS_IP_);

	return flags;
}

static inline
int hard_spin_trylock(struct raw_spinlock *rlock)
{
	hard_spin_trylock_prepare(rlock);

	if (do_raw_spin_trylock(rlock)) {
		hard_trylock_acquire(rlock, 1, _THIS_IP_);
		return 1;
	}

	hard_spin_trylock_fail(rlock);

	return 0;
}

static inline
unsigned long __hard_spin_trylock_irqsave(struct raw_spinlock *rlock,
					  int *locked)
{
	unsigned long flags = hard_local_irq_save();
	*locked = hard_spin_trylock(rlock);
	return *locked ? flags : ({ hard_local_irq_restore(flags); flags; });
}

static inline
int hard_spin_trylock_irq(struct raw_spinlock *rlock)
{
	hard_local_irq_disable();
	return hard_spin_trylock(rlock) ? : ({ hard_local_irq_enable(); 0; });
}

static inline
int hard_spin_is_locked(struct raw_spinlock *rlock)
{
	return arch_spin_is_locked(&rlock->raw_lock);
}

static inline
int hard_spin_is_contended(struct raw_spinlock *rlock)
{
#ifdef CONFIG_GENERIC_LOCKBREAK
	return rlock->break_lock;
#elif defined(arch_spin_is_contended)
	return arch_spin_is_contended(&rlock->raw_lock);
#else
	return 0;
#endif
}

#else  /* !SMP && !DEBUG_SPINLOCK */

#define hard_spin_lock_init(__rlock)	do { (void)(__rlock); } while (0)
#define hard_spin_lock(__rlock)		__HARD_LOCK(__rlock)
#define hard_spin_lock_nested(__rlock, __subclass)  \
	do { __HARD_LOCK(__rlock); (void)(__subclass); } while (0)
#define hard_spin_unlock(__rlock)	__HARD_UNLOCK(__rlock)
#define hard_spin_lock_irq(__rlock)	__HARD_LOCK_IRQ(__rlock)
#define hard_spin_unlock_irq(__rlock)	__HARD_UNLOCK_IRQ(__rlock)
#define hard_spin_unlock_irqrestore(__rlock, __flags)	\
	__HARD_UNLOCK_IRQRESTORE(__rlock, __flags)
#define __hard_spin_lock_irqsave(__rlock)		\
	({						\
		unsigned long __flags;			\
		__HARD_LOCK_IRQSAVE(__rlock, __flags);	\
		__flags;				\
	})
#define __hard_spin_trylock_irqsave(__rlock, __locked)	\
	({						\
		unsigned long __flags;			\
		__HARD_LOCK_IRQSAVE(__rlock, __flags);	\
		*(__locked) = 1;			\
		__flags;				\
	})
#define hard_spin_trylock(__rlock)	({ __HARD_LOCK(__rlock); 1; })
#define hard_spin_trylock_irq(__rlock)	({ __HARD_LOCK_IRQ(__rlock); 1; })
#define hard_spin_is_locked(__rlock)	((void)(__rlock), 0)
#define hard_spin_is_contended(__rlock)	((void)(__rlock), 0)
#endif	/* !SMP && !DEBUG_SPINLOCK */

/*
 * In the pipeline entry context, the regular preemption and root
 * stall logic do not apply since we may actually have preempted any
 * critical section of the kernel which is protected by regular
 * locking (spin or stall), or we may even have preempted the
 * out-of-band stage. Therefore, we just need to grab the raw spinlock
 * underlying a hybrid spinlock to exclude other CPUs.
 *
 * NOTE: When entering the pipeline, IRQs are already hard disabled.
 */

void __hybrid_spin_lock(struct raw_spinlock *rlock);
void __hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass);

static inline void hybrid_spin_lock(struct raw_spinlock *rlock)
{
	if (in_pipeline())
		hard_lock_acquire(rlock, 0, _THIS_IP_);
	else
		__hybrid_spin_lock(rlock);
}

#ifdef CONFIG_DEBUG_LOCK_ALLOC
static inline
void hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
{
	if (in_pipeline())
		hard_lock_acquire_nested(rlock, subclass, _THIS_IP_);
	else
		__hybrid_spin_lock_nested(rlock, subclass);
}
#else
static inline
void hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
{
	hybrid_spin_lock(rlock);
}
#endif

void __hybrid_spin_unlock(struct raw_spinlock *rlock);

static inline void hybrid_spin_unlock(struct raw_spinlock *rlock)
{
	if (in_pipeline())
		hard_lock_release(rlock, _THIS_IP_);
	else
		__hybrid_spin_unlock(rlock);
}

void __hybrid_spin_lock_irq(struct raw_spinlock *rlock);

static inline void hybrid_spin_lock_irq(struct raw_spinlock *rlock)
{
	if (in_pipeline())
		hard_lock_acquire(rlock, 0, _THIS_IP_);
	else
		__hybrid_spin_lock_irq(rlock);
}

void __hybrid_spin_unlock_irq(struct raw_spinlock *rlock);

static inline void hybrid_spin_unlock_irq(struct raw_spinlock *rlock)
{
	if (in_pipeline())
		hard_lock_release(rlock, _THIS_IP_);
	else
		__hybrid_spin_unlock_irq(rlock);
}

unsigned long __hybrid_spin_lock_irqsave(struct raw_spinlock *rlock);

#define hybrid_spin_lock_irqsave(__rlock, __flags)			\
	do {								\
		if (in_pipeline()) {					\
			hard_lock_acquire(__rlock, 0, _THIS_IP_);	\
			(__flags) = hard_local_save_flags();		\
		} else							\
			(__flags) = __hybrid_spin_lock_irqsave(__rlock); \
	} while (0)

void __hybrid_spin_unlock_irqrestore(struct raw_spinlock *rlock,
				      unsigned long flags);

static inline void hybrid_spin_unlock_irqrestore(struct raw_spinlock *rlock,
						  unsigned long flags)
{

	if (in_pipeline())
		hard_lock_release(rlock, _THIS_IP_);
	else
		__hybrid_spin_unlock_irqrestore(rlock, flags);
}

int __hybrid_spin_trylock(struct raw_spinlock *rlock);

static inline int hybrid_spin_trylock(struct raw_spinlock *rlock)
{
	if (in_pipeline()) {
		hard_spin_trylock_prepare(rlock);
		if (do_raw_spin_trylock(rlock)) {
			hard_trylock_acquire(rlock, 1, _THIS_IP_);
			return 1;
		}
		hard_spin_trylock_fail(rlock);
		return 0;
	}

	return __hybrid_spin_trylock(rlock);
}

int __hybrid_spin_trylock_irqsave(struct raw_spinlock *rlock,
				   unsigned long *flags);

#define hybrid_spin_trylock_irqsave(__rlock, __flags)			\
	({								\
		int __ret = 1;						\
		if (in_pipeline()) {					\
			hard_spin_trylock_prepare(__rlock);		\
			if (do_raw_spin_trylock(__rlock)) {		\
				hard_trylock_acquire(__rlock, 1, _THIS_IP_); \
				(__flags) = hard_local_save_flags();	\
			} else {					\
				hard_spin_trylock_fail(__rlock);	\
				__ret = 0;				\
			}						\
		} else {						\
			__ret = __hybrid_spin_trylock_irqsave(__rlock, &(__flags)); \
		}							\
		__ret;							\
	})

static inline int hybrid_spin_trylock_irq(struct raw_spinlock *rlock)
{
	unsigned long flags;
	return hybrid_spin_trylock_irqsave(rlock, flags);
}

static inline
int hybrid_spin_is_locked(struct raw_spinlock *rlock)
{
	return hard_spin_is_locked(rlock);
}

static inline
int hybrid_spin_is_contended(struct raw_spinlock *rlock)
{
	return hard_spin_is_contended(rlock);
}

#ifdef CONFIG_DEBUG_IRQ_PIPELINE
void check_spinlock_context(void);
#else
static inline void check_spinlock_context(void) { }
#endif

#endif /* __LINUX_SPINLOCK_PIPELINE_H */