/*
|
* SPDX-License-Identifier: GPL-2.0
|
*
|
* Copyright (C) 2016 Philippe Gerum <rpm@xenomai.org>.
|
*/
|
#include <linux/linkage.h>
|
#include <linux/preempt.h>
|
#include <linux/spinlock.h>
|
#include <linux/interrupt.h>
|
#include <linux/irq_pipeline.h>
|
#include <linux/kconfig.h>
|
|
/*
|
* A hybrid spinlock behaves in different ways depending on the
|
* current interrupt stage on entry.
|
*
|
* Such spinlock always leaves hard IRQs disabled once locked. In
|
* addition, it stalls the in-band stage when protecting a critical
|
* section there, disabling preemption like regular spinlocks do as
|
* well. This combination preserves the regular locking logic when
|
* called from the in-band stage, while fully disabling preemption by
|
* other interrupt stages.
|
*
|
* When taken from the pipeline entry context, a hybrid lock behaves
|
* like a hard spinlock, assuming that hard IRQs are already disabled.
|
*
|
* The irq descriptor lock (struct irq_desc) is a typical example of
|
* such lock, which properly serializes accesses regardless of the
|
* calling context.
|
*/
|
void __hybrid_spin_lock(struct raw_spinlock *rlock)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
if (running_inband())
|
preempt_disable();
|
|
__flags = hard_local_irq_save();
|
hard_lock_acquire(rlock, 0, _RET_IP_);
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
lock->hwflags = __flags;
|
}
|
EXPORT_SYMBOL(__hybrid_spin_lock);
|
|
void __hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
if (running_inband())
|
preempt_disable();
|
|
__flags = hard_local_irq_save();
|
hard_lock_acquire_nested(rlock, subclass, _RET_IP_);
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
lock->hwflags = __flags;
|
}
|
EXPORT_SYMBOL(__hybrid_spin_lock_nested);
|
|
void __hybrid_spin_unlock(struct raw_spinlock *rlock)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
/* Pick the flags before releasing the lock. */
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
__flags = lock->hwflags;
|
hard_lock_release(rlock, _RET_IP_);
|
hard_local_irq_restore(__flags);
|
|
if (running_inband())
|
preempt_enable();
|
}
|
EXPORT_SYMBOL(__hybrid_spin_unlock);
|
|
void __hybrid_spin_lock_irq(struct raw_spinlock *rlock)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
__flags = hard_local_irq_save();
|
|
if (running_inband()) {
|
stall_inband();
|
trace_hardirqs_off();
|
preempt_disable();
|
}
|
|
hard_lock_acquire(rlock, 0, _RET_IP_);
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
lock->hwflags = __flags;
|
}
|
EXPORT_SYMBOL(__hybrid_spin_lock_irq);
|
|
void __hybrid_spin_unlock_irq(struct raw_spinlock *rlock)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
/* Pick the flags before releasing the lock. */
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
__flags = lock->hwflags;
|
hard_lock_release(rlock, _RET_IP_);
|
|
if (running_inband()) {
|
trace_hardirqs_on();
|
unstall_inband_nocheck();
|
hard_local_irq_restore(__flags);
|
preempt_enable();
|
return;
|
}
|
|
hard_local_irq_restore(__flags);
|
}
|
EXPORT_SYMBOL(__hybrid_spin_unlock_irq);
|
|
unsigned long __hybrid_spin_lock_irqsave(struct raw_spinlock *rlock)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags, flags;
|
|
__flags = flags = hard_local_irq_save();
|
|
if (running_inband()) {
|
flags = test_and_stall_inband();
|
trace_hardirqs_off();
|
preempt_disable();
|
}
|
|
hard_lock_acquire(rlock, 0, _RET_IP_);
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
lock->hwflags = __flags;
|
|
return flags;
|
}
|
EXPORT_SYMBOL(__hybrid_spin_lock_irqsave);
|
|
void __hybrid_spin_unlock_irqrestore(struct raw_spinlock *rlock,
|
unsigned long flags)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
/* Pick the flags before releasing the lock. */
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
__flags = lock->hwflags;
|
hard_lock_release(rlock, _RET_IP_);
|
|
if (running_inband()) {
|
if (!flags) {
|
trace_hardirqs_on();
|
unstall_inband_nocheck();
|
}
|
hard_local_irq_restore(__flags);
|
preempt_enable();
|
return;
|
}
|
|
hard_local_irq_restore(__flags);
|
}
|
EXPORT_SYMBOL(__hybrid_spin_unlock_irqrestore);
|
|
int __hybrid_spin_trylock(struct raw_spinlock *rlock)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
|
if (running_inband())
|
preempt_disable();
|
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
__flags = hard_local_irq_save();
|
|
hard_spin_trylock_prepare(rlock);
|
if (do_raw_spin_trylock(rlock)) {
|
lock->hwflags = __flags;
|
hard_trylock_acquire(rlock, 1, _RET_IP_);
|
return 1;
|
}
|
|
hard_spin_trylock_fail(rlock);
|
hard_local_irq_restore(__flags);
|
|
if (running_inband())
|
preempt_enable();
|
|
return 0;
|
}
|
EXPORT_SYMBOL(__hybrid_spin_trylock);
|
|
int __hybrid_spin_trylock_irqsave(struct raw_spinlock *rlock,
|
unsigned long *flags)
|
{
|
struct hybrid_spinlock *lock;
|
unsigned long __flags;
|
bool inband;
|
|
inband = running_inband();
|
|
__flags = *flags = hard_local_irq_save();
|
|
lock = container_of(rlock, struct hybrid_spinlock, rlock);
|
if (inband) {
|
*flags = test_and_stall_inband();
|
trace_hardirqs_off();
|
preempt_disable();
|
}
|
|
hard_spin_trylock_prepare(rlock);
|
if (do_raw_spin_trylock(rlock)) {
|
hard_trylock_acquire(rlock, 1, _RET_IP_);
|
lock->hwflags = __flags;
|
return 1;
|
}
|
|
hard_spin_trylock_fail(rlock);
|
|
if (inband && !*flags) {
|
trace_hardirqs_on();
|
unstall_inband_nocheck();
|
}
|
|
hard_local_irq_restore(__flags);
|
|
if (inband)
|
preempt_enable();
|
|
return 0;
|
}
|
EXPORT_SYMBOL(__hybrid_spin_trylock_irqsave);
|
