~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,31 +1,24 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
3	4	* Author: Marc Zyngier <marc.zyngier@arm.com>
4		- *
5		- * This program is free software; you can redistribute it and/or modify
6		- * it under the terms of the GNU General Public License version 2 as
7		- * published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program. If not, see <http://www.gnu.org/licenses/>.
16	5	*/
17	6
18	7	#include <linux/acpi.h>
19	8	#include <linux/acpi_iort.h>
	9	+#include <linux/bitfield.h>
20	10	#include <linux/bitmap.h>
21	11	#include <linux/cpu.h>
	12	+#include <linux/crash_dump.h>
22	13	#include <linux/delay.h>
23	14	#include <linux/dma-iommu.h>
	15	+#include <linux/efi.h>
24	16	#include <linux/interrupt.h>
	17	+#include <linux/iopoll.h>
25	18	#include <linux/irqdomain.h>
26	19	#include <linux/list.h>
27		-#include <linux/list_sort.h>
28	20	#include <linux/log2.h>
	21	+#include <linux/memblock.h>
29	22	#include <linux/mm.h>
30	23	#include <linux/msi.h>
31	24	#include <linux/of.h>
..	..	@@ -51,6 +44,7 @@
51	44	#define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
52	45
53	46	#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
	47	+#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
54	48
55	49	static u32 lpi_id_bits;
56	50
..	..	@@ -63,7 +57,7 @@
63	57	#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
64	58	#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
65	59
66		-#define LPI_PROP_DEFAULT_PRIO 0xa0
	60	+#define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
67	61
68	62	/*
69	63	* Collection structure - just an ID, and a redistributor address to
..	..	@@ -102,6 +96,7 @@
102	96	struct mutex dev_alloc_lock;
103	97	struct list_head entry;
104	98	void __iomem *base;
	99	+ void __iomem *sgir_base;
105	100	phys_addr_t phys_base;
106	101	struct its_cmd_block *cmd_base;
107	102	struct its_cmd_block *cmd_write;
..	..	@@ -109,24 +104,36 @@
109	104	struct its_collection *collections;
110	105	struct fwnode_handle *fwnode_handle;
111	106	u64 (get_msi_base)(struct its_device its_dev);
	107	+ u64 typer;
112	108	u64 cbaser_save;
113	109	u32 ctlr_save;
	110	+ u32 mpidr;
114	111	struct list_head its_device_list;
115	112	u64 flags;
116	113	unsigned long list_nr;
117		- u32 ite_size;
118		- u32 device_ids;
119	114	int numa_node;
120	115	unsigned int msi_domain_flags;
121	116	u32 pre_its_base; /* for Socionext Synquacer */
122		- bool is_v4;
123	117	int vlpi_redist_offset;
124	118	};
	119	+
	120	+#define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
	121	+#define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
	122	+#define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
125	123
126	124	#define ITS_ITT_ALIGN SZ_256
127	125
128	126	/* The maximum number of VPEID bits supported by VLPI commands */
129		-#define ITS_MAX_VPEID_BITS (16)
	127	+#define ITS_MAX_VPEID_BITS \
	128	+ ({ \
	129	+ int nvpeid = 16; \
	130	+ if (gic_rdists->has_rvpeid && \
	131	+ gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
	132	+ nvpeid = 1 + (gic_rdists->gicd_typer2 & \
	133	+ GICD_TYPER2_VID); \
	134	+ \
	135	+ nvpeid; \
	136	+ })
130	137	#define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
131	138
132	139	/* Convert page order to size in bytes */
..	..	@@ -137,7 +144,7 @@
137	144	u16 *col_map;
138	145	irq_hw_number_t lpi_base;
139	146	int nr_lpis;
140		- struct mutex vlpi_lock;
	147	+ raw_spinlock_t vlpi_lock;
141	148	struct its_vm *vm;
142	149	struct its_vlpi_map *vlpi_maps;
143	150	int nr_vlpis;
..	..	@@ -167,6 +174,13 @@
167	174	int next_victim;
168	175	} vpe_proxy;
169	176
	177	+struct cpu_lpi_count {
	178	+ atomic_t managed;
	179	+ atomic_t unmanaged;
	180	+};
	181	+
	182	+static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
	183	+
170	184	static LIST_HEAD(its_nodes);
171	185	static DEFINE_RAW_SPINLOCK(its_lock);
172	186	static struct rdists *gic_rdists;
..	..	@@ -179,8 +193,18 @@
179	193	static DEFINE_IDA(its_vpeid_ida);
180	194
181	195	#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
	196	+#define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
182	197	#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
183	198	#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
	199	+
	200	+/*
	201	+ * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
	202	+ * always have vSGIs mapped.
	203	+ */
	204	+static bool require_its_list_vmovp(struct its_vm vm, struct its_node its)
	205	+{
	206	+ return (gic_rdists->has_rvpeid \|\| vm->vlpi_count[its->list_nr]);
	207	+}
184	208
185	209	static u16 get_its_list(struct its_vm *vm)
186	210	{
..	..	@@ -188,14 +212,20 @@
188	212	unsigned long its_list = 0;
189	213
190	214	list_for_each_entry(its, &its_nodes, entry) {
191		- if (!its->is_v4)
	215	+ if (!is_v4(its))
192	216	continue;
193	217
194		- if (vm->vlpi_count[its->list_nr])
	218	+ if (require_its_list_vmovp(vm, its))
195	219	__set_bit(its->list_nr, &its_list);
196	220	}
197	221
198	222	return (u16)its_list;
	223	+}
	224	+
	225	+static inline u32 its_get_event_id(struct irq_data *d)
	226	+{
	227	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	228	+ return d->hwirq - its_dev->event_map.lpi_base;
199	229	}
200	230
201	231	static struct its_collection dev_event_to_col(struct its_device its_dev,
..	..	@@ -204,6 +234,82 @@
204	234	struct its_node *its = its_dev->its;
205	235
206	236	return its->collections + its_dev->event_map.col_map[event];
	237	+}
	238	+
	239	+static struct its_vlpi_map dev_event_to_vlpi_map(struct its_device its_dev,
	240	+ u32 event)
	241	+{
	242	+ if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
	243	+ return NULL;
	244	+
	245	+ return &its_dev->event_map.vlpi_maps[event];
	246	+}
	247	+
	248	+static struct its_vlpi_map get_vlpi_map(struct irq_data d)
	249	+{
	250	+ if (irqd_is_forwarded_to_vcpu(d)) {
	251	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	252	+ u32 event = its_get_event_id(d);
	253	+
	254	+ return dev_event_to_vlpi_map(its_dev, event);
	255	+ }
	256	+
	257	+ return NULL;
	258	+}
	259	+
	260	+static int vpe_to_cpuid_lock(struct its_vpe vpe, unsigned long flags)
	261	+{
	262	+ raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
	263	+ return vpe->col_idx;
	264	+}
	265	+
	266	+static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
	267	+{
	268	+ raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
	269	+}
	270	+
	271	+static struct irq_chip its_vpe_irq_chip;
	272	+
	273	+static int irq_to_cpuid_lock(struct irq_data d, unsigned long flags)
	274	+{
	275	+ struct its_vpe *vpe = NULL;
	276	+ int cpu;
	277	+
	278	+ if (d->chip == &its_vpe_irq_chip) {
	279	+ vpe = irq_data_get_irq_chip_data(d);
	280	+ } else {
	281	+ struct its_vlpi_map *map = get_vlpi_map(d);
	282	+ if (map)
	283	+ vpe = map->vpe;
	284	+ }
	285	+
	286	+ if (vpe) {
	287	+ cpu = vpe_to_cpuid_lock(vpe, flags);
	288	+ } else {
	289	+ /* Physical LPIs are already locked via the irq_desc lock */
	290	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	291	+ cpu = its_dev->event_map.col_map[its_get_event_id(d)];
	292	+ /* Keep GCC quiet... */
	293	+ *flags = 0;
	294	+ }
	295	+
	296	+ return cpu;
	297	+}
	298	+
	299	+static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
	300	+{
	301	+ struct its_vpe *vpe = NULL;
	302	+
	303	+ if (d->chip == &its_vpe_irq_chip) {
	304	+ vpe = irq_data_get_irq_chip_data(d);
	305	+ } else {
	306	+ struct its_vlpi_map *map = get_vlpi_map(d);
	307	+ if (map)
	308	+ vpe = map->vpe;
	309	+ }
	310	+
	311	+ if (vpe)
	312	+ vpe_to_cpuid_unlock(vpe, flags);
207	313	}
208	314
209	315	static struct its_collection valid_col(struct its_collection col)
..	..	@@ -305,6 +411,19 @@
305	411	u16 seq_num;
306	412	u16 its_list;
307	413	} its_vmovp_cmd;
	414	+
	415	+ struct {
	416	+ struct its_vpe *vpe;
	417	+ } its_invdb_cmd;
	418	+
	419	+ struct {
	420	+ struct its_vpe *vpe;
	421	+ u8 sgi;
	422	+ u8 priority;
	423	+ bool enable;
	424	+ bool group;
	425	+ bool clear;
	426	+ } its_vsgi_cmd;
308	427	};
309	428	};
310	429
..	..	@@ -312,7 +431,10 @@
312	431	* The ITS command block, which is what the ITS actually parses.
313	432	*/
314	433	struct its_cmd_block {
315		- u64 raw_cmd[4];
	434	+ union {
	435	+ u64 raw_cmd[4];
	436	+ __le64 raw_cmd_le[4];
	437	+ };
316	438	};
317	439
318	440	#define ITS_CMD_QUEUE_SZ SZ_64K
..	..	@@ -418,13 +540,70 @@
418	540	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
419	541	}
420	542
	543	+static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
	544	+{
	545	+ its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
	546	+}
	547	+
	548	+static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
	549	+{
	550	+ its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
	551	+}
	552	+
	553	+static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
	554	+{
	555	+ its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
	556	+}
	557	+
	558	+static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
	559	+ u32 vpe_db_lpi)
	560	+{
	561	+ its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
	562	+}
	563	+
	564	+static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
	565	+ u32 vpe_db_lpi)
	566	+{
	567	+ its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
	568	+}
	569	+
	570	+static void its_encode_db(struct its_cmd_block *cmd, bool db)
	571	+{
	572	+ its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
	573	+}
	574	+
	575	+static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
	576	+{
	577	+ its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
	578	+}
	579	+
	580	+static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
	581	+{
	582	+ its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
	583	+}
	584	+
	585	+static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
	586	+{
	587	+ its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
	588	+}
	589	+
	590	+static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
	591	+{
	592	+ its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
	593	+}
	594	+
	595	+static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
	596	+{
	597	+ its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
	598	+}
	599	+
421	600	static inline void its_fixup_cmd(struct its_cmd_block *cmd)
422	601	{
423	602	/* Let's fixup BE commands */
424		- cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
425		- cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
426		- cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
427		- cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
	603	+ cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
	604	+ cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
	605	+ cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
	606	+ cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
428	607	}
429	608
430	609	static struct its_collection its_build_mapd_cmd(struct its_node its,
..	..	@@ -601,19 +780,45 @@
601	780	struct its_cmd_block *cmd,
602	781	struct its_cmd_desc *desc)
603	782	{
604		- unsigned long vpt_addr;
	783	+ unsigned long vpt_addr, vconf_addr;
605	784	u64 target;
606		-
607		- vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
608		- target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
	785	+ bool alloc;
609	786
610	787	its_encode_cmd(cmd, GITS_CMD_VMAPP);
611	788	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
612	789	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
	790	+
	791	+ if (!desc->its_vmapp_cmd.valid) {
	792	+ if (is_v4_1(its)) {
	793	+ alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
	794	+ its_encode_alloc(cmd, alloc);
	795	+ }
	796	+
	797	+ goto out;
	798	+ }
	799	+
	800	+ vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
	801	+ target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
	802	+
613	803	its_encode_target(cmd, target);
614	804	its_encode_vpt_addr(cmd, vpt_addr);
615	805	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
616	806
	807	+ if (!is_v4_1(its))
	808	+ goto out;
	809	+
	810	+ vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
	811	+
	812	+ alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
	813	+
	814	+ its_encode_alloc(cmd, alloc);
	815	+
	816	+ /* We can only signal PTZ when alloc==1. Why do we have two bits? */
	817	+ its_encode_ptz(cmd, alloc);
	818	+ its_encode_vconf_addr(cmd, vconf_addr);
	819	+ its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
	820	+
	821	+out:
617	822	its_fixup_cmd(cmd);
618	823
619	824	return valid_vpe(its, desc->its_vmapp_cmd.vpe);
..	..	@@ -625,7 +830,7 @@
625	830	{
626	831	u32 db;
627	832
628		- if (desc->its_vmapti_cmd.db_enabled)
	833	+ if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
629	834	db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
630	835	else
631	836	db = 1023;
..	..	@@ -648,7 +853,7 @@
648	853	{
649	854	u32 db;
650	855
651		- if (desc->its_vmovi_cmd.db_enabled)
	856	+ if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
652	857	db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
653	858	else
654	859	db = 1023;
..	..	@@ -678,9 +883,103 @@
678	883	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
679	884	its_encode_target(cmd, target);
680	885
	886	+ if (is_v4_1(its)) {
	887	+ its_encode_db(cmd, true);
	888	+ its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
	889	+ }
	890	+
681	891	its_fixup_cmd(cmd);
682	892
683	893	return valid_vpe(its, desc->its_vmovp_cmd.vpe);
	894	+}
	895	+
	896	+static struct its_vpe its_build_vinv_cmd(struct its_node its,
	897	+ struct its_cmd_block *cmd,
	898	+ struct its_cmd_desc *desc)
	899	+{
	900	+ struct its_vlpi_map *map;
	901	+
	902	+ map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
	903	+ desc->its_inv_cmd.event_id);
	904	+
	905	+ its_encode_cmd(cmd, GITS_CMD_INV);
	906	+ its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
	907	+ its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
	908	+
	909	+ its_fixup_cmd(cmd);
	910	+
	911	+ return valid_vpe(its, map->vpe);
	912	+}
	913	+
	914	+static struct its_vpe its_build_vint_cmd(struct its_node its,
	915	+ struct its_cmd_block *cmd,
	916	+ struct its_cmd_desc *desc)
	917	+{
	918	+ struct its_vlpi_map *map;
	919	+
	920	+ map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
	921	+ desc->its_int_cmd.event_id);
	922	+
	923	+ its_encode_cmd(cmd, GITS_CMD_INT);
	924	+ its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
	925	+ its_encode_event_id(cmd, desc->its_int_cmd.event_id);
	926	+
	927	+ its_fixup_cmd(cmd);
	928	+
	929	+ return valid_vpe(its, map->vpe);
	930	+}
	931	+
	932	+static struct its_vpe its_build_vclear_cmd(struct its_node its,
	933	+ struct its_cmd_block *cmd,
	934	+ struct its_cmd_desc *desc)
	935	+{
	936	+ struct its_vlpi_map *map;
	937	+
	938	+ map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
	939	+ desc->its_clear_cmd.event_id);
	940	+
	941	+ its_encode_cmd(cmd, GITS_CMD_CLEAR);
	942	+ its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
	943	+ its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
	944	+
	945	+ its_fixup_cmd(cmd);
	946	+
	947	+ return valid_vpe(its, map->vpe);
	948	+}
	949	+
	950	+static struct its_vpe its_build_invdb_cmd(struct its_node its,
	951	+ struct its_cmd_block *cmd,
	952	+ struct its_cmd_desc *desc)
	953	+{
	954	+ if (WARN_ON(!is_v4_1(its)))
	955	+ return NULL;
	956	+
	957	+ its_encode_cmd(cmd, GITS_CMD_INVDB);
	958	+ its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
	959	+
	960	+ its_fixup_cmd(cmd);
	961	+
	962	+ return valid_vpe(its, desc->its_invdb_cmd.vpe);
	963	+}
	964	+
	965	+static struct its_vpe its_build_vsgi_cmd(struct its_node its,
	966	+ struct its_cmd_block *cmd,
	967	+ struct its_cmd_desc *desc)
	968	+{
	969	+ if (WARN_ON(!is_v4_1(its)))
	970	+ return NULL;
	971	+
	972	+ its_encode_cmd(cmd, GITS_CMD_VSGI);
	973	+ its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
	974	+ its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
	975	+ its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
	976	+ its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
	977	+ its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
	978	+ its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
	979	+
	980	+ its_fixup_cmd(cmd);
	981	+
	982	+ return valid_vpe(its, desc->its_vsgi_cmd.vpe);
684	983	}
685	984
686	985	static u64 its_cmd_ptr_to_offset(struct its_node *its,
..	..	@@ -960,7 +1259,7 @@
960	1259
961	1260	static void its_send_vmapti(struct its_device *dev, u32 id)
962	1261	{
963		- struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
	1262	+ struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
964	1263	struct its_cmd_desc desc;
965	1264
966	1265	desc.its_vmapti_cmd.vpe = map->vpe;
..	..	@@ -974,7 +1273,7 @@
974	1273
975	1274	static void its_send_vmovi(struct its_device *dev, u32 id)
976	1275	{
977		- struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
	1276	+ struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
978	1277	struct its_cmd_desc desc;
979	1278
980	1279	desc.its_vmovi_cmd.vpe = map->vpe;
..	..	@@ -1028,10 +1327,10 @@
1028	1327
1029	1328	/* Emit VMOVPs */
1030	1329	list_for_each_entry(its, &its_nodes, entry) {
1031		- if (!its->is_v4)
	1330	+ if (!is_v4(its))
1032	1331	continue;
1033	1332
1034		- if (!vpe->its_vm->vlpi_count[its->list_nr])
	1333	+ if (!require_its_list_vmovp(vpe->its_vm, its))
1035	1334	continue;
1036	1335
1037	1336	desc.its_vmovp_cmd.col = &its->collections[col_id];
..	..	@@ -1049,40 +1348,79 @@
1049	1348	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1050	1349	}
1051	1350
	1351	+static void its_send_vinv(struct its_device *dev, u32 event_id)
	1352	+{
	1353	+ struct its_cmd_desc desc;
	1354	+
	1355	+ /*
	1356	+ * There is no real VINV command. This is just a normal INV,
	1357	+ * with a VSYNC instead of a SYNC.
	1358	+ */
	1359	+ desc.its_inv_cmd.dev = dev;
	1360	+ desc.its_inv_cmd.event_id = event_id;
	1361	+
	1362	+ its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
	1363	+}
	1364	+
	1365	+static void its_send_vint(struct its_device *dev, u32 event_id)
	1366	+{
	1367	+ struct its_cmd_desc desc;
	1368	+
	1369	+ /*
	1370	+ * There is no real VINT command. This is just a normal INT,
	1371	+ * with a VSYNC instead of a SYNC.
	1372	+ */
	1373	+ desc.its_int_cmd.dev = dev;
	1374	+ desc.its_int_cmd.event_id = event_id;
	1375	+
	1376	+ its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
	1377	+}
	1378	+
	1379	+static void its_send_vclear(struct its_device *dev, u32 event_id)
	1380	+{
	1381	+ struct its_cmd_desc desc;
	1382	+
	1383	+ /*
	1384	+ * There is no real VCLEAR command. This is just a normal CLEAR,
	1385	+ * with a VSYNC instead of a SYNC.
	1386	+ */
	1387	+ desc.its_clear_cmd.dev = dev;
	1388	+ desc.its_clear_cmd.event_id = event_id;
	1389	+
	1390	+ its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
	1391	+}
	1392	+
	1393	+static void its_send_invdb(struct its_node its, struct its_vpe vpe)
	1394	+{
	1395	+ struct its_cmd_desc desc;
	1396	+
	1397	+ desc.its_invdb_cmd.vpe = vpe;
	1398	+ its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
	1399	+}
	1400	+
1052	1401	/*
1053	1402	* irqchip functions - assumes MSI, mostly.
1054	1403	*/
1055		-
1056		-static inline u32 its_get_event_id(struct irq_data *d)
1057		-{
1058		- struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1059		- return d->hwirq - its_dev->event_map.lpi_base;
1060		-}
1061		-
1062	1404	static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1063	1405	{
	1406	+ struct its_vlpi_map *map = get_vlpi_map(d);
1064	1407	irq_hw_number_t hwirq;
1065		- struct page *prop_page;
	1408	+ void *va;
1066	1409	u8 *cfg;
1067	1410
1068		- if (irqd_is_forwarded_to_vcpu(d)) {
1069		- struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1070		- u32 event = its_get_event_id(d);
1071		- struct its_vlpi_map *map;
1072		-
1073		- prop_page = its_dev->event_map.vm->vprop_page;
1074		- map = &its_dev->event_map.vlpi_maps[event];
	1411	+ if (map) {
	1412	+ va = page_address(map->vm->vprop_page);
1075	1413	hwirq = map->vintid;
1076	1414
1077	1415	/* Remember the updated property */
1078	1416	map->properties &= ~clr;
1079	1417	map->properties \|= set \| LPI_PROP_GROUP1;
1080	1418	} else {
1081		- prop_page = gic_rdists->prop_page;
	1419	+ va = gic_rdists->prop_table_va;
1082	1420	hwirq = d->hwirq;
1083	1421	}
1084	1422
1085		- cfg = page_address(prop_page) + hwirq - 8192;
	1423	+ cfg = va + hwirq - 8192;
1086	1424	*cfg &= ~clr;
1087	1425	*cfg \|= set \| LPI_PROP_GROUP1;
1088	1426
..	..	@@ -1097,30 +1435,90 @@
1097	1435	dsb(ishst);
1098	1436	}
1099	1437
	1438	+static void wait_for_syncr(void __iomem *rdbase)
	1439	+{
	1440	+ while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
	1441	+ cpu_relax();
	1442	+}
	1443	+
	1444	+static void __direct_lpi_inv(struct irq_data *d, u64 val)
	1445	+{
	1446	+ void __iomem *rdbase;
	1447	+ unsigned long flags;
	1448	+ int cpu;
	1449	+
	1450	+ /* Target the redistributor this LPI is currently routed to */
	1451	+ cpu = irq_to_cpuid_lock(d, &flags);
	1452	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	1453	+
	1454	+ rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
	1455	+ gic_write_lpir(val, rdbase + GICR_INVLPIR);
	1456	+ wait_for_syncr(rdbase);
	1457	+
	1458	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	1459	+ irq_to_cpuid_unlock(d, flags);
	1460	+}
	1461	+
	1462	+static void direct_lpi_inv(struct irq_data *d)
	1463	+{
	1464	+ struct its_vlpi_map *map = get_vlpi_map(d);
	1465	+ u64 val;
	1466	+
	1467	+ if (map) {
	1468	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	1469	+
	1470	+ WARN_ON(!is_v4_1(its_dev->its));
	1471	+
	1472	+ val = GICR_INVLPIR_V;
	1473	+ val \|= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
	1474	+ val \|= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
	1475	+ } else {
	1476	+ val = d->hwirq;
	1477	+ }
	1478	+
	1479	+ __direct_lpi_inv(d, val);
	1480	+}
	1481	+
1100	1482	static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1101	1483	{
1102	1484	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1103	1485
1104	1486	lpi_write_config(d, clr, set);
1105		- its_send_inv(its_dev, its_get_event_id(d));
	1487	+ if (gic_rdists->has_direct_lpi &&
	1488	+ (is_v4_1(its_dev->its) \|\| !irqd_is_forwarded_to_vcpu(d)))
	1489	+ direct_lpi_inv(d);
	1490	+ else if (!irqd_is_forwarded_to_vcpu(d))
	1491	+ its_send_inv(its_dev, its_get_event_id(d));
	1492	+ else
	1493	+ its_send_vinv(its_dev, its_get_event_id(d));
1106	1494	}
1107	1495
1108	1496	static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1109	1497	{
1110	1498	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1111	1499	u32 event = its_get_event_id(d);
	1500	+ struct its_vlpi_map *map;
1112	1501
1113		- if (its_dev->event_map.vlpi_maps[event].db_enabled == enable)
	1502	+ /*
	1503	+ * GICv4.1 does away with the per-LPI nonsense, nothing to do
	1504	+ * here.
	1505	+ */
	1506	+ if (is_v4_1(its_dev->its))
1114	1507	return;
1115	1508
1116		- its_dev->event_map.vlpi_maps[event].db_enabled = enable;
	1509	+ map = dev_event_to_vlpi_map(its_dev, event);
	1510	+
	1511	+ if (map->db_enabled == enable)
	1512	+ return;
	1513	+
	1514	+ map->db_enabled = enable;
1117	1515
1118	1516	/*
1119	1517	* More fun with the architecture:
1120	1518	*
1121	1519	* Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1122	1520	* value or to 1023, depending on the enable bit. But that
1123		- * would be issueing a mapping for an /existing/ DevID+EventID
	1521	+ * would be issuing a mapping for an /existing/ DevID+EventID
1124	1522	* pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1125	1523	* to the /same/ vPE, using this opportunity to adjust the
1126	1524	* doorbell. Mouahahahaha. We loves it, Precious.
..	..	@@ -1144,42 +1542,159 @@
1144	1542	lpi_update_config(d, 0, LPI_PROP_ENABLED);
1145	1543	}
1146	1544
	1545	+static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
	1546	+{
	1547	+ if (irqd_affinity_is_managed(d))
	1548	+ return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1549	+
	1550	+ return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1551	+}
	1552	+
	1553	+static void its_inc_lpi_count(struct irq_data *d, int cpu)
	1554	+{
	1555	+ if (irqd_affinity_is_managed(d))
	1556	+ atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1557	+ else
	1558	+ atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1559	+}
	1560	+
	1561	+static void its_dec_lpi_count(struct irq_data *d, int cpu)
	1562	+{
	1563	+ if (irqd_affinity_is_managed(d))
	1564	+ atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1565	+ else
	1566	+ atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1567	+}
	1568	+
	1569	+static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
	1570	+ const struct cpumask *cpu_mask)
	1571	+{
	1572	+ unsigned int cpu = nr_cpu_ids, tmp;
	1573	+ int count = S32_MAX;
	1574	+
	1575	+ for_each_cpu(tmp, cpu_mask) {
	1576	+ int this_count = its_read_lpi_count(d, tmp);
	1577	+ if (this_count < count) {
	1578	+ cpu = tmp;
	1579	+ count = this_count;
	1580	+ }
	1581	+ }
	1582	+
	1583	+ return cpu;
	1584	+}
	1585	+
	1586	+/*
	1587	+ * As suggested by Thomas Gleixner in:
	1588	+ * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
	1589	+ */
	1590	+static int its_select_cpu(struct irq_data *d,
	1591	+ const struct cpumask *aff_mask)
	1592	+{
	1593	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	1594	+ cpumask_var_t tmpmask;
	1595	+ int cpu, node;
	1596	+
	1597	+ if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
	1598	+ return -ENOMEM;
	1599	+
	1600	+ node = its_dev->its->numa_node;
	1601	+
	1602	+ if (!irqd_affinity_is_managed(d)) {
	1603	+ /* First try the NUMA node */
	1604	+ if (node != NUMA_NO_NODE) {
	1605	+ /*
	1606	+ * Try the intersection of the affinity mask and the
	1607	+ * node mask (and the online mask, just to be safe).
	1608	+ */
	1609	+ cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
	1610	+ cpumask_and(tmpmask, tmpmask, cpu_online_mask);
	1611	+
	1612	+ /*
	1613	+ * Ideally, we would check if the mask is empty, and
	1614	+ * try again on the full node here.
	1615	+ *
	1616	+ * But it turns out that the way ACPI describes the
	1617	+ * affinity for ITSs only deals about memory, and
	1618	+ * not target CPUs, so it cannot describe a single
	1619	+ * ITS placed next to two NUMA nodes.
	1620	+ *
	1621	+ * Instead, just fallback on the online mask. This
	1622	+ * diverges from Thomas' suggestion above.
	1623	+ */
	1624	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1625	+ if (cpu < nr_cpu_ids)
	1626	+ goto out;
	1627	+
	1628	+ /* If we can't cross sockets, give up */
	1629	+ if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
	1630	+ goto out;
	1631	+
	1632	+ /* If the above failed, expand the search */
	1633	+ }
	1634	+
	1635	+ /* Try the intersection of the affinity and online masks */
	1636	+ cpumask_and(tmpmask, aff_mask, cpu_online_mask);
	1637	+
	1638	+ /* If that doesn't fly, the online mask is the last resort */
	1639	+ if (cpumask_empty(tmpmask))
	1640	+ cpumask_copy(tmpmask, cpu_online_mask);
	1641	+
	1642	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1643	+ } else {
	1644	+ cpumask_copy(tmpmask, aff_mask);
	1645	+
	1646	+ /* If we cannot cross sockets, limit the search to that node */
	1647	+ if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
	1648	+ node != NUMA_NO_NODE)
	1649	+ cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
	1650	+
	1651	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1652	+ }
	1653	+out:
	1654	+ free_cpumask_var(tmpmask);
	1655	+
	1656	+ pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
	1657	+ return cpu;
	1658	+}
	1659	+
1147	1660	static int its_set_affinity(struct irq_data d, const struct cpumask mask_val,
1148	1661	bool force)
1149	1662	{
1150		- unsigned int cpu;
1151		- const struct cpumask *cpu_mask = cpu_online_mask;
1152	1663	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1153	1664	struct its_collection *target_col;
1154	1665	u32 id = its_get_event_id(d);
	1666	+ int cpu, prev_cpu;
1155	1667
1156	1668	/* A forwarded interrupt should use irq_set_vcpu_affinity */
1157	1669	if (irqd_is_forwarded_to_vcpu(d))
1158	1670	return -EINVAL;
1159	1671
1160		- /* lpi cannot be routed to a redistributor that is on a foreign node */
1161		- if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
1162		- if (its_dev->its->numa_node >= 0) {
1163		- cpu_mask = cpumask_of_node(its_dev->its->numa_node);
1164		- if (!cpumask_intersects(mask_val, cpu_mask))
1165		- return -EINVAL;
1166		- }
1167		- }
	1672	+ prev_cpu = its_dev->event_map.col_map[id];
	1673	+ its_dec_lpi_count(d, prev_cpu);
1168	1674
1169		- cpu = cpumask_any_and(mask_val, cpu_mask);
	1675	+ if (!force)
	1676	+ cpu = its_select_cpu(d, mask_val);
	1677	+ else
	1678	+ cpu = cpumask_pick_least_loaded(d, mask_val);
1170	1679
1171		- if (cpu >= nr_cpu_ids)
1172		- return -EINVAL;
	1680	+ if (cpu < 0 \|\| cpu >= nr_cpu_ids)
	1681	+ goto err;
1173	1682
1174	1683	/* don't set the affinity when the target cpu is same as current one */
1175		- if (cpu != its_dev->event_map.col_map[id]) {
	1684	+ if (cpu != prev_cpu) {
1176	1685	target_col = &its_dev->its->collections[cpu];
1177	1686	its_send_movi(its_dev, target_col, id);
1178	1687	its_dev->event_map.col_map[id] = cpu;
1179	1688	irq_data_update_effective_affinity(d, cpumask_of(cpu));
1180	1689	}
1181	1690
	1691	+ its_inc_lpi_count(d, cpu);
	1692	+
1182	1693	return IRQ_SET_MASK_OK_DONE;
	1694	+
	1695	+err:
	1696	+ its_inc_lpi_count(d, prev_cpu);
	1697	+ return -EINVAL;
1183	1698	}
1184	1699
1185	1700	static u64 its_irq_get_msi_base(struct its_device *its_dev)
..	..	@@ -1202,7 +1717,7 @@
1202	1717	msg->address_hi = upper_32_bits(addr);
1203	1718	msg->data = its_get_event_id(d);
1204	1719
1205		- iommu_dma_map_msi_msg(d->irq, msg);
	1720	+ iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1206	1721	}
1207	1722
1208	1723	static int its_irq_set_irqchip_state(struct irq_data *d,
..	..	@@ -1215,20 +1730,51 @@
1215	1730	if (which != IRQCHIP_STATE_PENDING)
1216	1731	return -EINVAL;
1217	1732
1218		- if (state)
1219		- its_send_int(its_dev, event);
1220		- else
1221		- its_send_clear(its_dev, event);
	1733	+ if (irqd_is_forwarded_to_vcpu(d)) {
	1734	+ if (state)
	1735	+ its_send_vint(its_dev, event);
	1736	+ else
	1737	+ its_send_vclear(its_dev, event);
	1738	+ } else {
	1739	+ if (state)
	1740	+ its_send_int(its_dev, event);
	1741	+ else
	1742	+ its_send_clear(its_dev, event);
	1743	+ }
1222	1744
1223	1745	return 0;
	1746	+}
	1747	+
	1748	+static int its_irq_retrigger(struct irq_data *d)
	1749	+{
	1750	+ return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
	1751	+}
	1752	+
	1753	+/*
	1754	+ * Two favourable cases:
	1755	+ *
	1756	+ * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
	1757	+ * for vSGI delivery
	1758	+ *
	1759	+ * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
	1760	+ * and we're better off mapping all VPEs always
	1761	+ *
	1762	+ * If neither (a) nor (b) is true, then we map vPEs on demand.
	1763	+ *
	1764	+ */
	1765	+static bool gic_requires_eager_mapping(void)
	1766	+{
	1767	+ if (!its_list_map \|\| gic_rdists->has_rvpeid)
	1768	+ return true;
	1769	+
	1770	+ return false;
1224	1771	}
1225	1772
1226	1773	static void its_map_vm(struct its_node its, struct its_vm vm)
1227	1774	{
1228	1775	unsigned long flags;
1229	1776
1230		- /* Not using the ITS list? Everything is always mapped. */
1231		- if (!its_list_map)
	1777	+ if (gic_requires_eager_mapping())
1232	1778	return;
1233	1779
1234	1780	raw_spin_lock_irqsave(&vmovp_lock, flags);
..	..	@@ -1262,7 +1808,7 @@
1262	1808	unsigned long flags;
1263	1809
1264	1810	/* Not using the ITS list? Everything is always mapped. */
1265		- if (!its_list_map)
	1811	+ if (gic_requires_eager_mapping())
1266	1812	return;
1267	1813
1268	1814	raw_spin_lock_irqsave(&vmovp_lock, flags);
..	..	@@ -1286,13 +1832,13 @@
1286	1832	if (!info->map)
1287	1833	return -EINVAL;
1288	1834
1289		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1835	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1290	1836
1291	1837	if (!its_dev->event_map.vm) {
1292	1838	struct its_vlpi_map *maps;
1293	1839
1294	1840	maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1295		- GFP_KERNEL);
	1841	+ GFP_ATOMIC);
1296	1842	if (!maps) {
1297	1843	ret = -ENOMEM;
1298	1844	goto out;
..	..	@@ -1335,29 +1881,30 @@
1335	1881	}
1336	1882
1337	1883	out:
1338		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1884	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1339	1885	return ret;
1340	1886	}
1341	1887
1342	1888	static int its_vlpi_get(struct irq_data d, struct its_cmd_info info)
1343	1889	{
1344	1890	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1345		- u32 event = its_get_event_id(d);
	1891	+ struct its_vlpi_map *map;
1346	1892	int ret = 0;
1347	1893
1348		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1894	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1349	1895
1350		- if (!its_dev->event_map.vm \|\|
1351		- !its_dev->event_map.vlpi_maps[event].vm) {
	1896	+ map = get_vlpi_map(d);
	1897	+
	1898	+ if (!its_dev->event_map.vm \|\| !map) {
1352	1899	ret = -EINVAL;
1353	1900	goto out;
1354	1901	}
1355	1902
1356	1903	/* Copy our mapping information to the incoming request */
1357		- *info->map = its_dev->event_map.vlpi_maps[event];
	1904	+ info->map = map;
1358	1905
1359	1906	out:
1360		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1907	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1361	1908	return ret;
1362	1909	}
1363	1910
..	..	@@ -1367,7 +1914,7 @@
1367	1914	u32 event = its_get_event_id(d);
1368	1915	int ret = 0;
1369	1916
1370		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1917	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1371	1918
1372	1919	if (!its_dev->event_map.vm \|\| !irqd_is_forwarded_to_vcpu(d)) {
1373	1920	ret = -EINVAL;
..	..	@@ -1397,7 +1944,7 @@
1397	1944	}
1398	1945
1399	1946	out:
1400		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1947	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1401	1948	return ret;
1402	1949	}
1403	1950
..	..	@@ -1423,7 +1970,7 @@
1423	1970	struct its_cmd_info *info = vcpu_info;
1424	1971
1425	1972	/* Need a v4 ITS */
1426		- if (!its_dev->its->is_v4)
	1973	+ if (!is_v4(its_dev->its))
1427	1974	return -EINVAL;
1428	1975
1429	1976	/* Unmap request? */
..	..	@@ -1454,6 +2001,7 @@
1454	2001	.irq_set_affinity = its_set_affinity,
1455	2002	.irq_compose_msi_msg = its_irq_compose_msi_msg,
1456	2003	.irq_set_irqchip_state = its_irq_set_irqchip_state,
	2004	+ .irq_retrigger = its_irq_retrigger,
1457	2005	.irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
1458	2006	};
1459	2007
..	..	@@ -1488,39 +2036,13 @@
1488	2036	{
1489	2037	struct lpi_range *range;
1490	2038
1491		- range = kzalloc(sizeof(*range), GFP_KERNEL);
	2039	+ range = kmalloc(sizeof(*range), GFP_KERNEL);
1492	2040	if (range) {
1493		- INIT_LIST_HEAD(&range->entry);
1494	2041	range->base_id = base;
1495	2042	range->span = span;
1496	2043	}
1497	2044
1498	2045	return range;
1499		-}
1500		-
1501		-static int lpi_range_cmp(void priv, struct list_head a, struct list_head *b)
1502		-{
1503		- struct lpi_range ra, rb;
1504		-
1505		- ra = container_of(a, struct lpi_range, entry);
1506		- rb = container_of(b, struct lpi_range, entry);
1507		-
1508		- return ra->base_id - rb->base_id;
1509		-}
1510		-
1511		-static void merge_lpi_ranges(void)
1512		-{
1513		- struct lpi_range range, tmp;
1514		-
1515		- list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
1516		- if (!list_is_last(&range->entry, &lpi_range_list) &&
1517		- (tmp->base_id == (range->base_id + range->span))) {
1518		- tmp->base_id = range->base_id;
1519		- tmp->span += range->span;
1520		- list_del(&range->entry);
1521		- kfree(range);
1522		- }
1523		- }
1524	2046	}
1525	2047
1526	2048	static int alloc_lpi_range(u32 nr_lpis, u32 *base)
..	..	@@ -1552,25 +2074,49 @@
1552	2074	return err;
1553	2075	}
1554	2076
	2077	+static void merge_lpi_ranges(struct lpi_range a, struct lpi_range b)
	2078	+{
	2079	+ if (&a->entry == &lpi_range_list \|\| &b->entry == &lpi_range_list)
	2080	+ return;
	2081	+ if (a->base_id + a->span != b->base_id)
	2082	+ return;
	2083	+ b->base_id = a->base_id;
	2084	+ b->span += a->span;
	2085	+ list_del(&a->entry);
	2086	+ kfree(a);
	2087	+}
	2088	+
1555	2089	static int free_lpi_range(u32 base, u32 nr_lpis)
1556	2090	{
1557		- struct lpi_range *new;
1558		- int err = 0;
	2091	+ struct lpi_range new, old;
	2092	+
	2093	+ new = mk_lpi_range(base, nr_lpis);
	2094	+ if (!new)
	2095	+ return -ENOMEM;
1559	2096
1560	2097	mutex_lock(&lpi_range_lock);
1561	2098
1562		- new = mk_lpi_range(base, nr_lpis);
1563		- if (!new) {
1564		- err = -ENOMEM;
1565		- goto out;
	2099	+ list_for_each_entry_reverse(old, &lpi_range_list, entry) {
	2100	+ if (old->base_id < base)
	2101	+ break;
1566	2102	}
	2103	+ /*
	2104	+ * old is the last element with ->base_id smaller than base,
	2105	+ * so new goes right after it. If there are no elements with
	2106	+ * ->base_id smaller than base, &old->entry ends up pointing
	2107	+ * at the head of the list, and inserting new it the start of
	2108	+ * the list is the right thing to do in that case as well.
	2109	+ */
	2110	+ list_add(&new->entry, &old->entry);
	2111	+ /*
	2112	+ * Now check if we can merge with the preceding and/or
	2113	+ * following ranges.
	2114	+ */
	2115	+ merge_lpi_ranges(old, new);
	2116	+ merge_lpi_ranges(new, list_next_entry(new, entry));
1567	2117
1568		- list_add(&new->entry, &lpi_range_list);
1569		- list_sort(NULL, &lpi_range_list, lpi_range_cmp);
1570		- merge_lpi_ranges();
1571		-out:
1572	2118	mutex_unlock(&lpi_range_lock);
1573		- return err;
	2119	+ return 0;
1574	2120	}
1575	2121
1576	2122	static int __init its_lpi_init(u32 id_bits)
..	..	@@ -1634,23 +2180,28 @@
1634	2180	kfree(bitmap);
1635	2181	}
1636	2182
	2183	+static void gic_reset_prop_table(void *va)
	2184	+{
	2185	+ /* Priority 0xa0, Group-1, disabled */
	2186	+ memset(va, LPI_PROP_DEFAULT_PRIO \| LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
	2187	+
	2188	+ /* Make sure the GIC will observe the written configuration */
	2189	+ gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
	2190	+}
	2191	+
1637	2192	static struct page *its_allocate_prop_table(gfp_t gfp_flags)
1638	2193	{
1639	2194	struct page *prop_page;
1640	2195
1641		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2196	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2197	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2198	+ of_machine_is_compatible("rockchip,rk3566"))
1642	2199	gfp_flags \|= GFP_DMA32;
1643	2200	prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
1644	2201	if (!prop_page)
1645	2202	return NULL;
1646	2203
1647		- /* Priority 0xa0, Group-1, disabled */
1648		- memset(page_address(prop_page),
1649		- LPI_PROP_DEFAULT_PRIO \| LPI_PROP_GROUP1,
1650		- LPI_PROPBASE_SZ);
1651		-
1652		- /* Make sure the GIC will observe the written configuration */
1653		- gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ);
	2204	+ gic_reset_prop_table(page_address(prop_page));
1654	2205
1655	2206	return prop_page;
1656	2207	}
..	..	@@ -1661,20 +2212,74 @@
1661	2212	get_order(LPI_PROPBASE_SZ));
1662	2213	}
1663	2214
1664		-static int __init its_alloc_lpi_tables(void)
	2215	+static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
1665	2216	{
1666		- phys_addr_t paddr;
	2217	+ phys_addr_t start, end, addr_end;
	2218	+ u64 i;
1667	2219
1668		- lpi_id_bits = min_t(u32, GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
1669		- ITS_MAX_LPI_NRBITS);
1670		- gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT);
1671		- if (!gic_rdists->prop_page) {
1672		- pr_err("Failed to allocate PROPBASE\n");
1673		- return -ENOMEM;
	2220	+ /*
	2221	+ * We don't bother checking for a kdump kernel as by
	2222	+ * construction, the LPI tables are out of this kernel's
	2223	+ * memory map.
	2224	+ */
	2225	+ if (is_kdump_kernel())
	2226	+ return true;
	2227	+
	2228	+ addr_end = addr + size - 1;
	2229	+
	2230	+ for_each_reserved_mem_range(i, &start, &end) {
	2231	+ if (addr >= start && addr_end <= end)
	2232	+ return true;
1674	2233	}
1675	2234
1676		- paddr = page_to_phys(gic_rdists->prop_page);
1677		- pr_info("GIC: using LPI property table @%pa\n", &paddr);
	2235	+ /* Not found, not a good sign... */
	2236	+ pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
	2237	+ &addr, &addr_end);
	2238	+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
	2239	+ return false;
	2240	+}
	2241	+
	2242	+static int gic_reserve_range(phys_addr_t addr, unsigned long size)
	2243	+{
	2244	+ if (efi_enabled(EFI_CONFIG_TABLES))
	2245	+ return efi_mem_reserve_persistent(addr, size);
	2246	+
	2247	+ return 0;
	2248	+}
	2249	+
	2250	+static int __init its_setup_lpi_prop_table(void)
	2251	+{
	2252	+ if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
	2253	+ u64 val;
	2254	+
	2255	+ val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
	2256	+ lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
	2257	+
	2258	+ gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
	2259	+ gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
	2260	+ LPI_PROPBASE_SZ,
	2261	+ MEMREMAP_WB);
	2262	+ gic_reset_prop_table(gic_rdists->prop_table_va);
	2263	+ } else {
	2264	+ struct page *page;
	2265	+
	2266	+ lpi_id_bits = min_t(u32,
	2267	+ GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
	2268	+ ITS_MAX_LPI_NRBITS);
	2269	+ page = its_allocate_prop_table(GFP_NOWAIT);
	2270	+ if (!page) {
	2271	+ pr_err("Failed to allocate PROPBASE\n");
	2272	+ return -ENOMEM;
	2273	+ }
	2274	+
	2275	+ gic_rdists->prop_table_pa = page_to_phys(page);
	2276	+ gic_rdists->prop_table_va = page_address(page);
	2277	+ WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
	2278	+ LPI_PROPBASE_SZ));
	2279	+ }
	2280	+
	2281	+ pr_info("GICv3: using LPI property table @%pa\n",
	2282	+ &gic_rdists->prop_table_pa);
1678	2283
1679	2284	return its_lpi_init(lpi_id_bits);
1680	2285	}
..	..	@@ -1706,18 +2311,18 @@
1706	2311	}
1707	2312
1708	2313	static int its_setup_baser(struct its_node its, struct its_baser baser,
1709		- u64 cache, u64 shr, u32 psz, u32 order,
1710		- bool indirect)
	2314	+ u64 cache, u64 shr, u32 order, bool indirect)
1711	2315	{
1712	2316	u64 val = its_read_baser(its, baser);
1713	2317	u64 esz = GITS_BASER_ENTRY_SIZE(val);
1714	2318	u64 type = GITS_BASER_TYPE(val);
1715	2319	u64 baser_phys, tmp;
1716		- u32 alloc_pages;
	2320	+ u32 alloc_pages, psz;
	2321	+ struct page *page;
1717	2322	void *base;
1718	2323	gfp_t gfp_flags;
1719	2324
1720		-retry_alloc_baser:
	2325	+ psz = baser->psz;
1721	2326	alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
1722	2327	if (alloc_pages > GITS_BASER_PAGES_MAX) {
1723	2328	pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
..	..	@@ -1728,12 +2333,15 @@
1728	2333	}
1729	2334
1730	2335	gfp_flags = GFP_KERNEL \| __GFP_ZERO;
1731		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2336	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2337	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2338	+ of_machine_is_compatible("rockchip,rk3566"))
1732	2339	gfp_flags \|= GFP_DMA32;
1733		- base = (void *)__get_free_pages(gfp_flags, order);
1734		- if (!base)
	2340	+ page = alloc_pages_node(its->numa_node, gfp_flags, order);
	2341	+ if (!page)
1735	2342	return -ENOMEM;
1736	2343
	2344	+ base = (void *)page_address(page);
1737	2345	baser_phys = virt_to_phys(base);
1738	2346
1739	2347	/* Check if the physical address of the memory is above 48bits */
..	..	@@ -1776,8 +2384,11 @@
1776	2384	its_write_baser(its, baser, val);
1777	2385	tmp = baser->val;
1778	2386
1779		- if (of_machine_is_compatible("rockchip,rk3568") \|\|
1780		- of_machine_is_compatible("rockchip,rk3566")) {
	2387	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	2388	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	2389	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2390	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	2391	+ of_machine_is_compatible("rockchip,rk3588"))) {
1781	2392	if (tmp & GITS_BASER_SHAREABILITY_MASK)
1782	2393	tmp &= ~GITS_BASER_SHAREABILITY_MASK;
1783	2394	else
..	..	@@ -1798,25 +2409,6 @@
1798	2409	gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
1799	2410	}
1800	2411	goto retry_baser;
1801		- }
1802		-
1803		- if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
1804		- /*
1805		- * Page size didn't stick. Let's try a smaller
1806		- * size and retry. If we reach 4K, then
1807		- * something is horribly wrong...
1808		- */
1809		- free_pages((unsigned long)base, order);
1810		- baser->base = NULL;
1811		-
1812		- switch (psz) {
1813		- case SZ_16K:
1814		- psz = SZ_4K;
1815		- goto retry_alloc_baser;
1816		- case SZ_64K:
1817		- psz = SZ_16K;
1818		- goto retry_alloc_baser;
1819		- }
1820	2412	}
1821	2413
1822	2414	if (val != tmp) {
..	..	@@ -1844,13 +2436,14 @@
1844	2436
1845	2437	static bool its_parse_indirect_baser(struct its_node *its,
1846	2438	struct its_baser *baser,
1847		- u32 psz, u32 *order, u32 ids)
	2439	+ u32 *order, u32 ids)
1848	2440	{
1849	2441	u64 tmp = its_read_baser(its, baser);
1850	2442	u64 type = GITS_BASER_TYPE(tmp);
1851	2443	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
1852	2444	u64 val = GITS_BASER_InnerShareable \| GITS_BASER_RaWaWb;
1853	2445	u32 new_order = *order;
	2446	+ u32 psz = baser->psz;
1854	2447	bool indirect = false;
1855	2448
1856	2449	/* No need to enable Indirection if memory requirement < (psz2)bytes /
..	..	@@ -1886,14 +2479,73 @@
1886	2479	if (new_order >= MAX_ORDER) {
1887	2480	new_order = MAX_ORDER - 1;
1888	2481	ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
1889		- pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
	2482	+ pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
1890	2483	&its->phys_base, its_base_type_string[type],
1891		- its->device_ids, ids);
	2484	+ device_ids(its), ids);
1892	2485	}
1893	2486
1894	2487	*order = new_order;
1895	2488
1896	2489	return indirect;
	2490	+}
	2491	+
	2492	+static u32 compute_common_aff(u64 val)
	2493	+{
	2494	+ u32 aff, clpiaff;
	2495	+
	2496	+ aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
	2497	+ clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
	2498	+
	2499	+ return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
	2500	+}
	2501	+
	2502	+static u32 compute_its_aff(struct its_node *its)
	2503	+{
	2504	+ u64 val;
	2505	+ u32 svpet;
	2506	+
	2507	+ /*
	2508	+ * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
	2509	+ * the resulting affinity. We then use that to see if this match
	2510	+ * our own affinity.
	2511	+ */
	2512	+ svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
	2513	+ val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
	2514	+ val \|= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
	2515	+ return compute_common_aff(val);
	2516	+}
	2517	+
	2518	+static struct its_node find_sibling_its(struct its_node cur_its)
	2519	+{
	2520	+ struct its_node *its;
	2521	+ u32 aff;
	2522	+
	2523	+ if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
	2524	+ return NULL;
	2525	+
	2526	+ aff = compute_its_aff(cur_its);
	2527	+
	2528	+ list_for_each_entry(its, &its_nodes, entry) {
	2529	+ u64 baser;
	2530	+
	2531	+ if (!is_v4_1(its) \|\| its == cur_its)
	2532	+ continue;
	2533	+
	2534	+ if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
	2535	+ continue;
	2536	+
	2537	+ if (aff != compute_its_aff(its))
	2538	+ continue;
	2539	+
	2540	+ /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
	2541	+ baser = its->tables[2].val;
	2542	+ if (!(baser & GITS_BASER_VALID))
	2543	+ continue;
	2544	+
	2545	+ return its;
	2546	+ }
	2547	+
	2548	+ return NULL;
1897	2549	}
1898	2550
1899	2551	static void its_free_tables(struct its_node *its)
..	..	@@ -1909,11 +2561,58 @@
1909	2561	}
1910	2562	}
1911	2563
	2564	+static int its_probe_baser_psz(struct its_node its, struct its_baser baser)
	2565	+{
	2566	+ u64 psz = SZ_64K;
	2567	+
	2568	+ while (psz) {
	2569	+ u64 val, gpsz;
	2570	+
	2571	+ val = its_read_baser(its, baser);
	2572	+ val &= ~GITS_BASER_PAGE_SIZE_MASK;
	2573	+
	2574	+ switch (psz) {
	2575	+ case SZ_64K:
	2576	+ gpsz = GITS_BASER_PAGE_SIZE_64K;
	2577	+ break;
	2578	+ case SZ_16K:
	2579	+ gpsz = GITS_BASER_PAGE_SIZE_16K;
	2580	+ break;
	2581	+ case SZ_4K:
	2582	+ default:
	2583	+ gpsz = GITS_BASER_PAGE_SIZE_4K;
	2584	+ break;
	2585	+ }
	2586	+
	2587	+ gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
	2588	+
	2589	+ val \|= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
	2590	+ its_write_baser(its, baser, val);
	2591	+
	2592	+ if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
	2593	+ break;
	2594	+
	2595	+ switch (psz) {
	2596	+ case SZ_64K:
	2597	+ psz = SZ_16K;
	2598	+ break;
	2599	+ case SZ_16K:
	2600	+ psz = SZ_4K;
	2601	+ break;
	2602	+ case SZ_4K:
	2603	+ default:
	2604	+ return -1;
	2605	+ }
	2606	+ }
	2607	+
	2608	+ baser->psz = psz;
	2609	+ return 0;
	2610	+}
	2611	+
1912	2612	static int its_alloc_tables(struct its_node *its)
1913	2613	{
1914	2614	u64 shr = GITS_BASER_InnerShareable;
1915	2615	u64 cache = GITS_BASER_RaWaWb;
1916		- u32 psz = SZ_64K;
1917	2616	int err, i;
1918	2617
1919	2618	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
..	..	@@ -1924,37 +2623,337 @@
1924	2623	struct its_baser *baser = its->tables + i;
1925	2624	u64 val = its_read_baser(its, baser);
1926	2625	u64 type = GITS_BASER_TYPE(val);
1927		- u32 order = get_order(psz);
1928	2626	bool indirect = false;
	2627	+ u32 order;
1929	2628
1930		- switch (type) {
1931		- case GITS_BASER_TYPE_NONE:
	2629	+ if (type == GITS_BASER_TYPE_NONE)
1932	2630	continue;
1933	2631
	2632	+ if (its_probe_baser_psz(its, baser)) {
	2633	+ its_free_tables(its);
	2634	+ return -ENXIO;
	2635	+ }
	2636	+
	2637	+ order = get_order(baser->psz);
	2638	+
	2639	+ switch (type) {
1934	2640	case GITS_BASER_TYPE_DEVICE:
1935		- indirect = its_parse_indirect_baser(its, baser,
1936		- psz, &order,
1937		- its->device_ids);
	2641	+ indirect = its_parse_indirect_baser(its, baser, &order,
	2642	+ device_ids(its));
1938	2643	break;
1939	2644
1940	2645	case GITS_BASER_TYPE_VCPU:
1941		- indirect = its_parse_indirect_baser(its, baser,
1942		- psz, &order,
	2646	+ if (is_v4_1(its)) {
	2647	+ struct its_node *sibling;
	2648	+
	2649	+ WARN_ON(i != 2);
	2650	+ if ((sibling = find_sibling_its(its))) {
	2651	+ *baser = sibling->tables[2];
	2652	+ its_write_baser(its, baser, baser->val);
	2653	+ continue;
	2654	+ }
	2655	+ }
	2656	+
	2657	+ indirect = its_parse_indirect_baser(its, baser, &order,
1943	2658	ITS_MAX_VPEID_BITS);
1944	2659	break;
1945	2660	}
1946	2661
1947		- err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
	2662	+ err = its_setup_baser(its, baser, cache, shr, order, indirect);
1948	2663	if (err < 0) {
1949	2664	its_free_tables(its);
1950	2665	return err;
1951	2666	}
1952	2667
1953	2668	/* Update settings which will be used for next BASERn */
1954		- psz = baser->psz;
1955	2669	cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
1956	2670	shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
1957	2671	}
	2672	+
	2673	+ return 0;
	2674	+}
	2675	+
	2676	+static u64 inherit_vpe_l1_table_from_its(void)
	2677	+{
	2678	+ struct its_node *its;
	2679	+ u64 val;
	2680	+ u32 aff;
	2681	+
	2682	+ val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
	2683	+ aff = compute_common_aff(val);
	2684	+
	2685	+ list_for_each_entry(its, &its_nodes, entry) {
	2686	+ u64 baser, addr;
	2687	+
	2688	+ if (!is_v4_1(its))
	2689	+ continue;
	2690	+
	2691	+ if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
	2692	+ continue;
	2693	+
	2694	+ if (aff != compute_its_aff(its))
	2695	+ continue;
	2696	+
	2697	+ /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
	2698	+ baser = its->tables[2].val;
	2699	+ if (!(baser & GITS_BASER_VALID))
	2700	+ continue;
	2701	+
	2702	+ /* We have a winner! */
	2703	+ gic_data_rdist()->vpe_l1_base = its->tables[2].base;
	2704	+
	2705	+ val = GICR_VPROPBASER_4_1_VALID;
	2706	+ if (baser & GITS_BASER_INDIRECT)
	2707	+ val \|= GICR_VPROPBASER_4_1_INDIRECT;
	2708	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
	2709	+ FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
	2710	+ switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
	2711	+ case GIC_PAGE_SIZE_64K:
	2712	+ addr = GITS_BASER_ADDR_48_to_52(baser);
	2713	+ break;
	2714	+ default:
	2715	+ addr = baser & GENMASK_ULL(47, 12);
	2716	+ break;
	2717	+ }
	2718	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
	2719	+ val \|= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
	2720	+ FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
	2721	+ val \|= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
	2722	+ FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
	2723	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
	2724	+
	2725	+ return val;
	2726	+ }
	2727	+
	2728	+ return 0;
	2729	+}
	2730	+
	2731	+static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
	2732	+{
	2733	+ u32 aff;
	2734	+ u64 val;
	2735	+ int cpu;
	2736	+
	2737	+ val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
	2738	+ aff = compute_common_aff(val);
	2739	+
	2740	+ for_each_possible_cpu(cpu) {
	2741	+ void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
	2742	+
	2743	+ if (!base \|\| cpu == smp_processor_id())
	2744	+ continue;
	2745	+
	2746	+ val = gic_read_typer(base + GICR_TYPER);
	2747	+ if (aff != compute_common_aff(val))
	2748	+ continue;
	2749	+
	2750	+ /*
	2751	+ * At this point, we have a victim. This particular CPU
	2752	+ * has already booted, and has an affinity that matches
	2753	+ * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
	2754	+ * Make sure we don't write the Z bit in that case.
	2755	+ */
	2756	+ val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
	2757	+ val &= ~GICR_VPROPBASER_4_1_Z;
	2758	+
	2759	+ gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
	2760	+ *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
	2761	+
	2762	+ return val;
	2763	+ }
	2764	+
	2765	+ return 0;
	2766	+}
	2767	+
	2768	+static bool allocate_vpe_l2_table(int cpu, u32 id)
	2769	+{
	2770	+ void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
	2771	+ unsigned int psz, esz, idx, npg, gpsz;
	2772	+ u64 val;
	2773	+ struct page *page;
	2774	+ __le64 *table;
	2775	+
	2776	+ if (!gic_rdists->has_rvpeid)
	2777	+ return true;
	2778	+
	2779	+ /* Skip non-present CPUs */
	2780	+ if (!base)
	2781	+ return true;
	2782	+
	2783	+ val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
	2784	+
	2785	+ esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
	2786	+ gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
	2787	+ npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
	2788	+
	2789	+ switch (gpsz) {
	2790	+ default:
	2791	+ WARN_ON(1);
	2792	+ fallthrough;
	2793	+ case GIC_PAGE_SIZE_4K:
	2794	+ psz = SZ_4K;
	2795	+ break;
	2796	+ case GIC_PAGE_SIZE_16K:
	2797	+ psz = SZ_16K;
	2798	+ break;
	2799	+ case GIC_PAGE_SIZE_64K:
	2800	+ psz = SZ_64K;
	2801	+ break;
	2802	+ }
	2803	+
	2804	+ /* Don't allow vpe_id that exceeds single, flat table limit */
	2805	+ if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
	2806	+ return (id < (npg * psz / (esz * SZ_8)));
	2807	+
	2808	+ /* Compute 1st level table index & check if that exceeds table limit */
	2809	+ idx = id >> ilog2(psz / (esz * SZ_8));
	2810	+ if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
	2811	+ return false;
	2812	+
	2813	+ table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
	2814	+
	2815	+ /* Allocate memory for 2nd level table */
	2816	+ if (!table[idx]) {
	2817	+ page = alloc_pages(GFP_KERNEL \| __GFP_ZERO, get_order(psz));
	2818	+ if (!page)
	2819	+ return false;
	2820	+
	2821	+ /* Flush Lvl2 table to PoC if hw doesn't support coherency */
	2822	+ if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
	2823	+ gic_flush_dcache_to_poc(page_address(page), psz);
	2824	+
	2825	+ table[idx] = cpu_to_le64(page_to_phys(page) \| GITS_BASER_VALID);
	2826	+
	2827	+ /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
	2828	+ if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
	2829	+ gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
	2830	+
	2831	+ /* Ensure updated table contents are visible to RD hardware */
	2832	+ dsb(sy);
	2833	+ }
	2834	+
	2835	+ return true;
	2836	+}
	2837	+
	2838	+static int allocate_vpe_l1_table(void)
	2839	+{
	2840	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	2841	+ u64 val, gpsz, npg, pa;
	2842	+ unsigned int psz = SZ_64K;
	2843	+ unsigned int np, epp, esz;
	2844	+ struct page *page;
	2845	+
	2846	+ if (!gic_rdists->has_rvpeid)
	2847	+ return 0;
	2848	+
	2849	+ /*
	2850	+ * if VPENDBASER.Valid is set, disable any previously programmed
	2851	+ * VPE by setting PendingLast while clearing Valid. This has the
	2852	+ * effect of making sure no doorbell will be generated and we can
	2853	+ * then safely clear VPROPBASER.Valid.
	2854	+ */
	2855	+ if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
	2856	+ gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
	2857	+ vlpi_base + GICR_VPENDBASER);
	2858	+
	2859	+ /*
	2860	+ * If we can inherit the configuration from another RD, let's do
	2861	+ * so. Otherwise, we have to go through the allocation process. We
	2862	+ * assume that all RDs have the exact same requirements, as
	2863	+ * nothing will work otherwise.
	2864	+ */
	2865	+ val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
	2866	+ if (val & GICR_VPROPBASER_4_1_VALID)
	2867	+ goto out;
	2868	+
	2869	+ gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
	2870	+ if (!gic_data_rdist()->vpe_table_mask)
	2871	+ return -ENOMEM;
	2872	+
	2873	+ val = inherit_vpe_l1_table_from_its();
	2874	+ if (val & GICR_VPROPBASER_4_1_VALID)
	2875	+ goto out;
	2876	+
	2877	+ /* First probe the page size */
	2878	+ val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
	2879	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	2880	+ val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
	2881	+ gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
	2882	+ esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
	2883	+
	2884	+ switch (gpsz) {
	2885	+ default:
	2886	+ gpsz = GIC_PAGE_SIZE_4K;
	2887	+ fallthrough;
	2888	+ case GIC_PAGE_SIZE_4K:
	2889	+ psz = SZ_4K;
	2890	+ break;
	2891	+ case GIC_PAGE_SIZE_16K:
	2892	+ psz = SZ_16K;
	2893	+ break;
	2894	+ case GIC_PAGE_SIZE_64K:
	2895	+ psz = SZ_64K;
	2896	+ break;
	2897	+ }
	2898	+
	2899	+ /*
	2900	+ * Start populating the register from scratch, including RO fields
	2901	+ * (which we want to print in debug cases...)
	2902	+ */
	2903	+ val = 0;
	2904	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
	2905	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
	2906	+
	2907	+ /* How many entries per GIC page? */
	2908	+ esz++;
	2909	+ epp = psz / (esz * SZ_8);
	2910	+
	2911	+ /*
	2912	+ * If we need more than just a single L1 page, flag the table
	2913	+ * as indirect and compute the number of required L1 pages.
	2914	+ */
	2915	+ if (epp < ITS_MAX_VPEID) {
	2916	+ int nl2;
	2917	+
	2918	+ val \|= GICR_VPROPBASER_4_1_INDIRECT;
	2919	+
	2920	+ /* Number of L2 pages required to cover the VPEID space */
	2921	+ nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
	2922	+
	2923	+ /* Number of L1 pages to point to the L2 pages */
	2924	+ npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
	2925	+ } else {
	2926	+ npg = 1;
	2927	+ }
	2928	+
	2929	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
	2930	+
	2931	+ /* Right, that's the number of CPU pages we need for L1 */
	2932	+ np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
	2933	+
	2934	+ pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
	2935	+ np, npg, psz, epp, esz);
	2936	+ page = alloc_pages(GFP_ATOMIC \| __GFP_ZERO, get_order(np * PAGE_SIZE));
	2937	+ if (!page)
	2938	+ return -ENOMEM;
	2939	+
	2940	+ gic_data_rdist()->vpe_l1_base = page_address(page);
	2941	+ pa = virt_to_phys(page_address(page));
	2942	+ WARN_ON(!IS_ALIGNED(pa, psz));
	2943	+
	2944	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
	2945	+ val \|= GICR_VPROPBASER_RaWb;
	2946	+ val \|= GICR_VPROPBASER_InnerShareable;
	2947	+ val \|= GICR_VPROPBASER_4_1_Z;
	2948	+ val \|= GICR_VPROPBASER_4_1_VALID;
	2949	+
	2950	+out:
	2951	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	2952	+ cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
	2953	+
	2954	+ pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
	2955	+ smp_processor_id(), val,
	2956	+ cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
1958	2957
1959	2958	return 0;
1960	2959	}
..	..	@@ -1977,14 +2976,13 @@
1977	2976	static struct page *its_allocate_pending_table(gfp_t gfp_flags)
1978	2977	{
1979	2978	struct page *pend_page;
1980		- /*
1981		- * The pending pages have to be at least 64kB aligned,
1982		- * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1983		- */
1984		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2979	+
	2980	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2981	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2982	+ of_machine_is_compatible("rockchip,rk3566"))
1985	2983	gfp_flags \|= GFP_DMA32;
1986	2984	pend_page = alloc_pages(gfp_flags \| __GFP_ZERO,
1987		- get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
	2985	+ get_order(LPI_PENDBASE_SZ));
1988	2986	if (!pend_page)
1989	2987	return NULL;
1990	2988
..	..	@@ -1996,22 +2994,73 @@
1996	2994
1997	2995	static void its_free_pending_table(struct page *pt)
1998	2996	{
1999		- free_pages((unsigned long)page_address(pt),
2000		- get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
	2997	+ free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2001	2998	}
2002	2999
2003		-static u64 its_clear_vpend_valid(void __iomem *vlpi_base)
	3000	+/*
	3001	+ * Booting with kdump and LPIs enabled is generally fine. Any other
	3002	+ * case is wrong in the absence of firmware/EFI support.
	3003	+ */
	3004	+static bool enabled_lpis_allowed(void)
	3005	+{
	3006	+ phys_addr_t addr;
	3007	+ u64 val;
	3008	+
	3009	+ /* Check whether the property table is in a reserved region */
	3010	+ val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
	3011	+ addr = val & GENMASK_ULL(51, 12);
	3012	+
	3013	+ return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
	3014	+}
	3015	+
	3016	+static int __init allocate_lpi_tables(void)
	3017	+{
	3018	+ u64 val;
	3019	+ int err, cpu;
	3020	+
	3021	+ /*
	3022	+ * If LPIs are enabled while we run this from the boot CPU,
	3023	+ * flag the RD tables as pre-allocated if the stars do align.
	3024	+ */
	3025	+ val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
	3026	+ if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
	3027	+ gic_rdists->flags \|= (RDIST_FLAGS_RD_TABLES_PREALLOCATED \|
	3028	+ RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
	3029	+ pr_info("GICv3: Using preallocated redistributor tables\n");
	3030	+ }
	3031	+
	3032	+ err = its_setup_lpi_prop_table();
	3033	+ if (err)
	3034	+ return err;
	3035	+
	3036	+ /*
	3037	+ * We allocate all the pending tables anyway, as we may have a
	3038	+ * mix of RDs that have had LPIs enabled, and some that
	3039	+ * don't. We'll free the unused ones as each CPU comes online.
	3040	+ */
	3041	+ for_each_possible_cpu(cpu) {
	3042	+ struct page *pend_page;
	3043	+
	3044	+ pend_page = its_allocate_pending_table(GFP_NOWAIT);
	3045	+ if (!pend_page) {
	3046	+ pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
	3047	+ return -ENOMEM;
	3048	+ }
	3049	+
	3050	+ gic_data_rdist_cpu(cpu)->pend_page = pend_page;
	3051	+ }
	3052	+
	3053	+ return 0;
	3054	+}
	3055	+
	3056	+static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
2004	3057	{
2005	3058	u32 count = 1000000; /* 1s! */
2006	3059	bool clean;
2007	3060	u64 val;
2008	3061
2009		- val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2010		- val &= ~GICR_VPENDBASER_Valid;
2011		- gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2012		-
2013	3062	do {
2014		- val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
	3063	+ val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2015	3064	clean = !(val & GICR_VPENDBASER_Dirty);
2016	3065	if (!clean) {
2017	3066	count--;
..	..	@@ -2020,6 +3069,27 @@
2020	3069	}
2021	3070	} while (!clean && count);
2022	3071
	3072	+ if (unlikely(!clean))
	3073	+ pr_err_ratelimited("ITS virtual pending table not cleaning\n");
	3074	+
	3075	+ return val;
	3076	+}
	3077	+
	3078	+static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
	3079	+{
	3080	+ u64 val;
	3081	+
	3082	+ /* Make sure we wait until the RD is done with the initial scan */
	3083	+ val = read_vpend_dirty_clear(vlpi_base);
	3084	+ val &= ~GICR_VPENDBASER_Valid;
	3085	+ val &= ~clr;
	3086	+ val \|= set;
	3087	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3088	+
	3089	+ val = read_vpend_dirty_clear(vlpi_base);
	3090	+ if (unlikely(val & GICR_VPENDBASER_Dirty))
	3091	+ val \|= GICR_VPENDBASER_PendingLast;
	3092	+
2023	3093	return val;
2024	3094	}
2025	3095
..	..	@@ -2027,28 +3097,40 @@
2027	3097	{
2028	3098	void __iomem *rbase = gic_data_rdist_rd_base();
2029	3099	struct page *pend_page;
	3100	+ phys_addr_t paddr;
2030	3101	u64 val, tmp;
2031	3102
2032		- /* If we didn't allocate the pending table yet, do it now */
2033		- pend_page = gic_data_rdist()->pend_page;
2034		- if (!pend_page) {
2035		- phys_addr_t paddr;
	3103	+ if (gic_data_rdist()->lpi_enabled)
	3104	+ return;
2036	3105
2037		- pend_page = its_allocate_pending_table(GFP_NOWAIT);
2038		- if (!pend_page) {
2039		- pr_err("Failed to allocate PENDBASE for CPU%d\n",
2040		- smp_processor_id());
2041		- return;
2042		- }
	3106	+ val = readl_relaxed(rbase + GICR_CTLR);
	3107	+ if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
	3108	+ (val & GICR_CTLR_ENABLE_LPIS)) {
	3109	+ /*
	3110	+ * Check that we get the same property table on all
	3111	+ * RDs. If we don't, this is hopeless.
	3112	+ */
	3113	+ paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
	3114	+ paddr &= GENMASK_ULL(51, 12);
	3115	+ if (WARN_ON(gic_rdists->prop_table_pa != paddr))
	3116	+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2043	3117
2044		- paddr = page_to_phys(pend_page);
2045		- pr_info("CPU%d: using LPI pending table @%pa\n",
2046		- smp_processor_id(), &paddr);
2047		- gic_data_rdist()->pend_page = pend_page;
	3118	+ paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
	3119	+ paddr &= GENMASK_ULL(51, 16);
	3120	+
	3121	+ WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
	3122	+ its_free_pending_table(gic_data_rdist()->pend_page);
	3123	+ gic_data_rdist()->pend_page = NULL;
	3124	+
	3125	+ goto out;
2048	3126	}
2049	3127
	3128	+ pend_page = gic_data_rdist()->pend_page;
	3129	+ paddr = page_to_phys(pend_page);
	3130	+ WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
	3131	+
2050	3132	/* set PROPBASE */
2051		- val = (page_to_phys(gic_rdists->prop_page) \|
	3133	+ val = (gic_rdists->prop_table_pa \|
2052	3134	GICR_PROPBASER_InnerShareable \|
2053	3135	GICR_PROPBASER_RaWaWb \|
2054	3136	((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
..	..	@@ -2056,7 +3138,11 @@
2056	3138	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
2057	3139	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
2058	3140
2059		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3141	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	3142	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	3143	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3144	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	3145	+ of_machine_is_compatible("rockchip,rk3588")))
2060	3146	tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
2061	3147
2062	3148	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
..	..	@@ -2083,7 +3169,11 @@
2083	3169	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
2084	3170	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
2085	3171
2086		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3172	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	3173	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	3174	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3175	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	3176	+ of_machine_is_compatible("rockchip,rk3588")))
2087	3177	tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
2088	3178
2089	3179	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
..	..	@@ -2102,12 +3192,12 @@
2102	3192	val \|= GICR_CTLR_ENABLE_LPIS;
2103	3193	writel_relaxed(val, rbase + GICR_CTLR);
2104	3194
2105		- if (gic_rdists->has_vlpis) {
	3195	+ if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
2106	3196	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2107	3197
2108	3198	/*
2109	3199	* It's possible for CPU to receive VLPIs before it is
2110		- * sheduled as a vPE, especially for the first CPU, and the
	3200	+ * scheduled as a vPE, especially for the first CPU, and the
2111	3201	* VLPI with INTID larger than 2^(IDbits+1) will be considered
2112	3202	* as out of range and dropped by GIC.
2113	3203	* So we initialize IDbits to known value to avoid VLPI drop.
..	..	@@ -2115,19 +3205,34 @@
2115	3205	val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2116	3206	pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
2117	3207	smp_processor_id(), val);
2118		- gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	3208	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2119	3209
2120	3210	/*
2121	3211	* Also clear Valid bit of GICR_VPENDBASER, in case some
2122	3212	* ancient programming gets left in and has possibility of
2123	3213	* corrupting memory.
2124	3214	*/
2125		- val = its_clear_vpend_valid(vlpi_base);
2126		- WARN_ON(val & GICR_VPENDBASER_Dirty);
	3215	+ val = its_clear_vpend_valid(vlpi_base, 0, 0);
	3216	+ }
	3217	+
	3218	+ if (allocate_vpe_l1_table()) {
	3219	+ /*
	3220	+ * If the allocation has failed, we're in massive trouble.
	3221	+ * Disable direct injection, and pray that no VM was
	3222	+ * already running...
	3223	+ */
	3224	+ gic_rdists->has_rvpeid = false;
	3225	+ gic_rdists->has_vlpis = false;
2127	3226	}
2128	3227
2129	3228	/* Make sure the GIC has seen the above */
2130	3229	dsb(sy);
	3230	+out:
	3231	+ gic_data_rdist()->lpi_enabled = true;
	3232	+ pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
	3233	+ smp_processor_id(),
	3234	+ gic_data_rdist()->pend_page ? "allocated" : "reserved",
	3235	+ &paddr);
2131	3236	}
2132	3237
2133	3238	static void its_cpu_init_collection(struct its_node *its)
..	..	@@ -2212,7 +3317,8 @@
2212	3317	return NULL;
2213	3318	}
2214	3319
2215		-static bool its_alloc_table_entry(struct its_baser *baser, u32 id)
	3320	+static bool its_alloc_table_entry(struct its_node *its,
	3321	+ struct its_baser *baser, u32 id)
2216	3322	{
2217	3323	struct page *page;
2218	3324	u32 esz, idx;
..	..	@@ -2234,9 +3340,12 @@
2234	3340	if (!table[idx]) {
2235	3341	gfp_t gfp_flags = GFP_KERNEL \| __GFP_ZERO;
2236	3342
2237		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3343	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3344	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3345	+ of_machine_is_compatible("rockchip,rk3566"))
2238	3346	gfp_flags \|= GFP_DMA32;
2239		- page = alloc_pages(gfp_flags, get_order(baser->psz));
	3347	+ page = alloc_pages_node(its->numa_node, gfp_flags,
	3348	+ get_order(baser->psz));
2240	3349	if (!page)
2241	3350	return false;
2242	3351
..	..	@@ -2265,14 +3374,15 @@
2265	3374
2266	3375	/* Don't allow device id that exceeds ITS hardware limit */
2267	3376	if (!baser)
2268		- return (ilog2(dev_id) < its->device_ids);
	3377	+ return (ilog2(dev_id) < device_ids(its));
2269	3378
2270		- return its_alloc_table_entry(baser, dev_id);
	3379	+ return its_alloc_table_entry(its, baser, dev_id);
2271	3380	}
2272	3381
2273	3382	static bool its_alloc_vpe_table(u32 vpe_id)
2274	3383	{
2275	3384	struct its_node *its;
	3385	+ int cpu;
2276	3386
2277	3387	/*
2278	3388	* Make sure the L2 tables are allocated on all v4 ITSs. We
..	..	@@ -2284,14 +3394,27 @@
2284	3394	list_for_each_entry(its, &its_nodes, entry) {
2285	3395	struct its_baser *baser;
2286	3396
2287		- if (!its->is_v4)
	3397	+ if (!is_v4(its))
2288	3398	continue;
2289	3399
2290	3400	baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
2291	3401	if (!baser)
2292	3402	return false;
2293	3403
2294		- if (!its_alloc_table_entry(baser, vpe_id))
	3404	+ if (!its_alloc_table_entry(its, baser, vpe_id))
	3405	+ return false;
	3406	+ }
	3407	+
	3408	+ /* Non v4.1? No need to iterate RDs and go back early. */
	3409	+ if (!gic_rdists->has_rvpeid)
	3410	+ return true;
	3411	+
	3412	+ /*
	3413	+ * Make sure the L2 tables are allocated for all copies of
	3414	+ * the L1 table on all v4.1 RDs.
	3415	+ */
	3416	+ for_each_possible_cpu(cpu) {
	3417	+ if (!allocate_vpe_l2_table(cpu, vpe_id))
2295	3418	return false;
2296	3419	}
2297	3420
..	..	@@ -2324,12 +3447,18 @@
2324	3447	* sized as a power of two (and you need at least one bit...).
2325	3448	*/
2326	3449	nr_ites = max(2, nvecs);
2327		- sz = nr_ites * its->ite_size;
	3450	+ sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
2328	3451	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
2329	3452	gfp_flags = GFP_KERNEL;
2330		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3453	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3454	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3455	+ of_machine_is_compatible("rockchip,rk3566")) {
2331	3456	gfp_flags \|= GFP_DMA32;
2332		- itt = (void *)__get_free_pages(gfp_flags, get_order(sz));
	3457	+ itt = (void *)__get_free_pages(gfp_flags, get_order(sz));
	3458	+ } else {
	3459	+ itt = kzalloc_node(sz, gfp_flags, its->numa_node);
	3460	+ }
	3461	+
2333	3462	if (alloc_lpis) {
2334	3463	lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
2335	3464	if (lpi_map)
..	..	@@ -2343,7 +3472,14 @@
2343	3472
2344	3473	if (!dev \|\| !itt \|\| !col_map \|\| (!lpi_map && alloc_lpis)) {
2345	3474	kfree(dev);
2346		- free_pages((unsigned long)itt, get_order(sz));
	3475	+
	3476	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3477	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3478	+ of_machine_is_compatible("rockchip,rk3566"))
	3479	+ free_pages((unsigned long)itt, get_order(sz));
	3480	+ else
	3481	+ kfree(itt);
	3482	+
2347	3483	kfree(lpi_map);
2348	3484	kfree(col_map);
2349	3485	return NULL;
..	..	@@ -2359,7 +3495,7 @@
2359	3495	dev->event_map.col_map = col_map;
2360	3496	dev->event_map.lpi_base = lpi_base;
2361	3497	dev->event_map.nr_lpis = nr_lpis;
2362		- mutex_init(&dev->event_map.vlpi_lock);
	3498	+ raw_spin_lock_init(&dev->event_map.vlpi_lock);
2363	3499	dev->device_id = dev_id;
2364	3500	INIT_LIST_HEAD(&dev->entry);
2365	3501
..	..	@@ -2380,7 +3516,15 @@
2380	3516	raw_spin_lock_irqsave(&its_dev->its->lock, flags);
2381	3517	list_del(&its_dev->entry);
2382	3518	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
2383		- free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));
	3519	+ kfree(its_dev->event_map.col_map);
	3520	+
	3521	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3522	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3523	+ of_machine_is_compatible("rockchip,rk3566"))
	3524	+ free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));
	3525	+ else
	3526	+ kfree(its_dev->itt);
	3527	+
2384	3528	kfree(its_dev);
2385	3529	}
2386	3530
..	..	@@ -2388,6 +3532,7 @@
2388	3532	{
2389	3533	int idx;
2390	3534
	3535	+ /* Find a free LPI region in lpi_map and allocate them. */
2391	3536	idx = bitmap_find_free_region(dev->event_map.lpi_map,
2392	3537	dev->event_map.nr_lpis,
2393	3538	get_count_order(nvecs));
..	..	@@ -2395,7 +3540,6 @@
2395	3540	return -ENOSPC;
2396	3541
2397	3542	*hwirq = dev->event_map.lpi_base + idx;
2398		- set_bit(idx, dev->event_map.lpi_map);
2399	3543
2400	3544	return 0;
2401	3545	}
..	..	@@ -2410,7 +3554,7 @@
2410	3554	int err = 0;
2411	3555
2412	3556	/*
2413		- * We ignore "dev" entierely, and rely on the dev_id that has
	3557	+ * We ignore "dev" entirely, and rely on the dev_id that has
2414	3558	* been passed via the scratchpad. This limits this domain's
2415	3559	* usefulness to upper layers that definitely know that they
2416	3560	* are built on top of the ITS.
..	..	@@ -2489,12 +3633,17 @@
2489	3633	{
2490	3634	msi_alloc_info_t *info = args;
2491	3635	struct its_device *its_dev = info->scratchpad[0].ptr;
	3636	+ struct its_node *its = its_dev->its;
2492	3637	struct irq_data *irqd;
2493	3638	irq_hw_number_t hwirq;
2494	3639	int err;
2495	3640	int i;
2496	3641
2497	3642	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
	3643	+ if (err)
	3644	+ return err;
	3645	+
	3646	+ err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
2498	3647	if (err)
2499	3648	return err;
2500	3649
..	..	@@ -2521,22 +3670,13 @@
2521	3670	{
2522	3671	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2523	3672	u32 event = its_get_event_id(d);
2524		- const struct cpumask *cpu_mask = cpu_online_mask;
2525	3673	int cpu;
2526	3674
2527		- /* get the cpu_mask of local node */
2528		- if (its_dev->its->numa_node >= 0)
2529		- cpu_mask = cpumask_of_node(its_dev->its->numa_node);
	3675	+ cpu = its_select_cpu(d, cpu_online_mask);
	3676	+ if (cpu < 0 \|\| cpu >= nr_cpu_ids)
	3677	+ return -EINVAL;
2530	3678
2531		- /* Bind the LPI to the first possible CPU */
2532		- cpu = cpumask_first_and(cpu_mask, cpu_online_mask);
2533		- if (cpu >= nr_cpu_ids) {
2534		- if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)
2535		- return -EINVAL;
2536		-
2537		- cpu = cpumask_first(cpu_online_mask);
2538		- }
2539		-
	3679	+ its_inc_lpi_count(d, cpu);
2540	3680	its_dev->event_map.col_map[event] = cpu;
2541	3681	irq_data_update_effective_affinity(d, cpumask_of(cpu));
2542	3682
..	..	@@ -2551,6 +3691,7 @@
2551	3691	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2552	3692	u32 event = its_get_event_id(d);
2553	3693
	3694	+ its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
2554	3695	/* Stop the delivery of interrupts */
2555	3696	its_send_discard(its_dev, event);
2556	3697	}
..	..	@@ -2578,7 +3719,7 @@
2578	3719
2579	3720	/*
2580	3721	* If all interrupts have been freed, start mopping the
2581		- * floor. This is conditionned on the device not being shared.
	3722	+ * floor. This is conditioned on the device not being shared.
2582	3723	*/
2583	3724	if (!its_dev->shared &&
2584	3725	bitmap_empty(its_dev->event_map.lpi_map,
..	..	@@ -2586,7 +3727,6 @@
2586	3727	its_lpi_free(its_dev->event_map.lpi_map,
2587	3728	its_dev->event_map.lpi_base,
2588	3729	its_dev->event_map.nr_lpis);
2589		- kfree(its_dev->event_map.col_map);
2590	3730
2591	3731	/* Unmap device/itt */
2592	3732	its_send_mapd(its_dev, 0);
..	..	@@ -2608,7 +3748,7 @@
2608	3748	/*
2609	3749	* This is insane.
2610	3750	*
2611		- * If a GICv4 doesn't implement Direct LPIs (which is extremely
	3751	+ * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
2612	3752	* likely), the only way to perform an invalidate is to use a fake
2613	3753	* device to issue an INV command, implying that the LPI has first
2614	3754	* been mapped to some event on that device. Since this is not exactly
..	..	@@ -2616,9 +3756,20 @@
2616	3756	* only issue an UNMAP if we're short on available slots.
2617	3757	*
2618	3758	* Broken by design(tm).
	3759	+ *
	3760	+ * GICv4.1, on the other hand, mandates that we're able to invalidate
	3761	+ * by writing to a MMIO register. It doesn't implement the whole of
	3762	+ * DirectLPI, but that's good enough. And most of the time, we don't
	3763	+ * even have to invalidate anything, as the redistributor can be told
	3764	+ * whether to generate a doorbell or not (we thus leave it enabled,
	3765	+ * always).
2619	3766	*/
2620	3767	static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
2621	3768	{
	3769	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3770	+ if (gic_rdists->has_rvpeid)
	3771	+ return;
	3772	+
2622	3773	/* Already unmapped? */
2623	3774	if (vpe->vpe_proxy_event == -1)
2624	3775	return;
..	..	@@ -2641,6 +3792,10 @@
2641	3792
2642	3793	static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
2643	3794	{
	3795	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3796	+ if (gic_rdists->has_rvpeid)
	3797	+ return;
	3798	+
2644	3799	if (!gic_rdists->has_direct_lpi) {
2645	3800	unsigned long flags;
2646	3801
..	..	@@ -2652,6 +3807,10 @@
2652	3807
2653	3808	static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
2654	3809	{
	3810	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3811	+ if (gic_rdists->has_rvpeid)
	3812	+ return;
	3813	+
2655	3814	/* Already mapped? */
2656	3815	if (vpe->vpe_proxy_event != -1)
2657	3816	return;
..	..	@@ -2674,13 +3833,16 @@
2674	3833	unsigned long flags;
2675	3834	struct its_collection *target_col;
2676	3835
	3836	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3837	+ if (gic_rdists->has_rvpeid)
	3838	+ return;
	3839	+
2677	3840	if (gic_rdists->has_direct_lpi) {
2678	3841	void __iomem *rdbase;
2679	3842
2680	3843	rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
2681	3844	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2682		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2683		- cpu_relax();
	3845	+ wait_for_syncr(rdbase);
2684	3846
2685	3847	return;
2686	3848	}
..	..	@@ -2701,25 +3863,58 @@
2701	3863	bool force)
2702	3864	{
2703	3865	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2704		- int cpu = cpumask_first(mask_val);
	3866	+ int from, cpu = cpumask_first(mask_val);
	3867	+ unsigned long flags;
2705	3868
2706	3869	/*
2707	3870	* Changing affinity is mega expensive, so let's be as lazy as
2708	3871	* we can and only do it if we really have to. Also, if mapped
2709	3872	* into the proxy device, we need to move the doorbell
2710	3873	* interrupt to its new location.
	3874	+ *
	3875	+ * Another thing is that changing the affinity of a vPE affects
	3876	+ * other interrupts such as all the vLPIs that are routed to
	3877	+ * this vPE. This means that the irq_desc lock is not enough to
	3878	+ * protect us, and that we must ensure nobody samples vpe->col_idx
	3879	+ * during the update, hence the lock below which must also be
	3880	+ * taken on any vLPI handling path that evaluates vpe->col_idx.
2711	3881	*/
2712		- if (vpe->col_idx != cpu) {
2713		- int from = vpe->col_idx;
	3882	+ from = vpe_to_cpuid_lock(vpe, &flags);
	3883	+ if (from == cpu)
	3884	+ goto out;
2714	3885
2715		- vpe->col_idx = cpu;
2716		- its_send_vmovp(vpe);
2717		- its_vpe_db_proxy_move(vpe, from, cpu);
2718		- }
	3886	+ vpe->col_idx = cpu;
2719	3887
	3888	+ /*
	3889	+ * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
	3890	+ * is sharing its VPE table with the current one.
	3891	+ */
	3892	+ if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
	3893	+ cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
	3894	+ goto out;
	3895	+
	3896	+ its_send_vmovp(vpe);
	3897	+ its_vpe_db_proxy_move(vpe, from, cpu);
	3898	+
	3899	+out:
2720	3900	irq_data_update_effective_affinity(d, cpumask_of(cpu));
	3901	+ vpe_to_cpuid_unlock(vpe, flags);
2721	3902
2722	3903	return IRQ_SET_MASK_OK_DONE;
	3904	+}
	3905	+
	3906	+static void its_wait_vpt_parse_complete(void)
	3907	+{
	3908	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	3909	+ u64 val;
	3910	+
	3911	+ if (!gic_rdists->has_vpend_valid_dirty)
	3912	+ return;
	3913	+
	3914	+ WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
	3915	+ val,
	3916	+ !(val & GICR_VPENDBASER_Dirty),
	3917	+ 10, 500));
2723	3918	}
2724	3919
2725	3920	static void its_vpe_schedule(struct its_vpe *vpe)
..	..	@@ -2733,12 +3928,12 @@
2733	3928	val \|= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2734	3929	val \|= GICR_VPROPBASER_RaWb;
2735	3930	val \|= GICR_VPROPBASER_InnerShareable;
2736		- gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	3931	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2737	3932
2738	3933	val = virt_to_phys(page_address(vpe->vpt_page)) &
2739	3934	GENMASK_ULL(51, 16);
2740	3935	val \|= GICR_VPENDBASER_RaWaWb;
2741		- val \|= GICR_VPENDBASER_NonShareable;
	3936	+ val \|= GICR_VPENDBASER_InnerShareable;
2742	3937	/*
2743	3938	* There is no good way of finding out if the pending table is
2744	3939	* empty as we can race against the doorbell interrupt very
..	..	@@ -2751,7 +3946,7 @@
2751	3946	val \|= GICR_VPENDBASER_PendingLast;
2752	3947	val \|= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
2753	3948	val \|= GICR_VPENDBASER_Valid;
2754		- gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3949	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2755	3950	}
2756	3951
2757	3952	static void its_vpe_deschedule(struct its_vpe *vpe)
..	..	@@ -2759,16 +3954,10 @@
2759	3954	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2760	3955	u64 val;
2761	3956
2762		- val = its_clear_vpend_valid(vlpi_base);
	3957	+ val = its_clear_vpend_valid(vlpi_base, 0, 0);
2763	3958
2764		- if (unlikely(val & GICR_VPENDBASER_Dirty)) {
2765		- pr_err_ratelimited("ITS virtual pending table not cleaning\n");
2766		- vpe->idai = false;
2767		- vpe->pending_last = true;
2768		- } else {
2769		- vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
2770		- vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2771		- }
	3959	+ vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
	3960	+ vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2772	3961	}
2773	3962
2774	3963	static void its_vpe_invall(struct its_vpe *vpe)
..	..	@@ -2776,7 +3965,7 @@
2776	3965	struct its_node *its;
2777	3966
2778	3967	list_for_each_entry(its, &its_nodes, entry) {
2779		- if (!its->is_v4)
	3968	+ if (!is_v4(its))
2780	3969	continue;
2781	3970
2782	3971	if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
..	..	@@ -2805,6 +3994,10 @@
2805	3994	its_vpe_deschedule(vpe);
2806	3995	return 0;
2807	3996
	3997	+ case COMMIT_VPE:
	3998	+ its_wait_vpt_parse_complete();
	3999	+ return 0;
	4000	+
2808	4001	case INVALL_VPE:
2809	4002	its_vpe_invall(vpe);
2810	4003	return 0;
..	..	@@ -2831,16 +4024,10 @@
2831	4024	{
2832	4025	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2833	4026
2834		- if (gic_rdists->has_direct_lpi) {
2835		- void __iomem *rdbase;
2836		-
2837		- rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
2838		- gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);
2839		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2840		- cpu_relax();
2841		- } else {
	4027	+ if (gic_rdists->has_direct_lpi)
	4028	+ __direct_lpi_inv(d, d->parent_data->hwirq);
	4029	+ else
2842	4030	its_vpe_send_cmd(vpe, its_send_inv);
2843		- }
2844	4031	}
2845	4032
2846	4033	static void its_vpe_mask_irq(struct irq_data *d)
..	..	@@ -2879,8 +4066,7 @@
2879	4066	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
2880	4067	} else {
2881	4068	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2882		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2883		- cpu_relax();
	4069	+ wait_for_syncr(rdbase);
2884	4070	}
2885	4071	} else {
2886	4072	if (state)
..	..	@@ -2906,6 +4092,375 @@
2906	4092	.irq_retrigger = its_vpe_retrigger,
2907	4093	.irq_set_irqchip_state = its_vpe_set_irqchip_state,
2908	4094	.irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
	4095	+};
	4096	+
	4097	+static struct its_node *find_4_1_its(void)
	4098	+{
	4099	+ static struct its_node *its = NULL;
	4100	+
	4101	+ if (!its) {
	4102	+ list_for_each_entry(its, &its_nodes, entry) {
	4103	+ if (is_v4_1(its))
	4104	+ return its;
	4105	+ }
	4106	+
	4107	+ /* Oops? */
	4108	+ its = NULL;
	4109	+ }
	4110	+
	4111	+ return its;
	4112	+}
	4113	+
	4114	+static void its_vpe_4_1_send_inv(struct irq_data *d)
	4115	+{
	4116	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4117	+ struct its_node *its;
	4118	+
	4119	+ /*
	4120	+ * GICv4.1 wants doorbells to be invalidated using the
	4121	+ * INVDB command in order to be broadcast to all RDs. Send
	4122	+ * it to the first valid ITS, and let the HW do its magic.
	4123	+ */
	4124	+ its = find_4_1_its();
	4125	+ if (its)
	4126	+ its_send_invdb(its, vpe);
	4127	+}
	4128	+
	4129	+static void its_vpe_4_1_mask_irq(struct irq_data *d)
	4130	+{
	4131	+ lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
	4132	+ its_vpe_4_1_send_inv(d);
	4133	+}
	4134	+
	4135	+static void its_vpe_4_1_unmask_irq(struct irq_data *d)
	4136	+{
	4137	+ lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
	4138	+ its_vpe_4_1_send_inv(d);
	4139	+}
	4140	+
	4141	+static void its_vpe_4_1_schedule(struct its_vpe *vpe,
	4142	+ struct its_cmd_info *info)
	4143	+{
	4144	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	4145	+ u64 val = 0;
	4146	+
	4147	+ /* Schedule the VPE */
	4148	+ val \|= GICR_VPENDBASER_Valid;
	4149	+ val \|= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
	4150	+ val \|= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
	4151	+ val \|= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
	4152	+
	4153	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	4154	+}
	4155	+
	4156	+static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
	4157	+ struct its_cmd_info *info)
	4158	+{
	4159	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	4160	+ u64 val;
	4161	+
	4162	+ if (info->req_db) {
	4163	+ unsigned long flags;
	4164	+
	4165	+ /*
	4166	+ * vPE is going to block: make the vPE non-resident with
	4167	+ * PendingLast clear and DB set. The GIC guarantees that if
	4168	+ * we read-back PendingLast clear, then a doorbell will be
	4169	+ * delivered when an interrupt comes.
	4170	+ *
	4171	+ * Note the locking to deal with the concurrent update of
	4172	+ * pending_last from the doorbell interrupt handler that can
	4173	+ * run concurrently.
	4174	+ */
	4175	+ raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
	4176	+ val = its_clear_vpend_valid(vlpi_base,
	4177	+ GICR_VPENDBASER_PendingLast,
	4178	+ GICR_VPENDBASER_4_1_DB);
	4179	+ vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
	4180	+ raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
	4181	+ } else {
	4182	+ /*
	4183	+ * We're not blocking, so just make the vPE non-resident
	4184	+ * with PendingLast set, indicating that we'll be back.
	4185	+ */
	4186	+ val = its_clear_vpend_valid(vlpi_base,
	4187	+ 0,
	4188	+ GICR_VPENDBASER_PendingLast);
	4189	+ vpe->pending_last = true;
	4190	+ }
	4191	+}
	4192	+
	4193	+static void its_vpe_4_1_invall(struct its_vpe *vpe)
	4194	+{
	4195	+ void __iomem *rdbase;
	4196	+ unsigned long flags;
	4197	+ u64 val;
	4198	+ int cpu;
	4199	+
	4200	+ val = GICR_INVALLR_V;
	4201	+ val \|= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
	4202	+
	4203	+ /* Target the redistributor this vPE is currently known on */
	4204	+ cpu = vpe_to_cpuid_lock(vpe, &flags);
	4205	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4206	+ rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
	4207	+ gic_write_lpir(val, rdbase + GICR_INVALLR);
	4208	+
	4209	+ wait_for_syncr(rdbase);
	4210	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4211	+ vpe_to_cpuid_unlock(vpe, flags);
	4212	+}
	4213	+
	4214	+static int its_vpe_4_1_set_vcpu_affinity(struct irq_data d, void vcpu_info)
	4215	+{
	4216	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4217	+ struct its_cmd_info *info = vcpu_info;
	4218	+
	4219	+ switch (info->cmd_type) {
	4220	+ case SCHEDULE_VPE:
	4221	+ its_vpe_4_1_schedule(vpe, info);
	4222	+ return 0;
	4223	+
	4224	+ case DESCHEDULE_VPE:
	4225	+ its_vpe_4_1_deschedule(vpe, info);
	4226	+ return 0;
	4227	+
	4228	+ case COMMIT_VPE:
	4229	+ its_wait_vpt_parse_complete();
	4230	+ return 0;
	4231	+
	4232	+ case INVALL_VPE:
	4233	+ its_vpe_4_1_invall(vpe);
	4234	+ return 0;
	4235	+
	4236	+ default:
	4237	+ return -EINVAL;
	4238	+ }
	4239	+}
	4240	+
	4241	+static struct irq_chip its_vpe_4_1_irq_chip = {
	4242	+ .name = "GICv4.1-vpe",
	4243	+ .irq_mask = its_vpe_4_1_mask_irq,
	4244	+ .irq_unmask = its_vpe_4_1_unmask_irq,
	4245	+ .irq_eoi = irq_chip_eoi_parent,
	4246	+ .irq_set_affinity = its_vpe_set_affinity,
	4247	+ .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
	4248	+};
	4249	+
	4250	+static void its_configure_sgi(struct irq_data *d, bool clear)
	4251	+{
	4252	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4253	+ struct its_cmd_desc desc;
	4254	+
	4255	+ desc.its_vsgi_cmd.vpe = vpe;
	4256	+ desc.its_vsgi_cmd.sgi = d->hwirq;
	4257	+ desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
	4258	+ desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
	4259	+ desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
	4260	+ desc.its_vsgi_cmd.clear = clear;
	4261	+
	4262	+ /*
	4263	+ * GICv4.1 allows us to send VSGI commands to any ITS as long as the
	4264	+ * destination VPE is mapped there. Since we map them eagerly at
	4265	+ * activation time, we're pretty sure the first GICv4.1 ITS will do.
	4266	+ */
	4267	+ its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
	4268	+}
	4269	+
	4270	+static void its_sgi_mask_irq(struct irq_data *d)
	4271	+{
	4272	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4273	+
	4274	+ vpe->sgi_config[d->hwirq].enabled = false;
	4275	+ its_configure_sgi(d, false);
	4276	+}
	4277	+
	4278	+static void its_sgi_unmask_irq(struct irq_data *d)
	4279	+{
	4280	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4281	+
	4282	+ vpe->sgi_config[d->hwirq].enabled = true;
	4283	+ its_configure_sgi(d, false);
	4284	+}
	4285	+
	4286	+static int its_sgi_set_affinity(struct irq_data *d,
	4287	+ const struct cpumask *mask_val,
	4288	+ bool force)
	4289	+{
	4290	+ /*
	4291	+ * There is no notion of affinity for virtual SGIs, at least
	4292	+ * not on the host (since they can only be targeting a vPE).
	4293	+ * Tell the kernel we've done whatever it asked for.
	4294	+ */
	4295	+ irq_data_update_effective_affinity(d, mask_val);
	4296	+ return IRQ_SET_MASK_OK;
	4297	+}
	4298	+
	4299	+static int its_sgi_set_irqchip_state(struct irq_data *d,
	4300	+ enum irqchip_irq_state which,
	4301	+ bool state)
	4302	+{
	4303	+ if (which != IRQCHIP_STATE_PENDING)
	4304	+ return -EINVAL;
	4305	+
	4306	+ if (state) {
	4307	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4308	+ struct its_node *its = find_4_1_its();
	4309	+ u64 val;
	4310	+
	4311	+ val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
	4312	+ val \|= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
	4313	+ writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
	4314	+ } else {
	4315	+ its_configure_sgi(d, true);
	4316	+ }
	4317	+
	4318	+ return 0;
	4319	+}
	4320	+
	4321	+static int its_sgi_get_irqchip_state(struct irq_data *d,
	4322	+ enum irqchip_irq_state which, bool *val)
	4323	+{
	4324	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4325	+ void __iomem *base;
	4326	+ unsigned long flags;
	4327	+ u32 count = 1000000; /* 1s! */
	4328	+ u32 status;
	4329	+ int cpu;
	4330	+
	4331	+ if (which != IRQCHIP_STATE_PENDING)
	4332	+ return -EINVAL;
	4333	+
	4334	+ /*
	4335	+ * Locking galore! We can race against two different events:
	4336	+ *
	4337	+ * - Concurrent vPE affinity change: we must make sure it cannot
	4338	+ * happen, or we'll talk to the wrong redistributor. This is
	4339	+ * identical to what happens with vLPIs.
	4340	+ *
	4341	+ * - Concurrent VSGIPENDR access: As it involves accessing two
	4342	+ * MMIO registers, this must be made atomic one way or another.
	4343	+ */
	4344	+ cpu = vpe_to_cpuid_lock(vpe, &flags);
	4345	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4346	+ base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
	4347	+ writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
	4348	+ do {
	4349	+ status = readl_relaxed(base + GICR_VSGIPENDR);
	4350	+ if (!(status & GICR_VSGIPENDR_BUSY))
	4351	+ goto out;
	4352	+
	4353	+ count--;
	4354	+ if (!count) {
	4355	+ pr_err_ratelimited("Unable to get SGI status\n");
	4356	+ goto out;
	4357	+ }
	4358	+ cpu_relax();
	4359	+ udelay(1);
	4360	+ } while (count);
	4361	+
	4362	+out:
	4363	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4364	+ vpe_to_cpuid_unlock(vpe, flags);
	4365	+
	4366	+ if (!count)
	4367	+ return -ENXIO;
	4368	+
	4369	+ *val = !!(status & (1 << d->hwirq));
	4370	+
	4371	+ return 0;
	4372	+}
	4373	+
	4374	+static int its_sgi_set_vcpu_affinity(struct irq_data d, void vcpu_info)
	4375	+{
	4376	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4377	+ struct its_cmd_info *info = vcpu_info;
	4378	+
	4379	+ switch (info->cmd_type) {
	4380	+ case PROP_UPDATE_VSGI:
	4381	+ vpe->sgi_config[d->hwirq].priority = info->priority;
	4382	+ vpe->sgi_config[d->hwirq].group = info->group;
	4383	+ its_configure_sgi(d, false);
	4384	+ return 0;
	4385	+
	4386	+ default:
	4387	+ return -EINVAL;
	4388	+ }
	4389	+}
	4390	+
	4391	+static struct irq_chip its_sgi_irq_chip = {
	4392	+ .name = "GICv4.1-sgi",
	4393	+ .irq_mask = its_sgi_mask_irq,
	4394	+ .irq_unmask = its_sgi_unmask_irq,
	4395	+ .irq_set_affinity = its_sgi_set_affinity,
	4396	+ .irq_set_irqchip_state = its_sgi_set_irqchip_state,
	4397	+ .irq_get_irqchip_state = its_sgi_get_irqchip_state,
	4398	+ .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
	4399	+};
	4400	+
	4401	+static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
	4402	+ unsigned int virq, unsigned int nr_irqs,
	4403	+ void *args)
	4404	+{
	4405	+ struct its_vpe *vpe = args;
	4406	+ int i;
	4407	+
	4408	+ /* Yes, we do want 16 SGIs */
	4409	+ WARN_ON(nr_irqs != 16);
	4410	+
	4411	+ for (i = 0; i < 16; i++) {
	4412	+ vpe->sgi_config[i].priority = 0;
	4413	+ vpe->sgi_config[i].enabled = false;
	4414	+ vpe->sgi_config[i].group = false;
	4415	+
	4416	+ irq_domain_set_hwirq_and_chip(domain, virq + i, i,
	4417	+ &its_sgi_irq_chip, vpe);
	4418	+ irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
	4419	+ }
	4420	+
	4421	+ return 0;
	4422	+}
	4423	+
	4424	+static void its_sgi_irq_domain_free(struct irq_domain *domain,
	4425	+ unsigned int virq,
	4426	+ unsigned int nr_irqs)
	4427	+{
	4428	+ /* Nothing to do */
	4429	+}
	4430	+
	4431	+static int its_sgi_irq_domain_activate(struct irq_domain *domain,
	4432	+ struct irq_data *d, bool reserve)
	4433	+{
	4434	+ /* Write out the initial SGI configuration */
	4435	+ its_configure_sgi(d, false);
	4436	+ return 0;
	4437	+}
	4438	+
	4439	+static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
	4440	+ struct irq_data *d)
	4441	+{
	4442	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4443	+
	4444	+ /*
	4445	+ * The VSGI command is awkward:
	4446	+ *
	4447	+ * - To change the configuration, CLEAR must be set to false,
	4448	+ * leaving the pending bit unchanged.
	4449	+ * - To clear the pending bit, CLEAR must be set to true, leaving
	4450	+ * the configuration unchanged.
	4451	+ *
	4452	+ * You just can't do both at once, hence the two commands below.
	4453	+ */
	4454	+ vpe->sgi_config[d->hwirq].enabled = false;
	4455	+ its_configure_sgi(d, false);
	4456	+ its_configure_sgi(d, true);
	4457	+}
	4458	+
	4459	+static const struct irq_domain_ops its_sgi_domain_ops = {
	4460	+ .alloc = its_sgi_irq_domain_alloc,
	4461	+ .free = its_sgi_irq_domain_free,
	4462	+ .activate = its_sgi_irq_domain_activate,
	4463	+ .deactivate = its_sgi_irq_domain_deactivate,
2909	4464	};
2910	4465
2911	4466	static int its_vpe_id_alloc(void)
..	..	@@ -2941,9 +4496,13 @@
2941	4496	return -ENOMEM;
2942	4497	}
2943	4498
	4499	+ raw_spin_lock_init(&vpe->vpe_lock);
2944	4500	vpe->vpe_id = vpe_id;
2945	4501	vpe->vpt_page = vpt_page;
2946		- vpe->vpe_proxy_event = -1;
	4502	+ if (gic_rdists->has_rvpeid)
	4503	+ atomic_set(&vpe->vmapp_count, 0);
	4504	+ else
	4505	+ vpe->vpe_proxy_event = -1;
2947	4506
2948	4507	return 0;
2949	4508	}
..	..	@@ -2985,6 +4544,7 @@
2985	4544	static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
2986	4545	unsigned int nr_irqs, void *args)
2987	4546	{
	4547	+ struct irq_chip *irqchip = &its_vpe_irq_chip;
2988	4548	struct its_vm *vm = args;
2989	4549	unsigned long *bitmap;
2990	4550	struct page *vprop_page;
..	..	@@ -3012,6 +4572,9 @@
3012	4572	vm->nr_db_lpis = nr_ids;
3013	4573	vm->vprop_page = vprop_page;
3014	4574
	4575	+ if (gic_rdists->has_rvpeid)
	4576	+ irqchip = &its_vpe_4_1_irq_chip;
	4577	+
3015	4578	for (i = 0; i < nr_irqs; i++) {
3016	4579	vm->vpes[i]->vpe_db_lpi = base + i;
3017	4580	err = its_vpe_init(vm->vpes[i]);
..	..	@@ -3022,7 +4585,7 @@
3022	4585	if (err)
3023	4586	break;
3024	4587	irq_domain_set_hwirq_and_chip(domain, virq + i, i,
3025		- &its_vpe_irq_chip, vm->vpes[i]);
	4588	+ irqchip, vm->vpes[i]);
3026	4589	set_bit(i, bitmap);
3027	4590	}
3028	4591
..	..	@@ -3043,15 +4606,19 @@
3043	4606	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3044	4607	struct its_node *its;
3045	4608
3046		- /* If we use the list map, we issue VMAPP on demand... */
3047		- if (its_list_map)
	4609	+ /*
	4610	+ * If we use the list map, we issue VMAPP on demand... Unless
	4611	+ * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
	4612	+ * so that VSGIs can work.
	4613	+ */
	4614	+ if (!gic_requires_eager_mapping())
3048	4615	return 0;
3049	4616
3050	4617	/* Map the VPE to the first possible CPU */
3051	4618	vpe->col_idx = cpumask_first(cpu_online_mask);
3052	4619
3053	4620	list_for_each_entry(its, &its_nodes, entry) {
3054		- if (!its->is_v4)
	4621	+ if (!is_v4(its))
3055	4622	continue;
3056	4623
3057	4624	its_send_vmapp(its, vpe, true);
..	..	@@ -3070,14 +4637,14 @@
3070	4637	struct its_node *its;
3071	4638
3072	4639	/*
3073		- * If we use the list map, we unmap the VPE once no VLPIs are
3074		- * associated with the VM.
	4640	+ * If we use the list map on GICv4.0, we unmap the VPE once no
	4641	+ * VLPIs are associated with the VM.
3075	4642	*/
3076		- if (its_list_map)
	4643	+ if (!gic_requires_eager_mapping())
3077	4644	return;
3078	4645
3079	4646	list_for_each_entry(its, &its_nodes, entry) {
3080		- if (!its->is_v4)
	4647	+ if (!is_v4(its))
3081	4648	continue;
3082	4649
3083	4650	its_send_vmapp(its, vpe, false);
..	..	@@ -3128,8 +4695,9 @@
3128	4695	{
3129	4696	struct its_node *its = data;
3130	4697
3131		- /* erratum 22375: only alloc 8MB table size */
3132		- its->device_ids = 0x14; /* 20 bits, 8MB */
	4698	+ /* erratum 22375: only alloc 8MB table size (20 bits) */
	4699	+ its->typer &= ~GITS_TYPER_DEVBITS;
	4700	+ its->typer \|= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
3133	4701	its->flags \|= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
3134	4702
3135	4703	return true;
..	..	@@ -3149,7 +4717,8 @@
3149	4717	struct its_node *its = data;
3150	4718
3151	4719	/* On QDF2400, the size of the ITE is 16Bytes */
3152		- its->ite_size = 16;
	4720	+ its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
	4721	+ its->typer \|= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
3153	4722
3154	4723	return true;
3155	4724	}
..	..	@@ -3183,8 +4752,10 @@
3183	4752	its->get_msi_base = its_irq_get_msi_base_pre_its;
3184	4753
3185	4754	ids = ilog2(pre_its_window[1]) - 2;
3186		- if (its->device_ids > ids)
3187		- its->device_ids = ids;
	4755	+ if (device_ids(its) > ids) {
	4756	+ its->typer &= ~GITS_TYPER_DEVBITS;
	4757	+ its->typer \|= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
	4758	+ }
3188	4759
3189	4760	/* the pre-ITS breaks isolation, so disable MSI remapping */
3190	4761	its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
..	..	@@ -3411,7 +4982,7 @@
3411	4982	}
3412	4983
3413	4984	/* Use the last possible DevID */
3414		- devid = GENMASK(its->device_ids - 1, 0);
	4985	+ devid = GENMASK(device_ids(its) - 1, 0);
3415	4986	vpe_proxy.dev = its_create_device(its, devid, entries, false);
3416	4987	if (!vpe_proxy.dev) {
3417	4988	kfree(vpe_proxy.vpes);
..	..	@@ -3474,10 +5045,11 @@
3474	5045	void __iomem *its_base;
3475	5046	u32 val, ctlr;
3476	5047	u64 baser, tmp, typer;
	5048	+ struct page *page;
3477	5049	int err;
3478	5050	gfp_t gfp_flags;
3479	5051
3480		- its_base = ioremap(res->start, resource_size(res));
	5052	+ its_base = ioremap(res->start, SZ_64K);
3481	5053	if (!its_base) {
3482	5054	pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
3483	5055	return -ENOMEM;
..	..	@@ -3509,12 +5081,10 @@
3509	5081	INIT_LIST_HEAD(&its->entry);
3510	5082	INIT_LIST_HEAD(&its->its_device_list);
3511	5083	typer = gic_read_typer(its_base + GITS_TYPER);
	5084	+ its->typer = typer;
3512	5085	its->base = its_base;
3513	5086	its->phys_base = res->start;
3514		- its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
3515		- its->device_ids = GITS_TYPER_DEVBITS(typer);
3516		- its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
3517		- if (its->is_v4) {
	5087	+ if (is_v4(its)) {
3518	5088	if (!(typer & GITS_TYPER_VMOVP)) {
3519	5089	err = its_compute_its_list_map(res, its_base);
3520	5090	if (err < 0)
..	..	@@ -3527,19 +5097,37 @@
3527	5097	} else {
3528	5098	pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
3529	5099	}
	5100	+
	5101	+ if (is_v4_1(its)) {
	5102	+ u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
	5103	+
	5104	+ its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
	5105	+ if (!its->sgir_base) {
	5106	+ err = -ENOMEM;
	5107	+ goto out_free_its;
	5108	+ }
	5109	+
	5110	+ its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
	5111	+
	5112	+ pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
	5113	+ &res->start, its->mpidr, svpet);
	5114	+ }
3530	5115	}
3531	5116
3532	5117	its->numa_node = numa_node;
3533	5118
3534	5119	gfp_flags = GFP_KERNEL \| __GFP_ZERO;
3535		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	5120	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	5121	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	5122	+ of_machine_is_compatible("rockchip,rk3566"))
3536	5123	gfp_flags \|= GFP_DMA32;
3537		- its->cmd_base = (void *)__get_free_pages(gfp_flags,
3538		- get_order(ITS_CMD_QUEUE_SZ));
3539		- if (!its->cmd_base) {
	5124	+ page = alloc_pages_node(its->numa_node, gfp_flags,
	5125	+ get_order(ITS_CMD_QUEUE_SZ));
	5126	+ if (!page) {
3540	5127	err = -ENOMEM;
3541		- goto out_free_its;
	5128	+ goto out_unmap_sgir;
3542	5129	}
	5130	+ its->cmd_base = (void *)page_address(page);
3543	5131	its->cmd_write = its->cmd_base;
3544	5132	its->fwnode_handle = handle;
3545	5133	its->get_msi_base = its_irq_get_msi_base;
..	..	@@ -3564,7 +5152,11 @@
3564	5152	gits_write_cbaser(baser, its->base + GITS_CBASER);
3565	5153	tmp = gits_read_cbaser(its->base + GITS_CBASER);
3566	5154
3567		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	5155	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	5156	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	5157	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	5158	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	5159	+ of_machine_is_compatible("rockchip,rk3588")))
3568	5160	tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
3569	5161
3570	5162	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
..	..	@@ -3586,7 +5178,7 @@
3586	5178	gits_write_cwriter(0, its->base + GITS_CWRITER);
3587	5179	ctlr = readl_relaxed(its->base + GITS_CTLR);
3588	5180	ctlr \|= GITS_CTLR_ENABLE;
3589		- if (its->is_v4)
	5181	+ if (is_v4(its))
3590	5182	ctlr \|= GITS_CTLR_ImDe;
3591	5183	writel_relaxed(ctlr, its->base + GITS_CTLR);
3592	5184
..	..	@@ -3604,6 +5196,9 @@
3604	5196	its_free_tables(its);
3605	5197	out_free_cmd:
3606	5198	free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
	5199	+out_unmap_sgir:
	5200	+ if (its->sgir_base)
	5201	+ iounmap(its->sgir_base);
3607	5202	out_free_its:
3608	5203	kfree(its);
3609	5204	out_unmap:
..	..	@@ -3623,16 +5218,6 @@
3623	5218	u64 timeout = USEC_PER_SEC;
3624	5219	u64 val;
3625	5220
3626		- /*
3627		- * If coming via a CPU hotplug event, we don't need to disable
3628		- * LPIs before trying to re-enable them. They are already
3629		- * configured and all is well in the world. Detect this case
3630		- * by checking the allocation of the pending table for the
3631		- * current CPU.
3632		- */
3633		- if (gic_data_rdist()->pend_page)
3634		- return 0;
3635		-
3636	5221	if (!gic_rdists_supports_plpis()) {
3637	5222	pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
3638	5223	return -ENXIO;
..	..	@@ -3642,7 +5227,21 @@
3642	5227	if (!(val & GICR_CTLR_ENABLE_LPIS))
3643	5228	return 0;
3644	5229
3645		- pr_warn("CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
	5230	+ /*
	5231	+ * If coming via a CPU hotplug event, we don't need to disable
	5232	+ * LPIs before trying to re-enable them. They are already
	5233	+ * configured and all is well in the world.
	5234	+ *
	5235	+ * If running with preallocated tables, there is nothing to do.
	5236	+ */
	5237	+ if (gic_data_rdist()->lpi_enabled \|\|
	5238	+ (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
	5239	+ return 0;
	5240	+
	5241	+ /*
	5242	+ * From that point on, we only try to do some damage control.
	5243	+ */
	5244	+ pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
3646	5245	smp_processor_id());
3647	5246	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3648	5247
..	..	@@ -3753,13 +5352,13 @@
3753	5352	return NUMA_NO_NODE;
3754	5353	}
3755	5354
3756		-static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header,
	5355	+static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
3757	5356	const unsigned long end)
3758	5357	{
3759	5358	return 0;
3760	5359	}
3761	5360
3762		-static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
	5361	+static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
3763	5362	const unsigned long end)
3764	5363	{
3765	5364	int node;
..	..	@@ -3775,7 +5374,12 @@
3775	5374	return -EINVAL;
3776	5375	}
3777	5376
3778		- node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
	5377	+ /*
	5378	+ * Note that in theory a new proximity node could be created by this
	5379	+ * entry as it is an SRAT resource allocation structure.
	5380	+ * We do not currently support doing so.
	5381	+ */
	5382	+ node = pxm_to_node(its_affinity->proximity_domain);
3779	5383
3780	5384	if (node == NUMA_NO_NODE \|\| node >= MAX_NUMNODES) {
3781	5385	pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
..	..	@@ -3826,7 +5430,7 @@
3826	5430	static void __init acpi_its_srat_maps_free(void) { }
3827	5431	#endif
3828	5432
3829		-static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
	5433	+static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
3830	5434	const unsigned long end)
3831	5435	{
3832	5436	struct acpi_madt_generic_translator *its_entry;
..	..	@@ -3840,7 +5444,7 @@
3840	5444	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
3841	5445	res.flags = IORESOURCE_MEM;
3842	5446
3843		- dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address);
	5447	+ dom_handle = irq_domain_alloc_fwnode(&res.start);
3844	5448	if (!dom_handle) {
3845	5449	pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
3846	5450	&res.start);
..	..	@@ -3883,7 +5487,10 @@
3883	5487	struct device_node *of_node;
3884	5488	struct its_node *its;
3885	5489	bool has_v4 = false;
	5490	+ bool has_v4_1 = false;
3886	5491	int err;
	5492	+
	5493	+ gic_rdists = rdists;
3887	5494
3888	5495	its_parent = parent_domain;
3889	5496	of_node = to_of_node(handle);
..	..	@@ -3897,17 +5504,29 @@
3897	5504	return -ENXIO;
3898	5505	}
3899	5506
3900		- gic_rdists = rdists;
3901		- err = its_alloc_lpi_tables();
	5507	+ err = allocate_lpi_tables();
3902	5508	if (err)
3903	5509	return err;
3904	5510
3905		- list_for_each_entry(its, &its_nodes, entry)
3906		- has_v4 \|= its->is_v4;
	5511	+ list_for_each_entry(its, &its_nodes, entry) {
	5512	+ has_v4 \|= is_v4(its);
	5513	+ has_v4_1 \|= is_v4_1(its);
	5514	+ }
	5515	+
	5516	+ /* Don't bother with inconsistent systems */
	5517	+ if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
	5518	+ rdists->has_rvpeid = false;
3907	5519
3908	5520	if (has_v4 & rdists->has_vlpis) {
	5521	+ const struct irq_domain_ops *sgi_ops;
	5522	+
	5523	+ if (has_v4_1)
	5524	+ sgi_ops = &its_sgi_domain_ops;
	5525	+ else
	5526	+ sgi_ops = NULL;
	5527	+
3909	5528	if (its_init_vpe_domain() \|\|
3910		- its_init_v4(parent_domain, &its_vpe_domain_ops)) {
	5529	+ its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
3911	5530	rdists->has_vlpis = false;
3912	5531	pr_err("ITS: Disabling GICv4 support\n");
3913	5532	}