~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,31 +1,24 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
3	4	* Author: Marc Zyngier <marc.zyngier@arm.com>
4		- *
5		- * This program is free software; you can redistribute it and/or modify
6		- * it under the terms of the GNU General Public License version 2 as
7		- * published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program. If not, see <http://www.gnu.org/licenses/>.
16	5	*/
17	6
18	7	#include <linux/acpi.h>
19	8	#include <linux/acpi_iort.h>
	9	+#include <linux/bitfield.h>
20	10	#include <linux/bitmap.h>
21	11	#include <linux/cpu.h>
	12	+#include <linux/crash_dump.h>
22	13	#include <linux/delay.h>
23	14	#include <linux/dma-iommu.h>
	15	+#include <linux/efi.h>
24	16	#include <linux/interrupt.h>
	17	+#include <linux/iopoll.h>
25	18	#include <linux/irqdomain.h>
26	19	#include <linux/list.h>
27		-#include <linux/list_sort.h>
28	20	#include <linux/log2.h>
	21	+#include <linux/memblock.h>
29	22	#include <linux/mm.h>
30	23	#include <linux/msi.h>
31	24	#include <linux/of.h>
..	..	@@ -51,6 +44,7 @@
51	44	#define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
52	45
53	46	#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
	47	+#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
54	48
55	49	static u32 lpi_id_bits;
56	50
..	..	@@ -63,7 +57,7 @@
63	57	#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
64	58	#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
65	59
66		-#define LPI_PROP_DEFAULT_PRIO 0xa0
	60	+#define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
67	61
68	62	/*
69	63	* Collection structure - just an ID, and a redistributor address to
..	..	@@ -102,6 +96,7 @@
102	96	struct mutex dev_alloc_lock;
103	97	struct list_head entry;
104	98	void __iomem *base;
	99	+ void __iomem *sgir_base;
105	100	phys_addr_t phys_base;
106	101	struct its_cmd_block *cmd_base;
107	102	struct its_cmd_block *cmd_write;
..	..	@@ -109,24 +104,36 @@
109	104	struct its_collection *collections;
110	105	struct fwnode_handle *fwnode_handle;
111	106	u64 (get_msi_base)(struct its_device its_dev);
	107	+ u64 typer;
112	108	u64 cbaser_save;
113	109	u32 ctlr_save;
	110	+ u32 mpidr;
114	111	struct list_head its_device_list;
115	112	u64 flags;
116	113	unsigned long list_nr;
117		- u32 ite_size;
118		- u32 device_ids;
119	114	int numa_node;
120	115	unsigned int msi_domain_flags;
121	116	u32 pre_its_base; /* for Socionext Synquacer */
122		- bool is_v4;
123	117	int vlpi_redist_offset;
124	118	};
	119	+
	120	+#define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
	121	+#define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
	122	+#define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
125	123
126	124	#define ITS_ITT_ALIGN SZ_256
127	125
128	126	/* The maximum number of VPEID bits supported by VLPI commands */
129		-#define ITS_MAX_VPEID_BITS (16)
	127	+#define ITS_MAX_VPEID_BITS \
	128	+ ({ \
	129	+ int nvpeid = 16; \
	130	+ if (gic_rdists->has_rvpeid && \
	131	+ gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
	132	+ nvpeid = 1 + (gic_rdists->gicd_typer2 & \
	133	+ GICD_TYPER2_VID); \
	134	+ \
	135	+ nvpeid; \
	136	+ })
130	137	#define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
131	138
132	139	/* Convert page order to size in bytes */
..	..	@@ -137,7 +144,7 @@
137	144	u16 *col_map;
138	145	irq_hw_number_t lpi_base;
139	146	int nr_lpis;
140		- struct mutex vlpi_lock;
	147	+ raw_spinlock_t vlpi_lock;
141	148	struct its_vm *vm;
142	149	struct its_vlpi_map *vlpi_maps;
143	150	int nr_vlpis;
..	..	@@ -167,6 +174,13 @@
167	174	int next_victim;
168	175	} vpe_proxy;
169	176
	177	+struct cpu_lpi_count {
	178	+ atomic_t managed;
	179	+ atomic_t unmanaged;
	180	+};
	181	+
	182	+static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
	183	+
170	184	static LIST_HEAD(its_nodes);
171	185	static DEFINE_RAW_SPINLOCK(its_lock);
172	186	static struct rdists *gic_rdists;
..	..	@@ -183,20 +197,35 @@
183	197	#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
184	198	#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
185	199
	200	+/*
	201	+ * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
	202	+ * always have vSGIs mapped.
	203	+ */
	204	+static bool require_its_list_vmovp(struct its_vm vm, struct its_node its)
	205	+{
	206	+ return (gic_rdists->has_rvpeid \|\| vm->vlpi_count[its->list_nr]);
	207	+}
	208	+
186	209	static u16 get_its_list(struct its_vm *vm)
187	210	{
188	211	struct its_node *its;
189	212	unsigned long its_list = 0;
190	213
191	214	list_for_each_entry(its, &its_nodes, entry) {
192		- if (!its->is_v4)
	215	+ if (!is_v4(its))
193	216	continue;
194	217
195		- if (vm->vlpi_count[its->list_nr])
	218	+ if (require_its_list_vmovp(vm, its))
196	219	__set_bit(its->list_nr, &its_list);
197	220	}
198	221
199	222	return (u16)its_list;
	223	+}
	224	+
	225	+static inline u32 its_get_event_id(struct irq_data *d)
	226	+{
	227	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	228	+ return d->hwirq - its_dev->event_map.lpi_base;
200	229	}
201	230
202	231	static struct its_collection dev_event_to_col(struct its_device its_dev,
..	..	@@ -205,6 +234,82 @@
205	234	struct its_node *its = its_dev->its;
206	235
207	236	return its->collections + its_dev->event_map.col_map[event];
	237	+}
	238	+
	239	+static struct its_vlpi_map dev_event_to_vlpi_map(struct its_device its_dev,
	240	+ u32 event)
	241	+{
	242	+ if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
	243	+ return NULL;
	244	+
	245	+ return &its_dev->event_map.vlpi_maps[event];
	246	+}
	247	+
	248	+static struct its_vlpi_map get_vlpi_map(struct irq_data d)
	249	+{
	250	+ if (irqd_is_forwarded_to_vcpu(d)) {
	251	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	252	+ u32 event = its_get_event_id(d);
	253	+
	254	+ return dev_event_to_vlpi_map(its_dev, event);
	255	+ }
	256	+
	257	+ return NULL;
	258	+}
	259	+
	260	+static int vpe_to_cpuid_lock(struct its_vpe vpe, unsigned long flags)
	261	+{
	262	+ raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
	263	+ return vpe->col_idx;
	264	+}
	265	+
	266	+static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
	267	+{
	268	+ raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
	269	+}
	270	+
	271	+static struct irq_chip its_vpe_irq_chip;
	272	+
	273	+static int irq_to_cpuid_lock(struct irq_data d, unsigned long flags)
	274	+{
	275	+ struct its_vpe *vpe = NULL;
	276	+ int cpu;
	277	+
	278	+ if (d->chip == &its_vpe_irq_chip) {
	279	+ vpe = irq_data_get_irq_chip_data(d);
	280	+ } else {
	281	+ struct its_vlpi_map *map = get_vlpi_map(d);
	282	+ if (map)
	283	+ vpe = map->vpe;
	284	+ }
	285	+
	286	+ if (vpe) {
	287	+ cpu = vpe_to_cpuid_lock(vpe, flags);
	288	+ } else {
	289	+ /* Physical LPIs are already locked via the irq_desc lock */
	290	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	291	+ cpu = its_dev->event_map.col_map[its_get_event_id(d)];
	292	+ /* Keep GCC quiet... */
	293	+ *flags = 0;
	294	+ }
	295	+
	296	+ return cpu;
	297	+}
	298	+
	299	+static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
	300	+{
	301	+ struct its_vpe *vpe = NULL;
	302	+
	303	+ if (d->chip == &its_vpe_irq_chip) {
	304	+ vpe = irq_data_get_irq_chip_data(d);
	305	+ } else {
	306	+ struct its_vlpi_map *map = get_vlpi_map(d);
	307	+ if (map)
	308	+ vpe = map->vpe;
	309	+ }
	310	+
	311	+ if (vpe)
	312	+ vpe_to_cpuid_unlock(vpe, flags);
208	313	}
209	314
210	315	static struct its_collection valid_col(struct its_collection col)
..	..	@@ -306,6 +411,19 @@
306	411	u16 seq_num;
307	412	u16 its_list;
308	413	} its_vmovp_cmd;
	414	+
	415	+ struct {
	416	+ struct its_vpe *vpe;
	417	+ } its_invdb_cmd;
	418	+
	419	+ struct {
	420	+ struct its_vpe *vpe;
	421	+ u8 sgi;
	422	+ u8 priority;
	423	+ bool enable;
	424	+ bool group;
	425	+ bool clear;
	426	+ } its_vsgi_cmd;
309	427	};
310	428	};
311	429
..	..	@@ -313,7 +431,10 @@
313	431	* The ITS command block, which is what the ITS actually parses.
314	432	*/
315	433	struct its_cmd_block {
316		- u64 raw_cmd[4];
	434	+ union {
	435	+ u64 raw_cmd[4];
	436	+ __le64 raw_cmd_le[4];
	437	+ };
317	438	};
318	439
319	440	#define ITS_CMD_QUEUE_SZ SZ_64K
..	..	@@ -419,13 +540,70 @@
419	540	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
420	541	}
421	542
	543	+static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
	544	+{
	545	+ its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
	546	+}
	547	+
	548	+static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
	549	+{
	550	+ its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
	551	+}
	552	+
	553	+static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
	554	+{
	555	+ its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
	556	+}
	557	+
	558	+static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
	559	+ u32 vpe_db_lpi)
	560	+{
	561	+ its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
	562	+}
	563	+
	564	+static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
	565	+ u32 vpe_db_lpi)
	566	+{
	567	+ its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
	568	+}
	569	+
	570	+static void its_encode_db(struct its_cmd_block *cmd, bool db)
	571	+{
	572	+ its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
	573	+}
	574	+
	575	+static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
	576	+{
	577	+ its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
	578	+}
	579	+
	580	+static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
	581	+{
	582	+ its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
	583	+}
	584	+
	585	+static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
	586	+{
	587	+ its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
	588	+}
	589	+
	590	+static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
	591	+{
	592	+ its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
	593	+}
	594	+
	595	+static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
	596	+{
	597	+ its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
	598	+}
	599	+
422	600	static inline void its_fixup_cmd(struct its_cmd_block *cmd)
423	601	{
424	602	/* Let's fixup BE commands */
425		- cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
426		- cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
427		- cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
428		- cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
	603	+ cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
	604	+ cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
	605	+ cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
	606	+ cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
429	607	}
430	608
431	609	static struct its_collection its_build_mapd_cmd(struct its_node its,
..	..	@@ -602,19 +780,45 @@
602	780	struct its_cmd_block *cmd,
603	781	struct its_cmd_desc *desc)
604	782	{
605		- unsigned long vpt_addr;
	783	+ unsigned long vpt_addr, vconf_addr;
606	784	u64 target;
607		-
608		- vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
609		- target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
	785	+ bool alloc;
610	786
611	787	its_encode_cmd(cmd, GITS_CMD_VMAPP);
612	788	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
613	789	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
	790	+
	791	+ if (!desc->its_vmapp_cmd.valid) {
	792	+ if (is_v4_1(its)) {
	793	+ alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
	794	+ its_encode_alloc(cmd, alloc);
	795	+ }
	796	+
	797	+ goto out;
	798	+ }
	799	+
	800	+ vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
	801	+ target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
	802	+
614	803	its_encode_target(cmd, target);
615	804	its_encode_vpt_addr(cmd, vpt_addr);
616	805	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
617	806
	807	+ if (!is_v4_1(its))
	808	+ goto out;
	809	+
	810	+ vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
	811	+
	812	+ alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
	813	+
	814	+ its_encode_alloc(cmd, alloc);
	815	+
	816	+ /* We can only signal PTZ when alloc==1. Why do we have two bits? */
	817	+ its_encode_ptz(cmd, alloc);
	818	+ its_encode_vconf_addr(cmd, vconf_addr);
	819	+ its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
	820	+
	821	+out:
618	822	its_fixup_cmd(cmd);
619	823
620	824	return valid_vpe(its, desc->its_vmapp_cmd.vpe);
..	..	@@ -626,7 +830,7 @@
626	830	{
627	831	u32 db;
628	832
629		- if (desc->its_vmapti_cmd.db_enabled)
	833	+ if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
630	834	db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
631	835	else
632	836	db = 1023;
..	..	@@ -649,7 +853,7 @@
649	853	{
650	854	u32 db;
651	855
652		- if (desc->its_vmovi_cmd.db_enabled)
	856	+ if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
653	857	db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
654	858	else
655	859	db = 1023;
..	..	@@ -679,9 +883,103 @@
679	883	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
680	884	its_encode_target(cmd, target);
681	885
	886	+ if (is_v4_1(its)) {
	887	+ its_encode_db(cmd, true);
	888	+ its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
	889	+ }
	890	+
682	891	its_fixup_cmd(cmd);
683	892
684	893	return valid_vpe(its, desc->its_vmovp_cmd.vpe);
	894	+}
	895	+
	896	+static struct its_vpe its_build_vinv_cmd(struct its_node its,
	897	+ struct its_cmd_block *cmd,
	898	+ struct its_cmd_desc *desc)
	899	+{
	900	+ struct its_vlpi_map *map;
	901	+
	902	+ map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
	903	+ desc->its_inv_cmd.event_id);
	904	+
	905	+ its_encode_cmd(cmd, GITS_CMD_INV);
	906	+ its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
	907	+ its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
	908	+
	909	+ its_fixup_cmd(cmd);
	910	+
	911	+ return valid_vpe(its, map->vpe);
	912	+}
	913	+
	914	+static struct its_vpe its_build_vint_cmd(struct its_node its,
	915	+ struct its_cmd_block *cmd,
	916	+ struct its_cmd_desc *desc)
	917	+{
	918	+ struct its_vlpi_map *map;
	919	+
	920	+ map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
	921	+ desc->its_int_cmd.event_id);
	922	+
	923	+ its_encode_cmd(cmd, GITS_CMD_INT);
	924	+ its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
	925	+ its_encode_event_id(cmd, desc->its_int_cmd.event_id);
	926	+
	927	+ its_fixup_cmd(cmd);
	928	+
	929	+ return valid_vpe(its, map->vpe);
	930	+}
	931	+
	932	+static struct its_vpe its_build_vclear_cmd(struct its_node its,
	933	+ struct its_cmd_block *cmd,
	934	+ struct its_cmd_desc *desc)
	935	+{
	936	+ struct its_vlpi_map *map;
	937	+
	938	+ map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
	939	+ desc->its_clear_cmd.event_id);
	940	+
	941	+ its_encode_cmd(cmd, GITS_CMD_CLEAR);
	942	+ its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
	943	+ its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
	944	+
	945	+ its_fixup_cmd(cmd);
	946	+
	947	+ return valid_vpe(its, map->vpe);
	948	+}
	949	+
	950	+static struct its_vpe its_build_invdb_cmd(struct its_node its,
	951	+ struct its_cmd_block *cmd,
	952	+ struct its_cmd_desc *desc)
	953	+{
	954	+ if (WARN_ON(!is_v4_1(its)))
	955	+ return NULL;
	956	+
	957	+ its_encode_cmd(cmd, GITS_CMD_INVDB);
	958	+ its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
	959	+
	960	+ its_fixup_cmd(cmd);
	961	+
	962	+ return valid_vpe(its, desc->its_invdb_cmd.vpe);
	963	+}
	964	+
	965	+static struct its_vpe its_build_vsgi_cmd(struct its_node its,
	966	+ struct its_cmd_block *cmd,
	967	+ struct its_cmd_desc *desc)
	968	+{
	969	+ if (WARN_ON(!is_v4_1(its)))
	970	+ return NULL;
	971	+
	972	+ its_encode_cmd(cmd, GITS_CMD_VSGI);
	973	+ its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
	974	+ its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
	975	+ its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
	976	+ its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
	977	+ its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
	978	+ its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
	979	+
	980	+ its_fixup_cmd(cmd);
	981	+
	982	+ return valid_vpe(its, desc->its_vsgi_cmd.vpe);
685	983	}
686	984
687	985	static u64 its_cmd_ptr_to_offset(struct its_node *its,
..	..	@@ -961,7 +1259,7 @@
961	1259
962	1260	static void its_send_vmapti(struct its_device *dev, u32 id)
963	1261	{
964		- struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
	1262	+ struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
965	1263	struct its_cmd_desc desc;
966	1264
967	1265	desc.its_vmapti_cmd.vpe = map->vpe;
..	..	@@ -975,7 +1273,7 @@
975	1273
976	1274	static void its_send_vmovi(struct its_device *dev, u32 id)
977	1275	{
978		- struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
	1276	+ struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
979	1277	struct its_cmd_desc desc;
980	1278
981	1279	desc.its_vmovi_cmd.vpe = map->vpe;
..	..	@@ -1029,10 +1327,10 @@
1029	1327
1030	1328	/* Emit VMOVPs */
1031	1329	list_for_each_entry(its, &its_nodes, entry) {
1032		- if (!its->is_v4)
	1330	+ if (!is_v4(its))
1033	1331	continue;
1034	1332
1035		- if (!vpe->its_vm->vlpi_count[its->list_nr])
	1333	+ if (!require_its_list_vmovp(vpe->its_vm, its))
1036	1334	continue;
1037	1335
1038	1336	desc.its_vmovp_cmd.col = &its->collections[col_id];
..	..	@@ -1050,40 +1348,79 @@
1050	1348	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1051	1349	}
1052	1350
	1351	+static void its_send_vinv(struct its_device *dev, u32 event_id)
	1352	+{
	1353	+ struct its_cmd_desc desc;
	1354	+
	1355	+ /*
	1356	+ * There is no real VINV command. This is just a normal INV,
	1357	+ * with a VSYNC instead of a SYNC.
	1358	+ */
	1359	+ desc.its_inv_cmd.dev = dev;
	1360	+ desc.its_inv_cmd.event_id = event_id;
	1361	+
	1362	+ its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
	1363	+}
	1364	+
	1365	+static void its_send_vint(struct its_device *dev, u32 event_id)
	1366	+{
	1367	+ struct its_cmd_desc desc;
	1368	+
	1369	+ /*
	1370	+ * There is no real VINT command. This is just a normal INT,
	1371	+ * with a VSYNC instead of a SYNC.
	1372	+ */
	1373	+ desc.its_int_cmd.dev = dev;
	1374	+ desc.its_int_cmd.event_id = event_id;
	1375	+
	1376	+ its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
	1377	+}
	1378	+
	1379	+static void its_send_vclear(struct its_device *dev, u32 event_id)
	1380	+{
	1381	+ struct its_cmd_desc desc;
	1382	+
	1383	+ /*
	1384	+ * There is no real VCLEAR command. This is just a normal CLEAR,
	1385	+ * with a VSYNC instead of a SYNC.
	1386	+ */
	1387	+ desc.its_clear_cmd.dev = dev;
	1388	+ desc.its_clear_cmd.event_id = event_id;
	1389	+
	1390	+ its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
	1391	+}
	1392	+
	1393	+static void its_send_invdb(struct its_node its, struct its_vpe vpe)
	1394	+{
	1395	+ struct its_cmd_desc desc;
	1396	+
	1397	+ desc.its_invdb_cmd.vpe = vpe;
	1398	+ its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
	1399	+}
	1400	+
1053	1401	/*
1054	1402	* irqchip functions - assumes MSI, mostly.
1055	1403	*/
1056		-
1057		-static inline u32 its_get_event_id(struct irq_data *d)
1058		-{
1059		- struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1060		- return d->hwirq - its_dev->event_map.lpi_base;
1061		-}
1062		-
1063	1404	static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1064	1405	{
	1406	+ struct its_vlpi_map *map = get_vlpi_map(d);
1065	1407	irq_hw_number_t hwirq;
1066		- struct page *prop_page;
	1408	+ void *va;
1067	1409	u8 *cfg;
1068	1410
1069		- if (irqd_is_forwarded_to_vcpu(d)) {
1070		- struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1071		- u32 event = its_get_event_id(d);
1072		- struct its_vlpi_map *map;
1073		-
1074		- prop_page = its_dev->event_map.vm->vprop_page;
1075		- map = &its_dev->event_map.vlpi_maps[event];
	1411	+ if (map) {
	1412	+ va = page_address(map->vm->vprop_page);
1076	1413	hwirq = map->vintid;
1077	1414
1078	1415	/* Remember the updated property */
1079	1416	map->properties &= ~clr;
1080	1417	map->properties \|= set \| LPI_PROP_GROUP1;
1081	1418	} else {
1082		- prop_page = gic_rdists->prop_page;
	1419	+ va = gic_rdists->prop_table_va;
1083	1420	hwirq = d->hwirq;
1084	1421	}
1085	1422
1086		- cfg = page_address(prop_page) + hwirq - 8192;
	1423	+ cfg = va + hwirq - 8192;
1087	1424	*cfg &= ~clr;
1088	1425	*cfg \|= set \| LPI_PROP_GROUP1;
1089	1426
..	..	@@ -1098,30 +1435,90 @@
1098	1435	dsb(ishst);
1099	1436	}
1100	1437
	1438	+static void wait_for_syncr(void __iomem *rdbase)
	1439	+{
	1440	+ while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
	1441	+ cpu_relax();
	1442	+}
	1443	+
	1444	+static void __direct_lpi_inv(struct irq_data *d, u64 val)
	1445	+{
	1446	+ void __iomem *rdbase;
	1447	+ unsigned long flags;
	1448	+ int cpu;
	1449	+
	1450	+ /* Target the redistributor this LPI is currently routed to */
	1451	+ cpu = irq_to_cpuid_lock(d, &flags);
	1452	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	1453	+
	1454	+ rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
	1455	+ gic_write_lpir(val, rdbase + GICR_INVLPIR);
	1456	+ wait_for_syncr(rdbase);
	1457	+
	1458	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	1459	+ irq_to_cpuid_unlock(d, flags);
	1460	+}
	1461	+
	1462	+static void direct_lpi_inv(struct irq_data *d)
	1463	+{
	1464	+ struct its_vlpi_map *map = get_vlpi_map(d);
	1465	+ u64 val;
	1466	+
	1467	+ if (map) {
	1468	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	1469	+
	1470	+ WARN_ON(!is_v4_1(its_dev->its));
	1471	+
	1472	+ val = GICR_INVLPIR_V;
	1473	+ val \|= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
	1474	+ val \|= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
	1475	+ } else {
	1476	+ val = d->hwirq;
	1477	+ }
	1478	+
	1479	+ __direct_lpi_inv(d, val);
	1480	+}
	1481	+
1101	1482	static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1102	1483	{
1103	1484	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1104	1485
1105	1486	lpi_write_config(d, clr, set);
1106		- its_send_inv(its_dev, its_get_event_id(d));
	1487	+ if (gic_rdists->has_direct_lpi &&
	1488	+ (is_v4_1(its_dev->its) \|\| !irqd_is_forwarded_to_vcpu(d)))
	1489	+ direct_lpi_inv(d);
	1490	+ else if (!irqd_is_forwarded_to_vcpu(d))
	1491	+ its_send_inv(its_dev, its_get_event_id(d));
	1492	+ else
	1493	+ its_send_vinv(its_dev, its_get_event_id(d));
1107	1494	}
1108	1495
1109	1496	static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1110	1497	{
1111	1498	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1112	1499	u32 event = its_get_event_id(d);
	1500	+ struct its_vlpi_map *map;
1113	1501
1114		- if (its_dev->event_map.vlpi_maps[event].db_enabled == enable)
	1502	+ /*
	1503	+ * GICv4.1 does away with the per-LPI nonsense, nothing to do
	1504	+ * here.
	1505	+ */
	1506	+ if (is_v4_1(its_dev->its))
1115	1507	return;
1116	1508
1117		- its_dev->event_map.vlpi_maps[event].db_enabled = enable;
	1509	+ map = dev_event_to_vlpi_map(its_dev, event);
	1510	+
	1511	+ if (map->db_enabled == enable)
	1512	+ return;
	1513	+
	1514	+ map->db_enabled = enable;
1118	1515
1119	1516	/*
1120	1517	* More fun with the architecture:
1121	1518	*
1122	1519	* Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1123	1520	* value or to 1023, depending on the enable bit. But that
1124		- * would be issueing a mapping for an /existing/ DevID+EventID
	1521	+ * would be issuing a mapping for an /existing/ DevID+EventID
1125	1522	* pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1126	1523	* to the /same/ vPE, using this opportunity to adjust the
1127	1524	* doorbell. Mouahahahaha. We loves it, Precious.
..	..	@@ -1145,42 +1542,159 @@
1145	1542	lpi_update_config(d, 0, LPI_PROP_ENABLED);
1146	1543	}
1147	1544
	1545	+static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
	1546	+{
	1547	+ if (irqd_affinity_is_managed(d))
	1548	+ return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1549	+
	1550	+ return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1551	+}
	1552	+
	1553	+static void its_inc_lpi_count(struct irq_data *d, int cpu)
	1554	+{
	1555	+ if (irqd_affinity_is_managed(d))
	1556	+ atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1557	+ else
	1558	+ atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1559	+}
	1560	+
	1561	+static void its_dec_lpi_count(struct irq_data *d, int cpu)
	1562	+{
	1563	+ if (irqd_affinity_is_managed(d))
	1564	+ atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1565	+ else
	1566	+ atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1567	+}
	1568	+
	1569	+static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
	1570	+ const struct cpumask *cpu_mask)
	1571	+{
	1572	+ unsigned int cpu = nr_cpu_ids, tmp;
	1573	+ int count = S32_MAX;
	1574	+
	1575	+ for_each_cpu(tmp, cpu_mask) {
	1576	+ int this_count = its_read_lpi_count(d, tmp);
	1577	+ if (this_count < count) {
	1578	+ cpu = tmp;
	1579	+ count = this_count;
	1580	+ }
	1581	+ }
	1582	+
	1583	+ return cpu;
	1584	+}
	1585	+
	1586	+/*
	1587	+ * As suggested by Thomas Gleixner in:
	1588	+ * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
	1589	+ */
	1590	+static int its_select_cpu(struct irq_data *d,
	1591	+ const struct cpumask *aff_mask)
	1592	+{
	1593	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	1594	+ cpumask_var_t tmpmask;
	1595	+ int cpu, node;
	1596	+
	1597	+ if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
	1598	+ return -ENOMEM;
	1599	+
	1600	+ node = its_dev->its->numa_node;
	1601	+
	1602	+ if (!irqd_affinity_is_managed(d)) {
	1603	+ /* First try the NUMA node */
	1604	+ if (node != NUMA_NO_NODE) {
	1605	+ /*
	1606	+ * Try the intersection of the affinity mask and the
	1607	+ * node mask (and the online mask, just to be safe).
	1608	+ */
	1609	+ cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
	1610	+ cpumask_and(tmpmask, tmpmask, cpu_online_mask);
	1611	+
	1612	+ /*
	1613	+ * Ideally, we would check if the mask is empty, and
	1614	+ * try again on the full node here.
	1615	+ *
	1616	+ * But it turns out that the way ACPI describes the
	1617	+ * affinity for ITSs only deals about memory, and
	1618	+ * not target CPUs, so it cannot describe a single
	1619	+ * ITS placed next to two NUMA nodes.
	1620	+ *
	1621	+ * Instead, just fallback on the online mask. This
	1622	+ * diverges from Thomas' suggestion above.
	1623	+ */
	1624	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1625	+ if (cpu < nr_cpu_ids)
	1626	+ goto out;
	1627	+
	1628	+ /* If we can't cross sockets, give up */
	1629	+ if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
	1630	+ goto out;
	1631	+
	1632	+ /* If the above failed, expand the search */
	1633	+ }
	1634	+
	1635	+ /* Try the intersection of the affinity and online masks */
	1636	+ cpumask_and(tmpmask, aff_mask, cpu_online_mask);
	1637	+
	1638	+ /* If that doesn't fly, the online mask is the last resort */
	1639	+ if (cpumask_empty(tmpmask))
	1640	+ cpumask_copy(tmpmask, cpu_online_mask);
	1641	+
	1642	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1643	+ } else {
	1644	+ cpumask_copy(tmpmask, aff_mask);
	1645	+
	1646	+ /* If we cannot cross sockets, limit the search to that node */
	1647	+ if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
	1648	+ node != NUMA_NO_NODE)
	1649	+ cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
	1650	+
	1651	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1652	+ }
	1653	+out:
	1654	+ free_cpumask_var(tmpmask);
	1655	+
	1656	+ pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
	1657	+ return cpu;
	1658	+}
	1659	+
1148	1660	static int its_set_affinity(struct irq_data d, const struct cpumask mask_val,
1149	1661	bool force)
1150	1662	{
1151		- unsigned int cpu;
1152		- const struct cpumask *cpu_mask = cpu_online_mask;
1153	1663	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1154	1664	struct its_collection *target_col;
1155	1665	u32 id = its_get_event_id(d);
	1666	+ int cpu, prev_cpu;
1156	1667
1157	1668	/* A forwarded interrupt should use irq_set_vcpu_affinity */
1158	1669	if (irqd_is_forwarded_to_vcpu(d))
1159	1670	return -EINVAL;
1160	1671
1161		- /* lpi cannot be routed to a redistributor that is on a foreign node */
1162		- if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
1163		- if (its_dev->its->numa_node >= 0) {
1164		- cpu_mask = cpumask_of_node(its_dev->its->numa_node);
1165		- if (!cpumask_intersects(mask_val, cpu_mask))
1166		- return -EINVAL;
1167		- }
1168		- }
	1672	+ prev_cpu = its_dev->event_map.col_map[id];
	1673	+ its_dec_lpi_count(d, prev_cpu);
1169	1674
1170		- cpu = cpumask_any_and(mask_val, cpu_mask);
	1675	+ if (!force)
	1676	+ cpu = its_select_cpu(d, mask_val);
	1677	+ else
	1678	+ cpu = cpumask_pick_least_loaded(d, mask_val);
1171	1679
1172		- if (cpu >= nr_cpu_ids)
1173		- return -EINVAL;
	1680	+ if (cpu < 0 \|\| cpu >= nr_cpu_ids)
	1681	+ goto err;
1174	1682
1175	1683	/* don't set the affinity when the target cpu is same as current one */
1176		- if (cpu != its_dev->event_map.col_map[id]) {
	1684	+ if (cpu != prev_cpu) {
1177	1685	target_col = &its_dev->its->collections[cpu];
1178	1686	its_send_movi(its_dev, target_col, id);
1179	1687	its_dev->event_map.col_map[id] = cpu;
1180	1688	irq_data_update_effective_affinity(d, cpumask_of(cpu));
1181	1689	}
1182	1690
	1691	+ its_inc_lpi_count(d, cpu);
	1692	+
1183	1693	return IRQ_SET_MASK_OK_DONE;
	1694	+
	1695	+err:
	1696	+ its_inc_lpi_count(d, prev_cpu);
	1697	+ return -EINVAL;
1184	1698	}
1185	1699
1186	1700	static u64 its_irq_get_msi_base(struct its_device *its_dev)
..	..	@@ -1203,7 +1717,7 @@
1203	1717	msg->address_hi = upper_32_bits(addr);
1204	1718	msg->data = its_get_event_id(d);
1205	1719
1206		- iommu_dma_map_msi_msg(d->irq, msg);
	1720	+ iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1207	1721	}
1208	1722
1209	1723	static int its_irq_set_irqchip_state(struct irq_data *d,
..	..	@@ -1216,20 +1730,51 @@
1216	1730	if (which != IRQCHIP_STATE_PENDING)
1217	1731	return -EINVAL;
1218	1732
1219		- if (state)
1220		- its_send_int(its_dev, event);
1221		- else
1222		- its_send_clear(its_dev, event);
	1733	+ if (irqd_is_forwarded_to_vcpu(d)) {
	1734	+ if (state)
	1735	+ its_send_vint(its_dev, event);
	1736	+ else
	1737	+ its_send_vclear(its_dev, event);
	1738	+ } else {
	1739	+ if (state)
	1740	+ its_send_int(its_dev, event);
	1741	+ else
	1742	+ its_send_clear(its_dev, event);
	1743	+ }
1223	1744
1224	1745	return 0;
	1746	+}
	1747	+
	1748	+static int its_irq_retrigger(struct irq_data *d)
	1749	+{
	1750	+ return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
	1751	+}
	1752	+
	1753	+/*
	1754	+ * Two favourable cases:
	1755	+ *
	1756	+ * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
	1757	+ * for vSGI delivery
	1758	+ *
	1759	+ * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
	1760	+ * and we're better off mapping all VPEs always
	1761	+ *
	1762	+ * If neither (a) nor (b) is true, then we map vPEs on demand.
	1763	+ *
	1764	+ */
	1765	+static bool gic_requires_eager_mapping(void)
	1766	+{
	1767	+ if (!its_list_map \|\| gic_rdists->has_rvpeid)
	1768	+ return true;
	1769	+
	1770	+ return false;
1225	1771	}
1226	1772
1227	1773	static void its_map_vm(struct its_node its, struct its_vm vm)
1228	1774	{
1229	1775	unsigned long flags;
1230	1776
1231		- /* Not using the ITS list? Everything is always mapped. */
1232		- if (!its_list_map)
	1777	+ if (gic_requires_eager_mapping())
1233	1778	return;
1234	1779
1235	1780	raw_spin_lock_irqsave(&vmovp_lock, flags);
..	..	@@ -1263,7 +1808,7 @@
1263	1808	unsigned long flags;
1264	1809
1265	1810	/* Not using the ITS list? Everything is always mapped. */
1266		- if (!its_list_map)
	1811	+ if (gic_requires_eager_mapping())
1267	1812	return;
1268	1813
1269	1814	raw_spin_lock_irqsave(&vmovp_lock, flags);
..	..	@@ -1287,13 +1832,13 @@
1287	1832	if (!info->map)
1288	1833	return -EINVAL;
1289	1834
1290		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1835	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1291	1836
1292	1837	if (!its_dev->event_map.vm) {
1293	1838	struct its_vlpi_map *maps;
1294	1839
1295	1840	maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1296		- GFP_KERNEL);
	1841	+ GFP_ATOMIC);
1297	1842	if (!maps) {
1298	1843	ret = -ENOMEM;
1299	1844	goto out;
..	..	@@ -1336,29 +1881,30 @@
1336	1881	}
1337	1882
1338	1883	out:
1339		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1884	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1340	1885	return ret;
1341	1886	}
1342	1887
1343	1888	static int its_vlpi_get(struct irq_data d, struct its_cmd_info info)
1344	1889	{
1345	1890	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1346		- u32 event = its_get_event_id(d);
	1891	+ struct its_vlpi_map *map;
1347	1892	int ret = 0;
1348	1893
1349		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1894	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1350	1895
1351		- if (!its_dev->event_map.vm \|\|
1352		- !its_dev->event_map.vlpi_maps[event].vm) {
	1896	+ map = get_vlpi_map(d);
	1897	+
	1898	+ if (!its_dev->event_map.vm \|\| !map) {
1353	1899	ret = -EINVAL;
1354	1900	goto out;
1355	1901	}
1356	1902
1357	1903	/* Copy our mapping information to the incoming request */
1358		- *info->map = its_dev->event_map.vlpi_maps[event];
	1904	+ info->map = map;
1359	1905
1360	1906	out:
1361		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1907	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1362	1908	return ret;
1363	1909	}
1364	1910
..	..	@@ -1368,7 +1914,7 @@
1368	1914	u32 event = its_get_event_id(d);
1369	1915	int ret = 0;
1370	1916
1371		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1917	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1372	1918
1373	1919	if (!its_dev->event_map.vm \|\| !irqd_is_forwarded_to_vcpu(d)) {
1374	1920	ret = -EINVAL;
..	..	@@ -1398,7 +1944,7 @@
1398	1944	}
1399	1945
1400	1946	out:
1401		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1947	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1402	1948	return ret;
1403	1949	}
1404	1950
..	..	@@ -1424,7 +1970,7 @@
1424	1970	struct its_cmd_info *info = vcpu_info;
1425	1971
1426	1972	/* Need a v4 ITS */
1427		- if (!its_dev->its->is_v4)
	1973	+ if (!is_v4(its_dev->its))
1428	1974	return -EINVAL;
1429	1975
1430	1976	/* Unmap request? */
..	..	@@ -1455,6 +2001,7 @@
1455	2001	.irq_set_affinity = its_set_affinity,
1456	2002	.irq_compose_msi_msg = its_irq_compose_msi_msg,
1457	2003	.irq_set_irqchip_state = its_irq_set_irqchip_state,
	2004	+ .irq_retrigger = its_irq_retrigger,
1458	2005	.irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
1459	2006	};
1460	2007
..	..	@@ -1489,39 +2036,13 @@
1489	2036	{
1490	2037	struct lpi_range *range;
1491	2038
1492		- range = kzalloc(sizeof(*range), GFP_KERNEL);
	2039	+ range = kmalloc(sizeof(*range), GFP_KERNEL);
1493	2040	if (range) {
1494		- INIT_LIST_HEAD(&range->entry);
1495	2041	range->base_id = base;
1496	2042	range->span = span;
1497	2043	}
1498	2044
1499	2045	return range;
1500		-}
1501		-
1502		-static int lpi_range_cmp(void priv, struct list_head a, struct list_head *b)
1503		-{
1504		- struct lpi_range ra, rb;
1505		-
1506		- ra = container_of(a, struct lpi_range, entry);
1507		- rb = container_of(b, struct lpi_range, entry);
1508		-
1509		- return ra->base_id - rb->base_id;
1510		-}
1511		-
1512		-static void merge_lpi_ranges(void)
1513		-{
1514		- struct lpi_range range, tmp;
1515		-
1516		- list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
1517		- if (!list_is_last(&range->entry, &lpi_range_list) &&
1518		- (tmp->base_id == (range->base_id + range->span))) {
1519		- tmp->base_id = range->base_id;
1520		- tmp->span += range->span;
1521		- list_del(&range->entry);
1522		- kfree(range);
1523		- }
1524		- }
1525	2046	}
1526	2047
1527	2048	static int alloc_lpi_range(u32 nr_lpis, u32 *base)
..	..	@@ -1553,25 +2074,49 @@
1553	2074	return err;
1554	2075	}
1555	2076
	2077	+static void merge_lpi_ranges(struct lpi_range a, struct lpi_range b)
	2078	+{
	2079	+ if (&a->entry == &lpi_range_list \|\| &b->entry == &lpi_range_list)
	2080	+ return;
	2081	+ if (a->base_id + a->span != b->base_id)
	2082	+ return;
	2083	+ b->base_id = a->base_id;
	2084	+ b->span += a->span;
	2085	+ list_del(&a->entry);
	2086	+ kfree(a);
	2087	+}
	2088	+
1556	2089	static int free_lpi_range(u32 base, u32 nr_lpis)
1557	2090	{
1558		- struct lpi_range *new;
1559		- int err = 0;
	2091	+ struct lpi_range new, old;
	2092	+
	2093	+ new = mk_lpi_range(base, nr_lpis);
	2094	+ if (!new)
	2095	+ return -ENOMEM;
1560	2096
1561	2097	mutex_lock(&lpi_range_lock);
1562	2098
1563		- new = mk_lpi_range(base, nr_lpis);
1564		- if (!new) {
1565		- err = -ENOMEM;
1566		- goto out;
	2099	+ list_for_each_entry_reverse(old, &lpi_range_list, entry) {
	2100	+ if (old->base_id < base)
	2101	+ break;
1567	2102	}
	2103	+ /*
	2104	+ * old is the last element with ->base_id smaller than base,
	2105	+ * so new goes right after it. If there are no elements with
	2106	+ * ->base_id smaller than base, &old->entry ends up pointing
	2107	+ * at the head of the list, and inserting new it the start of
	2108	+ * the list is the right thing to do in that case as well.
	2109	+ */
	2110	+ list_add(&new->entry, &old->entry);
	2111	+ /*
	2112	+ * Now check if we can merge with the preceding and/or
	2113	+ * following ranges.
	2114	+ */
	2115	+ merge_lpi_ranges(old, new);
	2116	+ merge_lpi_ranges(new, list_next_entry(new, entry));
1568	2117
1569		- list_add(&new->entry, &lpi_range_list);
1570		- list_sort(NULL, &lpi_range_list, lpi_range_cmp);
1571		- merge_lpi_ranges();
1572		-out:
1573	2118	mutex_unlock(&lpi_range_lock);
1574		- return err;
	2119	+ return 0;
1575	2120	}
1576	2121
1577	2122	static int __init its_lpi_init(u32 id_bits)
..	..	@@ -1635,23 +2180,28 @@
1635	2180	kfree(bitmap);
1636	2181	}
1637	2182
	2183	+static void gic_reset_prop_table(void *va)
	2184	+{
	2185	+ /* Priority 0xa0, Group-1, disabled */
	2186	+ memset(va, LPI_PROP_DEFAULT_PRIO \| LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
	2187	+
	2188	+ /* Make sure the GIC will observe the written configuration */
	2189	+ gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
	2190	+}
	2191	+
1638	2192	static struct page *its_allocate_prop_table(gfp_t gfp_flags)
1639	2193	{
1640	2194	struct page *prop_page;
1641	2195
1642		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2196	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2197	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2198	+ of_machine_is_compatible("rockchip,rk3566"))
1643	2199	gfp_flags \|= GFP_DMA32;
1644	2200	prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
1645	2201	if (!prop_page)
1646	2202	return NULL;
1647	2203
1648		- /* Priority 0xa0, Group-1, disabled */
1649		- memset(page_address(prop_page),
1650		- LPI_PROP_DEFAULT_PRIO \| LPI_PROP_GROUP1,
1651		- LPI_PROPBASE_SZ);
1652		-
1653		- /* Make sure the GIC will observe the written configuration */
1654		- gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ);
	2204	+ gic_reset_prop_table(page_address(prop_page));
1655	2205
1656	2206	return prop_page;
1657	2207	}
..	..	@@ -1662,20 +2212,74 @@
1662	2212	get_order(LPI_PROPBASE_SZ));
1663	2213	}
1664	2214
1665		-static int __init its_alloc_lpi_prop_table(void)
	2215	+static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
1666	2216	{
1667		- phys_addr_t paddr;
	2217	+ phys_addr_t start, end, addr_end;
	2218	+ u64 i;
1668	2219
1669		- lpi_id_bits = min_t(u32, GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
1670		- ITS_MAX_LPI_NRBITS);
1671		- gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT);
1672		- if (!gic_rdists->prop_page) {
1673		- pr_err("Failed to allocate PROPBASE\n");
1674		- return -ENOMEM;
	2220	+ /*
	2221	+ * We don't bother checking for a kdump kernel as by
	2222	+ * construction, the LPI tables are out of this kernel's
	2223	+ * memory map.
	2224	+ */
	2225	+ if (is_kdump_kernel())
	2226	+ return true;
	2227	+
	2228	+ addr_end = addr + size - 1;
	2229	+
	2230	+ for_each_reserved_mem_range(i, &start, &end) {
	2231	+ if (addr >= start && addr_end <= end)
	2232	+ return true;
1675	2233	}
1676	2234
1677		- paddr = page_to_phys(gic_rdists->prop_page);
1678		- pr_info("GIC: using LPI property table @%pa\n", &paddr);
	2235	+ /* Not found, not a good sign... */
	2236	+ pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
	2237	+ &addr, &addr_end);
	2238	+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
	2239	+ return false;
	2240	+}
	2241	+
	2242	+static int gic_reserve_range(phys_addr_t addr, unsigned long size)
	2243	+{
	2244	+ if (efi_enabled(EFI_CONFIG_TABLES))
	2245	+ return efi_mem_reserve_persistent(addr, size);
	2246	+
	2247	+ return 0;
	2248	+}
	2249	+
	2250	+static int __init its_setup_lpi_prop_table(void)
	2251	+{
	2252	+ if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
	2253	+ u64 val;
	2254	+
	2255	+ val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
	2256	+ lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
	2257	+
	2258	+ gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
	2259	+ gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
	2260	+ LPI_PROPBASE_SZ,
	2261	+ MEMREMAP_WB);
	2262	+ gic_reset_prop_table(gic_rdists->prop_table_va);
	2263	+ } else {
	2264	+ struct page *page;
	2265	+
	2266	+ lpi_id_bits = min_t(u32,
	2267	+ GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
	2268	+ ITS_MAX_LPI_NRBITS);
	2269	+ page = its_allocate_prop_table(GFP_NOWAIT);
	2270	+ if (!page) {
	2271	+ pr_err("Failed to allocate PROPBASE\n");
	2272	+ return -ENOMEM;
	2273	+ }
	2274	+
	2275	+ gic_rdists->prop_table_pa = page_to_phys(page);
	2276	+ gic_rdists->prop_table_va = page_address(page);
	2277	+ WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
	2278	+ LPI_PROPBASE_SZ));
	2279	+ }
	2280	+
	2281	+ pr_info("GICv3: using LPI property table @%pa\n",
	2282	+ &gic_rdists->prop_table_pa);
1679	2283
1680	2284	return its_lpi_init(lpi_id_bits);
1681	2285	}
..	..	@@ -1707,18 +2311,18 @@
1707	2311	}
1708	2312
1709	2313	static int its_setup_baser(struct its_node its, struct its_baser baser,
1710		- u64 cache, u64 shr, u32 psz, u32 order,
1711		- bool indirect)
	2314	+ u64 cache, u64 shr, u32 order, bool indirect)
1712	2315	{
1713	2316	u64 val = its_read_baser(its, baser);
1714	2317	u64 esz = GITS_BASER_ENTRY_SIZE(val);
1715	2318	u64 type = GITS_BASER_TYPE(val);
1716	2319	u64 baser_phys, tmp;
1717		- u32 alloc_pages;
	2320	+ u32 alloc_pages, psz;
	2321	+ struct page *page;
1718	2322	void *base;
1719	2323	gfp_t gfp_flags;
1720	2324
1721		-retry_alloc_baser:
	2325	+ psz = baser->psz;
1722	2326	alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
1723	2327	if (alloc_pages > GITS_BASER_PAGES_MAX) {
1724	2328	pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
..	..	@@ -1729,12 +2333,15 @@
1729	2333	}
1730	2334
1731	2335	gfp_flags = GFP_KERNEL \| __GFP_ZERO;
1732		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2336	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2337	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2338	+ of_machine_is_compatible("rockchip,rk3566"))
1733	2339	gfp_flags \|= GFP_DMA32;
1734		- base = (void *)__get_free_pages(gfp_flags, order);
1735		- if (!base)
	2340	+ page = alloc_pages_node(its->numa_node, gfp_flags, order);
	2341	+ if (!page)
1736	2342	return -ENOMEM;
1737	2343
	2344	+ base = (void *)page_address(page);
1738	2345	baser_phys = virt_to_phys(base);
1739	2346
1740	2347	/* Check if the physical address of the memory is above 48bits */
..	..	@@ -1777,8 +2384,11 @@
1777	2384	its_write_baser(its, baser, val);
1778	2385	tmp = baser->val;
1779	2386
1780		- if (of_machine_is_compatible("rockchip,rk3568") \|\|
1781		- of_machine_is_compatible("rockchip,rk3566")) {
	2387	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	2388	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	2389	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2390	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	2391	+ of_machine_is_compatible("rockchip,rk3588"))) {
1782	2392	if (tmp & GITS_BASER_SHAREABILITY_MASK)
1783	2393	tmp &= ~GITS_BASER_SHAREABILITY_MASK;
1784	2394	else
..	..	@@ -1799,25 +2409,6 @@
1799	2409	gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
1800	2410	}
1801	2411	goto retry_baser;
1802		- }
1803		-
1804		- if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
1805		- /*
1806		- * Page size didn't stick. Let's try a smaller
1807		- * size and retry. If we reach 4K, then
1808		- * something is horribly wrong...
1809		- */
1810		- free_pages((unsigned long)base, order);
1811		- baser->base = NULL;
1812		-
1813		- switch (psz) {
1814		- case SZ_16K:
1815		- psz = SZ_4K;
1816		- goto retry_alloc_baser;
1817		- case SZ_64K:
1818		- psz = SZ_16K;
1819		- goto retry_alloc_baser;
1820		- }
1821	2412	}
1822	2413
1823	2414	if (val != tmp) {
..	..	@@ -1845,13 +2436,14 @@
1845	2436
1846	2437	static bool its_parse_indirect_baser(struct its_node *its,
1847	2438	struct its_baser *baser,
1848		- u32 psz, u32 *order, u32 ids)
	2439	+ u32 *order, u32 ids)
1849	2440	{
1850	2441	u64 tmp = its_read_baser(its, baser);
1851	2442	u64 type = GITS_BASER_TYPE(tmp);
1852	2443	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
1853	2444	u64 val = GITS_BASER_InnerShareable \| GITS_BASER_RaWaWb;
1854	2445	u32 new_order = *order;
	2446	+ u32 psz = baser->psz;
1855	2447	bool indirect = false;
1856	2448
1857	2449	/* No need to enable Indirection if memory requirement < (psz2)bytes /
..	..	@@ -1887,14 +2479,73 @@
1887	2479	if (new_order >= MAX_ORDER) {
1888	2480	new_order = MAX_ORDER - 1;
1889	2481	ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
1890		- pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
	2482	+ pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
1891	2483	&its->phys_base, its_base_type_string[type],
1892		- its->device_ids, ids);
	2484	+ device_ids(its), ids);
1893	2485	}
1894	2486
1895	2487	*order = new_order;
1896	2488
1897	2489	return indirect;
	2490	+}
	2491	+
	2492	+static u32 compute_common_aff(u64 val)
	2493	+{
	2494	+ u32 aff, clpiaff;
	2495	+
	2496	+ aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
	2497	+ clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
	2498	+
	2499	+ return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
	2500	+}
	2501	+
	2502	+static u32 compute_its_aff(struct its_node *its)
	2503	+{
	2504	+ u64 val;
	2505	+ u32 svpet;
	2506	+
	2507	+ /*
	2508	+ * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
	2509	+ * the resulting affinity. We then use that to see if this match
	2510	+ * our own affinity.
	2511	+ */
	2512	+ svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
	2513	+ val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
	2514	+ val \|= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
	2515	+ return compute_common_aff(val);
	2516	+}
	2517	+
	2518	+static struct its_node find_sibling_its(struct its_node cur_its)
	2519	+{
	2520	+ struct its_node *its;
	2521	+ u32 aff;
	2522	+
	2523	+ if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
	2524	+ return NULL;
	2525	+
	2526	+ aff = compute_its_aff(cur_its);
	2527	+
	2528	+ list_for_each_entry(its, &its_nodes, entry) {
	2529	+ u64 baser;
	2530	+
	2531	+ if (!is_v4_1(its) \|\| its == cur_its)
	2532	+ continue;
	2533	+
	2534	+ if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
	2535	+ continue;
	2536	+
	2537	+ if (aff != compute_its_aff(its))
	2538	+ continue;
	2539	+
	2540	+ /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
	2541	+ baser = its->tables[2].val;
	2542	+ if (!(baser & GITS_BASER_VALID))
	2543	+ continue;
	2544	+
	2545	+ return its;
	2546	+ }
	2547	+
	2548	+ return NULL;
1898	2549	}
1899	2550
1900	2551	static void its_free_tables(struct its_node *its)
..	..	@@ -1910,11 +2561,58 @@
1910	2561	}
1911	2562	}
1912	2563
	2564	+static int its_probe_baser_psz(struct its_node its, struct its_baser baser)
	2565	+{
	2566	+ u64 psz = SZ_64K;
	2567	+
	2568	+ while (psz) {
	2569	+ u64 val, gpsz;
	2570	+
	2571	+ val = its_read_baser(its, baser);
	2572	+ val &= ~GITS_BASER_PAGE_SIZE_MASK;
	2573	+
	2574	+ switch (psz) {
	2575	+ case SZ_64K:
	2576	+ gpsz = GITS_BASER_PAGE_SIZE_64K;
	2577	+ break;
	2578	+ case SZ_16K:
	2579	+ gpsz = GITS_BASER_PAGE_SIZE_16K;
	2580	+ break;
	2581	+ case SZ_4K:
	2582	+ default:
	2583	+ gpsz = GITS_BASER_PAGE_SIZE_4K;
	2584	+ break;
	2585	+ }
	2586	+
	2587	+ gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
	2588	+
	2589	+ val \|= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
	2590	+ its_write_baser(its, baser, val);
	2591	+
	2592	+ if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
	2593	+ break;
	2594	+
	2595	+ switch (psz) {
	2596	+ case SZ_64K:
	2597	+ psz = SZ_16K;
	2598	+ break;
	2599	+ case SZ_16K:
	2600	+ psz = SZ_4K;
	2601	+ break;
	2602	+ case SZ_4K:
	2603	+ default:
	2604	+ return -1;
	2605	+ }
	2606	+ }
	2607	+
	2608	+ baser->psz = psz;
	2609	+ return 0;
	2610	+}
	2611	+
1913	2612	static int its_alloc_tables(struct its_node *its)
1914	2613	{
1915	2614	u64 shr = GITS_BASER_InnerShareable;
1916	2615	u64 cache = GITS_BASER_RaWaWb;
1917		- u32 psz = SZ_64K;
1918	2616	int err, i;
1919	2617
1920	2618	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
..	..	@@ -1925,37 +2623,337 @@
1925	2623	struct its_baser *baser = its->tables + i;
1926	2624	u64 val = its_read_baser(its, baser);
1927	2625	u64 type = GITS_BASER_TYPE(val);
1928		- u32 order = get_order(psz);
1929	2626	bool indirect = false;
	2627	+ u32 order;
1930	2628
1931		- switch (type) {
1932		- case GITS_BASER_TYPE_NONE:
	2629	+ if (type == GITS_BASER_TYPE_NONE)
1933	2630	continue;
1934	2631
	2632	+ if (its_probe_baser_psz(its, baser)) {
	2633	+ its_free_tables(its);
	2634	+ return -ENXIO;
	2635	+ }
	2636	+
	2637	+ order = get_order(baser->psz);
	2638	+
	2639	+ switch (type) {
1935	2640	case GITS_BASER_TYPE_DEVICE:
1936		- indirect = its_parse_indirect_baser(its, baser,
1937		- psz, &order,
1938		- its->device_ids);
	2641	+ indirect = its_parse_indirect_baser(its, baser, &order,
	2642	+ device_ids(its));
1939	2643	break;
1940	2644
1941	2645	case GITS_BASER_TYPE_VCPU:
1942		- indirect = its_parse_indirect_baser(its, baser,
1943		- psz, &order,
	2646	+ if (is_v4_1(its)) {
	2647	+ struct its_node *sibling;
	2648	+
	2649	+ WARN_ON(i != 2);
	2650	+ if ((sibling = find_sibling_its(its))) {
	2651	+ *baser = sibling->tables[2];
	2652	+ its_write_baser(its, baser, baser->val);
	2653	+ continue;
	2654	+ }
	2655	+ }
	2656	+
	2657	+ indirect = its_parse_indirect_baser(its, baser, &order,
1944	2658	ITS_MAX_VPEID_BITS);
1945	2659	break;
1946	2660	}
1947	2661
1948		- err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
	2662	+ err = its_setup_baser(its, baser, cache, shr, order, indirect);
1949	2663	if (err < 0) {
1950	2664	its_free_tables(its);
1951	2665	return err;
1952	2666	}
1953	2667
1954	2668	/* Update settings which will be used for next BASERn */
1955		- psz = baser->psz;
1956	2669	cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
1957	2670	shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
1958	2671	}
	2672	+
	2673	+ return 0;
	2674	+}
	2675	+
	2676	+static u64 inherit_vpe_l1_table_from_its(void)
	2677	+{
	2678	+ struct its_node *its;
	2679	+ u64 val;
	2680	+ u32 aff;
	2681	+
	2682	+ val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
	2683	+ aff = compute_common_aff(val);
	2684	+
	2685	+ list_for_each_entry(its, &its_nodes, entry) {
	2686	+ u64 baser, addr;
	2687	+
	2688	+ if (!is_v4_1(its))
	2689	+ continue;
	2690	+
	2691	+ if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
	2692	+ continue;
	2693	+
	2694	+ if (aff != compute_its_aff(its))
	2695	+ continue;
	2696	+
	2697	+ /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
	2698	+ baser = its->tables[2].val;
	2699	+ if (!(baser & GITS_BASER_VALID))
	2700	+ continue;
	2701	+
	2702	+ /* We have a winner! */
	2703	+ gic_data_rdist()->vpe_l1_base = its->tables[2].base;
	2704	+
	2705	+ val = GICR_VPROPBASER_4_1_VALID;
	2706	+ if (baser & GITS_BASER_INDIRECT)
	2707	+ val \|= GICR_VPROPBASER_4_1_INDIRECT;
	2708	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
	2709	+ FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
	2710	+ switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
	2711	+ case GIC_PAGE_SIZE_64K:
	2712	+ addr = GITS_BASER_ADDR_48_to_52(baser);
	2713	+ break;
	2714	+ default:
	2715	+ addr = baser & GENMASK_ULL(47, 12);
	2716	+ break;
	2717	+ }
	2718	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
	2719	+ val \|= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
	2720	+ FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
	2721	+ val \|= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
	2722	+ FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
	2723	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
	2724	+
	2725	+ return val;
	2726	+ }
	2727	+
	2728	+ return 0;
	2729	+}
	2730	+
	2731	+static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
	2732	+{
	2733	+ u32 aff;
	2734	+ u64 val;
	2735	+ int cpu;
	2736	+
	2737	+ val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
	2738	+ aff = compute_common_aff(val);
	2739	+
	2740	+ for_each_possible_cpu(cpu) {
	2741	+ void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
	2742	+
	2743	+ if (!base \|\| cpu == smp_processor_id())
	2744	+ continue;
	2745	+
	2746	+ val = gic_read_typer(base + GICR_TYPER);
	2747	+ if (aff != compute_common_aff(val))
	2748	+ continue;
	2749	+
	2750	+ /*
	2751	+ * At this point, we have a victim. This particular CPU
	2752	+ * has already booted, and has an affinity that matches
	2753	+ * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
	2754	+ * Make sure we don't write the Z bit in that case.
	2755	+ */
	2756	+ val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
	2757	+ val &= ~GICR_VPROPBASER_4_1_Z;
	2758	+
	2759	+ gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
	2760	+ *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
	2761	+
	2762	+ return val;
	2763	+ }
	2764	+
	2765	+ return 0;
	2766	+}
	2767	+
	2768	+static bool allocate_vpe_l2_table(int cpu, u32 id)
	2769	+{
	2770	+ void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
	2771	+ unsigned int psz, esz, idx, npg, gpsz;
	2772	+ u64 val;
	2773	+ struct page *page;
	2774	+ __le64 *table;
	2775	+
	2776	+ if (!gic_rdists->has_rvpeid)
	2777	+ return true;
	2778	+
	2779	+ /* Skip non-present CPUs */
	2780	+ if (!base)
	2781	+ return true;
	2782	+
	2783	+ val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
	2784	+
	2785	+ esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
	2786	+ gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
	2787	+ npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
	2788	+
	2789	+ switch (gpsz) {
	2790	+ default:
	2791	+ WARN_ON(1);
	2792	+ fallthrough;
	2793	+ case GIC_PAGE_SIZE_4K:
	2794	+ psz = SZ_4K;
	2795	+ break;
	2796	+ case GIC_PAGE_SIZE_16K:
	2797	+ psz = SZ_16K;
	2798	+ break;
	2799	+ case GIC_PAGE_SIZE_64K:
	2800	+ psz = SZ_64K;
	2801	+ break;
	2802	+ }
	2803	+
	2804	+ /* Don't allow vpe_id that exceeds single, flat table limit */
	2805	+ if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
	2806	+ return (id < (npg * psz / (esz * SZ_8)));
	2807	+
	2808	+ /* Compute 1st level table index & check if that exceeds table limit */
	2809	+ idx = id >> ilog2(psz / (esz * SZ_8));
	2810	+ if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
	2811	+ return false;
	2812	+
	2813	+ table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
	2814	+
	2815	+ /* Allocate memory for 2nd level table */
	2816	+ if (!table[idx]) {
	2817	+ page = alloc_pages(GFP_KERNEL \| __GFP_ZERO, get_order(psz));
	2818	+ if (!page)
	2819	+ return false;
	2820	+
	2821	+ /* Flush Lvl2 table to PoC if hw doesn't support coherency */
	2822	+ if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
	2823	+ gic_flush_dcache_to_poc(page_address(page), psz);
	2824	+
	2825	+ table[idx] = cpu_to_le64(page_to_phys(page) \| GITS_BASER_VALID);
	2826	+
	2827	+ /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
	2828	+ if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
	2829	+ gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
	2830	+
	2831	+ /* Ensure updated table contents are visible to RD hardware */
	2832	+ dsb(sy);
	2833	+ }
	2834	+
	2835	+ return true;
	2836	+}
	2837	+
	2838	+static int allocate_vpe_l1_table(void)
	2839	+{
	2840	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	2841	+ u64 val, gpsz, npg, pa;
	2842	+ unsigned int psz = SZ_64K;
	2843	+ unsigned int np, epp, esz;
	2844	+ struct page *page;
	2845	+
	2846	+ if (!gic_rdists->has_rvpeid)
	2847	+ return 0;
	2848	+
	2849	+ /*
	2850	+ * if VPENDBASER.Valid is set, disable any previously programmed
	2851	+ * VPE by setting PendingLast while clearing Valid. This has the
	2852	+ * effect of making sure no doorbell will be generated and we can
	2853	+ * then safely clear VPROPBASER.Valid.
	2854	+ */
	2855	+ if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
	2856	+ gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
	2857	+ vlpi_base + GICR_VPENDBASER);
	2858	+
	2859	+ /*
	2860	+ * If we can inherit the configuration from another RD, let's do
	2861	+ * so. Otherwise, we have to go through the allocation process. We
	2862	+ * assume that all RDs have the exact same requirements, as
	2863	+ * nothing will work otherwise.
	2864	+ */
	2865	+ val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
	2866	+ if (val & GICR_VPROPBASER_4_1_VALID)
	2867	+ goto out;
	2868	+
	2869	+ gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
	2870	+ if (!gic_data_rdist()->vpe_table_mask)
	2871	+ return -ENOMEM;
	2872	+
	2873	+ val = inherit_vpe_l1_table_from_its();
	2874	+ if (val & GICR_VPROPBASER_4_1_VALID)
	2875	+ goto out;
	2876	+
	2877	+ /* First probe the page size */
	2878	+ val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
	2879	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	2880	+ val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
	2881	+ gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
	2882	+ esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
	2883	+
	2884	+ switch (gpsz) {
	2885	+ default:
	2886	+ gpsz = GIC_PAGE_SIZE_4K;
	2887	+ fallthrough;
	2888	+ case GIC_PAGE_SIZE_4K:
	2889	+ psz = SZ_4K;
	2890	+ break;
	2891	+ case GIC_PAGE_SIZE_16K:
	2892	+ psz = SZ_16K;
	2893	+ break;
	2894	+ case GIC_PAGE_SIZE_64K:
	2895	+ psz = SZ_64K;
	2896	+ break;
	2897	+ }
	2898	+
	2899	+ /*
	2900	+ * Start populating the register from scratch, including RO fields
	2901	+ * (which we want to print in debug cases...)
	2902	+ */
	2903	+ val = 0;
	2904	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
	2905	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
	2906	+
	2907	+ /* How many entries per GIC page? */
	2908	+ esz++;
	2909	+ epp = psz / (esz * SZ_8);
	2910	+
	2911	+ /*
	2912	+ * If we need more than just a single L1 page, flag the table
	2913	+ * as indirect and compute the number of required L1 pages.
	2914	+ */
	2915	+ if (epp < ITS_MAX_VPEID) {
	2916	+ int nl2;
	2917	+
	2918	+ val \|= GICR_VPROPBASER_4_1_INDIRECT;
	2919	+
	2920	+ /* Number of L2 pages required to cover the VPEID space */
	2921	+ nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
	2922	+
	2923	+ /* Number of L1 pages to point to the L2 pages */
	2924	+ npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
	2925	+ } else {
	2926	+ npg = 1;
	2927	+ }
	2928	+
	2929	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
	2930	+
	2931	+ /* Right, that's the number of CPU pages we need for L1 */
	2932	+ np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
	2933	+
	2934	+ pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
	2935	+ np, npg, psz, epp, esz);
	2936	+ page = alloc_pages(GFP_ATOMIC \| __GFP_ZERO, get_order(np * PAGE_SIZE));
	2937	+ if (!page)
	2938	+ return -ENOMEM;
	2939	+
	2940	+ gic_data_rdist()->vpe_l1_base = page_address(page);
	2941	+ pa = virt_to_phys(page_address(page));
	2942	+ WARN_ON(!IS_ALIGNED(pa, psz));
	2943	+
	2944	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
	2945	+ val \|= GICR_VPROPBASER_RaWb;
	2946	+ val \|= GICR_VPROPBASER_InnerShareable;
	2947	+ val \|= GICR_VPROPBASER_4_1_Z;
	2948	+ val \|= GICR_VPROPBASER_4_1_VALID;
	2949	+
	2950	+out:
	2951	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	2952	+ cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
	2953	+
	2954	+ pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
	2955	+ smp_processor_id(), val,
	2956	+ cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
1959	2957
1960	2958	return 0;
1961	2959	}
..	..	@@ -1978,14 +2976,13 @@
1978	2976	static struct page *its_allocate_pending_table(gfp_t gfp_flags)
1979	2977	{
1980	2978	struct page *pend_page;
1981		- /*
1982		- * The pending pages have to be at least 64kB aligned,
1983		- * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1984		- */
1985		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2979	+
	2980	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2981	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2982	+ of_machine_is_compatible("rockchip,rk3566"))
1986	2983	gfp_flags \|= GFP_DMA32;
1987	2984	pend_page = alloc_pages(gfp_flags \| __GFP_ZERO,
1988		- get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
	2985	+ get_order(LPI_PENDBASE_SZ));
1989	2986	if (!pend_page)
1990	2987	return NULL;
1991	2988
..	..	@@ -1997,38 +2994,42 @@
1997	2994
1998	2995	static void its_free_pending_table(struct page *pt)
1999	2996	{
2000		- free_pages((unsigned long)page_address(pt),
2001		- get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
	2997	+ free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2002	2998	}
2003	2999
2004		-static u64 its_clear_vpend_valid(void __iomem *vlpi_base)
	3000	+/*
	3001	+ * Booting with kdump and LPIs enabled is generally fine. Any other
	3002	+ * case is wrong in the absence of firmware/EFI support.
	3003	+ */
	3004	+static bool enabled_lpis_allowed(void)
2005	3005	{
2006		- u32 count = 1000000; /* 1s! */
2007		- bool clean;
	3006	+ phys_addr_t addr;
2008	3007	u64 val;
2009	3008
2010		- val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2011		- val &= ~GICR_VPENDBASER_Valid;
2012		- gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3009	+ /* Check whether the property table is in a reserved region */
	3010	+ val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
	3011	+ addr = val & GENMASK_ULL(51, 12);
2013	3012
2014		- do {
2015		- val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2016		- clean = !(val & GICR_VPENDBASER_Dirty);
2017		- if (!clean) {
2018		- count--;
2019		- cpu_relax();
2020		- udelay(1);
2021		- }
2022		- } while (!clean && count);
2023		-
2024		- return val;
	3013	+ return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
2025	3014	}
2026	3015
2027	3016	static int __init allocate_lpi_tables(void)
2028	3017	{
	3018	+ u64 val;
2029	3019	int err, cpu;
2030	3020
2031		- err = its_alloc_lpi_prop_table();
	3021	+ /*
	3022	+ * If LPIs are enabled while we run this from the boot CPU,
	3023	+ * flag the RD tables as pre-allocated if the stars do align.
	3024	+ */
	3025	+ val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
	3026	+ if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
	3027	+ gic_rdists->flags \|= (RDIST_FLAGS_RD_TABLES_PREALLOCATED \|
	3028	+ RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
	3029	+ pr_info("GICv3: Using preallocated redistributor tables\n");
	3030	+ }
	3031	+
	3032	+ err = its_setup_lpi_prop_table();
2032	3033	if (err)
2033	3034	return err;
2034	3035
..	..	@@ -2052,6 +3053,46 @@
2052	3053	return 0;
2053	3054	}
2054	3055
	3056	+static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
	3057	+{
	3058	+ u32 count = 1000000; /* 1s! */
	3059	+ bool clean;
	3060	+ u64 val;
	3061	+
	3062	+ do {
	3063	+ val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
	3064	+ clean = !(val & GICR_VPENDBASER_Dirty);
	3065	+ if (!clean) {
	3066	+ count--;
	3067	+ cpu_relax();
	3068	+ udelay(1);
	3069	+ }
	3070	+ } while (!clean && count);
	3071	+
	3072	+ if (unlikely(!clean))
	3073	+ pr_err_ratelimited("ITS virtual pending table not cleaning\n");
	3074	+
	3075	+ return val;
	3076	+}
	3077	+
	3078	+static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
	3079	+{
	3080	+ u64 val;
	3081	+
	3082	+ /* Make sure we wait until the RD is done with the initial scan */
	3083	+ val = read_vpend_dirty_clear(vlpi_base);
	3084	+ val &= ~GICR_VPENDBASER_Valid;
	3085	+ val &= ~clr;
	3086	+ val \|= set;
	3087	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3088	+
	3089	+ val = read_vpend_dirty_clear(vlpi_base);
	3090	+ if (unlikely(val & GICR_VPENDBASER_Dirty))
	3091	+ val \|= GICR_VPENDBASER_PendingLast;
	3092	+
	3093	+ return val;
	3094	+}
	3095	+
2055	3096	static void its_cpu_init_lpis(void)
2056	3097	{
2057	3098	void __iomem *rbase = gic_data_rdist_rd_base();
..	..	@@ -2062,11 +3103,34 @@
2062	3103	if (gic_data_rdist()->lpi_enabled)
2063	3104	return;
2064	3105
	3106	+ val = readl_relaxed(rbase + GICR_CTLR);
	3107	+ if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
	3108	+ (val & GICR_CTLR_ENABLE_LPIS)) {
	3109	+ /*
	3110	+ * Check that we get the same property table on all
	3111	+ * RDs. If we don't, this is hopeless.
	3112	+ */
	3113	+ paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
	3114	+ paddr &= GENMASK_ULL(51, 12);
	3115	+ if (WARN_ON(gic_rdists->prop_table_pa != paddr))
	3116	+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
	3117	+
	3118	+ paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
	3119	+ paddr &= GENMASK_ULL(51, 16);
	3120	+
	3121	+ WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
	3122	+ its_free_pending_table(gic_data_rdist()->pend_page);
	3123	+ gic_data_rdist()->pend_page = NULL;
	3124	+
	3125	+ goto out;
	3126	+ }
	3127	+
2065	3128	pend_page = gic_data_rdist()->pend_page;
2066	3129	paddr = page_to_phys(pend_page);
	3130	+ WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
2067	3131
2068	3132	/* set PROPBASE */
2069		- val = (page_to_phys(gic_rdists->prop_page) \|
	3133	+ val = (gic_rdists->prop_table_pa \|
2070	3134	GICR_PROPBASER_InnerShareable \|
2071	3135	GICR_PROPBASER_RaWaWb \|
2072	3136	((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
..	..	@@ -2074,7 +3138,11 @@
2074	3138	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
2075	3139	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
2076	3140
2077		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3141	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	3142	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	3143	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3144	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	3145	+ of_machine_is_compatible("rockchip,rk3588")))
2078	3146	tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
2079	3147
2080	3148	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
..	..	@@ -2101,7 +3169,11 @@
2101	3169	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
2102	3170	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
2103	3171
2104		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3172	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	3173	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	3174	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3175	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	3176	+ of_machine_is_compatible("rockchip,rk3588")))
2105	3177	tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
2106	3178
2107	3179	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
..	..	@@ -2120,12 +3192,12 @@
2120	3192	val \|= GICR_CTLR_ENABLE_LPIS;
2121	3193	writel_relaxed(val, rbase + GICR_CTLR);
2122	3194
2123		- if (gic_rdists->has_vlpis) {
	3195	+ if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
2124	3196	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2125	3197
2126	3198	/*
2127	3199	* It's possible for CPU to receive VLPIs before it is
2128		- * sheduled as a vPE, especially for the first CPU, and the
	3200	+ * scheduled as a vPE, especially for the first CPU, and the
2129	3201	* VLPI with INTID larger than 2^(IDbits+1) will be considered
2130	3202	* as out of range and dropped by GIC.
2131	3203	* So we initialize IDbits to known value to avoid VLPI drop.
..	..	@@ -2133,22 +3205,33 @@
2133	3205	val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2134	3206	pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
2135	3207	smp_processor_id(), val);
2136		- gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	3208	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2137	3209
2138	3210	/*
2139	3211	* Also clear Valid bit of GICR_VPENDBASER, in case some
2140	3212	* ancient programming gets left in and has possibility of
2141	3213	* corrupting memory.
2142	3214	*/
2143		- val = its_clear_vpend_valid(vlpi_base);
2144		- WARN_ON(val & GICR_VPENDBASER_Dirty);
	3215	+ val = its_clear_vpend_valid(vlpi_base, 0, 0);
	3216	+ }
	3217	+
	3218	+ if (allocate_vpe_l1_table()) {
	3219	+ /*
	3220	+ * If the allocation has failed, we're in massive trouble.
	3221	+ * Disable direct injection, and pray that no VM was
	3222	+ * already running...
	3223	+ */
	3224	+ gic_rdists->has_rvpeid = false;
	3225	+ gic_rdists->has_vlpis = false;
2145	3226	}
2146	3227
2147	3228	/* Make sure the GIC has seen the above */
2148	3229	dsb(sy);
	3230	+out:
2149	3231	gic_data_rdist()->lpi_enabled = true;
2150		- pr_info("GICv3: CPU%d: using LPI pending table @%pa\n",
	3232	+ pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
2151	3233	smp_processor_id(),
	3234	+ gic_data_rdist()->pend_page ? "allocated" : "reserved",
2152	3235	&paddr);
2153	3236	}
2154	3237
..	..	@@ -2234,7 +3317,8 @@
2234	3317	return NULL;
2235	3318	}
2236	3319
2237		-static bool its_alloc_table_entry(struct its_baser *baser, u32 id)
	3320	+static bool its_alloc_table_entry(struct its_node *its,
	3321	+ struct its_baser *baser, u32 id)
2238	3322	{
2239	3323	struct page *page;
2240	3324	u32 esz, idx;
..	..	@@ -2256,9 +3340,12 @@
2256	3340	if (!table[idx]) {
2257	3341	gfp_t gfp_flags = GFP_KERNEL \| __GFP_ZERO;
2258	3342
2259		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3343	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3344	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3345	+ of_machine_is_compatible("rockchip,rk3566"))
2260	3346	gfp_flags \|= GFP_DMA32;
2261		- page = alloc_pages(gfp_flags, get_order(baser->psz));
	3347	+ page = alloc_pages_node(its->numa_node, gfp_flags,
	3348	+ get_order(baser->psz));
2262	3349	if (!page)
2263	3350	return false;
2264	3351
..	..	@@ -2287,14 +3374,15 @@
2287	3374
2288	3375	/* Don't allow device id that exceeds ITS hardware limit */
2289	3376	if (!baser)
2290		- return (ilog2(dev_id) < its->device_ids);
	3377	+ return (ilog2(dev_id) < device_ids(its));
2291	3378
2292		- return its_alloc_table_entry(baser, dev_id);
	3379	+ return its_alloc_table_entry(its, baser, dev_id);
2293	3380	}
2294	3381
2295	3382	static bool its_alloc_vpe_table(u32 vpe_id)
2296	3383	{
2297	3384	struct its_node *its;
	3385	+ int cpu;
2298	3386
2299	3387	/*
2300	3388	* Make sure the L2 tables are allocated on all v4 ITSs. We
..	..	@@ -2306,14 +3394,27 @@
2306	3394	list_for_each_entry(its, &its_nodes, entry) {
2307	3395	struct its_baser *baser;
2308	3396
2309		- if (!its->is_v4)
	3397	+ if (!is_v4(its))
2310	3398	continue;
2311	3399
2312	3400	baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
2313	3401	if (!baser)
2314	3402	return false;
2315	3403
2316		- if (!its_alloc_table_entry(baser, vpe_id))
	3404	+ if (!its_alloc_table_entry(its, baser, vpe_id))
	3405	+ return false;
	3406	+ }
	3407	+
	3408	+ /* Non v4.1? No need to iterate RDs and go back early. */
	3409	+ if (!gic_rdists->has_rvpeid)
	3410	+ return true;
	3411	+
	3412	+ /*
	3413	+ * Make sure the L2 tables are allocated for all copies of
	3414	+ * the L1 table on all v4.1 RDs.
	3415	+ */
	3416	+ for_each_possible_cpu(cpu) {
	3417	+ if (!allocate_vpe_l2_table(cpu, vpe_id))
2317	3418	return false;
2318	3419	}
2319	3420
..	..	@@ -2346,12 +3447,18 @@
2346	3447	* sized as a power of two (and you need at least one bit...).
2347	3448	*/
2348	3449	nr_ites = max(2, nvecs);
2349		- sz = nr_ites * its->ite_size;
	3450	+ sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
2350	3451	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
2351	3452	gfp_flags = GFP_KERNEL;
2352		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3453	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3454	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3455	+ of_machine_is_compatible("rockchip,rk3566")) {
2353	3456	gfp_flags \|= GFP_DMA32;
2354		- itt = (void *)__get_free_pages(gfp_flags, get_order(sz));
	3457	+ itt = (void *)__get_free_pages(gfp_flags, get_order(sz));
	3458	+ } else {
	3459	+ itt = kzalloc_node(sz, gfp_flags, its->numa_node);
	3460	+ }
	3461	+
2355	3462	if (alloc_lpis) {
2356	3463	lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
2357	3464	if (lpi_map)
..	..	@@ -2365,7 +3472,14 @@
2365	3472
2366	3473	if (!dev \|\| !itt \|\| !col_map \|\| (!lpi_map && alloc_lpis)) {
2367	3474	kfree(dev);
2368		- free_pages((unsigned long)itt, get_order(sz));
	3475	+
	3476	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3477	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3478	+ of_machine_is_compatible("rockchip,rk3566"))
	3479	+ free_pages((unsigned long)itt, get_order(sz));
	3480	+ else
	3481	+ kfree(itt);
	3482	+
2369	3483	kfree(lpi_map);
2370	3484	kfree(col_map);
2371	3485	return NULL;
..	..	@@ -2381,7 +3495,7 @@
2381	3495	dev->event_map.col_map = col_map;
2382	3496	dev->event_map.lpi_base = lpi_base;
2383	3497	dev->event_map.nr_lpis = nr_lpis;
2384		- mutex_init(&dev->event_map.vlpi_lock);
	3498	+ raw_spin_lock_init(&dev->event_map.vlpi_lock);
2385	3499	dev->device_id = dev_id;
2386	3500	INIT_LIST_HEAD(&dev->entry);
2387	3501
..	..	@@ -2402,7 +3516,15 @@
2402	3516	raw_spin_lock_irqsave(&its_dev->its->lock, flags);
2403	3517	list_del(&its_dev->entry);
2404	3518	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
2405		- free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));
	3519	+ kfree(its_dev->event_map.col_map);
	3520	+
	3521	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3522	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3523	+ of_machine_is_compatible("rockchip,rk3566"))
	3524	+ free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));
	3525	+ else
	3526	+ kfree(its_dev->itt);
	3527	+
2406	3528	kfree(its_dev);
2407	3529	}
2408	3530
..	..	@@ -2410,6 +3532,7 @@
2410	3532	{
2411	3533	int idx;
2412	3534
	3535	+ /* Find a free LPI region in lpi_map and allocate them. */
2413	3536	idx = bitmap_find_free_region(dev->event_map.lpi_map,
2414	3537	dev->event_map.nr_lpis,
2415	3538	get_count_order(nvecs));
..	..	@@ -2417,7 +3540,6 @@
2417	3540	return -ENOSPC;
2418	3541
2419	3542	*hwirq = dev->event_map.lpi_base + idx;
2420		- set_bit(idx, dev->event_map.lpi_map);
2421	3543
2422	3544	return 0;
2423	3545	}
..	..	@@ -2432,7 +3554,7 @@
2432	3554	int err = 0;
2433	3555
2434	3556	/*
2435		- * We ignore "dev" entierely, and rely on the dev_id that has
	3557	+ * We ignore "dev" entirely, and rely on the dev_id that has
2436	3558	* been passed via the scratchpad. This limits this domain's
2437	3559	* usefulness to upper layers that definitely know that they
2438	3560	* are built on top of the ITS.
..	..	@@ -2511,12 +3633,17 @@
2511	3633	{
2512	3634	msi_alloc_info_t *info = args;
2513	3635	struct its_device *its_dev = info->scratchpad[0].ptr;
	3636	+ struct its_node *its = its_dev->its;
2514	3637	struct irq_data *irqd;
2515	3638	irq_hw_number_t hwirq;
2516	3639	int err;
2517	3640	int i;
2518	3641
2519	3642	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
	3643	+ if (err)
	3644	+ return err;
	3645	+
	3646	+ err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
2520	3647	if (err)
2521	3648	return err;
2522	3649
..	..	@@ -2543,22 +3670,13 @@
2543	3670	{
2544	3671	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2545	3672	u32 event = its_get_event_id(d);
2546		- const struct cpumask *cpu_mask = cpu_online_mask;
2547	3673	int cpu;
2548	3674
2549		- /* get the cpu_mask of local node */
2550		- if (its_dev->its->numa_node >= 0)
2551		- cpu_mask = cpumask_of_node(its_dev->its->numa_node);
	3675	+ cpu = its_select_cpu(d, cpu_online_mask);
	3676	+ if (cpu < 0 \|\| cpu >= nr_cpu_ids)
	3677	+ return -EINVAL;
2552	3678
2553		- /* Bind the LPI to the first possible CPU */
2554		- cpu = cpumask_first_and(cpu_mask, cpu_online_mask);
2555		- if (cpu >= nr_cpu_ids) {
2556		- if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)
2557		- return -EINVAL;
2558		-
2559		- cpu = cpumask_first(cpu_online_mask);
2560		- }
2561		-
	3679	+ its_inc_lpi_count(d, cpu);
2562	3680	its_dev->event_map.col_map[event] = cpu;
2563	3681	irq_data_update_effective_affinity(d, cpumask_of(cpu));
2564	3682
..	..	@@ -2573,6 +3691,7 @@
2573	3691	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2574	3692	u32 event = its_get_event_id(d);
2575	3693
	3694	+ its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
2576	3695	/* Stop the delivery of interrupts */
2577	3696	its_send_discard(its_dev, event);
2578	3697	}
..	..	@@ -2600,7 +3719,7 @@
2600	3719
2601	3720	/*
2602	3721	* If all interrupts have been freed, start mopping the
2603		- * floor. This is conditionned on the device not being shared.
	3722	+ * floor. This is conditioned on the device not being shared.
2604	3723	*/
2605	3724	if (!its_dev->shared &&
2606	3725	bitmap_empty(its_dev->event_map.lpi_map,
..	..	@@ -2608,7 +3727,6 @@
2608	3727	its_lpi_free(its_dev->event_map.lpi_map,
2609	3728	its_dev->event_map.lpi_base,
2610	3729	its_dev->event_map.nr_lpis);
2611		- kfree(its_dev->event_map.col_map);
2612	3730
2613	3731	/* Unmap device/itt */
2614	3732	its_send_mapd(its_dev, 0);
..	..	@@ -2630,7 +3748,7 @@
2630	3748	/*
2631	3749	* This is insane.
2632	3750	*
2633		- * If a GICv4 doesn't implement Direct LPIs (which is extremely
	3751	+ * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
2634	3752	* likely), the only way to perform an invalidate is to use a fake
2635	3753	* device to issue an INV command, implying that the LPI has first
2636	3754	* been mapped to some event on that device. Since this is not exactly
..	..	@@ -2638,9 +3756,20 @@
2638	3756	* only issue an UNMAP if we're short on available slots.
2639	3757	*
2640	3758	* Broken by design(tm).
	3759	+ *
	3760	+ * GICv4.1, on the other hand, mandates that we're able to invalidate
	3761	+ * by writing to a MMIO register. It doesn't implement the whole of
	3762	+ * DirectLPI, but that's good enough. And most of the time, we don't
	3763	+ * even have to invalidate anything, as the redistributor can be told
	3764	+ * whether to generate a doorbell or not (we thus leave it enabled,
	3765	+ * always).
2641	3766	*/
2642	3767	static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
2643	3768	{
	3769	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3770	+ if (gic_rdists->has_rvpeid)
	3771	+ return;
	3772	+
2644	3773	/* Already unmapped? */
2645	3774	if (vpe->vpe_proxy_event == -1)
2646	3775	return;
..	..	@@ -2663,6 +3792,10 @@
2663	3792
2664	3793	static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
2665	3794	{
	3795	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3796	+ if (gic_rdists->has_rvpeid)
	3797	+ return;
	3798	+
2666	3799	if (!gic_rdists->has_direct_lpi) {
2667	3800	unsigned long flags;
2668	3801
..	..	@@ -2674,6 +3807,10 @@
2674	3807
2675	3808	static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
2676	3809	{
	3810	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3811	+ if (gic_rdists->has_rvpeid)
	3812	+ return;
	3813	+
2677	3814	/* Already mapped? */
2678	3815	if (vpe->vpe_proxy_event != -1)
2679	3816	return;
..	..	@@ -2696,13 +3833,16 @@
2696	3833	unsigned long flags;
2697	3834	struct its_collection *target_col;
2698	3835
	3836	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3837	+ if (gic_rdists->has_rvpeid)
	3838	+ return;
	3839	+
2699	3840	if (gic_rdists->has_direct_lpi) {
2700	3841	void __iomem *rdbase;
2701	3842
2702	3843	rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
2703	3844	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2704		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2705		- cpu_relax();
	3845	+ wait_for_syncr(rdbase);
2706	3846
2707	3847	return;
2708	3848	}
..	..	@@ -2723,25 +3863,58 @@
2723	3863	bool force)
2724	3864	{
2725	3865	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2726		- int cpu = cpumask_first(mask_val);
	3866	+ int from, cpu = cpumask_first(mask_val);
	3867	+ unsigned long flags;
2727	3868
2728	3869	/*
2729	3870	* Changing affinity is mega expensive, so let's be as lazy as
2730	3871	* we can and only do it if we really have to. Also, if mapped
2731	3872	* into the proxy device, we need to move the doorbell
2732	3873	* interrupt to its new location.
	3874	+ *
	3875	+ * Another thing is that changing the affinity of a vPE affects
	3876	+ * other interrupts such as all the vLPIs that are routed to
	3877	+ * this vPE. This means that the irq_desc lock is not enough to
	3878	+ * protect us, and that we must ensure nobody samples vpe->col_idx
	3879	+ * during the update, hence the lock below which must also be
	3880	+ * taken on any vLPI handling path that evaluates vpe->col_idx.
2733	3881	*/
2734		- if (vpe->col_idx != cpu) {
2735		- int from = vpe->col_idx;
	3882	+ from = vpe_to_cpuid_lock(vpe, &flags);
	3883	+ if (from == cpu)
	3884	+ goto out;
2736	3885
2737		- vpe->col_idx = cpu;
2738		- its_send_vmovp(vpe);
2739		- its_vpe_db_proxy_move(vpe, from, cpu);
2740		- }
	3886	+ vpe->col_idx = cpu;
2741	3887
	3888	+ /*
	3889	+ * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
	3890	+ * is sharing its VPE table with the current one.
	3891	+ */
	3892	+ if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
	3893	+ cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
	3894	+ goto out;
	3895	+
	3896	+ its_send_vmovp(vpe);
	3897	+ its_vpe_db_proxy_move(vpe, from, cpu);
	3898	+
	3899	+out:
2742	3900	irq_data_update_effective_affinity(d, cpumask_of(cpu));
	3901	+ vpe_to_cpuid_unlock(vpe, flags);
2743	3902
2744	3903	return IRQ_SET_MASK_OK_DONE;
	3904	+}
	3905	+
	3906	+static void its_wait_vpt_parse_complete(void)
	3907	+{
	3908	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	3909	+ u64 val;
	3910	+
	3911	+ if (!gic_rdists->has_vpend_valid_dirty)
	3912	+ return;
	3913	+
	3914	+ WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
	3915	+ val,
	3916	+ !(val & GICR_VPENDBASER_Dirty),
	3917	+ 10, 500));
2745	3918	}
2746	3919
2747	3920	static void its_vpe_schedule(struct its_vpe *vpe)
..	..	@@ -2755,12 +3928,12 @@
2755	3928	val \|= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2756	3929	val \|= GICR_VPROPBASER_RaWb;
2757	3930	val \|= GICR_VPROPBASER_InnerShareable;
2758		- gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	3931	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2759	3932
2760	3933	val = virt_to_phys(page_address(vpe->vpt_page)) &
2761	3934	GENMASK_ULL(51, 16);
2762	3935	val \|= GICR_VPENDBASER_RaWaWb;
2763		- val \|= GICR_VPENDBASER_NonShareable;
	3936	+ val \|= GICR_VPENDBASER_InnerShareable;
2764	3937	/*
2765	3938	* There is no good way of finding out if the pending table is
2766	3939	* empty as we can race against the doorbell interrupt very
..	..	@@ -2773,7 +3946,7 @@
2773	3946	val \|= GICR_VPENDBASER_PendingLast;
2774	3947	val \|= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
2775	3948	val \|= GICR_VPENDBASER_Valid;
2776		- gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3949	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2777	3950	}
2778	3951
2779	3952	static void its_vpe_deschedule(struct its_vpe *vpe)
..	..	@@ -2781,16 +3954,10 @@
2781	3954	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2782	3955	u64 val;
2783	3956
2784		- val = its_clear_vpend_valid(vlpi_base);
	3957	+ val = its_clear_vpend_valid(vlpi_base, 0, 0);
2785	3958
2786		- if (unlikely(val & GICR_VPENDBASER_Dirty)) {
2787		- pr_err_ratelimited("ITS virtual pending table not cleaning\n");
2788		- vpe->idai = false;
2789		- vpe->pending_last = true;
2790		- } else {
2791		- vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
2792		- vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2793		- }
	3959	+ vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
	3960	+ vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2794	3961	}
2795	3962
2796	3963	static void its_vpe_invall(struct its_vpe *vpe)
..	..	@@ -2798,7 +3965,7 @@
2798	3965	struct its_node *its;
2799	3966
2800	3967	list_for_each_entry(its, &its_nodes, entry) {
2801		- if (!its->is_v4)
	3968	+ if (!is_v4(its))
2802	3969	continue;
2803	3970
2804	3971	if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
..	..	@@ -2827,6 +3994,10 @@
2827	3994	its_vpe_deschedule(vpe);
2828	3995	return 0;
2829	3996
	3997	+ case COMMIT_VPE:
	3998	+ its_wait_vpt_parse_complete();
	3999	+ return 0;
	4000	+
2830	4001	case INVALL_VPE:
2831	4002	its_vpe_invall(vpe);
2832	4003	return 0;
..	..	@@ -2853,16 +4024,10 @@
2853	4024	{
2854	4025	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2855	4026
2856		- if (gic_rdists->has_direct_lpi) {
2857		- void __iomem *rdbase;
2858		-
2859		- rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
2860		- gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);
2861		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2862		- cpu_relax();
2863		- } else {
	4027	+ if (gic_rdists->has_direct_lpi)
	4028	+ __direct_lpi_inv(d, d->parent_data->hwirq);
	4029	+ else
2864	4030	its_vpe_send_cmd(vpe, its_send_inv);
2865		- }
2866	4031	}
2867	4032
2868	4033	static void its_vpe_mask_irq(struct irq_data *d)
..	..	@@ -2901,8 +4066,7 @@
2901	4066	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
2902	4067	} else {
2903	4068	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2904		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2905		- cpu_relax();
	4069	+ wait_for_syncr(rdbase);
2906	4070	}
2907	4071	} else {
2908	4072	if (state)
..	..	@@ -2928,6 +4092,375 @@
2928	4092	.irq_retrigger = its_vpe_retrigger,
2929	4093	.irq_set_irqchip_state = its_vpe_set_irqchip_state,
2930	4094	.irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
	4095	+};
	4096	+
	4097	+static struct its_node *find_4_1_its(void)
	4098	+{
	4099	+ static struct its_node *its = NULL;
	4100	+
	4101	+ if (!its) {
	4102	+ list_for_each_entry(its, &its_nodes, entry) {
	4103	+ if (is_v4_1(its))
	4104	+ return its;
	4105	+ }
	4106	+
	4107	+ /* Oops? */
	4108	+ its = NULL;
	4109	+ }
	4110	+
	4111	+ return its;
	4112	+}
	4113	+
	4114	+static void its_vpe_4_1_send_inv(struct irq_data *d)
	4115	+{
	4116	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4117	+ struct its_node *its;
	4118	+
	4119	+ /*
	4120	+ * GICv4.1 wants doorbells to be invalidated using the
	4121	+ * INVDB command in order to be broadcast to all RDs. Send
	4122	+ * it to the first valid ITS, and let the HW do its magic.
	4123	+ */
	4124	+ its = find_4_1_its();
	4125	+ if (its)
	4126	+ its_send_invdb(its, vpe);
	4127	+}
	4128	+
	4129	+static void its_vpe_4_1_mask_irq(struct irq_data *d)
	4130	+{
	4131	+ lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
	4132	+ its_vpe_4_1_send_inv(d);
	4133	+}
	4134	+
	4135	+static void its_vpe_4_1_unmask_irq(struct irq_data *d)
	4136	+{
	4137	+ lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
	4138	+ its_vpe_4_1_send_inv(d);
	4139	+}
	4140	+
	4141	+static void its_vpe_4_1_schedule(struct its_vpe *vpe,
	4142	+ struct its_cmd_info *info)
	4143	+{
	4144	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	4145	+ u64 val = 0;
	4146	+
	4147	+ /* Schedule the VPE */
	4148	+ val \|= GICR_VPENDBASER_Valid;
	4149	+ val \|= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
	4150	+ val \|= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
	4151	+ val \|= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
	4152	+
	4153	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	4154	+}
	4155	+
	4156	+static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
	4157	+ struct its_cmd_info *info)
	4158	+{
	4159	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	4160	+ u64 val;
	4161	+
	4162	+ if (info->req_db) {
	4163	+ unsigned long flags;
	4164	+
	4165	+ /*
	4166	+ * vPE is going to block: make the vPE non-resident with
	4167	+ * PendingLast clear and DB set. The GIC guarantees that if
	4168	+ * we read-back PendingLast clear, then a doorbell will be
	4169	+ * delivered when an interrupt comes.
	4170	+ *
	4171	+ * Note the locking to deal with the concurrent update of
	4172	+ * pending_last from the doorbell interrupt handler that can
	4173	+ * run concurrently.
	4174	+ */
	4175	+ raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
	4176	+ val = its_clear_vpend_valid(vlpi_base,
	4177	+ GICR_VPENDBASER_PendingLast,
	4178	+ GICR_VPENDBASER_4_1_DB);
	4179	+ vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
	4180	+ raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
	4181	+ } else {
	4182	+ /*
	4183	+ * We're not blocking, so just make the vPE non-resident
	4184	+ * with PendingLast set, indicating that we'll be back.
	4185	+ */
	4186	+ val = its_clear_vpend_valid(vlpi_base,
	4187	+ 0,
	4188	+ GICR_VPENDBASER_PendingLast);
	4189	+ vpe->pending_last = true;
	4190	+ }
	4191	+}
	4192	+
	4193	+static void its_vpe_4_1_invall(struct its_vpe *vpe)
	4194	+{
	4195	+ void __iomem *rdbase;
	4196	+ unsigned long flags;
	4197	+ u64 val;
	4198	+ int cpu;
	4199	+
	4200	+ val = GICR_INVALLR_V;
	4201	+ val \|= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
	4202	+
	4203	+ /* Target the redistributor this vPE is currently known on */
	4204	+ cpu = vpe_to_cpuid_lock(vpe, &flags);
	4205	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4206	+ rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
	4207	+ gic_write_lpir(val, rdbase + GICR_INVALLR);
	4208	+
	4209	+ wait_for_syncr(rdbase);
	4210	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4211	+ vpe_to_cpuid_unlock(vpe, flags);
	4212	+}
	4213	+
	4214	+static int its_vpe_4_1_set_vcpu_affinity(struct irq_data d, void vcpu_info)
	4215	+{
	4216	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4217	+ struct its_cmd_info *info = vcpu_info;
	4218	+
	4219	+ switch (info->cmd_type) {
	4220	+ case SCHEDULE_VPE:
	4221	+ its_vpe_4_1_schedule(vpe, info);
	4222	+ return 0;
	4223	+
	4224	+ case DESCHEDULE_VPE:
	4225	+ its_vpe_4_1_deschedule(vpe, info);
	4226	+ return 0;
	4227	+
	4228	+ case COMMIT_VPE:
	4229	+ its_wait_vpt_parse_complete();
	4230	+ return 0;
	4231	+
	4232	+ case INVALL_VPE:
	4233	+ its_vpe_4_1_invall(vpe);
	4234	+ return 0;
	4235	+
	4236	+ default:
	4237	+ return -EINVAL;
	4238	+ }
	4239	+}
	4240	+
	4241	+static struct irq_chip its_vpe_4_1_irq_chip = {
	4242	+ .name = "GICv4.1-vpe",
	4243	+ .irq_mask = its_vpe_4_1_mask_irq,
	4244	+ .irq_unmask = its_vpe_4_1_unmask_irq,
	4245	+ .irq_eoi = irq_chip_eoi_parent,
	4246	+ .irq_set_affinity = its_vpe_set_affinity,
	4247	+ .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
	4248	+};
	4249	+
	4250	+static void its_configure_sgi(struct irq_data *d, bool clear)
	4251	+{
	4252	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4253	+ struct its_cmd_desc desc;
	4254	+
	4255	+ desc.its_vsgi_cmd.vpe = vpe;
	4256	+ desc.its_vsgi_cmd.sgi = d->hwirq;
	4257	+ desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
	4258	+ desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
	4259	+ desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
	4260	+ desc.its_vsgi_cmd.clear = clear;
	4261	+
	4262	+ /*
	4263	+ * GICv4.1 allows us to send VSGI commands to any ITS as long as the
	4264	+ * destination VPE is mapped there. Since we map them eagerly at
	4265	+ * activation time, we're pretty sure the first GICv4.1 ITS will do.
	4266	+ */
	4267	+ its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
	4268	+}
	4269	+
	4270	+static void its_sgi_mask_irq(struct irq_data *d)
	4271	+{
	4272	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4273	+
	4274	+ vpe->sgi_config[d->hwirq].enabled = false;
	4275	+ its_configure_sgi(d, false);
	4276	+}
	4277	+
	4278	+static void its_sgi_unmask_irq(struct irq_data *d)
	4279	+{
	4280	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4281	+
	4282	+ vpe->sgi_config[d->hwirq].enabled = true;
	4283	+ its_configure_sgi(d, false);
	4284	+}
	4285	+
	4286	+static int its_sgi_set_affinity(struct irq_data *d,
	4287	+ const struct cpumask *mask_val,
	4288	+ bool force)
	4289	+{
	4290	+ /*
	4291	+ * There is no notion of affinity for virtual SGIs, at least
	4292	+ * not on the host (since they can only be targeting a vPE).
	4293	+ * Tell the kernel we've done whatever it asked for.
	4294	+ */
	4295	+ irq_data_update_effective_affinity(d, mask_val);
	4296	+ return IRQ_SET_MASK_OK;
	4297	+}
	4298	+
	4299	+static int its_sgi_set_irqchip_state(struct irq_data *d,
	4300	+ enum irqchip_irq_state which,
	4301	+ bool state)
	4302	+{
	4303	+ if (which != IRQCHIP_STATE_PENDING)
	4304	+ return -EINVAL;
	4305	+
	4306	+ if (state) {
	4307	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4308	+ struct its_node *its = find_4_1_its();
	4309	+ u64 val;
	4310	+
	4311	+ val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
	4312	+ val \|= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
	4313	+ writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
	4314	+ } else {
	4315	+ its_configure_sgi(d, true);
	4316	+ }
	4317	+
	4318	+ return 0;
	4319	+}
	4320	+
	4321	+static int its_sgi_get_irqchip_state(struct irq_data *d,
	4322	+ enum irqchip_irq_state which, bool *val)
	4323	+{
	4324	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4325	+ void __iomem *base;
	4326	+ unsigned long flags;
	4327	+ u32 count = 1000000; /* 1s! */
	4328	+ u32 status;
	4329	+ int cpu;
	4330	+
	4331	+ if (which != IRQCHIP_STATE_PENDING)
	4332	+ return -EINVAL;
	4333	+
	4334	+ /*
	4335	+ * Locking galore! We can race against two different events:
	4336	+ *
	4337	+ * - Concurrent vPE affinity change: we must make sure it cannot
	4338	+ * happen, or we'll talk to the wrong redistributor. This is
	4339	+ * identical to what happens with vLPIs.
	4340	+ *
	4341	+ * - Concurrent VSGIPENDR access: As it involves accessing two
	4342	+ * MMIO registers, this must be made atomic one way or another.
	4343	+ */
	4344	+ cpu = vpe_to_cpuid_lock(vpe, &flags);
	4345	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4346	+ base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
	4347	+ writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
	4348	+ do {
	4349	+ status = readl_relaxed(base + GICR_VSGIPENDR);
	4350	+ if (!(status & GICR_VSGIPENDR_BUSY))
	4351	+ goto out;
	4352	+
	4353	+ count--;
	4354	+ if (!count) {
	4355	+ pr_err_ratelimited("Unable to get SGI status\n");
	4356	+ goto out;
	4357	+ }
	4358	+ cpu_relax();
	4359	+ udelay(1);
	4360	+ } while (count);
	4361	+
	4362	+out:
	4363	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4364	+ vpe_to_cpuid_unlock(vpe, flags);
	4365	+
	4366	+ if (!count)
	4367	+ return -ENXIO;
	4368	+
	4369	+ *val = !!(status & (1 << d->hwirq));
	4370	+
	4371	+ return 0;
	4372	+}
	4373	+
	4374	+static int its_sgi_set_vcpu_affinity(struct irq_data d, void vcpu_info)
	4375	+{
	4376	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4377	+ struct its_cmd_info *info = vcpu_info;
	4378	+
	4379	+ switch (info->cmd_type) {
	4380	+ case PROP_UPDATE_VSGI:
	4381	+ vpe->sgi_config[d->hwirq].priority = info->priority;
	4382	+ vpe->sgi_config[d->hwirq].group = info->group;
	4383	+ its_configure_sgi(d, false);
	4384	+ return 0;
	4385	+
	4386	+ default:
	4387	+ return -EINVAL;
	4388	+ }
	4389	+}
	4390	+
	4391	+static struct irq_chip its_sgi_irq_chip = {
	4392	+ .name = "GICv4.1-sgi",
	4393	+ .irq_mask = its_sgi_mask_irq,
	4394	+ .irq_unmask = its_sgi_unmask_irq,
	4395	+ .irq_set_affinity = its_sgi_set_affinity,
	4396	+ .irq_set_irqchip_state = its_sgi_set_irqchip_state,
	4397	+ .irq_get_irqchip_state = its_sgi_get_irqchip_state,
	4398	+ .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
	4399	+};
	4400	+
	4401	+static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
	4402	+ unsigned int virq, unsigned int nr_irqs,
	4403	+ void *args)
	4404	+{
	4405	+ struct its_vpe *vpe = args;
	4406	+ int i;
	4407	+
	4408	+ /* Yes, we do want 16 SGIs */
	4409	+ WARN_ON(nr_irqs != 16);
	4410	+
	4411	+ for (i = 0; i < 16; i++) {
	4412	+ vpe->sgi_config[i].priority = 0;
	4413	+ vpe->sgi_config[i].enabled = false;
	4414	+ vpe->sgi_config[i].group = false;
	4415	+
	4416	+ irq_domain_set_hwirq_and_chip(domain, virq + i, i,
	4417	+ &its_sgi_irq_chip, vpe);
	4418	+ irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
	4419	+ }
	4420	+
	4421	+ return 0;
	4422	+}
	4423	+
	4424	+static void its_sgi_irq_domain_free(struct irq_domain *domain,
	4425	+ unsigned int virq,
	4426	+ unsigned int nr_irqs)
	4427	+{
	4428	+ /* Nothing to do */
	4429	+}
	4430	+
	4431	+static int its_sgi_irq_domain_activate(struct irq_domain *domain,
	4432	+ struct irq_data *d, bool reserve)
	4433	+{
	4434	+ /* Write out the initial SGI configuration */
	4435	+ its_configure_sgi(d, false);
	4436	+ return 0;
	4437	+}
	4438	+
	4439	+static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
	4440	+ struct irq_data *d)
	4441	+{
	4442	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4443	+
	4444	+ /*
	4445	+ * The VSGI command is awkward:
	4446	+ *
	4447	+ * - To change the configuration, CLEAR must be set to false,
	4448	+ * leaving the pending bit unchanged.
	4449	+ * - To clear the pending bit, CLEAR must be set to true, leaving
	4450	+ * the configuration unchanged.
	4451	+ *
	4452	+ * You just can't do both at once, hence the two commands below.
	4453	+ */
	4454	+ vpe->sgi_config[d->hwirq].enabled = false;
	4455	+ its_configure_sgi(d, false);
	4456	+ its_configure_sgi(d, true);
	4457	+}
	4458	+
	4459	+static const struct irq_domain_ops its_sgi_domain_ops = {
	4460	+ .alloc = its_sgi_irq_domain_alloc,
	4461	+ .free = its_sgi_irq_domain_free,
	4462	+ .activate = its_sgi_irq_domain_activate,
	4463	+ .deactivate = its_sgi_irq_domain_deactivate,
2931	4464	};
2932	4465
2933	4466	static int its_vpe_id_alloc(void)
..	..	@@ -2963,9 +4496,13 @@
2963	4496	return -ENOMEM;
2964	4497	}
2965	4498
	4499	+ raw_spin_lock_init(&vpe->vpe_lock);
2966	4500	vpe->vpe_id = vpe_id;
2967	4501	vpe->vpt_page = vpt_page;
2968		- vpe->vpe_proxy_event = -1;
	4502	+ if (gic_rdists->has_rvpeid)
	4503	+ atomic_set(&vpe->vmapp_count, 0);
	4504	+ else
	4505	+ vpe->vpe_proxy_event = -1;
2969	4506
2970	4507	return 0;
2971	4508	}
..	..	@@ -3007,6 +4544,7 @@
3007	4544	static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3008	4545	unsigned int nr_irqs, void *args)
3009	4546	{
	4547	+ struct irq_chip *irqchip = &its_vpe_irq_chip;
3010	4548	struct its_vm *vm = args;
3011	4549	unsigned long *bitmap;
3012	4550	struct page *vprop_page;
..	..	@@ -3034,6 +4572,9 @@
3034	4572	vm->nr_db_lpis = nr_ids;
3035	4573	vm->vprop_page = vprop_page;
3036	4574
	4575	+ if (gic_rdists->has_rvpeid)
	4576	+ irqchip = &its_vpe_4_1_irq_chip;
	4577	+
3037	4578	for (i = 0; i < nr_irqs; i++) {
3038	4579	vm->vpes[i]->vpe_db_lpi = base + i;
3039	4580	err = its_vpe_init(vm->vpes[i]);
..	..	@@ -3044,7 +4585,7 @@
3044	4585	if (err)
3045	4586	break;
3046	4587	irq_domain_set_hwirq_and_chip(domain, virq + i, i,
3047		- &its_vpe_irq_chip, vm->vpes[i]);
	4588	+ irqchip, vm->vpes[i]);
3048	4589	set_bit(i, bitmap);
3049	4590	}
3050	4591
..	..	@@ -3065,15 +4606,19 @@
3065	4606	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3066	4607	struct its_node *its;
3067	4608
3068		- /* If we use the list map, we issue VMAPP on demand... */
3069		- if (its_list_map)
	4609	+ /*
	4610	+ * If we use the list map, we issue VMAPP on demand... Unless
	4611	+ * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
	4612	+ * so that VSGIs can work.
	4613	+ */
	4614	+ if (!gic_requires_eager_mapping())
3070	4615	return 0;
3071	4616
3072	4617	/* Map the VPE to the first possible CPU */
3073	4618	vpe->col_idx = cpumask_first(cpu_online_mask);
3074	4619
3075	4620	list_for_each_entry(its, &its_nodes, entry) {
3076		- if (!its->is_v4)
	4621	+ if (!is_v4(its))
3077	4622	continue;
3078	4623
3079	4624	its_send_vmapp(its, vpe, true);
..	..	@@ -3092,14 +4637,14 @@
3092	4637	struct its_node *its;
3093	4638
3094	4639	/*
3095		- * If we use the list map, we unmap the VPE once no VLPIs are
3096		- * associated with the VM.
	4640	+ * If we use the list map on GICv4.0, we unmap the VPE once no
	4641	+ * VLPIs are associated with the VM.
3097	4642	*/
3098		- if (its_list_map)
	4643	+ if (!gic_requires_eager_mapping())
3099	4644	return;
3100	4645
3101	4646	list_for_each_entry(its, &its_nodes, entry) {
3102		- if (!its->is_v4)
	4647	+ if (!is_v4(its))
3103	4648	continue;
3104	4649
3105	4650	its_send_vmapp(its, vpe, false);
..	..	@@ -3150,8 +4695,9 @@
3150	4695	{
3151	4696	struct its_node *its = data;
3152	4697
3153		- /* erratum 22375: only alloc 8MB table size */
3154		- its->device_ids = 0x14; /* 20 bits, 8MB */
	4698	+ /* erratum 22375: only alloc 8MB table size (20 bits) */
	4699	+ its->typer &= ~GITS_TYPER_DEVBITS;
	4700	+ its->typer \|= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
3155	4701	its->flags \|= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
3156	4702
3157	4703	return true;
..	..	@@ -3171,7 +4717,8 @@
3171	4717	struct its_node *its = data;
3172	4718
3173	4719	/* On QDF2400, the size of the ITE is 16Bytes */
3174		- its->ite_size = 16;
	4720	+ its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
	4721	+ its->typer \|= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
3175	4722
3176	4723	return true;
3177	4724	}
..	..	@@ -3205,8 +4752,10 @@
3205	4752	its->get_msi_base = its_irq_get_msi_base_pre_its;
3206	4753
3207	4754	ids = ilog2(pre_its_window[1]) - 2;
3208		- if (its->device_ids > ids)
3209		- its->device_ids = ids;
	4755	+ if (device_ids(its) > ids) {
	4756	+ its->typer &= ~GITS_TYPER_DEVBITS;
	4757	+ its->typer \|= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
	4758	+ }
3210	4759
3211	4760	/* the pre-ITS breaks isolation, so disable MSI remapping */
3212	4761	its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
..	..	@@ -3433,7 +4982,7 @@
3433	4982	}
3434	4983
3435	4984	/* Use the last possible DevID */
3436		- devid = GENMASK(its->device_ids - 1, 0);
	4985	+ devid = GENMASK(device_ids(its) - 1, 0);
3437	4986	vpe_proxy.dev = its_create_device(its, devid, entries, false);
3438	4987	if (!vpe_proxy.dev) {
3439	4988	kfree(vpe_proxy.vpes);
..	..	@@ -3496,10 +5045,11 @@
3496	5045	void __iomem *its_base;
3497	5046	u32 val, ctlr;
3498	5047	u64 baser, tmp, typer;
	5048	+ struct page *page;
3499	5049	int err;
3500	5050	gfp_t gfp_flags;
3501	5051
3502		- its_base = ioremap(res->start, resource_size(res));
	5052	+ its_base = ioremap(res->start, SZ_64K);
3503	5053	if (!its_base) {
3504	5054	pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
3505	5055	return -ENOMEM;
..	..	@@ -3531,12 +5081,10 @@
3531	5081	INIT_LIST_HEAD(&its->entry);
3532	5082	INIT_LIST_HEAD(&its->its_device_list);
3533	5083	typer = gic_read_typer(its_base + GITS_TYPER);
	5084	+ its->typer = typer;
3534	5085	its->base = its_base;
3535	5086	its->phys_base = res->start;
3536		- its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
3537		- its->device_ids = GITS_TYPER_DEVBITS(typer);
3538		- its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
3539		- if (its->is_v4) {
	5087	+ if (is_v4(its)) {
3540	5088	if (!(typer & GITS_TYPER_VMOVP)) {
3541	5089	err = its_compute_its_list_map(res, its_base);
3542	5090	if (err < 0)
..	..	@@ -3549,19 +5097,37 @@
3549	5097	} else {
3550	5098	pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
3551	5099	}
	5100	+
	5101	+ if (is_v4_1(its)) {
	5102	+ u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
	5103	+
	5104	+ its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
	5105	+ if (!its->sgir_base) {
	5106	+ err = -ENOMEM;
	5107	+ goto out_free_its;
	5108	+ }
	5109	+
	5110	+ its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
	5111	+
	5112	+ pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
	5113	+ &res->start, its->mpidr, svpet);
	5114	+ }
3552	5115	}
3553	5116
3554	5117	its->numa_node = numa_node;
3555	5118
3556	5119	gfp_flags = GFP_KERNEL \| __GFP_ZERO;
3557		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	5120	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	5121	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	5122	+ of_machine_is_compatible("rockchip,rk3566"))
3558	5123	gfp_flags \|= GFP_DMA32;
3559		- its->cmd_base = (void *)__get_free_pages(gfp_flags,
3560		- get_order(ITS_CMD_QUEUE_SZ));
3561		- if (!its->cmd_base) {
	5124	+ page = alloc_pages_node(its->numa_node, gfp_flags,
	5125	+ get_order(ITS_CMD_QUEUE_SZ));
	5126	+ if (!page) {
3562	5127	err = -ENOMEM;
3563		- goto out_free_its;
	5128	+ goto out_unmap_sgir;
3564	5129	}
	5130	+ its->cmd_base = (void *)page_address(page);
3565	5131	its->cmd_write = its->cmd_base;
3566	5132	its->fwnode_handle = handle;
3567	5133	its->get_msi_base = its_irq_get_msi_base;
..	..	@@ -3586,7 +5152,11 @@
3586	5152	gits_write_cbaser(baser, its->base + GITS_CBASER);
3587	5153	tmp = gits_read_cbaser(its->base + GITS_CBASER);
3588	5154
3589		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	5155	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	5156	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	5157	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	5158	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	5159	+ of_machine_is_compatible("rockchip,rk3588")))
3590	5160	tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
3591	5161
3592	5162	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
..	..	@@ -3608,7 +5178,7 @@
3608	5178	gits_write_cwriter(0, its->base + GITS_CWRITER);
3609	5179	ctlr = readl_relaxed(its->base + GITS_CTLR);
3610	5180	ctlr \|= GITS_CTLR_ENABLE;
3611		- if (its->is_v4)
	5181	+ if (is_v4(its))
3612	5182	ctlr \|= GITS_CTLR_ImDe;
3613	5183	writel_relaxed(ctlr, its->base + GITS_CTLR);
3614	5184
..	..	@@ -3626,6 +5196,9 @@
3626	5196	its_free_tables(its);
3627	5197	out_free_cmd:
3628	5198	free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
	5199	+out_unmap_sgir:
	5200	+ if (its->sgir_base)
	5201	+ iounmap(its->sgir_base);
3629	5202	out_free_its:
3630	5203	kfree(its);
3631	5204	out_unmap:
..	..	@@ -3658,8 +5231,11 @@
3658	5231	* If coming via a CPU hotplug event, we don't need to disable
3659	5232	* LPIs before trying to re-enable them. They are already
3660	5233	* configured and all is well in the world.
	5234	+ *
	5235	+ * If running with preallocated tables, there is nothing to do.
3661	5236	*/
3662		- if (gic_data_rdist()->lpi_enabled)
	5237	+ if (gic_data_rdist()->lpi_enabled \|\|
	5238	+ (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
3663	5239	return 0;
3664	5240
3665	5241	/*
..	..	@@ -3776,13 +5352,13 @@
3776	5352	return NUMA_NO_NODE;
3777	5353	}
3778	5354
3779		-static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header,
	5355	+static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
3780	5356	const unsigned long end)
3781	5357	{
3782	5358	return 0;
3783	5359	}
3784	5360
3785		-static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
	5361	+static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
3786	5362	const unsigned long end)
3787	5363	{
3788	5364	int node;
..	..	@@ -3798,7 +5374,12 @@
3798	5374	return -EINVAL;
3799	5375	}
3800	5376
3801		- node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
	5377	+ /*
	5378	+ * Note that in theory a new proximity node could be created by this
	5379	+ * entry as it is an SRAT resource allocation structure.
	5380	+ * We do not currently support doing so.
	5381	+ */
	5382	+ node = pxm_to_node(its_affinity->proximity_domain);
3802	5383
3803	5384	if (node == NUMA_NO_NODE \|\| node >= MAX_NUMNODES) {
3804	5385	pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
..	..	@@ -3849,7 +5430,7 @@
3849	5430	static void __init acpi_its_srat_maps_free(void) { }
3850	5431	#endif
3851	5432
3852		-static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
	5433	+static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
3853	5434	const unsigned long end)
3854	5435	{
3855	5436	struct acpi_madt_generic_translator *its_entry;
..	..	@@ -3863,7 +5444,7 @@
3863	5444	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
3864	5445	res.flags = IORESOURCE_MEM;
3865	5446
3866		- dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address);
	5447	+ dom_handle = irq_domain_alloc_fwnode(&res.start);
3867	5448	if (!dom_handle) {
3868	5449	pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
3869	5450	&res.start);
..	..	@@ -3906,7 +5487,10 @@
3906	5487	struct device_node *of_node;
3907	5488	struct its_node *its;
3908	5489	bool has_v4 = false;
	5490	+ bool has_v4_1 = false;
3909	5491	int err;
	5492	+
	5493	+ gic_rdists = rdists;
3910	5494
3911	5495	its_parent = parent_domain;
3912	5496	of_node = to_of_node(handle);
..	..	@@ -3920,18 +5504,29 @@
3920	5504	return -ENXIO;
3921	5505	}
3922	5506
3923		- gic_rdists = rdists;
3924		-
3925	5507	err = allocate_lpi_tables();
3926	5508	if (err)
3927	5509	return err;
3928	5510
3929		- list_for_each_entry(its, &its_nodes, entry)
3930		- has_v4 \|= its->is_v4;
	5511	+ list_for_each_entry(its, &its_nodes, entry) {
	5512	+ has_v4 \|= is_v4(its);
	5513	+ has_v4_1 \|= is_v4_1(its);
	5514	+ }
	5515	+
	5516	+ /* Don't bother with inconsistent systems */
	5517	+ if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
	5518	+ rdists->has_rvpeid = false;
3931	5519
3932	5520	if (has_v4 & rdists->has_vlpis) {
	5521	+ const struct irq_domain_ops *sgi_ops;
	5522	+
	5523	+ if (has_v4_1)
	5524	+ sgi_ops = &its_sgi_domain_ops;
	5525	+ else
	5526	+ sgi_ops = NULL;
	5527	+
3933	5528	if (its_init_vpe_domain() \|\|
3934		- its_init_v4(parent_domain, &its_vpe_domain_ops)) {
	5529	+ its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
3935	5530	rdists->has_vlpis = false;
3936	5531	pr_err("ITS: Disabling GICv4 support\n");
3937	5532	}