~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,31 +1,24 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
3	4	* Author: Marc Zyngier <marc.zyngier@arm.com>
4		- *
5		- * This program is free software; you can redistribute it and/or modify
6		- * it under the terms of the GNU General Public License version 2 as
7		- * published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program. If not, see <http://www.gnu.org/licenses/>.
16	5	*/
17	6
18	7	#include <linux/acpi.h>
19	8	#include <linux/acpi_iort.h>
	9	+#include <linux/bitfield.h>
20	10	#include <linux/bitmap.h>
21	11	#include <linux/cpu.h>
	12	+#include <linux/crash_dump.h>
22	13	#include <linux/delay.h>
23	14	#include <linux/dma-iommu.h>
	15	+#include <linux/efi.h>
24	16	#include <linux/interrupt.h>
	17	+#include <linux/iopoll.h>
25	18	#include <linux/irqdomain.h>
26	19	#include <linux/list.h>
27		-#include <linux/list_sort.h>
28	20	#include <linux/log2.h>
	21	+#include <linux/memblock.h>
29	22	#include <linux/mm.h>
30	23	#include <linux/msi.h>
31	24	#include <linux/of.h>
..	..	@@ -51,6 +44,7 @@
51	44	#define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
52	45
53	46	#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
	47	+#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
54	48
55	49	static u32 lpi_id_bits;
56	50
..	..	@@ -63,7 +57,7 @@
63	57	#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
64	58	#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
65	59
66		-#define LPI_PROP_DEFAULT_PRIO 0xa0
	60	+#define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
67	61
68	62	/*
69	63	* Collection structure - just an ID, and a redistributor address to
..	..	@@ -102,6 +96,7 @@
102	96	struct mutex dev_alloc_lock;
103	97	struct list_head entry;
104	98	void __iomem *base;
	99	+ void __iomem *sgir_base;
105	100	phys_addr_t phys_base;
106	101	struct its_cmd_block *cmd_base;
107	102	struct its_cmd_block *cmd_write;
..	..	@@ -109,24 +104,36 @@
109	104	struct its_collection *collections;
110	105	struct fwnode_handle *fwnode_handle;
111	106	u64 (get_msi_base)(struct its_device its_dev);
	107	+ u64 typer;
112	108	u64 cbaser_save;
113	109	u32 ctlr_save;
	110	+ u32 mpidr;
114	111	struct list_head its_device_list;
115	112	u64 flags;
116	113	unsigned long list_nr;
117		- u32 ite_size;
118		- u32 device_ids;
119	114	int numa_node;
120	115	unsigned int msi_domain_flags;
121	116	u32 pre_its_base; /* for Socionext Synquacer */
122		- bool is_v4;
123	117	int vlpi_redist_offset;
124	118	};
	119	+
	120	+#define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
	121	+#define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
	122	+#define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
125	123
126	124	#define ITS_ITT_ALIGN SZ_256
127	125
128	126	/* The maximum number of VPEID bits supported by VLPI commands */
129		-#define ITS_MAX_VPEID_BITS (16)
	127	+#define ITS_MAX_VPEID_BITS \
	128	+ ({ \
	129	+ int nvpeid = 16; \
	130	+ if (gic_rdists->has_rvpeid && \
	131	+ gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
	132	+ nvpeid = 1 + (gic_rdists->gicd_typer2 & \
	133	+ GICD_TYPER2_VID); \
	134	+ \
	135	+ nvpeid; \
	136	+ })
130	137	#define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
131	138
132	139	/* Convert page order to size in bytes */
..	..	@@ -137,7 +144,7 @@
137	144	u16 *col_map;
138	145	irq_hw_number_t lpi_base;
139	146	int nr_lpis;
140		- struct mutex vlpi_lock;
	147	+ raw_spinlock_t vlpi_lock;
141	148	struct its_vm *vm;
142	149	struct its_vlpi_map *vlpi_maps;
143	150	int nr_vlpis;
..	..	@@ -167,6 +174,13 @@
167	174	int next_victim;
168	175	} vpe_proxy;
169	176
	177	+struct cpu_lpi_count {
	178	+ atomic_t managed;
	179	+ atomic_t unmanaged;
	180	+};
	181	+
	182	+static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
	183	+
170	184	static LIST_HEAD(its_nodes);
171	185	static DEFINE_RAW_SPINLOCK(its_lock);
172	186	static struct rdists *gic_rdists;
..	..	@@ -179,8 +193,18 @@
179	193	static DEFINE_IDA(its_vpeid_ida);
180	194
181	195	#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
	196	+#define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
182	197	#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
183	198	#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
	199	+
	200	+/*
	201	+ * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
	202	+ * always have vSGIs mapped.
	203	+ */
	204	+static bool require_its_list_vmovp(struct its_vm vm, struct its_node its)
	205	+{
	206	+ return (gic_rdists->has_rvpeid \|\| vm->vlpi_count[its->list_nr]);
	207	+}
184	208
185	209	static u16 get_its_list(struct its_vm *vm)
186	210	{
..	..	@@ -188,14 +212,20 @@
188	212	unsigned long its_list = 0;
189	213
190	214	list_for_each_entry(its, &its_nodes, entry) {
191		- if (!its->is_v4)
	215	+ if (!is_v4(its))
192	216	continue;
193	217
194		- if (vm->vlpi_count[its->list_nr])
	218	+ if (require_its_list_vmovp(vm, its))
195	219	__set_bit(its->list_nr, &its_list);
196	220	}
197	221
198	222	return (u16)its_list;
	223	+}
	224	+
	225	+static inline u32 its_get_event_id(struct irq_data *d)
	226	+{
	227	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	228	+ return d->hwirq - its_dev->event_map.lpi_base;
199	229	}
200	230
201	231	static struct its_collection dev_event_to_col(struct its_device its_dev,
..	..	@@ -204,6 +234,64 @@
204	234	struct its_node *its = its_dev->its;
205	235
206	236	return its->collections + its_dev->event_map.col_map[event];
	237	+}
	238	+
	239	+static struct its_vlpi_map dev_event_to_vlpi_map(struct its_device its_dev,
	240	+ u32 event)
	241	+{
	242	+ if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
	243	+ return NULL;
	244	+
	245	+ return &its_dev->event_map.vlpi_maps[event];
	246	+}
	247	+
	248	+static struct its_vlpi_map get_vlpi_map(struct irq_data d)
	249	+{
	250	+ if (irqd_is_forwarded_to_vcpu(d)) {
	251	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	252	+ u32 event = its_get_event_id(d);
	253	+
	254	+ return dev_event_to_vlpi_map(its_dev, event);
	255	+ }
	256	+
	257	+ return NULL;
	258	+}
	259	+
	260	+static int vpe_to_cpuid_lock(struct its_vpe vpe, unsigned long flags)
	261	+{
	262	+ raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
	263	+ return vpe->col_idx;
	264	+}
	265	+
	266	+static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
	267	+{
	268	+ raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
	269	+}
	270	+
	271	+static int irq_to_cpuid_lock(struct irq_data d, unsigned long flags)
	272	+{
	273	+ struct its_vlpi_map *map = get_vlpi_map(d);
	274	+ int cpu;
	275	+
	276	+ if (map) {
	277	+ cpu = vpe_to_cpuid_lock(map->vpe, flags);
	278	+ } else {
	279	+ /* Physical LPIs are already locked via the irq_desc lock */
	280	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	281	+ cpu = its_dev->event_map.col_map[its_get_event_id(d)];
	282	+ /* Keep GCC quiet... */
	283	+ *flags = 0;
	284	+ }
	285	+
	286	+ return cpu;
	287	+}
	288	+
	289	+static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
	290	+{
	291	+ struct its_vlpi_map *map = get_vlpi_map(d);
	292	+
	293	+ if (map)
	294	+ vpe_to_cpuid_unlock(map->vpe, flags);
207	295	}
208	296
209	297	static struct its_collection valid_col(struct its_collection col)
..	..	@@ -305,6 +393,19 @@
305	393	u16 seq_num;
306	394	u16 its_list;
307	395	} its_vmovp_cmd;
	396	+
	397	+ struct {
	398	+ struct its_vpe *vpe;
	399	+ } its_invdb_cmd;
	400	+
	401	+ struct {
	402	+ struct its_vpe *vpe;
	403	+ u8 sgi;
	404	+ u8 priority;
	405	+ bool enable;
	406	+ bool group;
	407	+ bool clear;
	408	+ } its_vsgi_cmd;
308	409	};
309	410	};
310	411
..	..	@@ -312,7 +413,10 @@
312	413	* The ITS command block, which is what the ITS actually parses.
313	414	*/
314	415	struct its_cmd_block {
315		- u64 raw_cmd[4];
	416	+ union {
	417	+ u64 raw_cmd[4];
	418	+ __le64 raw_cmd_le[4];
	419	+ };
316	420	};
317	421
318	422	#define ITS_CMD_QUEUE_SZ SZ_64K
..	..	@@ -418,13 +522,70 @@
418	522	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
419	523	}
420	524
	525	+static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
	526	+{
	527	+ its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
	528	+}
	529	+
	530	+static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
	531	+{
	532	+ its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
	533	+}
	534	+
	535	+static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
	536	+{
	537	+ its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
	538	+}
	539	+
	540	+static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
	541	+ u32 vpe_db_lpi)
	542	+{
	543	+ its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
	544	+}
	545	+
	546	+static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
	547	+ u32 vpe_db_lpi)
	548	+{
	549	+ its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
	550	+}
	551	+
	552	+static void its_encode_db(struct its_cmd_block *cmd, bool db)
	553	+{
	554	+ its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
	555	+}
	556	+
	557	+static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
	558	+{
	559	+ its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
	560	+}
	561	+
	562	+static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
	563	+{
	564	+ its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
	565	+}
	566	+
	567	+static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
	568	+{
	569	+ its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
	570	+}
	571	+
	572	+static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
	573	+{
	574	+ its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
	575	+}
	576	+
	577	+static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
	578	+{
	579	+ its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
	580	+}
	581	+
421	582	static inline void its_fixup_cmd(struct its_cmd_block *cmd)
422	583	{
423	584	/* Let's fixup BE commands */
424		- cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
425		- cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
426		- cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
427		- cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
	585	+ cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
	586	+ cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
	587	+ cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
	588	+ cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
428	589	}
429	590
430	591	static struct its_collection its_build_mapd_cmd(struct its_node its,
..	..	@@ -601,19 +762,45 @@
601	762	struct its_cmd_block *cmd,
602	763	struct its_cmd_desc *desc)
603	764	{
604		- unsigned long vpt_addr;
	765	+ unsigned long vpt_addr, vconf_addr;
605	766	u64 target;
606		-
607		- vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
608		- target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
	767	+ bool alloc;
609	768
610	769	its_encode_cmd(cmd, GITS_CMD_VMAPP);
611	770	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
612	771	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
	772	+
	773	+ if (!desc->its_vmapp_cmd.valid) {
	774	+ if (is_v4_1(its)) {
	775	+ alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
	776	+ its_encode_alloc(cmd, alloc);
	777	+ }
	778	+
	779	+ goto out;
	780	+ }
	781	+
	782	+ vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
	783	+ target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
	784	+
613	785	its_encode_target(cmd, target);
614	786	its_encode_vpt_addr(cmd, vpt_addr);
615	787	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
616	788
	789	+ if (!is_v4_1(its))
	790	+ goto out;
	791	+
	792	+ vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
	793	+
	794	+ alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
	795	+
	796	+ its_encode_alloc(cmd, alloc);
	797	+
	798	+ /* We can only signal PTZ when alloc==1. Why do we have two bits? */
	799	+ its_encode_ptz(cmd, alloc);
	800	+ its_encode_vconf_addr(cmd, vconf_addr);
	801	+ its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
	802	+
	803	+out:
617	804	its_fixup_cmd(cmd);
618	805
619	806	return valid_vpe(its, desc->its_vmapp_cmd.vpe);
..	..	@@ -625,7 +812,7 @@
625	812	{
626	813	u32 db;
627	814
628		- if (desc->its_vmapti_cmd.db_enabled)
	815	+ if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
629	816	db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
630	817	else
631	818	db = 1023;
..	..	@@ -648,7 +835,7 @@
648	835	{
649	836	u32 db;
650	837
651		- if (desc->its_vmovi_cmd.db_enabled)
	838	+ if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
652	839	db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
653	840	else
654	841	db = 1023;
..	..	@@ -678,9 +865,103 @@
678	865	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
679	866	its_encode_target(cmd, target);
680	867
	868	+ if (is_v4_1(its)) {
	869	+ its_encode_db(cmd, true);
	870	+ its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
	871	+ }
	872	+
681	873	its_fixup_cmd(cmd);
682	874
683	875	return valid_vpe(its, desc->its_vmovp_cmd.vpe);
	876	+}
	877	+
	878	+static struct its_vpe its_build_vinv_cmd(struct its_node its,
	879	+ struct its_cmd_block *cmd,
	880	+ struct its_cmd_desc *desc)
	881	+{
	882	+ struct its_vlpi_map *map;
	883	+
	884	+ map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
	885	+ desc->its_inv_cmd.event_id);
	886	+
	887	+ its_encode_cmd(cmd, GITS_CMD_INV);
	888	+ its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
	889	+ its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
	890	+
	891	+ its_fixup_cmd(cmd);
	892	+
	893	+ return valid_vpe(its, map->vpe);
	894	+}
	895	+
	896	+static struct its_vpe its_build_vint_cmd(struct its_node its,
	897	+ struct its_cmd_block *cmd,
	898	+ struct its_cmd_desc *desc)
	899	+{
	900	+ struct its_vlpi_map *map;
	901	+
	902	+ map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
	903	+ desc->its_int_cmd.event_id);
	904	+
	905	+ its_encode_cmd(cmd, GITS_CMD_INT);
	906	+ its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
	907	+ its_encode_event_id(cmd, desc->its_int_cmd.event_id);
	908	+
	909	+ its_fixup_cmd(cmd);
	910	+
	911	+ return valid_vpe(its, map->vpe);
	912	+}
	913	+
	914	+static struct its_vpe its_build_vclear_cmd(struct its_node its,
	915	+ struct its_cmd_block *cmd,
	916	+ struct its_cmd_desc *desc)
	917	+{
	918	+ struct its_vlpi_map *map;
	919	+
	920	+ map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
	921	+ desc->its_clear_cmd.event_id);
	922	+
	923	+ its_encode_cmd(cmd, GITS_CMD_CLEAR);
	924	+ its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
	925	+ its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
	926	+
	927	+ its_fixup_cmd(cmd);
	928	+
	929	+ return valid_vpe(its, map->vpe);
	930	+}
	931	+
	932	+static struct its_vpe its_build_invdb_cmd(struct its_node its,
	933	+ struct its_cmd_block *cmd,
	934	+ struct its_cmd_desc *desc)
	935	+{
	936	+ if (WARN_ON(!is_v4_1(its)))
	937	+ return NULL;
	938	+
	939	+ its_encode_cmd(cmd, GITS_CMD_INVDB);
	940	+ its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
	941	+
	942	+ its_fixup_cmd(cmd);
	943	+
	944	+ return valid_vpe(its, desc->its_invdb_cmd.vpe);
	945	+}
	946	+
	947	+static struct its_vpe its_build_vsgi_cmd(struct its_node its,
	948	+ struct its_cmd_block *cmd,
	949	+ struct its_cmd_desc *desc)
	950	+{
	951	+ if (WARN_ON(!is_v4_1(its)))
	952	+ return NULL;
	953	+
	954	+ its_encode_cmd(cmd, GITS_CMD_VSGI);
	955	+ its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
	956	+ its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
	957	+ its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
	958	+ its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
	959	+ its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
	960	+ its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
	961	+
	962	+ its_fixup_cmd(cmd);
	963	+
	964	+ return valid_vpe(its, desc->its_vsgi_cmd.vpe);
684	965	}
685	966
686	967	static u64 its_cmd_ptr_to_offset(struct its_node *its,
..	..	@@ -960,7 +1241,7 @@
960	1241
961	1242	static void its_send_vmapti(struct its_device *dev, u32 id)
962	1243	{
963		- struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
	1244	+ struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
964	1245	struct its_cmd_desc desc;
965	1246
966	1247	desc.its_vmapti_cmd.vpe = map->vpe;
..	..	@@ -974,7 +1255,7 @@
974	1255
975	1256	static void its_send_vmovi(struct its_device *dev, u32 id)
976	1257	{
977		- struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
	1258	+ struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
978	1259	struct its_cmd_desc desc;
979	1260
980	1261	desc.its_vmovi_cmd.vpe = map->vpe;
..	..	@@ -1028,10 +1309,10 @@
1028	1309
1029	1310	/* Emit VMOVPs */
1030	1311	list_for_each_entry(its, &its_nodes, entry) {
1031		- if (!its->is_v4)
	1312	+ if (!is_v4(its))
1032	1313	continue;
1033	1314
1034		- if (!vpe->its_vm->vlpi_count[its->list_nr])
	1315	+ if (!require_its_list_vmovp(vpe->its_vm, its))
1035	1316	continue;
1036	1317
1037	1318	desc.its_vmovp_cmd.col = &its->collections[col_id];
..	..	@@ -1049,40 +1330,79 @@
1049	1330	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1050	1331	}
1051	1332
	1333	+static void its_send_vinv(struct its_device *dev, u32 event_id)
	1334	+{
	1335	+ struct its_cmd_desc desc;
	1336	+
	1337	+ /*
	1338	+ * There is no real VINV command. This is just a normal INV,
	1339	+ * with a VSYNC instead of a SYNC.
	1340	+ */
	1341	+ desc.its_inv_cmd.dev = dev;
	1342	+ desc.its_inv_cmd.event_id = event_id;
	1343	+
	1344	+ its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
	1345	+}
	1346	+
	1347	+static void its_send_vint(struct its_device *dev, u32 event_id)
	1348	+{
	1349	+ struct its_cmd_desc desc;
	1350	+
	1351	+ /*
	1352	+ * There is no real VINT command. This is just a normal INT,
	1353	+ * with a VSYNC instead of a SYNC.
	1354	+ */
	1355	+ desc.its_int_cmd.dev = dev;
	1356	+ desc.its_int_cmd.event_id = event_id;
	1357	+
	1358	+ its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
	1359	+}
	1360	+
	1361	+static void its_send_vclear(struct its_device *dev, u32 event_id)
	1362	+{
	1363	+ struct its_cmd_desc desc;
	1364	+
	1365	+ /*
	1366	+ * There is no real VCLEAR command. This is just a normal CLEAR,
	1367	+ * with a VSYNC instead of a SYNC.
	1368	+ */
	1369	+ desc.its_clear_cmd.dev = dev;
	1370	+ desc.its_clear_cmd.event_id = event_id;
	1371	+
	1372	+ its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
	1373	+}
	1374	+
	1375	+static void its_send_invdb(struct its_node its, struct its_vpe vpe)
	1376	+{
	1377	+ struct its_cmd_desc desc;
	1378	+
	1379	+ desc.its_invdb_cmd.vpe = vpe;
	1380	+ its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
	1381	+}
	1382	+
1052	1383	/*
1053	1384	* irqchip functions - assumes MSI, mostly.
1054	1385	*/
1055		-
1056		-static inline u32 its_get_event_id(struct irq_data *d)
1057		-{
1058		- struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1059		- return d->hwirq - its_dev->event_map.lpi_base;
1060		-}
1061		-
1062	1386	static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1063	1387	{
	1388	+ struct its_vlpi_map *map = get_vlpi_map(d);
1064	1389	irq_hw_number_t hwirq;
1065		- struct page *prop_page;
	1390	+ void *va;
1066	1391	u8 *cfg;
1067	1392
1068		- if (irqd_is_forwarded_to_vcpu(d)) {
1069		- struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1070		- u32 event = its_get_event_id(d);
1071		- struct its_vlpi_map *map;
1072		-
1073		- prop_page = its_dev->event_map.vm->vprop_page;
1074		- map = &its_dev->event_map.vlpi_maps[event];
	1393	+ if (map) {
	1394	+ va = page_address(map->vm->vprop_page);
1075	1395	hwirq = map->vintid;
1076	1396
1077	1397	/* Remember the updated property */
1078	1398	map->properties &= ~clr;
1079	1399	map->properties \|= set \| LPI_PROP_GROUP1;
1080	1400	} else {
1081		- prop_page = gic_rdists->prop_page;
	1401	+ va = gic_rdists->prop_table_va;
1082	1402	hwirq = d->hwirq;
1083	1403	}
1084	1404
1085		- cfg = page_address(prop_page) + hwirq - 8192;
	1405	+ cfg = va + hwirq - 8192;
1086	1406	*cfg &= ~clr;
1087	1407	*cfg \|= set \| LPI_PROP_GROUP1;
1088	1408
..	..	@@ -1097,23 +1417,76 @@
1097	1417	dsb(ishst);
1098	1418	}
1099	1419
	1420	+static void wait_for_syncr(void __iomem *rdbase)
	1421	+{
	1422	+ while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
	1423	+ cpu_relax();
	1424	+}
	1425	+
	1426	+static void direct_lpi_inv(struct irq_data *d)
	1427	+{
	1428	+ struct its_vlpi_map *map = get_vlpi_map(d);
	1429	+ void __iomem *rdbase;
	1430	+ unsigned long flags;
	1431	+ u64 val;
	1432	+ int cpu;
	1433	+
	1434	+ if (map) {
	1435	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	1436	+
	1437	+ WARN_ON(!is_v4_1(its_dev->its));
	1438	+
	1439	+ val = GICR_INVLPIR_V;
	1440	+ val \|= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
	1441	+ val \|= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
	1442	+ } else {
	1443	+ val = d->hwirq;
	1444	+ }
	1445	+
	1446	+ /* Target the redistributor this LPI is currently routed to */
	1447	+ cpu = irq_to_cpuid_lock(d, &flags);
	1448	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	1449	+ rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
	1450	+ gic_write_lpir(val, rdbase + GICR_INVLPIR);
	1451	+
	1452	+ wait_for_syncr(rdbase);
	1453	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	1454	+ irq_to_cpuid_unlock(d, flags);
	1455	+}
	1456	+
1100	1457	static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1101	1458	{
1102	1459	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1103	1460
1104	1461	lpi_write_config(d, clr, set);
1105		- its_send_inv(its_dev, its_get_event_id(d));
	1462	+ if (gic_rdists->has_direct_lpi &&
	1463	+ (is_v4_1(its_dev->its) \|\| !irqd_is_forwarded_to_vcpu(d)))
	1464	+ direct_lpi_inv(d);
	1465	+ else if (!irqd_is_forwarded_to_vcpu(d))
	1466	+ its_send_inv(its_dev, its_get_event_id(d));
	1467	+ else
	1468	+ its_send_vinv(its_dev, its_get_event_id(d));
1106	1469	}
1107	1470
1108	1471	static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1109	1472	{
1110	1473	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1111	1474	u32 event = its_get_event_id(d);
	1475	+ struct its_vlpi_map *map;
1112	1476
1113		- if (its_dev->event_map.vlpi_maps[event].db_enabled == enable)
	1477	+ /*
	1478	+ * GICv4.1 does away with the per-LPI nonsense, nothing to do
	1479	+ * here.
	1480	+ */
	1481	+ if (is_v4_1(its_dev->its))
1114	1482	return;
1115	1483
1116		- its_dev->event_map.vlpi_maps[event].db_enabled = enable;
	1484	+ map = dev_event_to_vlpi_map(its_dev, event);
	1485	+
	1486	+ if (map->db_enabled == enable)
	1487	+ return;
	1488	+
	1489	+ map->db_enabled = enable;
1117	1490
1118	1491	/*
1119	1492	* More fun with the architecture:
..	..	@@ -1144,42 +1517,159 @@
1144	1517	lpi_update_config(d, 0, LPI_PROP_ENABLED);
1145	1518	}
1146	1519
	1520	+static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
	1521	+{
	1522	+ if (irqd_affinity_is_managed(d))
	1523	+ return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1524	+
	1525	+ return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1526	+}
	1527	+
	1528	+static void its_inc_lpi_count(struct irq_data *d, int cpu)
	1529	+{
	1530	+ if (irqd_affinity_is_managed(d))
	1531	+ atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1532	+ else
	1533	+ atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1534	+}
	1535	+
	1536	+static void its_dec_lpi_count(struct irq_data *d, int cpu)
	1537	+{
	1538	+ if (irqd_affinity_is_managed(d))
	1539	+ atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
	1540	+ else
	1541	+ atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
	1542	+}
	1543	+
	1544	+static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
	1545	+ const struct cpumask *cpu_mask)
	1546	+{
	1547	+ unsigned int cpu = nr_cpu_ids, tmp;
	1548	+ int count = S32_MAX;
	1549	+
	1550	+ for_each_cpu(tmp, cpu_mask) {
	1551	+ int this_count = its_read_lpi_count(d, tmp);
	1552	+ if (this_count < count) {
	1553	+ cpu = tmp;
	1554	+ count = this_count;
	1555	+ }
	1556	+ }
	1557	+
	1558	+ return cpu;
	1559	+}
	1560	+
	1561	+/*
	1562	+ * As suggested by Thomas Gleixner in:
	1563	+ * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
	1564	+ */
	1565	+static int its_select_cpu(struct irq_data *d,
	1566	+ const struct cpumask *aff_mask)
	1567	+{
	1568	+ struct its_device *its_dev = irq_data_get_irq_chip_data(d);
	1569	+ cpumask_var_t tmpmask;
	1570	+ int cpu, node;
	1571	+
	1572	+ if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
	1573	+ return -ENOMEM;
	1574	+
	1575	+ node = its_dev->its->numa_node;
	1576	+
	1577	+ if (!irqd_affinity_is_managed(d)) {
	1578	+ /* First try the NUMA node */
	1579	+ if (node != NUMA_NO_NODE) {
	1580	+ /*
	1581	+ * Try the intersection of the affinity mask and the
	1582	+ * node mask (and the online mask, just to be safe).
	1583	+ */
	1584	+ cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
	1585	+ cpumask_and(tmpmask, tmpmask, cpu_online_mask);
	1586	+
	1587	+ /*
	1588	+ * Ideally, we would check if the mask is empty, and
	1589	+ * try again on the full node here.
	1590	+ *
	1591	+ * But it turns out that the way ACPI describes the
	1592	+ * affinity for ITSs only deals about memory, and
	1593	+ * not target CPUs, so it cannot describe a single
	1594	+ * ITS placed next to two NUMA nodes.
	1595	+ *
	1596	+ * Instead, just fallback on the online mask. This
	1597	+ * diverges from Thomas' suggestion above.
	1598	+ */
	1599	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1600	+ if (cpu < nr_cpu_ids)
	1601	+ goto out;
	1602	+
	1603	+ /* If we can't cross sockets, give up */
	1604	+ if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
	1605	+ goto out;
	1606	+
	1607	+ /* If the above failed, expand the search */
	1608	+ }
	1609	+
	1610	+ /* Try the intersection of the affinity and online masks */
	1611	+ cpumask_and(tmpmask, aff_mask, cpu_online_mask);
	1612	+
	1613	+ /* If that doesn't fly, the online mask is the last resort */
	1614	+ if (cpumask_empty(tmpmask))
	1615	+ cpumask_copy(tmpmask, cpu_online_mask);
	1616	+
	1617	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1618	+ } else {
	1619	+ cpumask_copy(tmpmask, aff_mask);
	1620	+
	1621	+ /* If we cannot cross sockets, limit the search to that node */
	1622	+ if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
	1623	+ node != NUMA_NO_NODE)
	1624	+ cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
	1625	+
	1626	+ cpu = cpumask_pick_least_loaded(d, tmpmask);
	1627	+ }
	1628	+out:
	1629	+ free_cpumask_var(tmpmask);
	1630	+
	1631	+ pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
	1632	+ return cpu;
	1633	+}
	1634	+
1147	1635	static int its_set_affinity(struct irq_data d, const struct cpumask mask_val,
1148	1636	bool force)
1149	1637	{
1150		- unsigned int cpu;
1151		- const struct cpumask *cpu_mask = cpu_online_mask;
1152	1638	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1153	1639	struct its_collection *target_col;
1154	1640	u32 id = its_get_event_id(d);
	1641	+ int cpu, prev_cpu;
1155	1642
1156	1643	/* A forwarded interrupt should use irq_set_vcpu_affinity */
1157	1644	if (irqd_is_forwarded_to_vcpu(d))
1158	1645	return -EINVAL;
1159	1646
1160		- /* lpi cannot be routed to a redistributor that is on a foreign node */
1161		- if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
1162		- if (its_dev->its->numa_node >= 0) {
1163		- cpu_mask = cpumask_of_node(its_dev->its->numa_node);
1164		- if (!cpumask_intersects(mask_val, cpu_mask))
1165		- return -EINVAL;
1166		- }
1167		- }
	1647	+ prev_cpu = its_dev->event_map.col_map[id];
	1648	+ its_dec_lpi_count(d, prev_cpu);
1168	1649
1169		- cpu = cpumask_any_and(mask_val, cpu_mask);
	1650	+ if (!force)
	1651	+ cpu = its_select_cpu(d, mask_val);
	1652	+ else
	1653	+ cpu = cpumask_pick_least_loaded(d, mask_val);
1170	1654
1171		- if (cpu >= nr_cpu_ids)
1172		- return -EINVAL;
	1655	+ if (cpu < 0 \|\| cpu >= nr_cpu_ids)
	1656	+ goto err;
1173	1657
1174	1658	/* don't set the affinity when the target cpu is same as current one */
1175		- if (cpu != its_dev->event_map.col_map[id]) {
	1659	+ if (cpu != prev_cpu) {
1176	1660	target_col = &its_dev->its->collections[cpu];
1177	1661	its_send_movi(its_dev, target_col, id);
1178	1662	its_dev->event_map.col_map[id] = cpu;
1179	1663	irq_data_update_effective_affinity(d, cpumask_of(cpu));
1180	1664	}
1181	1665
	1666	+ its_inc_lpi_count(d, cpu);
	1667	+
1182	1668	return IRQ_SET_MASK_OK_DONE;
	1669	+
	1670	+err:
	1671	+ its_inc_lpi_count(d, prev_cpu);
	1672	+ return -EINVAL;
1183	1673	}
1184	1674
1185	1675	static u64 its_irq_get_msi_base(struct its_device *its_dev)
..	..	@@ -1202,7 +1692,7 @@
1202	1692	msg->address_hi = upper_32_bits(addr);
1203	1693	msg->data = its_get_event_id(d);
1204	1694
1205		- iommu_dma_map_msi_msg(d->irq, msg);
	1695	+ iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1206	1696	}
1207	1697
1208	1698	static int its_irq_set_irqchip_state(struct irq_data *d,
..	..	@@ -1215,20 +1705,51 @@
1215	1705	if (which != IRQCHIP_STATE_PENDING)
1216	1706	return -EINVAL;
1217	1707
1218		- if (state)
1219		- its_send_int(its_dev, event);
1220		- else
1221		- its_send_clear(its_dev, event);
	1708	+ if (irqd_is_forwarded_to_vcpu(d)) {
	1709	+ if (state)
	1710	+ its_send_vint(its_dev, event);
	1711	+ else
	1712	+ its_send_vclear(its_dev, event);
	1713	+ } else {
	1714	+ if (state)
	1715	+ its_send_int(its_dev, event);
	1716	+ else
	1717	+ its_send_clear(its_dev, event);
	1718	+ }
1222	1719
1223	1720	return 0;
	1721	+}
	1722	+
	1723	+static int its_irq_retrigger(struct irq_data *d)
	1724	+{
	1725	+ return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
	1726	+}
	1727	+
	1728	+/*
	1729	+ * Two favourable cases:
	1730	+ *
	1731	+ * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
	1732	+ * for vSGI delivery
	1733	+ *
	1734	+ * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
	1735	+ * and we're better off mapping all VPEs always
	1736	+ *
	1737	+ * If neither (a) nor (b) is true, then we map vPEs on demand.
	1738	+ *
	1739	+ */
	1740	+static bool gic_requires_eager_mapping(void)
	1741	+{
	1742	+ if (!its_list_map \|\| gic_rdists->has_rvpeid)
	1743	+ return true;
	1744	+
	1745	+ return false;
1224	1746	}
1225	1747
1226	1748	static void its_map_vm(struct its_node its, struct its_vm vm)
1227	1749	{
1228	1750	unsigned long flags;
1229	1751
1230		- /* Not using the ITS list? Everything is always mapped. */
1231		- if (!its_list_map)
	1752	+ if (gic_requires_eager_mapping())
1232	1753	return;
1233	1754
1234	1755	raw_spin_lock_irqsave(&vmovp_lock, flags);
..	..	@@ -1262,7 +1783,7 @@
1262	1783	unsigned long flags;
1263	1784
1264	1785	/* Not using the ITS list? Everything is always mapped. */
1265		- if (!its_list_map)
	1786	+ if (gic_requires_eager_mapping())
1266	1787	return;
1267	1788
1268	1789	raw_spin_lock_irqsave(&vmovp_lock, flags);
..	..	@@ -1286,13 +1807,13 @@
1286	1807	if (!info->map)
1287	1808	return -EINVAL;
1288	1809
1289		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1810	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1290	1811
1291	1812	if (!its_dev->event_map.vm) {
1292	1813	struct its_vlpi_map *maps;
1293	1814
1294	1815	maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1295		- GFP_KERNEL);
	1816	+ GFP_ATOMIC);
1296	1817	if (!maps) {
1297	1818	ret = -ENOMEM;
1298	1819	goto out;
..	..	@@ -1335,29 +1856,30 @@
1335	1856	}
1336	1857
1337	1858	out:
1338		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1859	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1339	1860	return ret;
1340	1861	}
1341	1862
1342	1863	static int its_vlpi_get(struct irq_data d, struct its_cmd_info info)
1343	1864	{
1344	1865	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1345		- u32 event = its_get_event_id(d);
	1866	+ struct its_vlpi_map *map;
1346	1867	int ret = 0;
1347	1868
1348		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1869	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1349	1870
1350		- if (!its_dev->event_map.vm \|\|
1351		- !its_dev->event_map.vlpi_maps[event].vm) {
	1871	+ map = get_vlpi_map(d);
	1872	+
	1873	+ if (!its_dev->event_map.vm \|\| !map) {
1352	1874	ret = -EINVAL;
1353	1875	goto out;
1354	1876	}
1355	1877
1356	1878	/* Copy our mapping information to the incoming request */
1357		- *info->map = its_dev->event_map.vlpi_maps[event];
	1879	+ info->map = map;
1358	1880
1359	1881	out:
1360		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1882	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1361	1883	return ret;
1362	1884	}
1363	1885
..	..	@@ -1367,7 +1889,7 @@
1367	1889	u32 event = its_get_event_id(d);
1368	1890	int ret = 0;
1369	1891
1370		- mutex_lock(&its_dev->event_map.vlpi_lock);
	1892	+ raw_spin_lock(&its_dev->event_map.vlpi_lock);
1371	1893
1372	1894	if (!its_dev->event_map.vm \|\| !irqd_is_forwarded_to_vcpu(d)) {
1373	1895	ret = -EINVAL;
..	..	@@ -1397,7 +1919,7 @@
1397	1919	}
1398	1920
1399	1921	out:
1400		- mutex_unlock(&its_dev->event_map.vlpi_lock);
	1922	+ raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1401	1923	return ret;
1402	1924	}
1403	1925
..	..	@@ -1423,7 +1945,7 @@
1423	1945	struct its_cmd_info *info = vcpu_info;
1424	1946
1425	1947	/* Need a v4 ITS */
1426		- if (!its_dev->its->is_v4)
	1948	+ if (!is_v4(its_dev->its))
1427	1949	return -EINVAL;
1428	1950
1429	1951	/* Unmap request? */
..	..	@@ -1454,6 +1976,7 @@
1454	1976	.irq_set_affinity = its_set_affinity,
1455	1977	.irq_compose_msi_msg = its_irq_compose_msi_msg,
1456	1978	.irq_set_irqchip_state = its_irq_set_irqchip_state,
	1979	+ .irq_retrigger = its_irq_retrigger,
1457	1980	.irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
1458	1981	};
1459	1982
..	..	@@ -1488,39 +2011,13 @@
1488	2011	{
1489	2012	struct lpi_range *range;
1490	2013
1491		- range = kzalloc(sizeof(*range), GFP_KERNEL);
	2014	+ range = kmalloc(sizeof(*range), GFP_KERNEL);
1492	2015	if (range) {
1493		- INIT_LIST_HEAD(&range->entry);
1494	2016	range->base_id = base;
1495	2017	range->span = span;
1496	2018	}
1497	2019
1498	2020	return range;
1499		-}
1500		-
1501		-static int lpi_range_cmp(void priv, struct list_head a, struct list_head *b)
1502		-{
1503		- struct lpi_range ra, rb;
1504		-
1505		- ra = container_of(a, struct lpi_range, entry);
1506		- rb = container_of(b, struct lpi_range, entry);
1507		-
1508		- return ra->base_id - rb->base_id;
1509		-}
1510		-
1511		-static void merge_lpi_ranges(void)
1512		-{
1513		- struct lpi_range range, tmp;
1514		-
1515		- list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
1516		- if (!list_is_last(&range->entry, &lpi_range_list) &&
1517		- (tmp->base_id == (range->base_id + range->span))) {
1518		- tmp->base_id = range->base_id;
1519		- tmp->span += range->span;
1520		- list_del(&range->entry);
1521		- kfree(range);
1522		- }
1523		- }
1524	2021	}
1525	2022
1526	2023	static int alloc_lpi_range(u32 nr_lpis, u32 *base)
..	..	@@ -1552,25 +2049,49 @@
1552	2049	return err;
1553	2050	}
1554	2051
	2052	+static void merge_lpi_ranges(struct lpi_range a, struct lpi_range b)
	2053	+{
	2054	+ if (&a->entry == &lpi_range_list \|\| &b->entry == &lpi_range_list)
	2055	+ return;
	2056	+ if (a->base_id + a->span != b->base_id)
	2057	+ return;
	2058	+ b->base_id = a->base_id;
	2059	+ b->span += a->span;
	2060	+ list_del(&a->entry);
	2061	+ kfree(a);
	2062	+}
	2063	+
1555	2064	static int free_lpi_range(u32 base, u32 nr_lpis)
1556	2065	{
1557		- struct lpi_range *new;
1558		- int err = 0;
	2066	+ struct lpi_range new, old;
	2067	+
	2068	+ new = mk_lpi_range(base, nr_lpis);
	2069	+ if (!new)
	2070	+ return -ENOMEM;
1559	2071
1560	2072	mutex_lock(&lpi_range_lock);
1561	2073
1562		- new = mk_lpi_range(base, nr_lpis);
1563		- if (!new) {
1564		- err = -ENOMEM;
1565		- goto out;
	2074	+ list_for_each_entry_reverse(old, &lpi_range_list, entry) {
	2075	+ if (old->base_id < base)
	2076	+ break;
1566	2077	}
	2078	+ /*
	2079	+ * old is the last element with ->base_id smaller than base,
	2080	+ * so new goes right after it. If there are no elements with
	2081	+ * ->base_id smaller than base, &old->entry ends up pointing
	2082	+ * at the head of the list, and inserting new it the start of
	2083	+ * the list is the right thing to do in that case as well.
	2084	+ */
	2085	+ list_add(&new->entry, &old->entry);
	2086	+ /*
	2087	+ * Now check if we can merge with the preceding and/or
	2088	+ * following ranges.
	2089	+ */
	2090	+ merge_lpi_ranges(old, new);
	2091	+ merge_lpi_ranges(new, list_next_entry(new, entry));
1567	2092
1568		- list_add(&new->entry, &lpi_range_list);
1569		- list_sort(NULL, &lpi_range_list, lpi_range_cmp);
1570		- merge_lpi_ranges();
1571		-out:
1572	2093	mutex_unlock(&lpi_range_lock);
1573		- return err;
	2094	+ return 0;
1574	2095	}
1575	2096
1576	2097	static int __init its_lpi_init(u32 id_bits)
..	..	@@ -1634,23 +2155,28 @@
1634	2155	kfree(bitmap);
1635	2156	}
1636	2157
	2158	+static void gic_reset_prop_table(void *va)
	2159	+{
	2160	+ /* Priority 0xa0, Group-1, disabled */
	2161	+ memset(va, LPI_PROP_DEFAULT_PRIO \| LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
	2162	+
	2163	+ /* Make sure the GIC will observe the written configuration */
	2164	+ gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
	2165	+}
	2166	+
1637	2167	static struct page *its_allocate_prop_table(gfp_t gfp_flags)
1638	2168	{
1639	2169	struct page *prop_page;
1640	2170
1641		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2171	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2172	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2173	+ of_machine_is_compatible("rockchip,rk3566"))
1642	2174	gfp_flags \|= GFP_DMA32;
1643	2175	prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
1644	2176	if (!prop_page)
1645	2177	return NULL;
1646	2178
1647		- /* Priority 0xa0, Group-1, disabled */
1648		- memset(page_address(prop_page),
1649		- LPI_PROP_DEFAULT_PRIO \| LPI_PROP_GROUP1,
1650		- LPI_PROPBASE_SZ);
1651		-
1652		- /* Make sure the GIC will observe the written configuration */
1653		- gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ);
	2179	+ gic_reset_prop_table(page_address(prop_page));
1654	2180
1655	2181	return prop_page;
1656	2182	}
..	..	@@ -1661,20 +2187,74 @@
1661	2187	get_order(LPI_PROPBASE_SZ));
1662	2188	}
1663	2189
1664		-static int __init its_alloc_lpi_tables(void)
	2190	+static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
1665	2191	{
1666		- phys_addr_t paddr;
	2192	+ phys_addr_t start, end, addr_end;
	2193	+ u64 i;
1667	2194
1668		- lpi_id_bits = min_t(u32, GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
1669		- ITS_MAX_LPI_NRBITS);
1670		- gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT);
1671		- if (!gic_rdists->prop_page) {
1672		- pr_err("Failed to allocate PROPBASE\n");
1673		- return -ENOMEM;
	2195	+ /*
	2196	+ * We don't bother checking for a kdump kernel as by
	2197	+ * construction, the LPI tables are out of this kernel's
	2198	+ * memory map.
	2199	+ */
	2200	+ if (is_kdump_kernel())
	2201	+ return true;
	2202	+
	2203	+ addr_end = addr + size - 1;
	2204	+
	2205	+ for_each_reserved_mem_range(i, &start, &end) {
	2206	+ if (addr >= start && addr_end <= end)
	2207	+ return true;
1674	2208	}
1675	2209
1676		- paddr = page_to_phys(gic_rdists->prop_page);
1677		- pr_info("GIC: using LPI property table @%pa\n", &paddr);
	2210	+ /* Not found, not a good sign... */
	2211	+ pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
	2212	+ &addr, &addr_end);
	2213	+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
	2214	+ return false;
	2215	+}
	2216	+
	2217	+static int gic_reserve_range(phys_addr_t addr, unsigned long size)
	2218	+{
	2219	+ if (efi_enabled(EFI_CONFIG_TABLES))
	2220	+ return efi_mem_reserve_persistent(addr, size);
	2221	+
	2222	+ return 0;
	2223	+}
	2224	+
	2225	+static int __init its_setup_lpi_prop_table(void)
	2226	+{
	2227	+ if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
	2228	+ u64 val;
	2229	+
	2230	+ val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
	2231	+ lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
	2232	+
	2233	+ gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
	2234	+ gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
	2235	+ LPI_PROPBASE_SZ,
	2236	+ MEMREMAP_WB);
	2237	+ gic_reset_prop_table(gic_rdists->prop_table_va);
	2238	+ } else {
	2239	+ struct page *page;
	2240	+
	2241	+ lpi_id_bits = min_t(u32,
	2242	+ GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
	2243	+ ITS_MAX_LPI_NRBITS);
	2244	+ page = its_allocate_prop_table(GFP_NOWAIT);
	2245	+ if (!page) {
	2246	+ pr_err("Failed to allocate PROPBASE\n");
	2247	+ return -ENOMEM;
	2248	+ }
	2249	+
	2250	+ gic_rdists->prop_table_pa = page_to_phys(page);
	2251	+ gic_rdists->prop_table_va = page_address(page);
	2252	+ WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
	2253	+ LPI_PROPBASE_SZ));
	2254	+ }
	2255	+
	2256	+ pr_info("GICv3: using LPI property table @%pa\n",
	2257	+ &gic_rdists->prop_table_pa);
1678	2258
1679	2259	return its_lpi_init(lpi_id_bits);
1680	2260	}
..	..	@@ -1706,18 +2286,18 @@
1706	2286	}
1707	2287
1708	2288	static int its_setup_baser(struct its_node its, struct its_baser baser,
1709		- u64 cache, u64 shr, u32 psz, u32 order,
1710		- bool indirect)
	2289	+ u64 cache, u64 shr, u32 order, bool indirect)
1711	2290	{
1712	2291	u64 val = its_read_baser(its, baser);
1713	2292	u64 esz = GITS_BASER_ENTRY_SIZE(val);
1714	2293	u64 type = GITS_BASER_TYPE(val);
1715	2294	u64 baser_phys, tmp;
1716		- u32 alloc_pages;
	2295	+ u32 alloc_pages, psz;
	2296	+ struct page *page;
1717	2297	void *base;
1718	2298	gfp_t gfp_flags;
1719	2299
1720		-retry_alloc_baser:
	2300	+ psz = baser->psz;
1721	2301	alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
1722	2302	if (alloc_pages > GITS_BASER_PAGES_MAX) {
1723	2303	pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
..	..	@@ -1728,12 +2308,15 @@
1728	2308	}
1729	2309
1730	2310	gfp_flags = GFP_KERNEL \| __GFP_ZERO;
1731		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2311	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2312	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2313	+ of_machine_is_compatible("rockchip,rk3566"))
1732	2314	gfp_flags \|= GFP_DMA32;
1733		- base = (void *)__get_free_pages(gfp_flags, order);
1734		- if (!base)
	2315	+ page = alloc_pages_node(its->numa_node, gfp_flags, order);
	2316	+ if (!page)
1735	2317	return -ENOMEM;
1736	2318
	2319	+ base = (void *)page_address(page);
1737	2320	baser_phys = virt_to_phys(base);
1738	2321
1739	2322	/* Check if the physical address of the memory is above 48bits */
..	..	@@ -1776,8 +2359,11 @@
1776	2359	its_write_baser(its, baser, val);
1777	2360	tmp = baser->val;
1778	2361
1779		- if (of_machine_is_compatible("rockchip,rk3568") \|\|
1780		- of_machine_is_compatible("rockchip,rk3566")) {
	2362	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	2363	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	2364	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2365	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	2366	+ of_machine_is_compatible("rockchip,rk3588"))) {
1781	2367	if (tmp & GITS_BASER_SHAREABILITY_MASK)
1782	2368	tmp &= ~GITS_BASER_SHAREABILITY_MASK;
1783	2369	else
..	..	@@ -1798,25 +2384,6 @@
1798	2384	gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
1799	2385	}
1800	2386	goto retry_baser;
1801		- }
1802		-
1803		- if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
1804		- /*
1805		- * Page size didn't stick. Let's try a smaller
1806		- * size and retry. If we reach 4K, then
1807		- * something is horribly wrong...
1808		- */
1809		- free_pages((unsigned long)base, order);
1810		- baser->base = NULL;
1811		-
1812		- switch (psz) {
1813		- case SZ_16K:
1814		- psz = SZ_4K;
1815		- goto retry_alloc_baser;
1816		- case SZ_64K:
1817		- psz = SZ_16K;
1818		- goto retry_alloc_baser;
1819		- }
1820	2387	}
1821	2388
1822	2389	if (val != tmp) {
..	..	@@ -1844,13 +2411,14 @@
1844	2411
1845	2412	static bool its_parse_indirect_baser(struct its_node *its,
1846	2413	struct its_baser *baser,
1847		- u32 psz, u32 *order, u32 ids)
	2414	+ u32 *order, u32 ids)
1848	2415	{
1849	2416	u64 tmp = its_read_baser(its, baser);
1850	2417	u64 type = GITS_BASER_TYPE(tmp);
1851	2418	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
1852	2419	u64 val = GITS_BASER_InnerShareable \| GITS_BASER_RaWaWb;
1853	2420	u32 new_order = *order;
	2421	+ u32 psz = baser->psz;
1854	2422	bool indirect = false;
1855	2423
1856	2424	/* No need to enable Indirection if memory requirement < (psz2)bytes /
..	..	@@ -1886,14 +2454,73 @@
1886	2454	if (new_order >= MAX_ORDER) {
1887	2455	new_order = MAX_ORDER - 1;
1888	2456	ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
1889		- pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
	2457	+ pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
1890	2458	&its->phys_base, its_base_type_string[type],
1891		- its->device_ids, ids);
	2459	+ device_ids(its), ids);
1892	2460	}
1893	2461
1894	2462	*order = new_order;
1895	2463
1896	2464	return indirect;
	2465	+}
	2466	+
	2467	+static u32 compute_common_aff(u64 val)
	2468	+{
	2469	+ u32 aff, clpiaff;
	2470	+
	2471	+ aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
	2472	+ clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
	2473	+
	2474	+ return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
	2475	+}
	2476	+
	2477	+static u32 compute_its_aff(struct its_node *its)
	2478	+{
	2479	+ u64 val;
	2480	+ u32 svpet;
	2481	+
	2482	+ /*
	2483	+ * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
	2484	+ * the resulting affinity. We then use that to see if this match
	2485	+ * our own affinity.
	2486	+ */
	2487	+ svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
	2488	+ val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
	2489	+ val \|= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
	2490	+ return compute_common_aff(val);
	2491	+}
	2492	+
	2493	+static struct its_node find_sibling_its(struct its_node cur_its)
	2494	+{
	2495	+ struct its_node *its;
	2496	+ u32 aff;
	2497	+
	2498	+ if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
	2499	+ return NULL;
	2500	+
	2501	+ aff = compute_its_aff(cur_its);
	2502	+
	2503	+ list_for_each_entry(its, &its_nodes, entry) {
	2504	+ u64 baser;
	2505	+
	2506	+ if (!is_v4_1(its) \|\| its == cur_its)
	2507	+ continue;
	2508	+
	2509	+ if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
	2510	+ continue;
	2511	+
	2512	+ if (aff != compute_its_aff(its))
	2513	+ continue;
	2514	+
	2515	+ /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
	2516	+ baser = its->tables[2].val;
	2517	+ if (!(baser & GITS_BASER_VALID))
	2518	+ continue;
	2519	+
	2520	+ return its;
	2521	+ }
	2522	+
	2523	+ return NULL;
1897	2524	}
1898	2525
1899	2526	static void its_free_tables(struct its_node *its)
..	..	@@ -1909,11 +2536,58 @@
1909	2536	}
1910	2537	}
1911	2538
	2539	+static int its_probe_baser_psz(struct its_node its, struct its_baser baser)
	2540	+{
	2541	+ u64 psz = SZ_64K;
	2542	+
	2543	+ while (psz) {
	2544	+ u64 val, gpsz;
	2545	+
	2546	+ val = its_read_baser(its, baser);
	2547	+ val &= ~GITS_BASER_PAGE_SIZE_MASK;
	2548	+
	2549	+ switch (psz) {
	2550	+ case SZ_64K:
	2551	+ gpsz = GITS_BASER_PAGE_SIZE_64K;
	2552	+ break;
	2553	+ case SZ_16K:
	2554	+ gpsz = GITS_BASER_PAGE_SIZE_16K;
	2555	+ break;
	2556	+ case SZ_4K:
	2557	+ default:
	2558	+ gpsz = GITS_BASER_PAGE_SIZE_4K;
	2559	+ break;
	2560	+ }
	2561	+
	2562	+ gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
	2563	+
	2564	+ val \|= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
	2565	+ its_write_baser(its, baser, val);
	2566	+
	2567	+ if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
	2568	+ break;
	2569	+
	2570	+ switch (psz) {
	2571	+ case SZ_64K:
	2572	+ psz = SZ_16K;
	2573	+ break;
	2574	+ case SZ_16K:
	2575	+ psz = SZ_4K;
	2576	+ break;
	2577	+ case SZ_4K:
	2578	+ default:
	2579	+ return -1;
	2580	+ }
	2581	+ }
	2582	+
	2583	+ baser->psz = psz;
	2584	+ return 0;
	2585	+}
	2586	+
1912	2587	static int its_alloc_tables(struct its_node *its)
1913	2588	{
1914	2589	u64 shr = GITS_BASER_InnerShareable;
1915	2590	u64 cache = GITS_BASER_RaWaWb;
1916		- u32 psz = SZ_64K;
1917	2591	int err, i;
1918	2592
1919	2593	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
..	..	@@ -1924,37 +2598,337 @@
1924	2598	struct its_baser *baser = its->tables + i;
1925	2599	u64 val = its_read_baser(its, baser);
1926	2600	u64 type = GITS_BASER_TYPE(val);
1927		- u32 order = get_order(psz);
1928	2601	bool indirect = false;
	2602	+ u32 order;
1929	2603
1930		- switch (type) {
1931		- case GITS_BASER_TYPE_NONE:
	2604	+ if (type == GITS_BASER_TYPE_NONE)
1932	2605	continue;
1933	2606
	2607	+ if (its_probe_baser_psz(its, baser)) {
	2608	+ its_free_tables(its);
	2609	+ return -ENXIO;
	2610	+ }
	2611	+
	2612	+ order = get_order(baser->psz);
	2613	+
	2614	+ switch (type) {
1934	2615	case GITS_BASER_TYPE_DEVICE:
1935		- indirect = its_parse_indirect_baser(its, baser,
1936		- psz, &order,
1937		- its->device_ids);
	2616	+ indirect = its_parse_indirect_baser(its, baser, &order,
	2617	+ device_ids(its));
1938	2618	break;
1939	2619
1940	2620	case GITS_BASER_TYPE_VCPU:
1941		- indirect = its_parse_indirect_baser(its, baser,
1942		- psz, &order,
	2621	+ if (is_v4_1(its)) {
	2622	+ struct its_node *sibling;
	2623	+
	2624	+ WARN_ON(i != 2);
	2625	+ if ((sibling = find_sibling_its(its))) {
	2626	+ *baser = sibling->tables[2];
	2627	+ its_write_baser(its, baser, baser->val);
	2628	+ continue;
	2629	+ }
	2630	+ }
	2631	+
	2632	+ indirect = its_parse_indirect_baser(its, baser, &order,
1943	2633	ITS_MAX_VPEID_BITS);
1944	2634	break;
1945	2635	}
1946	2636
1947		- err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
	2637	+ err = its_setup_baser(its, baser, cache, shr, order, indirect);
1948	2638	if (err < 0) {
1949	2639	its_free_tables(its);
1950	2640	return err;
1951	2641	}
1952	2642
1953	2643	/* Update settings which will be used for next BASERn */
1954		- psz = baser->psz;
1955	2644	cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
1956	2645	shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
1957	2646	}
	2647	+
	2648	+ return 0;
	2649	+}
	2650	+
	2651	+static u64 inherit_vpe_l1_table_from_its(void)
	2652	+{
	2653	+ struct its_node *its;
	2654	+ u64 val;
	2655	+ u32 aff;
	2656	+
	2657	+ val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
	2658	+ aff = compute_common_aff(val);
	2659	+
	2660	+ list_for_each_entry(its, &its_nodes, entry) {
	2661	+ u64 baser, addr;
	2662	+
	2663	+ if (!is_v4_1(its))
	2664	+ continue;
	2665	+
	2666	+ if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
	2667	+ continue;
	2668	+
	2669	+ if (aff != compute_its_aff(its))
	2670	+ continue;
	2671	+
	2672	+ /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
	2673	+ baser = its->tables[2].val;
	2674	+ if (!(baser & GITS_BASER_VALID))
	2675	+ continue;
	2676	+
	2677	+ /* We have a winner! */
	2678	+ gic_data_rdist()->vpe_l1_base = its->tables[2].base;
	2679	+
	2680	+ val = GICR_VPROPBASER_4_1_VALID;
	2681	+ if (baser & GITS_BASER_INDIRECT)
	2682	+ val \|= GICR_VPROPBASER_4_1_INDIRECT;
	2683	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
	2684	+ FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
	2685	+ switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
	2686	+ case GIC_PAGE_SIZE_64K:
	2687	+ addr = GITS_BASER_ADDR_48_to_52(baser);
	2688	+ break;
	2689	+ default:
	2690	+ addr = baser & GENMASK_ULL(47, 12);
	2691	+ break;
	2692	+ }
	2693	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
	2694	+ val \|= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
	2695	+ FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
	2696	+ val \|= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
	2697	+ FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
	2698	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
	2699	+
	2700	+ return val;
	2701	+ }
	2702	+
	2703	+ return 0;
	2704	+}
	2705	+
	2706	+static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
	2707	+{
	2708	+ u32 aff;
	2709	+ u64 val;
	2710	+ int cpu;
	2711	+
	2712	+ val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
	2713	+ aff = compute_common_aff(val);
	2714	+
	2715	+ for_each_possible_cpu(cpu) {
	2716	+ void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
	2717	+
	2718	+ if (!base \|\| cpu == smp_processor_id())
	2719	+ continue;
	2720	+
	2721	+ val = gic_read_typer(base + GICR_TYPER);
	2722	+ if (aff != compute_common_aff(val))
	2723	+ continue;
	2724	+
	2725	+ /*
	2726	+ * At this point, we have a victim. This particular CPU
	2727	+ * has already booted, and has an affinity that matches
	2728	+ * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
	2729	+ * Make sure we don't write the Z bit in that case.
	2730	+ */
	2731	+ val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
	2732	+ val &= ~GICR_VPROPBASER_4_1_Z;
	2733	+
	2734	+ gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
	2735	+ *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
	2736	+
	2737	+ return val;
	2738	+ }
	2739	+
	2740	+ return 0;
	2741	+}
	2742	+
	2743	+static bool allocate_vpe_l2_table(int cpu, u32 id)
	2744	+{
	2745	+ void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
	2746	+ unsigned int psz, esz, idx, npg, gpsz;
	2747	+ u64 val;
	2748	+ struct page *page;
	2749	+ __le64 *table;
	2750	+
	2751	+ if (!gic_rdists->has_rvpeid)
	2752	+ return true;
	2753	+
	2754	+ /* Skip non-present CPUs */
	2755	+ if (!base)
	2756	+ return true;
	2757	+
	2758	+ val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
	2759	+
	2760	+ esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
	2761	+ gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
	2762	+ npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
	2763	+
	2764	+ switch (gpsz) {
	2765	+ default:
	2766	+ WARN_ON(1);
	2767	+ fallthrough;
	2768	+ case GIC_PAGE_SIZE_4K:
	2769	+ psz = SZ_4K;
	2770	+ break;
	2771	+ case GIC_PAGE_SIZE_16K:
	2772	+ psz = SZ_16K;
	2773	+ break;
	2774	+ case GIC_PAGE_SIZE_64K:
	2775	+ psz = SZ_64K;
	2776	+ break;
	2777	+ }
	2778	+
	2779	+ /* Don't allow vpe_id that exceeds single, flat table limit */
	2780	+ if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
	2781	+ return (id < (npg * psz / (esz * SZ_8)));
	2782	+
	2783	+ /* Compute 1st level table index & check if that exceeds table limit */
	2784	+ idx = id >> ilog2(psz / (esz * SZ_8));
	2785	+ if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
	2786	+ return false;
	2787	+
	2788	+ table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
	2789	+
	2790	+ /* Allocate memory for 2nd level table */
	2791	+ if (!table[idx]) {
	2792	+ page = alloc_pages(GFP_KERNEL \| __GFP_ZERO, get_order(psz));
	2793	+ if (!page)
	2794	+ return false;
	2795	+
	2796	+ /* Flush Lvl2 table to PoC if hw doesn't support coherency */
	2797	+ if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
	2798	+ gic_flush_dcache_to_poc(page_address(page), psz);
	2799	+
	2800	+ table[idx] = cpu_to_le64(page_to_phys(page) \| GITS_BASER_VALID);
	2801	+
	2802	+ /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
	2803	+ if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
	2804	+ gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
	2805	+
	2806	+ /* Ensure updated table contents are visible to RD hardware */
	2807	+ dsb(sy);
	2808	+ }
	2809	+
	2810	+ return true;
	2811	+}
	2812	+
	2813	+static int allocate_vpe_l1_table(void)
	2814	+{
	2815	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	2816	+ u64 val, gpsz, npg, pa;
	2817	+ unsigned int psz = SZ_64K;
	2818	+ unsigned int np, epp, esz;
	2819	+ struct page *page;
	2820	+
	2821	+ if (!gic_rdists->has_rvpeid)
	2822	+ return 0;
	2823	+
	2824	+ /*
	2825	+ * if VPENDBASER.Valid is set, disable any previously programmed
	2826	+ * VPE by setting PendingLast while clearing Valid. This has the
	2827	+ * effect of making sure no doorbell will be generated and we can
	2828	+ * then safely clear VPROPBASER.Valid.
	2829	+ */
	2830	+ if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
	2831	+ gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
	2832	+ vlpi_base + GICR_VPENDBASER);
	2833	+
	2834	+ /*
	2835	+ * If we can inherit the configuration from another RD, let's do
	2836	+ * so. Otherwise, we have to go through the allocation process. We
	2837	+ * assume that all RDs have the exact same requirements, as
	2838	+ * nothing will work otherwise.
	2839	+ */
	2840	+ val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
	2841	+ if (val & GICR_VPROPBASER_4_1_VALID)
	2842	+ goto out;
	2843	+
	2844	+ gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
	2845	+ if (!gic_data_rdist()->vpe_table_mask)
	2846	+ return -ENOMEM;
	2847	+
	2848	+ val = inherit_vpe_l1_table_from_its();
	2849	+ if (val & GICR_VPROPBASER_4_1_VALID)
	2850	+ goto out;
	2851	+
	2852	+ /* First probe the page size */
	2853	+ val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
	2854	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	2855	+ val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
	2856	+ gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
	2857	+ esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
	2858	+
	2859	+ switch (gpsz) {
	2860	+ default:
	2861	+ gpsz = GIC_PAGE_SIZE_4K;
	2862	+ fallthrough;
	2863	+ case GIC_PAGE_SIZE_4K:
	2864	+ psz = SZ_4K;
	2865	+ break;
	2866	+ case GIC_PAGE_SIZE_16K:
	2867	+ psz = SZ_16K;
	2868	+ break;
	2869	+ case GIC_PAGE_SIZE_64K:
	2870	+ psz = SZ_64K;
	2871	+ break;
	2872	+ }
	2873	+
	2874	+ /*
	2875	+ * Start populating the register from scratch, including RO fields
	2876	+ * (which we want to print in debug cases...)
	2877	+ */
	2878	+ val = 0;
	2879	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
	2880	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
	2881	+
	2882	+ /* How many entries per GIC page? */
	2883	+ esz++;
	2884	+ epp = psz / (esz * SZ_8);
	2885	+
	2886	+ /*
	2887	+ * If we need more than just a single L1 page, flag the table
	2888	+ * as indirect and compute the number of required L1 pages.
	2889	+ */
	2890	+ if (epp < ITS_MAX_VPEID) {
	2891	+ int nl2;
	2892	+
	2893	+ val \|= GICR_VPROPBASER_4_1_INDIRECT;
	2894	+
	2895	+ /* Number of L2 pages required to cover the VPEID space */
	2896	+ nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
	2897	+
	2898	+ /* Number of L1 pages to point to the L2 pages */
	2899	+ npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
	2900	+ } else {
	2901	+ npg = 1;
	2902	+ }
	2903	+
	2904	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
	2905	+
	2906	+ /* Right, that's the number of CPU pages we need for L1 */
	2907	+ np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
	2908	+
	2909	+ pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
	2910	+ np, npg, psz, epp, esz);
	2911	+ page = alloc_pages(GFP_ATOMIC \| __GFP_ZERO, get_order(np * PAGE_SIZE));
	2912	+ if (!page)
	2913	+ return -ENOMEM;
	2914	+
	2915	+ gic_data_rdist()->vpe_l1_base = page_address(page);
	2916	+ pa = virt_to_phys(page_address(page));
	2917	+ WARN_ON(!IS_ALIGNED(pa, psz));
	2918	+
	2919	+ val \|= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
	2920	+ val \|= GICR_VPROPBASER_RaWb;
	2921	+ val \|= GICR_VPROPBASER_InnerShareable;
	2922	+ val \|= GICR_VPROPBASER_4_1_Z;
	2923	+ val \|= GICR_VPROPBASER_4_1_VALID;
	2924	+
	2925	+out:
	2926	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	2927	+ cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
	2928	+
	2929	+ pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
	2930	+ smp_processor_id(), val,
	2931	+ cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
1958	2932
1959	2933	return 0;
1960	2934	}
..	..	@@ -1977,14 +2951,13 @@
1977	2951	static struct page *its_allocate_pending_table(gfp_t gfp_flags)
1978	2952	{
1979	2953	struct page *pend_page;
1980		- /*
1981		- * The pending pages have to be at least 64kB aligned,
1982		- * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1983		- */
1984		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	2954	+
	2955	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	2956	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	2957	+ of_machine_is_compatible("rockchip,rk3566"))
1985	2958	gfp_flags \|= GFP_DMA32;
1986	2959	pend_page = alloc_pages(gfp_flags \| __GFP_ZERO,
1987		- get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
	2960	+ get_order(LPI_PENDBASE_SZ));
1988	2961	if (!pend_page)
1989	2962	return NULL;
1990	2963
..	..	@@ -1996,22 +2969,73 @@
1996	2969
1997	2970	static void its_free_pending_table(struct page *pt)
1998	2971	{
1999		- free_pages((unsigned long)page_address(pt),
2000		- get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
	2972	+ free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2001	2973	}
2002	2974
2003		-static u64 its_clear_vpend_valid(void __iomem *vlpi_base)
	2975	+/*
	2976	+ * Booting with kdump and LPIs enabled is generally fine. Any other
	2977	+ * case is wrong in the absence of firmware/EFI support.
	2978	+ */
	2979	+static bool enabled_lpis_allowed(void)
	2980	+{
	2981	+ phys_addr_t addr;
	2982	+ u64 val;
	2983	+
	2984	+ /* Check whether the property table is in a reserved region */
	2985	+ val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
	2986	+ addr = val & GENMASK_ULL(51, 12);
	2987	+
	2988	+ return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
	2989	+}
	2990	+
	2991	+static int __init allocate_lpi_tables(void)
	2992	+{
	2993	+ u64 val;
	2994	+ int err, cpu;
	2995	+
	2996	+ /*
	2997	+ * If LPIs are enabled while we run this from the boot CPU,
	2998	+ * flag the RD tables as pre-allocated if the stars do align.
	2999	+ */
	3000	+ val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
	3001	+ if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
	3002	+ gic_rdists->flags \|= (RDIST_FLAGS_RD_TABLES_PREALLOCATED \|
	3003	+ RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
	3004	+ pr_info("GICv3: Using preallocated redistributor tables\n");
	3005	+ }
	3006	+
	3007	+ err = its_setup_lpi_prop_table();
	3008	+ if (err)
	3009	+ return err;
	3010	+
	3011	+ /*
	3012	+ * We allocate all the pending tables anyway, as we may have a
	3013	+ * mix of RDs that have had LPIs enabled, and some that
	3014	+ * don't. We'll free the unused ones as each CPU comes online.
	3015	+ */
	3016	+ for_each_possible_cpu(cpu) {
	3017	+ struct page *pend_page;
	3018	+
	3019	+ pend_page = its_allocate_pending_table(GFP_NOWAIT);
	3020	+ if (!pend_page) {
	3021	+ pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
	3022	+ return -ENOMEM;
	3023	+ }
	3024	+
	3025	+ gic_data_rdist_cpu(cpu)->pend_page = pend_page;
	3026	+ }
	3027	+
	3028	+ return 0;
	3029	+}
	3030	+
	3031	+static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
2004	3032	{
2005	3033	u32 count = 1000000; /* 1s! */
2006	3034	bool clean;
2007	3035	u64 val;
2008	3036
2009		- val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2010		- val &= ~GICR_VPENDBASER_Valid;
2011		- gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2012		-
2013	3037	do {
2014		- val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
	3038	+ val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2015	3039	clean = !(val & GICR_VPENDBASER_Dirty);
2016	3040	if (!clean) {
2017	3041	count--;
..	..	@@ -2020,6 +3044,27 @@
2020	3044	}
2021	3045	} while (!clean && count);
2022	3046
	3047	+ if (unlikely(!clean))
	3048	+ pr_err_ratelimited("ITS virtual pending table not cleaning\n");
	3049	+
	3050	+ return val;
	3051	+}
	3052	+
	3053	+static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
	3054	+{
	3055	+ u64 val;
	3056	+
	3057	+ /* Make sure we wait until the RD is done with the initial scan */
	3058	+ val = read_vpend_dirty_clear(vlpi_base);
	3059	+ val &= ~GICR_VPENDBASER_Valid;
	3060	+ val &= ~clr;
	3061	+ val \|= set;
	3062	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3063	+
	3064	+ val = read_vpend_dirty_clear(vlpi_base);
	3065	+ if (unlikely(val & GICR_VPENDBASER_Dirty))
	3066	+ val \|= GICR_VPENDBASER_PendingLast;
	3067	+
2023	3068	return val;
2024	3069	}
2025	3070
..	..	@@ -2027,28 +3072,40 @@
2027	3072	{
2028	3073	void __iomem *rbase = gic_data_rdist_rd_base();
2029	3074	struct page *pend_page;
	3075	+ phys_addr_t paddr;
2030	3076	u64 val, tmp;
2031	3077
2032		- /* If we didn't allocate the pending table yet, do it now */
2033		- pend_page = gic_data_rdist()->pend_page;
2034		- if (!pend_page) {
2035		- phys_addr_t paddr;
	3078	+ if (gic_data_rdist()->lpi_enabled)
	3079	+ return;
2036	3080
2037		- pend_page = its_allocate_pending_table(GFP_NOWAIT);
2038		- if (!pend_page) {
2039		- pr_err("Failed to allocate PENDBASE for CPU%d\n",
2040		- smp_processor_id());
2041		- return;
2042		- }
	3081	+ val = readl_relaxed(rbase + GICR_CTLR);
	3082	+ if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
	3083	+ (val & GICR_CTLR_ENABLE_LPIS)) {
	3084	+ /*
	3085	+ * Check that we get the same property table on all
	3086	+ * RDs. If we don't, this is hopeless.
	3087	+ */
	3088	+ paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
	3089	+ paddr &= GENMASK_ULL(51, 12);
	3090	+ if (WARN_ON(gic_rdists->prop_table_pa != paddr))
	3091	+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2043	3092
2044		- paddr = page_to_phys(pend_page);
2045		- pr_info("CPU%d: using LPI pending table @%pa\n",
2046		- smp_processor_id(), &paddr);
2047		- gic_data_rdist()->pend_page = pend_page;
	3093	+ paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
	3094	+ paddr &= GENMASK_ULL(51, 16);
	3095	+
	3096	+ WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
	3097	+ its_free_pending_table(gic_data_rdist()->pend_page);
	3098	+ gic_data_rdist()->pend_page = NULL;
	3099	+
	3100	+ goto out;
2048	3101	}
2049	3102
	3103	+ pend_page = gic_data_rdist()->pend_page;
	3104	+ paddr = page_to_phys(pend_page);
	3105	+ WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
	3106	+
2050	3107	/* set PROPBASE */
2051		- val = (page_to_phys(gic_rdists->prop_page) \|
	3108	+ val = (gic_rdists->prop_table_pa \|
2052	3109	GICR_PROPBASER_InnerShareable \|
2053	3110	GICR_PROPBASER_RaWaWb \|
2054	3111	((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
..	..	@@ -2056,7 +3113,11 @@
2056	3113	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
2057	3114	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
2058	3115
2059		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3116	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	3117	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	3118	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3119	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	3120	+ of_machine_is_compatible("rockchip,rk3588")))
2060	3121	tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
2061	3122
2062	3123	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
..	..	@@ -2083,7 +3144,11 @@
2083	3144	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
2084	3145	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
2085	3146
2086		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3147	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	3148	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	3149	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3150	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	3151	+ of_machine_is_compatible("rockchip,rk3588")))
2087	3152	tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
2088	3153
2089	3154	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
..	..	@@ -2102,7 +3167,7 @@
2102	3167	val \|= GICR_CTLR_ENABLE_LPIS;
2103	3168	writel_relaxed(val, rbase + GICR_CTLR);
2104	3169
2105		- if (gic_rdists->has_vlpis) {
	3170	+ if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
2106	3171	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2107	3172
2108	3173	/*
..	..	@@ -2115,19 +3180,34 @@
2115	3180	val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2116	3181	pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
2117	3182	smp_processor_id(), val);
2118		- gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	3183	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2119	3184
2120	3185	/*
2121	3186	* Also clear Valid bit of GICR_VPENDBASER, in case some
2122	3187	* ancient programming gets left in and has possibility of
2123	3188	* corrupting memory.
2124	3189	*/
2125		- val = its_clear_vpend_valid(vlpi_base);
2126		- WARN_ON(val & GICR_VPENDBASER_Dirty);
	3190	+ val = its_clear_vpend_valid(vlpi_base, 0, 0);
	3191	+ }
	3192	+
	3193	+ if (allocate_vpe_l1_table()) {
	3194	+ /*
	3195	+ * If the allocation has failed, we're in massive trouble.
	3196	+ * Disable direct injection, and pray that no VM was
	3197	+ * already running...
	3198	+ */
	3199	+ gic_rdists->has_rvpeid = false;
	3200	+ gic_rdists->has_vlpis = false;
2127	3201	}
2128	3202
2129	3203	/* Make sure the GIC has seen the above */
2130	3204	dsb(sy);
	3205	+out:
	3206	+ gic_data_rdist()->lpi_enabled = true;
	3207	+ pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
	3208	+ smp_processor_id(),
	3209	+ gic_data_rdist()->pend_page ? "allocated" : "reserved",
	3210	+ &paddr);
2131	3211	}
2132	3212
2133	3213	static void its_cpu_init_collection(struct its_node *its)
..	..	@@ -2212,7 +3292,8 @@
2212	3292	return NULL;
2213	3293	}
2214	3294
2215		-static bool its_alloc_table_entry(struct its_baser *baser, u32 id)
	3295	+static bool its_alloc_table_entry(struct its_node *its,
	3296	+ struct its_baser *baser, u32 id)
2216	3297	{
2217	3298	struct page *page;
2218	3299	u32 esz, idx;
..	..	@@ -2234,9 +3315,12 @@
2234	3315	if (!table[idx]) {
2235	3316	gfp_t gfp_flags = GFP_KERNEL \| __GFP_ZERO;
2236	3317
2237		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3318	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3319	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3320	+ of_machine_is_compatible("rockchip,rk3566"))
2238	3321	gfp_flags \|= GFP_DMA32;
2239		- page = alloc_pages(gfp_flags, get_order(baser->psz));
	3322	+ page = alloc_pages_node(its->numa_node, gfp_flags,
	3323	+ get_order(baser->psz));
2240	3324	if (!page)
2241	3325	return false;
2242	3326
..	..	@@ -2265,14 +3349,15 @@
2265	3349
2266	3350	/* Don't allow device id that exceeds ITS hardware limit */
2267	3351	if (!baser)
2268		- return (ilog2(dev_id) < its->device_ids);
	3352	+ return (ilog2(dev_id) < device_ids(its));
2269	3353
2270		- return its_alloc_table_entry(baser, dev_id);
	3354	+ return its_alloc_table_entry(its, baser, dev_id);
2271	3355	}
2272	3356
2273	3357	static bool its_alloc_vpe_table(u32 vpe_id)
2274	3358	{
2275	3359	struct its_node *its;
	3360	+ int cpu;
2276	3361
2277	3362	/*
2278	3363	* Make sure the L2 tables are allocated on all v4 ITSs. We
..	..	@@ -2284,14 +3369,27 @@
2284	3369	list_for_each_entry(its, &its_nodes, entry) {
2285	3370	struct its_baser *baser;
2286	3371
2287		- if (!its->is_v4)
	3372	+ if (!is_v4(its))
2288	3373	continue;
2289	3374
2290	3375	baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
2291	3376	if (!baser)
2292	3377	return false;
2293	3378
2294		- if (!its_alloc_table_entry(baser, vpe_id))
	3379	+ if (!its_alloc_table_entry(its, baser, vpe_id))
	3380	+ return false;
	3381	+ }
	3382	+
	3383	+ /* Non v4.1? No need to iterate RDs and go back early. */
	3384	+ if (!gic_rdists->has_rvpeid)
	3385	+ return true;
	3386	+
	3387	+ /*
	3388	+ * Make sure the L2 tables are allocated for all copies of
	3389	+ * the L1 table on all v4.1 RDs.
	3390	+ */
	3391	+ for_each_possible_cpu(cpu) {
	3392	+ if (!allocate_vpe_l2_table(cpu, vpe_id))
2295	3393	return false;
2296	3394	}
2297	3395
..	..	@@ -2324,12 +3422,18 @@
2324	3422	* sized as a power of two (and you need at least one bit...).
2325	3423	*/
2326	3424	nr_ites = max(2, nvecs);
2327		- sz = nr_ites * its->ite_size;
	3425	+ sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
2328	3426	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
2329	3427	gfp_flags = GFP_KERNEL;
2330		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	3428	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3429	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3430	+ of_machine_is_compatible("rockchip,rk3566")) {
2331	3431	gfp_flags \|= GFP_DMA32;
2332		- itt = (void *)__get_free_pages(gfp_flags, get_order(sz));
	3432	+ itt = (void *)__get_free_pages(gfp_flags, get_order(sz));
	3433	+ } else {
	3434	+ itt = kzalloc_node(sz, gfp_flags, its->numa_node);
	3435	+ }
	3436	+
2333	3437	if (alloc_lpis) {
2334	3438	lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
2335	3439	if (lpi_map)
..	..	@@ -2343,7 +3447,14 @@
2343	3447
2344	3448	if (!dev \|\| !itt \|\| !col_map \|\| (!lpi_map && alloc_lpis)) {
2345	3449	kfree(dev);
2346		- free_pages((unsigned long)itt, get_order(sz));
	3450	+
	3451	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3452	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3453	+ of_machine_is_compatible("rockchip,rk3566"))
	3454	+ free_pages((unsigned long)itt, get_order(sz));
	3455	+ else
	3456	+ kfree(itt);
	3457	+
2347	3458	kfree(lpi_map);
2348	3459	kfree(col_map);
2349	3460	return NULL;
..	..	@@ -2359,7 +3470,7 @@
2359	3470	dev->event_map.col_map = col_map;
2360	3471	dev->event_map.lpi_base = lpi_base;
2361	3472	dev->event_map.nr_lpis = nr_lpis;
2362		- mutex_init(&dev->event_map.vlpi_lock);
	3473	+ raw_spin_lock_init(&dev->event_map.vlpi_lock);
2363	3474	dev->device_id = dev_id;
2364	3475	INIT_LIST_HEAD(&dev->entry);
2365	3476
..	..	@@ -2380,7 +3491,15 @@
2380	3491	raw_spin_lock_irqsave(&its_dev->its->lock, flags);
2381	3492	list_del(&its_dev->entry);
2382	3493	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
2383		- free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));
	3494	+ kfree(its_dev->event_map.col_map);
	3495	+
	3496	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	3497	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	3498	+ of_machine_is_compatible("rockchip,rk3566"))
	3499	+ free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));
	3500	+ else
	3501	+ kfree(its_dev->itt);
	3502	+
2384	3503	kfree(its_dev);
2385	3504	}
2386	3505
..	..	@@ -2388,6 +3507,7 @@
2388	3507	{
2389	3508	int idx;
2390	3509
	3510	+ /* Find a free LPI region in lpi_map and allocate them. */
2391	3511	idx = bitmap_find_free_region(dev->event_map.lpi_map,
2392	3512	dev->event_map.nr_lpis,
2393	3513	get_count_order(nvecs));
..	..	@@ -2395,7 +3515,6 @@
2395	3515	return -ENOSPC;
2396	3516
2397	3517	*hwirq = dev->event_map.lpi_base + idx;
2398		- set_bit(idx, dev->event_map.lpi_map);
2399	3518
2400	3519	return 0;
2401	3520	}
..	..	@@ -2410,7 +3529,7 @@
2410	3529	int err = 0;
2411	3530
2412	3531	/*
2413		- * We ignore "dev" entierely, and rely on the dev_id that has
	3532	+ * We ignore "dev" entirely, and rely on the dev_id that has
2414	3533	* been passed via the scratchpad. This limits this domain's
2415	3534	* usefulness to upper layers that definitely know that they
2416	3535	* are built on top of the ITS.
..	..	@@ -2489,12 +3608,17 @@
2489	3608	{
2490	3609	msi_alloc_info_t *info = args;
2491	3610	struct its_device *its_dev = info->scratchpad[0].ptr;
	3611	+ struct its_node *its = its_dev->its;
2492	3612	struct irq_data *irqd;
2493	3613	irq_hw_number_t hwirq;
2494	3614	int err;
2495	3615	int i;
2496	3616
2497	3617	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
	3618	+ if (err)
	3619	+ return err;
	3620	+
	3621	+ err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
2498	3622	if (err)
2499	3623	return err;
2500	3624
..	..	@@ -2521,22 +3645,13 @@
2521	3645	{
2522	3646	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2523	3647	u32 event = its_get_event_id(d);
2524		- const struct cpumask *cpu_mask = cpu_online_mask;
2525	3648	int cpu;
2526	3649
2527		- /* get the cpu_mask of local node */
2528		- if (its_dev->its->numa_node >= 0)
2529		- cpu_mask = cpumask_of_node(its_dev->its->numa_node);
	3650	+ cpu = its_select_cpu(d, cpu_online_mask);
	3651	+ if (cpu < 0 \|\| cpu >= nr_cpu_ids)
	3652	+ return -EINVAL;
2530	3653
2531		- /* Bind the LPI to the first possible CPU */
2532		- cpu = cpumask_first_and(cpu_mask, cpu_online_mask);
2533		- if (cpu >= nr_cpu_ids) {
2534		- if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)
2535		- return -EINVAL;
2536		-
2537		- cpu = cpumask_first(cpu_online_mask);
2538		- }
2539		-
	3654	+ its_inc_lpi_count(d, cpu);
2540	3655	its_dev->event_map.col_map[event] = cpu;
2541	3656	irq_data_update_effective_affinity(d, cpumask_of(cpu));
2542	3657
..	..	@@ -2551,6 +3666,7 @@
2551	3666	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2552	3667	u32 event = its_get_event_id(d);
2553	3668
	3669	+ its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
2554	3670	/* Stop the delivery of interrupts */
2555	3671	its_send_discard(its_dev, event);
2556	3672	}
..	..	@@ -2586,7 +3702,6 @@
2586	3702	its_lpi_free(its_dev->event_map.lpi_map,
2587	3703	its_dev->event_map.lpi_base,
2588	3704	its_dev->event_map.nr_lpis);
2589		- kfree(its_dev->event_map.col_map);
2590	3705
2591	3706	/* Unmap device/itt */
2592	3707	its_send_mapd(its_dev, 0);
..	..	@@ -2608,7 +3723,7 @@
2608	3723	/*
2609	3724	* This is insane.
2610	3725	*
2611		- * If a GICv4 doesn't implement Direct LPIs (which is extremely
	3726	+ * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
2612	3727	* likely), the only way to perform an invalidate is to use a fake
2613	3728	* device to issue an INV command, implying that the LPI has first
2614	3729	* been mapped to some event on that device. Since this is not exactly
..	..	@@ -2616,9 +3731,20 @@
2616	3731	* only issue an UNMAP if we're short on available slots.
2617	3732	*
2618	3733	* Broken by design(tm).
	3734	+ *
	3735	+ * GICv4.1, on the other hand, mandates that we're able to invalidate
	3736	+ * by writing to a MMIO register. It doesn't implement the whole of
	3737	+ * DirectLPI, but that's good enough. And most of the time, we don't
	3738	+ * even have to invalidate anything, as the redistributor can be told
	3739	+ * whether to generate a doorbell or not (we thus leave it enabled,
	3740	+ * always).
2619	3741	*/
2620	3742	static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
2621	3743	{
	3744	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3745	+ if (gic_rdists->has_rvpeid)
	3746	+ return;
	3747	+
2622	3748	/* Already unmapped? */
2623	3749	if (vpe->vpe_proxy_event == -1)
2624	3750	return;
..	..	@@ -2641,6 +3767,10 @@
2641	3767
2642	3768	static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
2643	3769	{
	3770	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3771	+ if (gic_rdists->has_rvpeid)
	3772	+ return;
	3773	+
2644	3774	if (!gic_rdists->has_direct_lpi) {
2645	3775	unsigned long flags;
2646	3776
..	..	@@ -2652,6 +3782,10 @@
2652	3782
2653	3783	static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
2654	3784	{
	3785	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3786	+ if (gic_rdists->has_rvpeid)
	3787	+ return;
	3788	+
2655	3789	/* Already mapped? */
2656	3790	if (vpe->vpe_proxy_event != -1)
2657	3791	return;
..	..	@@ -2674,13 +3808,16 @@
2674	3808	unsigned long flags;
2675	3809	struct its_collection *target_col;
2676	3810
	3811	+ /* GICv4.1 doesn't use a proxy, so nothing to do here */
	3812	+ if (gic_rdists->has_rvpeid)
	3813	+ return;
	3814	+
2677	3815	if (gic_rdists->has_direct_lpi) {
2678	3816	void __iomem *rdbase;
2679	3817
2680	3818	rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
2681	3819	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2682		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2683		- cpu_relax();
	3820	+ wait_for_syncr(rdbase);
2684	3821
2685	3822	return;
2686	3823	}
..	..	@@ -2701,25 +3838,58 @@
2701	3838	bool force)
2702	3839	{
2703	3840	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2704		- int cpu = cpumask_first(mask_val);
	3841	+ int from, cpu = cpumask_first(mask_val);
	3842	+ unsigned long flags;
2705	3843
2706	3844	/*
2707	3845	* Changing affinity is mega expensive, so let's be as lazy as
2708	3846	* we can and only do it if we really have to. Also, if mapped
2709	3847	* into the proxy device, we need to move the doorbell
2710	3848	* interrupt to its new location.
	3849	+ *
	3850	+ * Another thing is that changing the affinity of a vPE affects
	3851	+ * other interrupts such as all the vLPIs that are routed to
	3852	+ * this vPE. This means that the irq_desc lock is not enough to
	3853	+ * protect us, and that we must ensure nobody samples vpe->col_idx
	3854	+ * during the update, hence the lock below which must also be
	3855	+ * taken on any vLPI handling path that evaluates vpe->col_idx.
2711	3856	*/
2712		- if (vpe->col_idx != cpu) {
2713		- int from = vpe->col_idx;
	3857	+ from = vpe_to_cpuid_lock(vpe, &flags);
	3858	+ if (from == cpu)
	3859	+ goto out;
2714	3860
2715		- vpe->col_idx = cpu;
2716		- its_send_vmovp(vpe);
2717		- its_vpe_db_proxy_move(vpe, from, cpu);
2718		- }
	3861	+ vpe->col_idx = cpu;
2719	3862
	3863	+ /*
	3864	+ * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
	3865	+ * is sharing its VPE table with the current one.
	3866	+ */
	3867	+ if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
	3868	+ cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
	3869	+ goto out;
	3870	+
	3871	+ its_send_vmovp(vpe);
	3872	+ its_vpe_db_proxy_move(vpe, from, cpu);
	3873	+
	3874	+out:
2720	3875	irq_data_update_effective_affinity(d, cpumask_of(cpu));
	3876	+ vpe_to_cpuid_unlock(vpe, flags);
2721	3877
2722	3878	return IRQ_SET_MASK_OK_DONE;
	3879	+}
	3880	+
	3881	+static void its_wait_vpt_parse_complete(void)
	3882	+{
	3883	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	3884	+ u64 val;
	3885	+
	3886	+ if (!gic_rdists->has_vpend_valid_dirty)
	3887	+ return;
	3888	+
	3889	+ WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
	3890	+ val,
	3891	+ !(val & GICR_VPENDBASER_Dirty),
	3892	+ 10, 500));
2723	3893	}
2724	3894
2725	3895	static void its_vpe_schedule(struct its_vpe *vpe)
..	..	@@ -2733,12 +3903,12 @@
2733	3903	val \|= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2734	3904	val \|= GICR_VPROPBASER_RaWb;
2735	3905	val \|= GICR_VPROPBASER_InnerShareable;
2736		- gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
	3906	+ gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2737	3907
2738	3908	val = virt_to_phys(page_address(vpe->vpt_page)) &
2739	3909	GENMASK_ULL(51, 16);
2740	3910	val \|= GICR_VPENDBASER_RaWaWb;
2741		- val \|= GICR_VPENDBASER_NonShareable;
	3911	+ val \|= GICR_VPENDBASER_InnerShareable;
2742	3912	/*
2743	3913	* There is no good way of finding out if the pending table is
2744	3914	* empty as we can race against the doorbell interrupt very
..	..	@@ -2751,7 +3921,7 @@
2751	3921	val \|= GICR_VPENDBASER_PendingLast;
2752	3922	val \|= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
2753	3923	val \|= GICR_VPENDBASER_Valid;
2754		- gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	3924	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2755	3925	}
2756	3926
2757	3927	static void its_vpe_deschedule(struct its_vpe *vpe)
..	..	@@ -2759,16 +3929,10 @@
2759	3929	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2760	3930	u64 val;
2761	3931
2762		- val = its_clear_vpend_valid(vlpi_base);
	3932	+ val = its_clear_vpend_valid(vlpi_base, 0, 0);
2763	3933
2764		- if (unlikely(val & GICR_VPENDBASER_Dirty)) {
2765		- pr_err_ratelimited("ITS virtual pending table not cleaning\n");
2766		- vpe->idai = false;
2767		- vpe->pending_last = true;
2768		- } else {
2769		- vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
2770		- vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2771		- }
	3934	+ vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
	3935	+ vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2772	3936	}
2773	3937
2774	3938	static void its_vpe_invall(struct its_vpe *vpe)
..	..	@@ -2776,7 +3940,7 @@
2776	3940	struct its_node *its;
2777	3941
2778	3942	list_for_each_entry(its, &its_nodes, entry) {
2779		- if (!its->is_v4)
	3943	+ if (!is_v4(its))
2780	3944	continue;
2781	3945
2782	3946	if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
..	..	@@ -2803,6 +3967,10 @@
2803	3967
2804	3968	case DESCHEDULE_VPE:
2805	3969	its_vpe_deschedule(vpe);
	3970	+ return 0;
	3971	+
	3972	+ case COMMIT_VPE:
	3973	+ its_wait_vpt_parse_complete();
2806	3974	return 0;
2807	3975
2808	3976	case INVALL_VPE:
..	..	@@ -2834,10 +4002,12 @@
2834	4002	if (gic_rdists->has_direct_lpi) {
2835	4003	void __iomem *rdbase;
2836	4004
	4005	+ /* Target the redistributor this VPE is currently known on */
	4006	+ raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
2837	4007	rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
2838		- gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);
2839		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2840		- cpu_relax();
	4008	+ gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR);
	4009	+ wait_for_syncr(rdbase);
	4010	+ raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
2841	4011	} else {
2842	4012	its_vpe_send_cmd(vpe, its_send_inv);
2843	4013	}
..	..	@@ -2879,8 +4049,7 @@
2879	4049	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
2880	4050	} else {
2881	4051	gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2882		- while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2883		- cpu_relax();
	4052	+ wait_for_syncr(rdbase);
2884	4053	}
2885	4054	} else {
2886	4055	if (state)
..	..	@@ -2906,6 +4075,375 @@
2906	4075	.irq_retrigger = its_vpe_retrigger,
2907	4076	.irq_set_irqchip_state = its_vpe_set_irqchip_state,
2908	4077	.irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
	4078	+};
	4079	+
	4080	+static struct its_node *find_4_1_its(void)
	4081	+{
	4082	+ static struct its_node *its = NULL;
	4083	+
	4084	+ if (!its) {
	4085	+ list_for_each_entry(its, &its_nodes, entry) {
	4086	+ if (is_v4_1(its))
	4087	+ return its;
	4088	+ }
	4089	+
	4090	+ /* Oops? */
	4091	+ its = NULL;
	4092	+ }
	4093	+
	4094	+ return its;
	4095	+}
	4096	+
	4097	+static void its_vpe_4_1_send_inv(struct irq_data *d)
	4098	+{
	4099	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4100	+ struct its_node *its;
	4101	+
	4102	+ /*
	4103	+ * GICv4.1 wants doorbells to be invalidated using the
	4104	+ * INVDB command in order to be broadcast to all RDs. Send
	4105	+ * it to the first valid ITS, and let the HW do its magic.
	4106	+ */
	4107	+ its = find_4_1_its();
	4108	+ if (its)
	4109	+ its_send_invdb(its, vpe);
	4110	+}
	4111	+
	4112	+static void its_vpe_4_1_mask_irq(struct irq_data *d)
	4113	+{
	4114	+ lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
	4115	+ its_vpe_4_1_send_inv(d);
	4116	+}
	4117	+
	4118	+static void its_vpe_4_1_unmask_irq(struct irq_data *d)
	4119	+{
	4120	+ lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
	4121	+ its_vpe_4_1_send_inv(d);
	4122	+}
	4123	+
	4124	+static void its_vpe_4_1_schedule(struct its_vpe *vpe,
	4125	+ struct its_cmd_info *info)
	4126	+{
	4127	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	4128	+ u64 val = 0;
	4129	+
	4130	+ /* Schedule the VPE */
	4131	+ val \|= GICR_VPENDBASER_Valid;
	4132	+ val \|= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
	4133	+ val \|= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
	4134	+ val \|= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
	4135	+
	4136	+ gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
	4137	+}
	4138	+
	4139	+static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
	4140	+ struct its_cmd_info *info)
	4141	+{
	4142	+ void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
	4143	+ u64 val;
	4144	+
	4145	+ if (info->req_db) {
	4146	+ unsigned long flags;
	4147	+
	4148	+ /*
	4149	+ * vPE is going to block: make the vPE non-resident with
	4150	+ * PendingLast clear and DB set. The GIC guarantees that if
	4151	+ * we read-back PendingLast clear, then a doorbell will be
	4152	+ * delivered when an interrupt comes.
	4153	+ *
	4154	+ * Note the locking to deal with the concurrent update of
	4155	+ * pending_last from the doorbell interrupt handler that can
	4156	+ * run concurrently.
	4157	+ */
	4158	+ raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
	4159	+ val = its_clear_vpend_valid(vlpi_base,
	4160	+ GICR_VPENDBASER_PendingLast,
	4161	+ GICR_VPENDBASER_4_1_DB);
	4162	+ vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
	4163	+ raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
	4164	+ } else {
	4165	+ /*
	4166	+ * We're not blocking, so just make the vPE non-resident
	4167	+ * with PendingLast set, indicating that we'll be back.
	4168	+ */
	4169	+ val = its_clear_vpend_valid(vlpi_base,
	4170	+ 0,
	4171	+ GICR_VPENDBASER_PendingLast);
	4172	+ vpe->pending_last = true;
	4173	+ }
	4174	+}
	4175	+
	4176	+static void its_vpe_4_1_invall(struct its_vpe *vpe)
	4177	+{
	4178	+ void __iomem *rdbase;
	4179	+ unsigned long flags;
	4180	+ u64 val;
	4181	+ int cpu;
	4182	+
	4183	+ val = GICR_INVALLR_V;
	4184	+ val \|= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
	4185	+
	4186	+ /* Target the redistributor this vPE is currently known on */
	4187	+ cpu = vpe_to_cpuid_lock(vpe, &flags);
	4188	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4189	+ rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
	4190	+ gic_write_lpir(val, rdbase + GICR_INVALLR);
	4191	+
	4192	+ wait_for_syncr(rdbase);
	4193	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4194	+ vpe_to_cpuid_unlock(vpe, flags);
	4195	+}
	4196	+
	4197	+static int its_vpe_4_1_set_vcpu_affinity(struct irq_data d, void vcpu_info)
	4198	+{
	4199	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4200	+ struct its_cmd_info *info = vcpu_info;
	4201	+
	4202	+ switch (info->cmd_type) {
	4203	+ case SCHEDULE_VPE:
	4204	+ its_vpe_4_1_schedule(vpe, info);
	4205	+ return 0;
	4206	+
	4207	+ case DESCHEDULE_VPE:
	4208	+ its_vpe_4_1_deschedule(vpe, info);
	4209	+ return 0;
	4210	+
	4211	+ case COMMIT_VPE:
	4212	+ its_wait_vpt_parse_complete();
	4213	+ return 0;
	4214	+
	4215	+ case INVALL_VPE:
	4216	+ its_vpe_4_1_invall(vpe);
	4217	+ return 0;
	4218	+
	4219	+ default:
	4220	+ return -EINVAL;
	4221	+ }
	4222	+}
	4223	+
	4224	+static struct irq_chip its_vpe_4_1_irq_chip = {
	4225	+ .name = "GICv4.1-vpe",
	4226	+ .irq_mask = its_vpe_4_1_mask_irq,
	4227	+ .irq_unmask = its_vpe_4_1_unmask_irq,
	4228	+ .irq_eoi = irq_chip_eoi_parent,
	4229	+ .irq_set_affinity = its_vpe_set_affinity,
	4230	+ .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
	4231	+};
	4232	+
	4233	+static void its_configure_sgi(struct irq_data *d, bool clear)
	4234	+{
	4235	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4236	+ struct its_cmd_desc desc;
	4237	+
	4238	+ desc.its_vsgi_cmd.vpe = vpe;
	4239	+ desc.its_vsgi_cmd.sgi = d->hwirq;
	4240	+ desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
	4241	+ desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
	4242	+ desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
	4243	+ desc.its_vsgi_cmd.clear = clear;
	4244	+
	4245	+ /*
	4246	+ * GICv4.1 allows us to send VSGI commands to any ITS as long as the
	4247	+ * destination VPE is mapped there. Since we map them eagerly at
	4248	+ * activation time, we're pretty sure the first GICv4.1 ITS will do.
	4249	+ */
	4250	+ its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
	4251	+}
	4252	+
	4253	+static void its_sgi_mask_irq(struct irq_data *d)
	4254	+{
	4255	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4256	+
	4257	+ vpe->sgi_config[d->hwirq].enabled = false;
	4258	+ its_configure_sgi(d, false);
	4259	+}
	4260	+
	4261	+static void its_sgi_unmask_irq(struct irq_data *d)
	4262	+{
	4263	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4264	+
	4265	+ vpe->sgi_config[d->hwirq].enabled = true;
	4266	+ its_configure_sgi(d, false);
	4267	+}
	4268	+
	4269	+static int its_sgi_set_affinity(struct irq_data *d,
	4270	+ const struct cpumask *mask_val,
	4271	+ bool force)
	4272	+{
	4273	+ /*
	4274	+ * There is no notion of affinity for virtual SGIs, at least
	4275	+ * not on the host (since they can only be targetting a vPE).
	4276	+ * Tell the kernel we've done whatever it asked for.
	4277	+ */
	4278	+ irq_data_update_effective_affinity(d, mask_val);
	4279	+ return IRQ_SET_MASK_OK;
	4280	+}
	4281	+
	4282	+static int its_sgi_set_irqchip_state(struct irq_data *d,
	4283	+ enum irqchip_irq_state which,
	4284	+ bool state)
	4285	+{
	4286	+ if (which != IRQCHIP_STATE_PENDING)
	4287	+ return -EINVAL;
	4288	+
	4289	+ if (state) {
	4290	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4291	+ struct its_node *its = find_4_1_its();
	4292	+ u64 val;
	4293	+
	4294	+ val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
	4295	+ val \|= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
	4296	+ writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
	4297	+ } else {
	4298	+ its_configure_sgi(d, true);
	4299	+ }
	4300	+
	4301	+ return 0;
	4302	+}
	4303	+
	4304	+static int its_sgi_get_irqchip_state(struct irq_data *d,
	4305	+ enum irqchip_irq_state which, bool *val)
	4306	+{
	4307	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4308	+ void __iomem *base;
	4309	+ unsigned long flags;
	4310	+ u32 count = 1000000; /* 1s! */
	4311	+ u32 status;
	4312	+ int cpu;
	4313	+
	4314	+ if (which != IRQCHIP_STATE_PENDING)
	4315	+ return -EINVAL;
	4316	+
	4317	+ /*
	4318	+ * Locking galore! We can race against two different events:
	4319	+ *
	4320	+ * - Concurent vPE affinity change: we must make sure it cannot
	4321	+ * happen, or we'll talk to the wrong redistributor. This is
	4322	+ * identical to what happens with vLPIs.
	4323	+ *
	4324	+ * - Concurrent VSGIPENDR access: As it involves accessing two
	4325	+ * MMIO registers, this must be made atomic one way or another.
	4326	+ */
	4327	+ cpu = vpe_to_cpuid_lock(vpe, &flags);
	4328	+ raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4329	+ base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
	4330	+ writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
	4331	+ do {
	4332	+ status = readl_relaxed(base + GICR_VSGIPENDR);
	4333	+ if (!(status & GICR_VSGIPENDR_BUSY))
	4334	+ goto out;
	4335	+
	4336	+ count--;
	4337	+ if (!count) {
	4338	+ pr_err_ratelimited("Unable to get SGI status\n");
	4339	+ goto out;
	4340	+ }
	4341	+ cpu_relax();
	4342	+ udelay(1);
	4343	+ } while (count);
	4344	+
	4345	+out:
	4346	+ raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
	4347	+ vpe_to_cpuid_unlock(vpe, flags);
	4348	+
	4349	+ if (!count)
	4350	+ return -ENXIO;
	4351	+
	4352	+ *val = !!(status & (1 << d->hwirq));
	4353	+
	4354	+ return 0;
	4355	+}
	4356	+
	4357	+static int its_sgi_set_vcpu_affinity(struct irq_data d, void vcpu_info)
	4358	+{
	4359	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4360	+ struct its_cmd_info *info = vcpu_info;
	4361	+
	4362	+ switch (info->cmd_type) {
	4363	+ case PROP_UPDATE_VSGI:
	4364	+ vpe->sgi_config[d->hwirq].priority = info->priority;
	4365	+ vpe->sgi_config[d->hwirq].group = info->group;
	4366	+ its_configure_sgi(d, false);
	4367	+ return 0;
	4368	+
	4369	+ default:
	4370	+ return -EINVAL;
	4371	+ }
	4372	+}
	4373	+
	4374	+static struct irq_chip its_sgi_irq_chip = {
	4375	+ .name = "GICv4.1-sgi",
	4376	+ .irq_mask = its_sgi_mask_irq,
	4377	+ .irq_unmask = its_sgi_unmask_irq,
	4378	+ .irq_set_affinity = its_sgi_set_affinity,
	4379	+ .irq_set_irqchip_state = its_sgi_set_irqchip_state,
	4380	+ .irq_get_irqchip_state = its_sgi_get_irqchip_state,
	4381	+ .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
	4382	+};
	4383	+
	4384	+static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
	4385	+ unsigned int virq, unsigned int nr_irqs,
	4386	+ void *args)
	4387	+{
	4388	+ struct its_vpe *vpe = args;
	4389	+ int i;
	4390	+
	4391	+ /* Yes, we do want 16 SGIs */
	4392	+ WARN_ON(nr_irqs != 16);
	4393	+
	4394	+ for (i = 0; i < 16; i++) {
	4395	+ vpe->sgi_config[i].priority = 0;
	4396	+ vpe->sgi_config[i].enabled = false;
	4397	+ vpe->sgi_config[i].group = false;
	4398	+
	4399	+ irq_domain_set_hwirq_and_chip(domain, virq + i, i,
	4400	+ &its_sgi_irq_chip, vpe);
	4401	+ irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
	4402	+ }
	4403	+
	4404	+ return 0;
	4405	+}
	4406	+
	4407	+static void its_sgi_irq_domain_free(struct irq_domain *domain,
	4408	+ unsigned int virq,
	4409	+ unsigned int nr_irqs)
	4410	+{
	4411	+ /* Nothing to do */
	4412	+}
	4413	+
	4414	+static int its_sgi_irq_domain_activate(struct irq_domain *domain,
	4415	+ struct irq_data *d, bool reserve)
	4416	+{
	4417	+ /* Write out the initial SGI configuration */
	4418	+ its_configure_sgi(d, false);
	4419	+ return 0;
	4420	+}
	4421	+
	4422	+static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
	4423	+ struct irq_data *d)
	4424	+{
	4425	+ struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
	4426	+
	4427	+ /*
	4428	+ * The VSGI command is awkward:
	4429	+ *
	4430	+ * - To change the configuration, CLEAR must be set to false,
	4431	+ * leaving the pending bit unchanged.
	4432	+ * - To clear the pending bit, CLEAR must be set to true, leaving
	4433	+ * the configuration unchanged.
	4434	+ *
	4435	+ * You just can't do both at once, hence the two commands below.
	4436	+ */
	4437	+ vpe->sgi_config[d->hwirq].enabled = false;
	4438	+ its_configure_sgi(d, false);
	4439	+ its_configure_sgi(d, true);
	4440	+}
	4441	+
	4442	+static const struct irq_domain_ops its_sgi_domain_ops = {
	4443	+ .alloc = its_sgi_irq_domain_alloc,
	4444	+ .free = its_sgi_irq_domain_free,
	4445	+ .activate = its_sgi_irq_domain_activate,
	4446	+ .deactivate = its_sgi_irq_domain_deactivate,
2909	4447	};
2910	4448
2911	4449	static int its_vpe_id_alloc(void)
..	..	@@ -2941,9 +4479,13 @@
2941	4479	return -ENOMEM;
2942	4480	}
2943	4481
	4482	+ raw_spin_lock_init(&vpe->vpe_lock);
2944	4483	vpe->vpe_id = vpe_id;
2945	4484	vpe->vpt_page = vpt_page;
2946		- vpe->vpe_proxy_event = -1;
	4485	+ if (gic_rdists->has_rvpeid)
	4486	+ atomic_set(&vpe->vmapp_count, 0);
	4487	+ else
	4488	+ vpe->vpe_proxy_event = -1;
2947	4489
2948	4490	return 0;
2949	4491	}
..	..	@@ -2985,6 +4527,7 @@
2985	4527	static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
2986	4528	unsigned int nr_irqs, void *args)
2987	4529	{
	4530	+ struct irq_chip *irqchip = &its_vpe_irq_chip;
2988	4531	struct its_vm *vm = args;
2989	4532	unsigned long *bitmap;
2990	4533	struct page *vprop_page;
..	..	@@ -3012,6 +4555,9 @@
3012	4555	vm->nr_db_lpis = nr_ids;
3013	4556	vm->vprop_page = vprop_page;
3014	4557
	4558	+ if (gic_rdists->has_rvpeid)
	4559	+ irqchip = &its_vpe_4_1_irq_chip;
	4560	+
3015	4561	for (i = 0; i < nr_irqs; i++) {
3016	4562	vm->vpes[i]->vpe_db_lpi = base + i;
3017	4563	err = its_vpe_init(vm->vpes[i]);
..	..	@@ -3022,7 +4568,7 @@
3022	4568	if (err)
3023	4569	break;
3024	4570	irq_domain_set_hwirq_and_chip(domain, virq + i, i,
3025		- &its_vpe_irq_chip, vm->vpes[i]);
	4571	+ irqchip, vm->vpes[i]);
3026	4572	set_bit(i, bitmap);
3027	4573	}
3028	4574
..	..	@@ -3043,15 +4589,19 @@
3043	4589	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3044	4590	struct its_node *its;
3045	4591
3046		- /* If we use the list map, we issue VMAPP on demand... */
3047		- if (its_list_map)
	4592	+ /*
	4593	+ * If we use the list map, we issue VMAPP on demand... Unless
	4594	+ * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
	4595	+ * so that VSGIs can work.
	4596	+ */
	4597	+ if (!gic_requires_eager_mapping())
3048	4598	return 0;
3049	4599
3050	4600	/* Map the VPE to the first possible CPU */
3051	4601	vpe->col_idx = cpumask_first(cpu_online_mask);
3052	4602
3053	4603	list_for_each_entry(its, &its_nodes, entry) {
3054		- if (!its->is_v4)
	4604	+ if (!is_v4(its))
3055	4605	continue;
3056	4606
3057	4607	its_send_vmapp(its, vpe, true);
..	..	@@ -3070,14 +4620,14 @@
3070	4620	struct its_node *its;
3071	4621
3072	4622	/*
3073		- * If we use the list map, we unmap the VPE once no VLPIs are
3074		- * associated with the VM.
	4623	+ * If we use the list map on GICv4.0, we unmap the VPE once no
	4624	+ * VLPIs are associated with the VM.
3075	4625	*/
3076		- if (its_list_map)
	4626	+ if (!gic_requires_eager_mapping())
3077	4627	return;
3078	4628
3079	4629	list_for_each_entry(its, &its_nodes, entry) {
3080		- if (!its->is_v4)
	4630	+ if (!is_v4(its))
3081	4631	continue;
3082	4632
3083	4633	its_send_vmapp(its, vpe, false);
..	..	@@ -3128,8 +4678,9 @@
3128	4678	{
3129	4679	struct its_node *its = data;
3130	4680
3131		- /* erratum 22375: only alloc 8MB table size */
3132		- its->device_ids = 0x14; /* 20 bits, 8MB */
	4681	+ /* erratum 22375: only alloc 8MB table size (20 bits) */
	4682	+ its->typer &= ~GITS_TYPER_DEVBITS;
	4683	+ its->typer \|= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
3133	4684	its->flags \|= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
3134	4685
3135	4686	return true;
..	..	@@ -3149,7 +4700,8 @@
3149	4700	struct its_node *its = data;
3150	4701
3151	4702	/* On QDF2400, the size of the ITE is 16Bytes */
3152		- its->ite_size = 16;
	4703	+ its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
	4704	+ its->typer \|= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
3153	4705
3154	4706	return true;
3155	4707	}
..	..	@@ -3183,8 +4735,10 @@
3183	4735	its->get_msi_base = its_irq_get_msi_base_pre_its;
3184	4736
3185	4737	ids = ilog2(pre_its_window[1]) - 2;
3186		- if (its->device_ids > ids)
3187		- its->device_ids = ids;
	4738	+ if (device_ids(its) > ids) {
	4739	+ its->typer &= ~GITS_TYPER_DEVBITS;
	4740	+ its->typer \|= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
	4741	+ }
3188	4742
3189	4743	/* the pre-ITS breaks isolation, so disable MSI remapping */
3190	4744	its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
..	..	@@ -3411,7 +4965,7 @@
3411	4965	}
3412	4966
3413	4967	/* Use the last possible DevID */
3414		- devid = GENMASK(its->device_ids - 1, 0);
	4968	+ devid = GENMASK(device_ids(its) - 1, 0);
3415	4969	vpe_proxy.dev = its_create_device(its, devid, entries, false);
3416	4970	if (!vpe_proxy.dev) {
3417	4971	kfree(vpe_proxy.vpes);
..	..	@@ -3474,10 +5028,11 @@
3474	5028	void __iomem *its_base;
3475	5029	u32 val, ctlr;
3476	5030	u64 baser, tmp, typer;
	5031	+ struct page *page;
3477	5032	int err;
3478	5033	gfp_t gfp_flags;
3479	5034
3480		- its_base = ioremap(res->start, resource_size(res));
	5035	+ its_base = ioremap(res->start, SZ_64K);
3481	5036	if (!its_base) {
3482	5037	pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
3483	5038	return -ENOMEM;
..	..	@@ -3509,12 +5064,10 @@
3509	5064	INIT_LIST_HEAD(&its->entry);
3510	5065	INIT_LIST_HEAD(&its->its_device_list);
3511	5066	typer = gic_read_typer(its_base + GITS_TYPER);
	5067	+ its->typer = typer;
3512	5068	its->base = its_base;
3513	5069	its->phys_base = res->start;
3514		- its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
3515		- its->device_ids = GITS_TYPER_DEVBITS(typer);
3516		- its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
3517		- if (its->is_v4) {
	5070	+ if (is_v4(its)) {
3518	5071	if (!(typer & GITS_TYPER_VMOVP)) {
3519	5072	err = its_compute_its_list_map(res, its_base);
3520	5073	if (err < 0)
..	..	@@ -3527,19 +5080,37 @@
3527	5080	} else {
3528	5081	pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
3529	5082	}
	5083	+
	5084	+ if (is_v4_1(its)) {
	5085	+ u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
	5086	+
	5087	+ its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
	5088	+ if (!its->sgir_base) {
	5089	+ err = -ENOMEM;
	5090	+ goto out_free_its;
	5091	+ }
	5092	+
	5093	+ its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
	5094	+
	5095	+ pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
	5096	+ &res->start, its->mpidr, svpet);
	5097	+ }
3530	5098	}
3531	5099
3532	5100	its->numa_node = numa_node;
3533	5101
3534	5102	gfp_flags = GFP_KERNEL \| __GFP_ZERO;
3535		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	5103	+ if (of_machine_is_compatible("rockchip,rk3568") \|\|
	5104	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	5105	+ of_machine_is_compatible("rockchip,rk3566"))
3536	5106	gfp_flags \|= GFP_DMA32;
3537		- its->cmd_base = (void *)__get_free_pages(gfp_flags,
3538		- get_order(ITS_CMD_QUEUE_SZ));
3539		- if (!its->cmd_base) {
	5107	+ page = alloc_pages_node(its->numa_node, gfp_flags,
	5108	+ get_order(ITS_CMD_QUEUE_SZ));
	5109	+ if (!page) {
3540	5110	err = -ENOMEM;
3541		- goto out_free_its;
	5111	+ goto out_unmap_sgir;
3542	5112	}
	5113	+ its->cmd_base = (void *)page_address(page);
3543	5114	its->cmd_write = its->cmd_base;
3544	5115	its->fwnode_handle = handle;
3545	5116	its->get_msi_base = its_irq_get_msi_base;
..	..	@@ -3564,7 +5135,11 @@
3564	5135	gits_write_cbaser(baser, its->base + GITS_CBASER);
3565	5136	tmp = gits_read_cbaser(its->base + GITS_CBASER);
3566	5137
3567		- if (of_machine_is_compatible("rockchip,rk3568") \|\| of_machine_is_compatible("rockchip,rk3566"))
	5138	+ if (IS_ENABLED(CONFIG_NO_GKI) &&
	5139	+ (of_machine_is_compatible("rockchip,rk3568") \|\|
	5140	+ of_machine_is_compatible("rockchip,rk3567") \|\|
	5141	+ of_machine_is_compatible("rockchip,rk3566") \|\|
	5142	+ of_machine_is_compatible("rockchip,rk3588")))
3568	5143	tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
3569	5144
3570	5145	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
..	..	@@ -3586,7 +5161,7 @@
3586	5161	gits_write_cwriter(0, its->base + GITS_CWRITER);
3587	5162	ctlr = readl_relaxed(its->base + GITS_CTLR);
3588	5163	ctlr \|= GITS_CTLR_ENABLE;
3589		- if (its->is_v4)
	5164	+ if (is_v4(its))
3590	5165	ctlr \|= GITS_CTLR_ImDe;
3591	5166	writel_relaxed(ctlr, its->base + GITS_CTLR);
3592	5167
..	..	@@ -3604,6 +5179,9 @@
3604	5179	its_free_tables(its);
3605	5180	out_free_cmd:
3606	5181	free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
	5182	+out_unmap_sgir:
	5183	+ if (its->sgir_base)
	5184	+ iounmap(its->sgir_base);
3607	5185	out_free_its:
3608	5186	kfree(its);
3609	5187	out_unmap:
..	..	@@ -3623,16 +5201,6 @@
3623	5201	u64 timeout = USEC_PER_SEC;
3624	5202	u64 val;
3625	5203
3626		- /*
3627		- * If coming via a CPU hotplug event, we don't need to disable
3628		- * LPIs before trying to re-enable them. They are already
3629		- * configured and all is well in the world. Detect this case
3630		- * by checking the allocation of the pending table for the
3631		- * current CPU.
3632		- */
3633		- if (gic_data_rdist()->pend_page)
3634		- return 0;
3635		-
3636	5204	if (!gic_rdists_supports_plpis()) {
3637	5205	pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
3638	5206	return -ENXIO;
..	..	@@ -3642,7 +5210,21 @@
3642	5210	if (!(val & GICR_CTLR_ENABLE_LPIS))
3643	5211	return 0;
3644	5212
3645		- pr_warn("CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
	5213	+ /*
	5214	+ * If coming via a CPU hotplug event, we don't need to disable
	5215	+ * LPIs before trying to re-enable them. They are already
	5216	+ * configured and all is well in the world.
	5217	+ *
	5218	+ * If running with preallocated tables, there is nothing to do.
	5219	+ */
	5220	+ if (gic_data_rdist()->lpi_enabled \|\|
	5221	+ (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
	5222	+ return 0;
	5223	+
	5224	+ /*
	5225	+ * From that point on, we only try to do some damage control.
	5226	+ */
	5227	+ pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
3646	5228	smp_processor_id());
3647	5229	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3648	5230
..	..	@@ -3753,13 +5335,13 @@
3753	5335	return NUMA_NO_NODE;
3754	5336	}
3755	5337
3756		-static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header,
	5338	+static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
3757	5339	const unsigned long end)
3758	5340	{
3759	5341	return 0;
3760	5342	}
3761	5343
3762		-static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
	5344	+static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
3763	5345	const unsigned long end)
3764	5346	{
3765	5347	int node;
..	..	@@ -3775,7 +5357,12 @@
3775	5357	return -EINVAL;
3776	5358	}
3777	5359
3778		- node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
	5360	+ /*
	5361	+ * Note that in theory a new proximity node could be created by this
	5362	+ * entry as it is an SRAT resource allocation structure.
	5363	+ * We do not currently support doing so.
	5364	+ */
	5365	+ node = pxm_to_node(its_affinity->proximity_domain);
3779	5366
3780	5367	if (node == NUMA_NO_NODE \|\| node >= MAX_NUMNODES) {
3781	5368	pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
..	..	@@ -3826,7 +5413,7 @@
3826	5413	static void __init acpi_its_srat_maps_free(void) { }
3827	5414	#endif
3828	5415
3829		-static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
	5416	+static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
3830	5417	const unsigned long end)
3831	5418	{
3832	5419	struct acpi_madt_generic_translator *its_entry;
..	..	@@ -3840,7 +5427,7 @@
3840	5427	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
3841	5428	res.flags = IORESOURCE_MEM;
3842	5429
3843		- dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address);
	5430	+ dom_handle = irq_domain_alloc_fwnode(&res.start);
3844	5431	if (!dom_handle) {
3845	5432	pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
3846	5433	&res.start);
..	..	@@ -3883,7 +5470,10 @@
3883	5470	struct device_node *of_node;
3884	5471	struct its_node *its;
3885	5472	bool has_v4 = false;
	5473	+ bool has_v4_1 = false;
3886	5474	int err;
	5475	+
	5476	+ gic_rdists = rdists;
3887	5477
3888	5478	its_parent = parent_domain;
3889	5479	of_node = to_of_node(handle);
..	..	@@ -3897,17 +5487,29 @@
3897	5487	return -ENXIO;
3898	5488	}
3899	5489
3900		- gic_rdists = rdists;
3901		- err = its_alloc_lpi_tables();
	5490	+ err = allocate_lpi_tables();
3902	5491	if (err)
3903	5492	return err;
3904	5493
3905		- list_for_each_entry(its, &its_nodes, entry)
3906		- has_v4 \|= its->is_v4;
	5494	+ list_for_each_entry(its, &its_nodes, entry) {
	5495	+ has_v4 \|= is_v4(its);
	5496	+ has_v4_1 \|= is_v4_1(its);
	5497	+ }
	5498	+
	5499	+ /* Don't bother with inconsistent systems */
	5500	+ if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
	5501	+ rdists->has_rvpeid = false;
3907	5502
3908	5503	if (has_v4 & rdists->has_vlpis) {
	5504	+ const struct irq_domain_ops *sgi_ops;
	5505	+
	5506	+ if (has_v4_1)
	5507	+ sgi_ops = &its_sgi_domain_ops;
	5508	+ else
	5509	+ sgi_ops = NULL;
	5510	+
3909	5511	if (its_init_vpe_domain() \|\|
3910		- its_init_v4(parent_domain, &its_vpe_domain_ops)) {
	5512	+ its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
3911	5513	rdists->has_vlpis = false;
3912	5514	pr_err("ITS: Disabling GICv4 support\n");
3913	5515	}