~hc/RK356X_SDK_RELEASE.git

..	..	@@ -24,11 +24,10 @@
24	24	#include <linux/err.h>
25	25	#include <linux/export.h>
26	26	#include <linux/delay.h>
27		-#include <linux/irq.h>
28		-#include <linux/kernel_stat.h>
29	27	#include <linux/seq_file.h>
	28	+#include <linux/jump_label.h>
30	29	#include <linux/pci.h>
31		-#include <linux/msi.h>
	30	+#include <linux/printk.h>
32	31
33	32	#include <asm/isc.h>
34	33	#include <asm/airq.h>
..	..	@@ -37,38 +36,28 @@
37	36	#include <asm/pci_clp.h>
38	37	#include <asm/pci_dma.h>
39	38
40		-#define DEBUG /* enable pr_debug */
41		-
42		-#define SIC_IRQ_MODE_ALL 0
43		-#define SIC_IRQ_MODE_SINGLE 1
44		-
45		-#define ZPCI_NR_DMA_SPACES 1
46		-#define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS
	39	+#include "pci_bus.h"
	40	+#include "pci_iov.h"
47	41
48	42	/* list of all detected zpci devices */
49	43	static LIST_HEAD(zpci_list);
50	44	static DEFINE_SPINLOCK(zpci_list_lock);
51	45
52		-static struct irq_chip zpci_irq_chip = {
53		- .name = "zPCI",
54		- .irq_unmask = pci_msi_unmask_irq,
55		- .irq_mask = pci_msi_mask_irq,
56		-};
57		-
58		-static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
	46	+static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE);
59	47	static DEFINE_SPINLOCK(zpci_domain_lock);
60	48
61		-static struct airq_iv *zpci_aisb_iv;
62		-static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
63		-
64	49	#define ZPCI_IOMAP_ENTRIES \
65		- min(((unsigned long) ZPCI_NR_DEVICES * PCI_BAR_COUNT / 2), \
	50	+ min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2), \
66	51	ZPCI_IOMAP_MAX_ENTRIES)
	52	+
	53	+unsigned int s390_pci_no_rid;
67	54
68	55	static DEFINE_SPINLOCK(zpci_iomap_lock);
69	56	static unsigned long *zpci_iomap_bitmap;
70	57	struct zpci_iomap_entry *zpci_iomap_start;
71	58	EXPORT_SYMBOL_GPL(zpci_iomap_start);
	59	+
	60	+DEFINE_STATIC_KEY_FALSE(have_mio);
72	61
73	62	static struct kmem_cache *zdev_fmb_cache;
74	63
..	..	@@ -80,6 +69,7 @@
80	69	list_for_each_entry(tmp, &zpci_list, entry) {
81	70	if (tmp->fid == fid) {
82	71	zdev = tmp;
	72	+ zpci_zdev_get(zdev);
83	73	break;
84	74	}
85	75	}
..	..	@@ -103,17 +93,12 @@
103	93	spin_unlock(&zpci_list_lock);
104	94
105	95	list_for_each_entry_safe(zdev, tmp, &remove, entry)
106		- zpci_remove_device(zdev);
107		-}
108		-
109		-static struct zpci_dev get_zdev_by_bus(struct pci_bus bus)
110		-{
111		- return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
	96	+ zpci_device_reserved(zdev);
112	97	}
113	98
114	99	int pci_domain_nr(struct pci_bus *bus)
115	100	{
116		- return ((struct zpci_dev *) bus->sysdata)->domain;
	101	+ return ((struct zpci_bus *) bus->sysdata)->domain_nr;
117	102	}
118	103	EXPORT_SYMBOL_GPL(pci_domain_nr);
119	104
..	..	@@ -122,39 +107,6 @@
122	107	return pci_domain_nr(bus);
123	108	}
124	109	EXPORT_SYMBOL_GPL(pci_proc_domain);
125		-
126		-/* Modify PCI: Register adapter interruptions */
127		-static int zpci_set_airq(struct zpci_dev *zdev)
128		-{
129		- u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
130		- struct zpci_fib fib = {0};
131		- u8 status;
132		-
133		- fib.isc = PCI_ISC;
134		- fib.sum = 1; /* enable summary notifications */
135		- fib.noi = airq_iv_end(zdev->aibv);
136		- fib.aibv = (unsigned long) zdev->aibv->vector;
137		- fib.aibvo = 0; /* each zdev has its own interrupt vector */
138		- fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
139		- fib.aisbo = zdev->aisb & 63;
140		-
141		- return zpci_mod_fc(req, &fib, &status) ? -EIO : 0;
142		-}
143		-
144		-/* Modify PCI: Unregister adapter interruptions */
145		-static int zpci_clear_airq(struct zpci_dev *zdev)
146		-{
147		- u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT);
148		- struct zpci_fib fib = {0};
149		- u8 cc, status;
150		-
151		- cc = zpci_mod_fc(req, &fib, &status);
152		- if (cc == 3 \|\| (cc == 1 && status == 24))
153		- /* Function already gone or IRQs already deregistered. */
154		- cc = 0;
155		-
156		- return cc ? -EIO : 0;
157		-}
158	110
159	111	/* Modify PCI: Register I/O address translation parameters */
160	112	int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
..	..	@@ -241,7 +193,7 @@
241	193	u64 data;
242	194	int rc;
243	195
244		- rc = zpci_load(&data, req, offset);
	196	+ rc = __zpci_load(&data, req, offset);
245	197	if (!rc) {
246	198	data = le64_to_cpu((__force __le64) data);
247	199	data >>= (8 - len) * 8;
..	..	@@ -259,7 +211,7 @@
259	211
260	212	data <<= (8 - len) * 8;
261	213	data = (__force u64) cpu_to_le64(data);
262		- rc = zpci_store(data, req, offset);
	214	+ rc = __zpci_store(data, req, offset);
263	215	return rc;
264	216	}
265	217
..	..	@@ -276,17 +228,71 @@
276	228	zpci_memcpy_toio(to, from, count);
277	229	}
278	230
	231	+static void __iomem *__ioremap(phys_addr_t addr, size_t size, pgprot_t prot)
	232	+{
	233	+ unsigned long offset, vaddr;
	234	+ struct vm_struct *area;
	235	+ phys_addr_t last_addr;
	236	+
	237	+ last_addr = addr + size - 1;
	238	+ if (!size \|\| last_addr < addr)
	239	+ return NULL;
	240	+
	241	+ if (!static_branch_unlikely(&have_mio))
	242	+ return (void __iomem *) addr;
	243	+
	244	+ offset = addr & ~PAGE_MASK;
	245	+ addr &= PAGE_MASK;
	246	+ size = PAGE_ALIGN(size + offset);
	247	+ area = get_vm_area(size, VM_IOREMAP);
	248	+ if (!area)
	249	+ return NULL;
	250	+
	251	+ vaddr = (unsigned long) area->addr;
	252	+ if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) {
	253	+ free_vm_area(area);
	254	+ return NULL;
	255	+ }
	256	+ return (void __iomem *) ((unsigned long) area->addr + offset);
	257	+}
	258	+
	259	+void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot)
	260	+{
	261	+ return __ioremap(addr, size, __pgprot(prot));
	262	+}
	263	+EXPORT_SYMBOL(ioremap_prot);
	264	+
	265	+void __iomem *ioremap(phys_addr_t addr, size_t size)
	266	+{
	267	+ return __ioremap(addr, size, PAGE_KERNEL);
	268	+}
	269	+EXPORT_SYMBOL(ioremap);
	270	+
	271	+void __iomem *ioremap_wc(phys_addr_t addr, size_t size)
	272	+{
	273	+ return __ioremap(addr, size, pgprot_writecombine(PAGE_KERNEL));
	274	+}
	275	+EXPORT_SYMBOL(ioremap_wc);
	276	+
	277	+void __iomem *ioremap_wt(phys_addr_t addr, size_t size)
	278	+{
	279	+ return __ioremap(addr, size, pgprot_writethrough(PAGE_KERNEL));
	280	+}
	281	+EXPORT_SYMBOL(ioremap_wt);
	282	+
	283	+void iounmap(volatile void __iomem *addr)
	284	+{
	285	+ if (static_branch_likely(&have_mio))
	286	+ vunmap((__force void *) ((unsigned long) addr & PAGE_MASK));
	287	+}
	288	+EXPORT_SYMBOL(iounmap);
	289	+
279	290	/* Create a virtual mapping cookie for a PCI BAR */
280		-void __iomem pci_iomap_range(struct pci_dev pdev,
281		- int bar,
282		- unsigned long offset,
283		- unsigned long max)
	291	+static void __iomem pci_iomap_range_fh(struct pci_dev pdev, int bar,
	292	+ unsigned long offset, unsigned long max)
284	293	{
285	294	struct zpci_dev *zdev = to_zpci(pdev);
286	295	int idx;
287		-
288		- if (!pci_resource_len(pdev, bar))
289		- return NULL;
290	296
291	297	idx = zdev->bars[bar].map_idx;
292	298	spin_lock(&zpci_iomap_lock);
..	..	@@ -298,6 +304,30 @@
298	304
299	305	return (void __iomem *) ZPCI_ADDR(idx) + offset;
300	306	}
	307	+
	308	+static void __iomem pci_iomap_range_mio(struct pci_dev pdev, int bar,
	309	+ unsigned long offset,
	310	+ unsigned long max)
	311	+{
	312	+ unsigned long barsize = pci_resource_len(pdev, bar);
	313	+ struct zpci_dev *zdev = to_zpci(pdev);
	314	+ void __iomem *iova;
	315	+
	316	+ iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
	317	+ return iova ? iova + offset : iova;
	318	+}
	319	+
	320	+void __iomem pci_iomap_range(struct pci_dev pdev, int bar,
	321	+ unsigned long offset, unsigned long max)
	322	+{
	323	+ if (bar >= PCI_STD_NUM_BARS \|\| !pci_resource_len(pdev, bar))
	324	+ return NULL;
	325	+
	326	+ if (static_branch_likely(&have_mio))
	327	+ return pci_iomap_range_mio(pdev, bar, offset, max);
	328	+ else
	329	+ return pci_iomap_range_fh(pdev, bar, offset, max);
	330	+}
301	331	EXPORT_SYMBOL(pci_iomap_range);
302	332
303	333	void __iomem pci_iomap(struct pci_dev dev, int bar, unsigned long maxlen)
..	..	@@ -306,7 +336,37 @@
306	336	}
307	337	EXPORT_SYMBOL(pci_iomap);
308	338
309		-void pci_iounmap(struct pci_dev pdev, void __iomem addr)
	339	+static void __iomem pci_iomap_wc_range_mio(struct pci_dev pdev, int bar,
	340	+ unsigned long offset, unsigned long max)
	341	+{
	342	+ unsigned long barsize = pci_resource_len(pdev, bar);
	343	+ struct zpci_dev *zdev = to_zpci(pdev);
	344	+ void __iomem *iova;
	345	+
	346	+ iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
	347	+ return iova ? iova + offset : iova;
	348	+}
	349	+
	350	+void __iomem pci_iomap_wc_range(struct pci_dev pdev, int bar,
	351	+ unsigned long offset, unsigned long max)
	352	+{
	353	+ if (bar >= PCI_STD_NUM_BARS \|\| !pci_resource_len(pdev, bar))
	354	+ return NULL;
	355	+
	356	+ if (static_branch_likely(&have_mio))
	357	+ return pci_iomap_wc_range_mio(pdev, bar, offset, max);
	358	+ else
	359	+ return pci_iomap_range_fh(pdev, bar, offset, max);
	360	+}
	361	+EXPORT_SYMBOL(pci_iomap_wc_range);
	362	+
	363	+void __iomem pci_iomap_wc(struct pci_dev dev, int bar, unsigned long maxlen)
	364	+{
	365	+ return pci_iomap_wc_range(dev, bar, 0, maxlen);
	366	+}
	367	+EXPORT_SYMBOL(pci_iomap_wc);
	368	+
	369	+static void pci_iounmap_fh(struct pci_dev pdev, void __iomem addr)
310	370	{
311	371	unsigned int idx = ZPCI_IDX(addr);
312	372
..	..	@@ -319,34 +379,35 @@
319	379	}
320	380	spin_unlock(&zpci_iomap_lock);
321	381	}
	382	+
	383	+static void pci_iounmap_mio(struct pci_dev pdev, void __iomem addr)
	384	+{
	385	+ iounmap(addr);
	386	+}
	387	+
	388	+void pci_iounmap(struct pci_dev pdev, void __iomem addr)
	389	+{
	390	+ if (static_branch_likely(&have_mio))
	391	+ pci_iounmap_mio(pdev, addr);
	392	+ else
	393	+ pci_iounmap_fh(pdev, addr);
	394	+}
322	395	EXPORT_SYMBOL(pci_iounmap);
323	396
324	397	static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
325	398	int size, u32 *val)
326	399	{
327		- struct zpci_dev *zdev = get_zdev_by_bus(bus);
328		- int ret;
	400	+ struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn);
329	401
330		- if (!zdev \|\| devfn != ZPCI_DEVFN)
331		- ret = -ENODEV;
332		- else
333		- ret = zpci_cfg_load(zdev, where, val, size);
334		-
335		- return ret;
	402	+ return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;
336	403	}
337	404
338	405	static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
339	406	int size, u32 val)
340	407	{
341		- struct zpci_dev *zdev = get_zdev_by_bus(bus);
342		- int ret;
	408	+ struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn);
343	409
344		- if (!zdev \|\| devfn != ZPCI_DEVFN)
345		- ret = -ENODEV;
346		- else
347		- ret = zpci_cfg_store(zdev, where, val, size);
348		-
349		- return ret;
	410	+ return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;
350	411	}
351	412
352	413	static struct pci_ops pci_root_ops = {
..	..	@@ -354,198 +415,45 @@
354	415	.write = pci_write,
355	416	};
356	417
357		-static void zpci_irq_handler(struct airq_struct *airq)
358		-{
359		- unsigned long si, ai;
360		- struct airq_iv *aibv;
361		- int irqs_on = 0;
362		-
363		- inc_irq_stat(IRQIO_PCI);
364		- for (si = 0;;) {
365		- /* Scan adapter summary indicator bit vector */
366		- si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
367		- if (si == -1UL) {
368		- if (irqs_on++)
369		- /* End of second scan with interrupts on. */
370		- break;
371		- /* First scan complete, reenable interrupts. */
372		- if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC))
373		- break;
374		- si = 0;
375		- continue;
376		- }
377		-
378		- /* Scan the adapter interrupt vector for this device. */
379		- aibv = zpci_aibv[si];
380		- for (ai = 0;;) {
381		- ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
382		- if (ai == -1UL)
383		- break;
384		- inc_irq_stat(IRQIO_MSI);
385		- airq_iv_lock(aibv, ai);
386		- generic_handle_irq(airq_iv_get_data(aibv, ai));
387		- airq_iv_unlock(aibv, ai);
388		- }
389		- }
390		-}
391		-
392		-int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
393		-{
394		- struct zpci_dev *zdev = to_zpci(pdev);
395		- unsigned int hwirq, msi_vecs;
396		- unsigned long aisb;
397		- struct msi_desc *msi;
398		- struct msi_msg msg;
399		- int rc, irq;
400		-
401		- zdev->aisb = -1UL;
402		- if (type == PCI_CAP_ID_MSI && nvec > 1)
403		- return 1;
404		- msi_vecs = min_t(unsigned int, nvec, zdev->max_msi);
405		-
406		- /* Allocate adapter summary indicator bit */
407		- aisb = airq_iv_alloc_bit(zpci_aisb_iv);
408		- if (aisb == -1UL)
409		- return -EIO;
410		- zdev->aisb = aisb;
411		-
412		- /* Create adapter interrupt vector */
413		- zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA \| AIRQ_IV_BITLOCK);
414		- if (!zdev->aibv)
415		- return -ENOMEM;
416		-
417		- /* Wire up shortcut pointer */
418		- zpci_aibv[aisb] = zdev->aibv;
419		-
420		- /* Request MSI interrupts */
421		- hwirq = 0;
422		- for_each_pci_msi_entry(msi, pdev) {
423		- if (hwirq >= msi_vecs)
424		- break;
425		- irq = irq_alloc_desc(0); /* Alloc irq on node 0 */
426		- if (irq < 0)
427		- return -ENOMEM;
428		- rc = irq_set_msi_desc(irq, msi);
429		- if (rc)
430		- return rc;
431		- irq_set_chip_and_handler(irq, &zpci_irq_chip,
432		- handle_simple_irq);
433		- msg.data = hwirq;
434		- msg.address_lo = zdev->msi_addr & 0xffffffff;
435		- msg.address_hi = zdev->msi_addr >> 32;
436		- pci_write_msi_msg(irq, &msg);
437		- airq_iv_set_data(zdev->aibv, hwirq, irq);
438		- hwirq++;
439		- }
440		-
441		- /* Enable adapter interrupts */
442		- rc = zpci_set_airq(zdev);
443		- if (rc)
444		- return rc;
445		-
446		- return (msi_vecs == nvec) ? 0 : msi_vecs;
447		-}
448		-
449		-void arch_teardown_msi_irqs(struct pci_dev *pdev)
450		-{
451		- struct zpci_dev *zdev = to_zpci(pdev);
452		- struct msi_desc *msi;
453		- int rc;
454		-
455		- /* Disable adapter interrupts */
456		- rc = zpci_clear_airq(zdev);
457		- if (rc)
458		- return;
459		-
460		- /* Release MSI interrupts */
461		- for_each_pci_msi_entry(msi, pdev) {
462		- if (!msi->irq)
463		- continue;
464		- if (msi->msi_attrib.is_msix)
465		- __pci_msix_desc_mask_irq(msi, 1);
466		- else
467		- __pci_msi_desc_mask_irq(msi, 1, 1);
468		- irq_set_msi_desc(msi->irq, NULL);
469		- irq_free_desc(msi->irq);
470		- msi->msg.address_lo = 0;
471		- msi->msg.address_hi = 0;
472		- msi->msg.data = 0;
473		- msi->irq = 0;
474		- }
475		-
476		- if (zdev->aisb != -1UL) {
477		- zpci_aibv[zdev->aisb] = NULL;
478		- airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
479		- zdev->aisb = -1UL;
480		- }
481		- if (zdev->aibv) {
482		- airq_iv_release(zdev->aibv);
483		- zdev->aibv = NULL;
484		- }
485		-}
486		-
487	418	static void zpci_map_resources(struct pci_dev *pdev)
488	419	{
	420	+ struct zpci_dev *zdev = to_zpci(pdev);
489	421	resource_size_t len;
490	422	int i;
491	423
492		- for (i = 0; i < PCI_BAR_COUNT; i++) {
	424	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
493	425	len = pci_resource_len(pdev, i);
494	426	if (!len)
495	427	continue;
496		- pdev->resource[i].start =
497		- (resource_size_t __force) pci_iomap(pdev, i, 0);
	428	+
	429	+ if (zpci_use_mio(zdev))
	430	+ pdev->resource[i].start =
	431	+ (resource_size_t __force) zdev->bars[i].mio_wt;
	432	+ else
	433	+ pdev->resource[i].start = (resource_size_t __force)
	434	+ pci_iomap_range_fh(pdev, i, 0, 0);
498	435	pdev->resource[i].end = pdev->resource[i].start + len - 1;
499	436	}
	437	+
	438	+ zpci_iov_map_resources(pdev);
500	439	}
501	440
502	441	static void zpci_unmap_resources(struct pci_dev *pdev)
503	442	{
	443	+ struct zpci_dev *zdev = to_zpci(pdev);
504	444	resource_size_t len;
505	445	int i;
506	446
507		- for (i = 0; i < PCI_BAR_COUNT; i++) {
	447	+ if (zpci_use_mio(zdev))
	448	+ return;
	449	+
	450	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
508	451	len = pci_resource_len(pdev, i);
509	452	if (!len)
510	453	continue;
511		- pci_iounmap(pdev, (void __iomem __force *)
512		- pdev->resource[i].start);
	454	+ pci_iounmap_fh(pdev, (void __iomem __force *)
	455	+ pdev->resource[i].start);
513	456	}
514		-}
515		-
516		-static struct airq_struct zpci_airq = {
517		- .handler = zpci_irq_handler,
518		- .isc = PCI_ISC,
519		-};
520		-
521		-static int __init zpci_irq_init(void)
522		-{
523		- int rc;
524		-
525		- rc = register_adapter_interrupt(&zpci_airq);
526		- if (rc)
527		- goto out;
528		- /* Set summary to 1 to be called every time for the ISC. */
529		- *zpci_airq.lsi_ptr = 1;
530		-
531		- rc = -ENOMEM;
532		- zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
533		- if (!zpci_aisb_iv)
534		- goto out_airq;
535		-
536		- zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
537		- return 0;
538		-
539		-out_airq:
540		- unregister_adapter_interrupt(&zpci_airq);
541		-out:
542		- return rc;
543		-}
544		-
545		-static void zpci_irq_exit(void)
546		-{
547		- airq_iv_release(zpci_aisb_iv);
548		- unregister_adapter_interrupt(&zpci_airq);
549	457	}
550	458
551	459	static int zpci_alloc_iomap(struct zpci_dev *zdev)
..	..	@@ -592,17 +500,17 @@
592	500	return r;
593	501	}
594	502
595		-static int zpci_setup_bus_resources(struct zpci_dev *zdev,
596		- struct list_head *resources)
	503	+int zpci_setup_bus_resources(struct zpci_dev *zdev,
	504	+ struct list_head *resources)
597	505	{
598	506	unsigned long addr, size, flags;
599	507	struct resource *res;
600	508	int i, entry;
601	509
602	510	snprintf(zdev->res_name, sizeof(zdev->res_name),
603		- "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);
	511	+ "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);
604	512
605		- for (i = 0; i < PCI_BAR_COUNT; i++) {
	513	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
606	514	if (!zdev->bars[i].size)
607	515	continue;
608	516	entry = zpci_alloc_iomap(zdev);
..	..	@@ -617,7 +525,10 @@
617	525	if (zdev->bars[i].val & 4)
618	526	flags \|= IORESOURCE_MEM_64;
619	527
620		- addr = ZPCI_ADDR(entry);
	528	+ if (zpci_use_mio(zdev))
	529	+ addr = (unsigned long) zdev->bars[i].mio_wt;
	530	+ else
	531	+ addr = ZPCI_ADDR(entry);
621	532	size = 1UL << zdev->bars[i].size;
622	533
623	534	res = __alloc_res(zdev, addr, size, flags);
..	..	@@ -636,7 +547,7 @@
636	547	{
637	548	int i;
638	549
639		- for (i = 0; i < PCI_BAR_COUNT; i++) {
	550	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
640	551	if (!zdev->bars[i].size \|\| !zdev->bars[i].res)
641	552	continue;
642	553
..	..	@@ -648,14 +559,20 @@
648	559
649	560	int pcibios_add_device(struct pci_dev *pdev)
650	561	{
	562	+ struct zpci_dev *zdev = to_zpci(pdev);
651	563	struct resource *res;
652	564	int i;
	565	+
	566	+ /* The pdev has a reference to the zdev via its bus */
	567	+ zpci_zdev_get(zdev);
	568	+ if (pdev->is_physfn)
	569	+ pdev->no_vf_scan = 1;
653	570
654	571	pdev->dev.groups = zpci_attr_groups;
655	572	pdev->dev.dma_ops = &s390_pci_dma_ops;
656	573	zpci_map_resources(pdev);
657	574
658		- for (i = 0; i < PCI_BAR_COUNT; i++) {
	575	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
659	576	res = &pdev->resource[i];
660	577	if (res->parent \|\| !res->flags)
661	578	continue;
..	..	@@ -667,7 +584,10 @@
667	584
668	585	void pcibios_release_device(struct pci_dev *pdev)
669	586	{
	587	+ struct zpci_dev *zdev = to_zpci(pdev);
	588	+
670	589	zpci_unmap_resources(pdev);
	590	+ zpci_zdev_put(zdev);
671	591	}
672	592
673	593	int pcibios_enable_device(struct pci_dev *pdev, int mask)
..	..	@@ -688,136 +608,62 @@
688	608	zpci_debug_exit_device(zdev);
689	609	}
690	610
691		-#ifdef CONFIG_HIBERNATE_CALLBACKS
692		-static int zpci_restore(struct device *dev)
	611	+static int __zpci_register_domain(int domain)
693	612	{
694		- struct pci_dev *pdev = to_pci_dev(dev);
695		- struct zpci_dev *zdev = to_zpci(pdev);
696		- int ret = 0;
697		-
698		- if (zdev->state != ZPCI_FN_STATE_ONLINE)
699		- goto out;
700		-
701		- ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
702		- if (ret)
703		- goto out;
704		-
705		- zpci_map_resources(pdev);
706		- zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
707		- (u64) zdev->dma_table);
708		-
709		-out:
710		- return ret;
	613	+ spin_lock(&zpci_domain_lock);
	614	+ if (test_bit(domain, zpci_domain)) {
	615	+ spin_unlock(&zpci_domain_lock);
	616	+ pr_err("Domain %04x is already assigned\n", domain);
	617	+ return -EEXIST;
	618	+ }
	619	+ set_bit(domain, zpci_domain);
	620	+ spin_unlock(&zpci_domain_lock);
	621	+ return domain;
711	622	}
712	623
713		-static int zpci_freeze(struct device *dev)
	624	+static int __zpci_alloc_domain(void)
714	625	{
715		- struct pci_dev *pdev = to_pci_dev(dev);
716		- struct zpci_dev *zdev = to_zpci(pdev);
	626	+ int domain;
717	627
718		- if (zdev->state != ZPCI_FN_STATE_ONLINE)
719		- return 0;
720		-
721		- zpci_unregister_ioat(zdev, 0);
722		- zpci_unmap_resources(pdev);
723		- return clp_disable_fh(zdev);
	628	+ spin_lock(&zpci_domain_lock);
	629	+ /*
	630	+ * We can always auto allocate domains below ZPCI_NR_DEVICES.
	631	+ * There is either a free domain or we have reached the maximum in
	632	+ * which case we would have bailed earlier.
	633	+ */
	634	+ domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
	635	+ set_bit(domain, zpci_domain);
	636	+ spin_unlock(&zpci_domain_lock);
	637	+ return domain;
724	638	}
725	639
726		-struct dev_pm_ops pcibios_pm_ops = {
727		- .thaw_noirq = zpci_restore,
728		- .freeze_noirq = zpci_freeze,
729		- .restore_noirq = zpci_restore,
730		- .poweroff_noirq = zpci_freeze,
731		-};
732		-#endif /* CONFIG_HIBERNATE_CALLBACKS */
733		-
734		-static int zpci_alloc_domain(struct zpci_dev *zdev)
	640	+int zpci_alloc_domain(int domain)
735	641	{
736	642	if (zpci_unique_uid) {
737		- zdev->domain = (u16) zdev->uid;
738		- if (zdev->domain >= ZPCI_NR_DEVICES)
739		- return 0;
740		-
741		- spin_lock(&zpci_domain_lock);
742		- if (test_bit(zdev->domain, zpci_domain)) {
743		- spin_unlock(&zpci_domain_lock);
744		- return -EEXIST;
745		- }
746		- set_bit(zdev->domain, zpci_domain);
747		- spin_unlock(&zpci_domain_lock);
748		- return 0;
	643	+ if (domain)
	644	+ return __zpci_register_domain(domain);
	645	+ pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n");
	646	+ update_uid_checking(false);
749	647	}
750		-
751		- spin_lock(&zpci_domain_lock);
752		- zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
753		- if (zdev->domain == ZPCI_NR_DEVICES) {
754		- spin_unlock(&zpci_domain_lock);
755		- return -ENOSPC;
756		- }
757		- set_bit(zdev->domain, zpci_domain);
758		- spin_unlock(&zpci_domain_lock);
759		- return 0;
	648	+ return __zpci_alloc_domain();
760	649	}
761	650
762		-static void zpci_free_domain(struct zpci_dev *zdev)
	651	+void zpci_free_domain(int domain)
763	652	{
764		- if (zdev->domain >= ZPCI_NR_DEVICES)
765		- return;
766		-
767	653	spin_lock(&zpci_domain_lock);
768		- clear_bit(zdev->domain, zpci_domain);
	654	+ clear_bit(domain, zpci_domain);
769	655	spin_unlock(&zpci_domain_lock);
770	656	}
771	657
772		-void pcibios_remove_bus(struct pci_bus *bus)
773		-{
774		- struct zpci_dev *zdev = get_zdev_by_bus(bus);
775		-
776		- zpci_exit_slot(zdev);
777		- zpci_cleanup_bus_resources(zdev);
778		- zpci_destroy_iommu(zdev);
779		- zpci_free_domain(zdev);
780		-
781		- spin_lock(&zpci_list_lock);
782		- list_del(&zdev->entry);
783		- spin_unlock(&zpci_list_lock);
784		-
785		- zpci_dbg(3, "rem fid:%x\n", zdev->fid);
786		- kfree(zdev);
787		-}
788		-
789		-static int zpci_scan_bus(struct zpci_dev *zdev)
790		-{
791		- LIST_HEAD(resources);
792		- int ret;
793		-
794		- ret = zpci_setup_bus_resources(zdev, &resources);
795		- if (ret)
796		- goto error;
797		-
798		- zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
799		- zdev, &resources);
800		- if (!zdev->bus) {
801		- ret = -EIO;
802		- goto error;
803		- }
804		- zdev->bus->max_bus_speed = zdev->max_bus_speed;
805		- pci_bus_add_devices(zdev->bus);
806		- return 0;
807		-
808		-error:
809		- zpci_cleanup_bus_resources(zdev);
810		- pci_free_resource_list(&resources);
811		- return ret;
812		-}
813	658
814	659	int zpci_enable_device(struct zpci_dev *zdev)
815	660	{
816	661	int rc;
817	662
818		- rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
819		- if (rc)
	663	+ if (clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES)) {
	664	+ rc = -EIO;
820	665	goto out;
	666	+ }
821	667
822	668	rc = zpci_dma_init_device(zdev);
823	669	if (rc)
..	..	@@ -836,58 +682,175 @@
836	682	int zpci_disable_device(struct zpci_dev *zdev)
837	683	{
838	684	zpci_dma_exit_device(zdev);
839		- return clp_disable_fh(zdev);
	685	+ /*
	686	+ * The zPCI function may already be disabled by the platform, this is
	687	+ * detected in clp_disable_fh() which becomes a no-op.
	688	+ */
	689	+ return clp_disable_fh(zdev) ? -EIO : 0;
840	690	}
841	691	EXPORT_SYMBOL_GPL(zpci_disable_device);
842	692
843		-int zpci_create_device(struct zpci_dev *zdev)
	693	+/* zpci_remove_device - Removes the given zdev from the PCI core
	694	+ * @zdev: the zdev to be removed from the PCI core
	695	+ * @set_error: if true the device's error state is set to permanent failure
	696	+ *
	697	+ * Sets a zPCI device to a configured but offline state; the zPCI
	698	+ * device is still accessible through its hotplug slot and the zPCI
	699	+ * API but is removed from the common code PCI bus, making it
	700	+ * no longer available to drivers.
	701	+ */
	702	+void zpci_remove_device(struct zpci_dev *zdev, bool set_error)
844	703	{
	704	+ struct zpci_bus *zbus = zdev->zbus;
	705	+ struct pci_dev *pdev;
	706	+
	707	+ if (!zdev->zbus->bus)
	708	+ return;
	709	+
	710	+ pdev = pci_get_slot(zbus->bus, zdev->devfn);
	711	+ if (pdev) {
	712	+ if (set_error)
	713	+ pdev->error_state = pci_channel_io_perm_failure;
	714	+ if (pdev->is_virtfn) {
	715	+ zpci_iov_remove_virtfn(pdev, zdev->vfn);
	716	+ /* balance pci_get_slot */
	717	+ pci_dev_put(pdev);
	718	+ return;
	719	+ }
	720	+ pci_stop_and_remove_bus_device_locked(pdev);
	721	+ /* balance pci_get_slot */
	722	+ pci_dev_put(pdev);
	723	+ }
	724	+}
	725	+
	726	+/**
	727	+ * zpci_create_device() - Create a new zpci_dev and add it to the zbus
	728	+ * @fid: Function ID of the device to be created
	729	+ * @fh: Current Function Handle of the device to be created
	730	+ * @state: Initial state after creation either Standby or Configured
	731	+ *
	732	+ * Creates a new zpci device and adds it to its, possibly newly created, zbus
	733	+ * as well as zpci_list.
	734	+ *
	735	+ * Returns: 0 on success, an error value otherwise
	736	+ */
	737	+int zpci_create_device(u32 fid, u32 fh, enum zpci_state state)
	738	+{
	739	+ struct zpci_dev *zdev;
845	740	int rc;
846	741
847		- rc = zpci_alloc_domain(zdev);
	742	+ zpci_dbg(3, "add fid:%x, fh:%x, c:%d\n", fid, fh, state);
	743	+ zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);
	744	+ if (!zdev)
	745	+ return -ENOMEM;
	746	+
	747	+ /* FID and Function Handle are the static/dynamic identifiers */
	748	+ zdev->fid = fid;
	749	+ zdev->fh = fh;
	750	+
	751	+ /* Query function properties and update zdev */
	752	+ rc = clp_query_pci_fn(zdev);
848	753	if (rc)
849		- goto out;
	754	+ goto error;
	755	+ zdev->state = state;
	756	+
	757	+ kref_init(&zdev->kref);
	758	+ mutex_init(&zdev->lock);
850	759
851	760	rc = zpci_init_iommu(zdev);
852	761	if (rc)
853		- goto out_free;
	762	+ goto error;
854	763
855		- mutex_init(&zdev->lock);
856	764	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
857	765	rc = zpci_enable_device(zdev);
858	766	if (rc)
859		- goto out_destroy_iommu;
	767	+ goto error_destroy_iommu;
860	768	}
861		- rc = zpci_scan_bus(zdev);
	769	+
	770	+ rc = zpci_bus_device_register(zdev, &pci_root_ops);
862	771	if (rc)
863		- goto out_disable;
	772	+ goto error_disable;
864	773
865	774	spin_lock(&zpci_list_lock);
866	775	list_add_tail(&zdev->entry, &zpci_list);
867	776	spin_unlock(&zpci_list_lock);
868	777
869		- zpci_init_slot(zdev);
870		-
871	778	return 0;
872	779
873		-out_disable:
	780	+error_disable:
874	781	if (zdev->state == ZPCI_FN_STATE_ONLINE)
875	782	zpci_disable_device(zdev);
876		-out_destroy_iommu:
	783	+error_destroy_iommu:
877	784	zpci_destroy_iommu(zdev);
878		-out_free:
879		- zpci_free_domain(zdev);
880		-out:
	785	+error:
	786	+ zpci_dbg(0, "add fid:%x, rc:%d\n", fid, rc);
	787	+ kfree(zdev);
881	788	return rc;
882	789	}
883	790
884		-void zpci_remove_device(struct zpci_dev *zdev)
	791	+bool zpci_is_device_configured(struct zpci_dev *zdev)
885	792	{
886		- if (!zdev->bus)
887		- return;
	793	+ enum zpci_state state = zdev->state;
888	794
889		- pci_stop_root_bus(zdev->bus);
890		- pci_remove_root_bus(zdev->bus);
	795	+ return state != ZPCI_FN_STATE_RESERVED &&
	796	+ state != ZPCI_FN_STATE_STANDBY;
	797	+}
	798	+
	799	+/**
	800	+ * zpci_device_reserved() - Mark device as resverved
	801	+ * @zdev: the zpci_dev that was reserved
	802	+ *
	803	+ * Handle the case that a given zPCI function was reserved by another system.
	804	+ * After a call to this function the zpci_dev can not be found via
	805	+ * get_zdev_by_fid() anymore but may still be accessible via existing
	806	+ * references though it will not be functional anymore.
	807	+ */
	808	+void zpci_device_reserved(struct zpci_dev *zdev)
	809	+{
	810	+ if (zdev->has_hp_slot)
	811	+ zpci_exit_slot(zdev);
	812	+ /*
	813	+ * Remove device from zpci_list as it is going away. This also
	814	+ * makes sure we ignore subsequent zPCI events for this device.
	815	+ */
	816	+ spin_lock(&zpci_list_lock);
	817	+ list_del(&zdev->entry);
	818	+ spin_unlock(&zpci_list_lock);
	819	+ zdev->state = ZPCI_FN_STATE_RESERVED;
	820	+ zpci_dbg(3, "rsv fid:%x\n", zdev->fid);
	821	+ zpci_zdev_put(zdev);
	822	+}
	823	+
	824	+void zpci_release_device(struct kref *kref)
	825	+{
	826	+ struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);
	827	+
	828	+ if (zdev->zbus->bus)
	829	+ zpci_remove_device(zdev, false);
	830	+
	831	+ switch (zdev->state) {
	832	+ case ZPCI_FN_STATE_ONLINE:
	833	+ case ZPCI_FN_STATE_CONFIGURED:
	834	+ zpci_disable_device(zdev);
	835	+ fallthrough;
	836	+ case ZPCI_FN_STATE_STANDBY:
	837	+ if (zdev->has_hp_slot)
	838	+ zpci_exit_slot(zdev);
	839	+ spin_lock(&zpci_list_lock);
	840	+ list_del(&zdev->entry);
	841	+ spin_unlock(&zpci_list_lock);
	842	+ zpci_dbg(3, "rsv fid:%x\n", zdev->fid);
	843	+ fallthrough;
	844	+ case ZPCI_FN_STATE_RESERVED:
	845	+ zpci_cleanup_bus_resources(zdev);
	846	+ zpci_bus_device_unregister(zdev);
	847	+ zpci_destroy_iommu(zdev);
	848	+ fallthrough;
	849	+ default:
	850	+ break;
	851	+ }
	852	+ zpci_dbg(3, "rem fid:%x\n", zdev->fid);
	853	+ kfree(zdev);
891	854	}
892	855
893	856	int zpci_report_error(struct pci_dev *pdev,
..	..	@@ -919,6 +882,9 @@
919	882	if (!zpci_iomap_bitmap)
920	883	goto error_iomap_bitmap;
921	884
	885	+ if (static_branch_likely(&have_mio))
	886	+ clp_setup_writeback_mio();
	887	+
922	888	return 0;
923	889	error_iomap_bitmap:
924	890	kfree(zpci_iomap_start);
..	..	@@ -935,13 +901,26 @@
935	901	kmem_cache_destroy(zdev_fmb_cache);
936	902	}
937	903
938		-static unsigned int s390_pci_probe = 1;
	904	+static unsigned int s390_pci_probe __initdata = 1;
	905	+unsigned int s390_pci_force_floating __initdata;
939	906	static unsigned int s390_pci_initialized;
940	907
941	908	char * __init pcibios_setup(char *str)
942	909	{
943	910	if (!strcmp(str, "off")) {
944	911	s390_pci_probe = 0;
	912	+ return NULL;
	913	+ }
	914	+ if (!strcmp(str, "nomio")) {
	915	+ S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO;
	916	+ return NULL;
	917	+ }
	918	+ if (!strcmp(str, "force_floating")) {
	919	+ s390_pci_force_floating = 1;
	920	+ return NULL;
	921	+ }
	922	+ if (!strcmp(str, "norid")) {
	923	+ s390_pci_no_rid = 1;
945	924	return NULL;
946	925	}
947	926	return str;
..	..	@@ -961,6 +940,11 @@
961	940
962	941	if (!test_facility(69) \|\| !test_facility(71))
963	942	return 0;
	943	+
	944	+ if (MACHINE_HAS_PCI_MIO) {
	945	+ static_branch_enable(&have_mio);
	946	+ ctl_set_bit(2, 5);
	947	+ }
964	948
965	949	rc = zpci_debug_init();
966	950	if (rc)
..	..	@@ -997,9 +981,3 @@
997	981	return rc;
998	982	}
999	983	subsys_initcall_sync(pci_base_init);
1000		-
1001		-void zpci_rescan(void)
1002		-{
1003		- if (zpci_is_enabled())
1004		- clp_rescan_pci_devices_simple();
1005		-}