~hc/RK356X_SDK_RELEASE.git

..	..	@@ -31,38 +31,41 @@
31	31	MODULE_LICENSE("GPL");
32	32
33	33	static inline int do_siga_sync(unsigned long schid,
34		- unsigned int out_mask, unsigned int in_mask,
	34	+ unsigned long out_mask, unsigned long in_mask,
35	35	unsigned int fc)
36	36	{
37		- register unsigned long __fc asm ("0") = fc;
38		- register unsigned long __schid asm ("1") = schid;
39		- register unsigned long out asm ("2") = out_mask;
40		- register unsigned long in asm ("3") = in_mask;
41	37	int cc;
42	38
43	39	asm volatile(
	40	+ " lgr 0,%[fc]\n"
	41	+ " lgr 1,%[schid]\n"
	42	+ " lgr 2,%[out]\n"
	43	+ " lgr 3,%[in]\n"
44	44	" siga 0\n"
45		- " ipm %0\n"
46		- " srl %0,28\n"
47		- : "=d" (cc)
48		- : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc");
	45	+ " ipm %[cc]\n"
	46	+ " srl %[cc],28\n"
	47	+ : [cc] "=&d" (cc)
	48	+ : [fc] "d" (fc), [schid] "d" (schid),
	49	+ [out] "d" (out_mask), [in] "d" (in_mask)
	50	+ : "cc", "0", "1", "2", "3");
49	51	return cc;
50	52	}
51	53
52		-static inline int do_siga_input(unsigned long schid, unsigned int mask,
53		- unsigned int fc)
	54	+static inline int do_siga_input(unsigned long schid, unsigned long mask,
	55	+ unsigned long fc)
54	56	{
55		- register unsigned long __fc asm ("0") = fc;
56		- register unsigned long __schid asm ("1") = schid;
57		- register unsigned long __mask asm ("2") = mask;
58	57	int cc;
59	58
60	59	asm volatile(
	60	+ " lgr 0,%[fc]\n"
	61	+ " lgr 1,%[schid]\n"
	62	+ " lgr 2,%[mask]\n"
61	63	" siga 0\n"
62		- " ipm %0\n"
63		- " srl %0,28\n"
64		- : "=d" (cc)
65		- : "d" (__fc), "d" (__schid), "d" (__mask) : "cc");
	64	+ " ipm %[cc]\n"
	65	+ " srl %[cc],28\n"
	66	+ : [cc] "=&d" (cc)
	67	+ : [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask)
	68	+ : "cc", "0", "1", "2");
66	69	return cc;
67	70	}
68	71
..	..	@@ -78,23 +81,24 @@
78	81	* Note: For IQDC unicast queues only the highest priority queue is processed.
79	82	*/
80	83	static inline int do_siga_output(unsigned long schid, unsigned long mask,
81		- unsigned int *bb, unsigned int fc,
	84	+ unsigned int *bb, unsigned long fc,
82	85	unsigned long aob)
83	86	{
84		- register unsigned long __fc asm("0") = fc;
85		- register unsigned long __schid asm("1") = schid;
86		- register unsigned long __mask asm("2") = mask;
87		- register unsigned long __aob asm("3") = aob;
88	87	int cc;
89	88
90	89	asm volatile(
	90	+ " lgr 0,%[fc]\n"
	91	+ " lgr 1,%[schid]\n"
	92	+ " lgr 2,%[mask]\n"
	93	+ " lgr 3,%[aob]\n"
91	94	" siga 0\n"
92		- " ipm %0\n"
93		- " srl %0,28\n"
94		- : "=d" (cc), "+d" (__fc), "+d" (__aob)
95		- : "d" (__schid), "d" (__mask)
96		- : "cc");
97		- *bb = __fc >> 31;
	95	+ " lgr %[fc],0\n"
	96	+ " ipm %[cc]\n"
	97	+ " srl %[cc],28\n"
	98	+ : [cc] "=&d" (cc), [fc] "+&d" (fc)
	99	+ : [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob)
	100	+ : "cc", "0", "1", "2", "3");
	101	+ *bb = fc >> 31;
98	102	return cc;
99	103	}
100	104
..	..	@@ -131,7 +135,7 @@
131	135	case 96:
132	136	/* not all buffers processed */
133	137	qperf_inc(q, eqbs_partial);
134		- DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS part:%02x",
	138	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",
135	139	tmp_count);
136	140	return count - tmp_count;
137	141	case 97:
..	..	@@ -143,7 +147,7 @@
143	147	DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
144	148	DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
145	149	q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
146		- q->first_to_kick, count, q->irq_ptr->int_parm);
	150	+ q->first_to_check, count, q->irq_ptr->int_parm);
147	151	return 0;
148	152	}
149	153	}
..	..	@@ -191,7 +195,7 @@
191	195	DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
192	196	DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
193	197	q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
194		- q->first_to_kick, count, q->irq_ptr->int_parm);
	198	+ q->first_to_check, count, q->irq_ptr->int_parm);
195	199	return 0;
196	200	}
197	201	}
..	..	@@ -205,17 +209,22 @@
205	209	int auto_ack, int merge_pending)
206	210	{
207	211	unsigned char __state = 0;
208		- int i;
	212	+ int i = 1;
209	213
210	214	if (is_qebsm(q))
211	215	return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
212	216
213	217	/* get initial state: */
214	218	__state = q->slsb.val[bufnr];
	219	+
	220	+ /* Bail out early if there is no work on the queue: */
	221	+ if (__state & SLSB_OWNER_CU)
	222	+ goto out;
	223	+
215	224	if (merge_pending && __state == SLSB_P_OUTPUT_PENDING)
216	225	__state = SLSB_P_OUTPUT_EMPTY;
217	226
218		- for (i = 1; i < count; i++) {
	227	+ for (; i < count; i++) {
219	228	bufnr = next_buf(bufnr);
220	229
221	230	/* merge PENDING into EMPTY: */
..	..	@@ -228,6 +237,8 @@
228	237	if (q->slsb.val[bufnr] != __state)
229	238	break;
230	239	}
	240	+
	241	+out:
231	242	*state = __state;
232	243	return i;
233	244	}
..	..	@@ -247,10 +258,17 @@
247	258	if (is_qebsm(q))
248	259	return qdio_do_sqbs(q, state, bufnr, count);
249	260
	261	+ /* Ensure that all preceding changes to the SBALs are visible: */
	262	+ mb();
	263	+
250	264	for (i = 0; i < count; i++) {
251		- xchg(&q->slsb.val[bufnr], state);
	265	+ WRITE_ONCE(q->slsb.val[bufnr], state);
252	266	bufnr = next_buf(bufnr);
253	267	}
	268	+
	269	+ /* Make our SLSB changes visible: */
	270	+ mb();
	271	+
254	272	return count;
255	273	}
256	274
..	..	@@ -303,20 +321,19 @@
303	321	return qdio_siga_sync(q, q->mask, 0);
304	322	}
305	323
306		-static int qdio_siga_output(struct qdio_q q, unsigned int busy_bit,
307		- unsigned long aob)
	324	+static int qdio_siga_output(struct qdio_q *q, unsigned int count,
	325	+ unsigned int *busy_bit, unsigned long aob)
308	326	{
309	327	unsigned long schid = ((u32 ) &q->irq_ptr->schid);
310	328	unsigned int fc = QDIO_SIGA_WRITE;
311	329	u64 start_time = 0;
312	330	int retries = 0, cc;
313		- unsigned long laob = 0;
314	331
315		- WARN_ON_ONCE(aob && ((queue_type(q) != QDIO_IQDIO_QFMT) \|\|
316		- !q->u.out.use_cq));
317		- if (q->u.out.use_cq && aob != 0) {
318		- fc = QDIO_SIGA_WRITEQ;
319		- laob = aob;
	332	+ if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {
	333	+ if (count > 1)
	334	+ fc = QDIO_SIGA_WRITEM;
	335	+ else if (aob)
	336	+ fc = QDIO_SIGA_WRITEQ;
320	337	}
321	338
322	339	if (is_qebsm(q)) {
..	..	@@ -324,7 +341,7 @@
324	341	fc \|= QDIO_SIGA_QEBSM_FLAG;
325	342	}
326	343	again:
327		- cc = do_siga_output(schid, q->mask, busy_bit, fc, laob);
	344	+ cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);
328	345
329	346	/* hipersocket busy condition */
330	347	if (unlikely(*busy_bit)) {
..	..	@@ -371,7 +388,7 @@
371	388	static inline void qdio_sync_queues(struct qdio_q *q)
372	389	{
373	390	/* PCI capable outbound queues will also be scanned so sync them too */
374		- if (pci_out_supported(q))
	391	+ if (pci_out_supported(q->irq_ptr))
375	392	qdio_siga_sync_all(q);
376	393	else
377	394	qdio_siga_sync_q(q);
..	..	@@ -382,186 +399,125 @@
382	399	{
383	400	if (need_siga_sync(q))
384	401	qdio_siga_sync_q(q);
385		- return get_buf_states(q, bufnr, state, 1, 0, 0);
	402	+ return get_buf_state(q, bufnr, state, 0);
386	403	}
387	404
388	405	static inline void qdio_stop_polling(struct qdio_q *q)
389	406	{
390		- if (!q->u.in.polling)
	407	+ if (!q->u.in.batch_count)
391	408	return;
392	409
393		- q->u.in.polling = 0;
394	410	qperf_inc(q, stop_polling);
395	411
396	412	/* show the card that we are not polling anymore */
397		- if (is_qebsm(q)) {
398		- set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
399		- q->u.in.ack_count);
400		- q->u.in.ack_count = 0;
401		- } else
402		- set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
	413	+ set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,
	414	+ q->u.in.batch_count);
	415	+ q->u.in.batch_count = 0;
403	416	}
404	417
405	418	static inline void account_sbals(struct qdio_q *q, unsigned int count)
406	419	{
407		- int pos;
408		-
409	420	q->q_stats.nr_sbal_total += count;
410		- if (count == QDIO_MAX_BUFFERS_MASK) {
411		- q->q_stats.nr_sbals[7]++;
412		- return;
413		- }
414		- pos = ilog2(count);
415		- q->q_stats.nr_sbals[pos]++;
	421	+ q->q_stats.nr_sbals[ilog2(count)]++;
416	422	}
417	423
418		-static void process_buffer_error(struct qdio_q *q, int count)
	424	+static void process_buffer_error(struct qdio_q *q, unsigned int start,
	425	+ int count)
419	426	{
420		- unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT :
421		- SLSB_P_OUTPUT_NOT_INIT;
422		-
423	427	q->qdio_error = QDIO_ERROR_SLSB_STATE;
424	428
425	429	/* special handling for no target buffer empty */
426	430	if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&
427		- q->sbal[q->first_to_check]->element[15].sflags == 0x10) {
	431	+ q->sbal[start]->element[15].sflags == 0x10) {
428	432	qperf_inc(q, target_full);
429		- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
430		- q->first_to_check);
431		- goto set;
	433	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);
	434	+ return;
432	435	}
433	436
434	437	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
435	438	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
436		- DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
	439	+ DBF_ERROR("FTC:%3d C:%3d", start, count);
437	440	DBF_ERROR("F14:%2x F15:%2x",
438		- q->sbal[q->first_to_check]->element[14].sflags,
439		- q->sbal[q->first_to_check]->element[15].sflags);
440		-
441		-set:
442		- /*
443		- * Interrupts may be avoided as long as the error is present
444		- * so change the buffer state immediately to avoid starvation.
445		- */
446		- set_buf_states(q, q->first_to_check, state, count);
	441	+ q->sbal[start]->element[14].sflags,
	442	+ q->sbal[start]->element[15].sflags);
447	443	}
448	444
449		-static inline void inbound_primed(struct qdio_q *q, int count)
	445	+static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
	446	+ int count, bool auto_ack)
450	447	{
451		- int new;
	448	+ /* ACK the newest SBAL: */
	449	+ if (!auto_ack)
	450	+ set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);
452	451
453		- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, count);
454		-
455		- /* for QEBSM the ACK was already set by EQBS */
456		- if (is_qebsm(q)) {
457		- if (!q->u.in.polling) {
458		- q->u.in.polling = 1;
459		- q->u.in.ack_count = count;
460		- q->u.in.ack_start = q->first_to_check;
461		- return;
462		- }
463		-
464		- /* delete the previous ACK's */
465		- set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
466		- q->u.in.ack_count);
467		- q->u.in.ack_count = count;
468		- q->u.in.ack_start = q->first_to_check;
469		- return;
470		- }
471		-
472		- /*
473		- * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
474		- * or by the next inbound run.
475		- */
476		- new = add_buf(q->first_to_check, count - 1);
477		- if (q->u.in.polling) {
478		- /* reset the previous ACK but first set the new one */
479		- set_buf_state(q, new, SLSB_P_INPUT_ACK);
480		- set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
481		- } else {
482		- q->u.in.polling = 1;
483		- set_buf_state(q, new, SLSB_P_INPUT_ACK);
484		- }
485		-
486		- q->u.in.ack_start = new;
487		- count--;
488		- if (!count)
489		- return;
490		- /* need to change ALL buffers to get more interrupts */
491		- set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
	452	+ if (!q->u.in.batch_count)
	453	+ q->u.in.batch_start = start;
	454	+ q->u.in.batch_count += count;
492	455	}
493	456
494		-static int get_inbound_buffer_frontier(struct qdio_q *q)
	457	+static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start)
495	458	{
496	459	unsigned char state = 0;
497	460	int count;
498	461
499	462	q->timestamp = get_tod_clock_fast();
500	463
501		- /*
502		- * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
503		- * would return 0.
504		- */
505		- count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
	464	+ count = atomic_read(&q->nr_buf_used);
506	465	if (!count)
507		- goto out;
	466	+ return 0;
508	467
509	468	/*
510	469	* No siga sync here, as a PCI or we after a thin interrupt
511	470	* already sync'ed the queues.
512	471	*/
513		- count = get_buf_states(q, q->first_to_check, &state, count, 1, 0);
	472	+ count = get_buf_states(q, start, &state, count, 1, 0);
514	473	if (!count)
515		- goto out;
	474	+ return 0;
516	475
517	476	switch (state) {
518	477	case SLSB_P_INPUT_PRIMED:
519		- inbound_primed(q, count);
520		- q->first_to_check = add_buf(q->first_to_check, count);
	478	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
	479	+ count);
	480	+
	481	+ inbound_handle_work(q, start, count, is_qebsm(q));
521	482	if (atomic_sub_return(count, &q->nr_buf_used) == 0)
522	483	qperf_inc(q, inbound_queue_full);
523	484	if (q->irq_ptr->perf_stat_enabled)
524	485	account_sbals(q, count);
525		- break;
	486	+ return count;
526	487	case SLSB_P_INPUT_ERROR:
527		- process_buffer_error(q, count);
528		- q->first_to_check = add_buf(q->first_to_check, count);
	488	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
	489	+ count);
	490	+
	491	+ process_buffer_error(q, start, count);
	492	+ inbound_handle_work(q, start, count, false);
529	493	if (atomic_sub_return(count, &q->nr_buf_used) == 0)
530	494	qperf_inc(q, inbound_queue_full);
531	495	if (q->irq_ptr->perf_stat_enabled)
532	496	account_sbals_error(q, count);
533		- break;
	497	+ return count;
534	498	case SLSB_CU_INPUT_EMPTY:
535		- case SLSB_P_INPUT_NOT_INIT:
536		- case SLSB_P_INPUT_ACK:
537	499	if (q->irq_ptr->perf_stat_enabled)
538	500	q->q_stats.nr_sbal_nop++;
539	501	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",
540		- q->nr, q->first_to_check);
541		- break;
542		- default:
543		- WARN_ON_ONCE(1);
544		- }
545		-out:
546		- return q->first_to_check;
547		-}
548		-
549		-static int qdio_inbound_q_moved(struct qdio_q *q)
550		-{
551		- int bufnr;
552		-
553		- bufnr = get_inbound_buffer_frontier(q);
554		-
555		- if (bufnr != q->last_move) {
556		- q->last_move = bufnr;
557		- if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
558		- q->u.in.timestamp = get_tod_clock();
559		- return 1;
560		- } else
	502	+ q->nr, start);
561	503	return 0;
	504	+ case SLSB_P_INPUT_NOT_INIT:
	505	+ case SLSB_P_INPUT_ACK:
	506	+ /* We should never see this state, throw a WARN: */
	507	+ default:
	508	+ dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
	509	+ "found state %#x at index %u on queue %u\n",
	510	+ state, start, q->nr);
	511	+ return 0;
	512	+ }
562	513	}
563	514
564		-static inline int qdio_inbound_q_done(struct qdio_q *q)
	515	+static int qdio_inbound_q_moved(struct qdio_q *q, unsigned int start)
	516	+{
	517	+ return get_inbound_buffer_frontier(q, start);
	518	+}
	519	+
	520	+static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)
565	521	{
566	522	unsigned char state = 0;
567	523
..	..	@@ -570,59 +526,13 @@
570	526
571	527	if (need_siga_sync(q))
572	528	qdio_siga_sync_q(q);
573		- get_buf_state(q, q->first_to_check, &state, 0);
	529	+ get_buf_state(q, start, &state, 0);
574	530
575	531	if (state == SLSB_P_INPUT_PRIMED \|\| state == SLSB_P_INPUT_ERROR)
576	532	/* more work coming */
577	533	return 0;
578	534
579		- if (is_thinint_irq(q->irq_ptr))
580		- return 1;
581		-
582		- /* don't poll under z/VM */
583		- if (MACHINE_IS_VM)
584		- return 1;
585		-
586		- /*
587		- * At this point we know, that inbound first_to_check
588		- * has (probably) not moved (see qdio_inbound_processing).
589		- */
590		- if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
591		- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
592		- q->first_to_check);
593		- return 1;
594		- } else
595		- return 0;
596		-}
597		-
598		-static inline int contains_aobs(struct qdio_q *q)
599		-{
600		- return !q->is_input_q && q->u.out.use_cq;
601		-}
602		-
603		-static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count)
604		-{
605		- unsigned char state = 0;
606		- int j, b = start;
607		-
608		- if (!contains_aobs(q))
609		- return;
610		-
611		- for (j = 0; j < count; ++j) {
612		- get_buf_state(q, b, &state, 0);
613		- if (state == SLSB_P_OUTPUT_PENDING) {
614		- struct qaob *aob = q->u.out.aobs[b];
615		- if (aob == NULL)
616		- continue;
617		-
618		- q->u.out.sbal_state[b].flags \|=
619		- QDIO_OUTBUF_STATE_FLAG_PENDING;
620		- q->u.out.aobs[b] = NULL;
621		- } else if (state == SLSB_P_OUTPUT_EMPTY) {
622		- q->u.out.sbal_state[b].aob = NULL;
623		- }
624		- b = next_buf(b);
625		- }
	535	+ return 1;
626	536	}
627	537
628	538	static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q,
..	..	@@ -630,15 +540,11 @@
630	540	{
631	541	unsigned long phys_aob = 0;
632	542
633		- if (!q->use_cq)
634		- return 0;
635		-
636	543	if (!q->aobs[bufnr]) {
637	544	struct qaob *aob = qdio_allocate_aob();
638	545	q->aobs[bufnr] = aob;
639	546	}
640	547	if (q->aobs[bufnr]) {
641		- q->sbal_state[bufnr].aob = q->aobs[bufnr];
642	548	q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user;
643	549	phys_aob = virt_to_phys(q->aobs[bufnr]);
644	550	WARN_ON_ONCE(phys_aob & 0xFF);
..	..	@@ -648,16 +554,11 @@
648	554	return phys_aob;
649	555	}
650	556
651		-static void qdio_kick_handler(struct qdio_q *q)
	557	+static void qdio_kick_handler(struct qdio_q *q, unsigned int start,
	558	+ unsigned int count)
652	559	{
653		- int start = q->first_to_kick;
654		- int end = q->first_to_check;
655		- int count;
656		-
657	560	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
658	561	return;
659		-
660		- count = sub_buf(end, start);
661	562
662	563	if (q->is_input_q) {
663	564	qperf_inc(q, inbound_handler);
..	..	@@ -668,13 +569,10 @@
668	569	start, count);
669	570	}
670	571
671		- qdio_handle_aobs(q, start, count);
672		-
673	572	q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
674	573	q->irq_ptr->int_parm);
675	574
676	575	/* for the next time */
677		- q->first_to_kick = end;
678	576	q->qdio_error = 0;
679	577	}
680	578
..	..	@@ -689,14 +587,20 @@
689	587
690	588	static void __qdio_inbound_processing(struct qdio_q *q)
691	589	{
	590	+ unsigned int start = q->first_to_check;
	591	+ int count;
	592	+
692	593	qperf_inc(q, tasklet_inbound);
693	594
694		- if (!qdio_inbound_q_moved(q))
	595	+ count = qdio_inbound_q_moved(q, start);
	596	+ if (count == 0)
695	597	return;
696	598
697		- qdio_kick_handler(q);
	599	+ qdio_kick_handler(q, start, count);
	600	+ start = add_buf(start, count);
	601	+ q->first_to_check = start;
698	602
699		- if (!qdio_inbound_q_done(q)) {
	603	+ if (!qdio_inbound_q_done(q, start)) {
700	604	/* means poll time is not yet over */
701	605	qperf_inc(q, tasklet_inbound_resched);
702	606	if (!qdio_tasklet_schedule(q))
..	..	@@ -708,7 +612,7 @@
708	612	* We need to check again to not lose initiative after
709	613	* resetting the ACK state.
710	614	*/
711		- if (!qdio_inbound_q_done(q)) {
	615	+ if (!qdio_inbound_q_done(q, start)) {
712	616	qperf_inc(q, tasklet_inbound_resched2);
713	617	qdio_tasklet_schedule(q);
714	618	}
..	..	@@ -720,7 +624,20 @@
720	624	__qdio_inbound_processing(q);
721	625	}
722	626
723		-static int get_outbound_buffer_frontier(struct qdio_q *q)
	627	+static void qdio_check_pending(struct qdio_q *q, unsigned int index)
	628	+{
	629	+ unsigned char state;
	630	+
	631	+ if (get_buf_state(q, index, &state, 0) > 0 &&
	632	+ state == SLSB_P_OUTPUT_PENDING &&
	633	+ q->u.out.aobs[index]) {
	634	+ q->u.out.sbal_state[index].flags \|=
	635	+ QDIO_OUTBUF_STATE_FLAG_PENDING;
	636	+ q->u.out.aobs[index] = NULL;
	637	+ }
	638	+}
	639	+
	640	+static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start)
724	641	{
725	642	unsigned char state = 0;
726	643	int count;
..	..	@@ -729,23 +646,18 @@
729	646
730	647	if (need_siga_sync(q))
731	648	if (((queue_type(q) != QDIO_IQDIO_QFMT) &&
732		- !pci_out_supported(q)) \|\|
	649	+ !pci_out_supported(q->irq_ptr)) \|\|
733	650	(queue_type(q) == QDIO_IQDIO_QFMT &&
734	651	multicast_outbound(q)))
735	652	qdio_siga_sync_q(q);
736	653
737		- /*
738		- * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
739		- * would return 0.
740		- */
741		- count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
	654	+ count = atomic_read(&q->nr_buf_used);
742	655	if (!count)
743		- goto out;
	656	+ return 0;
744	657
745		- count = get_buf_states(q, q->first_to_check, &state, count, 0,
746		- q->u.out.use_cq);
	658	+ count = get_buf_states(q, start, &state, count, 0, q->u.out.use_cq);
747	659	if (!count)
748		- goto out;
	660	+ return 0;
749	661
750	662	switch (state) {
751	663	case SLSB_P_OUTPUT_EMPTY:
..	..	@@ -755,34 +667,32 @@
755	667	"out empty:%1d %02x", q->nr, count);
756	668
757	669	atomic_sub(count, &q->nr_buf_used);
758		- q->first_to_check = add_buf(q->first_to_check, count);
759	670	if (q->irq_ptr->perf_stat_enabled)
760	671	account_sbals(q, count);
761		-
762		- break;
	672	+ return count;
763	673	case SLSB_P_OUTPUT_ERROR:
764		- process_buffer_error(q, count);
765		- q->first_to_check = add_buf(q->first_to_check, count);
	674	+ process_buffer_error(q, start, count);
766	675	atomic_sub(count, &q->nr_buf_used);
767	676	if (q->irq_ptr->perf_stat_enabled)
768	677	account_sbals_error(q, count);
769		- break;
	678	+ return count;
770	679	case SLSB_CU_OUTPUT_PRIMED:
771	680	/* the adapter has not fetched the output yet */
772	681	if (q->irq_ptr->perf_stat_enabled)
773	682	q->q_stats.nr_sbal_nop++;
774	683	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
775	684	q->nr);
776		- break;
777		- case SLSB_P_OUTPUT_NOT_INIT:
	685	+ return 0;
778	686	case SLSB_P_OUTPUT_HALTED:
779		- break;
	687	+ return 0;
	688	+ case SLSB_P_OUTPUT_NOT_INIT:
	689	+ /* We should never see this state, throw a WARN: */
780	690	default:
781		- WARN_ON_ONCE(1);
	691	+ dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
	692	+ "found state %#x at index %u on queue %u\n",
	693	+ state, start, q->nr);
	694	+ return 0;
782	695	}
783		-
784		-out:
785		- return q->first_to_check;
786	696	}
787	697
788	698	/* all buffers processed? */
..	..	@@ -791,21 +701,28 @@
791	701	return atomic_read(&q->nr_buf_used) == 0;
792	702	}
793	703
794		-static inline int qdio_outbound_q_moved(struct qdio_q *q)
	704	+static inline int qdio_outbound_q_moved(struct qdio_q *q, unsigned int start)
795	705	{
796		- int bufnr;
	706	+ int count;
797	707
798		- bufnr = get_outbound_buffer_frontier(q);
	708	+ count = get_outbound_buffer_frontier(q, start);
799	709
800		- if (bufnr != q->last_move) {
801		- q->last_move = bufnr;
	710	+ if (count) {
802	711	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
803		- return 1;
804		- } else
805		- return 0;
	712	+
	713	+ if (q->u.out.use_cq) {
	714	+ unsigned int i;
	715	+
	716	+ for (i = 0; i < count; i++)
	717	+ qdio_check_pending(q, QDIO_BUFNR(start + i));
	718	+ }
	719	+ }
	720	+
	721	+ return count;
806	722	}
807	723
808		-static int qdio_kick_outbound_q(struct qdio_q *q, unsigned long aob)
	724	+static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,
	725	+ unsigned long aob)
809	726	{
810	727	int retries = 0, cc;
811	728	unsigned int busy_bit;
..	..	@@ -817,7 +734,7 @@
817	734	retry:
818	735	qperf_inc(q, siga_write);
819	736
820		- cc = qdio_siga_output(q, &busy_bit, aob);
	737	+ cc = qdio_siga_output(q, count, &busy_bit, aob);
821	738	switch (cc) {
822	739	case 0:
823	740	break;
..	..	@@ -849,15 +766,21 @@
849	766
850	767	static void __qdio_outbound_processing(struct qdio_q *q)
851	768	{
	769	+ unsigned int start = q->first_to_check;
	770	+ int count;
	771	+
852	772	qperf_inc(q, tasklet_outbound);
853	773	WARN_ON_ONCE(atomic_read(&q->nr_buf_used) < 0);
854	774
855		- if (qdio_outbound_q_moved(q))
856		- qdio_kick_handler(q);
	775	+ count = qdio_outbound_q_moved(q, start);
	776	+ if (count) {
	777	+ q->first_to_check = add_buf(start, count);
	778	+ qdio_kick_handler(q, start, count);
	779	+ }
857	780
858		- if (queue_type(q) == QDIO_ZFCP_QFMT)
859		- if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
860		- goto sched;
	781	+ if (queue_type(q) == QDIO_ZFCP_QFMT && !pci_out_supported(q->irq_ptr) &&
	782	+ !qdio_outbound_q_done(q))
	783	+ goto sched;
861	784
862	785	if (q->u.out.pci_out_enabled)
863	786	return;
..	..	@@ -893,57 +816,30 @@
893	816	qdio_tasklet_schedule(q);
894	817	}
895	818
896		-static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
	819	+static inline void qdio_check_outbound_pci_queues(struct qdio_irq *irq)
897	820	{
898	821	struct qdio_q *out;
899	822	int i;
900	823
901		- if (!pci_out_supported(q))
	824	+ if (!pci_out_supported(irq) \|\| !irq->scan_threshold)
902	825	return;
903	826
904		- for_each_output_queue(q->irq_ptr, out, i)
	827	+ for_each_output_queue(irq, out, i)
905	828	if (!qdio_outbound_q_done(out))
906	829	qdio_tasklet_schedule(out);
907		-}
908		-
909		-static void __tiqdio_inbound_processing(struct qdio_q *q)
910		-{
911		- qperf_inc(q, tasklet_inbound);
912		- if (need_siga_sync(q) && need_siga_sync_after_ai(q))
913		- qdio_sync_queues(q);
914		-
915		- /*
916		- * The interrupt could be caused by a PCI request. Check the
917		- * PCI capable outbound queues.
918		- */
919		- qdio_check_outbound_after_thinint(q);
920		-
921		- if (!qdio_inbound_q_moved(q))
922		- return;
923		-
924		- qdio_kick_handler(q);
925		-
926		- if (!qdio_inbound_q_done(q)) {
927		- qperf_inc(q, tasklet_inbound_resched);
928		- if (!qdio_tasklet_schedule(q))
929		- return;
930		- }
931		-
932		- qdio_stop_polling(q);
933		- /*
934		- * We need to check again to not lose initiative after
935		- * resetting the ACK state.
936		- */
937		- if (!qdio_inbound_q_done(q)) {
938		- qperf_inc(q, tasklet_inbound_resched2);
939		- qdio_tasklet_schedule(q);
940		- }
941	830	}
942	831
943	832	void tiqdio_inbound_processing(unsigned long data)
944	833	{
945	834	struct qdio_q q = (struct qdio_q )data;
946		- __tiqdio_inbound_processing(q);
	835	+
	836	+ if (need_siga_sync(q) && need_siga_sync_after_ai(q))
	837	+ qdio_sync_queues(q);
	838	+
	839	+ /* The interrupt could be caused by a PCI request: */
	840	+ qdio_check_outbound_pci_queues(q->irq_ptr);
	841	+
	842	+ __qdio_inbound_processing(q);
947	843	}
948	844
949	845	static inline void qdio_set_state(struct qdio_irq *irq_ptr,
..	..	@@ -973,22 +869,17 @@
973	869	if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
974	870	return;
975	871
976		- for_each_input_queue(irq_ptr, q, i) {
977		- if (q->u.in.queue_start_poll) {
978		- /* skip if polling is enabled or already in work */
979		- if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
980		- &q->u.in.queue_irq_state)) {
981		- qperf_inc(q, int_discarded);
982		- continue;
983		- }
984		- q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
985		- q->irq_ptr->int_parm);
986		- } else {
	872	+ if (irq_ptr->irq_poll) {
	873	+ if (!test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
	874	+ irq_ptr->irq_poll(irq_ptr->cdev, irq_ptr->int_parm);
	875	+ else
	876	+ QDIO_PERF_STAT_INC(irq_ptr, int_discarded);
	877	+ } else {
	878	+ for_each_input_queue(irq_ptr, q, i)
987	879	tasklet_schedule(&q->tasklet);
988		- }
989	880	}
990	881
991		- if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED))
	882	+ if (!pci_out_supported(irq_ptr) \|\| !irq_ptr->scan_threshold)
992	883	return;
993	884
994	885	for_each_output_queue(irq_ptr, q, i) {
..	..	@@ -1000,12 +891,11 @@
1000	891	}
1001	892	}
1002	893
1003		-static void qdio_handle_activate_check(struct ccw_device *cdev,
1004		- unsigned long intparm, int cstat, int dstat)
	894	+static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,
	895	+ unsigned long intparm, int cstat,
	896	+ int dstat)
1005	897	{
1006		- struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1007	898	struct qdio_q *q;
1008		- int count;
1009	899
1010	900	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
1011	901	DBF_ERROR("intp :%lx", intparm);
..	..	@@ -1020,9 +910,8 @@
1020	910	goto no_handler;
1021	911	}
1022	912
1023		- count = sub_buf(q->first_to_check, q->first_to_kick);
1024	913	q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE,
1025		- q->nr, q->first_to_kick, count, irq_ptr->int_parm);
	914	+ q->nr, q->first_to_check, 0, irq_ptr->int_parm);
1026	915	no_handler:
1027	916	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1028	917	/*
..	..	@@ -1032,11 +921,9 @@
1032	921	lgr_info_log();
1033	922	}
1034	923
1035		-static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
	924	+static void qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat,
1036	925	int dstat)
1037	926	{
1038		- struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1039		-
1040	927	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
1041	928
1042	929	if (cstat)
..	..	@@ -1083,7 +970,7 @@
1083	970
1084	971	switch (irq_ptr->state) {
1085	972	case QDIO_IRQ_STATE_INACTIVE:
1086		- qdio_establish_handle_irq(cdev, cstat, dstat);
	973	+ qdio_establish_handle_irq(irq_ptr, cstat, dstat);
1087	974	break;
1088	975	case QDIO_IRQ_STATE_CLEANUP:
1089	976	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
..	..	@@ -1095,7 +982,7 @@
1095	982	return;
1096	983	}
1097	984	if (cstat \|\| dstat)
1098		- qdio_handle_activate_check(cdev, intparm, cstat,
	985	+ qdio_handle_activate_check(irq_ptr, intparm, cstat,
1099	986	dstat);
1100	987	break;
1101	988	case QDIO_IRQ_STATE_STOPPED:
..	..	@@ -1128,9 +1015,8 @@
1128	1015	}
1129	1016	EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
1130	1017
1131		-static void qdio_shutdown_queues(struct ccw_device *cdev)
	1018	+static void qdio_shutdown_queues(struct qdio_irq *irq_ptr)
1132	1019	{
1133		- struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1134	1020	struct qdio_q *q;
1135	1021	int i;
1136	1022
..	..	@@ -1141,6 +1027,33 @@
1141	1027	del_timer_sync(&q->u.out.timer);
1142	1028	tasklet_kill(&q->tasklet);
1143	1029	}
	1030	+}
	1031	+
	1032	+static int qdio_cancel_ccw(struct qdio_irq *irq, int how)
	1033	+{
	1034	+ struct ccw_device *cdev = irq->cdev;
	1035	+ int rc;
	1036	+
	1037	+ spin_lock_irq(get_ccwdev_lock(cdev));
	1038	+ qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP);
	1039	+ if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
	1040	+ rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
	1041	+ else
	1042	+ /* default behaviour is halt */
	1043	+ rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
	1044	+ spin_unlock_irq(get_ccwdev_lock(cdev));
	1045	+ if (rc) {
	1046	+ DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no);
	1047	+ DBF_ERROR("rc:%4d", rc);
	1048	+ return rc;
	1049	+ }
	1050	+
	1051	+ wait_event_interruptible_timeout(cdev->private->wait_q,
	1052	+ irq->state == QDIO_IRQ_STATE_INACTIVE \|\|
	1053	+ irq->state == QDIO_IRQ_STATE_ERR,
	1054	+ 10 * HZ);
	1055	+
	1056	+ return 0;
1144	1057	}
1145	1058
1146	1059	/**
..	..	@@ -1177,41 +1090,13 @@
1177	1090	*/
1178	1091	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1179	1092
1180		- tiqdio_remove_input_queues(irq_ptr);
1181		- qdio_shutdown_queues(cdev);
	1093	+ tiqdio_remove_device(irq_ptr);
	1094	+ qdio_shutdown_queues(irq_ptr);
1182	1095	qdio_shutdown_debug_entries(irq_ptr);
1183	1096
1184		- /* cleanup subchannel */
1185		- spin_lock_irq(get_ccwdev_lock(cdev));
1186		-
1187		- if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
1188		- rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
1189		- else
1190		- /* default behaviour is halt */
1191		- rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
1192		- if (rc) {
1193		- DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
1194		- DBF_ERROR("rc:%4d", rc);
1195		- goto no_cleanup;
1196		- }
1197		-
1198		- qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
1199		- spin_unlock_irq(get_ccwdev_lock(cdev));
1200		- wait_event_interruptible_timeout(cdev->private->wait_q,
1201		- irq_ptr->state == QDIO_IRQ_STATE_INACTIVE \|\|
1202		- irq_ptr->state == QDIO_IRQ_STATE_ERR,
1203		- 10 * HZ);
1204		- spin_lock_irq(get_ccwdev_lock(cdev));
1205		-
1206		-no_cleanup:
	1097	+ rc = qdio_cancel_ccw(irq_ptr, how);
1207	1098	qdio_shutdown_thinint(irq_ptr);
1208		-
1209		- /* restore interrupt handler */
1210		- if ((void )cdev->handler == (void )qdio_int_handler) {
1211		- cdev->handler = irq_ptr->orig_handler;
1212		- cdev->private->intparm = 0;
1213		- }
1214		- spin_unlock_irq(get_ccwdev_lock(cdev));
	1099	+ qdio_shutdown_irq(irq_ptr);
1215	1100
1216	1101	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1217	1102	mutex_unlock(&irq_ptr->setup_mutex);
..	..	@@ -1242,43 +1127,47 @@
1242	1127	cdev->private->qdio_data = NULL;
1243	1128	mutex_unlock(&irq_ptr->setup_mutex);
1244	1129
1245		- qdio_release_memory(irq_ptr);
	1130	+ qdio_free_async_data(irq_ptr);
	1131	+ qdio_free_queues(irq_ptr);
	1132	+ free_page((unsigned long) irq_ptr->qdr);
	1133	+ free_page(irq_ptr->chsc_page);
	1134	+ free_page((unsigned long) irq_ptr);
1246	1135	return 0;
1247	1136	}
1248	1137	EXPORT_SYMBOL_GPL(qdio_free);
1249	1138
1250	1139	/**
1251	1140	* qdio_allocate - allocate qdio queues and associated data
1252		- * @init_data: initialization data
	1141	+ * @cdev: associated ccw device
	1142	+ * @no_input_qs: allocate this number of Input Queues
	1143	+ * @no_output_qs: allocate this number of Output Queues
1253	1144	*/
1254		-int qdio_allocate(struct qdio_initialize *init_data)
	1145	+int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
	1146	+ unsigned int no_output_qs)
1255	1147	{
1256	1148	struct subchannel_id schid;
1257	1149	struct qdio_irq *irq_ptr;
	1150	+ int rc = -ENOMEM;
1258	1151
1259		- ccw_device_get_schid(init_data->cdev, &schid);
	1152	+ ccw_device_get_schid(cdev, &schid);
1260	1153	DBF_EVENT("qallocate:%4x", schid.sch_no);
1261	1154
1262		- if ((init_data->no_input_qs && !init_data->input_handler) \|\|
1263		- (init_data->no_output_qs && !init_data->output_handler))
1264		- return -EINVAL;
1265		-
1266		- if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) \|\|
1267		- (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ))
1268		- return -EINVAL;
1269		-
1270		- if ((!init_data->input_sbal_addr_array) \|\|
1271		- (!init_data->output_sbal_addr_array))
	1155	+ if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ \|\|
	1156	+ no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)
1272	1157	return -EINVAL;
1273	1158
1274	1159	/* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
1275	1160	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL \| GFP_DMA);
1276	1161	if (!irq_ptr)
1277		- goto out_err;
	1162	+ return -ENOMEM;
1278	1163
	1164	+ irq_ptr->cdev = cdev;
1279	1165	mutex_init(&irq_ptr->setup_mutex);
1280		- if (qdio_allocate_dbf(init_data, irq_ptr))
1281		- goto out_rel;
	1166	+ if (qdio_allocate_dbf(irq_ptr))
	1167	+ goto err_dbf;
	1168	+
	1169	+ DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,
	1170	+ no_output_qs);
1282	1171
1283	1172	/*
1284	1173	* Allocate a page for the chsc calls in qdio_establish.
..	..	@@ -1288,24 +1177,30 @@
1288	1177	*/
1289	1178	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
1290	1179	if (!irq_ptr->chsc_page)
1291		- goto out_rel;
	1180	+ goto err_chsc;
1292	1181
1293	1182	/* qdr is used in ccw1.cda which is u32 */
1294	1183	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL \| GFP_DMA);
1295	1184	if (!irq_ptr->qdr)
1296		- goto out_rel;
	1185	+ goto err_qdr;
1297	1186
1298		- if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs,
1299		- init_data->no_output_qs))
1300		- goto out_rel;
	1187	+ rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);
	1188	+ if (rc)
	1189	+ goto err_queues;
1301	1190
1302		- init_data->cdev->private->qdio_data = irq_ptr;
	1191	+ INIT_LIST_HEAD(&irq_ptr->entry);
	1192	+ cdev->private->qdio_data = irq_ptr;
1303	1193	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1304	1194	return 0;
1305		-out_rel:
1306		- qdio_release_memory(irq_ptr);
1307		-out_err:
1308		- return -ENOMEM;
	1195	+
	1196	+err_queues:
	1197	+ free_page((unsigned long) irq_ptr->qdr);
	1198	+err_qdr:
	1199	+ free_page(irq_ptr->chsc_page);
	1200	+err_chsc:
	1201	+err_dbf:
	1202	+ free_page((unsigned long) irq_ptr);
	1203	+ return rc;
1309	1204	}
1310	1205	EXPORT_SYMBOL_GPL(qdio_allocate);
1311	1206
..	..	@@ -1319,6 +1214,8 @@
1319	1214
1320	1215	for_each_output_queue(irq_ptr, q, i) {
1321	1216	if (use_cq) {
	1217	+ if (multicast_outbound(q))
	1218	+ continue;
1322	1219	if (qdio_enable_async_operation(&q->u.out) < 0) {
1323	1220	use_cq = 0;
1324	1221	continue;
..	..	@@ -1329,33 +1226,62 @@
1329	1226	DBF_EVENT("use_cq:%d", use_cq);
1330	1227	}
1331	1228
	1229	+static void qdio_trace_init_data(struct qdio_irq *irq,
	1230	+ struct qdio_initialize *data)
	1231	+{
	1232	+ DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format);
	1233	+ DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format);
	1234	+ DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR);
	1235	+ DBF_DEV_HEX(irq, &data->input_slib_elements, sizeof(void *), DBF_ERR);
	1236	+ DBF_DEV_HEX(irq, &data->output_slib_elements, sizeof(void *), DBF_ERR);
	1237	+ DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs,
	1238	+ data->no_output_qs);
	1239	+ DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR);
	1240	+ DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR);
	1241	+ DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR);
	1242	+ DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR);
	1243	+ DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *),
	1244	+ DBF_ERR);
	1245	+}
	1246	+
1332	1247	/**
1333	1248	* qdio_establish - establish queues on a qdio subchannel
	1249	+ * @cdev: associated ccw device
1334	1250	* @init_data: initialization data
1335	1251	*/
1336		-int qdio_establish(struct qdio_initialize *init_data)
	1252	+int qdio_establish(struct ccw_device *cdev,
	1253	+ struct qdio_initialize *init_data)
1337	1254	{
1338		- struct ccw_device *cdev = init_data->cdev;
	1255	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1339	1256	struct subchannel_id schid;
1340		- struct qdio_irq *irq_ptr;
	1257	+ long timeout;
1341	1258	int rc;
1342	1259
1343	1260	ccw_device_get_schid(cdev, &schid);
1344	1261	DBF_EVENT("qestablish:%4x", schid.sch_no);
1345	1262
1346		- irq_ptr = cdev->private->qdio_data;
1347	1263	if (!irq_ptr)
1348	1264	return -ENODEV;
1349	1265
	1266	+ if (init_data->no_input_qs > irq_ptr->max_input_qs \|\|
	1267	+ init_data->no_output_qs > irq_ptr->max_output_qs)
	1268	+ return -EINVAL;
	1269	+
	1270	+ if ((init_data->no_input_qs && !init_data->input_handler) \|\|
	1271	+ (init_data->no_output_qs && !init_data->output_handler))
	1272	+ return -EINVAL;
	1273	+
	1274	+ if (!init_data->input_sbal_addr_array \|\|
	1275	+ !init_data->output_sbal_addr_array)
	1276	+ return -EINVAL;
	1277	+
1350	1278	mutex_lock(&irq_ptr->setup_mutex);
1351		- qdio_setup_irq(init_data);
	1279	+ qdio_trace_init_data(irq_ptr, init_data);
	1280	+ qdio_setup_irq(irq_ptr, init_data);
1352	1281
1353	1282	rc = qdio_establish_thinint(irq_ptr);
1354		- if (rc) {
1355		- mutex_unlock(&irq_ptr->setup_mutex);
1356		- qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1357		- return rc;
1358		- }
	1283	+ if (rc)
	1284	+ goto err_thinint;
1359	1285
1360	1286	/* establish q */
1361	1287	irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd;
..	..	@@ -1371,14 +1297,16 @@
1371	1297	if (rc) {
1372	1298	DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1373	1299	DBF_ERROR("rc:%4x", rc);
1374		- mutex_unlock(&irq_ptr->setup_mutex);
1375		- qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1376		- return rc;
	1300	+ goto err_ccw_start;
1377	1301	}
1378	1302
1379		- wait_event_interruptible_timeout(cdev->private->wait_q,
1380		- irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED \|\|
1381		- irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
	1303	+ timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
	1304	+ irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED \|\|
	1305	+ irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
	1306	+ if (timeout <= 0) {
	1307	+ rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
	1308	+ goto err_ccw_timeout;
	1309	+ }
1382	1310
1383	1311	if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1384	1312	mutex_unlock(&irq_ptr->setup_mutex);
..	..	@@ -1394,9 +1322,19 @@
1394	1322	qdio_init_buf_states(irq_ptr);
1395	1323
1396	1324	mutex_unlock(&irq_ptr->setup_mutex);
1397		- qdio_print_subchannel_info(irq_ptr, cdev);
1398		- qdio_setup_debug_entries(irq_ptr, cdev);
	1325	+ qdio_print_subchannel_info(irq_ptr);
	1326	+ qdio_setup_debug_entries(irq_ptr);
1399	1327	return 0;
	1328	+
	1329	+err_ccw_timeout:
	1330	+ qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR);
	1331	+err_ccw_start:
	1332	+ qdio_shutdown_thinint(irq_ptr);
	1333	+err_thinint:
	1334	+ qdio_shutdown_irq(irq_ptr);
	1335	+ qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
	1336	+ mutex_unlock(&irq_ptr->setup_mutex);
	1337	+ return rc;
1400	1338	}
1401	1339	EXPORT_SYMBOL_GPL(qdio_establish);
1402	1340
..	..	@@ -1406,14 +1344,13 @@
1406	1344	*/
1407	1345	int qdio_activate(struct ccw_device *cdev)
1408	1346	{
	1347	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1409	1348	struct subchannel_id schid;
1410		- struct qdio_irq *irq_ptr;
1411	1349	int rc;
1412	1350
1413	1351	ccw_device_get_schid(cdev, &schid);
1414	1352	DBF_EVENT("qactivate:%4x", schid.sch_no);
1415	1353
1416		- irq_ptr = cdev->private->qdio_data;
1417	1354	if (!irq_ptr)
1418	1355	return -ENODEV;
1419	1356
..	..	@@ -1441,7 +1378,7 @@
1441	1378	}
1442	1379
1443	1380	if (is_thinint_irq(irq_ptr))
1444		- tiqdio_add_input_queues(irq_ptr);
	1381	+ tiqdio_add_device(irq_ptr);
1445	1382
1446	1383	/* wait for subchannel to become active */
1447	1384	msleep(5);
..	..	@@ -1461,25 +1398,6 @@
1461	1398	}
1462	1399	EXPORT_SYMBOL_GPL(qdio_activate);
1463	1400
1464		-static inline int buf_in_between(int bufnr, int start, int count)
1465		-{
1466		- int end = add_buf(start, count);
1467		-
1468		- if (end > start) {
1469		- if (bufnr >= start && bufnr < end)
1470		- return 1;
1471		- else
1472		- return 0;
1473		- }
1474		-
1475		- /* wrap-around case */
1476		- if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) \|\|
1477		- (bufnr < end))
1478		- return 1;
1479		- else
1480		- return 0;
1481		-}
1482		-
1483	1401	/**
1484	1402	* handle_inbound - reset processed input buffers
1485	1403	* @q: queue containing the buffers
..	..	@@ -1490,38 +1408,18 @@
1490	1408	static int handle_inbound(struct qdio_q *q, unsigned int callflags,
1491	1409	int bufnr, int count)
1492	1410	{
1493		- int diff;
	1411	+ int overlap;
1494	1412
1495	1413	qperf_inc(q, inbound_call);
1496	1414
1497		- if (!q->u.in.polling)
1498		- goto set;
1499		-
1500		- /* protect against stop polling setting an ACK for an emptied slsb */
1501		- if (count == QDIO_MAX_BUFFERS_PER_Q) {
1502		- /* overwriting everything, just delete polling status */
1503		- q->u.in.polling = 0;
1504		- q->u.in.ack_count = 0;
1505		- goto set;
1506		- } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
1507		- if (is_qebsm(q)) {
1508		- /* partial overwrite, just update ack_start */
1509		- diff = add_buf(bufnr, count);
1510		- diff = sub_buf(diff, q->u.in.ack_start);
1511		- q->u.in.ack_count -= diff;
1512		- if (q->u.in.ack_count <= 0) {
1513		- q->u.in.polling = 0;
1514		- q->u.in.ack_count = 0;
1515		- goto set;
1516		- }
1517		- q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
1518		- }
1519		- else
1520		- /* the only ACK will be deleted, so stop polling */
1521		- q->u.in.polling = 0;
	1415	+ /* If any processed SBALs are returned to HW, adjust our tracking: */
	1416	+ overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),
	1417	+ q->u.in.batch_count);
	1418	+ if (overlap > 0) {
	1419	+ q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);
	1420	+ q->u.in.batch_count -= overlap;
1522	1421	}
1523	1422
1524		-set:
1525	1423	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1526	1424	atomic_add(count, &q->nr_buf_used);
1527	1425
..	..	@@ -1539,8 +1437,9 @@
1539	1437	* @count: how many buffers are filled
1540	1438	*/
1541	1439	static int handle_outbound(struct qdio_q *q, unsigned int callflags,
1542		- int bufnr, int count)
	1440	+ unsigned int bufnr, unsigned int count)
1543	1441	{
	1442	+ const unsigned int scan_threshold = q->irq_ptr->scan_threshold;
1544	1443	unsigned char state = 0;
1545	1444	int used, rc = 0;
1546	1445
..	..	@@ -1561,12 +1460,10 @@
1561	1460	if (queue_type(q) == QDIO_IQDIO_QFMT) {
1562	1461	unsigned long phys_aob = 0;
1563	1462
1564		- /* One SIGA-W per buffer required for unicast HSI */
1565		- WARN_ON_ONCE(count > 1 && !multicast_outbound(q));
	1463	+ if (q->u.out.use_cq && count == 1)
	1464	+ phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);
1566	1465
1567		- phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);
1568		-
1569		- rc = qdio_kick_outbound_q(q, phys_aob);
	1466	+ rc = qdio_kick_outbound_q(q, count, phys_aob);
1570	1467	} else if (need_siga_sync(q)) {
1571	1468	rc = qdio_siga_sync_q(q);
1572	1469	} else if (count < QDIO_MAX_BUFFERS_PER_Q &&
..	..	@@ -1575,11 +1472,15 @@
1575	1472	/* The previous buffer is not processed yet, tack on. */
1576	1473	qperf_inc(q, fast_requeue);
1577	1474	} else {
1578		- rc = qdio_kick_outbound_q(q, 0);
	1475	+ rc = qdio_kick_outbound_q(q, count, 0);
1579	1476	}
1580	1477
	1478	+ /* Let drivers implement their own completion scanning: */
	1479	+ if (!scan_threshold)
	1480	+ return rc;
	1481	+
1581	1482	/* in case of SIGA errors we must process the error immediately */
1582		- if (used >= q->u.out.scan_threshold \|\| rc)
	1483	+ if (used >= scan_threshold \|\| rc)
1583	1484	qdio_tasklet_schedule(q);
1584	1485	else
1585	1486	/* free the SBALs in case of no further traffic */
..	..	@@ -1600,12 +1501,11 @@
1600	1501	int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
1601	1502	int q_nr, unsigned int bufnr, unsigned int count)
1602	1503	{
1603		- struct qdio_irq *irq_ptr;
	1504	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1604	1505
1605	1506	if (bufnr >= QDIO_MAX_BUFFERS_PER_Q \|\| count > QDIO_MAX_BUFFERS_PER_Q)
1606	1507	return -EINVAL;
1607	1508
1608		- irq_ptr = cdev->private->qdio_data;
1609	1509	if (!irq_ptr)
1610	1510	return -ENODEV;
1611	1511
..	..	@@ -1627,26 +1527,26 @@
1627	1527	EXPORT_SYMBOL_GPL(do_QDIO);
1628	1528
1629	1529	/**
1630		- * qdio_start_irq - process input buffers
	1530	+ * qdio_start_irq - enable interrupt processing for the device
1631	1531	* @cdev: associated ccw_device for the qdio subchannel
1632		- * @nr: input queue number
1633	1532	*
1634	1533	* Return codes
1635	1534	* 0 - success
1636	1535	* 1 - irqs not started since new data is available
1637	1536	*/
1638		-int qdio_start_irq(struct ccw_device *cdev, int nr)
	1537	+int qdio_start_irq(struct ccw_device *cdev)
1639	1538	{
1640	1539	struct qdio_q *q;
1641	1540	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
	1541	+ unsigned int i;
1642	1542
1643	1543	if (!irq_ptr)
1644	1544	return -ENODEV;
1645		- q = irq_ptr->input_qs[nr];
1646	1545
1647		- clear_nonshared_ind(irq_ptr);
1648		- qdio_stop_polling(q);
1649		- clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
	1546	+ for_each_input_queue(irq_ptr, q, i)
	1547	+ qdio_stop_polling(q);
	1548	+
	1549	+ clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state);
1650	1550
1651	1551	/*
1652	1552	* We need to check again to not lose initiative after
..	..	@@ -1654,19 +1554,60 @@
1654	1554	*/
1655	1555	if (test_nonshared_ind(irq_ptr))
1656	1556	goto rescan;
1657		- if (!qdio_inbound_q_done(q))
1658		- goto rescan;
	1557	+
	1558	+ for_each_input_queue(irq_ptr, q, i) {
	1559	+ if (!qdio_inbound_q_done(q, q->first_to_check))
	1560	+ goto rescan;
	1561	+ }
	1562	+
1659	1563	return 0;
1660	1564
1661	1565	rescan:
1662		- if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1663		- &q->u.in.queue_irq_state))
	1566	+ if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1664	1567	return 0;
1665	1568	else
1666	1569	return 1;
1667	1570
1668	1571	}
1669	1572	EXPORT_SYMBOL(qdio_start_irq);
	1573	+
	1574	+static int __qdio_inspect_queue(struct qdio_q q, unsigned int bufnr,
	1575	+ unsigned int *error)
	1576	+{
	1577	+ unsigned int start = q->first_to_check;
	1578	+ int count;
	1579	+
	1580	+ count = q->is_input_q ? qdio_inbound_q_moved(q, start) :
	1581	+ qdio_outbound_q_moved(q, start);
	1582	+ if (count == 0)
	1583	+ return 0;
	1584	+
	1585	+ *bufnr = start;
	1586	+ *error = q->qdio_error;
	1587	+
	1588	+ /* for the next time */
	1589	+ q->first_to_check = add_buf(start, count);
	1590	+ q->qdio_error = 0;
	1591	+
	1592	+ return count;
	1593	+}
	1594	+
	1595	+int qdio_inspect_queue(struct ccw_device *cdev, unsigned int nr, bool is_input,
	1596	+ unsigned int bufnr, unsigned int error)
	1597	+{
	1598	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
	1599	+ struct qdio_q *q;
	1600	+
	1601	+ if (!irq_ptr)
	1602	+ return -ENODEV;
	1603	+ q = is_input ? irq_ptr->input_qs[nr] : irq_ptr->output_qs[nr];
	1604	+
	1605	+ if (need_siga_sync(q))
	1606	+ qdio_siga_sync_q(q);
	1607	+
	1608	+ return __qdio_inspect_queue(q, bufnr, error);
	1609	+}
	1610	+EXPORT_SYMBOL_GPL(qdio_inspect_queue);
1670	1611
1671	1612	/**
1672	1613	* qdio_get_next_buffers - process input buffers
..	..	@@ -1684,7 +1625,6 @@
1684	1625	int *error)
1685	1626	{
1686	1627	struct qdio_q *q;
1687		- int start, end;
1688	1628	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1689	1629
1690	1630	if (!irq_ptr)
..	..	@@ -1698,141 +1638,37 @@
1698	1638	if (need_siga_sync(q))
1699	1639	qdio_sync_queues(q);
1700	1640
1701		- /* check the PCI capable outbound queues. */
1702		- qdio_check_outbound_after_thinint(q);
1703		-
1704		- if (!qdio_inbound_q_moved(q))
1705		- return 0;
	1641	+ qdio_check_outbound_pci_queues(irq_ptr);
1706	1642
1707	1643	/* Note: upper-layer MUST stop processing immediately here ... */
1708	1644	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
1709	1645	return -EIO;
1710	1646
1711		- start = q->first_to_kick;
1712		- end = q->first_to_check;
1713		- *bufnr = start;
1714		- *error = q->qdio_error;
1715		-
1716		- /* for the next time */
1717		- q->first_to_kick = end;
1718		- q->qdio_error = 0;
1719		- return sub_buf(end, start);
	1647	+ return __qdio_inspect_queue(q, bufnr, error);
1720	1648	}
1721	1649	EXPORT_SYMBOL(qdio_get_next_buffers);
1722	1650
1723	1651	/**
1724	1652	* qdio_stop_irq - disable interrupt processing for the device
1725	1653	* @cdev: associated ccw_device for the qdio subchannel
1726		- * @nr: input queue number
1727	1654	*
1728	1655	* Return codes
1729	1656	* 0 - interrupts were already disabled
1730	1657	* 1 - interrupts successfully disabled
1731	1658	*/
1732		-int qdio_stop_irq(struct ccw_device *cdev, int nr)
	1659	+int qdio_stop_irq(struct ccw_device *cdev)
1733	1660	{
1734		- struct qdio_q *q;
1735	1661	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1736	1662
1737	1663	if (!irq_ptr)
1738	1664	return -ENODEV;
1739		- q = irq_ptr->input_qs[nr];
1740	1665
1741		- if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1742		- &q->u.in.queue_irq_state))
	1666	+ if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1743	1667	return 0;
1744	1668	else
1745	1669	return 1;
1746	1670	}
1747	1671	EXPORT_SYMBOL(qdio_stop_irq);
1748		-
1749		-/**
1750		- * qdio_pnso_brinfo() - perform network subchannel op #0 - bridge info.
1751		- * @schid: Subchannel ID.
1752		- * @cnc: Boolean Change-Notification Control
1753		- * @response: Response code will be stored at this address
1754		- * @cb: Callback function will be executed for each element
1755		- * of the address list
1756		- * @priv: Pointer to pass to the callback function.
1757		- *
1758		- * Performs "Store-network-bridging-information list" operation and calls
1759		- * the callback function for every entry in the list. If "change-
1760		- * notification-control" is set, further changes in the address list
1761		- * will be reported via the IPA command.
1762		- */
1763		-int qdio_pnso_brinfo(struct subchannel_id schid,
1764		- int cnc, u16 *response,
1765		- void (cb)(void priv, enum qdio_brinfo_entry_type type,
1766		- void *entry),
1767		- void *priv)
1768		-{
1769		- struct chsc_pnso_area *rr;
1770		- int rc;
1771		- u32 prev_instance = 0;
1772		- int isfirstblock = 1;
1773		- int i, size, elems;
1774		-
1775		- rr = (struct chsc_pnso_area *)get_zeroed_page(GFP_KERNEL);
1776		- if (rr == NULL)
1777		- return -ENOMEM;
1778		- do {
1779		- /* on the first iteration, naihdr.resume_token will be zero */
1780		- rc = chsc_pnso_brinfo(schid, rr, rr->naihdr.resume_token, cnc);
1781		- if (rc != 0 && rc != -EBUSY)
1782		- goto out;
1783		- if (rr->response.code != 1) {
1784		- rc = -EIO;
1785		- continue;
1786		- } else
1787		- rc = 0;
1788		-
1789		- if (cb == NULL)
1790		- continue;
1791		-
1792		- size = rr->naihdr.naids;
1793		- elems = (rr->response.length -
1794		- sizeof(struct chsc_header) -
1795		- sizeof(struct chsc_brinfo_naihdr)) /
1796		- size;
1797		-
1798		- if (!isfirstblock && (rr->naihdr.instance != prev_instance)) {
1799		- /* Inform the caller that they need to scrap */
1800		- /* the data that was already reported via cb */
1801		- rc = -EAGAIN;
1802		- break;
1803		- }
1804		- isfirstblock = 0;
1805		- prev_instance = rr->naihdr.instance;
1806		- for (i = 0; i < elems; i++)
1807		- switch (size) {
1808		- case sizeof(struct qdio_brinfo_entry_l3_ipv6):
1809		- (*cb)(priv, l3_ipv6_addr,
1810		- &rr->entries.l3_ipv6[i]);
1811		- break;
1812		- case sizeof(struct qdio_brinfo_entry_l3_ipv4):
1813		- (*cb)(priv, l3_ipv4_addr,
1814		- &rr->entries.l3_ipv4[i]);
1815		- break;
1816		- case sizeof(struct qdio_brinfo_entry_l2):
1817		- (*cb)(priv, l2_addr_lnid,
1818		- &rr->entries.l2[i]);
1819		- break;
1820		- default:
1821		- WARN_ON_ONCE(1);
1822		- rc = -EIO;
1823		- goto out;
1824		- }
1825		- } while (rr->response.code == 0x0107 \|\| /* channel busy */
1826		- (rr->response.code == 1 && /* list stored */
1827		- /* resume token is non-zero => list incomplete */
1828		- (rr->naihdr.resume_token.t1 \|\| rr->naihdr.resume_token.t2)));
1829		- (*response) = rr->response.code;
1830		-
1831		-out:
1832		- free_page((unsigned long)rr);
1833		- return rc;
1834		-}
1835		-EXPORT_SYMBOL_GPL(qdio_pnso_brinfo);
1836	1672
1837	1673	static int __init init_QDIO(void)
1838	1674	{
..	..	@@ -1844,16 +1680,11 @@
1844	1680	rc = qdio_setup_init();
1845	1681	if (rc)
1846	1682	goto out_debug;
1847		- rc = tiqdio_allocate_memory();
	1683	+ rc = qdio_thinint_init();
1848	1684	if (rc)
1849	1685	goto out_cache;
1850		- rc = tiqdio_register_thinints();
1851		- if (rc)
1852		- goto out_ti;
1853	1686	return 0;
1854	1687
1855		-out_ti:
1856		- tiqdio_free_memory();
1857	1688	out_cache:
1858	1689	qdio_setup_exit();
1859	1690	out_debug:
..	..	@@ -1863,8 +1694,7 @@
1863	1694
1864	1695	static void __exit exit_QDIO(void)
1865	1696	{
1866		- tiqdio_unregister_thinints();
1867		- tiqdio_free_memory();
	1697	+ qdio_thinint_exit();
1868	1698	qdio_setup_exit();
1869	1699	qdio_debug_exit();
1870	1700	}