~hc/RK356X_SDK_RELEASE.git

..	..	@@ -131,7 +131,7 @@
131	131	case 96:
132	132	/* not all buffers processed */
133	133	qperf_inc(q, eqbs_partial);
134		- DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS part:%02x",
	134	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",
135	135	tmp_count);
136	136	return count - tmp_count;
137	137	case 97:
..	..	@@ -143,7 +143,7 @@
143	143	DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
144	144	DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
145	145	q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
146		- q->first_to_kick, count, q->irq_ptr->int_parm);
	146	+ q->first_to_check, count, q->irq_ptr->int_parm);
147	147	return 0;
148	148	}
149	149	}
..	..	@@ -191,7 +191,7 @@
191	191	DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
192	192	DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
193	193	q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
194		- q->first_to_kick, count, q->irq_ptr->int_parm);
	194	+ q->first_to_check, count, q->irq_ptr->int_parm);
195	195	return 0;
196	196	}
197	197	}
..	..	@@ -205,17 +205,22 @@
205	205	int auto_ack, int merge_pending)
206	206	{
207	207	unsigned char __state = 0;
208		- int i;
	208	+ int i = 1;
209	209
210	210	if (is_qebsm(q))
211	211	return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
212	212
213	213	/* get initial state: */
214	214	__state = q->slsb.val[bufnr];
	215	+
	216	+ /* Bail out early if there is no work on the queue: */
	217	+ if (__state & SLSB_OWNER_CU)
	218	+ goto out;
	219	+
215	220	if (merge_pending && __state == SLSB_P_OUTPUT_PENDING)
216	221	__state = SLSB_P_OUTPUT_EMPTY;
217	222
218		- for (i = 1; i < count; i++) {
	223	+ for (; i < count; i++) {
219	224	bufnr = next_buf(bufnr);
220	225
221	226	/* merge PENDING into EMPTY: */
..	..	@@ -228,6 +233,8 @@
228	233	if (q->slsb.val[bufnr] != __state)
229	234	break;
230	235	}
	236	+
	237	+out:
231	238	*state = __state;
232	239	return i;
233	240	}
..	..	@@ -247,10 +254,17 @@
247	254	if (is_qebsm(q))
248	255	return qdio_do_sqbs(q, state, bufnr, count);
249	256
	257	+ /* Ensure that all preceding changes to the SBALs are visible: */
	258	+ mb();
	259	+
250	260	for (i = 0; i < count; i++) {
251		- xchg(&q->slsb.val[bufnr], state);
	261	+ WRITE_ONCE(q->slsb.val[bufnr], state);
252	262	bufnr = next_buf(bufnr);
253	263	}
	264	+
	265	+ /* Make our SLSB changes visible: */
	266	+ mb();
	267	+
254	268	return count;
255	269	}
256	270
..	..	@@ -303,20 +317,19 @@
303	317	return qdio_siga_sync(q, q->mask, 0);
304	318	}
305	319
306		-static int qdio_siga_output(struct qdio_q q, unsigned int busy_bit,
307		- unsigned long aob)
	320	+static int qdio_siga_output(struct qdio_q *q, unsigned int count,
	321	+ unsigned int *busy_bit, unsigned long aob)
308	322	{
309	323	unsigned long schid = ((u32 ) &q->irq_ptr->schid);
310	324	unsigned int fc = QDIO_SIGA_WRITE;
311	325	u64 start_time = 0;
312	326	int retries = 0, cc;
313		- unsigned long laob = 0;
314	327
315		- WARN_ON_ONCE(aob && ((queue_type(q) != QDIO_IQDIO_QFMT) \|\|
316		- !q->u.out.use_cq));
317		- if (q->u.out.use_cq && aob != 0) {
318		- fc = QDIO_SIGA_WRITEQ;
319		- laob = aob;
	328	+ if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {
	329	+ if (count > 1)
	330	+ fc = QDIO_SIGA_WRITEM;
	331	+ else if (aob)
	332	+ fc = QDIO_SIGA_WRITEQ;
320	333	}
321	334
322	335	if (is_qebsm(q)) {
..	..	@@ -324,7 +337,7 @@
324	337	fc \|= QDIO_SIGA_QEBSM_FLAG;
325	338	}
326	339	again:
327		- cc = do_siga_output(schid, q->mask, busy_bit, fc, laob);
	340	+ cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);
328	341
329	342	/* hipersocket busy condition */
330	343	if (unlikely(*busy_bit)) {
..	..	@@ -371,7 +384,7 @@
371	384	static inline void qdio_sync_queues(struct qdio_q *q)
372	385	{
373	386	/* PCI capable outbound queues will also be scanned so sync them too */
374		- if (pci_out_supported(q))
	387	+ if (pci_out_supported(q->irq_ptr))
375	388	qdio_siga_sync_all(q);
376	389	else
377	390	qdio_siga_sync_q(q);
..	..	@@ -382,186 +395,125 @@
382	395	{
383	396	if (need_siga_sync(q))
384	397	qdio_siga_sync_q(q);
385		- return get_buf_states(q, bufnr, state, 1, 0, 0);
	398	+ return get_buf_state(q, bufnr, state, 0);
386	399	}
387	400
388	401	static inline void qdio_stop_polling(struct qdio_q *q)
389	402	{
390		- if (!q->u.in.polling)
	403	+ if (!q->u.in.batch_count)
391	404	return;
392	405
393		- q->u.in.polling = 0;
394	406	qperf_inc(q, stop_polling);
395	407
396	408	/* show the card that we are not polling anymore */
397		- if (is_qebsm(q)) {
398		- set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
399		- q->u.in.ack_count);
400		- q->u.in.ack_count = 0;
401		- } else
402		- set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
	409	+ set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,
	410	+ q->u.in.batch_count);
	411	+ q->u.in.batch_count = 0;
403	412	}
404	413
405	414	static inline void account_sbals(struct qdio_q *q, unsigned int count)
406	415	{
407		- int pos;
408		-
409	416	q->q_stats.nr_sbal_total += count;
410		- if (count == QDIO_MAX_BUFFERS_MASK) {
411		- q->q_stats.nr_sbals[7]++;
412		- return;
413		- }
414		- pos = ilog2(count);
415		- q->q_stats.nr_sbals[pos]++;
	417	+ q->q_stats.nr_sbals[ilog2(count)]++;
416	418	}
417	419
418		-static void process_buffer_error(struct qdio_q *q, int count)
	420	+static void process_buffer_error(struct qdio_q *q, unsigned int start,
	421	+ int count)
419	422	{
420		- unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT :
421		- SLSB_P_OUTPUT_NOT_INIT;
422		-
423	423	q->qdio_error = QDIO_ERROR_SLSB_STATE;
424	424
425	425	/* special handling for no target buffer empty */
426	426	if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&
427		- q->sbal[q->first_to_check]->element[15].sflags == 0x10) {
	427	+ q->sbal[start]->element[15].sflags == 0x10) {
428	428	qperf_inc(q, target_full);
429		- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
430		- q->first_to_check);
431		- goto set;
	429	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);
	430	+ return;
432	431	}
433	432
434	433	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
435	434	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
436		- DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
	435	+ DBF_ERROR("FTC:%3d C:%3d", start, count);
437	436	DBF_ERROR("F14:%2x F15:%2x",
438		- q->sbal[q->first_to_check]->element[14].sflags,
439		- q->sbal[q->first_to_check]->element[15].sflags);
440		-
441		-set:
442		- /*
443		- * Interrupts may be avoided as long as the error is present
444		- * so change the buffer state immediately to avoid starvation.
445		- */
446		- set_buf_states(q, q->first_to_check, state, count);
	437	+ q->sbal[start]->element[14].sflags,
	438	+ q->sbal[start]->element[15].sflags);
447	439	}
448	440
449		-static inline void inbound_primed(struct qdio_q *q, int count)
	441	+static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
	442	+ int count, bool auto_ack)
450	443	{
451		- int new;
	444	+ /* ACK the newest SBAL: */
	445	+ if (!auto_ack)
	446	+ set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);
452	447
453		- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, count);
454		-
455		- /* for QEBSM the ACK was already set by EQBS */
456		- if (is_qebsm(q)) {
457		- if (!q->u.in.polling) {
458		- q->u.in.polling = 1;
459		- q->u.in.ack_count = count;
460		- q->u.in.ack_start = q->first_to_check;
461		- return;
462		- }
463		-
464		- /* delete the previous ACK's */
465		- set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
466		- q->u.in.ack_count);
467		- q->u.in.ack_count = count;
468		- q->u.in.ack_start = q->first_to_check;
469		- return;
470		- }
471		-
472		- /*
473		- * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
474		- * or by the next inbound run.
475		- */
476		- new = add_buf(q->first_to_check, count - 1);
477		- if (q->u.in.polling) {
478		- /* reset the previous ACK but first set the new one */
479		- set_buf_state(q, new, SLSB_P_INPUT_ACK);
480		- set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
481		- } else {
482		- q->u.in.polling = 1;
483		- set_buf_state(q, new, SLSB_P_INPUT_ACK);
484		- }
485		-
486		- q->u.in.ack_start = new;
487		- count--;
488		- if (!count)
489		- return;
490		- /* need to change ALL buffers to get more interrupts */
491		- set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
	448	+ if (!q->u.in.batch_count)
	449	+ q->u.in.batch_start = start;
	450	+ q->u.in.batch_count += count;
492	451	}
493	452
494		-static int get_inbound_buffer_frontier(struct qdio_q *q)
	453	+static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start)
495	454	{
496	455	unsigned char state = 0;
497	456	int count;
498	457
499	458	q->timestamp = get_tod_clock_fast();
500	459
501		- /*
502		- * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
503		- * would return 0.
504		- */
505		- count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
	460	+ count = atomic_read(&q->nr_buf_used);
506	461	if (!count)
507		- goto out;
	462	+ return 0;
508	463
509	464	/*
510	465	* No siga sync here, as a PCI or we after a thin interrupt
511	466	* already sync'ed the queues.
512	467	*/
513		- count = get_buf_states(q, q->first_to_check, &state, count, 1, 0);
	468	+ count = get_buf_states(q, start, &state, count, 1, 0);
514	469	if (!count)
515		- goto out;
	470	+ return 0;
516	471
517	472	switch (state) {
518	473	case SLSB_P_INPUT_PRIMED:
519		- inbound_primed(q, count);
520		- q->first_to_check = add_buf(q->first_to_check, count);
	474	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
	475	+ count);
	476	+
	477	+ inbound_handle_work(q, start, count, is_qebsm(q));
521	478	if (atomic_sub_return(count, &q->nr_buf_used) == 0)
522	479	qperf_inc(q, inbound_queue_full);
523	480	if (q->irq_ptr->perf_stat_enabled)
524	481	account_sbals(q, count);
525		- break;
	482	+ return count;
526	483	case SLSB_P_INPUT_ERROR:
527		- process_buffer_error(q, count);
528		- q->first_to_check = add_buf(q->first_to_check, count);
	484	+ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
	485	+ count);
	486	+
	487	+ process_buffer_error(q, start, count);
	488	+ inbound_handle_work(q, start, count, false);
529	489	if (atomic_sub_return(count, &q->nr_buf_used) == 0)
530	490	qperf_inc(q, inbound_queue_full);
531	491	if (q->irq_ptr->perf_stat_enabled)
532	492	account_sbals_error(q, count);
533		- break;
	493	+ return count;
534	494	case SLSB_CU_INPUT_EMPTY:
535		- case SLSB_P_INPUT_NOT_INIT:
536		- case SLSB_P_INPUT_ACK:
537	495	if (q->irq_ptr->perf_stat_enabled)
538	496	q->q_stats.nr_sbal_nop++;
539	497	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",
540		- q->nr, q->first_to_check);
541		- break;
542		- default:
543		- WARN_ON_ONCE(1);
544		- }
545		-out:
546		- return q->first_to_check;
547		-}
548		-
549		-static int qdio_inbound_q_moved(struct qdio_q *q)
550		-{
551		- int bufnr;
552		-
553		- bufnr = get_inbound_buffer_frontier(q);
554		-
555		- if (bufnr != q->last_move) {
556		- q->last_move = bufnr;
557		- if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
558		- q->u.in.timestamp = get_tod_clock();
559		- return 1;
560		- } else
	498	+ q->nr, start);
561	499	return 0;
	500	+ case SLSB_P_INPUT_NOT_INIT:
	501	+ case SLSB_P_INPUT_ACK:
	502	+ /* We should never see this state, throw a WARN: */
	503	+ default:
	504	+ dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
	505	+ "found state %#x at index %u on queue %u\n",
	506	+ state, start, q->nr);
	507	+ return 0;
	508	+ }
562	509	}
563	510
564		-static inline int qdio_inbound_q_done(struct qdio_q *q)
	511	+static int qdio_inbound_q_moved(struct qdio_q *q, unsigned int start)
	512	+{
	513	+ return get_inbound_buffer_frontier(q, start);
	514	+}
	515	+
	516	+static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)
565	517	{
566	518	unsigned char state = 0;
567	519
..	..	@@ -570,59 +522,13 @@
570	522
571	523	if (need_siga_sync(q))
572	524	qdio_siga_sync_q(q);
573		- get_buf_state(q, q->first_to_check, &state, 0);
	525	+ get_buf_state(q, start, &state, 0);
574	526
575	527	if (state == SLSB_P_INPUT_PRIMED \|\| state == SLSB_P_INPUT_ERROR)
576	528	/* more work coming */
577	529	return 0;
578	530
579		- if (is_thinint_irq(q->irq_ptr))
580		- return 1;
581		-
582		- /* don't poll under z/VM */
583		- if (MACHINE_IS_VM)
584		- return 1;
585		-
586		- /*
587		- * At this point we know, that inbound first_to_check
588		- * has (probably) not moved (see qdio_inbound_processing).
589		- */
590		- if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
591		- DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
592		- q->first_to_check);
593		- return 1;
594		- } else
595		- return 0;
596		-}
597		-
598		-static inline int contains_aobs(struct qdio_q *q)
599		-{
600		- return !q->is_input_q && q->u.out.use_cq;
601		-}
602		-
603		-static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count)
604		-{
605		- unsigned char state = 0;
606		- int j, b = start;
607		-
608		- if (!contains_aobs(q))
609		- return;
610		-
611		- for (j = 0; j < count; ++j) {
612		- get_buf_state(q, b, &state, 0);
613		- if (state == SLSB_P_OUTPUT_PENDING) {
614		- struct qaob *aob = q->u.out.aobs[b];
615		- if (aob == NULL)
616		- continue;
617		-
618		- q->u.out.sbal_state[b].flags \|=
619		- QDIO_OUTBUF_STATE_FLAG_PENDING;
620		- q->u.out.aobs[b] = NULL;
621		- } else if (state == SLSB_P_OUTPUT_EMPTY) {
622		- q->u.out.sbal_state[b].aob = NULL;
623		- }
624		- b = next_buf(b);
625		- }
	531	+ return 1;
626	532	}
627	533
628	534	static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q,
..	..	@@ -630,15 +536,11 @@
630	536	{
631	537	unsigned long phys_aob = 0;
632	538
633		- if (!q->use_cq)
634		- return 0;
635		-
636	539	if (!q->aobs[bufnr]) {
637	540	struct qaob *aob = qdio_allocate_aob();
638	541	q->aobs[bufnr] = aob;
639	542	}
640	543	if (q->aobs[bufnr]) {
641		- q->sbal_state[bufnr].aob = q->aobs[bufnr];
642	544	q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user;
643	545	phys_aob = virt_to_phys(q->aobs[bufnr]);
644	546	WARN_ON_ONCE(phys_aob & 0xFF);
..	..	@@ -648,16 +550,11 @@
648	550	return phys_aob;
649	551	}
650	552
651		-static void qdio_kick_handler(struct qdio_q *q)
	553	+static void qdio_kick_handler(struct qdio_q *q, unsigned int start,
	554	+ unsigned int count)
652	555	{
653		- int start = q->first_to_kick;
654		- int end = q->first_to_check;
655		- int count;
656		-
657	556	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
658	557	return;
659		-
660		- count = sub_buf(end, start);
661	558
662	559	if (q->is_input_q) {
663	560	qperf_inc(q, inbound_handler);
..	..	@@ -668,13 +565,10 @@
668	565	start, count);
669	566	}
670	567
671		- qdio_handle_aobs(q, start, count);
672		-
673	568	q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
674	569	q->irq_ptr->int_parm);
675	570
676	571	/* for the next time */
677		- q->first_to_kick = end;
678	572	q->qdio_error = 0;
679	573	}
680	574
..	..	@@ -689,14 +583,20 @@
689	583
690	584	static void __qdio_inbound_processing(struct qdio_q *q)
691	585	{
	586	+ unsigned int start = q->first_to_check;
	587	+ int count;
	588	+
692	589	qperf_inc(q, tasklet_inbound);
693	590
694		- if (!qdio_inbound_q_moved(q))
	591	+ count = qdio_inbound_q_moved(q, start);
	592	+ if (count == 0)
695	593	return;
696	594
697		- qdio_kick_handler(q);
	595	+ qdio_kick_handler(q, start, count);
	596	+ start = add_buf(start, count);
	597	+ q->first_to_check = start;
698	598
699		- if (!qdio_inbound_q_done(q)) {
	599	+ if (!qdio_inbound_q_done(q, start)) {
700	600	/* means poll time is not yet over */
701	601	qperf_inc(q, tasklet_inbound_resched);
702	602	if (!qdio_tasklet_schedule(q))
..	..	@@ -708,7 +608,7 @@
708	608	* We need to check again to not lose initiative after
709	609	* resetting the ACK state.
710	610	*/
711		- if (!qdio_inbound_q_done(q)) {
	611	+ if (!qdio_inbound_q_done(q, start)) {
712	612	qperf_inc(q, tasklet_inbound_resched2);
713	613	qdio_tasklet_schedule(q);
714	614	}
..	..	@@ -720,7 +620,20 @@
720	620	__qdio_inbound_processing(q);
721	621	}
722	622
723		-static int get_outbound_buffer_frontier(struct qdio_q *q)
	623	+static void qdio_check_pending(struct qdio_q *q, unsigned int index)
	624	+{
	625	+ unsigned char state;
	626	+
	627	+ if (get_buf_state(q, index, &state, 0) > 0 &&
	628	+ state == SLSB_P_OUTPUT_PENDING &&
	629	+ q->u.out.aobs[index]) {
	630	+ q->u.out.sbal_state[index].flags \|=
	631	+ QDIO_OUTBUF_STATE_FLAG_PENDING;
	632	+ q->u.out.aobs[index] = NULL;
	633	+ }
	634	+}
	635	+
	636	+static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start)
724	637	{
725	638	unsigned char state = 0;
726	639	int count;
..	..	@@ -729,23 +642,18 @@
729	642
730	643	if (need_siga_sync(q))
731	644	if (((queue_type(q) != QDIO_IQDIO_QFMT) &&
732		- !pci_out_supported(q)) \|\|
	645	+ !pci_out_supported(q->irq_ptr)) \|\|
733	646	(queue_type(q) == QDIO_IQDIO_QFMT &&
734	647	multicast_outbound(q)))
735	648	qdio_siga_sync_q(q);
736	649
737		- /*
738		- * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
739		- * would return 0.
740		- */
741		- count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
	650	+ count = atomic_read(&q->nr_buf_used);
742	651	if (!count)
743		- goto out;
	652	+ return 0;
744	653
745		- count = get_buf_states(q, q->first_to_check, &state, count, 0,
746		- q->u.out.use_cq);
	654	+ count = get_buf_states(q, start, &state, count, 0, q->u.out.use_cq);
747	655	if (!count)
748		- goto out;
	656	+ return 0;
749	657
750	658	switch (state) {
751	659	case SLSB_P_OUTPUT_EMPTY:
..	..	@@ -755,34 +663,32 @@
755	663	"out empty:%1d %02x", q->nr, count);
756	664
757	665	atomic_sub(count, &q->nr_buf_used);
758		- q->first_to_check = add_buf(q->first_to_check, count);
759	666	if (q->irq_ptr->perf_stat_enabled)
760	667	account_sbals(q, count);
761		-
762		- break;
	668	+ return count;
763	669	case SLSB_P_OUTPUT_ERROR:
764		- process_buffer_error(q, count);
765		- q->first_to_check = add_buf(q->first_to_check, count);
	670	+ process_buffer_error(q, start, count);
766	671	atomic_sub(count, &q->nr_buf_used);
767	672	if (q->irq_ptr->perf_stat_enabled)
768	673	account_sbals_error(q, count);
769		- break;
	674	+ return count;
770	675	case SLSB_CU_OUTPUT_PRIMED:
771	676	/* the adapter has not fetched the output yet */
772	677	if (q->irq_ptr->perf_stat_enabled)
773	678	q->q_stats.nr_sbal_nop++;
774	679	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
775	680	q->nr);
776		- break;
777		- case SLSB_P_OUTPUT_NOT_INIT:
	681	+ return 0;
778	682	case SLSB_P_OUTPUT_HALTED:
779		- break;
	683	+ return 0;
	684	+ case SLSB_P_OUTPUT_NOT_INIT:
	685	+ /* We should never see this state, throw a WARN: */
780	686	default:
781		- WARN_ON_ONCE(1);
	687	+ dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
	688	+ "found state %#x at index %u on queue %u\n",
	689	+ state, start, q->nr);
	690	+ return 0;
782	691	}
783		-
784		-out:
785		- return q->first_to_check;
786	692	}
787	693
788	694	/* all buffers processed? */
..	..	@@ -791,21 +697,28 @@
791	697	return atomic_read(&q->nr_buf_used) == 0;
792	698	}
793	699
794		-static inline int qdio_outbound_q_moved(struct qdio_q *q)
	700	+static inline int qdio_outbound_q_moved(struct qdio_q *q, unsigned int start)
795	701	{
796		- int bufnr;
	702	+ int count;
797	703
798		- bufnr = get_outbound_buffer_frontier(q);
	704	+ count = get_outbound_buffer_frontier(q, start);
799	705
800		- if (bufnr != q->last_move) {
801		- q->last_move = bufnr;
	706	+ if (count) {
802	707	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
803		- return 1;
804		- } else
805		- return 0;
	708	+
	709	+ if (q->u.out.use_cq) {
	710	+ unsigned int i;
	711	+
	712	+ for (i = 0; i < count; i++)
	713	+ qdio_check_pending(q, QDIO_BUFNR(start + i));
	714	+ }
	715	+ }
	716	+
	717	+ return count;
806	718	}
807	719
808		-static int qdio_kick_outbound_q(struct qdio_q *q, unsigned long aob)
	720	+static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,
	721	+ unsigned long aob)
809	722	{
810	723	int retries = 0, cc;
811	724	unsigned int busy_bit;
..	..	@@ -817,7 +730,7 @@
817	730	retry:
818	731	qperf_inc(q, siga_write);
819	732
820		- cc = qdio_siga_output(q, &busy_bit, aob);
	733	+ cc = qdio_siga_output(q, count, &busy_bit, aob);
821	734	switch (cc) {
822	735	case 0:
823	736	break;
..	..	@@ -849,15 +762,21 @@
849	762
850	763	static void __qdio_outbound_processing(struct qdio_q *q)
851	764	{
	765	+ unsigned int start = q->first_to_check;
	766	+ int count;
	767	+
852	768	qperf_inc(q, tasklet_outbound);
853	769	WARN_ON_ONCE(atomic_read(&q->nr_buf_used) < 0);
854	770
855		- if (qdio_outbound_q_moved(q))
856		- qdio_kick_handler(q);
	771	+ count = qdio_outbound_q_moved(q, start);
	772	+ if (count) {
	773	+ q->first_to_check = add_buf(start, count);
	774	+ qdio_kick_handler(q, start, count);
	775	+ }
857	776
858		- if (queue_type(q) == QDIO_ZFCP_QFMT)
859		- if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
860		- goto sched;
	777	+ if (queue_type(q) == QDIO_ZFCP_QFMT && !pci_out_supported(q->irq_ptr) &&
	778	+ !qdio_outbound_q_done(q))
	779	+ goto sched;
861	780
862	781	if (q->u.out.pci_out_enabled)
863	782	return;
..	..	@@ -893,57 +812,30 @@
893	812	qdio_tasklet_schedule(q);
894	813	}
895	814
896		-static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
	815	+static inline void qdio_check_outbound_pci_queues(struct qdio_irq *irq)
897	816	{
898	817	struct qdio_q *out;
899	818	int i;
900	819
901		- if (!pci_out_supported(q))
	820	+ if (!pci_out_supported(irq) \|\| !irq->scan_threshold)
902	821	return;
903	822
904		- for_each_output_queue(q->irq_ptr, out, i)
	823	+ for_each_output_queue(irq, out, i)
905	824	if (!qdio_outbound_q_done(out))
906	825	qdio_tasklet_schedule(out);
907		-}
908		-
909		-static void __tiqdio_inbound_processing(struct qdio_q *q)
910		-{
911		- qperf_inc(q, tasklet_inbound);
912		- if (need_siga_sync(q) && need_siga_sync_after_ai(q))
913		- qdio_sync_queues(q);
914		-
915		- /*
916		- * The interrupt could be caused by a PCI request. Check the
917		- * PCI capable outbound queues.
918		- */
919		- qdio_check_outbound_after_thinint(q);
920		-
921		- if (!qdio_inbound_q_moved(q))
922		- return;
923		-
924		- qdio_kick_handler(q);
925		-
926		- if (!qdio_inbound_q_done(q)) {
927		- qperf_inc(q, tasklet_inbound_resched);
928		- if (!qdio_tasklet_schedule(q))
929		- return;
930		- }
931		-
932		- qdio_stop_polling(q);
933		- /*
934		- * We need to check again to not lose initiative after
935		- * resetting the ACK state.
936		- */
937		- if (!qdio_inbound_q_done(q)) {
938		- qperf_inc(q, tasklet_inbound_resched2);
939		- qdio_tasklet_schedule(q);
940		- }
941	826	}
942	827
943	828	void tiqdio_inbound_processing(unsigned long data)
944	829	{
945	830	struct qdio_q q = (struct qdio_q )data;
946		- __tiqdio_inbound_processing(q);
	831	+
	832	+ if (need_siga_sync(q) && need_siga_sync_after_ai(q))
	833	+ qdio_sync_queues(q);
	834	+
	835	+ /* The interrupt could be caused by a PCI request: */
	836	+ qdio_check_outbound_pci_queues(q->irq_ptr);
	837	+
	838	+ __qdio_inbound_processing(q);
947	839	}
948	840
949	841	static inline void qdio_set_state(struct qdio_irq *irq_ptr,
..	..	@@ -973,22 +865,17 @@
973	865	if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
974	866	return;
975	867
976		- for_each_input_queue(irq_ptr, q, i) {
977		- if (q->u.in.queue_start_poll) {
978		- /* skip if polling is enabled or already in work */
979		- if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
980		- &q->u.in.queue_irq_state)) {
981		- qperf_inc(q, int_discarded);
982		- continue;
983		- }
984		- q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
985		- q->irq_ptr->int_parm);
986		- } else {
	868	+ if (irq_ptr->irq_poll) {
	869	+ if (!test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
	870	+ irq_ptr->irq_poll(irq_ptr->cdev, irq_ptr->int_parm);
	871	+ else
	872	+ QDIO_PERF_STAT_INC(irq_ptr, int_discarded);
	873	+ } else {
	874	+ for_each_input_queue(irq_ptr, q, i)
987	875	tasklet_schedule(&q->tasklet);
988		- }
989	876	}
990	877
991		- if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED))
	878	+ if (!pci_out_supported(irq_ptr) \|\| !irq_ptr->scan_threshold)
992	879	return;
993	880
994	881	for_each_output_queue(irq_ptr, q, i) {
..	..	@@ -1000,12 +887,11 @@
1000	887	}
1001	888	}
1002	889
1003		-static void qdio_handle_activate_check(struct ccw_device *cdev,
1004		- unsigned long intparm, int cstat, int dstat)
	890	+static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,
	891	+ unsigned long intparm, int cstat,
	892	+ int dstat)
1005	893	{
1006		- struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1007	894	struct qdio_q *q;
1008		- int count;
1009	895
1010	896	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
1011	897	DBF_ERROR("intp :%lx", intparm);
..	..	@@ -1020,9 +906,8 @@
1020	906	goto no_handler;
1021	907	}
1022	908
1023		- count = sub_buf(q->first_to_check, q->first_to_kick);
1024	909	q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE,
1025		- q->nr, q->first_to_kick, count, irq_ptr->int_parm);
	910	+ q->nr, q->first_to_check, 0, irq_ptr->int_parm);
1026	911	no_handler:
1027	912	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1028	913	/*
..	..	@@ -1032,11 +917,9 @@
1032	917	lgr_info_log();
1033	918	}
1034	919
1035		-static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
	920	+static void qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat,
1036	921	int dstat)
1037	922	{
1038		- struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1039		-
1040	923	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
1041	924
1042	925	if (cstat)
..	..	@@ -1083,7 +966,7 @@
1083	966
1084	967	switch (irq_ptr->state) {
1085	968	case QDIO_IRQ_STATE_INACTIVE:
1086		- qdio_establish_handle_irq(cdev, cstat, dstat);
	969	+ qdio_establish_handle_irq(irq_ptr, cstat, dstat);
1087	970	break;
1088	971	case QDIO_IRQ_STATE_CLEANUP:
1089	972	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
..	..	@@ -1095,7 +978,7 @@
1095	978	return;
1096	979	}
1097	980	if (cstat \|\| dstat)
1098		- qdio_handle_activate_check(cdev, intparm, cstat,
	981	+ qdio_handle_activate_check(irq_ptr, intparm, cstat,
1099	982	dstat);
1100	983	break;
1101	984	case QDIO_IRQ_STATE_STOPPED:
..	..	@@ -1128,9 +1011,8 @@
1128	1011	}
1129	1012	EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
1130	1013
1131		-static void qdio_shutdown_queues(struct ccw_device *cdev)
	1014	+static void qdio_shutdown_queues(struct qdio_irq *irq_ptr)
1132	1015	{
1133		- struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1134	1016	struct qdio_q *q;
1135	1017	int i;
1136	1018
..	..	@@ -1141,6 +1023,33 @@
1141	1023	del_timer_sync(&q->u.out.timer);
1142	1024	tasklet_kill(&q->tasklet);
1143	1025	}
	1026	+}
	1027	+
	1028	+static int qdio_cancel_ccw(struct qdio_irq *irq, int how)
	1029	+{
	1030	+ struct ccw_device *cdev = irq->cdev;
	1031	+ int rc;
	1032	+
	1033	+ spin_lock_irq(get_ccwdev_lock(cdev));
	1034	+ qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP);
	1035	+ if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
	1036	+ rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
	1037	+ else
	1038	+ /* default behaviour is halt */
	1039	+ rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
	1040	+ spin_unlock_irq(get_ccwdev_lock(cdev));
	1041	+ if (rc) {
	1042	+ DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no);
	1043	+ DBF_ERROR("rc:%4d", rc);
	1044	+ return rc;
	1045	+ }
	1046	+
	1047	+ wait_event_interruptible_timeout(cdev->private->wait_q,
	1048	+ irq->state == QDIO_IRQ_STATE_INACTIVE \|\|
	1049	+ irq->state == QDIO_IRQ_STATE_ERR,
	1050	+ 10 * HZ);
	1051	+
	1052	+ return 0;
1144	1053	}
1145	1054
1146	1055	/**
..	..	@@ -1177,41 +1086,13 @@
1177	1086	*/
1178	1087	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1179	1088
1180		- tiqdio_remove_input_queues(irq_ptr);
1181		- qdio_shutdown_queues(cdev);
	1089	+ tiqdio_remove_device(irq_ptr);
	1090	+ qdio_shutdown_queues(irq_ptr);
1182	1091	qdio_shutdown_debug_entries(irq_ptr);
1183	1092
1184		- /* cleanup subchannel */
1185		- spin_lock_irq(get_ccwdev_lock(cdev));
1186		-
1187		- if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
1188		- rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
1189		- else
1190		- /* default behaviour is halt */
1191		- rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
1192		- if (rc) {
1193		- DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
1194		- DBF_ERROR("rc:%4d", rc);
1195		- goto no_cleanup;
1196		- }
1197		-
1198		- qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
1199		- spin_unlock_irq(get_ccwdev_lock(cdev));
1200		- wait_event_interruptible_timeout(cdev->private->wait_q,
1201		- irq_ptr->state == QDIO_IRQ_STATE_INACTIVE \|\|
1202		- irq_ptr->state == QDIO_IRQ_STATE_ERR,
1203		- 10 * HZ);
1204		- spin_lock_irq(get_ccwdev_lock(cdev));
1205		-
1206		-no_cleanup:
	1093	+ rc = qdio_cancel_ccw(irq_ptr, how);
1207	1094	qdio_shutdown_thinint(irq_ptr);
1208		-
1209		- /* restore interrupt handler */
1210		- if ((void )cdev->handler == (void )qdio_int_handler) {
1211		- cdev->handler = irq_ptr->orig_handler;
1212		- cdev->private->intparm = 0;
1213		- }
1214		- spin_unlock_irq(get_ccwdev_lock(cdev));
	1095	+ qdio_shutdown_irq(irq_ptr);
1215	1096
1216	1097	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1217	1098	mutex_unlock(&irq_ptr->setup_mutex);
..	..	@@ -1242,43 +1123,47 @@
1242	1123	cdev->private->qdio_data = NULL;
1243	1124	mutex_unlock(&irq_ptr->setup_mutex);
1244	1125
1245		- qdio_release_memory(irq_ptr);
	1126	+ qdio_free_async_data(irq_ptr);
	1127	+ qdio_free_queues(irq_ptr);
	1128	+ free_page((unsigned long) irq_ptr->qdr);
	1129	+ free_page(irq_ptr->chsc_page);
	1130	+ free_page((unsigned long) irq_ptr);
1246	1131	return 0;
1247	1132	}
1248	1133	EXPORT_SYMBOL_GPL(qdio_free);
1249	1134
1250	1135	/**
1251	1136	* qdio_allocate - allocate qdio queues and associated data
1252		- * @init_data: initialization data
	1137	+ * @cdev: associated ccw device
	1138	+ * @no_input_qs: allocate this number of Input Queues
	1139	+ * @no_output_qs: allocate this number of Output Queues
1253	1140	*/
1254		-int qdio_allocate(struct qdio_initialize *init_data)
	1141	+int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
	1142	+ unsigned int no_output_qs)
1255	1143	{
1256	1144	struct subchannel_id schid;
1257	1145	struct qdio_irq *irq_ptr;
	1146	+ int rc = -ENOMEM;
1258	1147
1259		- ccw_device_get_schid(init_data->cdev, &schid);
	1148	+ ccw_device_get_schid(cdev, &schid);
1260	1149	DBF_EVENT("qallocate:%4x", schid.sch_no);
1261	1150
1262		- if ((init_data->no_input_qs && !init_data->input_handler) \|\|
1263		- (init_data->no_output_qs && !init_data->output_handler))
1264		- return -EINVAL;
1265		-
1266		- if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) \|\|
1267		- (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ))
1268		- return -EINVAL;
1269		-
1270		- if ((!init_data->input_sbal_addr_array) \|\|
1271		- (!init_data->output_sbal_addr_array))
	1151	+ if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ \|\|
	1152	+ no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)
1272	1153	return -EINVAL;
1273	1154
1274	1155	/* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
1275	1156	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL \| GFP_DMA);
1276	1157	if (!irq_ptr)
1277		- goto out_err;
	1158	+ return -ENOMEM;
1278	1159
	1160	+ irq_ptr->cdev = cdev;
1279	1161	mutex_init(&irq_ptr->setup_mutex);
1280		- if (qdio_allocate_dbf(init_data, irq_ptr))
1281		- goto out_rel;
	1162	+ if (qdio_allocate_dbf(irq_ptr))
	1163	+ goto err_dbf;
	1164	+
	1165	+ DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,
	1166	+ no_output_qs);
1282	1167
1283	1168	/*
1284	1169	* Allocate a page for the chsc calls in qdio_establish.
..	..	@@ -1288,24 +1173,30 @@
1288	1173	*/
1289	1174	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
1290	1175	if (!irq_ptr->chsc_page)
1291		- goto out_rel;
	1176	+ goto err_chsc;
1292	1177
1293	1178	/* qdr is used in ccw1.cda which is u32 */
1294	1179	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL \| GFP_DMA);
1295	1180	if (!irq_ptr->qdr)
1296		- goto out_rel;
	1181	+ goto err_qdr;
1297	1182
1298		- if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs,
1299		- init_data->no_output_qs))
1300		- goto out_rel;
	1183	+ rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);
	1184	+ if (rc)
	1185	+ goto err_queues;
1301	1186
1302		- init_data->cdev->private->qdio_data = irq_ptr;
	1187	+ INIT_LIST_HEAD(&irq_ptr->entry);
	1188	+ cdev->private->qdio_data = irq_ptr;
1303	1189	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1304	1190	return 0;
1305		-out_rel:
1306		- qdio_release_memory(irq_ptr);
1307		-out_err:
1308		- return -ENOMEM;
	1191	+
	1192	+err_queues:
	1193	+ free_page((unsigned long) irq_ptr->qdr);
	1194	+err_qdr:
	1195	+ free_page(irq_ptr->chsc_page);
	1196	+err_chsc:
	1197	+err_dbf:
	1198	+ free_page((unsigned long) irq_ptr);
	1199	+ return rc;
1309	1200	}
1310	1201	EXPORT_SYMBOL_GPL(qdio_allocate);
1311	1202
..	..	@@ -1319,6 +1210,8 @@
1319	1210
1320	1211	for_each_output_queue(irq_ptr, q, i) {
1321	1212	if (use_cq) {
	1213	+ if (multicast_outbound(q))
	1214	+ continue;
1322	1215	if (qdio_enable_async_operation(&q->u.out) < 0) {
1323	1216	use_cq = 0;
1324	1217	continue;
..	..	@@ -1329,33 +1222,62 @@
1329	1222	DBF_EVENT("use_cq:%d", use_cq);
1330	1223	}
1331	1224
	1225	+static void qdio_trace_init_data(struct qdio_irq *irq,
	1226	+ struct qdio_initialize *data)
	1227	+{
	1228	+ DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format);
	1229	+ DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format);
	1230	+ DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR);
	1231	+ DBF_DEV_HEX(irq, &data->input_slib_elements, sizeof(void *), DBF_ERR);
	1232	+ DBF_DEV_HEX(irq, &data->output_slib_elements, sizeof(void *), DBF_ERR);
	1233	+ DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs,
	1234	+ data->no_output_qs);
	1235	+ DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR);
	1236	+ DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR);
	1237	+ DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR);
	1238	+ DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR);
	1239	+ DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *),
	1240	+ DBF_ERR);
	1241	+}
	1242	+
1332	1243	/**
1333	1244	* qdio_establish - establish queues on a qdio subchannel
	1245	+ * @cdev: associated ccw device
1334	1246	* @init_data: initialization data
1335	1247	*/
1336		-int qdio_establish(struct qdio_initialize *init_data)
	1248	+int qdio_establish(struct ccw_device *cdev,
	1249	+ struct qdio_initialize *init_data)
1337	1250	{
1338		- struct ccw_device *cdev = init_data->cdev;
	1251	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1339	1252	struct subchannel_id schid;
1340		- struct qdio_irq *irq_ptr;
	1253	+ long timeout;
1341	1254	int rc;
1342	1255
1343	1256	ccw_device_get_schid(cdev, &schid);
1344	1257	DBF_EVENT("qestablish:%4x", schid.sch_no);
1345	1258
1346		- irq_ptr = cdev->private->qdio_data;
1347	1259	if (!irq_ptr)
1348	1260	return -ENODEV;
1349	1261
	1262	+ if (init_data->no_input_qs > irq_ptr->max_input_qs \|\|
	1263	+ init_data->no_output_qs > irq_ptr->max_output_qs)
	1264	+ return -EINVAL;
	1265	+
	1266	+ if ((init_data->no_input_qs && !init_data->input_handler) \|\|
	1267	+ (init_data->no_output_qs && !init_data->output_handler))
	1268	+ return -EINVAL;
	1269	+
	1270	+ if (!init_data->input_sbal_addr_array \|\|
	1271	+ !init_data->output_sbal_addr_array)
	1272	+ return -EINVAL;
	1273	+
1350	1274	mutex_lock(&irq_ptr->setup_mutex);
1351		- qdio_setup_irq(init_data);
	1275	+ qdio_trace_init_data(irq_ptr, init_data);
	1276	+ qdio_setup_irq(irq_ptr, init_data);
1352	1277
1353	1278	rc = qdio_establish_thinint(irq_ptr);
1354		- if (rc) {
1355		- mutex_unlock(&irq_ptr->setup_mutex);
1356		- qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1357		- return rc;
1358		- }
	1279	+ if (rc)
	1280	+ goto err_thinint;
1359	1281
1360	1282	/* establish q */
1361	1283	irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd;
..	..	@@ -1371,14 +1293,16 @@
1371	1293	if (rc) {
1372	1294	DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1373	1295	DBF_ERROR("rc:%4x", rc);
1374		- mutex_unlock(&irq_ptr->setup_mutex);
1375		- qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1376		- return rc;
	1296	+ goto err_ccw_start;
1377	1297	}
1378	1298
1379		- wait_event_interruptible_timeout(cdev->private->wait_q,
1380		- irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED \|\|
1381		- irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
	1299	+ timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
	1300	+ irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED \|\|
	1301	+ irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
	1302	+ if (timeout <= 0) {
	1303	+ rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
	1304	+ goto err_ccw_timeout;
	1305	+ }
1382	1306
1383	1307	if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1384	1308	mutex_unlock(&irq_ptr->setup_mutex);
..	..	@@ -1394,9 +1318,19 @@
1394	1318	qdio_init_buf_states(irq_ptr);
1395	1319
1396	1320	mutex_unlock(&irq_ptr->setup_mutex);
1397		- qdio_print_subchannel_info(irq_ptr, cdev);
1398		- qdio_setup_debug_entries(irq_ptr, cdev);
	1321	+ qdio_print_subchannel_info(irq_ptr);
	1322	+ qdio_setup_debug_entries(irq_ptr);
1399	1323	return 0;
	1324	+
	1325	+err_ccw_timeout:
	1326	+ qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR);
	1327	+err_ccw_start:
	1328	+ qdio_shutdown_thinint(irq_ptr);
	1329	+err_thinint:
	1330	+ qdio_shutdown_irq(irq_ptr);
	1331	+ qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
	1332	+ mutex_unlock(&irq_ptr->setup_mutex);
	1333	+ return rc;
1400	1334	}
1401	1335	EXPORT_SYMBOL_GPL(qdio_establish);
1402	1336
..	..	@@ -1406,14 +1340,13 @@
1406	1340	*/
1407	1341	int qdio_activate(struct ccw_device *cdev)
1408	1342	{
	1343	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1409	1344	struct subchannel_id schid;
1410		- struct qdio_irq *irq_ptr;
1411	1345	int rc;
1412	1346
1413	1347	ccw_device_get_schid(cdev, &schid);
1414	1348	DBF_EVENT("qactivate:%4x", schid.sch_no);
1415	1349
1416		- irq_ptr = cdev->private->qdio_data;
1417	1350	if (!irq_ptr)
1418	1351	return -ENODEV;
1419	1352
..	..	@@ -1441,7 +1374,7 @@
1441	1374	}
1442	1375
1443	1376	if (is_thinint_irq(irq_ptr))
1444		- tiqdio_add_input_queues(irq_ptr);
	1377	+ tiqdio_add_device(irq_ptr);
1445	1378
1446	1379	/* wait for subchannel to become active */
1447	1380	msleep(5);
..	..	@@ -1461,25 +1394,6 @@
1461	1394	}
1462	1395	EXPORT_SYMBOL_GPL(qdio_activate);
1463	1396
1464		-static inline int buf_in_between(int bufnr, int start, int count)
1465		-{
1466		- int end = add_buf(start, count);
1467		-
1468		- if (end > start) {
1469		- if (bufnr >= start && bufnr < end)
1470		- return 1;
1471		- else
1472		- return 0;
1473		- }
1474		-
1475		- /* wrap-around case */
1476		- if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) \|\|
1477		- (bufnr < end))
1478		- return 1;
1479		- else
1480		- return 0;
1481		-}
1482		-
1483	1397	/**
1484	1398	* handle_inbound - reset processed input buffers
1485	1399	* @q: queue containing the buffers
..	..	@@ -1490,38 +1404,18 @@
1490	1404	static int handle_inbound(struct qdio_q *q, unsigned int callflags,
1491	1405	int bufnr, int count)
1492	1406	{
1493		- int diff;
	1407	+ int overlap;
1494	1408
1495	1409	qperf_inc(q, inbound_call);
1496	1410
1497		- if (!q->u.in.polling)
1498		- goto set;
1499		-
1500		- /* protect against stop polling setting an ACK for an emptied slsb */
1501		- if (count == QDIO_MAX_BUFFERS_PER_Q) {
1502		- /* overwriting everything, just delete polling status */
1503		- q->u.in.polling = 0;
1504		- q->u.in.ack_count = 0;
1505		- goto set;
1506		- } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
1507		- if (is_qebsm(q)) {
1508		- /* partial overwrite, just update ack_start */
1509		- diff = add_buf(bufnr, count);
1510		- diff = sub_buf(diff, q->u.in.ack_start);
1511		- q->u.in.ack_count -= diff;
1512		- if (q->u.in.ack_count <= 0) {
1513		- q->u.in.polling = 0;
1514		- q->u.in.ack_count = 0;
1515		- goto set;
1516		- }
1517		- q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
1518		- }
1519		- else
1520		- /* the only ACK will be deleted, so stop polling */
1521		- q->u.in.polling = 0;
	1411	+ /* If any processed SBALs are returned to HW, adjust our tracking: */
	1412	+ overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),
	1413	+ q->u.in.batch_count);
	1414	+ if (overlap > 0) {
	1415	+ q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);
	1416	+ q->u.in.batch_count -= overlap;
1522	1417	}
1523	1418
1524		-set:
1525	1419	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1526	1420	atomic_add(count, &q->nr_buf_used);
1527	1421
..	..	@@ -1539,8 +1433,9 @@
1539	1433	* @count: how many buffers are filled
1540	1434	*/
1541	1435	static int handle_outbound(struct qdio_q *q, unsigned int callflags,
1542		- int bufnr, int count)
	1436	+ unsigned int bufnr, unsigned int count)
1543	1437	{
	1438	+ const unsigned int scan_threshold = q->irq_ptr->scan_threshold;
1544	1439	unsigned char state = 0;
1545	1440	int used, rc = 0;
1546	1441
..	..	@@ -1561,12 +1456,10 @@
1561	1456	if (queue_type(q) == QDIO_IQDIO_QFMT) {
1562	1457	unsigned long phys_aob = 0;
1563	1458
1564		- /* One SIGA-W per buffer required for unicast HSI */
1565		- WARN_ON_ONCE(count > 1 && !multicast_outbound(q));
	1459	+ if (q->u.out.use_cq && count == 1)
	1460	+ phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);
1566	1461
1567		- phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);
1568		-
1569		- rc = qdio_kick_outbound_q(q, phys_aob);
	1462	+ rc = qdio_kick_outbound_q(q, count, phys_aob);
1570	1463	} else if (need_siga_sync(q)) {
1571	1464	rc = qdio_siga_sync_q(q);
1572	1465	} else if (count < QDIO_MAX_BUFFERS_PER_Q &&
..	..	@@ -1575,11 +1468,15 @@
1575	1468	/* The previous buffer is not processed yet, tack on. */
1576	1469	qperf_inc(q, fast_requeue);
1577	1470	} else {
1578		- rc = qdio_kick_outbound_q(q, 0);
	1471	+ rc = qdio_kick_outbound_q(q, count, 0);
1579	1472	}
1580	1473
	1474	+ /* Let drivers implement their own completion scanning: */
	1475	+ if (!scan_threshold)
	1476	+ return rc;
	1477	+
1581	1478	/* in case of SIGA errors we must process the error immediately */
1582		- if (used >= q->u.out.scan_threshold \|\| rc)
	1479	+ if (used >= scan_threshold \|\| rc)
1583	1480	qdio_tasklet_schedule(q);
1584	1481	else
1585	1482	/* free the SBALs in case of no further traffic */
..	..	@@ -1600,12 +1497,11 @@
1600	1497	int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
1601	1498	int q_nr, unsigned int bufnr, unsigned int count)
1602	1499	{
1603		- struct qdio_irq *irq_ptr;
	1500	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1604	1501
1605	1502	if (bufnr >= QDIO_MAX_BUFFERS_PER_Q \|\| count > QDIO_MAX_BUFFERS_PER_Q)
1606	1503	return -EINVAL;
1607	1504
1608		- irq_ptr = cdev->private->qdio_data;
1609	1505	if (!irq_ptr)
1610	1506	return -ENODEV;
1611	1507
..	..	@@ -1627,26 +1523,26 @@
1627	1523	EXPORT_SYMBOL_GPL(do_QDIO);
1628	1524
1629	1525	/**
1630		- * qdio_start_irq - process input buffers
	1526	+ * qdio_start_irq - enable interrupt processing for the device
1631	1527	* @cdev: associated ccw_device for the qdio subchannel
1632		- * @nr: input queue number
1633	1528	*
1634	1529	* Return codes
1635	1530	* 0 - success
1636	1531	* 1 - irqs not started since new data is available
1637	1532	*/
1638		-int qdio_start_irq(struct ccw_device *cdev, int nr)
	1533	+int qdio_start_irq(struct ccw_device *cdev)
1639	1534	{
1640	1535	struct qdio_q *q;
1641	1536	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
	1537	+ unsigned int i;
1642	1538
1643	1539	if (!irq_ptr)
1644	1540	return -ENODEV;
1645		- q = irq_ptr->input_qs[nr];
1646	1541
1647		- clear_nonshared_ind(irq_ptr);
1648		- qdio_stop_polling(q);
1649		- clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
	1542	+ for_each_input_queue(irq_ptr, q, i)
	1543	+ qdio_stop_polling(q);
	1544	+
	1545	+ clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state);
1650	1546
1651	1547	/*
1652	1548	* We need to check again to not lose initiative after
..	..	@@ -1654,19 +1550,60 @@
1654	1550	*/
1655	1551	if (test_nonshared_ind(irq_ptr))
1656	1552	goto rescan;
1657		- if (!qdio_inbound_q_done(q))
1658		- goto rescan;
	1553	+
	1554	+ for_each_input_queue(irq_ptr, q, i) {
	1555	+ if (!qdio_inbound_q_done(q, q->first_to_check))
	1556	+ goto rescan;
	1557	+ }
	1558	+
1659	1559	return 0;
1660	1560
1661	1561	rescan:
1662		- if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1663		- &q->u.in.queue_irq_state))
	1562	+ if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1664	1563	return 0;
1665	1564	else
1666	1565	return 1;
1667	1566
1668	1567	}
1669	1568	EXPORT_SYMBOL(qdio_start_irq);
	1569	+
	1570	+static int __qdio_inspect_queue(struct qdio_q q, unsigned int bufnr,
	1571	+ unsigned int *error)
	1572	+{
	1573	+ unsigned int start = q->first_to_check;
	1574	+ int count;
	1575	+
	1576	+ count = q->is_input_q ? qdio_inbound_q_moved(q, start) :
	1577	+ qdio_outbound_q_moved(q, start);
	1578	+ if (count == 0)
	1579	+ return 0;
	1580	+
	1581	+ *bufnr = start;
	1582	+ *error = q->qdio_error;
	1583	+
	1584	+ /* for the next time */
	1585	+ q->first_to_check = add_buf(start, count);
	1586	+ q->qdio_error = 0;
	1587	+
	1588	+ return count;
	1589	+}
	1590	+
	1591	+int qdio_inspect_queue(struct ccw_device *cdev, unsigned int nr, bool is_input,
	1592	+ unsigned int bufnr, unsigned int error)
	1593	+{
	1594	+ struct qdio_irq *irq_ptr = cdev->private->qdio_data;
	1595	+ struct qdio_q *q;
	1596	+
	1597	+ if (!irq_ptr)
	1598	+ return -ENODEV;
	1599	+ q = is_input ? irq_ptr->input_qs[nr] : irq_ptr->output_qs[nr];
	1600	+
	1601	+ if (need_siga_sync(q))
	1602	+ qdio_siga_sync_q(q);
	1603	+
	1604	+ return __qdio_inspect_queue(q, bufnr, error);
	1605	+}
	1606	+EXPORT_SYMBOL_GPL(qdio_inspect_queue);
1670	1607
1671	1608	/**
1672	1609	* qdio_get_next_buffers - process input buffers
..	..	@@ -1684,7 +1621,6 @@
1684	1621	int *error)
1685	1622	{
1686	1623	struct qdio_q *q;
1687		- int start, end;
1688	1624	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1689	1625
1690	1626	if (!irq_ptr)
..	..	@@ -1698,141 +1634,37 @@
1698	1634	if (need_siga_sync(q))
1699	1635	qdio_sync_queues(q);
1700	1636
1701		- /* check the PCI capable outbound queues. */
1702		- qdio_check_outbound_after_thinint(q);
1703		-
1704		- if (!qdio_inbound_q_moved(q))
1705		- return 0;
	1637	+ qdio_check_outbound_pci_queues(irq_ptr);
1706	1638
1707	1639	/* Note: upper-layer MUST stop processing immediately here ... */
1708	1640	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
1709	1641	return -EIO;
1710	1642
1711		- start = q->first_to_kick;
1712		- end = q->first_to_check;
1713		- *bufnr = start;
1714		- *error = q->qdio_error;
1715		-
1716		- /* for the next time */
1717		- q->first_to_kick = end;
1718		- q->qdio_error = 0;
1719		- return sub_buf(end, start);
	1643	+ return __qdio_inspect_queue(q, bufnr, error);
1720	1644	}
1721	1645	EXPORT_SYMBOL(qdio_get_next_buffers);
1722	1646
1723	1647	/**
1724	1648	* qdio_stop_irq - disable interrupt processing for the device
1725	1649	* @cdev: associated ccw_device for the qdio subchannel
1726		- * @nr: input queue number
1727	1650	*
1728	1651	* Return codes
1729	1652	* 0 - interrupts were already disabled
1730	1653	* 1 - interrupts successfully disabled
1731	1654	*/
1732		-int qdio_stop_irq(struct ccw_device *cdev, int nr)
	1655	+int qdio_stop_irq(struct ccw_device *cdev)
1733	1656	{
1734		- struct qdio_q *q;
1735	1657	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1736	1658
1737	1659	if (!irq_ptr)
1738	1660	return -ENODEV;
1739		- q = irq_ptr->input_qs[nr];
1740	1661
1741		- if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1742		- &q->u.in.queue_irq_state))
	1662	+ if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1743	1663	return 0;
1744	1664	else
1745	1665	return 1;
1746	1666	}
1747	1667	EXPORT_SYMBOL(qdio_stop_irq);
1748		-
1749		-/**
1750		- * qdio_pnso_brinfo() - perform network subchannel op #0 - bridge info.
1751		- * @schid: Subchannel ID.
1752		- * @cnc: Boolean Change-Notification Control
1753		- * @response: Response code will be stored at this address
1754		- * @cb: Callback function will be executed for each element
1755		- * of the address list
1756		- * @priv: Pointer to pass to the callback function.
1757		- *
1758		- * Performs "Store-network-bridging-information list" operation and calls
1759		- * the callback function for every entry in the list. If "change-
1760		- * notification-control" is set, further changes in the address list
1761		- * will be reported via the IPA command.
1762		- */
1763		-int qdio_pnso_brinfo(struct subchannel_id schid,
1764		- int cnc, u16 *response,
1765		- void (cb)(void priv, enum qdio_brinfo_entry_type type,
1766		- void *entry),
1767		- void *priv)
1768		-{
1769		- struct chsc_pnso_area *rr;
1770		- int rc;
1771		- u32 prev_instance = 0;
1772		- int isfirstblock = 1;
1773		- int i, size, elems;
1774		-
1775		- rr = (struct chsc_pnso_area *)get_zeroed_page(GFP_KERNEL);
1776		- if (rr == NULL)
1777		- return -ENOMEM;
1778		- do {
1779		- /* on the first iteration, naihdr.resume_token will be zero */
1780		- rc = chsc_pnso_brinfo(schid, rr, rr->naihdr.resume_token, cnc);
1781		- if (rc != 0 && rc != -EBUSY)
1782		- goto out;
1783		- if (rr->response.code != 1) {
1784		- rc = -EIO;
1785		- continue;
1786		- } else
1787		- rc = 0;
1788		-
1789		- if (cb == NULL)
1790		- continue;
1791		-
1792		- size = rr->naihdr.naids;
1793		- elems = (rr->response.length -
1794		- sizeof(struct chsc_header) -
1795		- sizeof(struct chsc_brinfo_naihdr)) /
1796		- size;
1797		-
1798		- if (!isfirstblock && (rr->naihdr.instance != prev_instance)) {
1799		- /* Inform the caller that they need to scrap */
1800		- /* the data that was already reported via cb */
1801		- rc = -EAGAIN;
1802		- break;
1803		- }
1804		- isfirstblock = 0;
1805		- prev_instance = rr->naihdr.instance;
1806		- for (i = 0; i < elems; i++)
1807		- switch (size) {
1808		- case sizeof(struct qdio_brinfo_entry_l3_ipv6):
1809		- (*cb)(priv, l3_ipv6_addr,
1810		- &rr->entries.l3_ipv6[i]);
1811		- break;
1812		- case sizeof(struct qdio_brinfo_entry_l3_ipv4):
1813		- (*cb)(priv, l3_ipv4_addr,
1814		- &rr->entries.l3_ipv4[i]);
1815		- break;
1816		- case sizeof(struct qdio_brinfo_entry_l2):
1817		- (*cb)(priv, l2_addr_lnid,
1818		- &rr->entries.l2[i]);
1819		- break;
1820		- default:
1821		- WARN_ON_ONCE(1);
1822		- rc = -EIO;
1823		- goto out;
1824		- }
1825		- } while (rr->response.code == 0x0107 \|\| /* channel busy */
1826		- (rr->response.code == 1 && /* list stored */
1827		- /* resume token is non-zero => list incomplete */
1828		- (rr->naihdr.resume_token.t1 \|\| rr->naihdr.resume_token.t2)));
1829		- (*response) = rr->response.code;
1830		-
1831		-out:
1832		- free_page((unsigned long)rr);
1833		- return rc;
1834		-}
1835		-EXPORT_SYMBOL_GPL(qdio_pnso_brinfo);
1836	1668
1837	1669	static int __init init_QDIO(void)
1838	1670	{
..	..	@@ -1844,16 +1676,11 @@
1844	1676	rc = qdio_setup_init();
1845	1677	if (rc)
1846	1678	goto out_debug;
1847		- rc = tiqdio_allocate_memory();
	1679	+ rc = qdio_thinint_init();
1848	1680	if (rc)
1849	1681	goto out_cache;
1850		- rc = tiqdio_register_thinints();
1851		- if (rc)
1852		- goto out_ti;
1853	1682	return 0;
1854	1683
1855		-out_ti:
1856		- tiqdio_free_memory();
1857	1684	out_cache:
1858	1685	qdio_setup_exit();
1859	1686	out_debug:
..	..	@@ -1863,8 +1690,7 @@
1863	1690
1864	1691	static void __exit exit_QDIO(void)
1865	1692	{
1866		- tiqdio_unregister_thinints();
1867		- tiqdio_free_memory();
	1693	+ qdio_thinint_exit();
1868	1694	qdio_setup_exit();
1869	1695	qdio_debug_exit();
1870	1696	}