forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/fs/nfs/direct.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * linux/fs/nfs/direct.c
34  *
....@@ -63,13 +64,6 @@
6364 
6465 static struct kmem_cache *nfs_direct_cachep;
6566 
66
-/*
67
- * This represents a set of asynchronous requests that we're waiting on
68
- */
69
-struct nfs_direct_mirror {
70
-	ssize_t count;
71
-};
72
-
7367 struct nfs_direct_req {
7468 	struct kref		kref;		/* release manager */
7569 
....@@ -82,9 +76,6 @@
8276 	/* completion state */
8377 	atomic_t		io_count;	/* i/os we're waiting for */
8478 	spinlock_t		lock;		/* protect completion state */
85
-
86
-	struct nfs_direct_mirror mirrors[NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX];
87
-	int			mirror_count;
8879 
8980 	loff_t			io_start;	/* Start offset for I/O */
9081 	ssize_t			count,		/* bytes actually processed */
....@@ -103,7 +94,7 @@
10394 #define NFS_ODIRECT_RESCHED_WRITES	(2)	/* write verification failed */
10495 	/* for read */
10596 #define NFS_ODIRECT_SHOULD_DIRTY	(3)	/* dirty user-space page after read */
106
-	struct nfs_writeverf	verf;		/* unstable write verifier */
97
+#define NFS_ODIRECT_DONE		INT_MAX	/* write verification failed */
10798 };
10899 
109100 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
....@@ -126,8 +117,6 @@
126117 			    const struct nfs_pgio_header *hdr,
127118 			    ssize_t dreq_len)
128119 {
129
-	struct nfs_direct_mirror *mirror = &dreq->mirrors[hdr->pgio_mirror_idx];
130
-
131120 	if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) ||
132121 	      test_bit(NFS_IOHDR_EOF, &hdr->flags)))
133122 		return;
....@@ -141,15 +130,12 @@
141130 		else /* Clear outstanding error if this is EOF */
142131 			dreq->error = 0;
143132 	}
144
-	if (mirror->count > dreq_len)
145
-		mirror->count = dreq_len;
146133 }
147134 
148135 static void
149136 nfs_direct_count_bytes(struct nfs_direct_req *dreq,
150137 		       const struct nfs_pgio_header *hdr)
151138 {
152
-	struct nfs_direct_mirror *mirror = &dreq->mirrors[hdr->pgio_mirror_idx];
153139 	loff_t hdr_end = hdr->io_start + hdr->good_bytes;
154140 	ssize_t dreq_len = 0;
155141 
....@@ -161,110 +147,8 @@
161147 	if (dreq_len > dreq->max_count)
162148 		dreq_len = dreq->max_count;
163149 
164
-	if (mirror->count < dreq_len)
165
-		mirror->count = dreq_len;
166150 	if (dreq->count < dreq_len)
167151 		dreq->count = dreq_len;
168
-}
169
-
170
-/*
171
- * nfs_direct_select_verf - select the right verifier
172
- * @dreq - direct request possibly spanning multiple servers
173
- * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs
174
- * @commit_idx - commit bucket index for the DS
175
- *
176
- * returns the correct verifier to use given the role of the server
177
- */
178
-static struct nfs_writeverf *
179
-nfs_direct_select_verf(struct nfs_direct_req *dreq,
180
-		       struct nfs_client *ds_clp,
181
-		       int commit_idx)
182
-{
183
-	struct nfs_writeverf *verfp = &dreq->verf;
184
-
185
-#ifdef CONFIG_NFS_V4_1
186
-	/*
187
-	 * pNFS is in use, use the DS verf except commit_through_mds is set
188
-	 * for layout segment where nbuckets is zero.
189
-	 */
190
-	if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {
191
-		if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)
192
-			verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;
193
-		else
194
-			WARN_ON_ONCE(1);
195
-	}
196
-#endif
197
-	return verfp;
198
-}
199
-
200
-
201
-/*
202
- * nfs_direct_set_hdr_verf - set the write/commit verifier
203
- * @dreq - direct request possibly spanning multiple servers
204
- * @hdr - pageio header to validate against previously seen verfs
205
- *
206
- * Set the server's (MDS or DS) "seen" verifier
207
- */
208
-static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,
209
-				    struct nfs_pgio_header *hdr)
210
-{
211
-	struct nfs_writeverf *verfp;
212
-
213
-	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
214
-	WARN_ON_ONCE(verfp->committed >= 0);
215
-	memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
216
-	WARN_ON_ONCE(verfp->committed < 0);
217
-}
218
-
219
-static int nfs_direct_cmp_verf(const struct nfs_writeverf *v1,
220
-		const struct nfs_writeverf *v2)
221
-{
222
-	return nfs_write_verifier_cmp(&v1->verifier, &v2->verifier);
223
-}
224
-
225
-/*
226
- * nfs_direct_cmp_hdr_verf - compare verifier for pgio header
227
- * @dreq - direct request possibly spanning multiple servers
228
- * @hdr - pageio header to validate against previously seen verf
229
- *
230
- * set the server's "seen" verf if not initialized.
231
- * returns result of comparison between @hdr->verf and the "seen"
232
- * verf of the server used by @hdr (DS or MDS)
233
- */
234
-static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,
235
-					  struct nfs_pgio_header *hdr)
236
-{
237
-	struct nfs_writeverf *verfp;
238
-
239
-	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
240
-	if (verfp->committed < 0) {
241
-		nfs_direct_set_hdr_verf(dreq, hdr);
242
-		return 0;
243
-	}
244
-	return nfs_direct_cmp_verf(verfp, &hdr->verf);
245
-}
246
-
247
-/*
248
- * nfs_direct_cmp_commit_data_verf - compare verifier for commit data
249
- * @dreq - direct request possibly spanning multiple servers
250
- * @data - commit data to validate against previously seen verf
251
- *
252
- * returns result of comparison between @data->verf and the verf of
253
- * the server used by @data (DS or MDS)
254
- */
255
-static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,
256
-					   struct nfs_commit_data *data)
257
-{
258
-	struct nfs_writeverf *verfp;
259
-
260
-	verfp = nfs_direct_select_verf(dreq, data->ds_clp,
261
-					 data->ds_commit_index);
262
-
263
-	/* verifier not set so always fail */
264
-	if (verfp->committed < 0 || data->res.verf->committed <= NFS_UNSTABLE)
265
-		return 1;
266
-
267
-	return nfs_direct_cmp_verf(verfp, data->res.verf);
268152 }
269153 
270154 /**
....@@ -288,8 +172,8 @@
288172 	VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
289173 
290174 	if (iov_iter_rw(iter) == READ)
291
-		return nfs_file_direct_read(iocb, iter);
292
-	return nfs_file_direct_write(iocb, iter);
175
+		return nfs_file_direct_read(iocb, iter, true);
176
+	return nfs_file_direct_write(iocb, iter, true);
293177 }
294178 
295179 static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
....@@ -309,18 +193,6 @@
309193 	cinfo->completion_ops = &nfs_direct_commit_completion_ops;
310194 }
311195 
312
-static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,
313
-					     struct nfs_pageio_descriptor *pgio,
314
-					     struct nfs_page *req)
315
-{
316
-	int mirror_count = 1;
317
-
318
-	if (pgio->pg_ops->pg_get_mirror_count)
319
-		mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
320
-
321
-	dreq->mirror_count = mirror_count;
322
-}
323
-
324196 static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
325197 {
326198 	struct nfs_direct_req *dreq;
....@@ -333,9 +205,8 @@
333205 	kref_get(&dreq->kref);
334206 	init_completion(&dreq->completion);
335207 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);
336
-	dreq->verf.committed = NFS_INVALID_STABLE_HOW;	/* not set yet */
208
+	pnfs_init_ds_commit_info(&dreq->ds_cinfo);
337209 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
338
-	dreq->mirror_count = 1;
339210 	spin_lock_init(&dreq->lock);
340211 
341212 	return dreq;
....@@ -345,7 +216,7 @@
345216 {
346217 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
347218 
348
-	nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);
219
+	pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);
349220 	if (dreq->l_ctx != NULL)
350221 		nfs_put_lock_context(dreq->l_ctx);
351222 	if (dreq->ctx != NULL)
....@@ -507,7 +378,7 @@
507378 			struct nfs_page *req;
508379 			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
509380 			/* XXX do we need to do the eof zeroing found in async_filler? */
510
-			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
381
+			req = nfs_create_request(dreq->ctx, pagevec[i],
511382 						 pgbase, req_len);
512383 			if (IS_ERR(req)) {
513384 				result = PTR_ERR(req);
....@@ -553,6 +424,7 @@
553424  * nfs_file_direct_read - file direct read operation for NFS files
554425  * @iocb: target I/O control block
555426  * @iter: vector of user buffers into which to read data
427
+ * @swap: flag indicating this is swap IO, not O_DIRECT IO
556428  *
557429  * We use this function for direct reads instead of calling
558430  * generic_file_aio_read() in order to avoid gfar's check to see if
....@@ -568,14 +440,15 @@
568440  * client must read the updated atime from the server back into its
569441  * cache.
570442  */
571
-ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
443
+ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
444
+			     bool swap)
572445 {
573446 	struct file *file = iocb->ki_filp;
574447 	struct address_space *mapping = file->f_mapping;
575448 	struct inode *inode = mapping->host;
576449 	struct nfs_direct_req *dreq;
577450 	struct nfs_lock_context *l_ctx;
578
-	ssize_t result = -EINVAL, requested;
451
+	ssize_t result, requested;
579452 	size_t count = iov_iter_count(iter);
580453 	nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
581454 
....@@ -610,12 +483,14 @@
610483 	if (iter_is_iovec(iter))
611484 		dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
612485 
613
-	nfs_start_io_direct(inode);
486
+	if (!swap)
487
+		nfs_start_io_direct(inode);
614488 
615489 	NFS_I(inode)->read_io += count;
616490 	requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
617491 
618
-	nfs_end_io_direct(inode);
492
+	if (!swap)
493
+		nfs_end_io_direct(inode);
619494 
620495 	if (requested > 0) {
621496 		result = nfs_direct_wait(dreq);
....@@ -634,16 +509,57 @@
634509 	return result;
635510 }
636511 
512
+static void nfs_direct_add_page_head(struct list_head *list,
513
+				     struct nfs_page *req)
514
+{
515
+	struct nfs_page *head = req->wb_head;
516
+
517
+	if (!list_empty(&head->wb_list) || !nfs_lock_request(head))
518
+		return;
519
+	if (!list_empty(&head->wb_list)) {
520
+		nfs_unlock_request(head);
521
+		return;
522
+	}
523
+	list_add(&head->wb_list, list);
524
+	kref_get(&head->wb_kref);
525
+	kref_get(&head->wb_kref);
526
+}
527
+
528
+static void nfs_direct_join_group(struct list_head *list,
529
+				  struct nfs_commit_info *cinfo,
530
+				  struct inode *inode)
531
+{
532
+	struct nfs_page *req, *subreq;
533
+
534
+	list_for_each_entry(req, list, wb_list) {
535
+		if (req->wb_head != req) {
536
+			nfs_direct_add_page_head(&req->wb_list, req);
537
+			continue;
538
+		}
539
+		subreq = req->wb_this_page;
540
+		if (subreq == req)
541
+			continue;
542
+		do {
543
+			/*
544
+			 * Remove subrequests from this list before freeing
545
+			 * them in the call to nfs_join_page_group().
546
+			 */
547
+			if (!list_empty(&subreq->wb_list)) {
548
+				nfs_list_remove_request(subreq);
549
+				nfs_release_request(subreq);
550
+			}
551
+		} while ((subreq = subreq->wb_this_page) != req);
552
+		nfs_join_page_group(req, cinfo, inode);
553
+	}
554
+}
555
+
637556 static void
638557 nfs_direct_write_scan_commit_list(struct inode *inode,
639558 				  struct list_head *list,
640559 				  struct nfs_commit_info *cinfo)
641560 {
642561 	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
643
-#ifdef CONFIG_NFS_V4_1
644
-	if (cinfo->ds != NULL && cinfo->ds->nwritten != 0)
645
-		NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
646
-#endif
562
+	pnfs_recover_commit_reqs(list, cinfo);
647563 	nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
648564 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
649565 }
....@@ -655,33 +571,26 @@
655571 	LIST_HEAD(reqs);
656572 	struct nfs_commit_info cinfo;
657573 	LIST_HEAD(failed);
658
-	int i;
659574 
660575 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
661576 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
577
+
578
+	nfs_direct_join_group(&reqs, &cinfo, dreq->inode);
662579 
663580 	dreq->count = 0;
664581 	dreq->max_count = 0;
665582 	list_for_each_entry(req, &reqs, wb_list)
666583 		dreq->max_count += req->wb_bytes;
667
-	dreq->verf.committed = NFS_INVALID_STABLE_HOW;
668584 	nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
669
-	for (i = 0; i < dreq->mirror_count; i++)
670
-		dreq->mirrors[i].count = 0;
671585 	get_dreq(dreq);
672586 
673587 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
674588 			      &nfs_direct_write_completion_ops);
675589 	desc.pg_dreq = dreq;
676590 
677
-	req = nfs_list_entry(reqs.next);
678
-	nfs_direct_setup_mirroring(dreq, &desc, req);
679
-	if (desc.pg_error < 0) {
680
-		list_splice_init(&reqs, &failed);
681
-		goto out_failed;
682
-	}
683
-
684591 	list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
592
+		/* Bump the transmission count */
593
+		req->wb_nio++;
685594 		if (!nfs_pageio_add_request(&desc, req)) {
686595 			nfs_list_move_request(req, &failed);
687596 			spin_lock(&cinfo.inode->i_lock);
....@@ -696,7 +605,6 @@
696605 	}
697606 	nfs_pageio_complete(&desc);
698607 
699
-out_failed:
700608 	while (!list_empty(&failed)) {
701609 		req = nfs_list_entry(failed.next);
702610 		nfs_list_remove_request(req);
....@@ -709,27 +617,40 @@
709617 
710618 static void nfs_direct_commit_complete(struct nfs_commit_data *data)
711619 {
620
+	const struct nfs_writeverf *verf = data->res.verf;
712621 	struct nfs_direct_req *dreq = data->dreq;
713622 	struct nfs_commit_info cinfo;
714623 	struct nfs_page *req;
715624 	int status = data->task.tk_status;
716625 
626
+	if (status < 0) {
627
+		/* Errors in commit are fatal */
628
+		dreq->error = status;
629
+		dreq->max_count = 0;
630
+		dreq->count = 0;
631
+		dreq->flags = NFS_ODIRECT_DONE;
632
+	} else if (dreq->flags == NFS_ODIRECT_DONE)
633
+		status = dreq->error;
634
+
717635 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
718
-	if (status < 0 || nfs_direct_cmp_commit_data_verf(dreq, data))
719
-		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
720636 
721637 	while (!list_empty(&data->pages)) {
722638 		req = nfs_list_entry(data->pages.next);
723639 		nfs_list_remove_request(req);
724
-		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
725
-			/* Note the rewrite will go through mds */
640
+		if (status >= 0 && !nfs_write_match_verf(verf, req)) {
641
+			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
642
+			/*
643
+			 * Despite the reboot, the write was successful,
644
+			 * so reset wb_nio.
645
+			 */
646
+			req->wb_nio = 0;
726647 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
727
-		} else
648
+		} else /* Error or match */
728649 			nfs_release_request(req);
729650 		nfs_unlock_and_release_request(req);
730651 	}
731652 
732
-	if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
653
+	if (nfs_commit_end(cinfo.mds))
733654 		nfs_direct_write_complete(dreq);
734655 }
735656 
....@@ -739,7 +660,8 @@
739660 	struct nfs_direct_req *dreq = cinfo->dreq;
740661 
741662 	spin_lock(&dreq->lock);
742
-	dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
663
+	if (dreq->flags != NFS_ODIRECT_DONE)
664
+		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
743665 	spin_unlock(&dreq->lock);
744666 	nfs_mark_request_commit(req, NULL, cinfo, 0);
745667 }
....@@ -762,6 +684,23 @@
762684 		nfs_direct_write_reschedule(dreq);
763685 }
764686 
687
+static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)
688
+{
689
+	struct nfs_commit_info cinfo;
690
+	struct nfs_page *req;
691
+	LIST_HEAD(reqs);
692
+
693
+	nfs_init_cinfo_from_dreq(&cinfo, dreq);
694
+	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
695
+
696
+	while (!list_empty(&reqs)) {
697
+		req = nfs_list_entry(reqs.next);
698
+		nfs_list_remove_request(req);
699
+		nfs_release_request(req);
700
+		nfs_unlock_and_release_request(req);
701
+	}
702
+}
703
+
765704 static void nfs_direct_write_schedule_work(struct work_struct *work)
766705 {
767706 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
....@@ -776,6 +715,7 @@
776715 			nfs_direct_write_reschedule(dreq);
777716 			break;
778717 		default:
718
+			nfs_direct_write_clear_reqs(dreq);
779719 			nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
780720 			nfs_direct_complete(dreq);
781721 	}
....@@ -790,8 +730,8 @@
790730 {
791731 	struct nfs_direct_req *dreq = hdr->dreq;
792732 	struct nfs_commit_info cinfo;
793
-	bool request_commit = false;
794733 	struct nfs_page *req = nfs_list_entry(hdr->pages.next);
734
+	int flags = NFS_ODIRECT_DONE;
795735 
796736 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
797737 
....@@ -802,21 +742,10 @@
802742 	}
803743 
804744 	nfs_direct_count_bytes(dreq, hdr);
805
-	if (hdr->good_bytes != 0) {
806
-		if (nfs_write_need_commit(hdr)) {
807
-			if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
808
-				request_commit = true;
809
-			else if (dreq->flags == 0) {
810
-				nfs_direct_set_hdr_verf(dreq, hdr);
811
-				request_commit = true;
812
-				dreq->flags = NFS_ODIRECT_DO_COMMIT;
813
-			} else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
814
-				request_commit = true;
815
-				if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))
816
-					dreq->flags =
817
-						NFS_ODIRECT_RESCHED_WRITES;
818
-			}
819
-		}
745
+	if (hdr->good_bytes != 0 && nfs_write_need_commit(hdr)) {
746
+		if (!dreq->flags)
747
+			dreq->flags = NFS_ODIRECT_DO_COMMIT;
748
+		flags = dreq->flags;
820749 	}
821750 	spin_unlock(&dreq->lock);
822751 
....@@ -824,10 +753,15 @@
824753 
825754 		req = nfs_list_entry(hdr->pages.next);
826755 		nfs_list_remove_request(req);
827
-		if (request_commit) {
756
+		if (flags == NFS_ODIRECT_DO_COMMIT) {
828757 			kref_get(&req->wb_kref);
758
+			memcpy(&req->wb_verf, &hdr->verf.verifier,
759
+			       sizeof(req->wb_verf));
829760 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
830761 				hdr->ds_commit_idx);
762
+		} else if (flags == NFS_ODIRECT_RESCHED_WRITES) {
763
+			kref_get(&req->wb_kref);
764
+			nfs_mark_request_commit(req, NULL, &cinfo, 0);
831765 		}
832766 		nfs_unlock_and_release_request(req);
833767 	}
....@@ -858,7 +792,8 @@
858792 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
859793 		/* fake unstable write to let common nfs resend pages */
860794 		hdr->verf.committed = NFS_UNSTABLE;
861
-		hdr->good_bytes = hdr->args.count;
795
+		hdr->good_bytes = hdr->args.offset + hdr->args.count -
796
+			hdr->io_start;
862797 	}
863798 	spin_unlock(&dreq->lock);
864799 }
....@@ -884,7 +819,7 @@
884819  */
885820 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
886821 					       struct iov_iter *iter,
887
-					       loff_t pos)
822
+					       loff_t pos, int ioflags)
888823 {
889824 	struct nfs_pageio_descriptor desc;
890825 	struct inode *inode = dreq->inode;
....@@ -892,7 +827,7 @@
892827 	size_t requested_bytes = 0;
893828 	size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
894829 
895
-	nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
830
+	nfs_pageio_init_write(&desc, inode, ioflags, false,
896831 			      &nfs_direct_write_completion_ops);
897832 	desc.pg_dreq = dreq;
898833 	get_dreq(dreq);
....@@ -917,14 +852,13 @@
917852 			struct nfs_page *req;
918853 			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
919854 
920
-			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
855
+			req = nfs_create_request(dreq->ctx, pagevec[i],
921856 						 pgbase, req_len);
922857 			if (IS_ERR(req)) {
923858 				result = PTR_ERR(req);
924859 				break;
925860 			}
926861 
927
-			nfs_direct_setup_mirroring(dreq, &desc, req);
928862 			if (desc.pg_error < 0) {
929863 				nfs_free_request(req);
930864 				result = desc.pg_error;
....@@ -971,6 +905,7 @@
971905  * nfs_file_direct_write - file direct write operation for NFS files
972906  * @iocb: target I/O control block
973907  * @iter: vector of user buffers from which to write data
908
+ * @swap: flag indicating this is swap IO, not O_DIRECT IO
974909  *
975910  * We use this function for direct writes instead of calling
976911  * generic_file_aio_write() in order to avoid taking the inode
....@@ -987,9 +922,10 @@
987922  * Note that O_APPEND is not supported for NFS direct writes, as there
988923  * is no atomic O_APPEND write facility in the NFS protocol.
989924  */
990
-ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
925
+ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
926
+			      bool swap)
991927 {
992
-	ssize_t result = -EINVAL, requested;
928
+	ssize_t result, requested;
993929 	size_t count;
994930 	struct file *file = iocb->ki_filp;
995931 	struct address_space *mapping = file->f_mapping;
....@@ -1001,7 +937,11 @@
1001937 	dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
1002938 		file, iov_iter_count(iter), (long long) iocb->ki_pos);
1003939 
1004
-	result = generic_write_checks(iocb, iter);
940
+	if (swap)
941
+		/* bypass generic checks */
942
+		result =  iov_iter_count(iter);
943
+	else
944
+		result = generic_write_checks(iocb, iter);
1005945 	if (result <= 0)
1006946 		return result;
1007947 	count = result;
....@@ -1030,17 +970,24 @@
1030970 	dreq->l_ctx = l_ctx;
1031971 	if (!is_sync_kiocb(iocb))
1032972 		dreq->iocb = iocb;
973
+	pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);
1033974 
1034
-	nfs_start_io_direct(inode);
975
+	if (swap) {
976
+		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
977
+							    FLUSH_STABLE);
978
+	} else {
979
+		nfs_start_io_direct(inode);
1035980 
1036
-	requested = nfs_direct_write_schedule_iovec(dreq, iter, pos);
981
+		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
982
+							    FLUSH_COND_STABLE);
1037983 
1038
-	if (mapping->nrpages) {
1039
-		invalidate_inode_pages2_range(mapping,
1040
-					      pos >> PAGE_SHIFT, end);
984
+		if (mapping->nrpages) {
985
+			invalidate_inode_pages2_range(mapping,
986
+						      pos >> PAGE_SHIFT, end);
987
+		}
988
+
989
+		nfs_end_io_direct(inode);
1041990 	}
1042
-
1043
-	nfs_end_io_direct(inode);
1044991 
1045992 	if (requested > 0) {
1046993 		result = nfs_direct_wait(dreq);