forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 9999e48639b3cecb08ffb37358bcba3b48161b29 
 kernel/fs/nfs/pagelist.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * linux/fs/nfs/pagelist.c
34  *
....@@ -16,27 +17,42 @@
1617 #include <linux/nfs.h>
1718 #include <linux/nfs3.h>
1819 #include <linux/nfs4.h>
19
-#include <linux/nfs_page.h>
2020 #include <linux/nfs_fs.h>
21
+#include <linux/nfs_page.h>
2122 #include <linux/nfs_mount.h>
2223 #include <linux/export.h>
2324 
2425 #include "internal.h"
2526 #include "pnfs.h"
27
+#include "nfstrace.h"
2628 
2729 #define NFSDBG_FACILITY		NFSDBG_PAGECACHE
2830 
2931 static struct kmem_cache *nfs_page_cachep;
3032 static const struct rpc_call_ops nfs_pgio_common_ops;
3133 
34
+static struct nfs_pgio_mirror *
35
+nfs_pgio_get_mirror(struct nfs_pageio_descriptor *desc, u32 idx)
36
+{
37
+	if (desc->pg_ops->pg_get_mirror)
38
+		return desc->pg_ops->pg_get_mirror(desc, idx);
39
+	return &desc->pg_mirrors[0];
40
+}
41
+
3242 struct nfs_pgio_mirror *
3343 nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc)
3444 {
35
-	return nfs_pgio_has_mirroring(desc) ?
36
-		&desc->pg_mirrors[desc->pg_mirror_idx] :
37
-		&desc->pg_mirrors[0];
45
+	return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx);
3846 }
3947 EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror);
48
+
49
+static u32
50
+nfs_pgio_set_current_mirror(struct nfs_pageio_descriptor *desc, u32 idx)
51
+{
52
+	if (desc->pg_ops->pg_set_mirror)
53
+		return desc->pg_ops->pg_set_mirror(desc, idx);
54
+	return desc->pg_mirror_idx;
55
+}
4056 
4157 void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
4258 		       struct nfs_pgio_header *hdr,
....@@ -47,7 +63,7 @@
4763 
4864 	hdr->req = nfs_list_entry(mirror->pg_list.next);
4965 	hdr->inode = desc->pg_inode;
50
-	hdr->cred = hdr->req->wb_context->cred;
66
+	hdr->cred = nfs_req_openctx(hdr->req)->cred;
5167 	hdr->io_start = req_offset(hdr->req);
5268 	hdr->good_bytes = mirror->pg_count;
5369 	hdr->io_completion = desc->pg_io_completion;
....@@ -63,20 +79,21 @@
6379 
6480 void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
6581 {
66
-	spin_lock(&hdr->lock);
67
-	if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags)
68
-	    || pos < hdr->io_start + hdr->good_bytes) {
82
+	unsigned int new = pos - hdr->io_start;
83
+
84
+	trace_nfs_pgio_error(hdr, error, pos);
85
+	if (hdr->good_bytes > new) {
86
+		hdr->good_bytes = new;
6987 		clear_bit(NFS_IOHDR_EOF, &hdr->flags);
70
-		hdr->good_bytes = pos - hdr->io_start;
71
-		hdr->error = error;
88
+		if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags))
89
+			hdr->error = error;
7290 	}
73
-	spin_unlock(&hdr->lock);
7491 }
7592 
76
-static inline struct nfs_page *
77
-nfs_page_alloc(void)
93
+static inline struct nfs_page *nfs_page_alloc(void)
7894 {
79
-	struct nfs_page	*p = kmem_cache_zalloc(nfs_page_cachep, GFP_NOIO);
95
+	struct nfs_page *p =
96
+		kmem_cache_zalloc(nfs_page_cachep, nfs_io_gfp_mask());
8097 	if (p)
8198 		INIT_LIST_HEAD(&p->wb_list);
8299 	return p;
....@@ -130,6 +147,102 @@
130147 	return ret;
131148 }
132149 EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait);
150
+
151
+/*
152
+ * nfs_page_lock_head_request - page lock the head of the page group
153
+ * @req: any member of the page group
154
+ */
155
+struct nfs_page *
156
+nfs_page_group_lock_head(struct nfs_page *req)
157
+{
158
+	struct nfs_page *head = req->wb_head;
159
+
160
+	while (!nfs_lock_request(head)) {
161
+		int ret = nfs_wait_on_request(head);
162
+		if (ret < 0)
163
+			return ERR_PTR(ret);
164
+	}
165
+	if (head != req)
166
+		kref_get(&head->wb_kref);
167
+	return head;
168
+}
169
+
170
+/*
171
+ * nfs_unroll_locks -  unlock all newly locked reqs and wait on @req
172
+ * @head: head request of page group, must be holding head lock
173
+ * @req: request that couldn't lock and needs to wait on the req bit lock
174
+ *
175
+ * This is a helper function for nfs_lock_and_join_requests
176
+ * returns 0 on success, < 0 on error.
177
+ */
178
+static void
179
+nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req)
180
+{
181
+	struct nfs_page *tmp;
182
+
183
+	/* relinquish all the locks successfully grabbed this run */
184
+	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
185
+		if (!kref_read(&tmp->wb_kref))
186
+			continue;
187
+		nfs_unlock_and_release_request(tmp);
188
+	}
189
+}
190
+
191
+/*
192
+ * nfs_page_group_lock_subreq -  try to lock a subrequest
193
+ * @head: head request of page group
194
+ * @subreq: request to lock
195
+ *
196
+ * This is a helper function for nfs_lock_and_join_requests which
197
+ * must be called with the head request and page group both locked.
198
+ * On error, it returns with the page group unlocked.
199
+ */
200
+static int
201
+nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq)
202
+{
203
+	int ret;
204
+
205
+	if (!kref_get_unless_zero(&subreq->wb_kref))
206
+		return 0;
207
+	while (!nfs_lock_request(subreq)) {
208
+		nfs_page_group_unlock(head);
209
+		ret = nfs_wait_on_request(subreq);
210
+		if (!ret)
211
+			ret = nfs_page_group_lock(head);
212
+		if (ret < 0) {
213
+			nfs_unroll_locks(head, subreq);
214
+			nfs_release_request(subreq);
215
+			return ret;
216
+		}
217
+	}
218
+	return 0;
219
+}
220
+
221
+/*
222
+ * nfs_page_group_lock_subrequests -  try to lock the subrequests
223
+ * @head: head request of page group
224
+ *
225
+ * This is a helper function for nfs_lock_and_join_requests which
226
+ * must be called with the head request locked.
227
+ */
228
+int nfs_page_group_lock_subrequests(struct nfs_page *head)
229
+{
230
+	struct nfs_page *subreq;
231
+	int ret;
232
+
233
+	ret = nfs_page_group_lock(head);
234
+	if (ret < 0)
235
+		return ret;
236
+	/* lock each request in the page group */
237
+	for (subreq = head->wb_this_page; subreq != head;
238
+			subreq = subreq->wb_this_page) {
239
+		ret = nfs_page_group_lock_subreq(head, subreq);
240
+		if (ret < 0)
241
+			return ret;
242
+	}
243
+	nfs_page_group_unlock(head);
244
+	return 0;
245
+}
133246 
134247 /*
135248  * nfs_page_set_headlock - set the request PG_HEADLOCK
....@@ -318,25 +431,13 @@
318431 		nfs_release_request(head);
319432 }
320433 
321
-/**
322
- * nfs_create_request - Create an NFS read/write request.
323
- * @ctx: open context to use
324
- * @page: page to write
325
- * @last: last nfs request created for this page group or NULL if head
326
- * @offset: starting offset within the page for the write
327
- * @count: number of bytes to read/write
328
- *
329
- * The page must be locked by the caller. This makes sure we never
330
- * create two different requests for the same page.
331
- * User should ensure it is safe to sleep in this function.
332
- */
333
-struct nfs_page *
334
-nfs_create_request(struct nfs_open_context *ctx, struct page *page,
335
-		   struct nfs_page *last, unsigned int offset,
434
+static struct nfs_page *
435
+__nfs_create_request(struct nfs_lock_context *l_ctx, struct page *page,
436
+		   unsigned int pgbase, unsigned int offset,
336437 		   unsigned int count)
337438 {
338439 	struct nfs_page		*req;
339
-	struct nfs_lock_context *l_ctx;
440
+	struct nfs_open_context *ctx = l_ctx->open_context;
340441 
341442 	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
342443 		return ERR_PTR(-EBADF);
....@@ -345,13 +446,8 @@
345446 	if (req == NULL)
346447 		return ERR_PTR(-ENOMEM);
347448 
348
-	/* get lock context early so we can deal with alloc failures */
349
-	l_ctx = nfs_get_lock_context(ctx);
350
-	if (IS_ERR(l_ctx)) {
351
-		nfs_page_free(req);
352
-		return ERR_CAST(l_ctx);
353
-	}
354449 	req->wb_lock_context = l_ctx;
450
+	refcount_inc(&l_ctx->count);
355451 	atomic_inc(&l_ctx->io_count);
356452 
357453 	/* Initialize the request struct. Initially, we assume a
....@@ -363,17 +459,69 @@
363459 		get_page(page);
364460 	}
365461 	req->wb_offset  = offset;
366
-	req->wb_pgbase	= offset;
462
+	req->wb_pgbase	= pgbase;
367463 	req->wb_bytes   = count;
368
-	req->wb_context = get_nfs_open_context(ctx);
369464 	kref_init(&req->wb_kref);
370
-	nfs_page_group_init(req, last);
465
+	req->wb_nio = 0;
371466 	return req;
372467 }
373468 
374469 /**
470
+ * nfs_create_request - Create an NFS read/write request.
471
+ * @ctx: open context to use
472
+ * @page: page to write
473
+ * @offset: starting offset within the page for the write
474
+ * @count: number of bytes to read/write
475
+ *
476
+ * The page must be locked by the caller. This makes sure we never
477
+ * create two different requests for the same page.
478
+ * User should ensure it is safe to sleep in this function.
479
+ */
480
+struct nfs_page *
481
+nfs_create_request(struct nfs_open_context *ctx, struct page *page,
482
+		   unsigned int offset, unsigned int count)
483
+{
484
+	struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx);
485
+	struct nfs_page *ret;
486
+
487
+	if (IS_ERR(l_ctx))
488
+		return ERR_CAST(l_ctx);
489
+	ret = __nfs_create_request(l_ctx, page, offset, offset, count);
490
+	if (!IS_ERR(ret))
491
+		nfs_page_group_init(ret, NULL);
492
+	nfs_put_lock_context(l_ctx);
493
+	return ret;
494
+}
495
+
496
+static struct nfs_page *
497
+nfs_create_subreq(struct nfs_page *req,
498
+		  unsigned int pgbase,
499
+		  unsigned int offset,
500
+		  unsigned int count)
501
+{
502
+	struct nfs_page *last;
503
+	struct nfs_page *ret;
504
+
505
+	ret = __nfs_create_request(req->wb_lock_context, req->wb_page,
506
+			pgbase, offset, count);
507
+	if (!IS_ERR(ret)) {
508
+		/* find the last request */
509
+		for (last = req->wb_head;
510
+		     last->wb_this_page != req->wb_head;
511
+		     last = last->wb_this_page)
512
+			;
513
+
514
+		nfs_lock_request(ret);
515
+		ret->wb_index = req->wb_index;
516
+		nfs_page_group_init(ret, last);
517
+		ret->wb_nio = req->wb_nio;
518
+	}
519
+	return ret;
520
+}
521
+
522
+/**
375523  * nfs_unlock_request - Unlock request and wake up sleepers.
376
- * @req:
524
+ * @req: pointer to request
377525  */
378526 void nfs_unlock_request(struct nfs_page *req)
379527 {
....@@ -391,7 +539,7 @@
391539 
392540 /**
393541  * nfs_unlock_and_release_request - Unlock request and release the nfs_page
394
- * @req:
542
+ * @req: pointer to request
395543  */
396544 void nfs_unlock_and_release_request(struct nfs_page *req)
397545 {
....@@ -409,8 +557,8 @@
409557 static void nfs_clear_request(struct nfs_page *req)
410558 {
411559 	struct page *page = req->wb_page;
412
-	struct nfs_open_context *ctx = req->wb_context;
413560 	struct nfs_lock_context *l_ctx = req->wb_lock_context;
561
+	struct nfs_open_context *ctx;
414562 
415563 	if (page != NULL) {
416564 		put_page(page);
....@@ -419,15 +567,12 @@
419567 	if (l_ctx != NULL) {
420568 		if (atomic_dec_and_test(&l_ctx->io_count)) {
421569 			wake_up_var(&l_ctx->io_count);
570
+			ctx = l_ctx->open_context;
422571 			if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))
423572 				rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq);
424573 		}
425574 		nfs_put_lock_context(l_ctx);
426575 		req->wb_lock_context = NULL;
427
-	}
428
-	if (ctx != NULL) {
429
-		put_nfs_open_context(ctx);
430
-		req->wb_context = NULL;
431576 	}
432577 }
433578 
....@@ -484,7 +629,7 @@
484629  * @prev: previous request in desc, or NULL
485630  * @req: this request
486631  *
487
- * Returns zero if @req can be coalesced into @desc, otherwise it returns
632
+ * Returns zero if @req cannot be coalesced into @desc, otherwise it returns
488633  * the size of the request.
489634  */
490635 size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
....@@ -517,7 +662,6 @@
517662 
518663 	if (hdr) {
519664 		INIT_LIST_HEAD(&hdr->pages);
520
-		spin_lock_init(&hdr->lock);
521665 		hdr->rw_ops = ops;
522666 	}
523667 	return hdr;
....@@ -555,7 +699,6 @@
555699  * nfs_pgio_rpcsetup - Set up arguments for a pageio call
556700  * @hdr: The pageio hdr
557701  * @count: Number of bytes to read
558
- * @offset: Initial offset
559702  * @how: How to commit data (writes only)
560703  * @cinfo: Commit information for the call (writes only)
561704  */
....@@ -575,7 +718,7 @@
575718 	hdr->args.pgbase = req->wb_pgbase;
576719 	hdr->args.pages  = hdr->page_array.pagevec;
577720 	hdr->args.count  = count;
578
-	hdr->args.context = get_nfs_open_context(req->wb_context);
721
+	hdr->args.context = get_nfs_open_context(nfs_req_openctx(req));
579722 	hdr->args.lock_context = req->wb_lock_context;
580723 	hdr->args.stable  = NFS_UNSTABLE;
581724 	switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
....@@ -584,7 +727,7 @@
584727 	case FLUSH_COND_STABLE:
585728 		if (nfs_reqs_to_commit(cinfo))
586729 			break;
587
-		/* fall through */
730
+		fallthrough;
588731 	default:
589732 		hdr->args.stable = NFS_FILE_SYNC;
590733 	}
....@@ -611,7 +754,7 @@
611754 }
612755 
613756 int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
614
-		      struct rpc_cred *cred, const struct nfs_rpc_ops *rpc_ops,
757
+		      const struct cred *cred, const struct nfs_rpc_ops *rpc_ops,
615758 		      const struct rpc_call_ops *call_ops, int how, int flags)
616759 {
617760 	struct rpc_task *task;
....@@ -629,7 +772,6 @@
629772 		.workqueue = nfsiod_workqueue,
630773 		.flags = RPC_TASK_ASYNC | flags,
631774 	};
632
-	int ret = 0;
633775 
634776 	hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
635777 
....@@ -641,24 +783,15 @@
641783 		(unsigned long long)hdr->args.offset);
642784 
643785 	task = rpc_run_task(&task_setup_data);
644
-	if (IS_ERR(task)) {
645
-		ret = PTR_ERR(task);
646
-		goto out;
647
-	}
648
-	if (how & FLUSH_SYNC) {
649
-		ret = rpc_wait_for_completion_task(task);
650
-		if (ret == 0)
651
-			ret = task->tk_status;
652
-	}
786
+	if (IS_ERR(task))
787
+		return PTR_ERR(task);
653788 	rpc_put_task(task);
654
-out:
655
-	return ret;
789
+	return 0;
656790 }
657791 EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
658792 
659793 /**
660794  * nfs_pgio_error - Clean up from a pageio error
661
- * @desc: IO descriptor
662795  * @hdr: pageio header
663796  */
664797 static void nfs_pgio_error(struct nfs_pgio_header *hdr)
....@@ -724,6 +857,7 @@
724857 	desc->pg_mirrors_dynamic = NULL;
725858 	desc->pg_mirrors = desc->pg_mirrors_static;
726859 	nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
860
+	desc->pg_maxretrans = 0;
727861 }
728862 
729863 /**
....@@ -767,7 +901,7 @@
767901 	struct nfs_commit_info cinfo;
768902 	struct nfs_page_array *pg_array = &hdr->page_array;
769903 	unsigned int pagecount, pageused;
770
-	gfp_t gfp_flags = GFP_KERNEL;
904
+	gfp_t gfp_flags = nfs_io_gfp_mask();
771905 
772906 	pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
773907 	pg_array->npages = pagecount;
....@@ -775,8 +909,6 @@
775909 	if (pagecount <= ARRAY_SIZE(pg_array->page_array))
776910 		pg_array->pagevec = pg_array->page_array;
777911 	else {
778
-		if (hdr->rw_mode == FMODE_WRITE)
779
-			gfp_flags = GFP_NOIO;
780912 		pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags);
781913 		if (!pg_array->pagevec) {
782914 			pg_array->npages = 0;
....@@ -836,7 +968,8 @@
836968 					hdr->cred,
837969 					NFS_PROTO(hdr->inode),
838970 					desc->pg_rpc_callops,
839
-					desc->pg_ioflags, 0);
971
+					desc->pg_ioflags,
972
+					RPC_TASK_CRED_NOREF);
840973 	return ret;
841974 }
842975 
....@@ -851,7 +984,7 @@
851984 	desc->pg_mirrors_dynamic = NULL;
852985 	if (mirror_count == 1)
853986 		return desc->pg_mirrors_static;
854
-	ret = kmalloc_array(mirror_count, sizeof(*ret), GFP_NOFS);
987
+	ret = kmalloc_array(mirror_count, sizeof(*ret), nfs_io_gfp_mask());
855988 	if (ret != NULL) {
856989 		for (i = 0; i < mirror_count; i++)
857990 			nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);
....@@ -904,49 +1037,45 @@
9041037 }
9051038 
9061039 /**
907
- * nfs_can_coalesce_requests - test two requests for compatibility
1040
+ * nfs_coalesce_size - test two requests for compatibility
9081041  * @prev: pointer to nfs_page
9091042  * @req: pointer to nfs_page
1043
+ * @pgio: pointer to nfs_pagio_descriptor
9101044  *
9111045  * The nfs_page structures 'prev' and 'req' are compared to ensure that the
9121046  * page data area they describe is contiguous, and that their RPC
9131047  * credentials, NFSv4 open state, and lockowners are the same.
9141048  *
915
- * Return 'true' if this is the case, else return 'false'.
1049
+ * Returns size of the request that can be coalesced
9161050  */
917
-static bool nfs_can_coalesce_requests(struct nfs_page *prev,
1051
+static unsigned int nfs_coalesce_size(struct nfs_page *prev,
9181052 				      struct nfs_page *req,
9191053 				      struct nfs_pageio_descriptor *pgio)
9201054 {
921
-	size_t size;
9221055 	struct file_lock_context *flctx;
9231056 
9241057 	if (prev) {
925
-		if (!nfs_match_open_context(req->wb_context, prev->wb_context))
926
-			return false;
927
-		flctx = d_inode(req->wb_context->dentry)->i_flctx;
1058
+		if (!nfs_match_open_context(nfs_req_openctx(req), nfs_req_openctx(prev)))
1059
+			return 0;
1060
+		flctx = d_inode(nfs_req_openctx(req)->dentry)->i_flctx;
9281061 		if (flctx != NULL &&
9291062 		    !(list_empty_careful(&flctx->flc_posix) &&
9301063 		      list_empty_careful(&flctx->flc_flock)) &&
9311064 		    !nfs_match_lock_context(req->wb_lock_context,
9321065 					    prev->wb_lock_context))
933
-			return false;
1066
+			return 0;
9341067 		if (req_offset(req) != req_offset(prev) + prev->wb_bytes)
935
-			return false;
1068
+			return 0;
9361069 		if (req->wb_page == prev->wb_page) {
9371070 			if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)
938
-				return false;
1071
+				return 0;
9391072 		} else {
9401073 			if (req->wb_pgbase != 0 ||
9411074 			    prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE)
942
-				return false;
1075
+				return 0;
9431076 		}
9441077 	}
945
-	size = pgio->pg_ops->pg_test(pgio, prev, req);
946
-	WARN_ON_ONCE(size > req->wb_bytes);
947
-	if (size && size < req->wb_bytes)
948
-		req->wb_bytes = size;
949
-	return size > 0;
1078
+	return pgio->pg_ops->pg_test(pgio, prev, req);
9501079 }
9511080 
9521081 /**
....@@ -954,30 +1083,42 @@
9541083  * @desc: destination io descriptor
9551084  * @req: request
9561085  *
957
- * Returns true if the request 'req' was successfully coalesced into the
958
- * existing list of pages 'desc'.
1086
+ * If the request 'req' was successfully coalesced into the existing list
1087
+ * of pages 'desc', it returns the size of req.
9591088  */
960
-static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
961
-				     struct nfs_page *req)
1089
+static unsigned int
1090
+nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
1091
+		struct nfs_page *req)
9621092 {
9631093 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
964
-
9651094 	struct nfs_page *prev = NULL;
1095
+	unsigned int size;
9661096 
967
-	if (mirror->pg_count != 0) {
968
-		prev = nfs_list_entry(mirror->pg_list.prev);
969
-	} else {
1097
+	if (list_empty(&mirror->pg_list)) {
9701098 		if (desc->pg_ops->pg_init)
9711099 			desc->pg_ops->pg_init(desc, req);
9721100 		if (desc->pg_error < 0)
9731101 			return 0;
9741102 		mirror->pg_base = req->wb_pgbase;
975
-	}
976
-	if (!nfs_can_coalesce_requests(prev, req, desc))
1103
+		mirror->pg_count = 0;
1104
+		mirror->pg_recoalesce = 0;
1105
+	} else
1106
+		prev = nfs_list_entry(mirror->pg_list.prev);
1107
+
1108
+	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {
1109
+		if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)
1110
+			desc->pg_error = -ETIMEDOUT;
1111
+		else
1112
+			desc->pg_error = -EIO;
9771113 		return 0;
1114
+	}
1115
+
1116
+	size = nfs_coalesce_size(prev, req, desc);
1117
+	if (size < req->wb_bytes)
1118
+		return size;
9781119 	nfs_list_move_request(req, &mirror->pg_list);
9791120 	mirror->pg_count += req->wb_bytes;
980
-	return 1;
1121
+	return req->wb_bytes;
9811122 }
9821123 
9831124 /*
....@@ -1016,7 +1157,8 @@
10161157  * @req: request
10171158  *
10181159  * This may split a request into subrequests which are all part of the
1019
- * same page group.
1160
+ * same page group. If so, it will submit @req as the last one, to ensure
1161
+ * the pointer to @req is still valid in case of failure.
10201162  *
10211163  * Returns true if the request 'req' was successfully coalesced into the
10221164  * existing list of pages 'desc'.
....@@ -1025,65 +1167,56 @@
10251167 			   struct nfs_page *req)
10261168 {
10271169 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1028
-
10291170 	struct nfs_page *subreq;
1030
-	unsigned int bytes_left = 0;
1031
-	unsigned int offset, pgbase;
1171
+	unsigned int size, subreq_size;
10321172 
10331173 	nfs_page_group_lock(req);
10341174 
10351175 	subreq = req;
1036
-	bytes_left = subreq->wb_bytes;
1037
-	offset = subreq->wb_offset;
1038
-	pgbase = subreq->wb_pgbase;
1039
-
1040
-	do {
1041
-		if (!nfs_pageio_do_add_request(desc, subreq)) {
1042
-			/* make sure pg_test call(s) did nothing */
1043
-			WARN_ON_ONCE(subreq->wb_bytes != bytes_left);
1044
-			WARN_ON_ONCE(subreq->wb_offset != offset);
1045
-			WARN_ON_ONCE(subreq->wb_pgbase != pgbase);
1046
-
1176
+	subreq_size = subreq->wb_bytes;
1177
+	for(;;) {
1178
+		size = nfs_pageio_do_add_request(desc, subreq);
1179
+		if (size == subreq_size) {
1180
+			/* We successfully submitted a request */
1181
+			if (subreq == req)
1182
+				break;
1183
+			req->wb_pgbase += size;
1184
+			req->wb_bytes -= size;
1185
+			req->wb_offset += size;
1186
+			subreq_size = req->wb_bytes;
1187
+			subreq = req;
1188
+			continue;
1189
+		}
1190
+		if (WARN_ON_ONCE(subreq != req)) {
1191
+			nfs_page_group_unlock(req);
1192
+			nfs_pageio_cleanup_request(desc, subreq);
1193
+			subreq = req;
1194
+			subreq_size = req->wb_bytes;
1195
+			nfs_page_group_lock(req);
1196
+		}
1197
+		if (!size) {
1198
+			/* Can't coalesce any more, so do I/O */
10471199 			nfs_page_group_unlock(req);
10481200 			desc->pg_moreio = 1;
10491201 			nfs_pageio_doio(desc);
10501202 			if (desc->pg_error < 0 || mirror->pg_recoalesce)
1051
-				goto out_cleanup_subreq;
1203
+				return 0;
10521204 			/* retry add_request for this subreq */
10531205 			nfs_page_group_lock(req);
10541206 			continue;
10551207 		}
1056
-
1057
-		/* check for buggy pg_test call(s) */
1058
-		WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE);
1059
-		WARN_ON_ONCE(subreq->wb_bytes > bytes_left);
1060
-		WARN_ON_ONCE(subreq->wb_bytes == 0);
1061
-
1062
-		bytes_left -= subreq->wb_bytes;
1063
-		offset += subreq->wb_bytes;
1064
-		pgbase += subreq->wb_bytes;
1065
-
1066
-		if (bytes_left) {
1067
-			subreq = nfs_create_request(req->wb_context,
1068
-					req->wb_page,
1069
-					subreq, pgbase, bytes_left);
1070
-			if (IS_ERR(subreq))
1071
-				goto err_ptr;
1072
-			nfs_lock_request(subreq);
1073
-			subreq->wb_offset  = offset;
1074
-			subreq->wb_index = req->wb_index;
1075
-		}
1076
-	} while (bytes_left > 0);
1208
+		subreq = nfs_create_subreq(req, req->wb_pgbase,
1209
+				req->wb_offset, size);
1210
+		if (IS_ERR(subreq))
1211
+			goto err_ptr;
1212
+		subreq_size = size;
1213
+	}
10771214 
10781215 	nfs_page_group_unlock(req);
10791216 	return 1;
10801217 err_ptr:
10811218 	desc->pg_error = PTR_ERR(subreq);
10821219 	nfs_page_group_unlock(req);
1083
-	return 0;
1084
-out_cleanup_subreq:
1085
-	if (req != subreq)
1086
-		nfs_pageio_cleanup_request(desc, subreq);
10871220 	return 0;
10881221 }
10891222 
....@@ -1141,7 +1274,7 @@
11411274 		return;
11421275 
11431276 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1144
-		mirror = &desc->pg_mirrors[midx];
1277
+		mirror = nfs_pgio_get_mirror(desc, midx);
11451278 		desc->pg_completion_ops->error_cleanup(&mirror->pg_list,
11461279 				desc->pg_error);
11471280 	}
....@@ -1152,7 +1285,7 @@
11521285 {
11531286 	u32 midx;
11541287 	unsigned int pgbase, offset, bytes;
1155
-	struct nfs_page *dupreq, *lastreq;
1288
+	struct nfs_page *dupreq;
11561289 
11571290 	pgbase = req->wb_pgbase;
11581291 	offset = req->wb_offset;
....@@ -1162,48 +1295,33 @@
11621295 	if (desc->pg_error < 0)
11631296 		goto out_failed;
11641297 
1165
-	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1166
-		if (midx) {
1167
-			nfs_page_group_lock(req);
1298
+	/* Create the mirror instances first, and fire them off */
1299
+	for (midx = 1; midx < desc->pg_mirror_count; midx++) {
1300
+		nfs_page_group_lock(req);
11681301 
1169
-			/* find the last request */
1170
-			for (lastreq = req->wb_head;
1171
-			     lastreq->wb_this_page != req->wb_head;
1172
-			     lastreq = lastreq->wb_this_page)
1173
-				;
1302
+		dupreq = nfs_create_subreq(req,
1303
+				pgbase, offset, bytes);
11741304 
1175
-			dupreq = nfs_create_request(req->wb_context,
1176
-					req->wb_page, lastreq, pgbase, bytes);
1305
+		nfs_page_group_unlock(req);
1306
+		if (IS_ERR(dupreq)) {
1307
+			desc->pg_error = PTR_ERR(dupreq);
1308
+			goto out_failed;
1309
+		}
11771310 
1178
-			if (IS_ERR(dupreq)) {
1179
-				nfs_page_group_unlock(req);
1180
-				desc->pg_error = PTR_ERR(dupreq);
1181
-				goto out_failed;
1182
-			}
1183
-
1184
-			nfs_lock_request(dupreq);
1185
-			nfs_page_group_unlock(req);
1186
-			dupreq->wb_offset = offset;
1187
-			dupreq->wb_index = req->wb_index;
1188
-		} else
1189
-			dupreq = req;
1190
-
1191
-		if (nfs_pgio_has_mirroring(desc))
1192
-			desc->pg_mirror_idx = midx;
1311
+		nfs_pgio_set_current_mirror(desc, midx);
11931312 		if (!nfs_pageio_add_request_mirror(desc, dupreq))
11941313 			goto out_cleanup_subreq;
11951314 	}
11961315 
1316
+	nfs_pgio_set_current_mirror(desc, 0);
1317
+	if (!nfs_pageio_add_request_mirror(desc, req))
1318
+		goto out_failed;
1319
+
11971320 	return 1;
11981321 
11991322 out_cleanup_subreq:
1200
-	if (req != dupreq)
1201
-		nfs_pageio_cleanup_request(desc, dupreq);
1323
+	nfs_pageio_cleanup_request(desc, dupreq);
12021324 out_failed:
1203
-	/* remember fatal errors */
1204
-	if (nfs_error_is_fatal(desc->pg_error))
1205
-		nfs_context_set_write_error(req->wb_context,
1206
-						desc->pg_error);
12071325 	nfs_pageio_error_cleanup(desc);
12081326 	return 0;
12091327 }
....@@ -1217,11 +1335,12 @@
12171335 static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
12181336 				       u32 mirror_idx)
12191337 {
1220
-	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx];
1221
-	u32 restore_idx = desc->pg_mirror_idx;
1338
+	struct nfs_pgio_mirror *mirror;
1339
+	u32 restore_idx;
12221340 
1223
-	if (nfs_pgio_has_mirroring(desc))
1224
-		desc->pg_mirror_idx = mirror_idx;
1341
+	restore_idx = nfs_pgio_set_current_mirror(desc, mirror_idx);
1342
+	mirror = nfs_pgio_current_mirror(desc);
1343
+
12251344 	for (;;) {
12261345 		nfs_pageio_doio(desc);
12271346 		if (desc->pg_error < 0 || !mirror->pg_recoalesce)
....@@ -1229,7 +1348,7 @@
12291348 		if (!nfs_do_recoalesce(desc))
12301349 			break;
12311350 	}
1232
-	desc->pg_mirror_idx = restore_idx;
1351
+	nfs_pgio_set_current_mirror(desc, restore_idx);
12331352 }
12341353 
12351354 /*
....@@ -1303,7 +1422,7 @@
13031422 	u32 midx;
13041423 
13051424 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1306
-		mirror = &desc->pg_mirrors[midx];
1425
+		mirror = nfs_pgio_get_mirror(desc, midx);
13071426 		if (!list_empty(&mirror->pg_list)) {
13081427 			prev = nfs_list_entry(mirror->pg_list.prev);
13091428 			if (index != prev->wb_index + 1) {