forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-10-22 8ac6c7a54ed1b98d142dce24b11c6de6a1e239a5 
 kernel/net/sunrpc/xdr.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * linux/net/sunrpc/xdr.c
34  *
....@@ -15,6 +16,11 @@
1516 #include <linux/errno.h>
1617 #include <linux/sunrpc/xdr.h>
1718 #include <linux/sunrpc/msg_prot.h>
19
+#include <linux/bvec.h>
20
+#include <trace/events/sunrpc.h>
21
+
22
+static void _copy_to_pages(struct page **, size_t, const char *, size_t);
23
+
1824 
1925 /*
2026  * XDR functions for basic NFS types
....@@ -128,6 +134,48 @@
128134 }
129135 EXPORT_SYMBOL_GPL(xdr_terminate_string);
130136 
137
+size_t
138
+xdr_buf_pagecount(struct xdr_buf *buf)
139
+{
140
+	if (!buf->page_len)
141
+		return 0;
142
+	return (buf->page_base + buf->page_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
143
+}
144
+
145
+int
146
+xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp)
147
+{
148
+	size_t i, n = xdr_buf_pagecount(buf);
149
+
150
+	if (n != 0 && buf->bvec == NULL) {
151
+		buf->bvec = kmalloc_array(n, sizeof(buf->bvec[0]), gfp);
152
+		if (!buf->bvec)
153
+			return -ENOMEM;
154
+		for (i = 0; i < n; i++) {
155
+			buf->bvec[i].bv_page = buf->pages[i];
156
+			buf->bvec[i].bv_len = PAGE_SIZE;
157
+			buf->bvec[i].bv_offset = 0;
158
+		}
159
+	}
160
+	return 0;
161
+}
162
+
163
+void
164
+xdr_free_bvec(struct xdr_buf *buf)
165
+{
166
+	kfree(buf->bvec);
167
+	buf->bvec = NULL;
168
+}
169
+
170
+/**
171
+ * xdr_inline_pages - Prepare receive buffer for a large reply
172
+ * @xdr: xdr_buf into which reply will be placed
173
+ * @offset: expected offset where data payload will start, in bytes
174
+ * @pages: vector of struct page pointers
175
+ * @base: offset in first page where receive should start, in bytes
176
+ * @len: expected size of the upper layer data payload, in bytes
177
+ *
178
+ */
131179 void
132180 xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,
133181 		 struct page **pages, unsigned int base, unsigned int len)
....@@ -145,6 +193,8 @@
145193 
146194 	tail->iov_base = buf + offset;
147195 	tail->iov_len = buflen - offset;
196
+	if ((xdr->page_len & 3) == 0)
197
+		tail->iov_len -= sizeof(__be32);
148198 
149199 	xdr->buflen += len;
150200 }
....@@ -153,6 +203,88 @@
153203 /*
154204  * Helper routines for doing 'memmove' like operations on a struct xdr_buf
155205  */
206
+
207
+/**
208
+ * _shift_data_left_pages
209
+ * @pages: vector of pages containing both the source and dest memory area.
210
+ * @pgto_base: page vector address of destination
211
+ * @pgfrom_base: page vector address of source
212
+ * @len: number of bytes to copy
213
+ *
214
+ * Note: the addresses pgto_base and pgfrom_base are both calculated in
215
+ *       the same way:
216
+ *            if a memory area starts at byte 'base' in page 'pages[i]',
217
+ *            then its address is given as (i << PAGE_CACHE_SHIFT) + base
218
+ * Alse note: pgto_base must be < pgfrom_base, but the memory areas
219
+ * 	they point to may overlap.
220
+ */
221
+static void
222
+_shift_data_left_pages(struct page **pages, size_t pgto_base,
223
+			size_t pgfrom_base, size_t len)
224
+{
225
+	struct page **pgfrom, **pgto;
226
+	char *vfrom, *vto;
227
+	size_t copy;
228
+
229
+	BUG_ON(pgfrom_base <= pgto_base);
230
+
231
+	pgto = pages + (pgto_base >> PAGE_SHIFT);
232
+	pgfrom = pages + (pgfrom_base >> PAGE_SHIFT);
233
+
234
+	pgto_base &= ~PAGE_MASK;
235
+	pgfrom_base &= ~PAGE_MASK;
236
+
237
+	do {
238
+		if (pgto_base >= PAGE_SIZE) {
239
+			pgto_base = 0;
240
+			pgto++;
241
+		}
242
+		if (pgfrom_base >= PAGE_SIZE){
243
+			pgfrom_base = 0;
244
+			pgfrom++;
245
+		}
246
+
247
+		copy = len;
248
+		if (copy > (PAGE_SIZE - pgto_base))
249
+			copy = PAGE_SIZE - pgto_base;
250
+		if (copy > (PAGE_SIZE - pgfrom_base))
251
+			copy = PAGE_SIZE - pgfrom_base;
252
+
253
+		vto = kmap_atomic(*pgto);
254
+		if (*pgto != *pgfrom) {
255
+			vfrom = kmap_atomic(*pgfrom);
256
+			memcpy(vto + pgto_base, vfrom + pgfrom_base, copy);
257
+			kunmap_atomic(vfrom);
258
+		} else
259
+			memmove(vto + pgto_base, vto + pgfrom_base, copy);
260
+		flush_dcache_page(*pgto);
261
+		kunmap_atomic(vto);
262
+
263
+		pgto_base += copy;
264
+		pgfrom_base += copy;
265
+
266
+	} while ((len -= copy) != 0);
267
+}
268
+
269
+static void
270
+_shift_data_left_tail(struct xdr_buf *buf, unsigned int pgto, size_t len)
271
+{
272
+	struct kvec *tail = buf->tail;
273
+
274
+	if (len > tail->iov_len)
275
+		len = tail->iov_len;
276
+
277
+	_copy_to_pages(buf->pages,
278
+		       buf->page_base + pgto,
279
+		       (char *)tail->iov_base,
280
+		       len);
281
+	tail->iov_len -= len;
282
+
283
+	if (tail->iov_len > 0)
284
+		memmove((char *)tail->iov_base,
285
+				tail->iov_base + len,
286
+				tail->iov_len);
287
+}
156288 
157289 /**
158290  * _shift_data_right_pages
....@@ -217,6 +349,46 @@
217349 		kunmap_atomic(vto);
218350 
219351 	} while ((len -= copy) != 0);
352
+}
353
+
354
+static unsigned int
355
+_shift_data_right_tail(struct xdr_buf *buf, unsigned int pgfrom, size_t len)
356
+{
357
+	struct kvec *tail = buf->tail;
358
+	unsigned int tailbuf_len;
359
+	unsigned int result = 0;
360
+	size_t copy;
361
+
362
+	tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;
363
+
364
+	/* Shift the tail first */
365
+	if (tailbuf_len != 0) {
366
+		unsigned int free_space = tailbuf_len - tail->iov_len;
367
+
368
+		if (len < free_space)
369
+			free_space = len;
370
+		if (len > free_space)
371
+			len = free_space;
372
+
373
+		tail->iov_len += free_space;
374
+		copy = len;
375
+
376
+		if (tail->iov_len > len) {
377
+			char *p = (char *)tail->iov_base + len;
378
+			memmove(p, tail->iov_base, tail->iov_len - free_space);
379
+			result += tail->iov_len - free_space;
380
+		} else
381
+			copy = tail->iov_len;
382
+
383
+		/* Copy from the inlined pages into the tail */
384
+		_copy_from_pages((char *)tail->iov_base,
385
+					 buf->pages,
386
+					 buf->page_base + pgfrom,
387
+					 copy);
388
+		result += copy;
389
+	}
390
+
391
+	return result;
220392 }
221393 
222394 /**
....@@ -304,6 +476,38 @@
304476 EXPORT_SYMBOL_GPL(_copy_from_pages);
305477 
306478 /**
479
+ * _zero_pages
480
+ * @pages: array of pages
481
+ * @pgbase: beginning page vector address
482
+ * @len: length
483
+ */
484
+static void
485
+_zero_pages(struct page **pages, size_t pgbase, size_t len)
486
+{
487
+	struct page **page;
488
+	char *vpage;
489
+	size_t zero;
490
+
491
+	page = pages + (pgbase >> PAGE_SHIFT);
492
+	pgbase &= ~PAGE_MASK;
493
+
494
+	do {
495
+		zero = PAGE_SIZE - pgbase;
496
+		if (zero > len)
497
+			zero = len;
498
+
499
+		vpage = kmap_atomic(*page);
500
+		memset(vpage + pgbase, 0, zero);
501
+		kunmap_atomic(vpage);
502
+
503
+		flush_dcache_page(*page);
504
+		pgbase = 0;
505
+		page++;
506
+
507
+	} while ((len -= zero) != 0);
508
+}
509
+
510
+/**
307511  * xdr_shrink_bufhead
308512  * @buf: xdr_buf
309513  * @len: bytes to remove from buf->head[0]
....@@ -312,13 +516,15 @@
312516  * 'len' bytes. The extra data is not lost, but is instead
313517  * moved into the inlined pages and/or the tail.
314518  */
315
-static void
519
+static unsigned int
316520 xdr_shrink_bufhead(struct xdr_buf *buf, size_t len)
317521 {
318522 	struct kvec *head, *tail;
319523 	size_t copy, offs;
320524 	unsigned int pglen = buf->page_len;
525
+	unsigned int result;
321526 
527
+	result = 0;
322528 	tail = buf->tail;
323529 	head = buf->head;
324530 
....@@ -332,6 +538,7 @@
332538 			copy = tail->iov_len - len;
333539 			memmove((char *)tail->iov_base + len,
334540 					tail->iov_base, copy);
541
+			result += copy;
335542 		}
336543 		/* Copy from the inlined pages into the tail */
337544 		copy = len;
....@@ -342,11 +549,13 @@
342549 			copy = 0;
343550 		else if (copy > tail->iov_len - offs)
344551 			copy = tail->iov_len - offs;
345
-		if (copy != 0)
552
+		if (copy != 0) {
346553 			_copy_from_pages((char *)tail->iov_base + offs,
347554 					buf->pages,
348555 					buf->page_base + pglen + offs - len,
349556 					copy);
557
+			result += copy;
558
+		}
350559 		/* Do we also need to copy data from the head into the tail ? */
351560 		if (len > pglen) {
352561 			offs = copy = len - pglen;
....@@ -356,6 +565,7 @@
356565 					(char *)head->iov_base +
357566 					head->iov_len - offs,
358567 					copy);
568
+			result += copy;
359569 		}
360570 	}
361571 	/* Now handle pages */
....@@ -371,60 +581,42 @@
371581 		_copy_to_pages(buf->pages, buf->page_base,
372582 				(char *)head->iov_base + head->iov_len - len,
373583 				copy);
584
+		result += copy;
374585 	}
375586 	head->iov_len -= len;
376587 	buf->buflen -= len;
377588 	/* Have we truncated the message? */
378589 	if (buf->len > buf->buflen)
379590 		buf->len = buf->buflen;
591
+
592
+	return result;
380593 }
381594 
382595 /**
383
- * xdr_shrink_pagelen
596
+ * xdr_shrink_pagelen - shrinks buf->pages by up to @len bytes
384597  * @buf: xdr_buf
385598  * @len: bytes to remove from buf->pages
386599  *
387
- * Shrinks XDR buffer's page array buf->pages by
388
- * 'len' bytes. The extra data is not lost, but is instead
389
- * moved into the tail.
600
+ * The extra data is not lost, but is instead moved into buf->tail.
601
+ * Returns the actual number of bytes moved.
390602  */
391
-static void
603
+static unsigned int
392604 xdr_shrink_pagelen(struct xdr_buf *buf, size_t len)
393605 {
394
-	struct kvec *tail;
395
-	size_t copy;
396606 	unsigned int pglen = buf->page_len;
397
-	unsigned int tailbuf_len;
607
+	unsigned int result;
398608 
399
-	tail = buf->tail;
400
-	BUG_ON (len > pglen);
609
+	if (len > buf->page_len)
610
+		len = buf-> page_len;
401611 
402
-	tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;
403
-
404
-	/* Shift the tail first */
405
-	if (tailbuf_len != 0) {
406
-		unsigned int free_space = tailbuf_len - tail->iov_len;
407
-
408
-		if (len < free_space)
409
-			free_space = len;
410
-		tail->iov_len += free_space;
411
-
412
-		copy = len;
413
-		if (tail->iov_len > len) {
414
-			char *p = (char *)tail->iov_base + len;
415
-			memmove(p, tail->iov_base, tail->iov_len - len);
416
-		} else
417
-			copy = tail->iov_len;
418
-		/* Copy from the inlined pages into the tail */
419
-		_copy_from_pages((char *)tail->iov_base,
420
-				buf->pages, buf->page_base + pglen - len,
421
-				copy);
422
-	}
612
+	result = _shift_data_right_tail(buf, pglen - len, len);
423613 	buf->page_len -= len;
424614 	buf->buflen -= len;
425615 	/* Have we truncated the message? */
426616 	if (buf->len > buf->buflen)
427617 		buf->len = buf->buflen;
618
+
619
+	return result;
428620 }
429621 
430622 void
....@@ -445,10 +637,24 @@
445637 EXPORT_SYMBOL_GPL(xdr_stream_pos);
446638 
447639 /**
640
+ * xdr_page_pos - Return the current offset from the start of the xdr pages
641
+ * @xdr: pointer to struct xdr_stream
642
+ */
643
+unsigned int xdr_page_pos(const struct xdr_stream *xdr)
644
+{
645
+	unsigned int pos = xdr_stream_pos(xdr);
646
+
647
+	WARN_ON(pos < xdr->buf->head[0].iov_len);
648
+	return pos - xdr->buf->head[0].iov_len;
649
+}
650
+EXPORT_SYMBOL_GPL(xdr_page_pos);
651
+
652
+/**
448653  * xdr_init_encode - Initialize a struct xdr_stream for sending data.
449654  * @xdr: pointer to xdr_stream struct
450655  * @buf: pointer to XDR buffer in which to encode data
451656  * @p: current pointer inside XDR buffer
657
+ * @rqst: pointer to controlling rpc_rqst, for debugging
452658  *
453659  * Note: at the moment the RPC client only passes the length of our
454660  *	 scratch buffer in the xdr_buf's header kvec. Previously this
....@@ -457,7 +663,8 @@
457663  *	 of the buffer length, and takes care of adjusting the kvec
458664  *	 length for us.
459665  */
460
-void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p)
666
+void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
667
+		     struct rpc_rqst *rqst)
461668 {
462669 	struct kvec *iov = buf->head;
463670 	int scratch_len = buf->buflen - buf->page_len - buf->tail[0].iov_len;
....@@ -479,6 +686,7 @@
479686 		buf->len += len;
480687 		iov->iov_len += len;
481688 	}
689
+	xdr->rqst = rqst;
482690 }
483691 EXPORT_SYMBOL_GPL(xdr_init_encode);
484692 
....@@ -495,7 +703,7 @@
495703  * required at the end of encoding, or any other time when the xdr_buf
496704  * data might be read.
497705  */
498
-void xdr_commit_encode(struct xdr_stream *xdr)
706
+inline void xdr_commit_encode(struct xdr_stream *xdr)
499707 {
500708 	int shift = xdr->scratch.iov_len;
501709 	void *page;
....@@ -517,9 +725,9 @@
517725 	int frag1bytes, frag2bytes;
518726 
519727 	if (nbytes > PAGE_SIZE)
520
-		return NULL; /* Bigger buffers require special handling */
728
+		goto out_overflow; /* Bigger buffers require special handling */
521729 	if (xdr->buf->len + nbytes > xdr->buf->buflen)
522
-		return NULL; /* Sorry, we're totally out of space */
730
+		goto out_overflow; /* Sorry, we're totally out of space */
523731 	frag1bytes = (xdr->end - xdr->p) << 2;
524732 	frag2bytes = nbytes - frag1bytes;
525733 	if (xdr->iov)
....@@ -544,10 +752,17 @@
544752 	 */
545753 	xdr->p = (void *)p + frag2bytes;
546754 	space_left = xdr->buf->buflen - xdr->buf->len;
547
-	xdr->end = (void *)p + min_t(int, space_left, PAGE_SIZE);
755
+	if (space_left - frag1bytes >= PAGE_SIZE)
756
+		xdr->end = (void *)p + PAGE_SIZE;
757
+	else
758
+		xdr->end = (void *)p + space_left - frag1bytes;
759
+
548760 	xdr->buf->page_len += frag2bytes;
549761 	xdr->buf->len += nbytes;
550762 	return p;
763
+out_overflow:
764
+	trace_rpc_xdr_overflow(xdr, nbytes);
765
+	return NULL;
551766 }
552767 
553768 /**
....@@ -581,6 +796,51 @@
581796 }
582797 EXPORT_SYMBOL_GPL(xdr_reserve_space);
583798 
799
+
800
+/**
801
+ * xdr_reserve_space_vec - Reserves a large amount of buffer space for sending
802
+ * @xdr: pointer to xdr_stream
803
+ * @vec: pointer to a kvec array
804
+ * @nbytes: number of bytes to reserve
805
+ *
806
+ * Reserves enough buffer space to encode 'nbytes' of data and stores the
807
+ * pointers in 'vec'. The size argument passed to xdr_reserve_space() is
808
+ * determined based on the number of bytes remaining in the current page to
809
+ * avoid invalidating iov_base pointers when xdr_commit_encode() is called.
810
+ */
811
+int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes)
812
+{
813
+	int thislen;
814
+	int v = 0;
815
+	__be32 *p;
816
+
817
+	/*
818
+	 * svcrdma requires every READ payload to start somewhere
819
+	 * in xdr->pages.
820
+	 */
821
+	if (xdr->iov == xdr->buf->head) {
822
+		xdr->iov = NULL;
823
+		xdr->end = xdr->p;
824
+	}
825
+
826
+	while (nbytes) {
827
+		thislen = xdr->buf->page_len % PAGE_SIZE;
828
+		thislen = min_t(size_t, nbytes, PAGE_SIZE - thislen);
829
+
830
+		p = xdr_reserve_space(xdr, thislen);
831
+		if (!p)
832
+			return -EIO;
833
+
834
+		vec[v].iov_base = p;
835
+		vec[v].iov_len = thislen;
836
+		v++;
837
+		nbytes -= thislen;
838
+	}
839
+
840
+	return v;
841
+}
842
+EXPORT_SYMBOL_GPL(xdr_reserve_space_vec);
843
+
584844 /**
585845  * xdr_truncate_encode - truncate an encode buffer
586846  * @xdr: pointer to xdr_stream
....@@ -591,7 +851,7 @@
591851  * head, tail, and page lengths are adjusted to correspond.
592852  *
593853  * If this means moving xdr->p to a different buffer, we assume that
594
- * that the end pointer should be set to the end of the current page,
854
+ * the end pointer should be set to the end of the current page,
595855  * except in the case of the head buffer when we assume the head
596856  * buffer's current length represents the end of the available buffer.
597857  *
....@@ -758,6 +1018,13 @@
7581018 	return 0;
7591019 }
7601020 
1021
+static void xdr_set_page(struct xdr_stream *xdr, unsigned int base,
1022
+			 unsigned int len)
1023
+{
1024
+	if (xdr_set_page_base(xdr, base, len) < 0)
1025
+		xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);
1026
+}
1027
+
7611028 static void xdr_set_next_page(struct xdr_stream *xdr)
7621029 {
7631030 	unsigned int newbase;
....@@ -765,8 +1032,7 @@
7651032 	newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;
7661033 	newbase -= xdr->buf->page_base;
7671034 
768
-	if (xdr_set_page_base(xdr, newbase, PAGE_SIZE) < 0)
769
-		xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);
1035
+	xdr_set_page(xdr, newbase, PAGE_SIZE);
7701036 }
7711037 
7721038 static bool xdr_set_next_buffer(struct xdr_stream *xdr)
....@@ -774,8 +1040,7 @@
7741040 	if (xdr->page_ptr != NULL)
7751041 		xdr_set_next_page(xdr);
7761042 	else if (xdr->iov == xdr->buf->head) {
777
-		if (xdr_set_page_base(xdr, 0, PAGE_SIZE) < 0)
778
-			xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);
1043
+		xdr_set_page(xdr, 0, PAGE_SIZE);
7791044 	}
7801045 	return xdr->p != xdr->end;
7811046 }
....@@ -785,8 +1050,10 @@
7851050  * @xdr: pointer to xdr_stream struct
7861051  * @buf: pointer to XDR buffer from which to decode data
7871052  * @p: current pointer inside XDR buffer
1053
+ * @rqst: pointer to controlling rpc_rqst, for debugging
7881054  */
789
-void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p)
1055
+void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
1056
+		     struct rpc_rqst *rqst)
7901057 {
7911058 	xdr->buf = buf;
7921059 	xdr->scratch.iov_base = NULL;
....@@ -802,6 +1069,7 @@
8021069 		xdr->nwords -= p - xdr->p;
8031070 		xdr->p = p;
8041071 	}
1072
+	xdr->rqst = rqst;
8051073 }
8061074 EXPORT_SYMBOL_GPL(xdr_init_decode);
8071075 
....@@ -820,7 +1088,7 @@
8201088 	buf->page_len =  len;
8211089 	buf->buflen =  len;
8221090 	buf->len = len;
823
-	xdr_init_decode(xdr, buf, NULL);
1091
+	xdr_init_decode(xdr, buf, NULL, NULL);
8241092 }
8251093 EXPORT_SYMBOL_GPL(xdr_init_decode_pages);
8261094 
....@@ -862,20 +1130,23 @@
8621130 	size_t cplen = (char *)xdr->end - (char *)xdr->p;
8631131 
8641132 	if (nbytes > xdr->scratch.iov_len)
865
-		return NULL;
1133
+		goto out_overflow;
8661134 	p = __xdr_inline_decode(xdr, cplen);
8671135 	if (p == NULL)
8681136 		return NULL;
8691137 	memcpy(cpdest, p, cplen);
1138
+	if (!xdr_set_next_buffer(xdr))
1139
+		goto out_overflow;
8701140 	cpdest += cplen;
8711141 	nbytes -= cplen;
872
-	if (!xdr_set_next_buffer(xdr))
873
-		return NULL;
8741142 	p = __xdr_inline_decode(xdr, nbytes);
8751143 	if (p == NULL)
8761144 		return NULL;
8771145 	memcpy(cpdest, p, nbytes);
8781146 	return xdr->scratch.iov_base;
1147
+out_overflow:
1148
+	trace_rpc_xdr_overflow(xdr, nbytes);
1149
+	return NULL;
8791150 }
8801151 
8811152 /**
....@@ -892,33 +1163,47 @@
8921163 {
8931164 	__be32 *p;
8941165 
895
-	if (nbytes == 0)
1166
+	if (unlikely(nbytes == 0))
8961167 		return xdr->p;
8971168 	if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))
898
-		return NULL;
1169
+		goto out_overflow;
8991170 	p = __xdr_inline_decode(xdr, nbytes);
9001171 	if (p != NULL)
9011172 		return p;
9021173 	return xdr_copy_to_scratch(xdr, nbytes);
1174
+out_overflow:
1175
+	trace_rpc_xdr_overflow(xdr, nbytes);
1176
+	return NULL;
9031177 }
9041178 EXPORT_SYMBOL_GPL(xdr_inline_decode);
1179
+
1180
+static void xdr_realign_pages(struct xdr_stream *xdr)
1181
+{
1182
+	struct xdr_buf *buf = xdr->buf;
1183
+	struct kvec *iov = buf->head;
1184
+	unsigned int cur = xdr_stream_pos(xdr);
1185
+	unsigned int copied, offset;
1186
+
1187
+	/* Realign pages to current pointer position */
1188
+	if (iov->iov_len > cur) {
1189
+		offset = iov->iov_len - cur;
1190
+		copied = xdr_shrink_bufhead(buf, offset);
1191
+		trace_rpc_xdr_alignment(xdr, offset, copied);
1192
+		xdr->nwords = XDR_QUADLEN(buf->len - cur);
1193
+	}
1194
+}
9051195 
9061196 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)
9071197 {
9081198 	struct xdr_buf *buf = xdr->buf;
909
-	struct kvec *iov;
9101199 	unsigned int nwords = XDR_QUADLEN(len);
9111200 	unsigned int cur = xdr_stream_pos(xdr);
1201
+	unsigned int copied, offset;
9121202 
9131203 	if (xdr->nwords == 0)
9141204 		return 0;
915
-	/* Realign pages to current pointer position */
916
-	iov  = buf->head;
917
-	if (iov->iov_len > cur) {
918
-		xdr_shrink_bufhead(buf, iov->iov_len - cur);
919
-		xdr->nwords = XDR_QUADLEN(buf->len - cur);
920
-	}
9211205 
1206
+	xdr_realign_pages(xdr);
9221207 	if (nwords > xdr->nwords) {
9231208 		nwords = xdr->nwords;
9241209 		len = nwords << 2;
....@@ -927,7 +1212,9 @@
9271212 		len = buf->page_len;
9281213 	else if (nwords < xdr->nwords) {
9291214 		/* Truncate page data and move it into the tail */
930
-		xdr_shrink_pagelen(buf, buf->page_len - len);
1215
+		offset = buf->page_len - len;
1216
+		copied = xdr_shrink_pagelen(buf, offset);
1217
+		trace_rpc_xdr_alignment(xdr, offset, copied);
9311218 		xdr->nwords = XDR_QUADLEN(buf->len - cur);
9321219 	}
9331220 	return len;
....@@ -975,6 +1262,79 @@
9751262 }
9761263 EXPORT_SYMBOL_GPL(xdr_read_pages);
9771264 
1265
+uint64_t xdr_align_data(struct xdr_stream *xdr, uint64_t offset, uint32_t length)
1266
+{
1267
+	struct xdr_buf *buf = xdr->buf;
1268
+	unsigned int from, bytes;
1269
+	unsigned int shift = 0;
1270
+
1271
+	if ((offset + length) < offset ||
1272
+	    (offset + length) > buf->page_len)
1273
+		length = buf->page_len - offset;
1274
+
1275
+	xdr_realign_pages(xdr);
1276
+	from = xdr_page_pos(xdr);
1277
+	bytes = xdr->nwords << 2;
1278
+	if (length < bytes)
1279
+		bytes = length;
1280
+
1281
+	/* Move page data to the left */
1282
+	if (from > offset) {
1283
+		shift = min_t(unsigned int, bytes, buf->page_len - from);
1284
+		_shift_data_left_pages(buf->pages,
1285
+				       buf->page_base + offset,
1286
+				       buf->page_base + from,
1287
+				       shift);
1288
+		bytes -= shift;
1289
+
1290
+		/* Move tail data into the pages, if necessary */
1291
+		if (bytes > 0)
1292
+			_shift_data_left_tail(buf, offset + shift, bytes);
1293
+	}
1294
+
1295
+	xdr->nwords -= XDR_QUADLEN(length);
1296
+	xdr_set_page(xdr, from + length, PAGE_SIZE);
1297
+	return length;
1298
+}
1299
+EXPORT_SYMBOL_GPL(xdr_align_data);
1300
+
1301
+uint64_t xdr_expand_hole(struct xdr_stream *xdr, uint64_t offset, uint64_t length)
1302
+{
1303
+	struct xdr_buf *buf = xdr->buf;
1304
+	unsigned int bytes;
1305
+	unsigned int from;
1306
+	unsigned int truncated = 0;
1307
+
1308
+	if ((offset + length) < offset ||
1309
+	    (offset + length) > buf->page_len)
1310
+		length = buf->page_len - offset;
1311
+
1312
+	xdr_realign_pages(xdr);
1313
+	from = xdr_page_pos(xdr);
1314
+	bytes = xdr->nwords << 2;
1315
+
1316
+	if (offset + length + bytes > buf->page_len) {
1317
+		unsigned int shift = (offset + length + bytes) - buf->page_len;
1318
+		unsigned int res = _shift_data_right_tail(buf, from + bytes - shift, shift);
1319
+		truncated = shift - res;
1320
+		xdr->nwords -= XDR_QUADLEN(truncated);
1321
+		bytes -= shift;
1322
+	}
1323
+
1324
+	/* Now move the page data over and zero pages */
1325
+	if (bytes > 0)
1326
+		_shift_data_right_pages(buf->pages,
1327
+					buf->page_base + offset + length,
1328
+					buf->page_base + from,
1329
+					bytes);
1330
+	_zero_pages(buf->pages, buf->page_base + offset, length);
1331
+
1332
+	buf->len += length - (from - offset) - truncated;
1333
+	xdr_set_page(xdr, offset + length, PAGE_SIZE);
1334
+	return length;
1335
+}
1336
+EXPORT_SYMBOL_GPL(xdr_expand_hole);
1337
+
9781338 /**
9791339  * xdr_enter_page - decode data from the XDR page
9801340  * @xdr: pointer to xdr_stream struct
....@@ -997,7 +1357,7 @@
9971357 }
9981358 EXPORT_SYMBOL_GPL(xdr_enter_page);
9991359 
1000
-static struct kvec empty_iov = {.iov_base = NULL, .iov_len = 0};
1360
+static const struct kvec empty_iov = {.iov_base = NULL, .iov_len = 0};
10011361 
10021362 void
10031363 xdr_buf_from_iov(struct kvec *iov, struct xdr_buf *buf)
....@@ -1197,44 +1557,6 @@
11971557 	return write_bytes_to_xdr_buf(buf, base, &raw, sizeof(obj));
11981558 }
11991559 EXPORT_SYMBOL_GPL(xdr_encode_word);
1200
-
1201
-/* If the netobj starting offset bytes from the start of xdr_buf is contained
1202
- * entirely in the head or the tail, set object to point to it; otherwise
1203
- * try to find space for it at the end of the tail, copy it there, and
1204
- * set obj to point to it. */
1205
-int xdr_buf_read_netobj(struct xdr_buf *buf, struct xdr_netobj *obj, unsigned int offset)
1206
-{
1207
-	struct xdr_buf subbuf;
1208
-
1209
-	if (xdr_decode_word(buf, offset, &obj->len))
1210
-		return -EFAULT;
1211
-	if (xdr_buf_subsegment(buf, &subbuf, offset + 4, obj->len))
1212
-		return -EFAULT;
1213
-
1214
-	/* Is the obj contained entirely in the head? */
1215
-	obj->data = subbuf.head[0].iov_base;
1216
-	if (subbuf.head[0].iov_len == obj->len)
1217
-		return 0;
1218
-	/* ..or is the obj contained entirely in the tail? */
1219
-	obj->data = subbuf.tail[0].iov_base;
1220
-	if (subbuf.tail[0].iov_len == obj->len)
1221
-		return 0;
1222
-
1223
-	/* use end of tail as storage for obj:
1224
-	 * (We don't copy to the beginning because then we'd have
1225
-	 * to worry about doing a potentially overlapping copy.
1226
-	 * This assumes the object is at most half the length of the
1227
-	 * tail.) */
1228
-	if (obj->len > buf->buflen - buf->len)
1229
-		return -ENOMEM;
1230
-	if (buf->tail[0].iov_len != 0)
1231
-		obj->data = buf->tail[0].iov_base + buf->tail[0].iov_len;
1232
-	else
1233
-		obj->data = buf->head[0].iov_base + buf->head[0].iov_len;
1234
-	__read_bytes_from_xdr_buf(&subbuf, obj->data, obj->len);
1235
-	return 0;
1236
-}
1237
-EXPORT_SYMBOL_GPL(xdr_buf_read_netobj);
12381560 
12391561 /* Returns 0 on success, or else a negative error code. */
12401562 static int