~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* raid5.c : Multiple Devices driver for Linux
3	4	* Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
..	..	@@ -7,15 +8,6 @@
7	8	* RAID-4/5/6 management functions.
8	9	* Thanks to Penguin Computing for making the RAID-6 development possible
9	10	* by donating a test server!
10		- *
11		- * This program is free software; you can redistribute it and/or modify
12		- * it under the terms of the GNU General Public License as published by
13		- * the Free Software Foundation; either version 2, or (at your option)
14		- * any later version.
15		- *
16		- * You should have received a copy of the GNU General Public License
17		- * (for example /usr/src/linux/COPYING); if not, write to the Free
18		- * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19	11	*/
20	12
21	13	/*
..	..	@@ -44,6 +36,7 @@
44	36	*/
45	37
46	38	#include <linux/blkdev.h>
	39	+#include <linux/delay.h>
47	40	#include <linux/kthread.h>
48	41	#include <linux/raid/pq.h>
49	42	#include <linux/async_tx.h>
..	..	@@ -54,7 +47,6 @@
54	47	#include <linux/slab.h>
55	48	#include <linux/ratelimit.h>
56	49	#include <linux/nodemask.h>
57		-#include <linux/flex_array.h>
58	50
59	51	#include <trace/events/block.h>
60	52	#include <linux/list_sort.h>
..	..	@@ -78,13 +70,13 @@
78	70
79	71	static inline struct hlist_head stripe_hash(struct r5conf conf, sector_t sect)
80	72	{
81		- int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	73	+ int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
82	74	return &conf->stripe_hashtbl[hash];
83	75	}
84	76
85		-static inline int stripe_hash_locks_hash(sector_t sect)
	77	+static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)
86	78	{
87		- return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
	79	+ return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;
88	80	}
89	81
90	82	static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
..	..	@@ -457,13 +449,74 @@
457	449	return sh;
458	450	}
459	451
	452	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	453	+static void free_stripe_pages(struct stripe_head *sh)
	454	+{
	455	+ int i;
	456	+ struct page *p;
	457	+
	458	+ /* Have not allocate page pool */
	459	+ if (!sh->pages)
	460	+ return;
	461	+
	462	+ for (i = 0; i < sh->nr_pages; i++) {
	463	+ p = sh->pages[i];
	464	+ if (p)
	465	+ put_page(p);
	466	+ sh->pages[i] = NULL;
	467	+ }
	468	+}
	469	+
	470	+static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
	471	+{
	472	+ int i;
	473	+ struct page *p;
	474	+
	475	+ for (i = 0; i < sh->nr_pages; i++) {
	476	+ /* The page have allocated. */
	477	+ if (sh->pages[i])
	478	+ continue;
	479	+
	480	+ p = alloc_page(gfp);
	481	+ if (!p) {
	482	+ free_stripe_pages(sh);
	483	+ return -ENOMEM;
	484	+ }
	485	+ sh->pages[i] = p;
	486	+ }
	487	+ return 0;
	488	+}
	489	+
	490	+static int
	491	+init_stripe_shared_pages(struct stripe_head sh, struct r5conf conf, int disks)
	492	+{
	493	+ int nr_pages, cnt;
	494	+
	495	+ if (sh->pages)
	496	+ return 0;
	497	+
	498	+ /* Each of the sh->dev[i] need one conf->stripe_size */
	499	+ cnt = PAGE_SIZE / conf->stripe_size;
	500	+ nr_pages = (disks + cnt - 1) / cnt;
	501	+
	502	+ sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
	503	+ if (!sh->pages)
	504	+ return -ENOMEM;
	505	+ sh->nr_pages = nr_pages;
	506	+ sh->stripes_per_page = cnt;
	507	+ return 0;
	508	+}
	509	+#endif
	510	+
460	511	static void shrink_buffers(struct stripe_head *sh)
461	512	{
462		- struct page *p;
463	513	int i;
464	514	int num = sh->raid_conf->pool_size;
465	515
	516	+#if PAGE_SIZE == DEFAULT_STRIPE_SIZE
466	517	for (i = 0; i < num ; i++) {
	518	+ struct page *p;
	519	+
467	520	WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
468	521	p = sh->dev[i].page;
469	522	if (!p)
..	..	@@ -471,6 +524,11 @@
471	524	sh->dev[i].page = NULL;
472	525	put_page(p);
473	526	}
	527	+#else
	528	+ for (i = 0; i < num; i++)
	529	+ sh->dev[i].page = NULL;
	530	+ free_stripe_pages(sh); /* Free pages */
	531	+#endif
474	532	}
475	533
476	534	static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
..	..	@@ -478,6 +536,7 @@
478	536	int i;
479	537	int num = sh->raid_conf->pool_size;
480	538
	539	+#if PAGE_SIZE == DEFAULT_STRIPE_SIZE
481	540	for (i = 0; i < num; i++) {
482	541	struct page *page;
483	542
..	..	@@ -486,8 +545,18 @@
486	545	}
487	546	sh->dev[i].page = page;
488	547	sh->dev[i].orig_page = page;
	548	+ sh->dev[i].offset = 0;
489	549	}
	550	+#else
	551	+ if (alloc_stripe_pages(sh, gfp))
	552	+ return -ENOMEM;
490	553
	554	+ for (i = 0; i < num; i++) {
	555	+ sh->dev[i].page = raid5_get_dev_page(sh, i);
	556	+ sh->dev[i].orig_page = sh->dev[i].page;
	557	+ sh->dev[i].offset = raid5_get_page_offset(sh, i);
	558	+ }
	559	+#endif
491	560	return 0;
492	561	}
493	562
..	..	@@ -618,17 +687,17 @@
618	687	return degraded;
619	688	}
620	689
621		-static int has_failed(struct r5conf *conf)
	690	+static bool has_failed(struct r5conf *conf)
622	691	{
623		- int degraded;
	692	+ int degraded = conf->mddev->degraded;
624	693
625		- if (conf->mddev->reshape_position == MaxSector)
626		- return conf->mddev->degraded > conf->max_degraded;
	694	+ if (test_bit(MD_BROKEN, &conf->mddev->flags))
	695	+ return true;
627	696
628		- degraded = raid5_calc_degraded(conf);
629		- if (degraded > conf->max_degraded)
630		- return 1;
631		- return 0;
	697	+ if (conf->mddev->reshape_position != MaxSector)
	698	+ degraded = raid5_calc_degraded(conf);
	699	+
	700	+ return degraded > conf->max_degraded;
632	701	}
633	702
634	703	struct stripe_head *
..	..	@@ -636,7 +705,7 @@
636	705	int previous, int noblock, int noquiesce)
637	706	{
638	707	struct stripe_head *sh;
639		- int hash = stripe_hash_locks_hash(sector);
	708	+ int hash = stripe_hash_locks_hash(conf, sector);
640	709	int inc_empty_inactive_list_flag;
641	710
642	711	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
..	..	@@ -712,6 +781,8 @@
712	781	}
713	782
714	783	static void lock_two_stripes(struct stripe_head sh1, struct stripe_head sh2)
	784	+ __acquires(&sh1->stripe_lock)
	785	+ __acquires(&sh2->stripe_lock)
715	786	{
716	787	if (sh1 > sh2) {
717	788	spin_lock_irq(&sh2->stripe_lock);
..	..	@@ -723,6 +794,8 @@
723	794	}
724	795
725	796	static void unlock_two_stripes(struct stripe_head sh1, struct stripe_head sh2)
	797	+ __releases(&sh1->stripe_lock)
	798	+ __releases(&sh2->stripe_lock)
726	799	{
727	800	spin_unlock(&sh1->stripe_lock);
728	801	spin_unlock_irq(&sh2->stripe_lock);
..	..	@@ -753,9 +826,9 @@
753	826	tmp_sec = sh->sector;
754	827	if (!sector_div(tmp_sec, conf->chunk_sectors))
755	828	return;
756		- head_sector = sh->sector - STRIPE_SECTORS;
	829	+ head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
757	830
758		- hash = stripe_hash_locks_hash(head_sector);
	831	+ hash = stripe_hash_locks_hash(conf, head_sector);
759	832	spin_lock_irq(conf->hash_locks + hash);
760	833	head = __find_stripe(conf, head_sector, conf->generation);
761	834	if (head && !atomic_inc_not_zero(&head->count)) {
..	..	@@ -878,7 +951,7 @@
878	951	struct bio *bio;
879	952
880	953	while ((bio = bio_list_pop(tmp)))
881		- generic_make_request(bio);
	954	+ submit_bio_noacct(bio);
882	955	}
883	956
884	957	static int cmp_stripe(void priv, struct list_head a, struct list_head *b)
..	..	@@ -1062,7 +1135,7 @@
1062	1135	test_bit(WriteErrorSeen, &rdev->flags)) {
1063	1136	sector_t first_bad;
1064	1137	int bad_sectors;
1065		- int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	1138	+ int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
1066	1139	&first_bad, &bad_sectors);
1067	1140	if (!bad)
1068	1141	break;
..	..	@@ -1094,7 +1167,7 @@
1094	1167	if (rdev) {
1095	1168	if (s->syncing \|\| s->expanding \|\| s->expanded
1096	1169	\|\| s->replacing)
1097		- md_sync_acct(rdev->bdev, STRIPE_SECTORS);
	1170	+ md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));
1098	1171
1099	1172	set_bit(STRIPE_IO_STARTED, &sh->state);
1100	1173
..	..	@@ -1134,12 +1207,12 @@
1134	1207	else
1135	1208	sh->dev[i].vec.bv_page = sh->dev[i].page;
1136	1209	bi->bi_vcnt = 1;
1137		- bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1138		- bi->bi_io_vec[0].bv_offset = 0;
1139		- bi->bi_iter.bi_size = STRIPE_SIZE;
	1210	+ bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
	1211	+ bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
	1212	+ bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
1140	1213	bi->bi_write_hint = sh->dev[i].write_hint;
1141	1214	if (!rrdev)
1142		- sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
	1215	+ sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1143	1216	/*
1144	1217	* If this is discard request, set bi_vcnt 0. We don't
1145	1218	* want to confuse SCSI because SCSI will replace payload
..	..	@@ -1156,12 +1229,12 @@
1156	1229	if (should_defer && op_is_write(op))
1157	1230	bio_list_add(&pending_bios, bi);
1158	1231	else
1159		- generic_make_request(bi);
	1232	+ submit_bio_noacct(bi);
1160	1233	}
1161	1234	if (rrdev) {
1162	1235	if (s->syncing \|\| s->expanding \|\| s->expanded
1163	1236	\|\| s->replacing)
1164		- md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
	1237	+ md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));
1165	1238
1166	1239	set_bit(STRIPE_IO_STARTED, &sh->state);
1167	1240
..	..	@@ -1188,11 +1261,11 @@
1188	1261	WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1189	1262	sh->dev[i].rvec.bv_page = sh->dev[i].page;
1190	1263	rbi->bi_vcnt = 1;
1191		- rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1192		- rbi->bi_io_vec[0].bv_offset = 0;
1193		- rbi->bi_iter.bi_size = STRIPE_SIZE;
	1264	+ rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
	1265	+ rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
	1266	+ rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
1194	1267	rbi->bi_write_hint = sh->dev[i].write_hint;
1195		- sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
	1268	+ sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1196	1269	/*
1197	1270	* If this is discard request, set bi_vcnt 0. We don't
1198	1271	* want to confuse SCSI because SCSI will replace payload
..	..	@@ -1206,7 +1279,7 @@
1206	1279	if (should_defer && op_is_write(op))
1207	1280	bio_list_add(&pending_bios, rbi);
1208	1281	else
1209		- generic_make_request(rbi);
	1282	+ submit_bio_noacct(rbi);
1210	1283	}
1211	1284	if (!rdev && !rrdev) {
1212	1285	if (op_is_write(op))
..	..	@@ -1231,7 +1304,7 @@
1231	1304
1232	1305	static struct dma_async_tx_descriptor *
1233	1306	async_copy_data(int frombio, struct bio bio, struct page *page,
1234		- sector_t sector, struct dma_async_tx_descriptor *tx,
	1307	+ unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
1235	1308	struct stripe_head *sh, int no_skipcopy)
1236	1309	{
1237	1310	struct bio_vec bvl;
..	..	@@ -1240,6 +1313,7 @@
1240	1313	int page_offset;
1241	1314	struct async_submit_ctl submit;
1242	1315	enum async_tx_flags flags = 0;
	1316	+ struct r5conf *conf = sh->raid_conf;
1243	1317
1244	1318	if (bio->bi_iter.bi_sector >= sector)
1245	1319	page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
..	..	@@ -1261,8 +1335,8 @@
1261	1335	len -= b_offset;
1262	1336	}
1263	1337
1264		- if (len > 0 && page_offset + len > STRIPE_SIZE)
1265		- clen = STRIPE_SIZE - page_offset;
	1338	+ if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))
	1339	+ clen = RAID5_STRIPE_SIZE(conf) - page_offset;
1266	1340	else
1267	1341	clen = len;
1268	1342
..	..	@@ -1270,17 +1344,17 @@
1270	1344	b_offset += bvl.bv_offset;
1271	1345	bio_page = bvl.bv_page;
1272	1346	if (frombio) {
1273		- if (sh->raid_conf->skip_copy &&
	1347	+ if (conf->skip_copy &&
1274	1348	b_offset == 0 && page_offset == 0 &&
1275		- clen == STRIPE_SIZE &&
	1349	+ clen == RAID5_STRIPE_SIZE(conf) &&
1276	1350	!no_skipcopy)
1277	1351	*page = bio_page;
1278	1352	else
1279		- tx = async_memcpy(*page, bio_page, page_offset,
	1353	+ tx = async_memcpy(*page, bio_page, page_offset + poff,
1280	1354	b_offset, clen, &submit);
1281	1355	} else
1282	1356	tx = async_memcpy(bio_page, *page, b_offset,
1283		- page_offset, clen, &submit);
	1357	+ page_offset + poff, clen, &submit);
1284	1358	}
1285	1359	/* chain the operations */
1286	1360	submit.depend_tx = tx;
..	..	@@ -1297,6 +1371,7 @@
1297	1371	{
1298	1372	struct stripe_head *sh = stripe_head_ref;
1299	1373	int i;
	1374	+ struct r5conf *conf = sh->raid_conf;
1300	1375
1301	1376	pr_debug("%s: stripe %llu\n", __func__,
1302	1377	(unsigned long long)sh->sector);
..	..	@@ -1317,8 +1392,8 @@
1317	1392	rbi = dev->read;
1318	1393	dev->read = NULL;
1319	1394	while (rbi && rbi->bi_iter.bi_sector <
1320		- dev->sector + STRIPE_SECTORS) {
1321		- rbi2 = r5_next_bio(rbi, dev->sector);
	1395	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
	1396	+ rbi2 = r5_next_bio(conf, rbi, dev->sector);
1322	1397	bio_endio(rbi);
1323	1398	rbi = rbi2;
1324	1399	}
..	..	@@ -1335,6 +1410,7 @@
1335	1410	struct dma_async_tx_descriptor *tx = NULL;
1336	1411	struct async_submit_ctl submit;
1337	1412	int i;
	1413	+ struct r5conf *conf = sh->raid_conf;
1338	1414
1339	1415	BUG_ON(sh->batch_head);
1340	1416	pr_debug("%s: stripe %llu\n", __func__,
..	..	@@ -1349,10 +1425,11 @@
1349	1425	dev->toread = NULL;
1350	1426	spin_unlock_irq(&sh->stripe_lock);
1351	1427	while (rbi && rbi->bi_iter.bi_sector <
1352		- dev->sector + STRIPE_SECTORS) {
	1428	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
1353	1429	tx = async_copy_data(0, rbi, &dev->page,
	1430	+ dev->offset,
1354	1431	dev->sector, tx, sh, 0);
1355		- rbi = r5_next_bio(rbi, dev->sector);
	1432	+ rbi = r5_next_bio(conf, rbi, dev->sector);
1356	1433	}
1357	1434	}
1358	1435	}
..	..	@@ -1394,22 +1471,25 @@
1394	1471	}
1395	1472
1396	1473	/* return a pointer to the address conversion region of the scribble buffer */
1397		-static addr_conv_t to_addr_conv(struct stripe_head sh,
1398		- struct raid5_percpu *percpu, int i)
	1474	+static struct page *to_addr_page(struct raid5_percpu percpu, int i)
1399	1475	{
1400		- void *addr;
1401		-
1402		- addr = flex_array_get(percpu->scribble, i);
1403		- return addr + sizeof(struct page ) (sh->disks + 2);
	1476	+ return percpu->scribble + i * percpu->scribble_obj_size;
1404	1477	}
1405	1478
1406	1479	/* return a pointer to the address conversion region of the scribble buffer */
1407		-static struct page *to_addr_page(struct raid5_percpu percpu, int i)
	1480	+static addr_conv_t to_addr_conv(struct stripe_head sh,
	1481	+ struct raid5_percpu *percpu, int i)
1408	1482	{
1409		- void *addr;
	1483	+ return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
	1484	+}
1410	1485
1411		- addr = flex_array_get(percpu->scribble, i);
1412		- return addr;
	1486	+/*
	1487	+ * Return a pointer to record offset address.
	1488	+ */
	1489	+static unsigned int *
	1490	+to_addr_offs(struct stripe_head sh, struct raid5_percpu percpu)
	1491	+{
	1492	+ return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
1413	1493	}
1414	1494
1415	1495	static struct dma_async_tx_descriptor *
..	..	@@ -1417,9 +1497,11 @@
1417	1497	{
1418	1498	int disks = sh->disks;
1419	1499	struct page **xor_srcs = to_addr_page(percpu, 0);
	1500	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
1420	1501	int target = sh->ops.target;
1421	1502	struct r5dev *tgt = &sh->dev[target];
1422	1503	struct page *xor_dest = tgt->page;
	1504	+ unsigned int off_dest = tgt->offset;
1423	1505	int count = 0;
1424	1506	struct dma_async_tx_descriptor *tx;
1425	1507	struct async_submit_ctl submit;
..	..	@@ -1431,24 +1513,30 @@
1431	1513	__func__, (unsigned long long)sh->sector, target);
1432	1514	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1433	1515
1434		- for (i = disks; i--; )
1435		- if (i != target)
	1516	+ for (i = disks; i--; ) {
	1517	+ if (i != target) {
	1518	+ off_srcs[count] = sh->dev[i].offset;
1436	1519	xor_srcs[count++] = sh->dev[i].page;
	1520	+ }
	1521	+ }
1437	1522
1438	1523	atomic_inc(&sh->count);
1439	1524
1440	1525	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST, NULL,
1441	1526	ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1442	1527	if (unlikely(count == 1))
1443		- tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	1528	+ tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
	1529	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1444	1530	else
1445		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	1531	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	1532	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1446	1533
1447	1534	return tx;
1448	1535	}
1449	1536
1450	1537	/* set_syndrome_sources - populate source buffers for gen_syndrome
1451	1538	* @srcs - (struct page *) array of size sh->disks
	1539	+ * @offs - (unsigned int) array of offset for each page
1452	1540	* @sh - stripe_head to parse
1453	1541	*
1454	1542	* Populates srcs in proper layout order for the stripe and returns the
..	..	@@ -1457,6 +1545,7 @@
1457	1545	* is recorded in srcs[count+1]].
1458	1546	*/
1459	1547	static int set_syndrome_sources(struct page **srcs,
	1548	+ unsigned int *offs,
1460	1549	struct stripe_head *sh,
1461	1550	int srctype)
1462	1551	{
..	..	@@ -1487,6 +1576,12 @@
1487	1576	srcs[slot] = sh->dev[i].orig_page;
1488	1577	else
1489	1578	srcs[slot] = sh->dev[i].page;
	1579	+ /*
	1580	+ * For R5_InJournal, PAGE_SIZE must be 4KB and will
	1581	+ * not shared page. In that case, dev[i].offset
	1582	+ * is 0.
	1583	+ */
	1584	+ offs[slot] = sh->dev[i].offset;
1490	1585	}
1491	1586	i = raid6_next_disk(i, disks);
1492	1587	} while (i != d0_idx);
..	..	@@ -1499,12 +1594,14 @@
1499	1594	{
1500	1595	int disks = sh->disks;
1501	1596	struct page **blocks = to_addr_page(percpu, 0);
	1597	+ unsigned int *offs = to_addr_offs(sh, percpu);
1502	1598	int target;
1503	1599	int qd_idx = sh->qd_idx;
1504	1600	struct dma_async_tx_descriptor *tx;
1505	1601	struct async_submit_ctl submit;
1506	1602	struct r5dev *tgt;
1507	1603	struct page *dest;
	1604	+ unsigned int dest_off;
1508	1605	int i;
1509	1606	int count;
1510	1607
..	..	@@ -1523,30 +1620,34 @@
1523	1620	tgt = &sh->dev[target];
1524	1621	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1525	1622	dest = tgt->page;
	1623	+ dest_off = tgt->offset;
1526	1624
1527	1625	atomic_inc(&sh->count);
1528	1626
1529	1627	if (target == qd_idx) {
1530		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
	1628	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1531	1629	blocks[count] = NULL; /* regenerating p is not necessary */
1532	1630	BUG_ON(blocks[count+1] != dest); /* q should already be set */
1533	1631	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1534	1632	ops_complete_compute, sh,
1535	1633	to_addr_conv(sh, percpu, 0));
1536		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	1634	+ tx = async_gen_syndrome(blocks, offs, count+2,
	1635	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1537	1636	} else {
1538	1637	/* Compute any data- or p-drive using XOR */
1539	1638	count = 0;
1540	1639	for (i = disks; i-- ; ) {
1541	1640	if (i == target \|\| i == qd_idx)
1542	1641	continue;
	1642	+ offs[count] = sh->dev[i].offset;
1543	1643	blocks[count++] = sh->dev[i].page;
1544	1644	}
1545	1645
1546	1646	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST,
1547	1647	NULL, ops_complete_compute, sh,
1548	1648	to_addr_conv(sh, percpu, 0));
1549		- tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	1649	+ tx = async_xor_offs(dest, dest_off, blocks, offs, count,
	1650	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1550	1651	}
1551	1652
1552	1653	return tx;
..	..	@@ -1565,6 +1666,7 @@
1565	1666	struct r5dev *tgt2 = &sh->dev[target2];
1566	1667	struct dma_async_tx_descriptor *tx;
1567	1668	struct page **blocks = to_addr_page(percpu, 0);
	1669	+ unsigned int *offs = to_addr_offs(sh, percpu);
1568	1670	struct async_submit_ctl submit;
1569	1671
1570	1672	BUG_ON(sh->batch_head);
..	..	@@ -1577,13 +1679,16 @@
1577	1679	/* we need to open-code set_syndrome_sources to handle the
1578	1680	* slot number conversion for 'faila' and 'failb'
1579	1681	*/
1580		- for (i = 0; i < disks ; i++)
	1682	+ for (i = 0; i < disks ; i++) {
	1683	+ offs[i] = 0;
1581	1684	blocks[i] = NULL;
	1685	+ }
1582	1686	count = 0;
1583	1687	i = d0_idx;
1584	1688	do {
1585	1689	int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1586	1690
	1691	+ offs[slot] = sh->dev[i].offset;
1587	1692	blocks[slot] = sh->dev[i].page;
1588	1693
1589	1694	if (i == target)
..	..	@@ -1608,10 +1713,12 @@
1608	1713	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1609	1714	ops_complete_compute, sh,
1610	1715	to_addr_conv(sh, percpu, 0));
1611		- return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1612		- STRIPE_SIZE, &submit);
	1716	+ return async_gen_syndrome(blocks, offs, syndrome_disks+2,
	1717	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1718	+ &submit);
1613	1719	} else {
1614	1720	struct page *dest;
	1721	+ unsigned int dest_off;
1615	1722	int data_target;
1616	1723	int qd_idx = sh->qd_idx;
1617	1724
..	..	@@ -1625,22 +1732,26 @@
1625	1732	for (i = disks; i-- ; ) {
1626	1733	if (i == data_target \|\| i == qd_idx)
1627	1734	continue;
	1735	+ offs[count] = sh->dev[i].offset;
1628	1736	blocks[count++] = sh->dev[i].page;
1629	1737	}
1630	1738	dest = sh->dev[data_target].page;
	1739	+ dest_off = sh->dev[data_target].offset;
1631	1740	init_async_submit(&submit,
1632	1741	ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST,
1633	1742	NULL, NULL, NULL,
1634	1743	to_addr_conv(sh, percpu, 0));
1635		- tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
	1744	+ tx = async_xor_offs(dest, dest_off, blocks, offs, count,
	1745	+ RAID5_STRIPE_SIZE(sh->raid_conf),
1636	1746	&submit);
1637	1747
1638		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
	1748	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1639	1749	init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1640	1750	ops_complete_compute, sh,
1641	1751	to_addr_conv(sh, percpu, 0));
1642		- return async_gen_syndrome(blocks, 0, count+2,
1643		- STRIPE_SIZE, &submit);
	1752	+ return async_gen_syndrome(blocks, offs, count+2,
	1753	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1754	+ &submit);
1644	1755	}
1645	1756	} else {
1646	1757	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
..	..	@@ -1649,13 +1760,15 @@
1649	1760	if (failb == syndrome_disks) {
1650	1761	/* We're missing D+P. */
1651	1762	return async_raid6_datap_recov(syndrome_disks+2,
1652		- STRIPE_SIZE, faila,
1653		- blocks, &submit);
	1763	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1764	+ faila,
	1765	+ blocks, offs, &submit);
1654	1766	} else {
1655	1767	/* We're missing D+D. */
1656	1768	return async_raid6_2data_recov(syndrome_disks+2,
1657		- STRIPE_SIZE, faila, failb,
1658		- blocks, &submit);
	1769	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1770	+ faila, failb,
	1771	+ blocks, offs, &submit);
1659	1772	}
1660	1773	}
1661	1774	}
..	..	@@ -1681,10 +1794,12 @@
1681	1794	{
1682	1795	int disks = sh->disks;
1683	1796	struct page **xor_srcs = to_addr_page(percpu, 0);
	1797	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
1684	1798	int count = 0, pd_idx = sh->pd_idx, i;
1685	1799	struct async_submit_ctl submit;
1686	1800
1687	1801	/* existing parity data subtracted */
	1802	+ unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1688	1803	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1689	1804
1690	1805	BUG_ON(sh->batch_head);
..	..	@@ -1694,15 +1809,23 @@
1694	1809	for (i = disks; i--; ) {
1695	1810	struct r5dev *dev = &sh->dev[i];
1696	1811	/* Only process blocks that are known to be uptodate */
1697		- if (test_bit(R5_InJournal, &dev->flags))
	1812	+ if (test_bit(R5_InJournal, &dev->flags)) {
	1813	+ /*
	1814	+ * For this case, PAGE_SIZE must be equal to 4KB and
	1815	+ * page offset is zero.
	1816	+ */
	1817	+ off_srcs[count] = dev->offset;
1698	1818	xor_srcs[count++] = dev->orig_page;
1699		- else if (test_bit(R5_Wantdrain, &dev->flags))
	1819	+ } else if (test_bit(R5_Wantdrain, &dev->flags)) {
	1820	+ off_srcs[count] = dev->offset;
1700	1821	xor_srcs[count++] = dev->page;
	1822	+ }
1701	1823	}
1702	1824
1703	1825	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
1704	1826	ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1705		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	1827	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	1828	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1706	1829
1707	1830	return tx;
1708	1831	}
..	..	@@ -1712,17 +1835,19 @@
1712	1835	struct dma_async_tx_descriptor *tx)
1713	1836	{
1714	1837	struct page **blocks = to_addr_page(percpu, 0);
	1838	+ unsigned int *offs = to_addr_offs(sh, percpu);
1715	1839	int count;
1716	1840	struct async_submit_ctl submit;
1717	1841
1718	1842	pr_debug("%s: stripe %llu\n", __func__,
1719	1843	(unsigned long long)sh->sector);
1720	1844
1721		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
	1845	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
1722	1846
1723	1847	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_PQ_XOR_DST, tx,
1724	1848	ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1725		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	1849	+ tx = async_gen_syndrome(blocks, offs, count+2,
	1850	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1726	1851
1727	1852	return tx;
1728	1853	}
..	..	@@ -1763,7 +1888,7 @@
1763	1888	WARN_ON(dev->page != dev->orig_page);
1764	1889
1765	1890	while (wbi && wbi->bi_iter.bi_sector <
1766		- dev->sector + STRIPE_SECTORS) {
	1891	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
1767	1892	if (wbi->bi_opf & REQ_FUA)
1768	1893	set_bit(R5_WantFUA, &dev->flags);
1769	1894	if (wbi->bi_opf & REQ_SYNC)
..	..	@@ -1772,6 +1897,7 @@
1772	1897	set_bit(R5_Discard, &dev->flags);
1773	1898	else {
1774	1899	tx = async_copy_data(1, wbi, &dev->page,
	1900	+ dev->offset,
1775	1901	dev->sector, tx, sh,
1776	1902	r5c_is_writeback(conf->log));
1777	1903	if (dev->page != dev->orig_page &&
..	..	@@ -1781,7 +1907,7 @@
1781	1907	clear_bit(R5_OVERWRITE, &dev->flags);
1782	1908	}
1783	1909	}
1784		- wbi = r5_next_bio(wbi, dev->sector);
	1910	+ wbi = r5_next_bio(conf, wbi, dev->sector);
1785	1911	}
1786	1912
1787	1913	if (head_sh->batch_head) {
..	..	@@ -1851,9 +1977,11 @@
1851	1977	{
1852	1978	int disks = sh->disks;
1853	1979	struct page **xor_srcs;
	1980	+ unsigned int *off_srcs;
1854	1981	struct async_submit_ctl submit;
1855	1982	int count, pd_idx = sh->pd_idx, i;
1856	1983	struct page *xor_dest;
	1984	+ unsigned int off_dest;
1857	1985	int prexor = 0;
1858	1986	unsigned long flags;
1859	1987	int j = 0;
..	..	@@ -1878,24 +2006,31 @@
1878	2006	again:
1879	2007	count = 0;
1880	2008	xor_srcs = to_addr_page(percpu, j);
	2009	+ off_srcs = to_addr_offs(sh, percpu);
1881	2010	/* check if prexor is active which means only process blocks
1882	2011	* that are part of a read-modify-write (written)
1883	2012	*/
1884	2013	if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1885	2014	prexor = 1;
	2015	+ off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1886	2016	xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1887	2017	for (i = disks; i--; ) {
1888	2018	struct r5dev *dev = &sh->dev[i];
1889	2019	if (head_sh->dev[i].written \|\|
1890		- test_bit(R5_InJournal, &head_sh->dev[i].flags))
	2020	+ test_bit(R5_InJournal, &head_sh->dev[i].flags)) {
	2021	+ off_srcs[count] = dev->offset;
1891	2022	xor_srcs[count++] = dev->page;
	2023	+ }
1892	2024	}
1893	2025	} else {
1894	2026	xor_dest = sh->dev[pd_idx].page;
	2027	+ off_dest = sh->dev[pd_idx].offset;
1895	2028	for (i = disks; i--; ) {
1896	2029	struct r5dev *dev = &sh->dev[i];
1897		- if (i != pd_idx)
	2030	+ if (i != pd_idx) {
	2031	+ off_srcs[count] = dev->offset;
1898	2032	xor_srcs[count++] = dev->page;
	2033	+ }
1899	2034	}
1900	2035	}
1901	2036
..	..	@@ -1921,9 +2056,11 @@
1921	2056	}
1922	2057
1923	2058	if (unlikely(count == 1))
1924		- tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	2059	+ tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
	2060	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1925	2061	else
1926		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	2062	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	2063	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1927	2064	if (!last_stripe) {
1928	2065	j++;
1929	2066	sh = list_first_entry(&sh->batch_list, struct stripe_head,
..	..	@@ -1938,6 +2075,7 @@
1938	2075	{
1939	2076	struct async_submit_ctl submit;
1940	2077	struct page **blocks;
	2078	+ unsigned int *offs;
1941	2079	int count, i, j = 0;
1942	2080	struct stripe_head *head_sh = sh;
1943	2081	int last_stripe;
..	..	@@ -1962,6 +2100,7 @@
1962	2100
1963	2101	again:
1964	2102	blocks = to_addr_page(percpu, j);
	2103	+ offs = to_addr_offs(sh, percpu);
1965	2104
1966	2105	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1967	2106	synflags = SYNDROME_SRC_WRITTEN;
..	..	@@ -1971,7 +2110,7 @@
1971	2110	txflags = ASYNC_TX_ACK;
1972	2111	}
1973	2112
1974		- count = set_syndrome_sources(blocks, sh, synflags);
	2113	+ count = set_syndrome_sources(blocks, offs, sh, synflags);
1975	2114	last_stripe = !head_sh->batch_head \|\|
1976	2115	list_first_entry(&sh->batch_list,
1977	2116	struct stripe_head, batch_list) == head_sh;
..	..	@@ -1983,7 +2122,8 @@
1983	2122	} else
1984	2123	init_async_submit(&submit, 0, tx, NULL, NULL,
1985	2124	to_addr_conv(sh, percpu, j));
1986		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	2125	+ tx = async_gen_syndrome(blocks, offs, count+2,
	2126	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1987	2127	if (!last_stripe) {
1988	2128	j++;
1989	2129	sh = list_first_entry(&sh->batch_list, struct stripe_head,
..	..	@@ -2010,7 +2150,9 @@
2010	2150	int pd_idx = sh->pd_idx;
2011	2151	int qd_idx = sh->qd_idx;
2012	2152	struct page *xor_dest;
	2153	+ unsigned int off_dest;
2013	2154	struct page **xor_srcs = to_addr_page(percpu, 0);
	2155	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
2014	2156	struct dma_async_tx_descriptor *tx;
2015	2157	struct async_submit_ctl submit;
2016	2158	int count;
..	..	@@ -2022,16 +2164,20 @@
2022	2164	BUG_ON(sh->batch_head);
2023	2165	count = 0;
2024	2166	xor_dest = sh->dev[pd_idx].page;
	2167	+ off_dest = sh->dev[pd_idx].offset;
	2168	+ off_srcs[count] = off_dest;
2025	2169	xor_srcs[count++] = xor_dest;
2026	2170	for (i = disks; i--; ) {
2027	2171	if (i == pd_idx \|\| i == qd_idx)
2028	2172	continue;
	2173	+ off_srcs[count] = sh->dev[i].offset;
2029	2174	xor_srcs[count++] = sh->dev[i].page;
2030	2175	}
2031	2176
2032	2177	init_async_submit(&submit, 0, NULL, NULL, NULL,
2033	2178	to_addr_conv(sh, percpu, 0));
2034		- tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
	2179	+ tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	2180	+ RAID5_STRIPE_SIZE(sh->raid_conf),
2035	2181	&sh->ops.zero_sum_result, &submit);
2036	2182
2037	2183	atomic_inc(&sh->count);
..	..	@@ -2042,6 +2188,7 @@
2042	2188	static void ops_run_check_pq(struct stripe_head sh, struct raid5_percpu percpu, int checkp)
2043	2189	{
2044	2190	struct page **srcs = to_addr_page(percpu, 0);
	2191	+ unsigned int *offs = to_addr_offs(sh, percpu);
2045	2192	struct async_submit_ctl submit;
2046	2193	int count;
2047	2194
..	..	@@ -2049,15 +2196,16 @@
2049	2196	(unsigned long long)sh->sector, checkp);
2050	2197
2051	2198	BUG_ON(sh->batch_head);
2052		- count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
	2199	+ count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
2053	2200	if (!checkp)
2054	2201	srcs[count] = NULL;
2055	2202
2056	2203	atomic_inc(&sh->count);
2057	2204	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
2058	2205	sh, to_addr_conv(sh, percpu, 0));
2059		- async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
2060		- &sh->ops.zero_sum_result, percpu->spare_page, &submit);
	2206	+ async_syndrome_val(srcs, offs, count+2,
	2207	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	2208	+ &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
2061	2209	}
2062	2210
2063	2211	static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
..	..	@@ -2069,8 +2217,9 @@
2069	2217	struct raid5_percpu *percpu;
2070	2218	unsigned long cpu;
2071	2219
2072		- cpu = get_cpu();
	2220	+ cpu = get_cpu_light();
2073	2221	percpu = per_cpu_ptr(conf->percpu, cpu);
	2222	+ spin_lock(&percpu->lock);
2074	2223	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
2075	2224	ops_run_biofill(sh);
2076	2225	overlap_clear++;
..	..	@@ -2129,11 +2278,15 @@
2129	2278	if (test_and_clear_bit(R5_Overlap, &dev->flags))
2130	2279	wake_up(&sh->raid_conf->wait_for_overlap);
2131	2280	}
2132		- put_cpu();
	2281	+ spin_unlock(&percpu->lock);
	2282	+ put_cpu_light();
2133	2283	}
2134	2284
2135	2285	static void free_stripe(struct kmem_cache sc, struct stripe_head sh)
2136	2286	{
	2287	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2288	+ kfree(sh->pages);
	2289	+#endif
2137	2290	if (sh->ppl_page)
2138	2291	__free_page(sh->ppl_page);
2139	2292	kmem_cache_free(sc, sh);
..	..	@@ -2167,9 +2320,15 @@
2167	2320	sh->ppl_page = alloc_page(gfp);
2168	2321	if (!sh->ppl_page) {
2169	2322	free_stripe(sc, sh);
2170		- sh = NULL;
	2323	+ return NULL;
2171	2324	}
2172	2325	}
	2326	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2327	+ if (init_stripe_shared_pages(sh, conf, disks)) {
	2328	+ free_stripe(sc, sh);
	2329	+ return NULL;
	2330	+ }
	2331	+#endif
2173	2332	}
2174	2333	return sh;
2175	2334	}
..	..	@@ -2226,10 +2385,13 @@
2226	2385	}
2227	2386
2228	2387	/**
2229		- * scribble_len - return the required size of the scribble region
2230		- * @num - total number of disks in the array
	2388	+ * scribble_alloc - allocate percpu scribble buffer for required size
	2389	+ * of the scribble region
	2390	+ * @percpu: from for_each_present_cpu() of the caller
	2391	+ * @num: total number of disks in the array
	2392	+ * @cnt: scribble objs count for required size of the scribble region
2231	2393	*
2232		- * The size must be enough to contain:
	2394	+ * The scribble buffer size must be enough to contain:
2233	2395	* 1/ a struct page pointer for each device in the array +2
2234	2396	* 2/ room to convert each entry in (1) to its corresponding dma
2235	2397	* (dma_map_page()) or page (page_address()) address.
..	..	@@ -2238,21 +2400,29 @@
2238	2400	* calculate over all devices (not just the data blocks), using zeros in place
2239	2401	* of the P and Q blocks.
2240	2402	*/
2241		-static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
	2403	+static int scribble_alloc(struct raid5_percpu *percpu,
	2404	+ int num, int cnt)
2242	2405	{
2243		- struct flex_array *ret;
2244		- size_t len;
	2406	+ size_t obj_size =
	2407	+ sizeof(struct page ) (num + 2) +
	2408	+ sizeof(addr_conv_t) * (num + 2) +
	2409	+ sizeof(unsigned int) * (num + 2);
	2410	+ void *scribble;
2245	2411
2246		- len = sizeof(struct page ) (num+2) + sizeof(addr_conv_t) * (num+2);
2247		- ret = flex_array_alloc(len, cnt, flags);
2248		- if (!ret)
2249		- return NULL;
2250		- /* always prealloc all elements, so no locking is required */
2251		- if (flex_array_prealloc(ret, 0, cnt, flags)) {
2252		- flex_array_free(ret);
2253		- return NULL;
2254		- }
2255		- return ret;
	2412	+ /*
	2413	+ * If here is in raid array suspend context, it is in memalloc noio
	2414	+ * context as well, there is no potential recursive memory reclaim
	2415	+ * I/Os with the GFP_KERNEL flag.
	2416	+ */
	2417	+ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
	2418	+ if (!scribble)
	2419	+ return -ENOMEM;
	2420	+
	2421	+ kvfree(percpu->scribble);
	2422	+
	2423	+ percpu->scribble = scribble;
	2424	+ percpu->scribble_obj_size = obj_size;
	2425	+ return 0;
2256	2426	}
2257	2427
2258	2428	static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
..	..	@@ -2270,23 +2440,17 @@
2270	2440	return 0;
2271	2441	mddev_suspend(conf->mddev);
2272	2442	get_online_cpus();
	2443	+
2273	2444	for_each_present_cpu(cpu) {
2274	2445	struct raid5_percpu *percpu;
2275		- struct flex_array *scribble;
2276	2446
2277	2447	percpu = per_cpu_ptr(conf->percpu, cpu);
2278		- scribble = scribble_alloc(new_disks,
2279		- new_sectors / STRIPE_SECTORS,
2280		- GFP_NOIO);
2281		-
2282		- if (scribble) {
2283		- flex_array_free(percpu->scribble);
2284		- percpu->scribble = scribble;
2285		- } else {
2286		- err = -ENOMEM;
	2448	+ err = scribble_alloc(percpu, new_disks,
	2449	+ new_sectors / RAID5_STRIPE_SECTORS(conf));
	2450	+ if (err)
2287	2451	break;
2288		- }
2289	2452	}
	2453	+
2290	2454	put_online_cpus();
2291	2455	mddev_resume(conf->mddev);
2292	2456	if (!err) {
..	..	@@ -2374,9 +2538,16 @@
2374	2538	osh = get_free_stripe(conf, hash);
2375	2539	unlock_device_hash_lock(conf, hash);
2376	2540
	2541	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2542	+ for (i = 0; i < osh->nr_pages; i++) {
	2543	+ nsh->pages[i] = osh->pages[i];
	2544	+ osh->pages[i] = NULL;
	2545	+ }
	2546	+#endif
2377	2547	for(i=0; i<conf->pool_size; i++) {
2378	2548	nsh->dev[i].page = osh->dev[i].page;
2379	2549	nsh->dev[i].orig_page = osh->dev[i].page;
	2550	+ nsh->dev[i].offset = osh->dev[i].offset;
2380	2551	}
2381	2552	nsh->hash_lock_index = hash;
2382	2553	free_stripe(conf->slab_cache, osh);
..	..	@@ -2425,14 +2596,33 @@
2425	2596	nsh = list_entry(newstripes.next, struct stripe_head, lru);
2426	2597	list_del_init(&nsh->lru);
2427	2598
	2599	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2600	+ for (i = 0; i < nsh->nr_pages; i++) {
	2601	+ if (nsh->pages[i])
	2602	+ continue;
	2603	+ nsh->pages[i] = alloc_page(GFP_NOIO);
	2604	+ if (!nsh->pages[i])
	2605	+ err = -ENOMEM;
	2606	+ }
	2607	+
	2608	+ for (i = conf->raid_disks; i < newsize; i++) {
	2609	+ if (nsh->dev[i].page)
	2610	+ continue;
	2611	+ nsh->dev[i].page = raid5_get_dev_page(nsh, i);
	2612	+ nsh->dev[i].orig_page = nsh->dev[i].page;
	2613	+ nsh->dev[i].offset = raid5_get_page_offset(nsh, i);
	2614	+ }
	2615	+#else
2428	2616	for (i=conf->raid_disks; i < newsize; i++)
2429	2617	if (nsh->dev[i].page == NULL) {
2430	2618	struct page *p = alloc_page(GFP_NOIO);
2431	2619	nsh->dev[i].page = p;
2432	2620	nsh->dev[i].orig_page = p;
	2621	+ nsh->dev[i].offset = 0;
2433	2622	if (!p)
2434	2623	err = -ENOMEM;
2435	2624	}
	2625	+#endif
2436	2626	raid5_release_stripe(nsh);
2437	2627	}
2438	2628	/* critical section pass, GFP_NOIO no longer needed */
..	..	@@ -2516,10 +2706,10 @@
2516	2706	*/
2517	2707	pr_info_ratelimited(
2518	2708	"md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
2519		- mdname(conf->mddev), STRIPE_SECTORS,
	2709	+ mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),
2520	2710	(unsigned long long)s,
2521	2711	bdevname(rdev->bdev, b));
2522		- atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
	2712	+ atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);
2523	2713	clear_bit(R5_ReadError, &sh->dev[i].flags);
2524	2714	clear_bit(R5_ReWrite, &sh->dev[i].flags);
2525	2715	} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
..	..	@@ -2564,10 +2754,16 @@
2564	2754	(unsigned long long)s,
2565	2755	bdn);
2566	2756	} else if (atomic_read(&rdev->read_errors)
2567		- > conf->max_nr_stripes)
2568		- pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
2569		- mdname(conf->mddev), bdn);
2570		- else
	2757	+ > conf->max_nr_stripes) {
	2758	+ if (!test_bit(Faulty, &rdev->flags)) {
	2759	+ pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
	2760	+ mdname(conf->mddev),
	2761	+ atomic_read(&rdev->read_errors),
	2762	+ conf->max_nr_stripes);
	2763	+ pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
	2764	+ mdname(conf->mddev), bdn);
	2765	+ }
	2766	+ } else
2571	2767	retry = 1;
2572	2768	if (set_bad && test_bit(In_sync, &rdev->flags)
2573	2769	&& !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
..	..	@@ -2586,7 +2782,7 @@
2586	2782	if (!(set_bad
2587	2783	&& test_bit(In_sync, &rdev->flags)
2588	2784	&& rdev_set_badblocks(
2589		- rdev, sh->sector, STRIPE_SECTORS, 0)))
	2785	+ rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
2590	2786	md_error(conf->mddev, rdev);
2591	2787	}
2592	2788	}
..	..	@@ -2602,7 +2798,7 @@
2602	2798	struct stripe_head *sh = bi->bi_private;
2603	2799	struct r5conf *conf = sh->raid_conf;
2604	2800	int disks = sh->disks, i;
2605		- struct md_rdev *uninitialized_var(rdev);
	2801	+ struct md_rdev *rdev;
2606	2802	sector_t first_bad;
2607	2803	int bad_sectors;
2608	2804	int replacement = 0;
..	..	@@ -2638,7 +2834,7 @@
2638	2834	if (bi->bi_status)
2639	2835	md_error(conf->mddev, rdev);
2640	2836	else if (is_badblock(rdev, sh->sector,
2641		- STRIPE_SECTORS,
	2837	+ RAID5_STRIPE_SECTORS(conf),
2642	2838	&first_bad, &bad_sectors))
2643	2839	set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2644	2840	} else {
..	..	@@ -2650,7 +2846,7 @@
2650	2846	set_bit(MD_RECOVERY_NEEDED,
2651	2847	&rdev->mddev->recovery);
2652	2848	} else if (is_badblock(rdev, sh->sector,
2653		- STRIPE_SECTORS,
	2849	+ RAID5_STRIPE_SECTORS(conf),
2654	2850	&first_bad, &bad_sectors)) {
2655	2851	set_bit(R5_MadeGood, &sh->dev[i].flags);
2656	2852	if (test_bit(R5_ReadError, &sh->dev[i].flags))
..	..	@@ -2670,10 +2866,10 @@
2670	2866	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2671	2867	clear_bit(R5_LOCKED, &sh->dev[i].flags);
2672	2868	set_bit(STRIPE_HANDLE, &sh->state);
2673		- raid5_release_stripe(sh);
2674	2869
2675	2870	if (sh->batch_head && sh != sh->batch_head)
2676	2871	raid5_release_stripe(sh->batch_head);
	2872	+ raid5_release_stripe(sh);
2677	2873	}
2678	2874
2679	2875	static void raid5_error(struct mddev mddev, struct md_rdev rdev)
..	..	@@ -2683,22 +2879,31 @@
2683	2879	unsigned long flags;
2684	2880	pr_debug("raid456: error called\n");
2685	2881
	2882	+ pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n",
	2883	+ mdname(mddev), bdevname(rdev->bdev, b));
	2884	+
2686	2885	spin_lock_irqsave(&conf->device_lock, flags);
2687	2886	set_bit(Faulty, &rdev->flags);
2688	2887	clear_bit(In_sync, &rdev->flags);
2689	2888	mddev->degraded = raid5_calc_degraded(conf);
	2889	+
	2890	+ if (has_failed(conf)) {
	2891	+ set_bit(MD_BROKEN, &conf->mddev->flags);
	2892	+ conf->recovery_disabled = mddev->recovery_disabled;
	2893	+
	2894	+ pr_crit("md/raid:%s: Cannot continue operation (%d/%d failed).\n",
	2895	+ mdname(mddev), mddev->degraded, conf->raid_disks);
	2896	+ } else {
	2897	+ pr_crit("md/raid:%s: Operation continuing on %d devices.\n",
	2898	+ mdname(mddev), conf->raid_disks - mddev->degraded);
	2899	+ }
	2900	+
2690	2901	spin_unlock_irqrestore(&conf->device_lock, flags);
2691	2902	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2692	2903
2693	2904	set_bit(Blocked, &rdev->flags);
2694	2905	set_mask_bits(&mddev->sb_flags, 0,
2695	2906	BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
2696		- pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
2697		- "md/raid:%s: Operation continuing on %d devices.\n",
2698		- mdname(mddev),
2699		- bdevname(rdev->bdev, b),
2700		- mdname(mddev),
2701		- conf->raid_disks - mddev->degraded);
2702	2907	r5c_update_on_rdev_error(mddev, rdev);
2703	2908	}
2704	2909
..	..	@@ -3272,13 +3477,13 @@
3272	3477	/* check if page is covered */
3273	3478	sector_t sector = sh->dev[dd_idx].sector;
3274	3479	for (bi=sh->dev[dd_idx].towrite;
3275		- sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
	3480	+ sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
3276	3481	bi && bi->bi_iter.bi_sector <= sector;
3277		- bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
	3482	+ bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
3278	3483	if (bio_end_sector(bi) >= sector)
3279	3484	sector = bio_end_sector(bi);
3280	3485	}
3281		- if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
	3486	+ if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
3282	3487	if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
3283	3488	sh->overwrite_disks++;
3284	3489	}
..	..	@@ -3303,7 +3508,7 @@
3303	3508	set_bit(STRIPE_BITMAP_PENDING, &sh->state);
3304	3509	spin_unlock_irq(&sh->stripe_lock);
3305	3510	md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
3306		- STRIPE_SECTORS, 0);
	3511	+ RAID5_STRIPE_SECTORS(conf), 0);
3307	3512	spin_lock_irq(&sh->stripe_lock);
3308	3513	clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
3309	3514	if (!sh->batch_head) {
..	..	@@ -3365,7 +3570,7 @@
3365	3570	if (!rdev_set_badblocks(
3366	3571	rdev,
3367	3572	sh->sector,
3368		- STRIPE_SECTORS, 0))
	3573	+ RAID5_STRIPE_SECTORS(conf), 0))
3369	3574	md_error(conf->mddev, rdev);
3370	3575	rdev_dec_pending(rdev, conf->mddev);
3371	3576	}
..	..	@@ -3385,8 +3590,8 @@
3385	3590	wake_up(&conf->wait_for_overlap);
3386	3591
3387	3592	while (bi && bi->bi_iter.bi_sector <
3388		- sh->dev[i].sector + STRIPE_SECTORS) {
3389		- struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
	3593	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
	3594	+ struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
3390	3595
3391	3596	md_write_end(conf->mddev);
3392	3597	bio_io_error(bi);
..	..	@@ -3394,7 +3599,7 @@
3394	3599	}
3395	3600	if (bitmap_end)
3396	3601	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3397		- STRIPE_SECTORS, 0, 0);
	3602	+ RAID5_STRIPE_SECTORS(conf), 0, 0);
3398	3603	bitmap_end = 0;
3399	3604	/* and fail all 'written' */
3400	3605	bi = sh->dev[i].written;
..	..	@@ -3406,8 +3611,8 @@
3406	3611
3407	3612	if (bi) bitmap_end = 1;
3408	3613	while (bi && bi->bi_iter.bi_sector <
3409		- sh->dev[i].sector + STRIPE_SECTORS) {
3410		- struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
	3614	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
	3615	+ struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
3411	3616
3412	3617	md_write_end(conf->mddev);
3413	3618	bio_io_error(bi);
..	..	@@ -3430,9 +3635,9 @@
3430	3635	if (bi)
3431	3636	s->to_read--;
3432	3637	while (bi && bi->bi_iter.bi_sector <
3433		- sh->dev[i].sector + STRIPE_SECTORS) {
	3638	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3434	3639	struct bio *nextbi =
3435		- r5_next_bio(bi, sh->dev[i].sector);
	3640	+ r5_next_bio(conf, bi, sh->dev[i].sector);
3436	3641
3437	3642	bio_io_error(bi);
3438	3643	bi = nextbi;
..	..	@@ -3440,7 +3645,7 @@
3440	3645	}
3441	3646	if (bitmap_end)
3442	3647	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3443		- STRIPE_SECTORS, 0, 0);
	3648	+ RAID5_STRIPE_SECTORS(conf), 0, 0);
3444	3649	/* If we were in the middle of a write the parity block might
3445	3650	* still be locked - so just clear all R5_LOCKED flags
3446	3651	*/
..	..	@@ -3485,14 +3690,14 @@
3485	3690	&& !test_bit(Faulty, &rdev->flags)
3486	3691	&& !test_bit(In_sync, &rdev->flags)
3487	3692	&& !rdev_set_badblocks(rdev, sh->sector,
3488		- STRIPE_SECTORS, 0))
	3693	+ RAID5_STRIPE_SECTORS(conf), 0))
3489	3694	abort = 1;
3490	3695	rdev = rcu_dereference(conf->disks[i].replacement);
3491	3696	if (rdev
3492	3697	&& !test_bit(Faulty, &rdev->flags)
3493	3698	&& !test_bit(In_sync, &rdev->flags)
3494	3699	&& !rdev_set_badblocks(rdev, sh->sector,
3495		- STRIPE_SECTORS, 0))
	3700	+ RAID5_STRIPE_SECTORS(conf), 0))
3496	3701	abort = 1;
3497	3702	}
3498	3703	rcu_read_unlock();
..	..	@@ -3500,7 +3705,7 @@
3500	3705	conf->recovery_disabled =
3501	3706	conf->mddev->recovery_disabled;
3502	3707	}
3503		- md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
	3708	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);
3504	3709	}
3505	3710
3506	3711	static int want_replace(struct stripe_head *sh, int disk_idx)
..	..	@@ -3527,6 +3732,7 @@
3527	3732	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3528	3733	&sh->dev[s->failed_num[1]] };
3529	3734	int i;
	3735	+ bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
3530	3736
3531	3737
3532	3738	if (test_bit(R5_LOCKED, &dev->flags) \|\|
..	..	@@ -3585,18 +3791,27 @@
3585	3791	* devices must be read.
3586	3792	*/
3587	3793	return 1;
	3794	+
	3795	+ if (s->failed >= 2 &&
	3796	+ (fdev[i]->towrite \|\|
	3797	+ s->failed_num[i] == sh->pd_idx \|\|
	3798	+ s->failed_num[i] == sh->qd_idx) &&
	3799	+ !test_bit(R5_UPTODATE, &fdev[i]->flags))
	3800	+ /* In max degraded raid6, If the failed disk is P, Q,
	3801	+ * or we want to read the failed disk, we need to do
	3802	+ * reconstruct-write.
	3803	+ */
	3804	+ force_rcw = true;
3588	3805	}
3589	3806
3590		- /* If we are forced to do a reconstruct-write, either because
3591		- * the current RAID6 implementation only supports that, or
3592		- * because parity cannot be trusted and we are currently
3593		- * recovering it, there is extra need to be careful.
	3807	+ /* If we are forced to do a reconstruct-write, because parity
	3808	+ * cannot be trusted and we are currently recovering it, there
	3809	+ * is extra need to be careful.
3594	3810	* If one of the devices that we would need to read, because
3595	3811	* it is not being overwritten (and maybe not written at all)
3596	3812	* is missing/faulty, then we need to read everything we can.
3597	3813	*/
3598		- if (sh->raid_conf->level != 6 &&
3599		- sh->raid_conf->rmw_level != PARITY_DISABLE_RMW &&
	3814	+ if (!force_rcw &&
3600	3815	sh->sector < sh->raid_conf->mddev->recovery_cp)
3601	3816	/* reconstruct-write isn't being forced */
3602	3817	return 0;
..	..	@@ -3700,7 +3915,7 @@
3700	3915	return 0;
3701	3916	}
3702	3917
3703		-/**
	3918	+/*
3704	3919	* handle_stripe_fill - read or compute data to satisfy pending requests.
3705	3920	*/
3706	3921	static void handle_stripe_fill(struct stripe_head *sh,
..	..	@@ -3723,7 +3938,7 @@
3723	3938	* back cache (prexor with orig_page, and then xor with
3724	3939	* page) in the read path
3725	3940	*/
3726		- if (s->injournal && s->failed) {
	3941	+ if (s->to_read && s->injournal && s->failed) {
3727	3942	if (test_bit(STRIPE_R5C_CACHING, &sh->state))
3728	3943	r5c_make_stripe_write_out(sh);
3729	3944	goto out;
..	..	@@ -3775,14 +3990,14 @@
3775	3990	wbi = dev->written;
3776	3991	dev->written = NULL;
3777	3992	while (wbi && wbi->bi_iter.bi_sector <
3778		- dev->sector + STRIPE_SECTORS) {
3779		- wbi2 = r5_next_bio(wbi, dev->sector);
	3993	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
	3994	+ wbi2 = r5_next_bio(conf, wbi, dev->sector);
3780	3995	md_write_end(conf->mddev);
3781	3996	bio_endio(wbi);
3782	3997	wbi = wbi2;
3783	3998	}
3784	3999	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3785		- STRIPE_SECTORS,
	4000	+ RAID5_STRIPE_SECTORS(conf),
3786	4001	!test_bit(STRIPE_DEGRADED, &sh->state),
3787	4002	0);
3788	4003	if (head_sh->batch_head) {
..	..	@@ -3966,10 +4181,8 @@
3966	4181	set_bit(R5_LOCKED, &dev->flags);
3967	4182	set_bit(R5_Wantread, &dev->flags);
3968	4183	s->locked++;
3969		- } else {
	4184	+ } else
3970	4185	set_bit(STRIPE_DELAYED, &sh->state);
3971		- set_bit(STRIPE_HANDLE, &sh->state);
3972		- }
3973	4186	}
3974	4187	}
3975	4188	}
..	..	@@ -3994,10 +4207,8 @@
3994	4207	set_bit(R5_Wantread, &dev->flags);
3995	4208	s->locked++;
3996	4209	qread++;
3997		- } else {
	4210	+ } else
3998	4211	set_bit(STRIPE_DELAYED, &sh->state);
3999		- set_bit(STRIPE_HANDLE, &sh->state);
4000		- }
4001	4212	}
4002	4213	}
4003	4214	if (rcw && conf->mddev->queue)
..	..	@@ -4047,7 +4258,7 @@
4047	4258	break;
4048	4259	}
4049	4260	dev = &sh->dev[s->failed_num[0]];
4050		- /* fall through */
	4261	+ fallthrough;
4051	4262	case check_state_compute_result:
4052	4263	sh->check_state = check_state_idle;
4053	4264	if (!dev)
..	..	@@ -4089,7 +4300,7 @@
4089	4300	*/
4090	4301	set_bit(STRIPE_INSYNC, &sh->state);
4091	4302	else {
4092		- atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
	4303	+ atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
4093	4304	if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
4094	4305	/* don't try to repair!! */
4095	4306	set_bit(STRIPE_INSYNC, &sh->state);
..	..	@@ -4097,7 +4308,7 @@
4097	4308	"%llu-%llu\n", mdname(conf->mddev),
4098	4309	(unsigned long long) sh->sector,
4099	4310	(unsigned long long) sh->sector +
4100		- STRIPE_SECTORS);
	4311	+ RAID5_STRIPE_SECTORS(conf));
4101	4312	} else {
4102	4313	sh->check_state = check_state_compute_run;
4103	4314	set_bit(STRIPE_COMPUTE_RUN, &sh->state);
..	..	@@ -4178,7 +4389,7 @@
4178	4389
4179	4390	/* we have 2-disk failure */
4180	4391	BUG_ON(s->failed != 2);
4181		- /* fall through */
	4392	+ fallthrough;
4182	4393	case check_state_compute_result:
4183	4394	sh->check_state = check_state_idle;
4184	4395
..	..	@@ -4254,7 +4465,7 @@
4254	4465	*/
4255	4466	}
4256	4467	} else {
4257		- atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
	4468	+ atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
4258	4469	if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
4259	4470	/* don't try to repair!! */
4260	4471	set_bit(STRIPE_INSYNC, &sh->state);
..	..	@@ -4262,7 +4473,7 @@
4262	4473	"%llu-%llu\n", mdname(conf->mddev),
4263	4474	(unsigned long long) sh->sector,
4264	4475	(unsigned long long) sh->sector +
4265		- STRIPE_SECTORS);
	4476	+ RAID5_STRIPE_SECTORS(conf));
4266	4477	} else {
4267	4478	int *target = &sh->ops.target;
4268	4479
..	..	@@ -4333,7 +4544,8 @@
4333	4544	/* place all the copies on one channel */
4334	4545	init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
4335	4546	tx = async_memcpy(sh2->dev[dd_idx].page,
4336		- sh->dev[i].page, 0, 0, STRIPE_SIZE,
	4547	+ sh->dev[i].page, sh2->dev[dd_idx].offset,
	4548	+ sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
4337	4549	&submit);
4338	4550
4339	4551	set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
..	..	@@ -4432,8 +4644,8 @@
4432	4644	*/
4433	4645	rdev = rcu_dereference(conf->disks[i].replacement);
4434	4646	if (rdev && !test_bit(Faulty, &rdev->flags) &&
4435		- rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
4436		- !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	4647	+ rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
	4648	+ !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4437	4649	&first_bad, &bad_sectors))
4438	4650	set_bit(R5_ReadRepl, &dev->flags);
4439	4651	else {
..	..	@@ -4447,7 +4659,7 @@
4447	4659	if (rdev && test_bit(Faulty, &rdev->flags))
4448	4660	rdev = NULL;
4449	4661	if (rdev) {
4450		- is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	4662	+ is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4451	4663	&first_bad, &bad_sectors);
4452	4664	if (s->blocked_rdev == NULL
4453	4665	&& (test_bit(Blocked, &rdev->flags)
..	..	@@ -4474,7 +4686,7 @@
4474	4686	}
4475	4687	} else if (test_bit(In_sync, &rdev->flags))
4476	4688	set_bit(R5_Insync, &dev->flags);
4477		- else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
	4689	+ else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
4478	4690	/* in sync if before recovery_offset */
4479	4691	set_bit(R5_Insync, &dev->flags);
4480	4692	else if (test_bit(R5_UPTODATE, &dev->flags) &&
..	..	@@ -4563,12 +4775,12 @@
4563	4775	rcu_read_unlock();
4564	4776	}
4565	4777
	4778	+/*
	4779	+ * Return '1' if this is a member of batch, or '0' if it is a lone stripe or
	4780	+ * a head which can now be handled.
	4781	+ */
4566	4782	static int clear_batch_ready(struct stripe_head *sh)
4567	4783	{
4568		- /* Return '1' if this is a member of batch, or
4569		- * '0' if it is a lone stripe or a head which can now be
4570		- * handled.
4571		- */
4572	4784	struct stripe_head *tmp;
4573	4785	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
4574	4786	return (sh->batch_head && sh->batch_head != sh);
..	..	@@ -4618,7 +4830,6 @@
4618	4830	(1 << STRIPE_FULL_WRITE) \|
4619	4831	(1 << STRIPE_BIOFILL_RUN) \|
4620	4832	(1 << STRIPE_COMPUTE_RUN) \|
4621		- (1 << STRIPE_OPS_REQ_PENDING) \|
4622	4833	(1 << STRIPE_DISCARD) \|
4623	4834	(1 << STRIPE_BATCH_READY) \|
4624	4835	(1 << STRIPE_BATCH_ERR) \|
..	..	@@ -4673,15 +4884,20 @@
4673	4884	struct r5dev pdev, qdev;
4674	4885
4675	4886	clear_bit(STRIPE_HANDLE, &sh->state);
	4887	+
	4888	+ /*
	4889	+ * handle_stripe should not continue handle the batched stripe, only
	4890	+ * the head of batch list or lone stripe can continue. Otherwise we
	4891	+ * could see break_stripe_batch_list warns about the STRIPE_ACTIVE
	4892	+ * is set for the batched stripe.
	4893	+ */
	4894	+ if (clear_batch_ready(sh))
	4895	+ return;
	4896	+
4676	4897	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4677	4898	/* already being handled, ensure it gets handled
4678	4899	* again when current action finishes */
4679	4900	set_bit(STRIPE_HANDLE, &sh->state);
4680		- return;
4681		- }
4682		-
4683		- if (clear_batch_ready(sh) ) {
4684		- clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4685	4901	return;
4686	4902	}
4687	4903
..	..	@@ -4918,7 +5134,7 @@
4918	5134	if ((s.syncing \|\| s.replacing) && s.locked == 0 &&
4919	5135	!test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4920	5136	test_bit(STRIPE_INSYNC, &sh->state)) {
4921		- md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	5137	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
4922	5138	clear_bit(STRIPE_SYNCING, &sh->state);
4923	5139	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
4924	5140	wake_up(&conf->wait_for_overlap);
..	..	@@ -4937,14 +5153,11 @@
4937	5153	if (!test_bit(R5_ReWrite, &dev->flags)) {
4938	5154	set_bit(R5_Wantwrite, &dev->flags);
4939	5155	set_bit(R5_ReWrite, &dev->flags);
4940		- set_bit(R5_LOCKED, &dev->flags);
4941		- s.locked++;
4942		- } else {
	5156	+ } else
4943	5157	/* let's read it back */
4944	5158	set_bit(R5_Wantread, &dev->flags);
4945		- set_bit(R5_LOCKED, &dev->flags);
4946		- s.locked++;
4947		- }
	5159	+ set_bit(R5_LOCKED, &dev->flags);
	5160	+ s.locked++;
4948	5161	}
4949	5162	}
4950	5163
..	..	@@ -4986,7 +5199,7 @@
4986	5199	clear_bit(STRIPE_EXPAND_READY, &sh->state);
4987	5200	atomic_dec(&conf->reshape_stripes);
4988	5201	wake_up(&conf->wait_for_overlap);
4989		- md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	5202	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
4990	5203	}
4991	5204
4992	5205	if (s.expanding && s.locked == 0 &&
..	..	@@ -5016,14 +5229,14 @@
5016	5229	/* We own a safe reference to the rdev */
5017	5230	rdev = conf->disks[i].rdev;
5018	5231	if (!rdev_set_badblocks(rdev, sh->sector,
5019		- STRIPE_SECTORS, 0))
	5232	+ RAID5_STRIPE_SECTORS(conf), 0))
5020	5233	md_error(conf->mddev, rdev);
5021	5234	rdev_dec_pending(rdev, conf->mddev);
5022	5235	}
5023	5236	if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
5024	5237	rdev = conf->disks[i].rdev;
5025	5238	rdev_clear_badblocks(rdev, sh->sector,
5026		- STRIPE_SECTORS, 0);
	5239	+ RAID5_STRIPE_SECTORS(conf), 0);
5027	5240	rdev_dec_pending(rdev, conf->mddev);
5028	5241	}
5029	5242	if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
..	..	@@ -5032,7 +5245,7 @@
5032	5245	/* rdev have been moved down */
5033	5246	rdev = conf->disks[i].rdev;
5034	5247	rdev_clear_badblocks(rdev, sh->sector,
5035		- STRIPE_SECTORS, 0);
	5248	+ RAID5_STRIPE_SECTORS(conf), 0);
5036	5249	rdev_dec_pending(rdev, conf->mddev);
5037	5250	}
5038	5251	}
..	..	@@ -5088,28 +5301,6 @@
5088	5301	hash = sh->hash_lock_index;
5089	5302	__release_stripe(conf, sh, &temp_inactive_list[hash]);
5090	5303	}
5091		-}
5092		-
5093		-static int raid5_congested(struct mddev *mddev, int bits)
5094		-{
5095		- struct r5conf *conf = mddev->private;
5096		-
5097		- /* No difference between reads and writes. Just check
5098		- * how busy the stripe_cache is
5099		- */
5100		-
5101		- if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
5102		- return 1;
5103		-
5104		- /* Also checks whether there is pressure on r5cache log space */
5105		- if (test_bit(R5C_LOG_TIGHT, &conf->cache_state))
5106		- return 1;
5107		- if (conf->quiesce)
5108		- return 1;
5109		- if (atomic_read(&conf->empty_inactive_list_nr))
5110		- return 1;
5111		-
5112		- return 0;
5113	5304	}
5114	5305
5115	5306	static int in_chunk_boundary(struct mddev mddev, struct bio bio)
..	..	@@ -5257,7 +5448,6 @@
5257	5448	rcu_read_unlock();
5258	5449	raid_bio->bi_next = (void*)rdev;
5259	5450	bio_set_dev(align_bi, rdev->bdev);
5260		- bio_clear_flag(align_bi, BIO_SEG_VALID);
5261	5451
5262	5452	if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
5263	5453	bio_sectors(align_bi),
..	..	@@ -5281,7 +5471,7 @@
5281	5471	trace_block_bio_remap(align_bi->bi_disk->queue,
5282	5472	align_bi, disk_devt(mddev->gendisk),
5283	5473	raid_bio->bi_iter.bi_sector);
5284		- generic_make_request(align_bi);
	5474	+ submit_bio_noacct(align_bi);
5285	5475	return 1;
5286	5476	} else {
5287	5477	rcu_read_unlock();
..	..	@@ -5301,7 +5491,7 @@
5301	5491	struct r5conf *conf = mddev->private;
5302	5492	split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
5303	5493	bio_chain(split, raid_bio);
5304		- generic_make_request(raid_bio);
	5494	+ submit_bio_noacct(raid_bio);
5305	5495	raid_bio = split;
5306	5496	}
5307	5497
..	..	@@ -5497,8 +5687,8 @@
5497	5687	/* Skip discard while reshape is happening */
5498	5688	return;
5499	5689
5500		- logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
5501		- last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
	5690	+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
	5691	+ last_sector = bio_end_sector(bi);
5502	5692
5503	5693	bi->bi_next = NULL;
5504	5694
..	..	@@ -5512,7 +5702,7 @@
5512	5702	last_sector *= conf->chunk_sectors;
5513	5703
5514	5704	for (; logical_sector < last_sector;
5515		- logical_sector += STRIPE_SECTORS) {
	5705	+ logical_sector += RAID5_STRIPE_SECTORS(conf)) {
5516	5706	DEFINE_WAIT(w);
5517	5707	int d;
5518	5708	again:
..	..	@@ -5557,7 +5747,7 @@
5557	5747	d++)
5558	5748	md_bitmap_startwrite(mddev->bitmap,
5559	5749	sh->sector,
5560		- STRIPE_SECTORS,
	5750	+ RAID5_STRIPE_SECTORS(conf),
5561	5751	0);
5562	5752	sh->bm_seq = conf->seq_flush + 1;
5563	5753	set_bit(STRIPE_BIT_DELAY, &sh->state);
..	..	@@ -5622,12 +5812,12 @@
5622	5812	return true;
5623	5813	}
5624	5814
5625		- logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	5815	+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
5626	5816	last_sector = bio_end_sector(bi);
5627	5817	bi->bi_next = NULL;
5628	5818
5629	5819	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
5630		- for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
	5820	+ for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
5631	5821	int previous;
5632	5822	int seq;
5633	5823
..	..	@@ -5725,8 +5915,7 @@
5725	5915	do_flush = false;
5726	5916	}
5727	5917
5728		- if (!sh->batch_head \|\| sh == sh->batch_head)
5729		- set_bit(STRIPE_HANDLE, &sh->state);
	5918	+ set_bit(STRIPE_HANDLE, &sh->state);
5730	5919	clear_bit(STRIPE_DELAYED, &sh->state);
5731	5920	if ((!sh->batch_head \|\| sh == sh->batch_head) &&
5732	5921	(bi->bi_opf & REQ_SYNC) &&
..	..	@@ -5791,7 +5980,7 @@
5791	5980	sector_div(sector_nr, new_data_disks);
5792	5981	if (sector_nr) {
5793	5982	mddev->curr_resync_completed = sector_nr;
5794		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	5983	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
5795	5984	*skipped = 1;
5796	5985	retn = sector_nr;
5797	5986	goto finish;
..	..	@@ -5905,11 +6094,11 @@
5905	6094	conf->reshape_safe = mddev->reshape_position;
5906	6095	spin_unlock_irq(&conf->device_lock);
5907	6096	wake_up(&conf->wait_for_overlap);
5908		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	6097	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
5909	6098	}
5910	6099
5911	6100	INIT_LIST_HEAD(&stripes);
5912		- for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
	6101	+ for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {
5913	6102	int j;
5914	6103	int skipped_disk = 0;
5915	6104	sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
..	..	@@ -5930,7 +6119,7 @@
5930	6119	skipped_disk = 1;
5931	6120	continue;
5932	6121	}
5933		- memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
	6122	+ memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
5934	6123	set_bit(R5_Expanded, &sh->dev[j].flags);
5935	6124	set_bit(R5_UPTODATE, &sh->dev[j].flags);
5936	6125	}
..	..	@@ -5965,7 +6154,7 @@
5965	6154	set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
5966	6155	set_bit(STRIPE_HANDLE, &sh->state);
5967	6156	raid5_release_stripe(sh);
5968		- first_sector += STRIPE_SECTORS;
	6157	+ first_sector += RAID5_STRIPE_SECTORS(conf);
5969	6158	}
5970	6159	/* Now that the sources are clearly marked, we can release
5971	6160	* the destination stripes
..	..	@@ -6012,7 +6201,7 @@
6012	6201	conf->reshape_safe = mddev->reshape_position;
6013	6202	spin_unlock_irq(&conf->device_lock);
6014	6203	wake_up(&conf->wait_for_overlap);
6015		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	6204	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
6016	6205	}
6017	6206	ret:
6018	6207	return retn;
..	..	@@ -6071,11 +6260,12 @@
6071	6260	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
6072	6261	!conf->fullsync &&
6073	6262	!md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
6074		- sync_blocks >= STRIPE_SECTORS) {
	6263	+ sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {
6075	6264	/* we can skip this block, and probably more */
6076		- sync_blocks /= STRIPE_SECTORS;
	6265	+ do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));
6077	6266	*skipped = 1;
6078		- return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	6267	+ /* keep things rounded to whole stripes */
	6268	+ return sync_blocks * RAID5_STRIPE_SECTORS(conf);
6079	6269	}
6080	6270
6081	6271	md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
..	..	@@ -6108,7 +6298,7 @@
6108	6298
6109	6299	raid5_release_stripe(sh);
6110	6300
6111		- return STRIPE_SECTORS;
	6301	+ return RAID5_STRIPE_SECTORS(conf);
6112	6302	}
6113	6303
6114	6304	static int retry_aligned_read(struct r5conf conf, struct bio raid_bio,
..	..	@@ -6131,14 +6321,14 @@
6131	6321	int handled = 0;
6132	6322
6133	6323	logical_sector = raid_bio->bi_iter.bi_sector &
6134		- ~((sector_t)STRIPE_SECTORS-1);
	6324	+ ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
6135	6325	sector = raid5_compute_sector(conf, logical_sector,
6136	6326	0, &dd_idx, NULL);
6137	6327	last_sector = bio_end_sector(raid_bio);
6138	6328
6139	6329	for (; logical_sector < last_sector;
6140		- logical_sector += STRIPE_SECTORS,
6141		- sector += STRIPE_SECTORS,
	6330	+ logical_sector += RAID5_STRIPE_SECTORS(conf),
	6331	+ sector += RAID5_STRIPE_SECTORS(conf),
6142	6332	scnt++) {
6143	6333
6144	6334	if (scnt < offset)
..	..	@@ -6177,6 +6367,8 @@
6177	6367	static int handle_active_stripes(struct r5conf *conf, int group,
6178	6368	struct r5worker *worker,
6179	6369	struct list_head *temp_inactive_list)
	6370	+ __releases(&conf->device_lock)
	6371	+ __acquires(&conf->device_lock)
6180	6372	{
6181	6373	struct stripe_head batch[MAX_STRIPE_BATCH], sh;
6182	6374	int i, batch_size = 0, hash;
..	..	@@ -6329,7 +6521,18 @@
6329	6521	spin_unlock_irq(&conf->device_lock);
6330	6522	md_check_recovery(mddev);
6331	6523	spin_lock_irq(&conf->device_lock);
	6524	+
	6525	+ /*
	6526	+ * Waiting on MD_SB_CHANGE_PENDING below may deadlock
	6527	+ * seeing md_check_recovery() is needed to clear
	6528	+ * the flag when using mdmon.
	6529	+ */
	6530	+ continue;
6332	6531	}
	6532	+
	6533	+ wait_event_lock_irq(mddev->sb_wait,
	6534	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
	6535	+ conf->device_lock);
6333	6536	}
6334	6537	pr_debug("%d stripes handled\n", handled);
6335	6538
..	..	@@ -6469,6 +6672,100 @@
6469	6672	raid5_show_rmw_level,
6470	6673	raid5_store_rmw_level);
6471	6674
	6675	+static ssize_t
	6676	+raid5_show_stripe_size(struct mddev mddev, char page)
	6677	+{
	6678	+ struct r5conf *conf;
	6679	+ int ret = 0;
	6680	+
	6681	+ spin_lock(&mddev->lock);
	6682	+ conf = mddev->private;
	6683	+ if (conf)
	6684	+ ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));
	6685	+ spin_unlock(&mddev->lock);
	6686	+ return ret;
	6687	+}
	6688	+
	6689	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	6690	+static ssize_t
	6691	+raid5_store_stripe_size(struct mddev mddev, const char page, size_t len)
	6692	+{
	6693	+ struct r5conf *conf;
	6694	+ unsigned long new;
	6695	+ int err;
	6696	+ int size;
	6697	+
	6698	+ if (len >= PAGE_SIZE)
	6699	+ return -EINVAL;
	6700	+ if (kstrtoul(page, 10, &new))
	6701	+ return -EINVAL;
	6702	+
	6703	+ /*
	6704	+ * The value should not be bigger than PAGE_SIZE. It requires to
	6705	+ * be multiple of DEFAULT_STRIPE_SIZE and the value should be power
	6706	+ * of two.
	6707	+ */
	6708	+ if (new % DEFAULT_STRIPE_SIZE != 0 \|\|
	6709	+ new > PAGE_SIZE \|\| new == 0 \|\|
	6710	+ new != roundup_pow_of_two(new))
	6711	+ return -EINVAL;
	6712	+
	6713	+ err = mddev_lock(mddev);
	6714	+ if (err)
	6715	+ return err;
	6716	+
	6717	+ conf = mddev->private;
	6718	+ if (!conf) {
	6719	+ err = -ENODEV;
	6720	+ goto out_unlock;
	6721	+ }
	6722	+
	6723	+ if (new == conf->stripe_size)
	6724	+ goto out_unlock;
	6725	+
	6726	+ pr_debug("md/raid: change stripe_size from %lu to %lu\n",
	6727	+ conf->stripe_size, new);
	6728	+
	6729	+ if (mddev->sync_thread \|\|
	6730	+ test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) \|\|
	6731	+ mddev->reshape_position != MaxSector \|\|
	6732	+ mddev->sysfs_active) {
	6733	+ err = -EBUSY;
	6734	+ goto out_unlock;
	6735	+ }
	6736	+
	6737	+ mddev_suspend(mddev);
	6738	+ mutex_lock(&conf->cache_size_mutex);
	6739	+ size = conf->max_nr_stripes;
	6740	+
	6741	+ shrink_stripes(conf);
	6742	+
	6743	+ conf->stripe_size = new;
	6744	+ conf->stripe_shift = ilog2(new) - 9;
	6745	+ conf->stripe_sectors = new >> 9;
	6746	+ if (grow_stripes(conf, size)) {
	6747	+ pr_warn("md/raid:%s: couldn't allocate buffers\n",
	6748	+ mdname(mddev));
	6749	+ err = -ENOMEM;
	6750	+ }
	6751	+ mutex_unlock(&conf->cache_size_mutex);
	6752	+ mddev_resume(mddev);
	6753	+
	6754	+out_unlock:
	6755	+ mddev_unlock(mddev);
	6756	+ return err ?: len;
	6757	+}
	6758	+
	6759	+static struct md_sysfs_entry
	6760	+raid5_stripe_size = __ATTR(stripe_size, 0644,
	6761	+ raid5_show_stripe_size,
	6762	+ raid5_store_stripe_size);
	6763	+#else
	6764	+static struct md_sysfs_entry
	6765	+raid5_stripe_size = __ATTR(stripe_size, 0444,
	6766	+ raid5_show_stripe_size,
	6767	+ NULL);
	6768	+#endif
6472	6769
6473	6770	static ssize_t
6474	6771	raid5_show_preread_threshold(struct mddev mddev, char page)
..	..	@@ -6548,14 +6845,14 @@
6548	6845	if (!conf)
6549	6846	err = -ENODEV;
6550	6847	else if (new != conf->skip_copy) {
	6848	+ struct request_queue *q = mddev->queue;
	6849	+
6551	6850	mddev_suspend(mddev);
6552	6851	conf->skip_copy = new;
6553	6852	if (new)
6554		- mddev->queue->backing_dev_info->capabilities \|=
6555		- BDI_CAP_STABLE_WRITES;
	6853	+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
6556	6854	else
6557		- mddev->queue->backing_dev_info->capabilities &=
6558		- ~BDI_CAP_STABLE_WRITES;
	6855	+ blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
6559	6856	mddev_resume(mddev);
6560	6857	}
6561	6858	mddev_unlock(mddev);
..	..	@@ -6595,7 +6892,6 @@
6595	6892
6596	6893	static int alloc_thread_groups(struct r5conf *conf, int cnt,
6597	6894	int *group_cnt,
6598		- int *worker_cnt_per_group,
6599	6895	struct r5worker_group **worker_groups);
6600	6896	static ssize_t
6601	6897	raid5_store_group_thread_cnt(struct mddev mddev, const char page, size_t len)
..	..	@@ -6604,7 +6900,7 @@
6604	6900	unsigned int new;
6605	6901	int err;
6606	6902	struct r5worker_group new_groups, old_groups;
6607		- int group_cnt, worker_cnt_per_group;
	6903	+ int group_cnt;
6608	6904
6609	6905	if (len >= PAGE_SIZE)
6610	6906	return -EINVAL;
..	..	@@ -6627,13 +6923,11 @@
6627	6923	if (old_groups)
6628	6924	flush_workqueue(raid5_wq);
6629	6925
6630		- err = alloc_thread_groups(conf, new,
6631		- &group_cnt, &worker_cnt_per_group,
6632		- &new_groups);
	6926	+ err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
6633	6927	if (!err) {
6634	6928	spin_lock_irq(&conf->device_lock);
6635	6929	conf->group_cnt = group_cnt;
6636		- conf->worker_cnt_per_group = worker_cnt_per_group;
	6930	+ conf->worker_cnt_per_group = new;
6637	6931	conf->worker_groups = new_groups;
6638	6932	spin_unlock_irq(&conf->device_lock);
6639	6933
..	..	@@ -6660,7 +6954,9 @@
6660	6954	&raid5_group_thread_cnt.attr,
6661	6955	&raid5_skip_copy.attr,
6662	6956	&raid5_rmw_level.attr,
	6957	+ &raid5_stripe_size.attr,
6663	6958	&r5c_journal_mode.attr,
	6959	+ &ppl_write_hint.attr,
6664	6960	NULL,
6665	6961	};
6666	6962	static struct attribute_group raid5_attrs_group = {
..	..	@@ -6668,16 +6964,13 @@
6668	6964	.attrs = raid5_attrs,
6669	6965	};
6670	6966
6671		-static int alloc_thread_groups(struct r5conf *conf, int cnt,
6672		- int *group_cnt,
6673		- int *worker_cnt_per_group,
	6967	+static int alloc_thread_groups(struct r5conf conf, int cnt, int group_cnt,
6674	6968	struct r5worker_group **worker_groups)
6675	6969	{
6676	6970	int i, j, k;
6677	6971	ssize_t size;
6678	6972	struct r5worker *workers;
6679	6973
6680		- *worker_cnt_per_group = cnt;
6681	6974	if (cnt == 0) {
6682	6975	*group_cnt = 0;
6683	6976	*worker_groups = NULL;
..	..	@@ -6743,25 +7036,25 @@
6743	7036	static void free_scratch_buffer(struct r5conf conf, struct raid5_percpu percpu)
6744	7037	{
6745	7038	safe_put_page(percpu->spare_page);
6746		- if (percpu->scribble)
6747		- flex_array_free(percpu->scribble);
6748	7039	percpu->spare_page = NULL;
	7040	+ kvfree(percpu->scribble);
6749	7041	percpu->scribble = NULL;
6750	7042	}
6751	7043
6752	7044	static int alloc_scratch_buffer(struct r5conf conf, struct raid5_percpu percpu)
6753	7045	{
6754		- if (conf->level == 6 && !percpu->spare_page)
	7046	+ if (conf->level == 6 && !percpu->spare_page) {
6755	7047	percpu->spare_page = alloc_page(GFP_KERNEL);
6756		- if (!percpu->scribble)
6757		- percpu->scribble = scribble_alloc(max(conf->raid_disks,
6758		- conf->previous_raid_disks),
6759		- max(conf->chunk_sectors,
6760		- conf->prev_chunk_sectors)
6761		- / STRIPE_SECTORS,
6762		- GFP_KERNEL);
	7048	+ if (!percpu->spare_page)
	7049	+ return -ENOMEM;
	7050	+ }
6763	7051
6764		- if (!percpu->scribble \|\| (conf->level == 6 && !percpu->spare_page)) {
	7052	+ if (scribble_alloc(percpu,
	7053	+ max(conf->raid_disks,
	7054	+ conf->previous_raid_disks),
	7055	+ max(conf->chunk_sectors,
	7056	+ conf->prev_chunk_sectors)
	7057	+ / RAID5_STRIPE_SECTORS(conf))) {
6765	7058	free_scratch_buffer(conf, percpu);
6766	7059	return -ENOMEM;
6767	7060	}
..	..	@@ -6816,6 +7109,7 @@
6816	7109	__func__, cpu);
6817	7110	return -ENOMEM;
6818	7111	}
	7112	+ spin_lock_init(&per_cpu_ptr(conf->percpu, cpu)->lock);
6819	7113	return 0;
6820	7114	}
6821	7115
..	..	@@ -6877,7 +7171,7 @@
6877	7171	struct disk_info *disk;
6878	7172	char pers_name[6];
6879	7173	int i;
6880		- int group_cnt, worker_cnt_per_group;
	7174	+ int group_cnt;
6881	7175	struct r5worker_group *new_group;
6882	7176	int ret;
6883	7177
..	..	@@ -6913,6 +7207,12 @@
6913	7207	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
6914	7208	if (conf == NULL)
6915	7209	goto abort;
	7210	+
	7211	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	7212	+ conf->stripe_size = DEFAULT_STRIPE_SIZE;
	7213	+ conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
	7214	+ conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;
	7215	+#endif
6916	7216	INIT_LIST_HEAD(&conf->free_list);
6917	7217	INIT_LIST_HEAD(&conf->pending_list);
6918	7218	conf->pending_data = kcalloc(PENDING_IO_MAX,
..	..	@@ -6923,15 +7223,14 @@
6923	7223	for (i = 0; i < PENDING_IO_MAX; i++)
6924	7224	list_add(&conf->pending_data[i].sibling, &conf->free_list);
6925	7225	/* Don't enable multi-threading by default*/
6926		- if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6927		- &new_group)) {
	7226	+ if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
6928	7227	conf->group_cnt = group_cnt;
6929		- conf->worker_cnt_per_group = worker_cnt_per_group;
	7228	+ conf->worker_cnt_per_group = 0;
6930	7229	conf->worker_groups = new_group;
6931	7230	} else
6932	7231	goto abort;
6933	7232	spin_lock_init(&conf->device_lock);
6934		- seqcount_init(&conf->gen_lock);
	7233	+ seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);
6935	7234	mutex_init(&conf->cache_size_mutex);
6936	7235	init_waitqueue_head(&conf->wait_for_quiescent);
6937	7236	init_waitqueue_head(&conf->wait_for_stripe);
..	..	@@ -7065,8 +7364,8 @@
7065	7364	conf->min_nr_stripes = NR_STRIPES;
7066	7365	if (mddev->reshape_position != MaxSector) {
7067	7366	int stripes = max_t(int,
7068		- ((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4,
7069		- ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4);
	7367	+ ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,
	7368	+ ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);
7070	7369	conf->min_nr_stripes = max(NR_STRIPES, stripes);
7071	7370	if (conf->min_nr_stripes != NR_STRIPES)
7072	7371	pr_info("md/raid:%s: force stripe size %d for reshape\n",
..	..	@@ -7139,6 +7438,12 @@
7139	7438	return 1;
7140	7439	}
7141	7440	return 0;
	7441	+}
	7442	+
	7443	+static void raid5_set_io_opt(struct r5conf *conf)
	7444	+{
	7445	+ blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
	7446	+ (conf->raid_disks - conf->max_degraded));
7142	7447	}
7143	7448
7144	7449	static int raid5_run(struct mddev *mddev)
..	..	@@ -7425,13 +7730,10 @@
7425	7730	int data_disks = conf->previous_raid_disks - conf->max_degraded;
7426	7731	int stripe = data_disks *
7427	7732	((mddev->chunk_sectors << 9) / PAGE_SIZE);
7428		- if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
7429		- mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
7430	7733
7431	7734	chunk_size = mddev->chunk_sectors << 9;
7432	7735	blk_queue_io_min(mddev->queue, chunk_size);
7433		- blk_queue_io_opt(mddev->queue, chunk_size *
7434		- (conf->raid_disks - conf->max_degraded));
	7736	+ raid5_set_io_opt(conf);
7435	7737	mddev->queue->limits.raid_partial_stripes_expensive = 1;
7436	7738	/*
7437	7739	* We can only discard a whole stripe. It doesn't make sense to
..	..	@@ -7716,6 +8018,7 @@
7716	8018	*/
7717	8019	if (rdev->saved_raid_disk >= 0 &&
7718	8020	rdev->saved_raid_disk >= first &&
	8021	+ rdev->saved_raid_disk <= last &&
7719	8022	conf->disks[rdev->saved_raid_disk].rdev == NULL)
7720	8023	first = rdev->saved_raid_disk;
7721	8024
..	..	@@ -7797,14 +8100,14 @@
7797	8100	* stripe_heads first.
7798	8101	*/
7799	8102	struct r5conf *conf = mddev->private;
7800		- if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	8103	+ if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
7801	8104	> conf->min_nr_stripes \|\|
7802		- ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	8105	+ ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
7803	8106	> conf->min_nr_stripes) {
7804	8107	pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
7805	8108	mdname(mddev),
7806	8109	((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
7807		- / STRIPE_SIZE)*4);
	8110	+ / RAID5_STRIPE_SIZE(conf))*4);
7808	8111	return 0;
7809	8112	}
7810	8113	return 1;
..	..	@@ -7940,8 +8243,8 @@
7940	8243	else
7941	8244	rdev->recovery_offset = 0;
7942	8245
7943		- if (sysfs_link_rdev(mddev, rdev))
7944		- /* Failure here is OK */;
	8246	+ /* Failure here is OK */
	8247	+ sysfs_link_rdev(mddev, rdev);
7945	8248	}
7946	8249	} else if (rdev->raid_disk >= conf->previous_raid_disks
7947	8250	&& !test_bit(Faulty, &rdev->flags)) {
..	..	@@ -8015,16 +8318,8 @@
8015	8318	spin_unlock_irq(&conf->device_lock);
8016	8319	wake_up(&conf->wait_for_overlap);
8017	8320
8018		- /* read-ahead size must cover two whole stripes, which is
8019		- * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
8020		- */
8021		- if (conf->mddev->queue) {
8022		- int data_disks = conf->raid_disks - conf->max_degraded;
8023		- int stripe = data_disks * ((conf->chunk_sectors << 9)
8024		- / PAGE_SIZE);
8025		- if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
8026		- conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
8027		- }
	8321	+ if (conf->mddev->queue)
	8322	+ raid5_set_io_opt(conf);
8028	8323	}
8029	8324	}
8030	8325
..	..	@@ -8136,7 +8431,7 @@
8136	8431	while (chunksect && (mddev->array_sectors & (chunksect-1)))
8137	8432	chunksect >>= 1;
8138	8433
8139		- if ((chunksect<<9) < STRIPE_SIZE)
	8434	+ if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))
8140	8435	/* array size does not allow a suitable chunk size */
8141	8436	return ERR_PTR(-EINVAL);
8142	8437
..	..	@@ -8423,7 +8718,6 @@
8423	8718	.finish_reshape = raid5_finish_reshape,
8424	8719	.quiesce = raid5_quiesce,
8425	8720	.takeover = raid6_takeover,
8426		- .congested = raid5_congested,
8427	8721	.change_consistency_policy = raid5_change_consistency_policy,
8428	8722	};
8429	8723	static struct md_personality raid5_personality =
..	..	@@ -8448,7 +8742,6 @@
8448	8742	.finish_reshape = raid5_finish_reshape,
8449	8743	.quiesce = raid5_quiesce,
8450	8744	.takeover = raid5_takeover,
8451		- .congested = raid5_congested,
8452	8745	.change_consistency_policy = raid5_change_consistency_policy,
8453	8746	};
8454	8747
..	..	@@ -8474,7 +8767,6 @@
8474	8767	.finish_reshape = raid5_finish_reshape,
8475	8768	.quiesce = raid5_quiesce,
8476	8769	.takeover = raid4_takeover,
8477		- .congested = raid5_congested,
8478	8770	.change_consistency_policy = raid5_change_consistency_policy,
8479	8771	};
8480	8772