~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* raid5.c : Multiple Devices driver for Linux
3	4	* Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
..	..	@@ -7,15 +8,6 @@
7	8	* RAID-4/5/6 management functions.
8	9	* Thanks to Penguin Computing for making the RAID-6 development possible
9	10	* by donating a test server!
10		- *
11		- * This program is free software; you can redistribute it and/or modify
12		- * it under the terms of the GNU General Public License as published by
13		- * the Free Software Foundation; either version 2, or (at your option)
14		- * any later version.
15		- *
16		- * You should have received a copy of the GNU General Public License
17		- * (for example /usr/src/linux/COPYING); if not, write to the Free
18		- * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19	11	*/
20	12
21	13	/*
..	..	@@ -44,6 +36,7 @@
44	36	*/
45	37
46	38	#include <linux/blkdev.h>
	39	+#include <linux/delay.h>
47	40	#include <linux/kthread.h>
48	41	#include <linux/raid/pq.h>
49	42	#include <linux/async_tx.h>
..	..	@@ -54,7 +47,6 @@
54	47	#include <linux/slab.h>
55	48	#include <linux/ratelimit.h>
56	49	#include <linux/nodemask.h>
57		-#include <linux/flex_array.h>
58	50
59	51	#include <trace/events/block.h>
60	52	#include <linux/list_sort.h>
..	..	@@ -78,13 +70,13 @@
78	70
79	71	static inline struct hlist_head stripe_hash(struct r5conf conf, sector_t sect)
80	72	{
81		- int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	73	+ int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
82	74	return &conf->stripe_hashtbl[hash];
83	75	}
84	76
85		-static inline int stripe_hash_locks_hash(sector_t sect)
	77	+static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)
86	78	{
87		- return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
	79	+ return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;
88	80	}
89	81
90	82	static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
..	..	@@ -457,13 +449,74 @@
457	449	return sh;
458	450	}
459	451
	452	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	453	+static void free_stripe_pages(struct stripe_head *sh)
	454	+{
	455	+ int i;
	456	+ struct page *p;
	457	+
	458	+ /* Have not allocate page pool */
	459	+ if (!sh->pages)
	460	+ return;
	461	+
	462	+ for (i = 0; i < sh->nr_pages; i++) {
	463	+ p = sh->pages[i];
	464	+ if (p)
	465	+ put_page(p);
	466	+ sh->pages[i] = NULL;
	467	+ }
	468	+}
	469	+
	470	+static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
	471	+{
	472	+ int i;
	473	+ struct page *p;
	474	+
	475	+ for (i = 0; i < sh->nr_pages; i++) {
	476	+ /* The page have allocated. */
	477	+ if (sh->pages[i])
	478	+ continue;
	479	+
	480	+ p = alloc_page(gfp);
	481	+ if (!p) {
	482	+ free_stripe_pages(sh);
	483	+ return -ENOMEM;
	484	+ }
	485	+ sh->pages[i] = p;
	486	+ }
	487	+ return 0;
	488	+}
	489	+
	490	+static int
	491	+init_stripe_shared_pages(struct stripe_head sh, struct r5conf conf, int disks)
	492	+{
	493	+ int nr_pages, cnt;
	494	+
	495	+ if (sh->pages)
	496	+ return 0;
	497	+
	498	+ /* Each of the sh->dev[i] need one conf->stripe_size */
	499	+ cnt = PAGE_SIZE / conf->stripe_size;
	500	+ nr_pages = (disks + cnt - 1) / cnt;
	501	+
	502	+ sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
	503	+ if (!sh->pages)
	504	+ return -ENOMEM;
	505	+ sh->nr_pages = nr_pages;
	506	+ sh->stripes_per_page = cnt;
	507	+ return 0;
	508	+}
	509	+#endif
	510	+
460	511	static void shrink_buffers(struct stripe_head *sh)
461	512	{
462		- struct page *p;
463	513	int i;
464	514	int num = sh->raid_conf->pool_size;
465	515
	516	+#if PAGE_SIZE == DEFAULT_STRIPE_SIZE
466	517	for (i = 0; i < num ; i++) {
	518	+ struct page *p;
	519	+
467	520	WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
468	521	p = sh->dev[i].page;
469	522	if (!p)
..	..	@@ -471,6 +524,11 @@
471	524	sh->dev[i].page = NULL;
472	525	put_page(p);
473	526	}
	527	+#else
	528	+ for (i = 0; i < num; i++)
	529	+ sh->dev[i].page = NULL;
	530	+ free_stripe_pages(sh); /* Free pages */
	531	+#endif
474	532	}
475	533
476	534	static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
..	..	@@ -478,6 +536,7 @@
478	536	int i;
479	537	int num = sh->raid_conf->pool_size;
480	538
	539	+#if PAGE_SIZE == DEFAULT_STRIPE_SIZE
481	540	for (i = 0; i < num; i++) {
482	541	struct page *page;
483	542
..	..	@@ -486,8 +545,18 @@
486	545	}
487	546	sh->dev[i].page = page;
488	547	sh->dev[i].orig_page = page;
	548	+ sh->dev[i].offset = 0;
489	549	}
	550	+#else
	551	+ if (alloc_stripe_pages(sh, gfp))
	552	+ return -ENOMEM;
490	553
	554	+ for (i = 0; i < num; i++) {
	555	+ sh->dev[i].page = raid5_get_dev_page(sh, i);
	556	+ sh->dev[i].orig_page = sh->dev[i].page;
	557	+ sh->dev[i].offset = raid5_get_page_offset(sh, i);
	558	+ }
	559	+#endif
491	560	return 0;
492	561	}
493	562
..	..	@@ -618,17 +687,17 @@
618	687	return degraded;
619	688	}
620	689
621		-static int has_failed(struct r5conf *conf)
	690	+static bool has_failed(struct r5conf *conf)
622	691	{
623		- int degraded;
	692	+ int degraded = conf->mddev->degraded;
624	693
625		- if (conf->mddev->reshape_position == MaxSector)
626		- return conf->mddev->degraded > conf->max_degraded;
	694	+ if (test_bit(MD_BROKEN, &conf->mddev->flags))
	695	+ return true;
627	696
628		- degraded = raid5_calc_degraded(conf);
629		- if (degraded > conf->max_degraded)
630		- return 1;
631		- return 0;
	697	+ if (conf->mddev->reshape_position != MaxSector)
	698	+ degraded = raid5_calc_degraded(conf);
	699	+
	700	+ return degraded > conf->max_degraded;
632	701	}
633	702
634	703	struct stripe_head *
..	..	@@ -636,7 +705,7 @@
636	705	int previous, int noblock, int noquiesce)
637	706	{
638	707	struct stripe_head *sh;
639		- int hash = stripe_hash_locks_hash(sector);
	708	+ int hash = stripe_hash_locks_hash(conf, sector);
640	709	int inc_empty_inactive_list_flag;
641	710
642	711	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
..	..	@@ -712,6 +781,8 @@
712	781	}
713	782
714	783	static void lock_two_stripes(struct stripe_head sh1, struct stripe_head sh2)
	784	+ __acquires(&sh1->stripe_lock)
	785	+ __acquires(&sh2->stripe_lock)
715	786	{
716	787	if (sh1 > sh2) {
717	788	spin_lock_irq(&sh2->stripe_lock);
..	..	@@ -723,6 +794,8 @@
723	794	}
724	795
725	796	static void unlock_two_stripes(struct stripe_head sh1, struct stripe_head sh2)
	797	+ __releases(&sh1->stripe_lock)
	798	+ __releases(&sh2->stripe_lock)
726	799	{
727	800	spin_unlock(&sh1->stripe_lock);
728	801	spin_unlock_irq(&sh2->stripe_lock);
..	..	@@ -753,9 +826,9 @@
753	826	tmp_sec = sh->sector;
754	827	if (!sector_div(tmp_sec, conf->chunk_sectors))
755	828	return;
756		- head_sector = sh->sector - STRIPE_SECTORS;
	829	+ head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
757	830
758		- hash = stripe_hash_locks_hash(head_sector);
	831	+ hash = stripe_hash_locks_hash(conf, head_sector);
759	832	spin_lock_irq(conf->hash_locks + hash);
760	833	head = __find_stripe(conf, head_sector, conf->generation);
761	834	if (head && !atomic_inc_not_zero(&head->count)) {
..	..	@@ -878,7 +951,7 @@
878	951	struct bio *bio;
879	952
880	953	while ((bio = bio_list_pop(tmp)))
881		- generic_make_request(bio);
	954	+ submit_bio_noacct(bio);
882	955	}
883	956
884	957	static int cmp_stripe(void priv, struct list_head a, struct list_head *b)
..	..	@@ -1062,7 +1135,7 @@
1062	1135	test_bit(WriteErrorSeen, &rdev->flags)) {
1063	1136	sector_t first_bad;
1064	1137	int bad_sectors;
1065		- int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	1138	+ int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
1066	1139	&first_bad, &bad_sectors);
1067	1140	if (!bad)
1068	1141	break;
..	..	@@ -1094,7 +1167,7 @@
1094	1167	if (rdev) {
1095	1168	if (s->syncing \|\| s->expanding \|\| s->expanded
1096	1169	\|\| s->replacing)
1097		- md_sync_acct(rdev->bdev, STRIPE_SECTORS);
	1170	+ md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));
1098	1171
1099	1172	set_bit(STRIPE_IO_STARTED, &sh->state);
1100	1173
..	..	@@ -1134,12 +1207,12 @@
1134	1207	else
1135	1208	sh->dev[i].vec.bv_page = sh->dev[i].page;
1136	1209	bi->bi_vcnt = 1;
1137		- bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1138		- bi->bi_io_vec[0].bv_offset = 0;
1139		- bi->bi_iter.bi_size = STRIPE_SIZE;
	1210	+ bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
	1211	+ bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
	1212	+ bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
1140	1213	bi->bi_write_hint = sh->dev[i].write_hint;
1141	1214	if (!rrdev)
1142		- sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
	1215	+ sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1143	1216	/*
1144	1217	* If this is discard request, set bi_vcnt 0. We don't
1145	1218	* want to confuse SCSI because SCSI will replace payload
..	..	@@ -1156,12 +1229,12 @@
1156	1229	if (should_defer && op_is_write(op))
1157	1230	bio_list_add(&pending_bios, bi);
1158	1231	else
1159		- generic_make_request(bi);
	1232	+ submit_bio_noacct(bi);
1160	1233	}
1161	1234	if (rrdev) {
1162	1235	if (s->syncing \|\| s->expanding \|\| s->expanded
1163	1236	\|\| s->replacing)
1164		- md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
	1237	+ md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));
1165	1238
1166	1239	set_bit(STRIPE_IO_STARTED, &sh->state);
1167	1240
..	..	@@ -1188,11 +1261,11 @@
1188	1261	WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1189	1262	sh->dev[i].rvec.bv_page = sh->dev[i].page;
1190	1263	rbi->bi_vcnt = 1;
1191		- rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1192		- rbi->bi_io_vec[0].bv_offset = 0;
1193		- rbi->bi_iter.bi_size = STRIPE_SIZE;
	1264	+ rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
	1265	+ rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
	1266	+ rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
1194	1267	rbi->bi_write_hint = sh->dev[i].write_hint;
1195		- sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
	1268	+ sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1196	1269	/*
1197	1270	* If this is discard request, set bi_vcnt 0. We don't
1198	1271	* want to confuse SCSI because SCSI will replace payload
..	..	@@ -1206,7 +1279,7 @@
1206	1279	if (should_defer && op_is_write(op))
1207	1280	bio_list_add(&pending_bios, rbi);
1208	1281	else
1209		- generic_make_request(rbi);
	1282	+ submit_bio_noacct(rbi);
1210	1283	}
1211	1284	if (!rdev && !rrdev) {
1212	1285	if (op_is_write(op))
..	..	@@ -1231,7 +1304,7 @@
1231	1304
1232	1305	static struct dma_async_tx_descriptor *
1233	1306	async_copy_data(int frombio, struct bio bio, struct page *page,
1234		- sector_t sector, struct dma_async_tx_descriptor *tx,
	1307	+ unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
1235	1308	struct stripe_head *sh, int no_skipcopy)
1236	1309	{
1237	1310	struct bio_vec bvl;
..	..	@@ -1240,6 +1313,7 @@
1240	1313	int page_offset;
1241	1314	struct async_submit_ctl submit;
1242	1315	enum async_tx_flags flags = 0;
	1316	+ struct r5conf *conf = sh->raid_conf;
1243	1317
1244	1318	if (bio->bi_iter.bi_sector >= sector)
1245	1319	page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
..	..	@@ -1261,8 +1335,8 @@
1261	1335	len -= b_offset;
1262	1336	}
1263	1337
1264		- if (len > 0 && page_offset + len > STRIPE_SIZE)
1265		- clen = STRIPE_SIZE - page_offset;
	1338	+ if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))
	1339	+ clen = RAID5_STRIPE_SIZE(conf) - page_offset;
1266	1340	else
1267	1341	clen = len;
1268	1342
..	..	@@ -1270,17 +1344,17 @@
1270	1344	b_offset += bvl.bv_offset;
1271	1345	bio_page = bvl.bv_page;
1272	1346	if (frombio) {
1273		- if (sh->raid_conf->skip_copy &&
	1347	+ if (conf->skip_copy &&
1274	1348	b_offset == 0 && page_offset == 0 &&
1275		- clen == STRIPE_SIZE &&
	1349	+ clen == RAID5_STRIPE_SIZE(conf) &&
1276	1350	!no_skipcopy)
1277	1351	*page = bio_page;
1278	1352	else
1279		- tx = async_memcpy(*page, bio_page, page_offset,
	1353	+ tx = async_memcpy(*page, bio_page, page_offset + poff,
1280	1354	b_offset, clen, &submit);
1281	1355	} else
1282	1356	tx = async_memcpy(bio_page, *page, b_offset,
1283		- page_offset, clen, &submit);
	1357	+ page_offset + poff, clen, &submit);
1284	1358	}
1285	1359	/* chain the operations */
1286	1360	submit.depend_tx = tx;
..	..	@@ -1297,6 +1371,7 @@
1297	1371	{
1298	1372	struct stripe_head *sh = stripe_head_ref;
1299	1373	int i;
	1374	+ struct r5conf *conf = sh->raid_conf;
1300	1375
1301	1376	pr_debug("%s: stripe %llu\n", __func__,
1302	1377	(unsigned long long)sh->sector);
..	..	@@ -1317,8 +1392,8 @@
1317	1392	rbi = dev->read;
1318	1393	dev->read = NULL;
1319	1394	while (rbi && rbi->bi_iter.bi_sector <
1320		- dev->sector + STRIPE_SECTORS) {
1321		- rbi2 = r5_next_bio(rbi, dev->sector);
	1395	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
	1396	+ rbi2 = r5_next_bio(conf, rbi, dev->sector);
1322	1397	bio_endio(rbi);
1323	1398	rbi = rbi2;
1324	1399	}
..	..	@@ -1335,6 +1410,7 @@
1335	1410	struct dma_async_tx_descriptor *tx = NULL;
1336	1411	struct async_submit_ctl submit;
1337	1412	int i;
	1413	+ struct r5conf *conf = sh->raid_conf;
1338	1414
1339	1415	BUG_ON(sh->batch_head);
1340	1416	pr_debug("%s: stripe %llu\n", __func__,
..	..	@@ -1349,10 +1425,11 @@
1349	1425	dev->toread = NULL;
1350	1426	spin_unlock_irq(&sh->stripe_lock);
1351	1427	while (rbi && rbi->bi_iter.bi_sector <
1352		- dev->sector + STRIPE_SECTORS) {
	1428	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
1353	1429	tx = async_copy_data(0, rbi, &dev->page,
	1430	+ dev->offset,
1354	1431	dev->sector, tx, sh, 0);
1355		- rbi = r5_next_bio(rbi, dev->sector);
	1432	+ rbi = r5_next_bio(conf, rbi, dev->sector);
1356	1433	}
1357	1434	}
1358	1435	}
..	..	@@ -1394,22 +1471,25 @@
1394	1471	}
1395	1472
1396	1473	/* return a pointer to the address conversion region of the scribble buffer */
1397		-static addr_conv_t to_addr_conv(struct stripe_head sh,
1398		- struct raid5_percpu *percpu, int i)
	1474	+static struct page *to_addr_page(struct raid5_percpu percpu, int i)
1399	1475	{
1400		- void *addr;
1401		-
1402		- addr = flex_array_get(percpu->scribble, i);
1403		- return addr + sizeof(struct page ) (sh->disks + 2);
	1476	+ return percpu->scribble + i * percpu->scribble_obj_size;
1404	1477	}
1405	1478
1406	1479	/* return a pointer to the address conversion region of the scribble buffer */
1407		-static struct page *to_addr_page(struct raid5_percpu percpu, int i)
	1480	+static addr_conv_t to_addr_conv(struct stripe_head sh,
	1481	+ struct raid5_percpu *percpu, int i)
1408	1482	{
1409		- void *addr;
	1483	+ return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
	1484	+}
1410	1485
1411		- addr = flex_array_get(percpu->scribble, i);
1412		- return addr;
	1486	+/*
	1487	+ * Return a pointer to record offset address.
	1488	+ */
	1489	+static unsigned int *
	1490	+to_addr_offs(struct stripe_head sh, struct raid5_percpu percpu)
	1491	+{
	1492	+ return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
1413	1493	}
1414	1494
1415	1495	static struct dma_async_tx_descriptor *
..	..	@@ -1417,9 +1497,11 @@
1417	1497	{
1418	1498	int disks = sh->disks;
1419	1499	struct page **xor_srcs = to_addr_page(percpu, 0);
	1500	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
1420	1501	int target = sh->ops.target;
1421	1502	struct r5dev *tgt = &sh->dev[target];
1422	1503	struct page *xor_dest = tgt->page;
	1504	+ unsigned int off_dest = tgt->offset;
1423	1505	int count = 0;
1424	1506	struct dma_async_tx_descriptor *tx;
1425	1507	struct async_submit_ctl submit;
..	..	@@ -1431,24 +1513,30 @@
1431	1513	__func__, (unsigned long long)sh->sector, target);
1432	1514	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1433	1515
1434		- for (i = disks; i--; )
1435		- if (i != target)
	1516	+ for (i = disks; i--; ) {
	1517	+ if (i != target) {
	1518	+ off_srcs[count] = sh->dev[i].offset;
1436	1519	xor_srcs[count++] = sh->dev[i].page;
	1520	+ }
	1521	+ }
1437	1522
1438	1523	atomic_inc(&sh->count);
1439	1524
1440	1525	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST, NULL,
1441	1526	ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1442	1527	if (unlikely(count == 1))
1443		- tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	1528	+ tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
	1529	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1444	1530	else
1445		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	1531	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	1532	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1446	1533
1447	1534	return tx;
1448	1535	}
1449	1536
1450	1537	/* set_syndrome_sources - populate source buffers for gen_syndrome
1451	1538	* @srcs - (struct page *) array of size sh->disks
	1539	+ * @offs - (unsigned int) array of offset for each page
1452	1540	* @sh - stripe_head to parse
1453	1541	*
1454	1542	* Populates srcs in proper layout order for the stripe and returns the
..	..	@@ -1457,6 +1545,7 @@
1457	1545	* is recorded in srcs[count+1]].
1458	1546	*/
1459	1547	static int set_syndrome_sources(struct page **srcs,
	1548	+ unsigned int *offs,
1460	1549	struct stripe_head *sh,
1461	1550	int srctype)
1462	1551	{
..	..	@@ -1487,6 +1576,12 @@
1487	1576	srcs[slot] = sh->dev[i].orig_page;
1488	1577	else
1489	1578	srcs[slot] = sh->dev[i].page;
	1579	+ /*
	1580	+ * For R5_InJournal, PAGE_SIZE must be 4KB and will
	1581	+ * not shared page. In that case, dev[i].offset
	1582	+ * is 0.
	1583	+ */
	1584	+ offs[slot] = sh->dev[i].offset;
1490	1585	}
1491	1586	i = raid6_next_disk(i, disks);
1492	1587	} while (i != d0_idx);
..	..	@@ -1499,12 +1594,14 @@
1499	1594	{
1500	1595	int disks = sh->disks;
1501	1596	struct page **blocks = to_addr_page(percpu, 0);
	1597	+ unsigned int *offs = to_addr_offs(sh, percpu);
1502	1598	int target;
1503	1599	int qd_idx = sh->qd_idx;
1504	1600	struct dma_async_tx_descriptor *tx;
1505	1601	struct async_submit_ctl submit;
1506	1602	struct r5dev *tgt;
1507	1603	struct page *dest;
	1604	+ unsigned int dest_off;
1508	1605	int i;
1509	1606	int count;
1510	1607
..	..	@@ -1523,30 +1620,34 @@
1523	1620	tgt = &sh->dev[target];
1524	1621	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1525	1622	dest = tgt->page;
	1623	+ dest_off = tgt->offset;
1526	1624
1527	1625	atomic_inc(&sh->count);
1528	1626
1529	1627	if (target == qd_idx) {
1530		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
	1628	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1531	1629	blocks[count] = NULL; /* regenerating p is not necessary */
1532	1630	BUG_ON(blocks[count+1] != dest); /* q should already be set */
1533	1631	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1534	1632	ops_complete_compute, sh,
1535	1633	to_addr_conv(sh, percpu, 0));
1536		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	1634	+ tx = async_gen_syndrome(blocks, offs, count+2,
	1635	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1537	1636	} else {
1538	1637	/* Compute any data- or p-drive using XOR */
1539	1638	count = 0;
1540	1639	for (i = disks; i-- ; ) {
1541	1640	if (i == target \|\| i == qd_idx)
1542	1641	continue;
	1642	+ offs[count] = sh->dev[i].offset;
1543	1643	blocks[count++] = sh->dev[i].page;
1544	1644	}
1545	1645
1546	1646	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST,
1547	1647	NULL, ops_complete_compute, sh,
1548	1648	to_addr_conv(sh, percpu, 0));
1549		- tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	1649	+ tx = async_xor_offs(dest, dest_off, blocks, offs, count,
	1650	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1550	1651	}
1551	1652
1552	1653	return tx;
..	..	@@ -1565,6 +1666,7 @@
1565	1666	struct r5dev *tgt2 = &sh->dev[target2];
1566	1667	struct dma_async_tx_descriptor *tx;
1567	1668	struct page **blocks = to_addr_page(percpu, 0);
	1669	+ unsigned int *offs = to_addr_offs(sh, percpu);
1568	1670	struct async_submit_ctl submit;
1569	1671
1570	1672	BUG_ON(sh->batch_head);
..	..	@@ -1577,13 +1679,16 @@
1577	1679	/* we need to open-code set_syndrome_sources to handle the
1578	1680	* slot number conversion for 'faila' and 'failb'
1579	1681	*/
1580		- for (i = 0; i < disks ; i++)
	1682	+ for (i = 0; i < disks ; i++) {
	1683	+ offs[i] = 0;
1581	1684	blocks[i] = NULL;
	1685	+ }
1582	1686	count = 0;
1583	1687	i = d0_idx;
1584	1688	do {
1585	1689	int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1586	1690
	1691	+ offs[slot] = sh->dev[i].offset;
1587	1692	blocks[slot] = sh->dev[i].page;
1588	1693
1589	1694	if (i == target)
..	..	@@ -1608,10 +1713,12 @@
1608	1713	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1609	1714	ops_complete_compute, sh,
1610	1715	to_addr_conv(sh, percpu, 0));
1611		- return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1612		- STRIPE_SIZE, &submit);
	1716	+ return async_gen_syndrome(blocks, offs, syndrome_disks+2,
	1717	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1718	+ &submit);
1613	1719	} else {
1614	1720	struct page *dest;
	1721	+ unsigned int dest_off;
1615	1722	int data_target;
1616	1723	int qd_idx = sh->qd_idx;
1617	1724
..	..	@@ -1625,22 +1732,26 @@
1625	1732	for (i = disks; i-- ; ) {
1626	1733	if (i == data_target \|\| i == qd_idx)
1627	1734	continue;
	1735	+ offs[count] = sh->dev[i].offset;
1628	1736	blocks[count++] = sh->dev[i].page;
1629	1737	}
1630	1738	dest = sh->dev[data_target].page;
	1739	+ dest_off = sh->dev[data_target].offset;
1631	1740	init_async_submit(&submit,
1632	1741	ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST,
1633	1742	NULL, NULL, NULL,
1634	1743	to_addr_conv(sh, percpu, 0));
1635		- tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
	1744	+ tx = async_xor_offs(dest, dest_off, blocks, offs, count,
	1745	+ RAID5_STRIPE_SIZE(sh->raid_conf),
1636	1746	&submit);
1637	1747
1638		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
	1748	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1639	1749	init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1640	1750	ops_complete_compute, sh,
1641	1751	to_addr_conv(sh, percpu, 0));
1642		- return async_gen_syndrome(blocks, 0, count+2,
1643		- STRIPE_SIZE, &submit);
	1752	+ return async_gen_syndrome(blocks, offs, count+2,
	1753	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1754	+ &submit);
1644	1755	}
1645	1756	} else {
1646	1757	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
..	..	@@ -1649,13 +1760,15 @@
1649	1760	if (failb == syndrome_disks) {
1650	1761	/* We're missing D+P. */
1651	1762	return async_raid6_datap_recov(syndrome_disks+2,
1652		- STRIPE_SIZE, faila,
1653		- blocks, &submit);
	1763	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1764	+ faila,
	1765	+ blocks, offs, &submit);
1654	1766	} else {
1655	1767	/* We're missing D+D. */
1656	1768	return async_raid6_2data_recov(syndrome_disks+2,
1657		- STRIPE_SIZE, faila, failb,
1658		- blocks, &submit);
	1769	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1770	+ faila, failb,
	1771	+ blocks, offs, &submit);
1659	1772	}
1660	1773	}
1661	1774	}
..	..	@@ -1681,10 +1794,12 @@
1681	1794	{
1682	1795	int disks = sh->disks;
1683	1796	struct page **xor_srcs = to_addr_page(percpu, 0);
	1797	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
1684	1798	int count = 0, pd_idx = sh->pd_idx, i;
1685	1799	struct async_submit_ctl submit;
1686	1800
1687	1801	/* existing parity data subtracted */
	1802	+ unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1688	1803	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1689	1804
1690	1805	BUG_ON(sh->batch_head);
..	..	@@ -1694,15 +1809,23 @@
1694	1809	for (i = disks; i--; ) {
1695	1810	struct r5dev *dev = &sh->dev[i];
1696	1811	/* Only process blocks that are known to be uptodate */
1697		- if (test_bit(R5_InJournal, &dev->flags))
	1812	+ if (test_bit(R5_InJournal, &dev->flags)) {
	1813	+ /*
	1814	+ * For this case, PAGE_SIZE must be equal to 4KB and
	1815	+ * page offset is zero.
	1816	+ */
	1817	+ off_srcs[count] = dev->offset;
1698	1818	xor_srcs[count++] = dev->orig_page;
1699		- else if (test_bit(R5_Wantdrain, &dev->flags))
	1819	+ } else if (test_bit(R5_Wantdrain, &dev->flags)) {
	1820	+ off_srcs[count] = dev->offset;
1700	1821	xor_srcs[count++] = dev->page;
	1822	+ }
1701	1823	}
1702	1824
1703	1825	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
1704	1826	ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1705		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	1827	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	1828	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1706	1829
1707	1830	return tx;
1708	1831	}
..	..	@@ -1712,17 +1835,19 @@
1712	1835	struct dma_async_tx_descriptor *tx)
1713	1836	{
1714	1837	struct page **blocks = to_addr_page(percpu, 0);
	1838	+ unsigned int *offs = to_addr_offs(sh, percpu);
1715	1839	int count;
1716	1840	struct async_submit_ctl submit;
1717	1841
1718	1842	pr_debug("%s: stripe %llu\n", __func__,
1719	1843	(unsigned long long)sh->sector);
1720	1844
1721		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
	1845	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
1722	1846
1723	1847	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_PQ_XOR_DST, tx,
1724	1848	ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1725		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	1849	+ tx = async_gen_syndrome(blocks, offs, count+2,
	1850	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1726	1851
1727	1852	return tx;
1728	1853	}
..	..	@@ -1763,7 +1888,7 @@
1763	1888	WARN_ON(dev->page != dev->orig_page);
1764	1889
1765	1890	while (wbi && wbi->bi_iter.bi_sector <
1766		- dev->sector + STRIPE_SECTORS) {
	1891	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
1767	1892	if (wbi->bi_opf & REQ_FUA)
1768	1893	set_bit(R5_WantFUA, &dev->flags);
1769	1894	if (wbi->bi_opf & REQ_SYNC)
..	..	@@ -1772,6 +1897,7 @@
1772	1897	set_bit(R5_Discard, &dev->flags);
1773	1898	else {
1774	1899	tx = async_copy_data(1, wbi, &dev->page,
	1900	+ dev->offset,
1775	1901	dev->sector, tx, sh,
1776	1902	r5c_is_writeback(conf->log));
1777	1903	if (dev->page != dev->orig_page &&
..	..	@@ -1781,7 +1907,7 @@
1781	1907	clear_bit(R5_OVERWRITE, &dev->flags);
1782	1908	}
1783	1909	}
1784		- wbi = r5_next_bio(wbi, dev->sector);
	1910	+ wbi = r5_next_bio(conf, wbi, dev->sector);
1785	1911	}
1786	1912
1787	1913	if (head_sh->batch_head) {
..	..	@@ -1851,9 +1977,11 @@
1851	1977	{
1852	1978	int disks = sh->disks;
1853	1979	struct page **xor_srcs;
	1980	+ unsigned int *off_srcs;
1854	1981	struct async_submit_ctl submit;
1855	1982	int count, pd_idx = sh->pd_idx, i;
1856	1983	struct page *xor_dest;
	1984	+ unsigned int off_dest;
1857	1985	int prexor = 0;
1858	1986	unsigned long flags;
1859	1987	int j = 0;
..	..	@@ -1878,24 +2006,31 @@
1878	2006	again:
1879	2007	count = 0;
1880	2008	xor_srcs = to_addr_page(percpu, j);
	2009	+ off_srcs = to_addr_offs(sh, percpu);
1881	2010	/* check if prexor is active which means only process blocks
1882	2011	* that are part of a read-modify-write (written)
1883	2012	*/
1884	2013	if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1885	2014	prexor = 1;
	2015	+ off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1886	2016	xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1887	2017	for (i = disks; i--; ) {
1888	2018	struct r5dev *dev = &sh->dev[i];
1889	2019	if (head_sh->dev[i].written \|\|
1890		- test_bit(R5_InJournal, &head_sh->dev[i].flags))
	2020	+ test_bit(R5_InJournal, &head_sh->dev[i].flags)) {
	2021	+ off_srcs[count] = dev->offset;
1891	2022	xor_srcs[count++] = dev->page;
	2023	+ }
1892	2024	}
1893	2025	} else {
1894	2026	xor_dest = sh->dev[pd_idx].page;
	2027	+ off_dest = sh->dev[pd_idx].offset;
1895	2028	for (i = disks; i--; ) {
1896	2029	struct r5dev *dev = &sh->dev[i];
1897		- if (i != pd_idx)
	2030	+ if (i != pd_idx) {
	2031	+ off_srcs[count] = dev->offset;
1898	2032	xor_srcs[count++] = dev->page;
	2033	+ }
1899	2034	}
1900	2035	}
1901	2036
..	..	@@ -1921,9 +2056,11 @@
1921	2056	}
1922	2057
1923	2058	if (unlikely(count == 1))
1924		- tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	2059	+ tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
	2060	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1925	2061	else
1926		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	2062	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	2063	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1927	2064	if (!last_stripe) {
1928	2065	j++;
1929	2066	sh = list_first_entry(&sh->batch_list, struct stripe_head,
..	..	@@ -1938,6 +2075,7 @@
1938	2075	{
1939	2076	struct async_submit_ctl submit;
1940	2077	struct page **blocks;
	2078	+ unsigned int *offs;
1941	2079	int count, i, j = 0;
1942	2080	struct stripe_head *head_sh = sh;
1943	2081	int last_stripe;
..	..	@@ -1962,6 +2100,7 @@
1962	2100
1963	2101	again:
1964	2102	blocks = to_addr_page(percpu, j);
	2103	+ offs = to_addr_offs(sh, percpu);
1965	2104
1966	2105	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1967	2106	synflags = SYNDROME_SRC_WRITTEN;
..	..	@@ -1971,7 +2110,7 @@
1971	2110	txflags = ASYNC_TX_ACK;
1972	2111	}
1973	2112
1974		- count = set_syndrome_sources(blocks, sh, synflags);
	2113	+ count = set_syndrome_sources(blocks, offs, sh, synflags);
1975	2114	last_stripe = !head_sh->batch_head \|\|
1976	2115	list_first_entry(&sh->batch_list,
1977	2116	struct stripe_head, batch_list) == head_sh;
..	..	@@ -1983,7 +2122,8 @@
1983	2122	} else
1984	2123	init_async_submit(&submit, 0, tx, NULL, NULL,
1985	2124	to_addr_conv(sh, percpu, j));
1986		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	2125	+ tx = async_gen_syndrome(blocks, offs, count+2,
	2126	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1987	2127	if (!last_stripe) {
1988	2128	j++;
1989	2129	sh = list_first_entry(&sh->batch_list, struct stripe_head,
..	..	@@ -2010,7 +2150,9 @@
2010	2150	int pd_idx = sh->pd_idx;
2011	2151	int qd_idx = sh->qd_idx;
2012	2152	struct page *xor_dest;
	2153	+ unsigned int off_dest;
2013	2154	struct page **xor_srcs = to_addr_page(percpu, 0);
	2155	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
2014	2156	struct dma_async_tx_descriptor *tx;
2015	2157	struct async_submit_ctl submit;
2016	2158	int count;
..	..	@@ -2022,16 +2164,20 @@
2022	2164	BUG_ON(sh->batch_head);
2023	2165	count = 0;
2024	2166	xor_dest = sh->dev[pd_idx].page;
	2167	+ off_dest = sh->dev[pd_idx].offset;
	2168	+ off_srcs[count] = off_dest;
2025	2169	xor_srcs[count++] = xor_dest;
2026	2170	for (i = disks; i--; ) {
2027	2171	if (i == pd_idx \|\| i == qd_idx)
2028	2172	continue;
	2173	+ off_srcs[count] = sh->dev[i].offset;
2029	2174	xor_srcs[count++] = sh->dev[i].page;
2030	2175	}
2031	2176
2032	2177	init_async_submit(&submit, 0, NULL, NULL, NULL,
2033	2178	to_addr_conv(sh, percpu, 0));
2034		- tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
	2179	+ tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	2180	+ RAID5_STRIPE_SIZE(sh->raid_conf),
2035	2181	&sh->ops.zero_sum_result, &submit);
2036	2182
2037	2183	atomic_inc(&sh->count);
..	..	@@ -2042,6 +2188,7 @@
2042	2188	static void ops_run_check_pq(struct stripe_head sh, struct raid5_percpu percpu, int checkp)
2043	2189	{
2044	2190	struct page **srcs = to_addr_page(percpu, 0);
	2191	+ unsigned int *offs = to_addr_offs(sh, percpu);
2045	2192	struct async_submit_ctl submit;
2046	2193	int count;
2047	2194
..	..	@@ -2049,15 +2196,16 @@
2049	2196	(unsigned long long)sh->sector, checkp);
2050	2197
2051	2198	BUG_ON(sh->batch_head);
2052		- count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
	2199	+ count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
2053	2200	if (!checkp)
2054	2201	srcs[count] = NULL;
2055	2202
2056	2203	atomic_inc(&sh->count);
2057	2204	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
2058	2205	sh, to_addr_conv(sh, percpu, 0));
2059		- async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
2060		- &sh->ops.zero_sum_result, percpu->spare_page, &submit);
	2206	+ async_syndrome_val(srcs, offs, count+2,
	2207	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	2208	+ &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
2061	2209	}
2062	2210
2063	2211	static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
..	..	@@ -2136,6 +2284,9 @@
2136	2284
2137	2285	static void free_stripe(struct kmem_cache sc, struct stripe_head sh)
2138	2286	{
	2287	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2288	+ kfree(sh->pages);
	2289	+#endif
2139	2290	if (sh->ppl_page)
2140	2291	__free_page(sh->ppl_page);
2141	2292	kmem_cache_free(sc, sh);
..	..	@@ -2169,9 +2320,15 @@
2169	2320	sh->ppl_page = alloc_page(gfp);
2170	2321	if (!sh->ppl_page) {
2171	2322	free_stripe(sc, sh);
2172		- sh = NULL;
	2323	+ return NULL;
2173	2324	}
2174	2325	}
	2326	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2327	+ if (init_stripe_shared_pages(sh, conf, disks)) {
	2328	+ free_stripe(sc, sh);
	2329	+ return NULL;
	2330	+ }
	2331	+#endif
2175	2332	}
2176	2333	return sh;
2177	2334	}
..	..	@@ -2228,10 +2385,13 @@
2228	2385	}
2229	2386
2230	2387	/**
2231		- * scribble_len - return the required size of the scribble region
2232		- * @num - total number of disks in the array
	2388	+ * scribble_alloc - allocate percpu scribble buffer for required size
	2389	+ * of the scribble region
	2390	+ * @percpu: from for_each_present_cpu() of the caller
	2391	+ * @num: total number of disks in the array
	2392	+ * @cnt: scribble objs count for required size of the scribble region
2233	2393	*
2234		- * The size must be enough to contain:
	2394	+ * The scribble buffer size must be enough to contain:
2235	2395	* 1/ a struct page pointer for each device in the array +2
2236	2396	* 2/ room to convert each entry in (1) to its corresponding dma
2237	2397	* (dma_map_page()) or page (page_address()) address.
..	..	@@ -2240,21 +2400,29 @@
2240	2400	* calculate over all devices (not just the data blocks), using zeros in place
2241	2401	* of the P and Q blocks.
2242	2402	*/
2243		-static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
	2403	+static int scribble_alloc(struct raid5_percpu *percpu,
	2404	+ int num, int cnt)
2244	2405	{
2245		- struct flex_array *ret;
2246		- size_t len;
	2406	+ size_t obj_size =
	2407	+ sizeof(struct page ) (num + 2) +
	2408	+ sizeof(addr_conv_t) * (num + 2) +
	2409	+ sizeof(unsigned int) * (num + 2);
	2410	+ void *scribble;
2247	2411
2248		- len = sizeof(struct page ) (num+2) + sizeof(addr_conv_t) * (num+2);
2249		- ret = flex_array_alloc(len, cnt, flags);
2250		- if (!ret)
2251		- return NULL;
2252		- /* always prealloc all elements, so no locking is required */
2253		- if (flex_array_prealloc(ret, 0, cnt, flags)) {
2254		- flex_array_free(ret);
2255		- return NULL;
2256		- }
2257		- return ret;
	2412	+ /*
	2413	+ * If here is in raid array suspend context, it is in memalloc noio
	2414	+ * context as well, there is no potential recursive memory reclaim
	2415	+ * I/Os with the GFP_KERNEL flag.
	2416	+ */
	2417	+ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
	2418	+ if (!scribble)
	2419	+ return -ENOMEM;
	2420	+
	2421	+ kvfree(percpu->scribble);
	2422	+
	2423	+ percpu->scribble = scribble;
	2424	+ percpu->scribble_obj_size = obj_size;
	2425	+ return 0;
2258	2426	}
2259	2427
2260	2428	static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
..	..	@@ -2272,23 +2440,17 @@
2272	2440	return 0;
2273	2441	mddev_suspend(conf->mddev);
2274	2442	get_online_cpus();
	2443	+
2275	2444	for_each_present_cpu(cpu) {
2276	2445	struct raid5_percpu *percpu;
2277		- struct flex_array *scribble;
2278	2446
2279	2447	percpu = per_cpu_ptr(conf->percpu, cpu);
2280		- scribble = scribble_alloc(new_disks,
2281		- new_sectors / STRIPE_SECTORS,
2282		- GFP_NOIO);
2283		-
2284		- if (scribble) {
2285		- flex_array_free(percpu->scribble);
2286		- percpu->scribble = scribble;
2287		- } else {
2288		- err = -ENOMEM;
	2448	+ err = scribble_alloc(percpu, new_disks,
	2449	+ new_sectors / RAID5_STRIPE_SECTORS(conf));
	2450	+ if (err)
2289	2451	break;
2290		- }
2291	2452	}
	2453	+
2292	2454	put_online_cpus();
2293	2455	mddev_resume(conf->mddev);
2294	2456	if (!err) {
..	..	@@ -2376,9 +2538,16 @@
2376	2538	osh = get_free_stripe(conf, hash);
2377	2539	unlock_device_hash_lock(conf, hash);
2378	2540
	2541	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2542	+ for (i = 0; i < osh->nr_pages; i++) {
	2543	+ nsh->pages[i] = osh->pages[i];
	2544	+ osh->pages[i] = NULL;
	2545	+ }
	2546	+#endif
2379	2547	for(i=0; i<conf->pool_size; i++) {
2380	2548	nsh->dev[i].page = osh->dev[i].page;
2381	2549	nsh->dev[i].orig_page = osh->dev[i].page;
	2550	+ nsh->dev[i].offset = osh->dev[i].offset;
2382	2551	}
2383	2552	nsh->hash_lock_index = hash;
2384	2553	free_stripe(conf->slab_cache, osh);
..	..	@@ -2427,14 +2596,33 @@
2427	2596	nsh = list_entry(newstripes.next, struct stripe_head, lru);
2428	2597	list_del_init(&nsh->lru);
2429	2598
	2599	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2600	+ for (i = 0; i < nsh->nr_pages; i++) {
	2601	+ if (nsh->pages[i])
	2602	+ continue;
	2603	+ nsh->pages[i] = alloc_page(GFP_NOIO);
	2604	+ if (!nsh->pages[i])
	2605	+ err = -ENOMEM;
	2606	+ }
	2607	+
	2608	+ for (i = conf->raid_disks; i < newsize; i++) {
	2609	+ if (nsh->dev[i].page)
	2610	+ continue;
	2611	+ nsh->dev[i].page = raid5_get_dev_page(nsh, i);
	2612	+ nsh->dev[i].orig_page = nsh->dev[i].page;
	2613	+ nsh->dev[i].offset = raid5_get_page_offset(nsh, i);
	2614	+ }
	2615	+#else
2430	2616	for (i=conf->raid_disks; i < newsize; i++)
2431	2617	if (nsh->dev[i].page == NULL) {
2432	2618	struct page *p = alloc_page(GFP_NOIO);
2433	2619	nsh->dev[i].page = p;
2434	2620	nsh->dev[i].orig_page = p;
	2621	+ nsh->dev[i].offset = 0;
2435	2622	if (!p)
2436	2623	err = -ENOMEM;
2437	2624	}
	2625	+#endif
2438	2626	raid5_release_stripe(nsh);
2439	2627	}
2440	2628	/* critical section pass, GFP_NOIO no longer needed */
..	..	@@ -2518,10 +2706,10 @@
2518	2706	*/
2519	2707	pr_info_ratelimited(
2520	2708	"md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
2521		- mdname(conf->mddev), STRIPE_SECTORS,
	2709	+ mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),
2522	2710	(unsigned long long)s,
2523	2711	bdevname(rdev->bdev, b));
2524		- atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
	2712	+ atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);
2525	2713	clear_bit(R5_ReadError, &sh->dev[i].flags);
2526	2714	clear_bit(R5_ReWrite, &sh->dev[i].flags);
2527	2715	} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
..	..	@@ -2566,10 +2754,16 @@
2566	2754	(unsigned long long)s,
2567	2755	bdn);
2568	2756	} else if (atomic_read(&rdev->read_errors)
2569		- > conf->max_nr_stripes)
2570		- pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
2571		- mdname(conf->mddev), bdn);
2572		- else
	2757	+ > conf->max_nr_stripes) {
	2758	+ if (!test_bit(Faulty, &rdev->flags)) {
	2759	+ pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
	2760	+ mdname(conf->mddev),
	2761	+ atomic_read(&rdev->read_errors),
	2762	+ conf->max_nr_stripes);
	2763	+ pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
	2764	+ mdname(conf->mddev), bdn);
	2765	+ }
	2766	+ } else
2573	2767	retry = 1;
2574	2768	if (set_bad && test_bit(In_sync, &rdev->flags)
2575	2769	&& !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
..	..	@@ -2588,7 +2782,7 @@
2588	2782	if (!(set_bad
2589	2783	&& test_bit(In_sync, &rdev->flags)
2590	2784	&& rdev_set_badblocks(
2591		- rdev, sh->sector, STRIPE_SECTORS, 0)))
	2785	+ rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
2592	2786	md_error(conf->mddev, rdev);
2593	2787	}
2594	2788	}
..	..	@@ -2604,7 +2798,7 @@
2604	2798	struct stripe_head *sh = bi->bi_private;
2605	2799	struct r5conf *conf = sh->raid_conf;
2606	2800	int disks = sh->disks, i;
2607		- struct md_rdev *uninitialized_var(rdev);
	2801	+ struct md_rdev *rdev;
2608	2802	sector_t first_bad;
2609	2803	int bad_sectors;
2610	2804	int replacement = 0;
..	..	@@ -2640,7 +2834,7 @@
2640	2834	if (bi->bi_status)
2641	2835	md_error(conf->mddev, rdev);
2642	2836	else if (is_badblock(rdev, sh->sector,
2643		- STRIPE_SECTORS,
	2837	+ RAID5_STRIPE_SECTORS(conf),
2644	2838	&first_bad, &bad_sectors))
2645	2839	set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2646	2840	} else {
..	..	@@ -2652,7 +2846,7 @@
2652	2846	set_bit(MD_RECOVERY_NEEDED,
2653	2847	&rdev->mddev->recovery);
2654	2848	} else if (is_badblock(rdev, sh->sector,
2655		- STRIPE_SECTORS,
	2849	+ RAID5_STRIPE_SECTORS(conf),
2656	2850	&first_bad, &bad_sectors)) {
2657	2851	set_bit(R5_MadeGood, &sh->dev[i].flags);
2658	2852	if (test_bit(R5_ReadError, &sh->dev[i].flags))
..	..	@@ -2672,10 +2866,10 @@
2672	2866	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2673	2867	clear_bit(R5_LOCKED, &sh->dev[i].flags);
2674	2868	set_bit(STRIPE_HANDLE, &sh->state);
2675		- raid5_release_stripe(sh);
2676	2869
2677	2870	if (sh->batch_head && sh != sh->batch_head)
2678	2871	raid5_release_stripe(sh->batch_head);
	2872	+ raid5_release_stripe(sh);
2679	2873	}
2680	2874
2681	2875	static void raid5_error(struct mddev mddev, struct md_rdev rdev)
..	..	@@ -2685,22 +2879,31 @@
2685	2879	unsigned long flags;
2686	2880	pr_debug("raid456: error called\n");
2687	2881
	2882	+ pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n",
	2883	+ mdname(mddev), bdevname(rdev->bdev, b));
	2884	+
2688	2885	spin_lock_irqsave(&conf->device_lock, flags);
2689	2886	set_bit(Faulty, &rdev->flags);
2690	2887	clear_bit(In_sync, &rdev->flags);
2691	2888	mddev->degraded = raid5_calc_degraded(conf);
	2889	+
	2890	+ if (has_failed(conf)) {
	2891	+ set_bit(MD_BROKEN, &conf->mddev->flags);
	2892	+ conf->recovery_disabled = mddev->recovery_disabled;
	2893	+
	2894	+ pr_crit("md/raid:%s: Cannot continue operation (%d/%d failed).\n",
	2895	+ mdname(mddev), mddev->degraded, conf->raid_disks);
	2896	+ } else {
	2897	+ pr_crit("md/raid:%s: Operation continuing on %d devices.\n",
	2898	+ mdname(mddev), conf->raid_disks - mddev->degraded);
	2899	+ }
	2900	+
2692	2901	spin_unlock_irqrestore(&conf->device_lock, flags);
2693	2902	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2694	2903
2695	2904	set_bit(Blocked, &rdev->flags);
2696	2905	set_mask_bits(&mddev->sb_flags, 0,
2697	2906	BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
2698		- pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
2699		- "md/raid:%s: Operation continuing on %d devices.\n",
2700		- mdname(mddev),
2701		- bdevname(rdev->bdev, b),
2702		- mdname(mddev),
2703		- conf->raid_disks - mddev->degraded);
2704	2907	r5c_update_on_rdev_error(mddev, rdev);
2705	2908	}
2706	2909
..	..	@@ -3274,13 +3477,13 @@
3274	3477	/* check if page is covered */
3275	3478	sector_t sector = sh->dev[dd_idx].sector;
3276	3479	for (bi=sh->dev[dd_idx].towrite;
3277		- sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
	3480	+ sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
3278	3481	bi && bi->bi_iter.bi_sector <= sector;
3279		- bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
	3482	+ bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
3280	3483	if (bio_end_sector(bi) >= sector)
3281	3484	sector = bio_end_sector(bi);
3282	3485	}
3283		- if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
	3486	+ if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
3284	3487	if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
3285	3488	sh->overwrite_disks++;
3286	3489	}
..	..	@@ -3305,7 +3508,7 @@
3305	3508	set_bit(STRIPE_BITMAP_PENDING, &sh->state);
3306	3509	spin_unlock_irq(&sh->stripe_lock);
3307	3510	md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
3308		- STRIPE_SECTORS, 0);
	3511	+ RAID5_STRIPE_SECTORS(conf), 0);
3309	3512	spin_lock_irq(&sh->stripe_lock);
3310	3513	clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
3311	3514	if (!sh->batch_head) {
..	..	@@ -3367,7 +3570,7 @@
3367	3570	if (!rdev_set_badblocks(
3368	3571	rdev,
3369	3572	sh->sector,
3370		- STRIPE_SECTORS, 0))
	3573	+ RAID5_STRIPE_SECTORS(conf), 0))
3371	3574	md_error(conf->mddev, rdev);
3372	3575	rdev_dec_pending(rdev, conf->mddev);
3373	3576	}
..	..	@@ -3387,8 +3590,8 @@
3387	3590	wake_up(&conf->wait_for_overlap);
3388	3591
3389	3592	while (bi && bi->bi_iter.bi_sector <
3390		- sh->dev[i].sector + STRIPE_SECTORS) {
3391		- struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
	3593	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
	3594	+ struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
3392	3595
3393	3596	md_write_end(conf->mddev);
3394	3597	bio_io_error(bi);
..	..	@@ -3396,7 +3599,7 @@
3396	3599	}
3397	3600	if (bitmap_end)
3398	3601	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3399		- STRIPE_SECTORS, 0, 0);
	3602	+ RAID5_STRIPE_SECTORS(conf), 0, 0);
3400	3603	bitmap_end = 0;
3401	3604	/* and fail all 'written' */
3402	3605	bi = sh->dev[i].written;
..	..	@@ -3408,8 +3611,8 @@
3408	3611
3409	3612	if (bi) bitmap_end = 1;
3410	3613	while (bi && bi->bi_iter.bi_sector <
3411		- sh->dev[i].sector + STRIPE_SECTORS) {
3412		- struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
	3614	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
	3615	+ struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
3413	3616
3414	3617	md_write_end(conf->mddev);
3415	3618	bio_io_error(bi);
..	..	@@ -3432,9 +3635,9 @@
3432	3635	if (bi)
3433	3636	s->to_read--;
3434	3637	while (bi && bi->bi_iter.bi_sector <
3435		- sh->dev[i].sector + STRIPE_SECTORS) {
	3638	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3436	3639	struct bio *nextbi =
3437		- r5_next_bio(bi, sh->dev[i].sector);
	3640	+ r5_next_bio(conf, bi, sh->dev[i].sector);
3438	3641
3439	3642	bio_io_error(bi);
3440	3643	bi = nextbi;
..	..	@@ -3442,7 +3645,7 @@
3442	3645	}
3443	3646	if (bitmap_end)
3444	3647	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3445		- STRIPE_SECTORS, 0, 0);
	3648	+ RAID5_STRIPE_SECTORS(conf), 0, 0);
3446	3649	/* If we were in the middle of a write the parity block might
3447	3650	* still be locked - so just clear all R5_LOCKED flags
3448	3651	*/
..	..	@@ -3487,14 +3690,14 @@
3487	3690	&& !test_bit(Faulty, &rdev->flags)
3488	3691	&& !test_bit(In_sync, &rdev->flags)
3489	3692	&& !rdev_set_badblocks(rdev, sh->sector,
3490		- STRIPE_SECTORS, 0))
	3693	+ RAID5_STRIPE_SECTORS(conf), 0))
3491	3694	abort = 1;
3492	3695	rdev = rcu_dereference(conf->disks[i].replacement);
3493	3696	if (rdev
3494	3697	&& !test_bit(Faulty, &rdev->flags)
3495	3698	&& !test_bit(In_sync, &rdev->flags)
3496	3699	&& !rdev_set_badblocks(rdev, sh->sector,
3497		- STRIPE_SECTORS, 0))
	3700	+ RAID5_STRIPE_SECTORS(conf), 0))
3498	3701	abort = 1;
3499	3702	}
3500	3703	rcu_read_unlock();
..	..	@@ -3502,7 +3705,7 @@
3502	3705	conf->recovery_disabled =
3503	3706	conf->mddev->recovery_disabled;
3504	3707	}
3505		- md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
	3708	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);
3506	3709	}
3507	3710
3508	3711	static int want_replace(struct stripe_head *sh, int disk_idx)
..	..	@@ -3529,6 +3732,7 @@
3529	3732	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3530	3733	&sh->dev[s->failed_num[1]] };
3531	3734	int i;
	3735	+ bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
3532	3736
3533	3737
3534	3738	if (test_bit(R5_LOCKED, &dev->flags) \|\|
..	..	@@ -3587,18 +3791,27 @@
3587	3791	* devices must be read.
3588	3792	*/
3589	3793	return 1;
	3794	+
	3795	+ if (s->failed >= 2 &&
	3796	+ (fdev[i]->towrite \|\|
	3797	+ s->failed_num[i] == sh->pd_idx \|\|
	3798	+ s->failed_num[i] == sh->qd_idx) &&
	3799	+ !test_bit(R5_UPTODATE, &fdev[i]->flags))
	3800	+ /* In max degraded raid6, If the failed disk is P, Q,
	3801	+ * or we want to read the failed disk, we need to do
	3802	+ * reconstruct-write.
	3803	+ */
	3804	+ force_rcw = true;
3590	3805	}
3591	3806
3592		- /* If we are forced to do a reconstruct-write, either because
3593		- * the current RAID6 implementation only supports that, or
3594		- * because parity cannot be trusted and we are currently
3595		- * recovering it, there is extra need to be careful.
	3807	+ /* If we are forced to do a reconstruct-write, because parity
	3808	+ * cannot be trusted and we are currently recovering it, there
	3809	+ * is extra need to be careful.
3596	3810	* If one of the devices that we would need to read, because
3597	3811	* it is not being overwritten (and maybe not written at all)
3598	3812	* is missing/faulty, then we need to read everything we can.
3599	3813	*/
3600		- if (sh->raid_conf->level != 6 &&
3601		- sh->raid_conf->rmw_level != PARITY_DISABLE_RMW &&
	3814	+ if (!force_rcw &&
3602	3815	sh->sector < sh->raid_conf->mddev->recovery_cp)
3603	3816	/* reconstruct-write isn't being forced */
3604	3817	return 0;
..	..	@@ -3702,7 +3915,7 @@
3702	3915	return 0;
3703	3916	}
3704	3917
3705		-/**
	3918	+/*
3706	3919	* handle_stripe_fill - read or compute data to satisfy pending requests.
3707	3920	*/
3708	3921	static void handle_stripe_fill(struct stripe_head *sh,
..	..	@@ -3725,7 +3938,7 @@
3725	3938	* back cache (prexor with orig_page, and then xor with
3726	3939	* page) in the read path
3727	3940	*/
3728		- if (s->injournal && s->failed) {
	3941	+ if (s->to_read && s->injournal && s->failed) {
3729	3942	if (test_bit(STRIPE_R5C_CACHING, &sh->state))
3730	3943	r5c_make_stripe_write_out(sh);
3731	3944	goto out;
..	..	@@ -3777,14 +3990,14 @@
3777	3990	wbi = dev->written;
3778	3991	dev->written = NULL;
3779	3992	while (wbi && wbi->bi_iter.bi_sector <
3780		- dev->sector + STRIPE_SECTORS) {
3781		- wbi2 = r5_next_bio(wbi, dev->sector);
	3993	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
	3994	+ wbi2 = r5_next_bio(conf, wbi, dev->sector);
3782	3995	md_write_end(conf->mddev);
3783	3996	bio_endio(wbi);
3784	3997	wbi = wbi2;
3785	3998	}
3786	3999	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3787		- STRIPE_SECTORS,
	4000	+ RAID5_STRIPE_SECTORS(conf),
3788	4001	!test_bit(STRIPE_DEGRADED, &sh->state),
3789	4002	0);
3790	4003	if (head_sh->batch_head) {
..	..	@@ -3968,10 +4181,8 @@
3968	4181	set_bit(R5_LOCKED, &dev->flags);
3969	4182	set_bit(R5_Wantread, &dev->flags);
3970	4183	s->locked++;
3971		- } else {
	4184	+ } else
3972	4185	set_bit(STRIPE_DELAYED, &sh->state);
3973		- set_bit(STRIPE_HANDLE, &sh->state);
3974		- }
3975	4186	}
3976	4187	}
3977	4188	}
..	..	@@ -3996,10 +4207,8 @@
3996	4207	set_bit(R5_Wantread, &dev->flags);
3997	4208	s->locked++;
3998	4209	qread++;
3999		- } else {
	4210	+ } else
4000	4211	set_bit(STRIPE_DELAYED, &sh->state);
4001		- set_bit(STRIPE_HANDLE, &sh->state);
4002		- }
4003	4212	}
4004	4213	}
4005	4214	if (rcw && conf->mddev->queue)
..	..	@@ -4049,7 +4258,7 @@
4049	4258	break;
4050	4259	}
4051	4260	dev = &sh->dev[s->failed_num[0]];
4052		- /* fall through */
	4261	+ fallthrough;
4053	4262	case check_state_compute_result:
4054	4263	sh->check_state = check_state_idle;
4055	4264	if (!dev)
..	..	@@ -4091,7 +4300,7 @@
4091	4300	*/
4092	4301	set_bit(STRIPE_INSYNC, &sh->state);
4093	4302	else {
4094		- atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
	4303	+ atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
4095	4304	if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
4096	4305	/* don't try to repair!! */
4097	4306	set_bit(STRIPE_INSYNC, &sh->state);
..	..	@@ -4099,7 +4308,7 @@
4099	4308	"%llu-%llu\n", mdname(conf->mddev),
4100	4309	(unsigned long long) sh->sector,
4101	4310	(unsigned long long) sh->sector +
4102		- STRIPE_SECTORS);
	4311	+ RAID5_STRIPE_SECTORS(conf));
4103	4312	} else {
4104	4313	sh->check_state = check_state_compute_run;
4105	4314	set_bit(STRIPE_COMPUTE_RUN, &sh->state);
..	..	@@ -4180,7 +4389,7 @@
4180	4389
4181	4390	/* we have 2-disk failure */
4182	4391	BUG_ON(s->failed != 2);
4183		- /* fall through */
	4392	+ fallthrough;
4184	4393	case check_state_compute_result:
4185	4394	sh->check_state = check_state_idle;
4186	4395
..	..	@@ -4256,7 +4465,7 @@
4256	4465	*/
4257	4466	}
4258	4467	} else {
4259		- atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
	4468	+ atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
4260	4469	if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
4261	4470	/* don't try to repair!! */
4262	4471	set_bit(STRIPE_INSYNC, &sh->state);
..	..	@@ -4264,7 +4473,7 @@
4264	4473	"%llu-%llu\n", mdname(conf->mddev),
4265	4474	(unsigned long long) sh->sector,
4266	4475	(unsigned long long) sh->sector +
4267		- STRIPE_SECTORS);
	4476	+ RAID5_STRIPE_SECTORS(conf));
4268	4477	} else {
4269	4478	int *target = &sh->ops.target;
4270	4479
..	..	@@ -4335,7 +4544,8 @@
4335	4544	/* place all the copies on one channel */
4336	4545	init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
4337	4546	tx = async_memcpy(sh2->dev[dd_idx].page,
4338		- sh->dev[i].page, 0, 0, STRIPE_SIZE,
	4547	+ sh->dev[i].page, sh2->dev[dd_idx].offset,
	4548	+ sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
4339	4549	&submit);
4340	4550
4341	4551	set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
..	..	@@ -4434,8 +4644,8 @@
4434	4644	*/
4435	4645	rdev = rcu_dereference(conf->disks[i].replacement);
4436	4646	if (rdev && !test_bit(Faulty, &rdev->flags) &&
4437		- rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
4438		- !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	4647	+ rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
	4648	+ !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4439	4649	&first_bad, &bad_sectors))
4440	4650	set_bit(R5_ReadRepl, &dev->flags);
4441	4651	else {
..	..	@@ -4449,7 +4659,7 @@
4449	4659	if (rdev && test_bit(Faulty, &rdev->flags))
4450	4660	rdev = NULL;
4451	4661	if (rdev) {
4452		- is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	4662	+ is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4453	4663	&first_bad, &bad_sectors);
4454	4664	if (s->blocked_rdev == NULL
4455	4665	&& (test_bit(Blocked, &rdev->flags)
..	..	@@ -4476,7 +4686,7 @@
4476	4686	}
4477	4687	} else if (test_bit(In_sync, &rdev->flags))
4478	4688	set_bit(R5_Insync, &dev->flags);
4479		- else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
	4689	+ else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
4480	4690	/* in sync if before recovery_offset */
4481	4691	set_bit(R5_Insync, &dev->flags);
4482	4692	else if (test_bit(R5_UPTODATE, &dev->flags) &&
..	..	@@ -4565,12 +4775,12 @@
4565	4775	rcu_read_unlock();
4566	4776	}
4567	4777
	4778	+/*
	4779	+ * Return '1' if this is a member of batch, or '0' if it is a lone stripe or
	4780	+ * a head which can now be handled.
	4781	+ */
4568	4782	static int clear_batch_ready(struct stripe_head *sh)
4569	4783	{
4570		- /* Return '1' if this is a member of batch, or
4571		- * '0' if it is a lone stripe or a head which can now be
4572		- * handled.
4573		- */
4574	4784	struct stripe_head *tmp;
4575	4785	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
4576	4786	return (sh->batch_head && sh->batch_head != sh);
..	..	@@ -4620,7 +4830,6 @@
4620	4830	(1 << STRIPE_FULL_WRITE) \|
4621	4831	(1 << STRIPE_BIOFILL_RUN) \|
4622	4832	(1 << STRIPE_COMPUTE_RUN) \|
4623		- (1 << STRIPE_OPS_REQ_PENDING) \|
4624	4833	(1 << STRIPE_DISCARD) \|
4625	4834	(1 << STRIPE_BATCH_READY) \|
4626	4835	(1 << STRIPE_BATCH_ERR) \|
..	..	@@ -4675,15 +4884,20 @@
4675	4884	struct r5dev pdev, qdev;
4676	4885
4677	4886	clear_bit(STRIPE_HANDLE, &sh->state);
	4887	+
	4888	+ /*
	4889	+ * handle_stripe should not continue handle the batched stripe, only
	4890	+ * the head of batch list or lone stripe can continue. Otherwise we
	4891	+ * could see break_stripe_batch_list warns about the STRIPE_ACTIVE
	4892	+ * is set for the batched stripe.
	4893	+ */
	4894	+ if (clear_batch_ready(sh))
	4895	+ return;
	4896	+
4678	4897	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4679	4898	/* already being handled, ensure it gets handled
4680	4899	* again when current action finishes */
4681	4900	set_bit(STRIPE_HANDLE, &sh->state);
4682		- return;
4683		- }
4684		-
4685		- if (clear_batch_ready(sh) ) {
4686		- clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4687	4901	return;
4688	4902	}
4689	4903
..	..	@@ -4920,7 +5134,7 @@
4920	5134	if ((s.syncing \|\| s.replacing) && s.locked == 0 &&
4921	5135	!test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4922	5136	test_bit(STRIPE_INSYNC, &sh->state)) {
4923		- md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	5137	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
4924	5138	clear_bit(STRIPE_SYNCING, &sh->state);
4925	5139	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
4926	5140	wake_up(&conf->wait_for_overlap);
..	..	@@ -4939,14 +5153,11 @@
4939	5153	if (!test_bit(R5_ReWrite, &dev->flags)) {
4940	5154	set_bit(R5_Wantwrite, &dev->flags);
4941	5155	set_bit(R5_ReWrite, &dev->flags);
4942		- set_bit(R5_LOCKED, &dev->flags);
4943		- s.locked++;
4944		- } else {
	5156	+ } else
4945	5157	/* let's read it back */
4946	5158	set_bit(R5_Wantread, &dev->flags);
4947		- set_bit(R5_LOCKED, &dev->flags);
4948		- s.locked++;
4949		- }
	5159	+ set_bit(R5_LOCKED, &dev->flags);
	5160	+ s.locked++;
4950	5161	}
4951	5162	}
4952	5163
..	..	@@ -4988,7 +5199,7 @@
4988	5199	clear_bit(STRIPE_EXPAND_READY, &sh->state);
4989	5200	atomic_dec(&conf->reshape_stripes);
4990	5201	wake_up(&conf->wait_for_overlap);
4991		- md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	5202	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
4992	5203	}
4993	5204
4994	5205	if (s.expanding && s.locked == 0 &&
..	..	@@ -5018,14 +5229,14 @@
5018	5229	/* We own a safe reference to the rdev */
5019	5230	rdev = conf->disks[i].rdev;
5020	5231	if (!rdev_set_badblocks(rdev, sh->sector,
5021		- STRIPE_SECTORS, 0))
	5232	+ RAID5_STRIPE_SECTORS(conf), 0))
5022	5233	md_error(conf->mddev, rdev);
5023	5234	rdev_dec_pending(rdev, conf->mddev);
5024	5235	}
5025	5236	if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
5026	5237	rdev = conf->disks[i].rdev;
5027	5238	rdev_clear_badblocks(rdev, sh->sector,
5028		- STRIPE_SECTORS, 0);
	5239	+ RAID5_STRIPE_SECTORS(conf), 0);
5029	5240	rdev_dec_pending(rdev, conf->mddev);
5030	5241	}
5031	5242	if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
..	..	@@ -5034,7 +5245,7 @@
5034	5245	/* rdev have been moved down */
5035	5246	rdev = conf->disks[i].rdev;
5036	5247	rdev_clear_badblocks(rdev, sh->sector,
5037		- STRIPE_SECTORS, 0);
	5248	+ RAID5_STRIPE_SECTORS(conf), 0);
5038	5249	rdev_dec_pending(rdev, conf->mddev);
5039	5250	}
5040	5251	}
..	..	@@ -5090,28 +5301,6 @@
5090	5301	hash = sh->hash_lock_index;
5091	5302	__release_stripe(conf, sh, &temp_inactive_list[hash]);
5092	5303	}
5093		-}
5094		-
5095		-static int raid5_congested(struct mddev *mddev, int bits)
5096		-{
5097		- struct r5conf *conf = mddev->private;
5098		-
5099		- /* No difference between reads and writes. Just check
5100		- * how busy the stripe_cache is
5101		- */
5102		-
5103		- if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
5104		- return 1;
5105		-
5106		- /* Also checks whether there is pressure on r5cache log space */
5107		- if (test_bit(R5C_LOG_TIGHT, &conf->cache_state))
5108		- return 1;
5109		- if (conf->quiesce)
5110		- return 1;
5111		- if (atomic_read(&conf->empty_inactive_list_nr))
5112		- return 1;
5113		-
5114		- return 0;
5115	5304	}
5116	5305
5117	5306	static int in_chunk_boundary(struct mddev mddev, struct bio bio)
..	..	@@ -5259,7 +5448,6 @@
5259	5448	rcu_read_unlock();
5260	5449	raid_bio->bi_next = (void*)rdev;
5261	5450	bio_set_dev(align_bi, rdev->bdev);
5262		- bio_clear_flag(align_bi, BIO_SEG_VALID);
5263	5451
5264	5452	if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
5265	5453	bio_sectors(align_bi),
..	..	@@ -5283,7 +5471,7 @@
5283	5471	trace_block_bio_remap(align_bi->bi_disk->queue,
5284	5472	align_bi, disk_devt(mddev->gendisk),
5285	5473	raid_bio->bi_iter.bi_sector);
5286		- generic_make_request(align_bi);
	5474	+ submit_bio_noacct(align_bi);
5287	5475	return 1;
5288	5476	} else {
5289	5477	rcu_read_unlock();
..	..	@@ -5303,7 +5491,7 @@
5303	5491	struct r5conf *conf = mddev->private;
5304	5492	split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
5305	5493	bio_chain(split, raid_bio);
5306		- generic_make_request(raid_bio);
	5494	+ submit_bio_noacct(raid_bio);
5307	5495	raid_bio = split;
5308	5496	}
5309	5497
..	..	@@ -5499,8 +5687,8 @@
5499	5687	/* Skip discard while reshape is happening */
5500	5688	return;
5501	5689
5502		- logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
5503		- last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
	5690	+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
	5691	+ last_sector = bio_end_sector(bi);
5504	5692
5505	5693	bi->bi_next = NULL;
5506	5694
..	..	@@ -5514,7 +5702,7 @@
5514	5702	last_sector *= conf->chunk_sectors;
5515	5703
5516	5704	for (; logical_sector < last_sector;
5517		- logical_sector += STRIPE_SECTORS) {
	5705	+ logical_sector += RAID5_STRIPE_SECTORS(conf)) {
5518	5706	DEFINE_WAIT(w);
5519	5707	int d;
5520	5708	again:
..	..	@@ -5559,7 +5747,7 @@
5559	5747	d++)
5560	5748	md_bitmap_startwrite(mddev->bitmap,
5561	5749	sh->sector,
5562		- STRIPE_SECTORS,
	5750	+ RAID5_STRIPE_SECTORS(conf),
5563	5751	0);
5564	5752	sh->bm_seq = conf->seq_flush + 1;
5565	5753	set_bit(STRIPE_BIT_DELAY, &sh->state);
..	..	@@ -5624,12 +5812,12 @@
5624	5812	return true;
5625	5813	}
5626	5814
5627		- logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	5815	+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
5628	5816	last_sector = bio_end_sector(bi);
5629	5817	bi->bi_next = NULL;
5630	5818
5631	5819	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
5632		- for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
	5820	+ for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
5633	5821	int previous;
5634	5822	int seq;
5635	5823
..	..	@@ -5727,8 +5915,7 @@
5727	5915	do_flush = false;
5728	5916	}
5729	5917
5730		- if (!sh->batch_head \|\| sh == sh->batch_head)
5731		- set_bit(STRIPE_HANDLE, &sh->state);
	5918	+ set_bit(STRIPE_HANDLE, &sh->state);
5732	5919	clear_bit(STRIPE_DELAYED, &sh->state);
5733	5920	if ((!sh->batch_head \|\| sh == sh->batch_head) &&
5734	5921	(bi->bi_opf & REQ_SYNC) &&
..	..	@@ -5793,7 +5980,7 @@
5793	5980	sector_div(sector_nr, new_data_disks);
5794	5981	if (sector_nr) {
5795	5982	mddev->curr_resync_completed = sector_nr;
5796		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	5983	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
5797	5984	*skipped = 1;
5798	5985	retn = sector_nr;
5799	5986	goto finish;
..	..	@@ -5907,11 +6094,11 @@
5907	6094	conf->reshape_safe = mddev->reshape_position;
5908	6095	spin_unlock_irq(&conf->device_lock);
5909	6096	wake_up(&conf->wait_for_overlap);
5910		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	6097	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
5911	6098	}
5912	6099
5913	6100	INIT_LIST_HEAD(&stripes);
5914		- for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
	6101	+ for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {
5915	6102	int j;
5916	6103	int skipped_disk = 0;
5917	6104	sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
..	..	@@ -5932,7 +6119,7 @@
5932	6119	skipped_disk = 1;
5933	6120	continue;
5934	6121	}
5935		- memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
	6122	+ memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
5936	6123	set_bit(R5_Expanded, &sh->dev[j].flags);
5937	6124	set_bit(R5_UPTODATE, &sh->dev[j].flags);
5938	6125	}
..	..	@@ -5967,7 +6154,7 @@
5967	6154	set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
5968	6155	set_bit(STRIPE_HANDLE, &sh->state);
5969	6156	raid5_release_stripe(sh);
5970		- first_sector += STRIPE_SECTORS;
	6157	+ first_sector += RAID5_STRIPE_SECTORS(conf);
5971	6158	}
5972	6159	/* Now that the sources are clearly marked, we can release
5973	6160	* the destination stripes
..	..	@@ -6014,7 +6201,7 @@
6014	6201	conf->reshape_safe = mddev->reshape_position;
6015	6202	spin_unlock_irq(&conf->device_lock);
6016	6203	wake_up(&conf->wait_for_overlap);
6017		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	6204	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
6018	6205	}
6019	6206	ret:
6020	6207	return retn;
..	..	@@ -6073,11 +6260,12 @@
6073	6260	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
6074	6261	!conf->fullsync &&
6075	6262	!md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
6076		- sync_blocks >= STRIPE_SECTORS) {
	6263	+ sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {
6077	6264	/* we can skip this block, and probably more */
6078		- sync_blocks /= STRIPE_SECTORS;
	6265	+ do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));
6079	6266	*skipped = 1;
6080		- return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	6267	+ /* keep things rounded to whole stripes */
	6268	+ return sync_blocks * RAID5_STRIPE_SECTORS(conf);
6081	6269	}
6082	6270
6083	6271	md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
..	..	@@ -6110,7 +6298,7 @@
6110	6298
6111	6299	raid5_release_stripe(sh);
6112	6300
6113		- return STRIPE_SECTORS;
	6301	+ return RAID5_STRIPE_SECTORS(conf);
6114	6302	}
6115	6303
6116	6304	static int retry_aligned_read(struct r5conf conf, struct bio raid_bio,
..	..	@@ -6133,14 +6321,14 @@
6133	6321	int handled = 0;
6134	6322
6135	6323	logical_sector = raid_bio->bi_iter.bi_sector &
6136		- ~((sector_t)STRIPE_SECTORS-1);
	6324	+ ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
6137	6325	sector = raid5_compute_sector(conf, logical_sector,
6138	6326	0, &dd_idx, NULL);
6139	6327	last_sector = bio_end_sector(raid_bio);
6140	6328
6141	6329	for (; logical_sector < last_sector;
6142		- logical_sector += STRIPE_SECTORS,
6143		- sector += STRIPE_SECTORS,
	6330	+ logical_sector += RAID5_STRIPE_SECTORS(conf),
	6331	+ sector += RAID5_STRIPE_SECTORS(conf),
6144	6332	scnt++) {
6145	6333
6146	6334	if (scnt < offset)
..	..	@@ -6179,6 +6367,8 @@
6179	6367	static int handle_active_stripes(struct r5conf *conf, int group,
6180	6368	struct r5worker *worker,
6181	6369	struct list_head *temp_inactive_list)
	6370	+ __releases(&conf->device_lock)
	6371	+ __acquires(&conf->device_lock)
6182	6372	{
6183	6373	struct stripe_head batch[MAX_STRIPE_BATCH], sh;
6184	6374	int i, batch_size = 0, hash;
..	..	@@ -6331,7 +6521,18 @@
6331	6521	spin_unlock_irq(&conf->device_lock);
6332	6522	md_check_recovery(mddev);
6333	6523	spin_lock_irq(&conf->device_lock);
	6524	+
	6525	+ /*
	6526	+ * Waiting on MD_SB_CHANGE_PENDING below may deadlock
	6527	+ * seeing md_check_recovery() is needed to clear
	6528	+ * the flag when using mdmon.
	6529	+ */
	6530	+ continue;
6334	6531	}
	6532	+
	6533	+ wait_event_lock_irq(mddev->sb_wait,
	6534	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
	6535	+ conf->device_lock);
6335	6536	}
6336	6537	pr_debug("%d stripes handled\n", handled);
6337	6538
..	..	@@ -6471,6 +6672,100 @@
6471	6672	raid5_show_rmw_level,
6472	6673	raid5_store_rmw_level);
6473	6674
	6675	+static ssize_t
	6676	+raid5_show_stripe_size(struct mddev mddev, char page)
	6677	+{
	6678	+ struct r5conf *conf;
	6679	+ int ret = 0;
	6680	+
	6681	+ spin_lock(&mddev->lock);
	6682	+ conf = mddev->private;
	6683	+ if (conf)
	6684	+ ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));
	6685	+ spin_unlock(&mddev->lock);
	6686	+ return ret;
	6687	+}
	6688	+
	6689	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	6690	+static ssize_t
	6691	+raid5_store_stripe_size(struct mddev mddev, const char page, size_t len)
	6692	+{
	6693	+ struct r5conf *conf;
	6694	+ unsigned long new;
	6695	+ int err;
	6696	+ int size;
	6697	+
	6698	+ if (len >= PAGE_SIZE)
	6699	+ return -EINVAL;
	6700	+ if (kstrtoul(page, 10, &new))
	6701	+ return -EINVAL;
	6702	+
	6703	+ /*
	6704	+ * The value should not be bigger than PAGE_SIZE. It requires to
	6705	+ * be multiple of DEFAULT_STRIPE_SIZE and the value should be power
	6706	+ * of two.
	6707	+ */
	6708	+ if (new % DEFAULT_STRIPE_SIZE != 0 \|\|
	6709	+ new > PAGE_SIZE \|\| new == 0 \|\|
	6710	+ new != roundup_pow_of_two(new))
	6711	+ return -EINVAL;
	6712	+
	6713	+ err = mddev_lock(mddev);
	6714	+ if (err)
	6715	+ return err;
	6716	+
	6717	+ conf = mddev->private;
	6718	+ if (!conf) {
	6719	+ err = -ENODEV;
	6720	+ goto out_unlock;
	6721	+ }
	6722	+
	6723	+ if (new == conf->stripe_size)
	6724	+ goto out_unlock;
	6725	+
	6726	+ pr_debug("md/raid: change stripe_size from %lu to %lu\n",
	6727	+ conf->stripe_size, new);
	6728	+
	6729	+ if (mddev->sync_thread \|\|
	6730	+ test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) \|\|
	6731	+ mddev->reshape_position != MaxSector \|\|
	6732	+ mddev->sysfs_active) {
	6733	+ err = -EBUSY;
	6734	+ goto out_unlock;
	6735	+ }
	6736	+
	6737	+ mddev_suspend(mddev);
	6738	+ mutex_lock(&conf->cache_size_mutex);
	6739	+ size = conf->max_nr_stripes;
	6740	+
	6741	+ shrink_stripes(conf);
	6742	+
	6743	+ conf->stripe_size = new;
	6744	+ conf->stripe_shift = ilog2(new) - 9;
	6745	+ conf->stripe_sectors = new >> 9;
	6746	+ if (grow_stripes(conf, size)) {
	6747	+ pr_warn("md/raid:%s: couldn't allocate buffers\n",
	6748	+ mdname(mddev));
	6749	+ err = -ENOMEM;
	6750	+ }
	6751	+ mutex_unlock(&conf->cache_size_mutex);
	6752	+ mddev_resume(mddev);
	6753	+
	6754	+out_unlock:
	6755	+ mddev_unlock(mddev);
	6756	+ return err ?: len;
	6757	+}
	6758	+
	6759	+static struct md_sysfs_entry
	6760	+raid5_stripe_size = __ATTR(stripe_size, 0644,
	6761	+ raid5_show_stripe_size,
	6762	+ raid5_store_stripe_size);
	6763	+#else
	6764	+static struct md_sysfs_entry
	6765	+raid5_stripe_size = __ATTR(stripe_size, 0444,
	6766	+ raid5_show_stripe_size,
	6767	+ NULL);
	6768	+#endif
6474	6769
6475	6770	static ssize_t
6476	6771	raid5_show_preread_threshold(struct mddev mddev, char page)
..	..	@@ -6550,14 +6845,14 @@
6550	6845	if (!conf)
6551	6846	err = -ENODEV;
6552	6847	else if (new != conf->skip_copy) {
	6848	+ struct request_queue *q = mddev->queue;
	6849	+
6553	6850	mddev_suspend(mddev);
6554	6851	conf->skip_copy = new;
6555	6852	if (new)
6556		- mddev->queue->backing_dev_info->capabilities \|=
6557		- BDI_CAP_STABLE_WRITES;
	6853	+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
6558	6854	else
6559		- mddev->queue->backing_dev_info->capabilities &=
6560		- ~BDI_CAP_STABLE_WRITES;
	6855	+ blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
6561	6856	mddev_resume(mddev);
6562	6857	}
6563	6858	mddev_unlock(mddev);
..	..	@@ -6597,7 +6892,6 @@
6597	6892
6598	6893	static int alloc_thread_groups(struct r5conf *conf, int cnt,
6599	6894	int *group_cnt,
6600		- int *worker_cnt_per_group,
6601	6895	struct r5worker_group **worker_groups);
6602	6896	static ssize_t
6603	6897	raid5_store_group_thread_cnt(struct mddev mddev, const char page, size_t len)
..	..	@@ -6606,7 +6900,7 @@
6606	6900	unsigned int new;
6607	6901	int err;
6608	6902	struct r5worker_group new_groups, old_groups;
6609		- int group_cnt, worker_cnt_per_group;
	6903	+ int group_cnt;
6610	6904
6611	6905	if (len >= PAGE_SIZE)
6612	6906	return -EINVAL;
..	..	@@ -6629,13 +6923,11 @@
6629	6923	if (old_groups)
6630	6924	flush_workqueue(raid5_wq);
6631	6925
6632		- err = alloc_thread_groups(conf, new,
6633		- &group_cnt, &worker_cnt_per_group,
6634		- &new_groups);
	6926	+ err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
6635	6927	if (!err) {
6636	6928	spin_lock_irq(&conf->device_lock);
6637	6929	conf->group_cnt = group_cnt;
6638		- conf->worker_cnt_per_group = worker_cnt_per_group;
	6930	+ conf->worker_cnt_per_group = new;
6639	6931	conf->worker_groups = new_groups;
6640	6932	spin_unlock_irq(&conf->device_lock);
6641	6933
..	..	@@ -6662,7 +6954,9 @@
6662	6954	&raid5_group_thread_cnt.attr,
6663	6955	&raid5_skip_copy.attr,
6664	6956	&raid5_rmw_level.attr,
	6957	+ &raid5_stripe_size.attr,
6665	6958	&r5c_journal_mode.attr,
	6959	+ &ppl_write_hint.attr,
6666	6960	NULL,
6667	6961	};
6668	6962	static struct attribute_group raid5_attrs_group = {
..	..	@@ -6670,16 +6964,13 @@
6670	6964	.attrs = raid5_attrs,
6671	6965	};
6672	6966
6673		-static int alloc_thread_groups(struct r5conf *conf, int cnt,
6674		- int *group_cnt,
6675		- int *worker_cnt_per_group,
	6967	+static int alloc_thread_groups(struct r5conf conf, int cnt, int group_cnt,
6676	6968	struct r5worker_group **worker_groups)
6677	6969	{
6678	6970	int i, j, k;
6679	6971	ssize_t size;
6680	6972	struct r5worker *workers;
6681	6973
6682		- *worker_cnt_per_group = cnt;
6683	6974	if (cnt == 0) {
6684	6975	*group_cnt = 0;
6685	6976	*worker_groups = NULL;
..	..	@@ -6745,25 +7036,25 @@
6745	7036	static void free_scratch_buffer(struct r5conf conf, struct raid5_percpu percpu)
6746	7037	{
6747	7038	safe_put_page(percpu->spare_page);
6748		- if (percpu->scribble)
6749		- flex_array_free(percpu->scribble);
6750	7039	percpu->spare_page = NULL;
	7040	+ kvfree(percpu->scribble);
6751	7041	percpu->scribble = NULL;
6752	7042	}
6753	7043
6754	7044	static int alloc_scratch_buffer(struct r5conf conf, struct raid5_percpu percpu)
6755	7045	{
6756		- if (conf->level == 6 && !percpu->spare_page)
	7046	+ if (conf->level == 6 && !percpu->spare_page) {
6757	7047	percpu->spare_page = alloc_page(GFP_KERNEL);
6758		- if (!percpu->scribble)
6759		- percpu->scribble = scribble_alloc(max(conf->raid_disks,
6760		- conf->previous_raid_disks),
6761		- max(conf->chunk_sectors,
6762		- conf->prev_chunk_sectors)
6763		- / STRIPE_SECTORS,
6764		- GFP_KERNEL);
	7048	+ if (!percpu->spare_page)
	7049	+ return -ENOMEM;
	7050	+ }
6765	7051
6766		- if (!percpu->scribble \|\| (conf->level == 6 && !percpu->spare_page)) {
	7052	+ if (scribble_alloc(percpu,
	7053	+ max(conf->raid_disks,
	7054	+ conf->previous_raid_disks),
	7055	+ max(conf->chunk_sectors,
	7056	+ conf->prev_chunk_sectors)
	7057	+ / RAID5_STRIPE_SECTORS(conf))) {
6767	7058	free_scratch_buffer(conf, percpu);
6768	7059	return -ENOMEM;
6769	7060	}
..	..	@@ -6829,6 +7120,7 @@
6829	7120	conf->percpu = alloc_percpu(struct raid5_percpu);
6830	7121	if (!conf->percpu)
6831	7122	return -ENOMEM;
	7123	+
6832	7124	err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
6833	7125	if (!err) {
6834	7126	conf->scribble_disks = max(conf->raid_disks,
..	..	@@ -6879,7 +7171,7 @@
6879	7171	struct disk_info *disk;
6880	7172	char pers_name[6];
6881	7173	int i;
6882		- int group_cnt, worker_cnt_per_group;
	7174	+ int group_cnt;
6883	7175	struct r5worker_group *new_group;
6884	7176	int ret;
6885	7177
..	..	@@ -6915,6 +7207,12 @@
6915	7207	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
6916	7208	if (conf == NULL)
6917	7209	goto abort;
	7210	+
	7211	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	7212	+ conf->stripe_size = DEFAULT_STRIPE_SIZE;
	7213	+ conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
	7214	+ conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;
	7215	+#endif
6918	7216	INIT_LIST_HEAD(&conf->free_list);
6919	7217	INIT_LIST_HEAD(&conf->pending_list);
6920	7218	conf->pending_data = kcalloc(PENDING_IO_MAX,
..	..	@@ -6925,15 +7223,14 @@
6925	7223	for (i = 0; i < PENDING_IO_MAX; i++)
6926	7224	list_add(&conf->pending_data[i].sibling, &conf->free_list);
6927	7225	/* Don't enable multi-threading by default*/
6928		- if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6929		- &new_group)) {
	7226	+ if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
6930	7227	conf->group_cnt = group_cnt;
6931		- conf->worker_cnt_per_group = worker_cnt_per_group;
	7228	+ conf->worker_cnt_per_group = 0;
6932	7229	conf->worker_groups = new_group;
6933	7230	} else
6934	7231	goto abort;
6935	7232	spin_lock_init(&conf->device_lock);
6936		- seqcount_init(&conf->gen_lock);
	7233	+ seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);
6937	7234	mutex_init(&conf->cache_size_mutex);
6938	7235	init_waitqueue_head(&conf->wait_for_quiescent);
6939	7236	init_waitqueue_head(&conf->wait_for_stripe);
..	..	@@ -7067,8 +7364,8 @@
7067	7364	conf->min_nr_stripes = NR_STRIPES;
7068	7365	if (mddev->reshape_position != MaxSector) {
7069	7366	int stripes = max_t(int,
7070		- ((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4,
7071		- ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4);
	7367	+ ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,
	7368	+ ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);
7072	7369	conf->min_nr_stripes = max(NR_STRIPES, stripes);
7073	7370	if (conf->min_nr_stripes != NR_STRIPES)
7074	7371	pr_info("md/raid:%s: force stripe size %d for reshape\n",
..	..	@@ -7141,6 +7438,12 @@
7141	7438	return 1;
7142	7439	}
7143	7440	return 0;
	7441	+}
	7442	+
	7443	+static void raid5_set_io_opt(struct r5conf *conf)
	7444	+{
	7445	+ blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
	7446	+ (conf->raid_disks - conf->max_degraded));
7144	7447	}
7145	7448
7146	7449	static int raid5_run(struct mddev *mddev)
..	..	@@ -7427,13 +7730,10 @@
7427	7730	int data_disks = conf->previous_raid_disks - conf->max_degraded;
7428	7731	int stripe = data_disks *
7429	7732	((mddev->chunk_sectors << 9) / PAGE_SIZE);
7430		- if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
7431		- mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
7432	7733
7433	7734	chunk_size = mddev->chunk_sectors << 9;
7434	7735	blk_queue_io_min(mddev->queue, chunk_size);
7435		- blk_queue_io_opt(mddev->queue, chunk_size *
7436		- (conf->raid_disks - conf->max_degraded));
	7736	+ raid5_set_io_opt(conf);
7437	7737	mddev->queue->limits.raid_partial_stripes_expensive = 1;
7438	7738	/*
7439	7739	* We can only discard a whole stripe. It doesn't make sense to
..	..	@@ -7718,6 +8018,7 @@
7718	8018	*/
7719	8019	if (rdev->saved_raid_disk >= 0 &&
7720	8020	rdev->saved_raid_disk >= first &&
	8021	+ rdev->saved_raid_disk <= last &&
7721	8022	conf->disks[rdev->saved_raid_disk].rdev == NULL)
7722	8023	first = rdev->saved_raid_disk;
7723	8024
..	..	@@ -7799,14 +8100,14 @@
7799	8100	* stripe_heads first.
7800	8101	*/
7801	8102	struct r5conf *conf = mddev->private;
7802		- if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	8103	+ if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
7803	8104	> conf->min_nr_stripes \|\|
7804		- ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	8105	+ ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
7805	8106	> conf->min_nr_stripes) {
7806	8107	pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
7807	8108	mdname(mddev),
7808	8109	((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
7809		- / STRIPE_SIZE)*4);
	8110	+ / RAID5_STRIPE_SIZE(conf))*4);
7810	8111	return 0;
7811	8112	}
7812	8113	return 1;
..	..	@@ -7942,8 +8243,8 @@
7942	8243	else
7943	8244	rdev->recovery_offset = 0;
7944	8245
7945		- if (sysfs_link_rdev(mddev, rdev))
7946		- /* Failure here is OK */;
	8246	+ /* Failure here is OK */
	8247	+ sysfs_link_rdev(mddev, rdev);
7947	8248	}
7948	8249	} else if (rdev->raid_disk >= conf->previous_raid_disks
7949	8250	&& !test_bit(Faulty, &rdev->flags)) {
..	..	@@ -8017,16 +8318,8 @@
8017	8318	spin_unlock_irq(&conf->device_lock);
8018	8319	wake_up(&conf->wait_for_overlap);
8019	8320
8020		- /* read-ahead size must cover two whole stripes, which is
8021		- * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
8022		- */
8023		- if (conf->mddev->queue) {
8024		- int data_disks = conf->raid_disks - conf->max_degraded;
8025		- int stripe = data_disks * ((conf->chunk_sectors << 9)
8026		- / PAGE_SIZE);
8027		- if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
8028		- conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
8029		- }
	8321	+ if (conf->mddev->queue)
	8322	+ raid5_set_io_opt(conf);
8030	8323	}
8031	8324	}
8032	8325
..	..	@@ -8138,7 +8431,7 @@
8138	8431	while (chunksect && (mddev->array_sectors & (chunksect-1)))
8139	8432	chunksect >>= 1;
8140	8433
8141		- if ((chunksect<<9) < STRIPE_SIZE)
	8434	+ if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))
8142	8435	/* array size does not allow a suitable chunk size */
8143	8436	return ERR_PTR(-EINVAL);
8144	8437
..	..	@@ -8425,7 +8718,6 @@
8425	8718	.finish_reshape = raid5_finish_reshape,
8426	8719	.quiesce = raid5_quiesce,
8427	8720	.takeover = raid6_takeover,
8428		- .congested = raid5_congested,
8429	8721	.change_consistency_policy = raid5_change_consistency_policy,
8430	8722	};
8431	8723	static struct md_personality raid5_personality =
..	..	@@ -8450,7 +8742,6 @@
8450	8742	.finish_reshape = raid5_finish_reshape,
8451	8743	.quiesce = raid5_quiesce,
8452	8744	.takeover = raid5_takeover,
8453		- .congested = raid5_congested,
8454	8745	.change_consistency_policy = raid5_change_consistency_policy,
8455	8746	};
8456	8747
..	..	@@ -8476,7 +8767,6 @@
8476	8767	.finish_reshape = raid5_finish_reshape,
8477	8768	.quiesce = raid5_quiesce,
8478	8769	.takeover = raid4_takeover,
8479		- .congested = raid5_congested,
8480	8770	.change_consistency_policy = raid5_change_consistency_policy,
8481	8771	};
8482	8772