~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* raid5.c : Multiple Devices driver for Linux
3	4	* Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
..	..	@@ -7,15 +8,6 @@
7	8	* RAID-4/5/6 management functions.
8	9	* Thanks to Penguin Computing for making the RAID-6 development possible
9	10	* by donating a test server!
10		- *
11		- * This program is free software; you can redistribute it and/or modify
12		- * it under the terms of the GNU General Public License as published by
13		- * the Free Software Foundation; either version 2, or (at your option)
14		- * any later version.
15		- *
16		- * You should have received a copy of the GNU General Public License
17		- * (for example /usr/src/linux/COPYING); if not, write to the Free
18		- * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19	11	*/
20	12
21	13	/*
..	..	@@ -44,6 +36,7 @@
44	36	*/
45	37
46	38	#include <linux/blkdev.h>
	39	+#include <linux/delay.h>
47	40	#include <linux/kthread.h>
48	41	#include <linux/raid/pq.h>
49	42	#include <linux/async_tx.h>
..	..	@@ -54,7 +47,6 @@
54	47	#include <linux/slab.h>
55	48	#include <linux/ratelimit.h>
56	49	#include <linux/nodemask.h>
57		-#include <linux/flex_array.h>
58	50
59	51	#include <trace/events/block.h>
60	52	#include <linux/list_sort.h>
..	..	@@ -78,13 +70,13 @@
78	70
79	71	static inline struct hlist_head stripe_hash(struct r5conf conf, sector_t sect)
80	72	{
81		- int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	73	+ int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
82	74	return &conf->stripe_hashtbl[hash];
83	75	}
84	76
85		-static inline int stripe_hash_locks_hash(sector_t sect)
	77	+static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)
86	78	{
87		- return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
	79	+ return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;
88	80	}
89	81
90	82	static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
..	..	@@ -457,13 +449,74 @@
457	449	return sh;
458	450	}
459	451
	452	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	453	+static void free_stripe_pages(struct stripe_head *sh)
	454	+{
	455	+ int i;
	456	+ struct page *p;
	457	+
	458	+ /* Have not allocate page pool */
	459	+ if (!sh->pages)
	460	+ return;
	461	+
	462	+ for (i = 0; i < sh->nr_pages; i++) {
	463	+ p = sh->pages[i];
	464	+ if (p)
	465	+ put_page(p);
	466	+ sh->pages[i] = NULL;
	467	+ }
	468	+}
	469	+
	470	+static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
	471	+{
	472	+ int i;
	473	+ struct page *p;
	474	+
	475	+ for (i = 0; i < sh->nr_pages; i++) {
	476	+ /* The page have allocated. */
	477	+ if (sh->pages[i])
	478	+ continue;
	479	+
	480	+ p = alloc_page(gfp);
	481	+ if (!p) {
	482	+ free_stripe_pages(sh);
	483	+ return -ENOMEM;
	484	+ }
	485	+ sh->pages[i] = p;
	486	+ }
	487	+ return 0;
	488	+}
	489	+
	490	+static int
	491	+init_stripe_shared_pages(struct stripe_head sh, struct r5conf conf, int disks)
	492	+{
	493	+ int nr_pages, cnt;
	494	+
	495	+ if (sh->pages)
	496	+ return 0;
	497	+
	498	+ /* Each of the sh->dev[i] need one conf->stripe_size */
	499	+ cnt = PAGE_SIZE / conf->stripe_size;
	500	+ nr_pages = (disks + cnt - 1) / cnt;
	501	+
	502	+ sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
	503	+ if (!sh->pages)
	504	+ return -ENOMEM;
	505	+ sh->nr_pages = nr_pages;
	506	+ sh->stripes_per_page = cnt;
	507	+ return 0;
	508	+}
	509	+#endif
	510	+
460	511	static void shrink_buffers(struct stripe_head *sh)
461	512	{
462		- struct page *p;
463	513	int i;
464	514	int num = sh->raid_conf->pool_size;
465	515
	516	+#if PAGE_SIZE == DEFAULT_STRIPE_SIZE
466	517	for (i = 0; i < num ; i++) {
	518	+ struct page *p;
	519	+
467	520	WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
468	521	p = sh->dev[i].page;
469	522	if (!p)
..	..	@@ -471,6 +524,11 @@
471	524	sh->dev[i].page = NULL;
472	525	put_page(p);
473	526	}
	527	+#else
	528	+ for (i = 0; i < num; i++)
	529	+ sh->dev[i].page = NULL;
	530	+ free_stripe_pages(sh); /* Free pages */
	531	+#endif
474	532	}
475	533
476	534	static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
..	..	@@ -478,6 +536,7 @@
478	536	int i;
479	537	int num = sh->raid_conf->pool_size;
480	538
	539	+#if PAGE_SIZE == DEFAULT_STRIPE_SIZE
481	540	for (i = 0; i < num; i++) {
482	541	struct page *page;
483	542
..	..	@@ -486,8 +545,18 @@
486	545	}
487	546	sh->dev[i].page = page;
488	547	sh->dev[i].orig_page = page;
	548	+ sh->dev[i].offset = 0;
489	549	}
	550	+#else
	551	+ if (alloc_stripe_pages(sh, gfp))
	552	+ return -ENOMEM;
490	553
	554	+ for (i = 0; i < num; i++) {
	555	+ sh->dev[i].page = raid5_get_dev_page(sh, i);
	556	+ sh->dev[i].orig_page = sh->dev[i].page;
	557	+ sh->dev[i].offset = raid5_get_page_offset(sh, i);
	558	+ }
	559	+#endif
491	560	return 0;
492	561	}
493	562
..	..	@@ -618,17 +687,17 @@
618	687	return degraded;
619	688	}
620	689
621		-static int has_failed(struct r5conf *conf)
	690	+static bool has_failed(struct r5conf *conf)
622	691	{
623		- int degraded;
	692	+ int degraded = conf->mddev->degraded;
624	693
625		- if (conf->mddev->reshape_position == MaxSector)
626		- return conf->mddev->degraded > conf->max_degraded;
	694	+ if (test_bit(MD_BROKEN, &conf->mddev->flags))
	695	+ return true;
627	696
628		- degraded = raid5_calc_degraded(conf);
629		- if (degraded > conf->max_degraded)
630		- return 1;
631		- return 0;
	697	+ if (conf->mddev->reshape_position != MaxSector)
	698	+ degraded = raid5_calc_degraded(conf);
	699	+
	700	+ return degraded > conf->max_degraded;
632	701	}
633	702
634	703	struct stripe_head *
..	..	@@ -636,7 +705,7 @@
636	705	int previous, int noblock, int noquiesce)
637	706	{
638	707	struct stripe_head *sh;
639		- int hash = stripe_hash_locks_hash(sector);
	708	+ int hash = stripe_hash_locks_hash(conf, sector);
640	709	int inc_empty_inactive_list_flag;
641	710
642	711	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
..	..	@@ -712,6 +781,8 @@
712	781	}
713	782
714	783	static void lock_two_stripes(struct stripe_head sh1, struct stripe_head sh2)
	784	+ __acquires(&sh1->stripe_lock)
	785	+ __acquires(&sh2->stripe_lock)
715	786	{
716	787	if (sh1 > sh2) {
717	788	spin_lock_irq(&sh2->stripe_lock);
..	..	@@ -723,6 +794,8 @@
723	794	}
724	795
725	796	static void unlock_two_stripes(struct stripe_head sh1, struct stripe_head sh2)
	797	+ __releases(&sh1->stripe_lock)
	798	+ __releases(&sh2->stripe_lock)
726	799	{
727	800	spin_unlock(&sh1->stripe_lock);
728	801	spin_unlock_irq(&sh2->stripe_lock);
..	..	@@ -753,9 +826,9 @@
753	826	tmp_sec = sh->sector;
754	827	if (!sector_div(tmp_sec, conf->chunk_sectors))
755	828	return;
756		- head_sector = sh->sector - STRIPE_SECTORS;
	829	+ head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
757	830
758		- hash = stripe_hash_locks_hash(head_sector);
	831	+ hash = stripe_hash_locks_hash(conf, head_sector);
759	832	spin_lock_irq(conf->hash_locks + hash);
760	833	head = __find_stripe(conf, head_sector, conf->generation);
761	834	if (head && !atomic_inc_not_zero(&head->count)) {
..	..	@@ -878,7 +951,7 @@
878	951	struct bio *bio;
879	952
880	953	while ((bio = bio_list_pop(tmp)))
881		- generic_make_request(bio);
	954	+ submit_bio_noacct(bio);
882	955	}
883	956
884	957	static int cmp_stripe(void priv, struct list_head a, struct list_head *b)
..	..	@@ -1062,7 +1135,7 @@
1062	1135	test_bit(WriteErrorSeen, &rdev->flags)) {
1063	1136	sector_t first_bad;
1064	1137	int bad_sectors;
1065		- int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	1138	+ int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
1066	1139	&first_bad, &bad_sectors);
1067	1140	if (!bad)
1068	1141	break;
..	..	@@ -1094,7 +1167,7 @@
1094	1167	if (rdev) {
1095	1168	if (s->syncing \|\| s->expanding \|\| s->expanded
1096	1169	\|\| s->replacing)
1097		- md_sync_acct(rdev->bdev, STRIPE_SECTORS);
	1170	+ md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));
1098	1171
1099	1172	set_bit(STRIPE_IO_STARTED, &sh->state);
1100	1173
..	..	@@ -1134,12 +1207,12 @@
1134	1207	else
1135	1208	sh->dev[i].vec.bv_page = sh->dev[i].page;
1136	1209	bi->bi_vcnt = 1;
1137		- bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1138		- bi->bi_io_vec[0].bv_offset = 0;
1139		- bi->bi_iter.bi_size = STRIPE_SIZE;
	1210	+ bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
	1211	+ bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
	1212	+ bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
1140	1213	bi->bi_write_hint = sh->dev[i].write_hint;
1141	1214	if (!rrdev)
1142		- sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
	1215	+ sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1143	1216	/*
1144	1217	* If this is discard request, set bi_vcnt 0. We don't
1145	1218	* want to confuse SCSI because SCSI will replace payload
..	..	@@ -1156,12 +1229,12 @@
1156	1229	if (should_defer && op_is_write(op))
1157	1230	bio_list_add(&pending_bios, bi);
1158	1231	else
1159		- generic_make_request(bi);
	1232	+ submit_bio_noacct(bi);
1160	1233	}
1161	1234	if (rrdev) {
1162	1235	if (s->syncing \|\| s->expanding \|\| s->expanded
1163	1236	\|\| s->replacing)
1164		- md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
	1237	+ md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));
1165	1238
1166	1239	set_bit(STRIPE_IO_STARTED, &sh->state);
1167	1240
..	..	@@ -1188,11 +1261,11 @@
1188	1261	WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1189	1262	sh->dev[i].rvec.bv_page = sh->dev[i].page;
1190	1263	rbi->bi_vcnt = 1;
1191		- rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1192		- rbi->bi_io_vec[0].bv_offset = 0;
1193		- rbi->bi_iter.bi_size = STRIPE_SIZE;
	1264	+ rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
	1265	+ rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
	1266	+ rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
1194	1267	rbi->bi_write_hint = sh->dev[i].write_hint;
1195		- sh->dev[i].write_hint = RWF_WRITE_LIFE_NOT_SET;
	1268	+ sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1196	1269	/*
1197	1270	* If this is discard request, set bi_vcnt 0. We don't
1198	1271	* want to confuse SCSI because SCSI will replace payload
..	..	@@ -1206,7 +1279,7 @@
1206	1279	if (should_defer && op_is_write(op))
1207	1280	bio_list_add(&pending_bios, rbi);
1208	1281	else
1209		- generic_make_request(rbi);
	1282	+ submit_bio_noacct(rbi);
1210	1283	}
1211	1284	if (!rdev && !rrdev) {
1212	1285	if (op_is_write(op))
..	..	@@ -1231,7 +1304,7 @@
1231	1304
1232	1305	static struct dma_async_tx_descriptor *
1233	1306	async_copy_data(int frombio, struct bio bio, struct page *page,
1234		- sector_t sector, struct dma_async_tx_descriptor *tx,
	1307	+ unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
1235	1308	struct stripe_head *sh, int no_skipcopy)
1236	1309	{
1237	1310	struct bio_vec bvl;
..	..	@@ -1240,6 +1313,7 @@
1240	1313	int page_offset;
1241	1314	struct async_submit_ctl submit;
1242	1315	enum async_tx_flags flags = 0;
	1316	+ struct r5conf *conf = sh->raid_conf;
1243	1317
1244	1318	if (bio->bi_iter.bi_sector >= sector)
1245	1319	page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
..	..	@@ -1261,8 +1335,8 @@
1261	1335	len -= b_offset;
1262	1336	}
1263	1337
1264		- if (len > 0 && page_offset + len > STRIPE_SIZE)
1265		- clen = STRIPE_SIZE - page_offset;
	1338	+ if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))
	1339	+ clen = RAID5_STRIPE_SIZE(conf) - page_offset;
1266	1340	else
1267	1341	clen = len;
1268	1342
..	..	@@ -1270,17 +1344,17 @@
1270	1344	b_offset += bvl.bv_offset;
1271	1345	bio_page = bvl.bv_page;
1272	1346	if (frombio) {
1273		- if (sh->raid_conf->skip_copy &&
	1347	+ if (conf->skip_copy &&
1274	1348	b_offset == 0 && page_offset == 0 &&
1275		- clen == STRIPE_SIZE &&
	1349	+ clen == RAID5_STRIPE_SIZE(conf) &&
1276	1350	!no_skipcopy)
1277	1351	*page = bio_page;
1278	1352	else
1279		- tx = async_memcpy(*page, bio_page, page_offset,
	1353	+ tx = async_memcpy(*page, bio_page, page_offset + poff,
1280	1354	b_offset, clen, &submit);
1281	1355	} else
1282	1356	tx = async_memcpy(bio_page, *page, b_offset,
1283		- page_offset, clen, &submit);
	1357	+ page_offset + poff, clen, &submit);
1284	1358	}
1285	1359	/* chain the operations */
1286	1360	submit.depend_tx = tx;
..	..	@@ -1297,6 +1371,7 @@
1297	1371	{
1298	1372	struct stripe_head *sh = stripe_head_ref;
1299	1373	int i;
	1374	+ struct r5conf *conf = sh->raid_conf;
1300	1375
1301	1376	pr_debug("%s: stripe %llu\n", __func__,
1302	1377	(unsigned long long)sh->sector);
..	..	@@ -1317,8 +1392,8 @@
1317	1392	rbi = dev->read;
1318	1393	dev->read = NULL;
1319	1394	while (rbi && rbi->bi_iter.bi_sector <
1320		- dev->sector + STRIPE_SECTORS) {
1321		- rbi2 = r5_next_bio(rbi, dev->sector);
	1395	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
	1396	+ rbi2 = r5_next_bio(conf, rbi, dev->sector);
1322	1397	bio_endio(rbi);
1323	1398	rbi = rbi2;
1324	1399	}
..	..	@@ -1335,6 +1410,7 @@
1335	1410	struct dma_async_tx_descriptor *tx = NULL;
1336	1411	struct async_submit_ctl submit;
1337	1412	int i;
	1413	+ struct r5conf *conf = sh->raid_conf;
1338	1414
1339	1415	BUG_ON(sh->batch_head);
1340	1416	pr_debug("%s: stripe %llu\n", __func__,
..	..	@@ -1349,10 +1425,11 @@
1349	1425	dev->toread = NULL;
1350	1426	spin_unlock_irq(&sh->stripe_lock);
1351	1427	while (rbi && rbi->bi_iter.bi_sector <
1352		- dev->sector + STRIPE_SECTORS) {
	1428	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
1353	1429	tx = async_copy_data(0, rbi, &dev->page,
	1430	+ dev->offset,
1354	1431	dev->sector, tx, sh, 0);
1355		- rbi = r5_next_bio(rbi, dev->sector);
	1432	+ rbi = r5_next_bio(conf, rbi, dev->sector);
1356	1433	}
1357	1434	}
1358	1435	}
..	..	@@ -1394,22 +1471,25 @@
1394	1471	}
1395	1472
1396	1473	/* return a pointer to the address conversion region of the scribble buffer */
1397		-static addr_conv_t to_addr_conv(struct stripe_head sh,
1398		- struct raid5_percpu *percpu, int i)
	1474	+static struct page *to_addr_page(struct raid5_percpu percpu, int i)
1399	1475	{
1400		- void *addr;
1401		-
1402		- addr = flex_array_get(percpu->scribble, i);
1403		- return addr + sizeof(struct page ) (sh->disks + 2);
	1476	+ return percpu->scribble + i * percpu->scribble_obj_size;
1404	1477	}
1405	1478
1406	1479	/* return a pointer to the address conversion region of the scribble buffer */
1407		-static struct page *to_addr_page(struct raid5_percpu percpu, int i)
	1480	+static addr_conv_t to_addr_conv(struct stripe_head sh,
	1481	+ struct raid5_percpu *percpu, int i)
1408	1482	{
1409		- void *addr;
	1483	+ return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
	1484	+}
1410	1485
1411		- addr = flex_array_get(percpu->scribble, i);
1412		- return addr;
	1486	+/*
	1487	+ * Return a pointer to record offset address.
	1488	+ */
	1489	+static unsigned int *
	1490	+to_addr_offs(struct stripe_head sh, struct raid5_percpu percpu)
	1491	+{
	1492	+ return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
1413	1493	}
1414	1494
1415	1495	static struct dma_async_tx_descriptor *
..	..	@@ -1417,9 +1497,11 @@
1417	1497	{
1418	1498	int disks = sh->disks;
1419	1499	struct page **xor_srcs = to_addr_page(percpu, 0);
	1500	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
1420	1501	int target = sh->ops.target;
1421	1502	struct r5dev *tgt = &sh->dev[target];
1422	1503	struct page *xor_dest = tgt->page;
	1504	+ unsigned int off_dest = tgt->offset;
1423	1505	int count = 0;
1424	1506	struct dma_async_tx_descriptor *tx;
1425	1507	struct async_submit_ctl submit;
..	..	@@ -1431,24 +1513,30 @@
1431	1513	__func__, (unsigned long long)sh->sector, target);
1432	1514	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1433	1515
1434		- for (i = disks; i--; )
1435		- if (i != target)
	1516	+ for (i = disks; i--; ) {
	1517	+ if (i != target) {
	1518	+ off_srcs[count] = sh->dev[i].offset;
1436	1519	xor_srcs[count++] = sh->dev[i].page;
	1520	+ }
	1521	+ }
1437	1522
1438	1523	atomic_inc(&sh->count);
1439	1524
1440	1525	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST, NULL,
1441	1526	ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1442	1527	if (unlikely(count == 1))
1443		- tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	1528	+ tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
	1529	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1444	1530	else
1445		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	1531	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	1532	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1446	1533
1447	1534	return tx;
1448	1535	}
1449	1536
1450	1537	/* set_syndrome_sources - populate source buffers for gen_syndrome
1451	1538	* @srcs - (struct page *) array of size sh->disks
	1539	+ * @offs - (unsigned int) array of offset for each page
1452	1540	* @sh - stripe_head to parse
1453	1541	*
1454	1542	* Populates srcs in proper layout order for the stripe and returns the
..	..	@@ -1457,6 +1545,7 @@
1457	1545	* is recorded in srcs[count+1]].
1458	1546	*/
1459	1547	static int set_syndrome_sources(struct page **srcs,
	1548	+ unsigned int *offs,
1460	1549	struct stripe_head *sh,
1461	1550	int srctype)
1462	1551	{
..	..	@@ -1487,6 +1576,12 @@
1487	1576	srcs[slot] = sh->dev[i].orig_page;
1488	1577	else
1489	1578	srcs[slot] = sh->dev[i].page;
	1579	+ /*
	1580	+ * For R5_InJournal, PAGE_SIZE must be 4KB and will
	1581	+ * not shared page. In that case, dev[i].offset
	1582	+ * is 0.
	1583	+ */
	1584	+ offs[slot] = sh->dev[i].offset;
1490	1585	}
1491	1586	i = raid6_next_disk(i, disks);
1492	1587	} while (i != d0_idx);
..	..	@@ -1499,12 +1594,14 @@
1499	1594	{
1500	1595	int disks = sh->disks;
1501	1596	struct page **blocks = to_addr_page(percpu, 0);
	1597	+ unsigned int *offs = to_addr_offs(sh, percpu);
1502	1598	int target;
1503	1599	int qd_idx = sh->qd_idx;
1504	1600	struct dma_async_tx_descriptor *tx;
1505	1601	struct async_submit_ctl submit;
1506	1602	struct r5dev *tgt;
1507	1603	struct page *dest;
	1604	+ unsigned int dest_off;
1508	1605	int i;
1509	1606	int count;
1510	1607
..	..	@@ -1523,30 +1620,34 @@
1523	1620	tgt = &sh->dev[target];
1524	1621	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1525	1622	dest = tgt->page;
	1623	+ dest_off = tgt->offset;
1526	1624
1527	1625	atomic_inc(&sh->count);
1528	1626
1529	1627	if (target == qd_idx) {
1530		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
	1628	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1531	1629	blocks[count] = NULL; /* regenerating p is not necessary */
1532	1630	BUG_ON(blocks[count+1] != dest); /* q should already be set */
1533	1631	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1534	1632	ops_complete_compute, sh,
1535	1633	to_addr_conv(sh, percpu, 0));
1536		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	1634	+ tx = async_gen_syndrome(blocks, offs, count+2,
	1635	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1537	1636	} else {
1538	1637	/* Compute any data- or p-drive using XOR */
1539	1638	count = 0;
1540	1639	for (i = disks; i-- ; ) {
1541	1640	if (i == target \|\| i == qd_idx)
1542	1641	continue;
	1642	+ offs[count] = sh->dev[i].offset;
1543	1643	blocks[count++] = sh->dev[i].page;
1544	1644	}
1545	1645
1546	1646	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST,
1547	1647	NULL, ops_complete_compute, sh,
1548	1648	to_addr_conv(sh, percpu, 0));
1549		- tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	1649	+ tx = async_xor_offs(dest, dest_off, blocks, offs, count,
	1650	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1550	1651	}
1551	1652
1552	1653	return tx;
..	..	@@ -1565,6 +1666,7 @@
1565	1666	struct r5dev *tgt2 = &sh->dev[target2];
1566	1667	struct dma_async_tx_descriptor *tx;
1567	1668	struct page **blocks = to_addr_page(percpu, 0);
	1669	+ unsigned int *offs = to_addr_offs(sh, percpu);
1568	1670	struct async_submit_ctl submit;
1569	1671
1570	1672	BUG_ON(sh->batch_head);
..	..	@@ -1577,13 +1679,16 @@
1577	1679	/* we need to open-code set_syndrome_sources to handle the
1578	1680	* slot number conversion for 'faila' and 'failb'
1579	1681	*/
1580		- for (i = 0; i < disks ; i++)
	1682	+ for (i = 0; i < disks ; i++) {
	1683	+ offs[i] = 0;
1581	1684	blocks[i] = NULL;
	1685	+ }
1582	1686	count = 0;
1583	1687	i = d0_idx;
1584	1688	do {
1585	1689	int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1586	1690
	1691	+ offs[slot] = sh->dev[i].offset;
1587	1692	blocks[slot] = sh->dev[i].page;
1588	1693
1589	1694	if (i == target)
..	..	@@ -1608,10 +1713,12 @@
1608	1713	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1609	1714	ops_complete_compute, sh,
1610	1715	to_addr_conv(sh, percpu, 0));
1611		- return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1612		- STRIPE_SIZE, &submit);
	1716	+ return async_gen_syndrome(blocks, offs, syndrome_disks+2,
	1717	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1718	+ &submit);
1613	1719	} else {
1614	1720	struct page *dest;
	1721	+ unsigned int dest_off;
1615	1722	int data_target;
1616	1723	int qd_idx = sh->qd_idx;
1617	1724
..	..	@@ -1625,22 +1732,26 @@
1625	1732	for (i = disks; i-- ; ) {
1626	1733	if (i == data_target \|\| i == qd_idx)
1627	1734	continue;
	1735	+ offs[count] = sh->dev[i].offset;
1628	1736	blocks[count++] = sh->dev[i].page;
1629	1737	}
1630	1738	dest = sh->dev[data_target].page;
	1739	+ dest_off = sh->dev[data_target].offset;
1631	1740	init_async_submit(&submit,
1632	1741	ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST,
1633	1742	NULL, NULL, NULL,
1634	1743	to_addr_conv(sh, percpu, 0));
1635		- tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
	1744	+ tx = async_xor_offs(dest, dest_off, blocks, offs, count,
	1745	+ RAID5_STRIPE_SIZE(sh->raid_conf),
1636	1746	&submit);
1637	1747
1638		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
	1748	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1639	1749	init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1640	1750	ops_complete_compute, sh,
1641	1751	to_addr_conv(sh, percpu, 0));
1642		- return async_gen_syndrome(blocks, 0, count+2,
1643		- STRIPE_SIZE, &submit);
	1752	+ return async_gen_syndrome(blocks, offs, count+2,
	1753	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1754	+ &submit);
1644	1755	}
1645	1756	} else {
1646	1757	init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
..	..	@@ -1649,13 +1760,15 @@
1649	1760	if (failb == syndrome_disks) {
1650	1761	/* We're missing D+P. */
1651	1762	return async_raid6_datap_recov(syndrome_disks+2,
1652		- STRIPE_SIZE, faila,
1653		- blocks, &submit);
	1763	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1764	+ faila,
	1765	+ blocks, offs, &submit);
1654	1766	} else {
1655	1767	/* We're missing D+D. */
1656	1768	return async_raid6_2data_recov(syndrome_disks+2,
1657		- STRIPE_SIZE, faila, failb,
1658		- blocks, &submit);
	1769	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	1770	+ faila, failb,
	1771	+ blocks, offs, &submit);
1659	1772	}
1660	1773	}
1661	1774	}
..	..	@@ -1681,10 +1794,12 @@
1681	1794	{
1682	1795	int disks = sh->disks;
1683	1796	struct page **xor_srcs = to_addr_page(percpu, 0);
	1797	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
1684	1798	int count = 0, pd_idx = sh->pd_idx, i;
1685	1799	struct async_submit_ctl submit;
1686	1800
1687	1801	/* existing parity data subtracted */
	1802	+ unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1688	1803	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1689	1804
1690	1805	BUG_ON(sh->batch_head);
..	..	@@ -1694,15 +1809,23 @@
1694	1809	for (i = disks; i--; ) {
1695	1810	struct r5dev *dev = &sh->dev[i];
1696	1811	/* Only process blocks that are known to be uptodate */
1697		- if (test_bit(R5_InJournal, &dev->flags))
	1812	+ if (test_bit(R5_InJournal, &dev->flags)) {
	1813	+ /*
	1814	+ * For this case, PAGE_SIZE must be equal to 4KB and
	1815	+ * page offset is zero.
	1816	+ */
	1817	+ off_srcs[count] = dev->offset;
1698	1818	xor_srcs[count++] = dev->orig_page;
1699		- else if (test_bit(R5_Wantdrain, &dev->flags))
	1819	+ } else if (test_bit(R5_Wantdrain, &dev->flags)) {
	1820	+ off_srcs[count] = dev->offset;
1700	1821	xor_srcs[count++] = dev->page;
	1822	+ }
1701	1823	}
1702	1824
1703	1825	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_DROP_DST, tx,
1704	1826	ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1705		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	1827	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	1828	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1706	1829
1707	1830	return tx;
1708	1831	}
..	..	@@ -1712,17 +1835,19 @@
1712	1835	struct dma_async_tx_descriptor *tx)
1713	1836	{
1714	1837	struct page **blocks = to_addr_page(percpu, 0);
	1838	+ unsigned int *offs = to_addr_offs(sh, percpu);
1715	1839	int count;
1716	1840	struct async_submit_ctl submit;
1717	1841
1718	1842	pr_debug("%s: stripe %llu\n", __func__,
1719	1843	(unsigned long long)sh->sector);
1720	1844
1721		- count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
	1845	+ count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
1722	1846
1723	1847	init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_PQ_XOR_DST, tx,
1724	1848	ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1725		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	1849	+ tx = async_gen_syndrome(blocks, offs, count+2,
	1850	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1726	1851
1727	1852	return tx;
1728	1853	}
..	..	@@ -1763,7 +1888,7 @@
1763	1888	WARN_ON(dev->page != dev->orig_page);
1764	1889
1765	1890	while (wbi && wbi->bi_iter.bi_sector <
1766		- dev->sector + STRIPE_SECTORS) {
	1891	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
1767	1892	if (wbi->bi_opf & REQ_FUA)
1768	1893	set_bit(R5_WantFUA, &dev->flags);
1769	1894	if (wbi->bi_opf & REQ_SYNC)
..	..	@@ -1772,6 +1897,7 @@
1772	1897	set_bit(R5_Discard, &dev->flags);
1773	1898	else {
1774	1899	tx = async_copy_data(1, wbi, &dev->page,
	1900	+ dev->offset,
1775	1901	dev->sector, tx, sh,
1776	1902	r5c_is_writeback(conf->log));
1777	1903	if (dev->page != dev->orig_page &&
..	..	@@ -1781,7 +1907,7 @@
1781	1907	clear_bit(R5_OVERWRITE, &dev->flags);
1782	1908	}
1783	1909	}
1784		- wbi = r5_next_bio(wbi, dev->sector);
	1910	+ wbi = r5_next_bio(conf, wbi, dev->sector);
1785	1911	}
1786	1912
1787	1913	if (head_sh->batch_head) {
..	..	@@ -1851,9 +1977,11 @@
1851	1977	{
1852	1978	int disks = sh->disks;
1853	1979	struct page **xor_srcs;
	1980	+ unsigned int *off_srcs;
1854	1981	struct async_submit_ctl submit;
1855	1982	int count, pd_idx = sh->pd_idx, i;
1856	1983	struct page *xor_dest;
	1984	+ unsigned int off_dest;
1857	1985	int prexor = 0;
1858	1986	unsigned long flags;
1859	1987	int j = 0;
..	..	@@ -1878,24 +2006,31 @@
1878	2006	again:
1879	2007	count = 0;
1880	2008	xor_srcs = to_addr_page(percpu, j);
	2009	+ off_srcs = to_addr_offs(sh, percpu);
1881	2010	/* check if prexor is active which means only process blocks
1882	2011	* that are part of a read-modify-write (written)
1883	2012	*/
1884	2013	if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1885	2014	prexor = 1;
	2015	+ off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1886	2016	xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1887	2017	for (i = disks; i--; ) {
1888	2018	struct r5dev *dev = &sh->dev[i];
1889	2019	if (head_sh->dev[i].written \|\|
1890		- test_bit(R5_InJournal, &head_sh->dev[i].flags))
	2020	+ test_bit(R5_InJournal, &head_sh->dev[i].flags)) {
	2021	+ off_srcs[count] = dev->offset;
1891	2022	xor_srcs[count++] = dev->page;
	2023	+ }
1892	2024	}
1893	2025	} else {
1894	2026	xor_dest = sh->dev[pd_idx].page;
	2027	+ off_dest = sh->dev[pd_idx].offset;
1895	2028	for (i = disks; i--; ) {
1896	2029	struct r5dev *dev = &sh->dev[i];
1897		- if (i != pd_idx)
	2030	+ if (i != pd_idx) {
	2031	+ off_srcs[count] = dev->offset;
1898	2032	xor_srcs[count++] = dev->page;
	2033	+ }
1899	2034	}
1900	2035	}
1901	2036
..	..	@@ -1921,9 +2056,11 @@
1921	2056	}
1922	2057
1923	2058	if (unlikely(count == 1))
1924		- tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	2059	+ tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
	2060	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1925	2061	else
1926		- tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
	2062	+ tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	2063	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1927	2064	if (!last_stripe) {
1928	2065	j++;
1929	2066	sh = list_first_entry(&sh->batch_list, struct stripe_head,
..	..	@@ -1938,6 +2075,7 @@
1938	2075	{
1939	2076	struct async_submit_ctl submit;
1940	2077	struct page **blocks;
	2078	+ unsigned int *offs;
1941	2079	int count, i, j = 0;
1942	2080	struct stripe_head *head_sh = sh;
1943	2081	int last_stripe;
..	..	@@ -1962,6 +2100,7 @@
1962	2100
1963	2101	again:
1964	2102	blocks = to_addr_page(percpu, j);
	2103	+ offs = to_addr_offs(sh, percpu);
1965	2104
1966	2105	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1967	2106	synflags = SYNDROME_SRC_WRITTEN;
..	..	@@ -1971,7 +2110,7 @@
1971	2110	txflags = ASYNC_TX_ACK;
1972	2111	}
1973	2112
1974		- count = set_syndrome_sources(blocks, sh, synflags);
	2113	+ count = set_syndrome_sources(blocks, offs, sh, synflags);
1975	2114	last_stripe = !head_sh->batch_head \|\|
1976	2115	list_first_entry(&sh->batch_list,
1977	2116	struct stripe_head, batch_list) == head_sh;
..	..	@@ -1983,7 +2122,8 @@
1983	2122	} else
1984	2123	init_async_submit(&submit, 0, tx, NULL, NULL,
1985	2124	to_addr_conv(sh, percpu, j));
1986		- tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	2125	+ tx = async_gen_syndrome(blocks, offs, count+2,
	2126	+ RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1987	2127	if (!last_stripe) {
1988	2128	j++;
1989	2129	sh = list_first_entry(&sh->batch_list, struct stripe_head,
..	..	@@ -2010,7 +2150,9 @@
2010	2150	int pd_idx = sh->pd_idx;
2011	2151	int qd_idx = sh->qd_idx;
2012	2152	struct page *xor_dest;
	2153	+ unsigned int off_dest;
2013	2154	struct page **xor_srcs = to_addr_page(percpu, 0);
	2155	+ unsigned int *off_srcs = to_addr_offs(sh, percpu);
2014	2156	struct dma_async_tx_descriptor *tx;
2015	2157	struct async_submit_ctl submit;
2016	2158	int count;
..	..	@@ -2022,16 +2164,20 @@
2022	2164	BUG_ON(sh->batch_head);
2023	2165	count = 0;
2024	2166	xor_dest = sh->dev[pd_idx].page;
	2167	+ off_dest = sh->dev[pd_idx].offset;
	2168	+ off_srcs[count] = off_dest;
2025	2169	xor_srcs[count++] = xor_dest;
2026	2170	for (i = disks; i--; ) {
2027	2171	if (i == pd_idx \|\| i == qd_idx)
2028	2172	continue;
	2173	+ off_srcs[count] = sh->dev[i].offset;
2029	2174	xor_srcs[count++] = sh->dev[i].page;
2030	2175	}
2031	2176
2032	2177	init_async_submit(&submit, 0, NULL, NULL, NULL,
2033	2178	to_addr_conv(sh, percpu, 0));
2034		- tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
	2179	+ tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
	2180	+ RAID5_STRIPE_SIZE(sh->raid_conf),
2035	2181	&sh->ops.zero_sum_result, &submit);
2036	2182
2037	2183	atomic_inc(&sh->count);
..	..	@@ -2042,6 +2188,7 @@
2042	2188	static void ops_run_check_pq(struct stripe_head sh, struct raid5_percpu percpu, int checkp)
2043	2189	{
2044	2190	struct page **srcs = to_addr_page(percpu, 0);
	2191	+ unsigned int *offs = to_addr_offs(sh, percpu);
2045	2192	struct async_submit_ctl submit;
2046	2193	int count;
2047	2194
..	..	@@ -2049,15 +2196,16 @@
2049	2196	(unsigned long long)sh->sector, checkp);
2050	2197
2051	2198	BUG_ON(sh->batch_head);
2052		- count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
	2199	+ count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
2053	2200	if (!checkp)
2054	2201	srcs[count] = NULL;
2055	2202
2056	2203	atomic_inc(&sh->count);
2057	2204	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
2058	2205	sh, to_addr_conv(sh, percpu, 0));
2059		- async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
2060		- &sh->ops.zero_sum_result, percpu->spare_page, &submit);
	2206	+ async_syndrome_val(srcs, offs, count+2,
	2207	+ RAID5_STRIPE_SIZE(sh->raid_conf),
	2208	+ &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
2061	2209	}
2062	2210
2063	2211	static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
..	..	@@ -2069,9 +2217,8 @@
2069	2217	struct raid5_percpu *percpu;
2070	2218	unsigned long cpu;
2071	2219
2072		- cpu = get_cpu_light();
	2220	+ cpu = get_cpu();
2073	2221	percpu = per_cpu_ptr(conf->percpu, cpu);
2074		- spin_lock(&percpu->lock);
2075	2222	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
2076	2223	ops_run_biofill(sh);
2077	2224	overlap_clear++;
..	..	@@ -2130,12 +2277,14 @@
2130	2277	if (test_and_clear_bit(R5_Overlap, &dev->flags))
2131	2278	wake_up(&sh->raid_conf->wait_for_overlap);
2132	2279	}
2133		- spin_unlock(&percpu->lock);
2134		- put_cpu_light();
	2280	+ put_cpu();
2135	2281	}
2136	2282
2137	2283	static void free_stripe(struct kmem_cache sc, struct stripe_head sh)
2138	2284	{
	2285	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2286	+ kfree(sh->pages);
	2287	+#endif
2139	2288	if (sh->ppl_page)
2140	2289	__free_page(sh->ppl_page);
2141	2290	kmem_cache_free(sc, sh);
..	..	@@ -2169,9 +2318,15 @@
2169	2318	sh->ppl_page = alloc_page(gfp);
2170	2319	if (!sh->ppl_page) {
2171	2320	free_stripe(sc, sh);
2172		- sh = NULL;
	2321	+ return NULL;
2173	2322	}
2174	2323	}
	2324	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2325	+ if (init_stripe_shared_pages(sh, conf, disks)) {
	2326	+ free_stripe(sc, sh);
	2327	+ return NULL;
	2328	+ }
	2329	+#endif
2175	2330	}
2176	2331	return sh;
2177	2332	}
..	..	@@ -2228,10 +2383,13 @@
2228	2383	}
2229	2384
2230	2385	/**
2231		- * scribble_len - return the required size of the scribble region
2232		- * @num - total number of disks in the array
	2386	+ * scribble_alloc - allocate percpu scribble buffer for required size
	2387	+ * of the scribble region
	2388	+ * @percpu: from for_each_present_cpu() of the caller
	2389	+ * @num: total number of disks in the array
	2390	+ * @cnt: scribble objs count for required size of the scribble region
2233	2391	*
2234		- * The size must be enough to contain:
	2392	+ * The scribble buffer size must be enough to contain:
2235	2393	* 1/ a struct page pointer for each device in the array +2
2236	2394	* 2/ room to convert each entry in (1) to its corresponding dma
2237	2395	* (dma_map_page()) or page (page_address()) address.
..	..	@@ -2240,21 +2398,29 @@
2240	2398	* calculate over all devices (not just the data blocks), using zeros in place
2241	2399	* of the P and Q blocks.
2242	2400	*/
2243		-static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
	2401	+static int scribble_alloc(struct raid5_percpu *percpu,
	2402	+ int num, int cnt)
2244	2403	{
2245		- struct flex_array *ret;
2246		- size_t len;
	2404	+ size_t obj_size =
	2405	+ sizeof(struct page ) (num + 2) +
	2406	+ sizeof(addr_conv_t) * (num + 2) +
	2407	+ sizeof(unsigned int) * (num + 2);
	2408	+ void *scribble;
2247	2409
2248		- len = sizeof(struct page ) (num+2) + sizeof(addr_conv_t) * (num+2);
2249		- ret = flex_array_alloc(len, cnt, flags);
2250		- if (!ret)
2251		- return NULL;
2252		- /* always prealloc all elements, so no locking is required */
2253		- if (flex_array_prealloc(ret, 0, cnt, flags)) {
2254		- flex_array_free(ret);
2255		- return NULL;
2256		- }
2257		- return ret;
	2410	+ /*
	2411	+ * If here is in raid array suspend context, it is in memalloc noio
	2412	+ * context as well, there is no potential recursive memory reclaim
	2413	+ * I/Os with the GFP_KERNEL flag.
	2414	+ */
	2415	+ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
	2416	+ if (!scribble)
	2417	+ return -ENOMEM;
	2418	+
	2419	+ kvfree(percpu->scribble);
	2420	+
	2421	+ percpu->scribble = scribble;
	2422	+ percpu->scribble_obj_size = obj_size;
	2423	+ return 0;
2258	2424	}
2259	2425
2260	2426	static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
..	..	@@ -2272,23 +2438,17 @@
2272	2438	return 0;
2273	2439	mddev_suspend(conf->mddev);
2274	2440	get_online_cpus();
	2441	+
2275	2442	for_each_present_cpu(cpu) {
2276	2443	struct raid5_percpu *percpu;
2277		- struct flex_array *scribble;
2278	2444
2279	2445	percpu = per_cpu_ptr(conf->percpu, cpu);
2280		- scribble = scribble_alloc(new_disks,
2281		- new_sectors / STRIPE_SECTORS,
2282		- GFP_NOIO);
2283		-
2284		- if (scribble) {
2285		- flex_array_free(percpu->scribble);
2286		- percpu->scribble = scribble;
2287		- } else {
2288		- err = -ENOMEM;
	2446	+ err = scribble_alloc(percpu, new_disks,
	2447	+ new_sectors / RAID5_STRIPE_SECTORS(conf));
	2448	+ if (err)
2289	2449	break;
2290		- }
2291	2450	}
	2451	+
2292	2452	put_online_cpus();
2293	2453	mddev_resume(conf->mddev);
2294	2454	if (!err) {
..	..	@@ -2376,9 +2536,16 @@
2376	2536	osh = get_free_stripe(conf, hash);
2377	2537	unlock_device_hash_lock(conf, hash);
2378	2538
	2539	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2540	+ for (i = 0; i < osh->nr_pages; i++) {
	2541	+ nsh->pages[i] = osh->pages[i];
	2542	+ osh->pages[i] = NULL;
	2543	+ }
	2544	+#endif
2379	2545	for(i=0; i<conf->pool_size; i++) {
2380	2546	nsh->dev[i].page = osh->dev[i].page;
2381	2547	nsh->dev[i].orig_page = osh->dev[i].page;
	2548	+ nsh->dev[i].offset = osh->dev[i].offset;
2382	2549	}
2383	2550	nsh->hash_lock_index = hash;
2384	2551	free_stripe(conf->slab_cache, osh);
..	..	@@ -2427,14 +2594,33 @@
2427	2594	nsh = list_entry(newstripes.next, struct stripe_head, lru);
2428	2595	list_del_init(&nsh->lru);
2429	2596
	2597	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	2598	+ for (i = 0; i < nsh->nr_pages; i++) {
	2599	+ if (nsh->pages[i])
	2600	+ continue;
	2601	+ nsh->pages[i] = alloc_page(GFP_NOIO);
	2602	+ if (!nsh->pages[i])
	2603	+ err = -ENOMEM;
	2604	+ }
	2605	+
	2606	+ for (i = conf->raid_disks; i < newsize; i++) {
	2607	+ if (nsh->dev[i].page)
	2608	+ continue;
	2609	+ nsh->dev[i].page = raid5_get_dev_page(nsh, i);
	2610	+ nsh->dev[i].orig_page = nsh->dev[i].page;
	2611	+ nsh->dev[i].offset = raid5_get_page_offset(nsh, i);
	2612	+ }
	2613	+#else
2430	2614	for (i=conf->raid_disks; i < newsize; i++)
2431	2615	if (nsh->dev[i].page == NULL) {
2432	2616	struct page *p = alloc_page(GFP_NOIO);
2433	2617	nsh->dev[i].page = p;
2434	2618	nsh->dev[i].orig_page = p;
	2619	+ nsh->dev[i].offset = 0;
2435	2620	if (!p)
2436	2621	err = -ENOMEM;
2437	2622	}
	2623	+#endif
2438	2624	raid5_release_stripe(nsh);
2439	2625	}
2440	2626	/* critical section pass, GFP_NOIO no longer needed */
..	..	@@ -2518,10 +2704,10 @@
2518	2704	*/
2519	2705	pr_info_ratelimited(
2520	2706	"md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
2521		- mdname(conf->mddev), STRIPE_SECTORS,
	2707	+ mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),
2522	2708	(unsigned long long)s,
2523	2709	bdevname(rdev->bdev, b));
2524		- atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
	2710	+ atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);
2525	2711	clear_bit(R5_ReadError, &sh->dev[i].flags);
2526	2712	clear_bit(R5_ReWrite, &sh->dev[i].flags);
2527	2713	} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
..	..	@@ -2566,10 +2752,16 @@
2566	2752	(unsigned long long)s,
2567	2753	bdn);
2568	2754	} else if (atomic_read(&rdev->read_errors)
2569		- > conf->max_nr_stripes)
2570		- pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
2571		- mdname(conf->mddev), bdn);
2572		- else
	2755	+ > conf->max_nr_stripes) {
	2756	+ if (!test_bit(Faulty, &rdev->flags)) {
	2757	+ pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
	2758	+ mdname(conf->mddev),
	2759	+ atomic_read(&rdev->read_errors),
	2760	+ conf->max_nr_stripes);
	2761	+ pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
	2762	+ mdname(conf->mddev), bdn);
	2763	+ }
	2764	+ } else
2573	2765	retry = 1;
2574	2766	if (set_bad && test_bit(In_sync, &rdev->flags)
2575	2767	&& !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
..	..	@@ -2588,7 +2780,7 @@
2588	2780	if (!(set_bad
2589	2781	&& test_bit(In_sync, &rdev->flags)
2590	2782	&& rdev_set_badblocks(
2591		- rdev, sh->sector, STRIPE_SECTORS, 0)))
	2783	+ rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
2592	2784	md_error(conf->mddev, rdev);
2593	2785	}
2594	2786	}
..	..	@@ -2604,7 +2796,7 @@
2604	2796	struct stripe_head *sh = bi->bi_private;
2605	2797	struct r5conf *conf = sh->raid_conf;
2606	2798	int disks = sh->disks, i;
2607		- struct md_rdev *uninitialized_var(rdev);
	2799	+ struct md_rdev *rdev;
2608	2800	sector_t first_bad;
2609	2801	int bad_sectors;
2610	2802	int replacement = 0;
..	..	@@ -2640,7 +2832,7 @@
2640	2832	if (bi->bi_status)
2641	2833	md_error(conf->mddev, rdev);
2642	2834	else if (is_badblock(rdev, sh->sector,
2643		- STRIPE_SECTORS,
	2835	+ RAID5_STRIPE_SECTORS(conf),
2644	2836	&first_bad, &bad_sectors))
2645	2837	set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2646	2838	} else {
..	..	@@ -2652,7 +2844,7 @@
2652	2844	set_bit(MD_RECOVERY_NEEDED,
2653	2845	&rdev->mddev->recovery);
2654	2846	} else if (is_badblock(rdev, sh->sector,
2655		- STRIPE_SECTORS,
	2847	+ RAID5_STRIPE_SECTORS(conf),
2656	2848	&first_bad, &bad_sectors)) {
2657	2849	set_bit(R5_MadeGood, &sh->dev[i].flags);
2658	2850	if (test_bit(R5_ReadError, &sh->dev[i].flags))
..	..	@@ -2672,10 +2864,10 @@
2672	2864	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2673	2865	clear_bit(R5_LOCKED, &sh->dev[i].flags);
2674	2866	set_bit(STRIPE_HANDLE, &sh->state);
2675		- raid5_release_stripe(sh);
2676	2867
2677	2868	if (sh->batch_head && sh != sh->batch_head)
2678	2869	raid5_release_stripe(sh->batch_head);
	2870	+ raid5_release_stripe(sh);
2679	2871	}
2680	2872
2681	2873	static void raid5_error(struct mddev mddev, struct md_rdev rdev)
..	..	@@ -2685,22 +2877,31 @@
2685	2877	unsigned long flags;
2686	2878	pr_debug("raid456: error called\n");
2687	2879
	2880	+ pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n",
	2881	+ mdname(mddev), bdevname(rdev->bdev, b));
	2882	+
2688	2883	spin_lock_irqsave(&conf->device_lock, flags);
2689	2884	set_bit(Faulty, &rdev->flags);
2690	2885	clear_bit(In_sync, &rdev->flags);
2691	2886	mddev->degraded = raid5_calc_degraded(conf);
	2887	+
	2888	+ if (has_failed(conf)) {
	2889	+ set_bit(MD_BROKEN, &conf->mddev->flags);
	2890	+ conf->recovery_disabled = mddev->recovery_disabled;
	2891	+
	2892	+ pr_crit("md/raid:%s: Cannot continue operation (%d/%d failed).\n",
	2893	+ mdname(mddev), mddev->degraded, conf->raid_disks);
	2894	+ } else {
	2895	+ pr_crit("md/raid:%s: Operation continuing on %d devices.\n",
	2896	+ mdname(mddev), conf->raid_disks - mddev->degraded);
	2897	+ }
	2898	+
2692	2899	spin_unlock_irqrestore(&conf->device_lock, flags);
2693	2900	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2694	2901
2695	2902	set_bit(Blocked, &rdev->flags);
2696	2903	set_mask_bits(&mddev->sb_flags, 0,
2697	2904	BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
2698		- pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
2699		- "md/raid:%s: Operation continuing on %d devices.\n",
2700		- mdname(mddev),
2701		- bdevname(rdev->bdev, b),
2702		- mdname(mddev),
2703		- conf->raid_disks - mddev->degraded);
2704	2905	r5c_update_on_rdev_error(mddev, rdev);
2705	2906	}
2706	2907
..	..	@@ -3274,13 +3475,13 @@
3274	3475	/* check if page is covered */
3275	3476	sector_t sector = sh->dev[dd_idx].sector;
3276	3477	for (bi=sh->dev[dd_idx].towrite;
3277		- sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
	3478	+ sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
3278	3479	bi && bi->bi_iter.bi_sector <= sector;
3279		- bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
	3480	+ bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
3280	3481	if (bio_end_sector(bi) >= sector)
3281	3482	sector = bio_end_sector(bi);
3282	3483	}
3283		- if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
	3484	+ if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
3284	3485	if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
3285	3486	sh->overwrite_disks++;
3286	3487	}
..	..	@@ -3305,7 +3506,7 @@
3305	3506	set_bit(STRIPE_BITMAP_PENDING, &sh->state);
3306	3507	spin_unlock_irq(&sh->stripe_lock);
3307	3508	md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
3308		- STRIPE_SECTORS, 0);
	3509	+ RAID5_STRIPE_SECTORS(conf), 0);
3309	3510	spin_lock_irq(&sh->stripe_lock);
3310	3511	clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
3311	3512	if (!sh->batch_head) {
..	..	@@ -3367,7 +3568,7 @@
3367	3568	if (!rdev_set_badblocks(
3368	3569	rdev,
3369	3570	sh->sector,
3370		- STRIPE_SECTORS, 0))
	3571	+ RAID5_STRIPE_SECTORS(conf), 0))
3371	3572	md_error(conf->mddev, rdev);
3372	3573	rdev_dec_pending(rdev, conf->mddev);
3373	3574	}
..	..	@@ -3387,8 +3588,8 @@
3387	3588	wake_up(&conf->wait_for_overlap);
3388	3589
3389	3590	while (bi && bi->bi_iter.bi_sector <
3390		- sh->dev[i].sector + STRIPE_SECTORS) {
3391		- struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
	3591	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
	3592	+ struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
3392	3593
3393	3594	md_write_end(conf->mddev);
3394	3595	bio_io_error(bi);
..	..	@@ -3396,7 +3597,7 @@
3396	3597	}
3397	3598	if (bitmap_end)
3398	3599	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3399		- STRIPE_SECTORS, 0, 0);
	3600	+ RAID5_STRIPE_SECTORS(conf), 0, 0);
3400	3601	bitmap_end = 0;
3401	3602	/* and fail all 'written' */
3402	3603	bi = sh->dev[i].written;
..	..	@@ -3408,8 +3609,8 @@
3408	3609
3409	3610	if (bi) bitmap_end = 1;
3410	3611	while (bi && bi->bi_iter.bi_sector <
3411		- sh->dev[i].sector + STRIPE_SECTORS) {
3412		- struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
	3612	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
	3613	+ struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
3413	3614
3414	3615	md_write_end(conf->mddev);
3415	3616	bio_io_error(bi);
..	..	@@ -3432,9 +3633,9 @@
3432	3633	if (bi)
3433	3634	s->to_read--;
3434	3635	while (bi && bi->bi_iter.bi_sector <
3435		- sh->dev[i].sector + STRIPE_SECTORS) {
	3636	+ sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3436	3637	struct bio *nextbi =
3437		- r5_next_bio(bi, sh->dev[i].sector);
	3638	+ r5_next_bio(conf, bi, sh->dev[i].sector);
3438	3639
3439	3640	bio_io_error(bi);
3440	3641	bi = nextbi;
..	..	@@ -3442,7 +3643,7 @@
3442	3643	}
3443	3644	if (bitmap_end)
3444	3645	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3445		- STRIPE_SECTORS, 0, 0);
	3646	+ RAID5_STRIPE_SECTORS(conf), 0, 0);
3446	3647	/* If we were in the middle of a write the parity block might
3447	3648	* still be locked - so just clear all R5_LOCKED flags
3448	3649	*/
..	..	@@ -3487,14 +3688,14 @@
3487	3688	&& !test_bit(Faulty, &rdev->flags)
3488	3689	&& !test_bit(In_sync, &rdev->flags)
3489	3690	&& !rdev_set_badblocks(rdev, sh->sector,
3490		- STRIPE_SECTORS, 0))
	3691	+ RAID5_STRIPE_SECTORS(conf), 0))
3491	3692	abort = 1;
3492	3693	rdev = rcu_dereference(conf->disks[i].replacement);
3493	3694	if (rdev
3494	3695	&& !test_bit(Faulty, &rdev->flags)
3495	3696	&& !test_bit(In_sync, &rdev->flags)
3496	3697	&& !rdev_set_badblocks(rdev, sh->sector,
3497		- STRIPE_SECTORS, 0))
	3698	+ RAID5_STRIPE_SECTORS(conf), 0))
3498	3699	abort = 1;
3499	3700	}
3500	3701	rcu_read_unlock();
..	..	@@ -3502,7 +3703,7 @@
3502	3703	conf->recovery_disabled =
3503	3704	conf->mddev->recovery_disabled;
3504	3705	}
3505		- md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
	3706	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);
3506	3707	}
3507	3708
3508	3709	static int want_replace(struct stripe_head *sh, int disk_idx)
..	..	@@ -3529,6 +3730,7 @@
3529	3730	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3530	3731	&sh->dev[s->failed_num[1]] };
3531	3732	int i;
	3733	+ bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
3532	3734
3533	3735
3534	3736	if (test_bit(R5_LOCKED, &dev->flags) \|\|
..	..	@@ -3587,18 +3789,27 @@
3587	3789	* devices must be read.
3588	3790	*/
3589	3791	return 1;
	3792	+
	3793	+ if (s->failed >= 2 &&
	3794	+ (fdev[i]->towrite \|\|
	3795	+ s->failed_num[i] == sh->pd_idx \|\|
	3796	+ s->failed_num[i] == sh->qd_idx) &&
	3797	+ !test_bit(R5_UPTODATE, &fdev[i]->flags))
	3798	+ /* In max degraded raid6, If the failed disk is P, Q,
	3799	+ * or we want to read the failed disk, we need to do
	3800	+ * reconstruct-write.
	3801	+ */
	3802	+ force_rcw = true;
3590	3803	}
3591	3804
3592		- /* If we are forced to do a reconstruct-write, either because
3593		- * the current RAID6 implementation only supports that, or
3594		- * because parity cannot be trusted and we are currently
3595		- * recovering it, there is extra need to be careful.
	3805	+ /* If we are forced to do a reconstruct-write, because parity
	3806	+ * cannot be trusted and we are currently recovering it, there
	3807	+ * is extra need to be careful.
3596	3808	* If one of the devices that we would need to read, because
3597	3809	* it is not being overwritten (and maybe not written at all)
3598	3810	* is missing/faulty, then we need to read everything we can.
3599	3811	*/
3600		- if (sh->raid_conf->level != 6 &&
3601		- sh->raid_conf->rmw_level != PARITY_DISABLE_RMW &&
	3812	+ if (!force_rcw &&
3602	3813	sh->sector < sh->raid_conf->mddev->recovery_cp)
3603	3814	/* reconstruct-write isn't being forced */
3604	3815	return 0;
..	..	@@ -3702,7 +3913,7 @@
3702	3913	return 0;
3703	3914	}
3704	3915
3705		-/**
	3916	+/*
3706	3917	* handle_stripe_fill - read or compute data to satisfy pending requests.
3707	3918	*/
3708	3919	static void handle_stripe_fill(struct stripe_head *sh,
..	..	@@ -3725,7 +3936,7 @@
3725	3936	* back cache (prexor with orig_page, and then xor with
3726	3937	* page) in the read path
3727	3938	*/
3728		- if (s->injournal && s->failed) {
	3939	+ if (s->to_read && s->injournal && s->failed) {
3729	3940	if (test_bit(STRIPE_R5C_CACHING, &sh->state))
3730	3941	r5c_make_stripe_write_out(sh);
3731	3942	goto out;
..	..	@@ -3777,14 +3988,14 @@
3777	3988	wbi = dev->written;
3778	3989	dev->written = NULL;
3779	3990	while (wbi && wbi->bi_iter.bi_sector <
3780		- dev->sector + STRIPE_SECTORS) {
3781		- wbi2 = r5_next_bio(wbi, dev->sector);
	3991	+ dev->sector + RAID5_STRIPE_SECTORS(conf)) {
	3992	+ wbi2 = r5_next_bio(conf, wbi, dev->sector);
3782	3993	md_write_end(conf->mddev);
3783	3994	bio_endio(wbi);
3784	3995	wbi = wbi2;
3785	3996	}
3786	3997	md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3787		- STRIPE_SECTORS,
	3998	+ RAID5_STRIPE_SECTORS(conf),
3788	3999	!test_bit(STRIPE_DEGRADED, &sh->state),
3789	4000	0);
3790	4001	if (head_sh->batch_head) {
..	..	@@ -3968,10 +4179,8 @@
3968	4179	set_bit(R5_LOCKED, &dev->flags);
3969	4180	set_bit(R5_Wantread, &dev->flags);
3970	4181	s->locked++;
3971		- } else {
	4182	+ } else
3972	4183	set_bit(STRIPE_DELAYED, &sh->state);
3973		- set_bit(STRIPE_HANDLE, &sh->state);
3974		- }
3975	4184	}
3976	4185	}
3977	4186	}
..	..	@@ -3996,10 +4205,8 @@
3996	4205	set_bit(R5_Wantread, &dev->flags);
3997	4206	s->locked++;
3998	4207	qread++;
3999		- } else {
	4208	+ } else
4000	4209	set_bit(STRIPE_DELAYED, &sh->state);
4001		- set_bit(STRIPE_HANDLE, &sh->state);
4002		- }
4003	4210	}
4004	4211	}
4005	4212	if (rcw && conf->mddev->queue)
..	..	@@ -4049,7 +4256,7 @@
4049	4256	break;
4050	4257	}
4051	4258	dev = &sh->dev[s->failed_num[0]];
4052		- /* fall through */
	4259	+ fallthrough;
4053	4260	case check_state_compute_result:
4054	4261	sh->check_state = check_state_idle;
4055	4262	if (!dev)
..	..	@@ -4091,7 +4298,7 @@
4091	4298	*/
4092	4299	set_bit(STRIPE_INSYNC, &sh->state);
4093	4300	else {
4094		- atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
	4301	+ atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
4095	4302	if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
4096	4303	/* don't try to repair!! */
4097	4304	set_bit(STRIPE_INSYNC, &sh->state);
..	..	@@ -4099,7 +4306,7 @@
4099	4306	"%llu-%llu\n", mdname(conf->mddev),
4100	4307	(unsigned long long) sh->sector,
4101	4308	(unsigned long long) sh->sector +
4102		- STRIPE_SECTORS);
	4309	+ RAID5_STRIPE_SECTORS(conf));
4103	4310	} else {
4104	4311	sh->check_state = check_state_compute_run;
4105	4312	set_bit(STRIPE_COMPUTE_RUN, &sh->state);
..	..	@@ -4180,7 +4387,7 @@
4180	4387
4181	4388	/* we have 2-disk failure */
4182	4389	BUG_ON(s->failed != 2);
4183		- /* fall through */
	4390	+ fallthrough;
4184	4391	case check_state_compute_result:
4185	4392	sh->check_state = check_state_idle;
4186	4393
..	..	@@ -4256,7 +4463,7 @@
4256	4463	*/
4257	4464	}
4258	4465	} else {
4259		- atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
	4466	+ atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
4260	4467	if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
4261	4468	/* don't try to repair!! */
4262	4469	set_bit(STRIPE_INSYNC, &sh->state);
..	..	@@ -4264,7 +4471,7 @@
4264	4471	"%llu-%llu\n", mdname(conf->mddev),
4265	4472	(unsigned long long) sh->sector,
4266	4473	(unsigned long long) sh->sector +
4267		- STRIPE_SECTORS);
	4474	+ RAID5_STRIPE_SECTORS(conf));
4268	4475	} else {
4269	4476	int *target = &sh->ops.target;
4270	4477
..	..	@@ -4335,7 +4542,8 @@
4335	4542	/* place all the copies on one channel */
4336	4543	init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
4337	4544	tx = async_memcpy(sh2->dev[dd_idx].page,
4338		- sh->dev[i].page, 0, 0, STRIPE_SIZE,
	4545	+ sh->dev[i].page, sh2->dev[dd_idx].offset,
	4546	+ sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
4339	4547	&submit);
4340	4548
4341	4549	set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
..	..	@@ -4434,8 +4642,8 @@
4434	4642	*/
4435	4643	rdev = rcu_dereference(conf->disks[i].replacement);
4436	4644	if (rdev && !test_bit(Faulty, &rdev->flags) &&
4437		- rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
4438		- !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	4645	+ rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
	4646	+ !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4439	4647	&first_bad, &bad_sectors))
4440	4648	set_bit(R5_ReadRepl, &dev->flags);
4441	4649	else {
..	..	@@ -4449,7 +4657,7 @@
4449	4657	if (rdev && test_bit(Faulty, &rdev->flags))
4450	4658	rdev = NULL;
4451	4659	if (rdev) {
4452		- is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
	4660	+ is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4453	4661	&first_bad, &bad_sectors);
4454	4662	if (s->blocked_rdev == NULL
4455	4663	&& (test_bit(Blocked, &rdev->flags)
..	..	@@ -4476,7 +4684,7 @@
4476	4684	}
4477	4685	} else if (test_bit(In_sync, &rdev->flags))
4478	4686	set_bit(R5_Insync, &dev->flags);
4479		- else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
	4687	+ else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
4480	4688	/* in sync if before recovery_offset */
4481	4689	set_bit(R5_Insync, &dev->flags);
4482	4690	else if (test_bit(R5_UPTODATE, &dev->flags) &&
..	..	@@ -4565,12 +4773,12 @@
4565	4773	rcu_read_unlock();
4566	4774	}
4567	4775
	4776	+/*
	4777	+ * Return '1' if this is a member of batch, or '0' if it is a lone stripe or
	4778	+ * a head which can now be handled.
	4779	+ */
4568	4780	static int clear_batch_ready(struct stripe_head *sh)
4569	4781	{
4570		- /* Return '1' if this is a member of batch, or
4571		- * '0' if it is a lone stripe or a head which can now be
4572		- * handled.
4573		- */
4574	4782	struct stripe_head *tmp;
4575	4783	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
4576	4784	return (sh->batch_head && sh->batch_head != sh);
..	..	@@ -4620,7 +4828,6 @@
4620	4828	(1 << STRIPE_FULL_WRITE) \|
4621	4829	(1 << STRIPE_BIOFILL_RUN) \|
4622	4830	(1 << STRIPE_COMPUTE_RUN) \|
4623		- (1 << STRIPE_OPS_REQ_PENDING) \|
4624	4831	(1 << STRIPE_DISCARD) \|
4625	4832	(1 << STRIPE_BATCH_READY) \|
4626	4833	(1 << STRIPE_BATCH_ERR) \|
..	..	@@ -4675,15 +4882,20 @@
4675	4882	struct r5dev pdev, qdev;
4676	4883
4677	4884	clear_bit(STRIPE_HANDLE, &sh->state);
	4885	+
	4886	+ /*
	4887	+ * handle_stripe should not continue handle the batched stripe, only
	4888	+ * the head of batch list or lone stripe can continue. Otherwise we
	4889	+ * could see break_stripe_batch_list warns about the STRIPE_ACTIVE
	4890	+ * is set for the batched stripe.
	4891	+ */
	4892	+ if (clear_batch_ready(sh))
	4893	+ return;
	4894	+
4678	4895	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4679	4896	/* already being handled, ensure it gets handled
4680	4897	* again when current action finishes */
4681	4898	set_bit(STRIPE_HANDLE, &sh->state);
4682		- return;
4683		- }
4684		-
4685		- if (clear_batch_ready(sh) ) {
4686		- clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4687	4899	return;
4688	4900	}
4689	4901
..	..	@@ -4920,7 +5132,7 @@
4920	5132	if ((s.syncing \|\| s.replacing) && s.locked == 0 &&
4921	5133	!test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4922	5134	test_bit(STRIPE_INSYNC, &sh->state)) {
4923		- md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	5135	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
4924	5136	clear_bit(STRIPE_SYNCING, &sh->state);
4925	5137	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
4926	5138	wake_up(&conf->wait_for_overlap);
..	..	@@ -4939,14 +5151,11 @@
4939	5151	if (!test_bit(R5_ReWrite, &dev->flags)) {
4940	5152	set_bit(R5_Wantwrite, &dev->flags);
4941	5153	set_bit(R5_ReWrite, &dev->flags);
4942		- set_bit(R5_LOCKED, &dev->flags);
4943		- s.locked++;
4944		- } else {
	5154	+ } else
4945	5155	/* let's read it back */
4946	5156	set_bit(R5_Wantread, &dev->flags);
4947		- set_bit(R5_LOCKED, &dev->flags);
4948		- s.locked++;
4949		- }
	5157	+ set_bit(R5_LOCKED, &dev->flags);
	5158	+ s.locked++;
4950	5159	}
4951	5160	}
4952	5161
..	..	@@ -4988,7 +5197,7 @@
4988	5197	clear_bit(STRIPE_EXPAND_READY, &sh->state);
4989	5198	atomic_dec(&conf->reshape_stripes);
4990	5199	wake_up(&conf->wait_for_overlap);
4991		- md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	5200	+ md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
4992	5201	}
4993	5202
4994	5203	if (s.expanding && s.locked == 0 &&
..	..	@@ -5018,14 +5227,14 @@
5018	5227	/* We own a safe reference to the rdev */
5019	5228	rdev = conf->disks[i].rdev;
5020	5229	if (!rdev_set_badblocks(rdev, sh->sector,
5021		- STRIPE_SECTORS, 0))
	5230	+ RAID5_STRIPE_SECTORS(conf), 0))
5022	5231	md_error(conf->mddev, rdev);
5023	5232	rdev_dec_pending(rdev, conf->mddev);
5024	5233	}
5025	5234	if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
5026	5235	rdev = conf->disks[i].rdev;
5027	5236	rdev_clear_badblocks(rdev, sh->sector,
5028		- STRIPE_SECTORS, 0);
	5237	+ RAID5_STRIPE_SECTORS(conf), 0);
5029	5238	rdev_dec_pending(rdev, conf->mddev);
5030	5239	}
5031	5240	if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
..	..	@@ -5034,7 +5243,7 @@
5034	5243	/* rdev have been moved down */
5035	5244	rdev = conf->disks[i].rdev;
5036	5245	rdev_clear_badblocks(rdev, sh->sector,
5037		- STRIPE_SECTORS, 0);
	5246	+ RAID5_STRIPE_SECTORS(conf), 0);
5038	5247	rdev_dec_pending(rdev, conf->mddev);
5039	5248	}
5040	5249	}
..	..	@@ -5090,28 +5299,6 @@
5090	5299	hash = sh->hash_lock_index;
5091	5300	__release_stripe(conf, sh, &temp_inactive_list[hash]);
5092	5301	}
5093		-}
5094		-
5095		-static int raid5_congested(struct mddev *mddev, int bits)
5096		-{
5097		- struct r5conf *conf = mddev->private;
5098		-
5099		- /* No difference between reads and writes. Just check
5100		- * how busy the stripe_cache is
5101		- */
5102		-
5103		- if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
5104		- return 1;
5105		-
5106		- /* Also checks whether there is pressure on r5cache log space */
5107		- if (test_bit(R5C_LOG_TIGHT, &conf->cache_state))
5108		- return 1;
5109		- if (conf->quiesce)
5110		- return 1;
5111		- if (atomic_read(&conf->empty_inactive_list_nr))
5112		- return 1;
5113		-
5114		- return 0;
5115	5302	}
5116	5303
5117	5304	static int in_chunk_boundary(struct mddev mddev, struct bio bio)
..	..	@@ -5259,7 +5446,6 @@
5259	5446	rcu_read_unlock();
5260	5447	raid_bio->bi_next = (void*)rdev;
5261	5448	bio_set_dev(align_bi, rdev->bdev);
5262		- bio_clear_flag(align_bi, BIO_SEG_VALID);
5263	5449
5264	5450	if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
5265	5451	bio_sectors(align_bi),
..	..	@@ -5283,7 +5469,7 @@
5283	5469	trace_block_bio_remap(align_bi->bi_disk->queue,
5284	5470	align_bi, disk_devt(mddev->gendisk),
5285	5471	raid_bio->bi_iter.bi_sector);
5286		- generic_make_request(align_bi);
	5472	+ submit_bio_noacct(align_bi);
5287	5473	return 1;
5288	5474	} else {
5289	5475	rcu_read_unlock();
..	..	@@ -5303,7 +5489,7 @@
5303	5489	struct r5conf *conf = mddev->private;
5304	5490	split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
5305	5491	bio_chain(split, raid_bio);
5306		- generic_make_request(raid_bio);
	5492	+ submit_bio_noacct(raid_bio);
5307	5493	raid_bio = split;
5308	5494	}
5309	5495
..	..	@@ -5499,8 +5685,8 @@
5499	5685	/* Skip discard while reshape is happening */
5500	5686	return;
5501	5687
5502		- logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
5503		- last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
	5688	+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
	5689	+ last_sector = bio_end_sector(bi);
5504	5690
5505	5691	bi->bi_next = NULL;
5506	5692
..	..	@@ -5514,7 +5700,7 @@
5514	5700	last_sector *= conf->chunk_sectors;
5515	5701
5516	5702	for (; logical_sector < last_sector;
5517		- logical_sector += STRIPE_SECTORS) {
	5703	+ logical_sector += RAID5_STRIPE_SECTORS(conf)) {
5518	5704	DEFINE_WAIT(w);
5519	5705	int d;
5520	5706	again:
..	..	@@ -5559,7 +5745,7 @@
5559	5745	d++)
5560	5746	md_bitmap_startwrite(mddev->bitmap,
5561	5747	sh->sector,
5562		- STRIPE_SECTORS,
	5748	+ RAID5_STRIPE_SECTORS(conf),
5563	5749	0);
5564	5750	sh->bm_seq = conf->seq_flush + 1;
5565	5751	set_bit(STRIPE_BIT_DELAY, &sh->state);
..	..	@@ -5624,12 +5810,12 @@
5624	5810	return true;
5625	5811	}
5626	5812
5627		- logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	5813	+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
5628	5814	last_sector = bio_end_sector(bi);
5629	5815	bi->bi_next = NULL;
5630	5816
5631	5817	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
5632		- for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
	5818	+ for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
5633	5819	int previous;
5634	5820	int seq;
5635	5821
..	..	@@ -5727,8 +5913,7 @@
5727	5913	do_flush = false;
5728	5914	}
5729	5915
5730		- if (!sh->batch_head \|\| sh == sh->batch_head)
5731		- set_bit(STRIPE_HANDLE, &sh->state);
	5916	+ set_bit(STRIPE_HANDLE, &sh->state);
5732	5917	clear_bit(STRIPE_DELAYED, &sh->state);
5733	5918	if ((!sh->batch_head \|\| sh == sh->batch_head) &&
5734	5919	(bi->bi_opf & REQ_SYNC) &&
..	..	@@ -5793,7 +5978,7 @@
5793	5978	sector_div(sector_nr, new_data_disks);
5794	5979	if (sector_nr) {
5795	5980	mddev->curr_resync_completed = sector_nr;
5796		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	5981	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
5797	5982	*skipped = 1;
5798	5983	retn = sector_nr;
5799	5984	goto finish;
..	..	@@ -5907,11 +6092,11 @@
5907	6092	conf->reshape_safe = mddev->reshape_position;
5908	6093	spin_unlock_irq(&conf->device_lock);
5909	6094	wake_up(&conf->wait_for_overlap);
5910		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	6095	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
5911	6096	}
5912	6097
5913	6098	INIT_LIST_HEAD(&stripes);
5914		- for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
	6099	+ for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {
5915	6100	int j;
5916	6101	int skipped_disk = 0;
5917	6102	sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
..	..	@@ -5932,7 +6117,7 @@
5932	6117	skipped_disk = 1;
5933	6118	continue;
5934	6119	}
5935		- memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
	6120	+ memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
5936	6121	set_bit(R5_Expanded, &sh->dev[j].flags);
5937	6122	set_bit(R5_UPTODATE, &sh->dev[j].flags);
5938	6123	}
..	..	@@ -5967,7 +6152,7 @@
5967	6152	set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
5968	6153	set_bit(STRIPE_HANDLE, &sh->state);
5969	6154	raid5_release_stripe(sh);
5970		- first_sector += STRIPE_SECTORS;
	6155	+ first_sector += RAID5_STRIPE_SECTORS(conf);
5971	6156	}
5972	6157	/* Now that the sources are clearly marked, we can release
5973	6158	* the destination stripes
..	..	@@ -6014,7 +6199,7 @@
6014	6199	conf->reshape_safe = mddev->reshape_position;
6015	6200	spin_unlock_irq(&conf->device_lock);
6016	6201	wake_up(&conf->wait_for_overlap);
6017		- sysfs_notify(&mddev->kobj, NULL, "sync_completed");
	6202	+ sysfs_notify_dirent_safe(mddev->sysfs_completed);
6018	6203	}
6019	6204	ret:
6020	6205	return retn;
..	..	@@ -6073,11 +6258,12 @@
6073	6258	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
6074	6259	!conf->fullsync &&
6075	6260	!md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
6076		- sync_blocks >= STRIPE_SECTORS) {
	6261	+ sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {
6077	6262	/* we can skip this block, and probably more */
6078		- sync_blocks /= STRIPE_SECTORS;
	6263	+ do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));
6079	6264	*skipped = 1;
6080		- return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	6265	+ /* keep things rounded to whole stripes */
	6266	+ return sync_blocks * RAID5_STRIPE_SECTORS(conf);
6081	6267	}
6082	6268
6083	6269	md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
..	..	@@ -6110,7 +6296,7 @@
6110	6296
6111	6297	raid5_release_stripe(sh);
6112	6298
6113		- return STRIPE_SECTORS;
	6299	+ return RAID5_STRIPE_SECTORS(conf);
6114	6300	}
6115	6301
6116	6302	static int retry_aligned_read(struct r5conf conf, struct bio raid_bio,
..	..	@@ -6133,14 +6319,14 @@
6133	6319	int handled = 0;
6134	6320
6135	6321	logical_sector = raid_bio->bi_iter.bi_sector &
6136		- ~((sector_t)STRIPE_SECTORS-1);
	6322	+ ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
6137	6323	sector = raid5_compute_sector(conf, logical_sector,
6138	6324	0, &dd_idx, NULL);
6139	6325	last_sector = bio_end_sector(raid_bio);
6140	6326
6141	6327	for (; logical_sector < last_sector;
6142		- logical_sector += STRIPE_SECTORS,
6143		- sector += STRIPE_SECTORS,
	6328	+ logical_sector += RAID5_STRIPE_SECTORS(conf),
	6329	+ sector += RAID5_STRIPE_SECTORS(conf),
6144	6330	scnt++) {
6145	6331
6146	6332	if (scnt < offset)
..	..	@@ -6179,6 +6365,8 @@
6179	6365	static int handle_active_stripes(struct r5conf *conf, int group,
6180	6366	struct r5worker *worker,
6181	6367	struct list_head *temp_inactive_list)
	6368	+ __releases(&conf->device_lock)
	6369	+ __acquires(&conf->device_lock)
6182	6370	{
6183	6371	struct stripe_head batch[MAX_STRIPE_BATCH], sh;
6184	6372	int i, batch_size = 0, hash;
..	..	@@ -6331,7 +6519,18 @@
6331	6519	spin_unlock_irq(&conf->device_lock);
6332	6520	md_check_recovery(mddev);
6333	6521	spin_lock_irq(&conf->device_lock);
	6522	+
	6523	+ /*
	6524	+ * Waiting on MD_SB_CHANGE_PENDING below may deadlock
	6525	+ * seeing md_check_recovery() is needed to clear
	6526	+ * the flag when using mdmon.
	6527	+ */
	6528	+ continue;
6334	6529	}
	6530	+
	6531	+ wait_event_lock_irq(mddev->sb_wait,
	6532	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
	6533	+ conf->device_lock);
6335	6534	}
6336	6535	pr_debug("%d stripes handled\n", handled);
6337	6536
..	..	@@ -6471,6 +6670,100 @@
6471	6670	raid5_show_rmw_level,
6472	6671	raid5_store_rmw_level);
6473	6672
	6673	+static ssize_t
	6674	+raid5_show_stripe_size(struct mddev mddev, char page)
	6675	+{
	6676	+ struct r5conf *conf;
	6677	+ int ret = 0;
	6678	+
	6679	+ spin_lock(&mddev->lock);
	6680	+ conf = mddev->private;
	6681	+ if (conf)
	6682	+ ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));
	6683	+ spin_unlock(&mddev->lock);
	6684	+ return ret;
	6685	+}
	6686	+
	6687	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	6688	+static ssize_t
	6689	+raid5_store_stripe_size(struct mddev mddev, const char page, size_t len)
	6690	+{
	6691	+ struct r5conf *conf;
	6692	+ unsigned long new;
	6693	+ int err;
	6694	+ int size;
	6695	+
	6696	+ if (len >= PAGE_SIZE)
	6697	+ return -EINVAL;
	6698	+ if (kstrtoul(page, 10, &new))
	6699	+ return -EINVAL;
	6700	+
	6701	+ /*
	6702	+ * The value should not be bigger than PAGE_SIZE. It requires to
	6703	+ * be multiple of DEFAULT_STRIPE_SIZE and the value should be power
	6704	+ * of two.
	6705	+ */
	6706	+ if (new % DEFAULT_STRIPE_SIZE != 0 \|\|
	6707	+ new > PAGE_SIZE \|\| new == 0 \|\|
	6708	+ new != roundup_pow_of_two(new))
	6709	+ return -EINVAL;
	6710	+
	6711	+ err = mddev_lock(mddev);
	6712	+ if (err)
	6713	+ return err;
	6714	+
	6715	+ conf = mddev->private;
	6716	+ if (!conf) {
	6717	+ err = -ENODEV;
	6718	+ goto out_unlock;
	6719	+ }
	6720	+
	6721	+ if (new == conf->stripe_size)
	6722	+ goto out_unlock;
	6723	+
	6724	+ pr_debug("md/raid: change stripe_size from %lu to %lu\n",
	6725	+ conf->stripe_size, new);
	6726	+
	6727	+ if (mddev->sync_thread \|\|
	6728	+ test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) \|\|
	6729	+ mddev->reshape_position != MaxSector \|\|
	6730	+ mddev->sysfs_active) {
	6731	+ err = -EBUSY;
	6732	+ goto out_unlock;
	6733	+ }
	6734	+
	6735	+ mddev_suspend(mddev);
	6736	+ mutex_lock(&conf->cache_size_mutex);
	6737	+ size = conf->max_nr_stripes;
	6738	+
	6739	+ shrink_stripes(conf);
	6740	+
	6741	+ conf->stripe_size = new;
	6742	+ conf->stripe_shift = ilog2(new) - 9;
	6743	+ conf->stripe_sectors = new >> 9;
	6744	+ if (grow_stripes(conf, size)) {
	6745	+ pr_warn("md/raid:%s: couldn't allocate buffers\n",
	6746	+ mdname(mddev));
	6747	+ err = -ENOMEM;
	6748	+ }
	6749	+ mutex_unlock(&conf->cache_size_mutex);
	6750	+ mddev_resume(mddev);
	6751	+
	6752	+out_unlock:
	6753	+ mddev_unlock(mddev);
	6754	+ return err ?: len;
	6755	+}
	6756	+
	6757	+static struct md_sysfs_entry
	6758	+raid5_stripe_size = __ATTR(stripe_size, 0644,
	6759	+ raid5_show_stripe_size,
	6760	+ raid5_store_stripe_size);
	6761	+#else
	6762	+static struct md_sysfs_entry
	6763	+raid5_stripe_size = __ATTR(stripe_size, 0444,
	6764	+ raid5_show_stripe_size,
	6765	+ NULL);
	6766	+#endif
6474	6767
6475	6768	static ssize_t
6476	6769	raid5_show_preread_threshold(struct mddev mddev, char page)
..	..	@@ -6550,14 +6843,14 @@
6550	6843	if (!conf)
6551	6844	err = -ENODEV;
6552	6845	else if (new != conf->skip_copy) {
	6846	+ struct request_queue *q = mddev->queue;
	6847	+
6553	6848	mddev_suspend(mddev);
6554	6849	conf->skip_copy = new;
6555	6850	if (new)
6556		- mddev->queue->backing_dev_info->capabilities \|=
6557		- BDI_CAP_STABLE_WRITES;
	6851	+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
6558	6852	else
6559		- mddev->queue->backing_dev_info->capabilities &=
6560		- ~BDI_CAP_STABLE_WRITES;
	6853	+ blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
6561	6854	mddev_resume(mddev);
6562	6855	}
6563	6856	mddev_unlock(mddev);
..	..	@@ -6597,7 +6890,6 @@
6597	6890
6598	6891	static int alloc_thread_groups(struct r5conf *conf, int cnt,
6599	6892	int *group_cnt,
6600		- int *worker_cnt_per_group,
6601	6893	struct r5worker_group **worker_groups);
6602	6894	static ssize_t
6603	6895	raid5_store_group_thread_cnt(struct mddev mddev, const char page, size_t len)
..	..	@@ -6606,7 +6898,7 @@
6606	6898	unsigned int new;
6607	6899	int err;
6608	6900	struct r5worker_group new_groups, old_groups;
6609		- int group_cnt, worker_cnt_per_group;
	6901	+ int group_cnt;
6610	6902
6611	6903	if (len >= PAGE_SIZE)
6612	6904	return -EINVAL;
..	..	@@ -6629,13 +6921,11 @@
6629	6921	if (old_groups)
6630	6922	flush_workqueue(raid5_wq);
6631	6923
6632		- err = alloc_thread_groups(conf, new,
6633		- &group_cnt, &worker_cnt_per_group,
6634		- &new_groups);
	6924	+ err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
6635	6925	if (!err) {
6636	6926	spin_lock_irq(&conf->device_lock);
6637	6927	conf->group_cnt = group_cnt;
6638		- conf->worker_cnt_per_group = worker_cnt_per_group;
	6928	+ conf->worker_cnt_per_group = new;
6639	6929	conf->worker_groups = new_groups;
6640	6930	spin_unlock_irq(&conf->device_lock);
6641	6931
..	..	@@ -6662,7 +6952,9 @@
6662	6952	&raid5_group_thread_cnt.attr,
6663	6953	&raid5_skip_copy.attr,
6664	6954	&raid5_rmw_level.attr,
	6955	+ &raid5_stripe_size.attr,
6665	6956	&r5c_journal_mode.attr,
	6957	+ &ppl_write_hint.attr,
6666	6958	NULL,
6667	6959	};
6668	6960	static struct attribute_group raid5_attrs_group = {
..	..	@@ -6670,16 +6962,13 @@
6670	6962	.attrs = raid5_attrs,
6671	6963	};
6672	6964
6673		-static int alloc_thread_groups(struct r5conf *conf, int cnt,
6674		- int *group_cnt,
6675		- int *worker_cnt_per_group,
	6965	+static int alloc_thread_groups(struct r5conf conf, int cnt, int group_cnt,
6676	6966	struct r5worker_group **worker_groups)
6677	6967	{
6678	6968	int i, j, k;
6679	6969	ssize_t size;
6680	6970	struct r5worker *workers;
6681	6971
6682		- *worker_cnt_per_group = cnt;
6683	6972	if (cnt == 0) {
6684	6973	*group_cnt = 0;
6685	6974	*worker_groups = NULL;
..	..	@@ -6745,25 +7034,25 @@
6745	7034	static void free_scratch_buffer(struct r5conf conf, struct raid5_percpu percpu)
6746	7035	{
6747	7036	safe_put_page(percpu->spare_page);
6748		- if (percpu->scribble)
6749		- flex_array_free(percpu->scribble);
6750	7037	percpu->spare_page = NULL;
	7038	+ kvfree(percpu->scribble);
6751	7039	percpu->scribble = NULL;
6752	7040	}
6753	7041
6754	7042	static int alloc_scratch_buffer(struct r5conf conf, struct raid5_percpu percpu)
6755	7043	{
6756		- if (conf->level == 6 && !percpu->spare_page)
	7044	+ if (conf->level == 6 && !percpu->spare_page) {
6757	7045	percpu->spare_page = alloc_page(GFP_KERNEL);
6758		- if (!percpu->scribble)
6759		- percpu->scribble = scribble_alloc(max(conf->raid_disks,
6760		- conf->previous_raid_disks),
6761		- max(conf->chunk_sectors,
6762		- conf->prev_chunk_sectors)
6763		- / STRIPE_SECTORS,
6764		- GFP_KERNEL);
	7046	+ if (!percpu->spare_page)
	7047	+ return -ENOMEM;
	7048	+ }
6765	7049
6766		- if (!percpu->scribble \|\| (conf->level == 6 && !percpu->spare_page)) {
	7050	+ if (scribble_alloc(percpu,
	7051	+ max(conf->raid_disks,
	7052	+ conf->previous_raid_disks),
	7053	+ max(conf->chunk_sectors,
	7054	+ conf->prev_chunk_sectors)
	7055	+ / RAID5_STRIPE_SECTORS(conf))) {
6767	7056	free_scratch_buffer(conf, percpu);
6768	7057	return -ENOMEM;
6769	7058	}
..	..	@@ -6818,7 +7107,6 @@
6818	7107	__func__, cpu);
6819	7108	return -ENOMEM;
6820	7109	}
6821		- spin_lock_init(&per_cpu_ptr(conf->percpu, cpu)->lock);
6822	7110	return 0;
6823	7111	}
6824	7112
..	..	@@ -6829,6 +7117,7 @@
6829	7117	conf->percpu = alloc_percpu(struct raid5_percpu);
6830	7118	if (!conf->percpu)
6831	7119	return -ENOMEM;
	7120	+
6832	7121	err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
6833	7122	if (!err) {
6834	7123	conf->scribble_disks = max(conf->raid_disks,
..	..	@@ -6879,7 +7168,7 @@
6879	7168	struct disk_info *disk;
6880	7169	char pers_name[6];
6881	7170	int i;
6882		- int group_cnt, worker_cnt_per_group;
	7171	+ int group_cnt;
6883	7172	struct r5worker_group *new_group;
6884	7173	int ret;
6885	7174
..	..	@@ -6915,6 +7204,12 @@
6915	7204	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
6916	7205	if (conf == NULL)
6917	7206	goto abort;
	7207	+
	7208	+#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
	7209	+ conf->stripe_size = DEFAULT_STRIPE_SIZE;
	7210	+ conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
	7211	+ conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;
	7212	+#endif
6918	7213	INIT_LIST_HEAD(&conf->free_list);
6919	7214	INIT_LIST_HEAD(&conf->pending_list);
6920	7215	conf->pending_data = kcalloc(PENDING_IO_MAX,
..	..	@@ -6925,15 +7220,14 @@
6925	7220	for (i = 0; i < PENDING_IO_MAX; i++)
6926	7221	list_add(&conf->pending_data[i].sibling, &conf->free_list);
6927	7222	/* Don't enable multi-threading by default*/
6928		- if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6929		- &new_group)) {
	7223	+ if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
6930	7224	conf->group_cnt = group_cnt;
6931		- conf->worker_cnt_per_group = worker_cnt_per_group;
	7225	+ conf->worker_cnt_per_group = 0;
6932	7226	conf->worker_groups = new_group;
6933	7227	} else
6934	7228	goto abort;
6935	7229	spin_lock_init(&conf->device_lock);
6936		- seqcount_init(&conf->gen_lock);
	7230	+ seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);
6937	7231	mutex_init(&conf->cache_size_mutex);
6938	7232	init_waitqueue_head(&conf->wait_for_quiescent);
6939	7233	init_waitqueue_head(&conf->wait_for_stripe);
..	..	@@ -7067,8 +7361,8 @@
7067	7361	conf->min_nr_stripes = NR_STRIPES;
7068	7362	if (mddev->reshape_position != MaxSector) {
7069	7363	int stripes = max_t(int,
7070		- ((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4,
7071		- ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4);
	7364	+ ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,
	7365	+ ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);
7072	7366	conf->min_nr_stripes = max(NR_STRIPES, stripes);
7073	7367	if (conf->min_nr_stripes != NR_STRIPES)
7074	7368	pr_info("md/raid:%s: force stripe size %d for reshape\n",
..	..	@@ -7141,6 +7435,12 @@
7141	7435	return 1;
7142	7436	}
7143	7437	return 0;
	7438	+}
	7439	+
	7440	+static void raid5_set_io_opt(struct r5conf *conf)
	7441	+{
	7442	+ blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
	7443	+ (conf->raid_disks - conf->max_degraded));
7144	7444	}
7145	7445
7146	7446	static int raid5_run(struct mddev *mddev)
..	..	@@ -7427,13 +7727,10 @@
7427	7727	int data_disks = conf->previous_raid_disks - conf->max_degraded;
7428	7728	int stripe = data_disks *
7429	7729	((mddev->chunk_sectors << 9) / PAGE_SIZE);
7430		- if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
7431		- mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
7432	7730
7433	7731	chunk_size = mddev->chunk_sectors << 9;
7434	7732	blk_queue_io_min(mddev->queue, chunk_size);
7435		- blk_queue_io_opt(mddev->queue, chunk_size *
7436		- (conf->raid_disks - conf->max_degraded));
	7733	+ raid5_set_io_opt(conf);
7437	7734	mddev->queue->limits.raid_partial_stripes_expensive = 1;
7438	7735	/*
7439	7736	* We can only discard a whole stripe. It doesn't make sense to
..	..	@@ -7718,6 +8015,7 @@
7718	8015	*/
7719	8016	if (rdev->saved_raid_disk >= 0 &&
7720	8017	rdev->saved_raid_disk >= first &&
	8018	+ rdev->saved_raid_disk <= last &&
7721	8019	conf->disks[rdev->saved_raid_disk].rdev == NULL)
7722	8020	first = rdev->saved_raid_disk;
7723	8021
..	..	@@ -7799,14 +8097,14 @@
7799	8097	* stripe_heads first.
7800	8098	*/
7801	8099	struct r5conf *conf = mddev->private;
7802		- if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	8100	+ if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
7803	8101	> conf->min_nr_stripes \|\|
7804		- ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	8102	+ ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
7805	8103	> conf->min_nr_stripes) {
7806	8104	pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
7807	8105	mdname(mddev),
7808	8106	((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
7809		- / STRIPE_SIZE)*4);
	8107	+ / RAID5_STRIPE_SIZE(conf))*4);
7810	8108	return 0;
7811	8109	}
7812	8110	return 1;
..	..	@@ -7942,8 +8240,8 @@
7942	8240	else
7943	8241	rdev->recovery_offset = 0;
7944	8242
7945		- if (sysfs_link_rdev(mddev, rdev))
7946		- /* Failure here is OK */;
	8243	+ /* Failure here is OK */
	8244	+ sysfs_link_rdev(mddev, rdev);
7947	8245	}
7948	8246	} else if (rdev->raid_disk >= conf->previous_raid_disks
7949	8247	&& !test_bit(Faulty, &rdev->flags)) {
..	..	@@ -8017,16 +8315,8 @@
8017	8315	spin_unlock_irq(&conf->device_lock);
8018	8316	wake_up(&conf->wait_for_overlap);
8019	8317
8020		- /* read-ahead size must cover two whole stripes, which is
8021		- * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
8022		- */
8023		- if (conf->mddev->queue) {
8024		- int data_disks = conf->raid_disks - conf->max_degraded;
8025		- int stripe = data_disks * ((conf->chunk_sectors << 9)
8026		- / PAGE_SIZE);
8027		- if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
8028		- conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
8029		- }
	8318	+ if (conf->mddev->queue)
	8319	+ raid5_set_io_opt(conf);
8030	8320	}
8031	8321	}
8032	8322
..	..	@@ -8138,7 +8428,7 @@
8138	8428	while (chunksect && (mddev->array_sectors & (chunksect-1)))
8139	8429	chunksect >>= 1;
8140	8430
8141		- if ((chunksect<<9) < STRIPE_SIZE)
	8431	+ if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))
8142	8432	/* array size does not allow a suitable chunk size */
8143	8433	return ERR_PTR(-EINVAL);
8144	8434
..	..	@@ -8425,7 +8715,6 @@
8425	8715	.finish_reshape = raid5_finish_reshape,
8426	8716	.quiesce = raid5_quiesce,
8427	8717	.takeover = raid6_takeover,
8428		- .congested = raid5_congested,
8429	8718	.change_consistency_policy = raid5_change_consistency_policy,
8430	8719	};
8431	8720	static struct md_personality raid5_personality =
..	..	@@ -8450,7 +8739,6 @@
8450	8739	.finish_reshape = raid5_finish_reshape,
8451	8740	.quiesce = raid5_quiesce,
8452	8741	.takeover = raid5_takeover,
8453		- .congested = raid5_congested,
8454	8742	.change_consistency_policy = raid5_change_consistency_policy,
8455	8743	};
8456	8744
..	..	@@ -8476,7 +8764,6 @@
8476	8764	.finish_reshape = raid5_finish_reshape,
8477	8765	.quiesce = raid5_quiesce,
8478	8766	.takeover = raid4_takeover,
8479		- .congested = raid5_congested,
8480	8767	.change_consistency_policy = raid5_change_consistency_policy,
8481	8768	};
8482	8769