forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/fs/f2fs/f2fs.h
....@@ -22,6 +22,7 @@
2222 #include <linux/bio.h>
2323 #include <linux/blkdev.h>
2424 #include <linux/quotaops.h>
25
+#include <linux/part_stat.h>
2526 #include <crypto/hash.h>
2627 
2728 #include <linux/fscrypt.h>
....@@ -32,10 +33,8 @@
3233 #else
3334 #define f2fs_bug_on(sbi, condition)					\
3435 	do {								\
35
-		if (unlikely(condition)) {				\
36
-			WARN_ON(1);					\
36
+		if (WARN_ON(condition))					\
3737 			set_sbi_flag(sbi, SBI_NEED_FSCK);		\
38
-		}							\
3938 	} while (0)
4039 #endif
4140 
....@@ -44,7 +43,6 @@
4443 	FAULT_KVMALLOC,
4544 	FAULT_PAGE_ALLOC,
4645 	FAULT_PAGE_GET,
47
-	FAULT_ALLOC_BIO,
4846 	FAULT_ALLOC_NID,
4947 	FAULT_ORPHAN,
5048 	FAULT_BLOCK,
....@@ -71,7 +69,7 @@
7169 #define IS_FAULT_SET(fi, type) ((fi)->inject_type & (1 << (type)))
7270 #endif
7371 
74
-#define MIN_ROOT_RESERVED_BLOCKS (128 * 1024 * 1024U)
72
+#define MIN_ROOT_RESERVED_BLOCKS (128 * 1024 * 1024)
7573 
7674 /*
7775  * For mount options
....@@ -88,7 +86,7 @@
8886 #define F2FS_MOUNT_FLUSH_MERGE		0x00000400
8987 #define F2FS_MOUNT_NOBARRIER		0x00000800
9088 #define F2FS_MOUNT_FASTBOOT		0x00001000
91
-#define F2FS_MOUNT_EXTENT_CACHE		0x00002000
89
+#define F2FS_MOUNT_READ_EXTENT_CACHE	0x00002000
9290 #define F2FS_MOUNT_DATA_FLUSH		0x00008000
9391 #define F2FS_MOUNT_FAULT_INJECTION	0x00010000
9492 #define F2FS_MOUNT_USRQUOTA		0x00080000
....@@ -99,6 +97,11 @@
9997 #define F2FS_MOUNT_RESERVE_ROOT		0x01000000
10098 #define F2FS_MOUNT_DISABLE_CHECKPOINT	0x02000000
10199 #define F2FS_MOUNT_NORECOVERY		0x04000000
100
+#define F2FS_MOUNT_ATGC			0x08000000
101
+#define F2FS_MOUNT_MERGE_CHECKPOINT	0x10000000
102
+#define	F2FS_MOUNT_GC_MERGE		0x20000000
103
+#define F2FS_MOUNT_COMPRESS_CACHE	0x40000000
104
+#define F2FS_MOUNT_AGE_EXTENT_CACHE	0x80000000
102105 
103106 #define F2FS_OPTION(sbi)	((sbi)->mount_opt)
104107 #define clear_opt(sbi, option)	(F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
....@@ -116,6 +119,18 @@
116119 typedef u32 nid_t;
117120 
118121 #define COMPRESS_EXT_NUM		16
122
+
123
+/*
124
+ * An implementation of an rwsem that is explicitly unfair to readers. This
125
+ * prevents priority inversion when a low-priority reader acquires the read lock
126
+ * while sleeping on the write lock but the write lock is needed by
127
+ * higher-priority clients.
128
+ */
129
+
130
+struct f2fs_rwsem {
131
+        struct rw_semaphore internal_rwsem;
132
+        wait_queue_head_t read_waiters;
133
+};
119134 
120135 struct f2fs_mount_info {
121136 	unsigned int opt;
....@@ -139,10 +154,8 @@
139154 	int fsync_mode;			/* fsync policy */
140155 	int fs_mode;			/* fs mode: LFS or ADAPTIVE */
141156 	int bggc_mode;			/* bggc mode: off, on or sync */
142
-	struct fscrypt_dummy_context dummy_enc_ctx; /* test dummy encryption */
143
-#ifdef CONFIG_FS_ENCRYPTION
144
-	bool inlinecrypt;		/* inline encryption enabled */
145
-#endif
157
+	int memory_mode;		/* memory mode */
158
+	struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
146159 	block_t unusable_cap_perc;	/* percentage for cap */
147160 	block_t unusable_cap;		/* Amount of space allowed to be
148161 					 * unusable when disabling checkpoint
....@@ -150,8 +163,11 @@
150163 
151164 	/* For compression */
152165 	unsigned char compress_algorithm;	/* algorithm type */
153
-	unsigned compress_log_size;		/* cluster log size */
166
+	unsigned char compress_log_size;	/* cluster log size */
167
+	unsigned char compress_level;		/* compress level */
168
+	bool compress_chksum;			/* compressed data chksum */
154169 	unsigned char compress_ext_cnt;		/* extension count */
170
+	int compress_mode;			/* compression mode */
155171 	unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN];	/* extensions */
156172 };
157173 
....@@ -169,6 +185,7 @@
169185 #define F2FS_FEATURE_SB_CHKSUM		0x0800
170186 #define F2FS_FEATURE_CASEFOLD		0x1000
171187 #define F2FS_FEATURE_COMPRESSION	0x2000
188
+#define F2FS_FEATURE_RO			0x4000
172189 
173190 #define __F2FS_HAS_FEATURE(raw_super, mask)				\
174191 	((raw_super->feature & cpu_to_le32(mask)) != 0)
....@@ -238,6 +255,10 @@
238255 					 * condition of read on truncated area
239256 					 * by extent_cache
240257 					 */
258
+	DATA_GENERIC_ENHANCE_UPDATE,	/*
259
+					 * strong check on range and segment
260
+					 * bitmap for update case
261
+					 */
241262 	META_GENERIC,
242263 };
243264 
....@@ -267,6 +288,26 @@
267288 	struct list_head list;	/* list head */
268289 	struct page *page;	/* warm node page pointer */
269290 	unsigned int seq_id;	/* sequence id */
291
+};
292
+
293
+struct ckpt_req {
294
+	struct completion wait;		/* completion for checkpoint done */
295
+	struct llist_node llnode;	/* llist_node to be linked in wait queue */
296
+	int ret;			/* return code of checkpoint */
297
+	ktime_t queue_time;		/* request queued time */
298
+};
299
+
300
+struct ckpt_req_control {
301
+	struct task_struct *f2fs_issue_ckpt;	/* checkpoint task */
302
+	int ckpt_thread_ioprio;			/* checkpoint merge thread ioprio */
303
+	wait_queue_head_t ckpt_wait_queue;	/* waiting queue for wake-up */
304
+	atomic_t issued_ckpt;		/* # of actually issued ckpts */
305
+	atomic_t total_ckpt;		/* # of total ckpts */
306
+	atomic_t queued_ckpt;		/* # of queued ckpts */
307
+	struct llist_head issue_list;	/* list for command issue */
308
+	spinlock_t stat_lock;		/* lock for below checkpoint time stats */
309
+	unsigned int cur_time;		/* cur wait time in msec for currently issued checkpoint */
310
+	unsigned int peak_time;		/* peak wait time in msec until now */
270311 };
271312 
272313 /* for the bitmap indicate blocks to be discarded */
....@@ -405,93 +446,6 @@
405446 	return size <= MAX_SIT_JENTRIES(journal);
406447 }
407448 
408
-/*
409
- * ioctl commands
410
- */
411
-#define F2FS_IOC_GETFLAGS		FS_IOC_GETFLAGS
412
-#define F2FS_IOC_SETFLAGS		FS_IOC_SETFLAGS
413
-#define F2FS_IOC_GETVERSION		FS_IOC_GETVERSION
414
-
415
-#define F2FS_IOCTL_MAGIC		0xf5
416
-#define F2FS_IOC_START_ATOMIC_WRITE	_IO(F2FS_IOCTL_MAGIC, 1)
417
-#define F2FS_IOC_COMMIT_ATOMIC_WRITE	_IO(F2FS_IOCTL_MAGIC, 2)
418
-#define F2FS_IOC_START_VOLATILE_WRITE	_IO(F2FS_IOCTL_MAGIC, 3)
419
-#define F2FS_IOC_RELEASE_VOLATILE_WRITE	_IO(F2FS_IOCTL_MAGIC, 4)
420
-#define F2FS_IOC_ABORT_VOLATILE_WRITE	_IO(F2FS_IOCTL_MAGIC, 5)
421
-#define F2FS_IOC_GARBAGE_COLLECT	_IOW(F2FS_IOCTL_MAGIC, 6, __u32)
422
-#define F2FS_IOC_WRITE_CHECKPOINT	_IO(F2FS_IOCTL_MAGIC, 7)
423
-#define F2FS_IOC_DEFRAGMENT		_IOWR(F2FS_IOCTL_MAGIC, 8,	\
424
-						struct f2fs_defragment)
425
-#define F2FS_IOC_MOVE_RANGE		_IOWR(F2FS_IOCTL_MAGIC, 9,	\
426
-						struct f2fs_move_range)
427
-#define F2FS_IOC_FLUSH_DEVICE		_IOW(F2FS_IOCTL_MAGIC, 10,	\
428
-						struct f2fs_flush_device)
429
-#define F2FS_IOC_GARBAGE_COLLECT_RANGE	_IOW(F2FS_IOCTL_MAGIC, 11,	\
430
-						struct f2fs_gc_range)
431
-#define F2FS_IOC_GET_FEATURES		_IOR(F2FS_IOCTL_MAGIC, 12, __u32)
432
-#define F2FS_IOC_SET_PIN_FILE		_IOW(F2FS_IOCTL_MAGIC, 13, __u32)
433
-#define F2FS_IOC_GET_PIN_FILE		_IOR(F2FS_IOCTL_MAGIC, 14, __u32)
434
-#define F2FS_IOC_PRECACHE_EXTENTS	_IO(F2FS_IOCTL_MAGIC, 15)
435
-#define F2FS_IOC_RESIZE_FS		_IOW(F2FS_IOCTL_MAGIC, 16, __u64)
436
-#define F2FS_IOC_GET_COMPRESS_BLOCKS	_IOR(F2FS_IOCTL_MAGIC, 17, __u64)
437
-#define F2FS_IOC_RELEASE_COMPRESS_BLOCKS				\
438
-					_IOR(F2FS_IOCTL_MAGIC, 18, __u64)
439
-#define F2FS_IOC_RESERVE_COMPRESS_BLOCKS				\
440
-					_IOR(F2FS_IOCTL_MAGIC, 19, __u64)
441
-
442
-#define F2FS_IOC_GET_VOLUME_NAME	FS_IOC_GETFSLABEL
443
-#define F2FS_IOC_SET_VOLUME_NAME	FS_IOC_SETFSLABEL
444
-
445
-#define F2FS_IOC_SET_ENCRYPTION_POLICY	FS_IOC_SET_ENCRYPTION_POLICY
446
-#define F2FS_IOC_GET_ENCRYPTION_POLICY	FS_IOC_GET_ENCRYPTION_POLICY
447
-#define F2FS_IOC_GET_ENCRYPTION_PWSALT	FS_IOC_GET_ENCRYPTION_PWSALT
448
-
449
-/*
450
- * should be same as XFS_IOC_GOINGDOWN.
451
- * Flags for going down operation used by FS_IOC_GOINGDOWN
452
- */
453
-#define F2FS_IOC_SHUTDOWN	_IOR('X', 125, __u32)	/* Shutdown */
454
-#define F2FS_GOING_DOWN_FULLSYNC	0x0	/* going down with full sync */
455
-#define F2FS_GOING_DOWN_METASYNC	0x1	/* going down with metadata */
456
-#define F2FS_GOING_DOWN_NOSYNC		0x2	/* going down */
457
-#define F2FS_GOING_DOWN_METAFLUSH	0x3	/* going down with meta flush */
458
-#define F2FS_GOING_DOWN_NEED_FSCK	0x4	/* going down to trigger fsck */
459
-
460
-#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
461
-/*
462
- * ioctl commands in 32 bit emulation
463
- */
464
-#define F2FS_IOC32_GETFLAGS		FS_IOC32_GETFLAGS
465
-#define F2FS_IOC32_SETFLAGS		FS_IOC32_SETFLAGS
466
-#define F2FS_IOC32_GETVERSION		FS_IOC32_GETVERSION
467
-#endif
468
-
469
-#define F2FS_IOC_FSGETXATTR		FS_IOC_FSGETXATTR
470
-#define F2FS_IOC_FSSETXATTR		FS_IOC_FSSETXATTR
471
-
472
-struct f2fs_gc_range {
473
-	u32 sync;
474
-	u64 start;
475
-	u64 len;
476
-};
477
-
478
-struct f2fs_defragment {
479
-	u64 start;
480
-	u64 len;
481
-};
482
-
483
-struct f2fs_move_range {
484
-	u32 dst_fd;		/* destination fd */
485
-	u64 pos_in;		/* start position in src_fd */
486
-	u64 pos_out;		/* start position in dst_fd */
487
-	u64 len;		/* size to move */
488
-};
489
-
490
-struct f2fs_flush_device {
491
-	u32 dev_num;		/* device number to flush */
492
-	u32 segments;		/* # of segments to flush */
493
-};
494
-
495449 /* for inline stuff */
496450 #define DEF_INLINE_RESERVED_SIZE	1
497451 static inline int get_extra_isize(struct inode *inode);
....@@ -544,11 +498,11 @@
544498 #ifdef CONFIG_UNICODE
545499 	/*
546500 	 * For casefolded directories: the casefolded name, but it's left NULL
547
-	 * if the original name is not valid Unicode, if the directory is both
548
-	 * casefolded and encrypted and its encryption key is unavailable, or if
549
-	 * the filesystem is doing an internal operation where usr_fname is also
550
-	 * NULL.  In all these cases we fall back to treating the name as an
551
-	 * opaque byte sequence.
501
+	 * if the original name is not valid Unicode, if the original name is
502
+	 * "." or "..", if the directory is both casefolded and encrypted and
503
+	 * its encryption key is unavailable, or if the filesystem is doing an
504
+	 * internal operation where usr_fname is also NULL.  In all these cases
505
+	 * we fall back to treating the name as an opaque byte sequence.
552506 	 */
553507 	struct fscrypt_str cf_name;
554508 #endif
....@@ -622,18 +576,59 @@
622576 #define F2FS_MIN_EXTENT_LEN	64	/* minimum extent length */
623577 
624578 /* number of extent info in extent cache we try to shrink */
625
-#define EXTENT_CACHE_SHRINK_NUMBER	128
579
+#define READ_EXTENT_CACHE_SHRINK_NUMBER	128
580
+
581
+/* number of age extent info in extent cache we try to shrink */
582
+#define AGE_EXTENT_CACHE_SHRINK_NUMBER	128
583
+#define LAST_AGE_WEIGHT			30
584
+#define SAME_AGE_REGION			1024
585
+
586
+/*
587
+ * Define data block with age less than 1GB as hot data
588
+ * define data block with age less than 10GB but more than 1GB as warm data
589
+ */
590
+#define DEF_HOT_DATA_AGE_THRESHOLD	262144
591
+#define DEF_WARM_DATA_AGE_THRESHOLD	2621440
592
+
593
+/* extent cache type */
594
+enum extent_type {
595
+	EX_READ,
596
+	EX_BLOCK_AGE,
597
+	NR_EXTENT_CACHES,
598
+};
626599 
627600 struct rb_entry {
628601 	struct rb_node rb_node;		/* rb node located in rb-tree */
629
-	unsigned int ofs;		/* start offset of the entry */
630
-	unsigned int len;		/* length of the entry */
602
+	union {
603
+		struct {
604
+			unsigned int ofs;	/* start offset of the entry */
605
+			unsigned int len;	/* length of the entry */
606
+		};
607
+		unsigned long long key;		/* 64-bits key */
608
+	} __packed;
631609 };
632610 
633611 struct extent_info {
634612 	unsigned int fofs;		/* start offset in a file */
635613 	unsigned int len;		/* length of the extent */
636
-	u32 blk;			/* start block address of the extent */
614
+	union {
615
+		/* read extent_cache */
616
+		struct {
617
+			/* start block address of the extent */
618
+			block_t blk;
619
+#ifdef CONFIG_F2FS_FS_COMPRESSION
620
+			/* physical extent length of compressed blocks */
621
+			unsigned int c_len;
622
+#endif
623
+		};
624
+		/* block age extent_cache */
625
+		struct {
626
+			/* block age of the extent */
627
+			unsigned long long age;
628
+			/* last total blocks allocated */
629
+			unsigned long long last_blocks;
630
+		};
631
+	};
637632 };
638633 
639634 struct extent_node {
....@@ -645,13 +640,25 @@
645640 
646641 struct extent_tree {
647642 	nid_t ino;			/* inode number */
643
+	enum extent_type type;		/* keep the extent tree type */
648644 	struct rb_root_cached root;	/* root of extent info rb-tree */
649645 	struct extent_node *cached_en;	/* recently accessed extent node */
650
-	struct extent_info largest;	/* largested extent info */
651646 	struct list_head list;		/* to be used by sbi->zombie_list */
652647 	rwlock_t lock;			/* protect extent info rb-tree */
653648 	atomic_t node_cnt;		/* # of extent node in rb-tree*/
654649 	bool largest_updated;		/* largest extent updated */
650
+	struct extent_info largest;	/* largest cached extent for EX_READ */
651
+};
652
+
653
+struct extent_tree_info {
654
+	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
655
+	struct mutex extent_tree_lock;	/* locking extent radix tree */
656
+	struct list_head extent_list;		/* lru list for shrinker */
657
+	spinlock_t extent_lock;			/* locking extent lru list */
658
+	atomic_t total_ext_tree;		/* extent tree count */
659
+	struct list_head zombie_list;		/* extent zombie tree list */
660
+	atomic_t total_zombie_tree;		/* extent zombie tree count */
661
+	atomic_t total_ext_node;		/* extent info count */
655662 };
656663 
657664 /*
....@@ -701,21 +708,26 @@
701708 #define FADVISE_MODIFIABLE_BITS	(FADVISE_COLD_BIT | FADVISE_HOT_BIT)
702709 
703710 #define file_is_cold(inode)	is_file(inode, FADVISE_COLD_BIT)
704
-#define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
705711 #define file_set_cold(inode)	set_file(inode, FADVISE_COLD_BIT)
706
-#define file_lost_pino(inode)	set_file(inode, FADVISE_LOST_PINO_BIT)
707712 #define file_clear_cold(inode)	clear_file(inode, FADVISE_COLD_BIT)
713
+
714
+#define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
715
+#define file_lost_pino(inode)	set_file(inode, FADVISE_LOST_PINO_BIT)
708716 #define file_got_pino(inode)	clear_file(inode, FADVISE_LOST_PINO_BIT)
717
+
709718 #define file_is_encrypt(inode)	is_file(inode, FADVISE_ENCRYPT_BIT)
710719 #define file_set_encrypt(inode)	set_file(inode, FADVISE_ENCRYPT_BIT)
711
-#define file_clear_encrypt(inode) clear_file(inode, FADVISE_ENCRYPT_BIT)
720
+
712721 #define file_enc_name(inode)	is_file(inode, FADVISE_ENC_NAME_BIT)
713722 #define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
723
+
714724 #define file_keep_isize(inode)	is_file(inode, FADVISE_KEEP_SIZE_BIT)
715725 #define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
726
+
716727 #define file_is_hot(inode)	is_file(inode, FADVISE_HOT_BIT)
717728 #define file_set_hot(inode)	set_file(inode, FADVISE_HOT_BIT)
718729 #define file_clear_hot(inode)	clear_file(inode, FADVISE_HOT_BIT)
730
+
719731 #define file_is_verity(inode)	is_file(inode, FADVISE_VERITY_BIT)
720732 #define file_set_verity(inode)	set_file(inode, FADVISE_VERITY_BIT)
721733 
....@@ -751,7 +763,8 @@
751763 	FI_DROP_CACHE,		/* drop dirty page cache */
752764 	FI_DATA_EXIST,		/* indicate data exists */
753765 	FI_INLINE_DOTS,		/* indicate inline dot dentries */
754
-	FI_DO_DEFRAG,		/* indicate defragment is running */
766
+	FI_SKIP_WRITES,		/* should skip data page writeback */
767
+	FI_OPU_WRITE,		/* used for opu per file */
755768 	FI_DIRTY_FILE,		/* indicate regular/symlink has dirty pages */
756769 	FI_NO_PREALLOC,		/* indicate skipped preallocated blocks */
757770 	FI_HOT_DATA,		/* indicate file is hot */
....@@ -761,7 +774,11 @@
761774 	FI_ATOMIC_REVOKE_REQUEST, /* request to drop atomic data */
762775 	FI_VERITY_IN_PROGRESS,	/* building fs-verity Merkle tree */
763776 	FI_COMPRESSED_FILE,	/* indicate file's data can be compressed */
777
+	FI_COMPRESS_CORRUPT,	/* indicate compressed cluster is corrupted */
764778 	FI_MMAP_FILE,		/* indicate file was mmapped */
779
+	FI_ENABLE_COMPRESS,	/* enable compression in "user" compression mode */
780
+	FI_COMPRESS_RELEASED,	/* compressed blocks were released */
781
+	FI_ALIGNED_WRITE,	/* enable aligned write */
765782 	FI_MAX,			/* max flag, never be used */
766783 };
767784 
....@@ -778,12 +795,13 @@
778795 
779796 	/* Use below internally in f2fs*/
780797 	unsigned long flags[BITS_TO_LONGS(FI_MAX)];	/* use to pass per-file flags */
781
-	struct rw_semaphore i_sem;	/* protect fi info */
798
+	struct f2fs_rwsem i_sem;	/* protect fi info */
782799 	atomic_t dirty_pages;		/* # of dirty pages */
783800 	f2fs_hash_t chash;		/* hash value of given file name */
784801 	unsigned int clevel;		/* maximum level of given file name */
785802 	struct task_struct *task;	/* lookup and create consistency */
786803 	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
804
+	struct task_struct *wb_task;	/* indicate inode is in context of writeback */
787805 	nid_t i_xattr_nid;		/* node id that contains xattrs */
788806 	loff_t	last_disk_size;		/* lastly written file size */
789807 	spinlock_t i_size_lock;		/* protect last_disk_size */
....@@ -800,12 +818,13 @@
800818 	struct list_head inmem_pages;	/* inmemory pages managed by f2fs */
801819 	struct task_struct *inmem_task;	/* store inmemory task */
802820 	struct mutex inmem_lock;	/* lock for inmemory pages */
803
-	struct extent_tree *extent_tree;	/* cached extent_tree entry */
821
+	struct extent_tree *extent_tree[NR_EXTENT_CACHES];
822
+					/* cached extent_tree entry */
804823 
805824 	/* avoid racing between foreground op and gc */
806
-	struct rw_semaphore i_gc_rwsem[2];
807
-	struct rw_semaphore i_mmap_sem;
808
-	struct rw_semaphore i_xattr_sem; /* avoid racing between reading and changing EAs */
825
+	struct f2fs_rwsem i_gc_rwsem[2];
826
+	struct f2fs_rwsem i_mmap_sem;
827
+	struct f2fs_rwsem i_xattr_sem; /* avoid racing between reading and changing EAs */
809828 
810829 	int i_extra_isize;		/* size of extra space located in i_addr */
811830 	kprojid_t i_projid;		/* id for project quota */
....@@ -814,13 +833,15 @@
814833 	struct timespec64 i_disk_time[4];/* inode disk times */
815834 
816835 	/* for file compress */
817
-	u64 i_compr_blocks;			/* # of compressed blocks */
836
+	atomic_t i_compr_blocks;		/* # of compressed blocks */
818837 	unsigned char i_compress_algorithm;	/* algorithm type */
819838 	unsigned char i_log_cluster_size;	/* log of cluster size */
839
+	unsigned char i_compress_level;		/* compress level (lz4hc,zstd) */
840
+	unsigned short i_compress_flag;		/* compress flag */
820841 	unsigned int i_cluster_size;		/* cluster size */
821842 };
822843 
823
-static inline void get_extent_info(struct extent_info *ext,
844
+static inline void get_read_extent_info(struct extent_info *ext,
824845 					struct f2fs_extent *i_ext)
825846 {
826847 	ext->fofs = le32_to_cpu(i_ext->fofs);
....@@ -828,20 +849,12 @@
828849 	ext->len = le32_to_cpu(i_ext->len);
829850 }
830851 
831
-static inline void set_raw_extent(struct extent_info *ext,
852
+static inline void set_raw_read_extent(struct extent_info *ext,
832853 					struct f2fs_extent *i_ext)
833854 {
834855 	i_ext->fofs = cpu_to_le32(ext->fofs);
835856 	i_ext->blk = cpu_to_le32(ext->blk);
836857 	i_ext->len = cpu_to_le32(ext->len);
837
-}
838
-
839
-static inline void set_extent_info(struct extent_info *ei, unsigned int fofs,
840
-						u32 blk, unsigned int len)
841
-{
842
-	ei->fofs = fofs;
843
-	ei->blk = blk;
844
-	ei->len = len;
845858 }
846859 
847860 static inline bool __is_discard_mergeable(struct discard_info *back,
....@@ -863,35 +876,6 @@
863876 	return __is_discard_mergeable(cur, front, max_len);
864877 }
865878 
866
-static inline bool __is_extent_mergeable(struct extent_info *back,
867
-						struct extent_info *front)
868
-{
869
-	return (back->fofs + back->len == front->fofs &&
870
-			back->blk + back->len == front->blk);
871
-}
872
-
873
-static inline bool __is_back_mergeable(struct extent_info *cur,
874
-						struct extent_info *back)
875
-{
876
-	return __is_extent_mergeable(back, cur);
877
-}
878
-
879
-static inline bool __is_front_mergeable(struct extent_info *cur,
880
-						struct extent_info *front)
881
-{
882
-	return __is_extent_mergeable(cur, front);
883
-}
884
-
885
-extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
886
-static inline void __try_update_largest_extent(struct extent_tree *et,
887
-						struct extent_node *en)
888
-{
889
-	if (en->ei.len > et->largest.len) {
890
-		et->largest = en->ei;
891
-		et->largest_updated = true;
892
-	}
893
-}
894
-
895879 /*
896880  * For free nid management
897881  */
....@@ -899,6 +883,13 @@
899883 	FREE_NID,		/* newly added to free nid list */
900884 	PREALLOC_NID,		/* it is preallocated */
901885 	MAX_NID_STATE,
886
+};
887
+
888
+enum nat_state {
889
+	TOTAL_NAT,
890
+	DIRTY_NAT,
891
+	RECLAIMABLE_NAT,
892
+	MAX_NAT_STATE,
902893 };
903894 
904895 struct f2fs_nm_info {
....@@ -913,11 +904,10 @@
913904 	/* NAT cache management */
914905 	struct radix_tree_root nat_root;/* root of the nat entry cache */
915906 	struct radix_tree_root nat_set_root;/* root of the nat set cache */
916
-	struct rw_semaphore nat_tree_lock;	/* protect nat_tree_lock */
907
+	struct f2fs_rwsem nat_tree_lock;	/* protect nat entry tree */
917908 	struct list_head nat_entries;	/* cached nat entry list (clean) */
918909 	spinlock_t nat_list_lock;	/* protect clean nat entry list */
919
-	unsigned int nat_cnt;		/* the # of cached nat entries */
920
-	unsigned int dirty_nat_cnt;	/* total num of nat entries in set */
910
+	unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
921911 	unsigned int nat_blocks;	/* # of nat blocks */
922912 
923913 	/* free node ids management */
....@@ -986,7 +976,10 @@
986976  */
987977 #define	NR_CURSEG_DATA_TYPE	(3)
988978 #define NR_CURSEG_NODE_TYPE	(3)
989
-#define NR_CURSEG_TYPE	(NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
979
+#define NR_CURSEG_INMEM_TYPE	(2)
980
+#define NR_CURSEG_RO_TYPE	(2)
981
+#define NR_CURSEG_PERSIST_TYPE	(NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
982
+#define NR_CURSEG_TYPE		(NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
990983 
991984 enum {
992985 	CURSEG_HOT_DATA	= 0,	/* directory entry blocks */
....@@ -995,8 +988,11 @@
995988 	CURSEG_HOT_NODE,	/* direct node blocks of directory files */
996989 	CURSEG_WARM_NODE,	/* direct node blocks of normal files */
997990 	CURSEG_COLD_NODE,	/* indirect node blocks */
998
-	NO_CHECK_TYPE,
999
-	CURSEG_COLD_DATA_PINNED,/* cold data for pinned file */
991
+	NR_PERSISTENT_LOG,	/* number of persistent log */
992
+	CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
993
+				/* pinned file that needs consecutive block address */
994
+	CURSEG_ALL_DATA_ATGC,	/* SSR alloctor in hot/warm/cold data area */
995
+	NO_CHECK_TYPE,		/* number of persistent & inmem log */
1000996 };
1001997 
1002998 struct flush_cmd {
....@@ -1021,7 +1017,7 @@
10211017 	struct dirty_seglist_info *dirty_info;	/* dirty segment information */
10221018 	struct curseg_info *curseg_array;	/* active segment information */
10231019 
1024
-	struct rw_semaphore curseg_lock;	/* for preventing curseg change */
1020
+	struct f2fs_rwsem curseg_lock;	/* for preventing curseg change */
10251021 
10261022 	block_t seg0_blkaddr;		/* block address of 0'th segment */
10271023 	block_t main_blkaddr;		/* start block address of main area */
....@@ -1030,6 +1026,7 @@
10301026 	unsigned int segment_count;	/* total # of segments */
10311027 	unsigned int main_segments;	/* # of segments in main area */
10321028 	unsigned int reserved_segments;	/* # of reserved segments */
1029
+	unsigned int additional_reserved_segments;/* reserved segs for IO align feature */
10331030 	unsigned int ovp_segments;	/* # of overprovision segments */
10341031 
10351032 	/* a threshold to reclaim prefree segments */
....@@ -1095,8 +1092,8 @@
10951092  */
10961093 #define PAGE_TYPE_OF_BIO(type)	((type) > META ? META : (type))
10971094 enum page_type {
1098
-	DATA,
1099
-	NODE,
1095
+	DATA = 0,
1096
+	NODE = 1,	/* should not change this */
11001097 	META,
11011098 	NR_PAGE_TYPE,
11021099 	META_FLUSH,
....@@ -1186,6 +1183,7 @@
11861183 	bool retry;		/* need to reallocate block address */
11871184 	int compr_blocks;	/* # of compressed block addresses */
11881185 	bool encrypted;		/* indicate file is encrypted */
1186
+	bool post_read;		/* require post read */
11891187 	enum iostat_type io_type;	/* io type */
11901188 	struct writeback_control *io_wbc; /* writeback control */
11911189 	struct bio **bio;		/* bio for ipu */
....@@ -1204,11 +1202,11 @@
12041202 	struct bio *bio;		/* bios to merge */
12051203 	sector_t last_block_in_bio;	/* last block number */
12061204 	struct f2fs_io_info fio;	/* store buffered io info. */
1207
-	struct rw_semaphore io_rwsem;	/* blocking op for bio */
1205
+	struct f2fs_rwsem io_rwsem;	/* blocking op for bio */
12081206 	spinlock_t io_lock;		/* serialize DATA/NODE IOs */
12091207 	struct list_head io_list;	/* track fios */
12101208 	struct list_head bio_list;	/* bio entry list head */
1211
-	struct rw_semaphore bio_list_lock;	/* lock to protect bio entry list */
1209
+	struct f2fs_rwsem bio_list_lock;	/* lock to protect bio entry list */
12121210 };
12131211 
12141212 #define FDEV(i)				(sbi->devs[i])
....@@ -1222,6 +1220,7 @@
12221220 #ifdef CONFIG_BLK_DEV_ZONED
12231221 	unsigned int nr_blkz;		/* Total number of zones */
12241222 	unsigned long *blkz_seq;	/* Bitmap indicating sequential zones */
1223
+	block_t *zone_capacity_blocks;  /* Array of zone capacity in blks */
12251224 #endif
12261225 };
12271226 
....@@ -1241,6 +1240,18 @@
12411240 	unsigned long ino_num;			/* number of entries */
12421241 };
12431242 
1243
+/* for GC_AT */
1244
+struct atgc_management {
1245
+	bool atgc_enabled;			/* ATGC is enabled or not */
1246
+	struct rb_root_cached root;		/* root of victim rb-tree */
1247
+	struct list_head victim_list;		/* linked with all victim entries */
1248
+	unsigned int victim_count;		/* victim count in rb-tree */
1249
+	unsigned int candidate_ratio;		/* candidate ratio */
1250
+	unsigned int max_candidate_count;	/* max candidate count */
1251
+	unsigned int age_weight;		/* age weight, vblock_weight = 100 - age_weight */
1252
+	unsigned long long age_threshold;	/* age threshold */
1253
+};
1254
+
12441255 /* For s_flag in struct f2fs_sb_info */
12451256 enum {
12461257 	SBI_IS_DIRTY,				/* dirty flag for checkpoint */
....@@ -1257,6 +1268,7 @@
12571268 	SBI_QUOTA_SKIP_FLUSH,			/* skip flushing quota in current CP */
12581269 	SBI_QUOTA_NEED_REPAIR,			/* quota file may be corrupted */
12591270 	SBI_IS_RESIZEFS,			/* resizefs is in process */
1271
+	SBI_IS_FREEZING,			/* freezefs is in process */
12601272 };
12611273 
12621274 enum {
....@@ -1273,7 +1285,10 @@
12731285 	GC_NORMAL,
12741286 	GC_IDLE_CB,
12751287 	GC_IDLE_GREEDY,
1276
-	GC_URGENT,
1288
+	GC_IDLE_AT,
1289
+	GC_URGENT_HIGH,
1290
+	GC_URGENT_LOW,
1291
+	MAX_GC_MODE,
12771292 };
12781293 
12791294 enum {
....@@ -1307,36 +1322,158 @@
13071322 	FSYNC_MODE_NOBARRIER,	/* fsync behaves nobarrier based on posix */
13081323 };
13091324 
1325
+enum {
1326
+	COMPR_MODE_FS,		/*
1327
+				 * automatically compress compression
1328
+				 * enabled files
1329
+				 */
1330
+	COMPR_MODE_USER,	/*
1331
+				 * automatical compression is disabled.
1332
+				 * user can control the file compression
1333
+				 * using ioctls
1334
+				 */
1335
+};
1336
+
1337
+enum {
1338
+	MEMORY_MODE_NORMAL,	/* memory mode for normal devices */
1339
+	MEMORY_MODE_LOW,	/* memory mode for low memry devices */
1340
+};
1341
+
1342
+static inline int f2fs_test_bit(unsigned int nr, char *addr);
1343
+static inline void f2fs_set_bit(unsigned int nr, char *addr);
1344
+static inline void f2fs_clear_bit(unsigned int nr, char *addr);
1345
+
13101346 /*
1311
- * this value is set in page as a private data which indicate that
1312
- * the page is atomically written, and it is in inmem_pages list.
1347
+ * Layout of f2fs page.private:
1348
+ *
1349
+ * Layout A: lowest bit should be 1
1350
+ * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
1351
+ * bit 0	PAGE_PRIVATE_NOT_POINTER
1352
+ * bit 1	PAGE_PRIVATE_ATOMIC_WRITE
1353
+ * bit 2	PAGE_PRIVATE_DUMMY_WRITE
1354
+ * bit 3	PAGE_PRIVATE_ONGOING_MIGRATION
1355
+ * bit 4	PAGE_PRIVATE_INLINE_INODE
1356
+ * bit 5	PAGE_PRIVATE_REF_RESOURCE
1357
+ * bit 6-	f2fs private data
1358
+ *
1359
+ * Layout B: lowest bit should be 0
1360
+ * page.private is a wrapped pointer.
13131361  */
1314
-#define ATOMIC_WRITTEN_PAGE		((unsigned long)-1)
1315
-#define DUMMY_WRITTEN_PAGE		((unsigned long)-2)
1362
+enum {
1363
+	PAGE_PRIVATE_NOT_POINTER,		/* private contains non-pointer data */
1364
+	PAGE_PRIVATE_ATOMIC_WRITE,		/* data page from atomic write path */
1365
+	PAGE_PRIVATE_DUMMY_WRITE,		/* data page for padding aligned IO */
1366
+	PAGE_PRIVATE_ONGOING_MIGRATION,		/* data page which is on-going migrating */
1367
+	PAGE_PRIVATE_INLINE_INODE,		/* inode page contains inline data */
1368
+	PAGE_PRIVATE_REF_RESOURCE,		/* dirty page has referenced resources */
1369
+	PAGE_PRIVATE_MAX
1370
+};
13161371 
1317
-#define IS_ATOMIC_WRITTEN_PAGE(page)			\
1318
-		(page_private(page) == (unsigned long)ATOMIC_WRITTEN_PAGE)
1319
-#define IS_DUMMY_WRITTEN_PAGE(page)			\
1320
-		(page_private(page) == (unsigned long)DUMMY_WRITTEN_PAGE)
1372
+#define PAGE_PRIVATE_GET_FUNC(name, flagname) \
1373
+static inline bool page_private_##name(struct page *page) \
1374
+{ \
1375
+	return PagePrivate(page) && \
1376
+		test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
1377
+		test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1378
+}
13211379 
1322
-#ifdef CONFIG_FS_ENCRYPTION
1323
-#define DUMMY_ENCRYPTION_ENABLED(sbi) \
1324
-	(unlikely(F2FS_OPTION(sbi).dummy_enc_ctx.ctx != NULL))
1325
-#else
1326
-#define DUMMY_ENCRYPTION_ENABLED(sbi) (0)
1327
-#endif
1380
+#define PAGE_PRIVATE_SET_FUNC(name, flagname) \
1381
+static inline void set_page_private_##name(struct page *page) \
1382
+{ \
1383
+	if (!PagePrivate(page)) { \
1384
+		get_page(page); \
1385
+		SetPagePrivate(page); \
1386
+		set_page_private(page, 0); \
1387
+	} \
1388
+	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
1389
+	set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1390
+}
1391
+
1392
+#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
1393
+static inline void clear_page_private_##name(struct page *page) \
1394
+{ \
1395
+	clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1396
+	if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) { \
1397
+		set_page_private(page, 0); \
1398
+		if (PagePrivate(page)) { \
1399
+			ClearPagePrivate(page); \
1400
+			put_page(page); \
1401
+		}\
1402
+	} \
1403
+}
1404
+
1405
+PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
1406
+PAGE_PRIVATE_GET_FUNC(reference, REF_RESOURCE);
1407
+PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
1408
+PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
1409
+PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
1410
+PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE);
1411
+
1412
+PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
1413
+PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
1414
+PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
1415
+PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
1416
+PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE);
1417
+
1418
+PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
1419
+PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
1420
+PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
1421
+PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
1422
+PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE);
1423
+
1424
+static inline unsigned long get_page_private_data(struct page *page)
1425
+{
1426
+	unsigned long data = page_private(page);
1427
+
1428
+	if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
1429
+		return 0;
1430
+	return data >> PAGE_PRIVATE_MAX;
1431
+}
1432
+
1433
+static inline void set_page_private_data(struct page *page, unsigned long data)
1434
+{
1435
+	if (!PagePrivate(page)) {
1436
+		get_page(page);
1437
+		SetPagePrivate(page);
1438
+		set_page_private(page, 0);
1439
+	}
1440
+	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page));
1441
+	page_private(page) |= data << PAGE_PRIVATE_MAX;
1442
+}
1443
+
1444
+static inline void clear_page_private_data(struct page *page)
1445
+{
1446
+	page_private(page) &= (1 << PAGE_PRIVATE_MAX) - 1;
1447
+	if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) {
1448
+		set_page_private(page, 0);
1449
+		if (PagePrivate(page)) {
1450
+			ClearPagePrivate(page);
1451
+			put_page(page);
1452
+		}
1453
+	}
1454
+}
13281455 
13291456 /* For compression */
13301457 enum compress_algorithm_type {
13311458 	COMPRESS_LZO,
13321459 	COMPRESS_LZ4,
13331460 	COMPRESS_ZSTD,
1461
+	COMPRESS_LZORLE,
13341462 	COMPRESS_MAX,
13351463 };
13361464 
1337
-#define COMPRESS_DATA_RESERVED_SIZE		5
1465
+enum compress_flag {
1466
+	COMPRESS_CHKSUM,
1467
+	COMPRESS_MAX_FLAG,
1468
+};
1469
+
1470
+#define	COMPRESS_WATERMARK			20
1471
+#define	COMPRESS_PERCENT			20
1472
+
1473
+#define COMPRESS_DATA_RESERVED_SIZE		4
13381474 struct compress_data {
13391475 	__le32 clen;			/* compressed data size */
1476
+	__le32 chksum;			/* compressed data chksum */
13401477 	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
13411478 	u8 cdata[];			/* compressed data */
13421479 };
....@@ -1344,6 +1481,8 @@
13441481 #define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
13451482 
13461483 #define F2FS_COMPRESSED_PAGE_MAGIC	0xF5F2C000
1484
+
1485
+#define	COMPRESS_LEVEL_OFFSET	8
13471486 
13481487 /* compress context */
13491488 struct compress_ctx {
....@@ -1369,10 +1508,10 @@
13691508 	struct inode *inode;		/* inode the context belong to */
13701509 	struct page **rpages;		/* pages store raw data in cluster */
13711510 	unsigned int nr_rpages;		/* total page number in rpages */
1372
-	refcount_t ref;			/* referrence count of raw page */
1511
+	atomic_t pending_pages;		/* in-flight compressed page count */
13731512 };
13741513 
1375
-/* decompress io context for read IO path */
1514
+/* Context for decompressing one cluster on the read IO path */
13761515 struct decompress_io_ctx {
13771516 	u32 magic;			/* magic number to indicate page is compressed */
13781517 	struct inode *inode;		/* inode the context belong to */
....@@ -1388,22 +1527,50 @@
13881527 	struct compress_data *cbuf;	/* virtual mapped address on cpages */
13891528 	size_t rlen;			/* valid data length in rbuf */
13901529 	size_t clen;			/* valid data length in cbuf */
1391
-	refcount_t ref;			/* referrence count of compressed page */
1392
-	bool failed;			/* indicate IO error during decompression */
1530
+
1531
+	/*
1532
+	 * The number of compressed pages remaining to be read in this cluster.
1533
+	 * This is initially nr_cpages.  It is decremented by 1 each time a page
1534
+	 * has been read (or failed to be read).  When it reaches 0, the cluster
1535
+	 * is decompressed (or an error is reported).
1536
+	 *
1537
+	 * If an error occurs before all the pages have been submitted for I/O,
1538
+	 * then this will never reach 0.  In this case the I/O submitter is
1539
+	 * responsible for calling f2fs_decompress_end_io() instead.
1540
+	 */
1541
+	atomic_t remaining_pages;
1542
+
1543
+	/*
1544
+	 * Number of references to this decompress_io_ctx.
1545
+	 *
1546
+	 * One reference is held for I/O completion.  This reference is dropped
1547
+	 * after the pagecache pages are updated and unlocked -- either after
1548
+	 * decompression (and verity if enabled), or after an error.
1549
+	 *
1550
+	 * In addition, each compressed page holds a reference while it is in a
1551
+	 * bio.  These references are necessary prevent compressed pages from
1552
+	 * being freed while they are still in a bio.
1553
+	 */
1554
+	refcount_t refcnt;
1555
+
1556
+	bool failed;			/* IO error occurred before decompression? */
1557
+	bool need_verity;		/* need fs-verity verification after decompression? */
13931558 	void *private;			/* payload buffer for specified decompression algorithm */
13941559 	void *private2;			/* extra payload buffer */
1560
+	struct work_struct verity_work;	/* work to verify the decompressed pages */
1561
+	struct work_struct free_work;	/* work for late free this structure itself */
13951562 };
13961563 
13971564 #define NULL_CLUSTER			((unsigned int)(~0))
13981565 #define MIN_COMPRESS_LOG_SIZE		2
13991566 #define MAX_COMPRESS_LOG_SIZE		8
1400
-#define MAX_COMPRESS_WINDOW_SIZE	((PAGE_SIZE) << MAX_COMPRESS_LOG_SIZE)
1567
+#define MAX_COMPRESS_WINDOW_SIZE(log_size)	((PAGE_SIZE) << (log_size))
14011568 
14021569 struct f2fs_sb_info {
14031570 	struct super_block *sb;			/* pointer to VFS super block */
14041571 	struct proc_dir_entry *s_proc;		/* proc entry */
14051572 	struct f2fs_super_block *raw_super;	/* raw super block pointer */
1406
-	struct rw_semaphore sb_lock;		/* lock for raw super block */
1573
+	struct f2fs_rwsem sb_lock;		/* lock for raw super block */
14071574 	int valid_super_block;			/* valid super block no */
14081575 	unsigned long s_flag;				/* flags for sbi */
14091576 	struct mutex writepages;		/* mutex for writepages() */
....@@ -1423,7 +1590,7 @@
14231590 	/* for bio operations */
14241591 	struct f2fs_bio_info *write_io[NR_PAGE_TYPE];	/* for write bios */
14251592 	/* keep migration IO order for LFS mode */
1426
-	struct rw_semaphore io_order_lock;
1593
+	struct f2fs_rwsem io_order_lock;
14271594 	mempool_t *write_io_dummy;		/* Dummy pages */
14281595 
14291596 	/* for checkpoint */
....@@ -1431,15 +1598,16 @@
14311598 	int cur_cp_pack;			/* remain current cp pack */
14321599 	spinlock_t cp_lock;			/* for flag in ckpt */
14331600 	struct inode *meta_inode;		/* cache meta blocks */
1434
-	struct mutex cp_mutex;			/* checkpoint procedure lock */
1435
-	struct rw_semaphore cp_rwsem;		/* blocking FS operations */
1436
-	struct rw_semaphore node_write;		/* locking node writes */
1437
-	struct rw_semaphore node_change;	/* locking node change */
1601
+	struct f2fs_rwsem cp_global_sem;	/* checkpoint procedure lock */
1602
+	struct f2fs_rwsem cp_rwsem;		/* blocking FS operations */
1603
+	struct f2fs_rwsem node_write;		/* locking node writes */
1604
+	struct f2fs_rwsem node_change;	/* locking node change */
14381605 	wait_queue_head_t cp_wait;
14391606 	unsigned long last_time[MAX_TIME];	/* to store time in jiffies */
14401607 	long interval_time[MAX_TIME];		/* to store thresholds */
1608
+	struct ckpt_req_control cprc_info;	/* for checkpoint request control */
14411609 
1442
-	struct inode_management im[MAX_INO_ENTRY];      /* manage inode cache */
1610
+	struct inode_management im[MAX_INO_ENTRY];	/* manage inode cache */
14431611 
14441612 	spinlock_t fsync_node_lock;		/* for node entry lock */
14451613 	struct list_head fsync_node_list;	/* node list head */
....@@ -1455,14 +1623,13 @@
14551623 	struct mutex flush_lock;		/* for flush exclusion */
14561624 
14571625 	/* for extent tree cache */
1458
-	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
1459
-	struct mutex extent_tree_lock;	/* locking extent radix tree */
1460
-	struct list_head extent_list;		/* lru list for shrinker */
1461
-	spinlock_t extent_lock;			/* locking extent lru list */
1462
-	atomic_t total_ext_tree;		/* extent tree count */
1463
-	struct list_head zombie_list;		/* extent zombie tree list */
1464
-	atomic_t total_zombie_tree;		/* extent zombie tree count */
1465
-	atomic_t total_ext_node;		/* extent info count */
1626
+	struct extent_tree_info extent_tree[NR_EXTENT_CACHES];
1627
+	atomic64_t allocated_data_blocks;	/* for block age extent_cache */
1628
+
1629
+	/* The threshold used for hot and warm data seperation*/
1630
+	unsigned int hot_data_age_threshold;
1631
+	unsigned int warm_data_age_threshold;
1632
+	unsigned int last_age_weight;
14661633 
14671634 	/* basic filesystem units */
14681635 	unsigned int log_sectors_per_block;	/* log2 sectors per block */
....@@ -1478,9 +1645,9 @@
14781645 	unsigned int total_sections;		/* total section count */
14791646 	unsigned int total_node_count;		/* total node block count */
14801647 	unsigned int total_valid_node_count;	/* valid node block count */
1481
-	loff_t max_file_blocks;			/* max block index of file */
14821648 	int dir_level;				/* directory level */
14831649 	int readdir_ra;				/* readahead inode in readdir */
1650
+	u64 max_io_bytes;			/* max io bytes to merge IOs */
14841651 
14851652 	block_t user_block_count;		/* # of user blocks */
14861653 	block_t total_valid_block_count;	/* # of valid blocks */
....@@ -1493,7 +1660,7 @@
14931660 	block_t unusable_block_count;		/* # of blocks saved by last cp */
14941661 
14951662 	unsigned int nquota_files;		/* # of quota sysfile */
1496
-	struct rw_semaphore quota_sem;		/* blocking cp for flags */
1663
+	struct f2fs_rwsem quota_sem;		/* blocking cp for flags */
14971664 
14981665 	/* # of pages, see count_type */
14991666 	atomic_t nr_pages[NR_COUNT_TYPE];
....@@ -1509,22 +1676,24 @@
15091676 	struct f2fs_mount_info mount_opt;	/* mount options */
15101677 
15111678 	/* for cleaning operations */
1512
-	struct rw_semaphore gc_lock;		/*
1679
+	struct f2fs_rwsem gc_lock;		/*
15131680 						 * semaphore for GC, avoid
15141681 						 * race between GC and GC or CP
15151682 						 */
15161683 	struct f2fs_gc_kthread	*gc_thread;	/* GC thread */
1684
+	struct atgc_management am;		/* atgc management */
15171685 	unsigned int cur_victim_sec;		/* current victim section num */
15181686 	unsigned int gc_mode;			/* current GC state */
15191687 	unsigned int next_victim_seg[2];	/* next segment in victim section */
1688
+
15201689 	/* for skip statistic */
1521
-	unsigned int atomic_files;              /* # of opened atomic file */
1690
+	unsigned int atomic_files;		/* # of opened atomic file */
15221691 	unsigned long long skipped_atomic_files[2];	/* FG_GC and BG_GC */
15231692 	unsigned long long skipped_gc_rwsem;		/* FG_GC only */
15241693 
15251694 	/* threshold for gc trials on pinned files */
15261695 	u64 gc_pin_file_threshold;
1527
-	struct rw_semaphore pin_sem;
1696
+	struct f2fs_rwsem pin_sem;
15281697 
15291698 	/* maximum # of trials to find a victim segment for SSR and GC */
15301699 	unsigned int max_victim_search;
....@@ -1543,15 +1712,19 @@
15431712 	unsigned int segment_count[2];		/* # of allocated segments */
15441713 	unsigned int block_count[2];		/* # of allocated blocks */
15451714 	atomic_t inplace_count;		/* # of inplace update */
1546
-	atomic64_t total_hit_ext;		/* # of lookup extent cache */
1547
-	atomic64_t read_hit_rbtree;		/* # of hit rbtree extent node */
1548
-	atomic64_t read_hit_largest;		/* # of hit largest extent node */
1549
-	atomic64_t read_hit_cached;		/* # of hit cached extent node */
1715
+	/* # of lookup extent cache */
1716
+	atomic64_t total_hit_ext[NR_EXTENT_CACHES];
1717
+	/* # of hit rbtree extent node */
1718
+	atomic64_t read_hit_rbtree[NR_EXTENT_CACHES];
1719
+	/* # of hit cached extent node */
1720
+	atomic64_t read_hit_cached[NR_EXTENT_CACHES];
1721
+	/* # of hit largest extent node in read extent cache */
1722
+	atomic64_t read_hit_largest;
15501723 	atomic_t inline_xattr;			/* # of inline_xattr inodes */
15511724 	atomic_t inline_inode;			/* # of inline_data inodes */
15521725 	atomic_t inline_dir;			/* # of inline_dentry inodes */
15531726 	atomic_t compr_inode;			/* # of compressed inodes */
1554
-	atomic_t compr_blocks;			/* # of compressed blocks */
1727
+	atomic64_t compr_blocks;		/* # of compressed blocks */
15551728 	atomic_t vw_cnt;			/* # of volatile writes */
15561729 	atomic_t max_aw_cnt;			/* max # of atomic writes */
15571730 	atomic_t max_vw_cnt;			/* max # of volatile writes */
....@@ -1574,8 +1747,14 @@
15741747 	unsigned int node_io_flag;
15751748 
15761749 	/* For sysfs suppport */
1577
-	struct kobject s_kobj;
1750
+	struct kobject s_kobj;			/* /sys/fs/f2fs/<devname> */
15781751 	struct completion s_kobj_unregister;
1752
+
1753
+	struct kobject s_stat_kobj;		/* /sys/fs/f2fs/<devname>/stat */
1754
+	struct completion s_stat_kobj_unregister;
1755
+
1756
+	struct kobject s_feature_list_kobj;		/* /sys/fs/f2fs/<devname>/feature_list */
1757
+	struct completion s_feature_list_kobj_unregister;
15791758 
15801759 	/* For shrinker support */
15811760 	struct list_head s_list;
....@@ -1600,6 +1779,26 @@
16001779 
16011780 	struct kmem_cache *inline_xattr_slab;	/* inline xattr entry */
16021781 	unsigned int inline_xattr_slab_size;	/* default inline xattr slab size */
1782
+
1783
+	/* For reclaimed segs statistics per each GC mode */
1784
+	unsigned int gc_segment_mode;		/* GC state for reclaimed segments */
1785
+	unsigned int gc_reclaimed_segs[MAX_GC_MODE];	/* Reclaimed segs for each mode */
1786
+
1787
+#ifdef CONFIG_F2FS_FS_COMPRESSION
1788
+	struct kmem_cache *page_array_slab;	/* page array entry */
1789
+	unsigned int page_array_slab_size;	/* default page array slab size */
1790
+
1791
+	/* For runtime compression statistics */
1792
+	u64 compr_written_block;
1793
+	u64 compr_saved_block;
1794
+	u32 compr_new_inode;
1795
+
1796
+	/* For compressed block cache */
1797
+	struct inode *compress_inode;		/* cache compressed blocks */
1798
+	unsigned int compress_percent;		/* cache page percentage */
1799
+	unsigned int compress_watermark;	/* cache page watermark */
1800
+	atomic_t compress_page_hit;		/* cache hit count */
1801
+#endif
16031802 };
16041803 
16051804 struct f2fs_private_dio {
....@@ -1651,13 +1850,6 @@
16511850 	return sbi->s_ndevs > 1;
16521851 }
16531852 
1654
-/* For write statistics. Suppose sector size is 512 bytes,
1655
- * and the return value is in kbytes. s is of struct f2fs_sb_info.
1656
- */
1657
-#define BD_PART_WRITTEN(s)						 \
1658
-(((u64)part_stat_read((s)->sb->s_bdev->bd_part, sectors[STAT_WRITE]) -   \
1659
-		(s)->sectors_written_start) >> 1)
1660
-
16611853 static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
16621854 {
16631855 	unsigned long now = jiffies;
....@@ -1707,7 +1899,6 @@
17071899 	BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
17081900 
17091901 	desc.shash.tfm = sbi->s_chksum_driver;
1710
-	desc.shash.flags = 0;
17111902 	*(u32 *)desc.ctx = crc;
17121903 
17131904 	err = crypto_shash_update(&desc.shash, address, length);
....@@ -1925,29 +2116,93 @@
19252116 	return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set;
19262117 }
19272118 
2119
+#define init_f2fs_rwsem(sem)					\
2120
+do {								\
2121
+	static struct lock_class_key __key;			\
2122
+								\
2123
+	__init_f2fs_rwsem((sem), #sem, &__key);			\
2124
+} while (0)
2125
+
2126
+static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem,
2127
+		const char *sem_name, struct lock_class_key *key)
2128
+{
2129
+	__init_rwsem(&sem->internal_rwsem, sem_name, key);
2130
+	init_waitqueue_head(&sem->read_waiters);
2131
+}
2132
+
2133
+static inline int f2fs_rwsem_is_locked(struct f2fs_rwsem *sem)
2134
+{
2135
+	return rwsem_is_locked(&sem->internal_rwsem);
2136
+}
2137
+
2138
+static inline int f2fs_rwsem_is_contended(struct f2fs_rwsem *sem)
2139
+{
2140
+	return rwsem_is_contended(&sem->internal_rwsem);
2141
+}
2142
+
2143
+static inline void f2fs_down_read(struct f2fs_rwsem *sem)
2144
+{
2145
+	wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem));
2146
+}
2147
+
2148
+static inline int f2fs_down_read_trylock(struct f2fs_rwsem *sem)
2149
+{
2150
+	return down_read_trylock(&sem->internal_rwsem);
2151
+}
2152
+
2153
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
2154
+static inline void f2fs_down_read_nested(struct f2fs_rwsem *sem, int subclass)
2155
+{
2156
+	down_read_nested(&sem->internal_rwsem, subclass);
2157
+}
2158
+#else
2159
+#define f2fs_down_read_nested(sem, subclass) f2fs_down_read(sem)
2160
+#endif
2161
+
2162
+static inline void f2fs_up_read(struct f2fs_rwsem *sem)
2163
+{
2164
+	up_read(&sem->internal_rwsem);
2165
+}
2166
+
2167
+static inline void f2fs_down_write(struct f2fs_rwsem *sem)
2168
+{
2169
+	down_write(&sem->internal_rwsem);
2170
+}
2171
+
2172
+static inline int f2fs_down_write_trylock(struct f2fs_rwsem *sem)
2173
+{
2174
+	return down_write_trylock(&sem->internal_rwsem);
2175
+}
2176
+
2177
+static inline void f2fs_up_write(struct f2fs_rwsem *sem)
2178
+{
2179
+	up_write(&sem->internal_rwsem);
2180
+	wake_up_all(&sem->read_waiters);
2181
+}
2182
+
19282183 static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
19292184 {
1930
-	down_read(&sbi->cp_rwsem);
2185
+	f2fs_down_read(&sbi->cp_rwsem);
19312186 }
19322187 
19332188 static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
19342189 {
1935
-	return down_read_trylock(&sbi->cp_rwsem);
2190
+	return f2fs_down_read_trylock(&sbi->cp_rwsem);
19362191 }
19372192 
19382193 static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
19392194 {
1940
-	up_read(&sbi->cp_rwsem);
2195
+	f2fs_up_read(&sbi->cp_rwsem);
19412196 }
19422197 
19432198 static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
19442199 {
1945
-	down_write(&sbi->cp_rwsem);
2200
+	f2fs_down_write(&sbi->cp_rwsem);
19462201 }
19472202 
19482203 static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
19492204 {
1950
-	up_write(&sbi->cp_rwsem);
2205
+	f2fs_up_write(&sbi->cp_rwsem);
19512206 }
19522207 
19532208 static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
....@@ -2037,6 +2292,11 @@
20372292 
20382293 	if (!__allow_reserved_blocks(sbi, inode, true))
20392294 		avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2295
+
2296
+	if (F2FS_IO_ALIGNED(sbi))
2297
+		avail_user_block_count -= sbi->blocks_per_seg *
2298
+				SM_I(sbi)->additional_reserved_segments;
2299
+
20402300 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
20412301 		if (avail_user_block_count > sbi->unusable_block_count)
20422302 			avail_user_block_count -= sbi->unusable_block_count;
....@@ -2199,6 +2459,7 @@
21992459 static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
22002460 {
22012461 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2462
+	void *tmp_ptr = &ckpt->sit_nat_version_bitmap;
22022463 	int offset;
22032464 
22042465 	if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
....@@ -2208,7 +2469,7 @@
22082469 		 * if large_nat_bitmap feature is enabled, leave checksum
22092470 		 * protection for all nat/sit bitmaps.
22102471 		 */
2211
-		return &ckpt->sit_nat_version_bitmap + offset + sizeof(__le32);
2472
+		return tmp_ptr + offset + sizeof(__le32);
22122473 	}
22132474 
22142475 	if (__cp_payload(sbi) > 0) {
....@@ -2219,7 +2480,7 @@
22192480 	} else {
22202481 		offset = (flag == NAT_BITMAP) ?
22212482 			le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
2222
-		return &ckpt->sit_nat_version_bitmap + offset;
2483
+		return tmp_ptr + offset;
22232484 	}
22242485 }
22252486 
....@@ -2251,6 +2512,7 @@
22512512 	return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
22522513 }
22532514 
2515
+extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
22542516 static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
22552517 					struct inode *inode, bool is_inode)
22562518 {
....@@ -2282,6 +2544,11 @@
22822544 
22832545 	if (!__allow_reserved_blocks(sbi, inode, false))
22842546 		valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
2547
+
2548
+	if (F2FS_IO_ALIGNED(sbi))
2549
+		valid_block_count += sbi->blocks_per_seg *
2550
+				SM_I(sbi)->additional_reserved_segments;
2551
+
22852552 	user_block_count = sbi->user_block_count;
22862553 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
22872554 		user_block_count -= sbi->unusable_block_count;
....@@ -2326,11 +2593,17 @@
23262593 {
23272594 	spin_lock(&sbi->stat_lock);
23282595 
2329
-	f2fs_bug_on(sbi, !sbi->total_valid_block_count);
2330
-	f2fs_bug_on(sbi, !sbi->total_valid_node_count);
2596
+	if (unlikely(!sbi->total_valid_block_count ||
2597
+			!sbi->total_valid_node_count)) {
2598
+		f2fs_warn(sbi, "dec_valid_node_count: inconsistent block counts, total_valid_block:%u, total_valid_node:%u",
2599
+			  sbi->total_valid_block_count,
2600
+			  sbi->total_valid_node_count);
2601
+		set_sbi_flag(sbi, SBI_NEED_FSCK);
2602
+	} else {
2603
+		sbi->total_valid_block_count--;
2604
+		sbi->total_valid_node_count--;
2605
+	}
23312606 
2332
-	sbi->total_valid_node_count--;
2333
-	sbi->total_valid_block_count--;
23342607 	if (sbi->reserved_blocks &&
23352608 		sbi->current_reserved_blocks < sbi->reserved_blocks)
23362609 		sbi->current_reserved_blocks++;
....@@ -2458,25 +2731,36 @@
24582731 	return entry;
24592732 }
24602733 
2461
-static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2734
+static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
24622735 {
2463
-	if (sbi->gc_mode == GC_URGENT)
2464
-		return true;
2465
-
24662736 	if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
24672737 		get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
24682738 		get_pages(sbi, F2FS_WB_CP_DATA) ||
24692739 		get_pages(sbi, F2FS_DIO_READ) ||
24702740 		get_pages(sbi, F2FS_DIO_WRITE))
2471
-		return false;
2741
+		return true;
24722742 
24732743 	if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
24742744 			atomic_read(&SM_I(sbi)->dcc_info->queued_discard))
2475
-		return false;
2745
+		return true;
24762746 
24772747 	if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
24782748 			atomic_read(&SM_I(sbi)->fcc_info->queued_flush))
2749
+		return true;
2750
+	return false;
2751
+}
2752
+
2753
+static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2754
+{
2755
+	if (sbi->gc_mode == GC_URGENT_HIGH)
2756
+		return true;
2757
+
2758
+	if (is_inflight_io(sbi, type))
24792759 		return false;
2760
+
2761
+	if (sbi->gc_mode == GC_URGENT_LOW &&
2762
+			(type == DISCARD_TIME || type == GC_TIME))
2763
+		return true;
24802764 
24812765 	return f2fs_time_over(sbi, type);
24822766 }
....@@ -2643,17 +2927,18 @@
26432927 	case FI_NEW_INODE:
26442928 		if (set)
26452929 			return;
2646
-		/* fall through */
2930
+		fallthrough;
26472931 	case FI_DATA_EXIST:
26482932 	case FI_INLINE_DOTS:
26492933 	case FI_PIN_FILE:
2934
+	case FI_COMPRESS_RELEASED:
26502935 		f2fs_mark_inode_dirty_sync(inode, true);
26512936 	}
26522937 }
26532938 
26542939 static inline void set_inode_flag(struct inode *inode, int flag)
26552940 {
2656
-	test_and_set_bit(flag, F2FS_I(inode)->flags);
2941
+	set_bit(flag, F2FS_I(inode)->flags);
26572942 	__mark_inode_dirty_flag(inode, flag, true);
26582943 }
26592944 
....@@ -2664,7 +2949,7 @@
26642949 
26652950 static inline void clear_inode_flag(struct inode *inode, int flag)
26662951 {
2667
-	test_and_clear_bit(flag, F2FS_I(inode)->flags);
2952
+	clear_bit(flag, F2FS_I(inode)->flags);
26682953 	__mark_inode_dirty_flag(inode, flag, false);
26692954 }
26702955 
....@@ -2768,6 +3053,8 @@
27683053 		set_bit(FI_EXTRA_ATTR, fi->flags);
27693054 	if (ri->i_inline & F2FS_PIN_FILE)
27703055 		set_bit(FI_PIN_FILE, fi->flags);
3056
+	if (ri->i_inline & F2FS_COMPRESS_RELEASED)
3057
+		set_bit(FI_COMPRESS_RELEASED, fi->flags);
27713058 }
27723059 
27733060 static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
....@@ -2788,6 +3075,8 @@
27883075 		ri->i_inline |= F2FS_EXTRA_ATTR;
27893076 	if (is_inode_flag_set(inode, FI_PIN_FILE))
27903077 		ri->i_inline |= F2FS_PIN_FILE;
3078
+	if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED))
3079
+		ri->i_inline |= F2FS_COMPRESS_RELEASED;
27913080 }
27923081 
27933082 static inline int f2fs_has_extra_attr(struct inode *inode)
....@@ -2804,6 +3093,22 @@
28043093 {
28053094 	return S_ISREG(inode->i_mode) &&
28063095 		is_inode_flag_set(inode, FI_COMPRESSED_FILE);
3096
+}
3097
+
3098
+static inline bool f2fs_need_compress_data(struct inode *inode)
3099
+{
3100
+	int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
3101
+
3102
+	if (!f2fs_compressed_file(inode))
3103
+		return false;
3104
+
3105
+	if (compress_mode == COMPR_MODE_FS)
3106
+		return true;
3107
+	else if (compress_mode == COMPR_MODE_USER &&
3108
+			is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
3109
+		return true;
3110
+
3111
+	return false;
28073112 }
28083113 
28093114 static inline unsigned int addrs_per_inode(struct inode *inode)
....@@ -2980,25 +3285,6 @@
29803285 	return false;
29813286 }
29823287 
2983
-static inline bool f2fs_may_extent_tree(struct inode *inode)
2984
-{
2985
-	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2986
-
2987
-	if (!test_opt(sbi, EXTENT_CACHE) ||
2988
-			is_inode_flag_set(inode, FI_NO_EXTENT) ||
2989
-			is_inode_flag_set(inode, FI_COMPRESSED_FILE))
2990
-		return false;
2991
-
2992
-	/*
2993
-	 * for recovered files during mount do not create extents
2994
-	 * if shrinker is not registered.
2995
-	 */
2996
-	if (list_empty(&sbi->s_list))
2997
-		return false;
2998
-
2999
-	return S_ISREG(inode->i_mode);
3000
-}
3001
-
30023288 static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
30033289 					size_t size, gfp_t flags)
30043290 {
....@@ -3122,27 +3408,6 @@
31223408 	return true;
31233409 }
31243410 
3125
-static inline void f2fs_set_page_private(struct page *page,
3126
-						unsigned long data)
3127
-{
3128
-	if (PagePrivate(page))
3129
-		return;
3130
-
3131
-	get_page(page);
3132
-	SetPagePrivate(page);
3133
-	set_page_private(page, data);
3134
-}
3135
-
3136
-static inline void f2fs_clear_page_private(struct page *page)
3137
-{
3138
-	if (!PagePrivate(page))
3139
-		return;
3140
-
3141
-	set_page_private(page, 0);
3142
-	ClearPagePrivate(page);
3143
-	f2fs_put_page(page, 0);
3144
-}
3145
-
31463411 /*
31473412  * file.c
31483413  */
....@@ -3248,7 +3513,9 @@
32483513 void f2fs_inode_synced(struct inode *inode);
32493514 int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
32503515 int f2fs_quota_sync(struct super_block *sb, int type);
3516
+loff_t max_file_blocks(struct inode *inode);
32513517 void f2fs_quota_off_umount(struct super_block *sb);
3518
+void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason);
32523519 int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
32533520 int f2fs_sync_fs(struct super_block *sb, int sync);
32543521 int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
....@@ -3261,7 +3528,6 @@
32613528 /*
32623529  * node.c
32633530  */
3264
-struct dnode_of_data;
32653531 struct node_info;
32663532 
32673533 int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
....@@ -3274,7 +3540,7 @@
32743540 bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
32753541 bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
32763542 int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3277
-						struct node_info *ni);
3543
+				struct node_info *ni, bool checkpoint_context);
32783544 pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
32793545 int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
32803546 int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
....@@ -3288,7 +3554,7 @@
32883554 struct page *f2fs_get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid);
32893555 struct page *f2fs_get_node_page_ra(struct page *parent, int start);
32903556 int f2fs_move_node_page(struct page *node_page, int gc_type);
3291
-int f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
3557
+void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
32923558 int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
32933559 			struct writeback_control *wbc, bool atomic,
32943560 			unsigned int *seq_id);
....@@ -3338,9 +3604,16 @@
33383604 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
33393605 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
33403606 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
3341
-void allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3607
+bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
3608
+void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3609
+void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3610
+void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3611
+void f2fs_get_new_segment(struct f2fs_sb_info *sbi,
3612
+			unsigned int *newseg, bool new_sec, int dir);
3613
+void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
33423614 					unsigned int start, unsigned int end);
3343
-void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi, int type);
3615
+void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force);
3616
+void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
33443617 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
33453618 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
33463619 					struct cp_control *cpc);
....@@ -3355,7 +3628,8 @@
33553628 int f2fs_inplace_write_data(struct f2fs_io_info *fio);
33563629 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
33573630 			block_t old_blkaddr, block_t new_blkaddr,
3358
-			bool recover_curseg, bool recover_newaddr);
3631
+			bool recover_curseg, bool recover_newaddr,
3632
+			bool from_gc);
33593633 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
33603634 			block_t old_addr, block_t new_addr,
33613635 			unsigned char version, bool recover_curseg,
....@@ -3363,7 +3637,7 @@
33633637 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
33643638 			block_t old_blkaddr, block_t *new_blkaddr,
33653639 			struct f2fs_summary *sum, int type,
3366
-			struct f2fs_io_info *fio, bool add_list);
3640
+			struct f2fs_io_info *fio);
33673641 void f2fs_wait_on_page_writeback(struct page *page,
33683642 			enum page_type type, bool ordered, bool locked);
33693643 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
....@@ -3374,6 +3648,8 @@
33743648 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
33753649 			unsigned int val, int alloc);
33763650 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3651
+int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi);
3652
+int f2fs_check_write_pointer(struct f2fs_sb_info *sbi);
33773653 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
33783654 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
33793655 int __init f2fs_create_segment_manager_caches(void);
....@@ -3381,14 +3657,20 @@
33813657 int f2fs_rw_hint_to_seg_type(enum rw_hint hint);
33823658 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
33833659 			enum page_type type, enum temp_type temp);
3660
+unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
3661
+			unsigned int segno);
3662
+unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3663
+			unsigned int segno);
33843664 
33853665 /*
33863666  * checkpoint.c
33873667  */
3388
-void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io);
3668
+void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
3669
+							unsigned char reason);
3670
+void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
33893671 struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
33903672 struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3391
-struct page *f2fs_get_meta_page_nofail(struct f2fs_sb_info *sbi, pgoff_t index);
3673
+struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
33923674 struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index);
33933675 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
33943676 					block_t blkaddr, int type);
....@@ -3414,19 +3696,24 @@
34143696 int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
34153697 void f2fs_update_dirty_page(struct inode *inode, struct page *page);
34163698 void f2fs_remove_dirty_inode(struct inode *inode);
3417
-int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type);
3699
+int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
3700
+								bool from_cp);
34183701 void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
3702
+u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
34193703 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
34203704 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
34213705 int __init f2fs_create_checkpoint_caches(void);
34223706 void f2fs_destroy_checkpoint_caches(void);
3707
+int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
3708
+int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
3709
+void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
3710
+void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
34233711 
34243712 /*
34253713  * data.c
34263714  */
34273715 int __init f2fs_init_bioset(void);
34283716 void f2fs_destroy_bioset(void);
3429
-struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio);
34303717 int f2fs_init_bio_entry_cache(void);
34313718 void f2fs_destroy_bio_entry_cache(void);
34323719 void f2fs_submit_bio(struct f2fs_sb_info *sbi,
....@@ -3451,9 +3738,6 @@
34513738 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
34523739 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from);
34533740 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
3454
-int f2fs_mpage_readpages(struct address_space *mapping,
3455
-			struct list_head *pages, struct page *page,
3456
-			unsigned nr_pages, bool is_readahead);
34573741 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
34583742 			int op_flags, bool for_write);
34593743 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);
....@@ -3462,7 +3746,7 @@
34623746 struct page *f2fs_get_new_data_page(struct inode *inode,
34633747 			struct page *ipage, pgoff_t index, bool new_i_size);
34643748 int f2fs_do_write_data_page(struct f2fs_io_info *fio);
3465
-void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
3749
+void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
34663750 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
34673751 			int create, int flag);
34683752 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
....@@ -3474,7 +3758,7 @@
34743758 				struct bio **bio, sector_t *last_block,
34753759 				struct writeback_control *wbc,
34763760 				enum iostat_type io_type,
3477
-				int compr_blocks);
3761
+				int compr_blocks, bool allow_balance);
34783762 void f2fs_invalidate_page(struct page *page, unsigned int offset,
34793763 			unsigned int length);
34803764 int f2fs_release_page(struct page *page, gfp_t wait);
....@@ -3483,7 +3767,7 @@
34833767 			struct page *page, enum migrate_mode mode);
34843768 #endif
34853769 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
3486
-void f2fs_clear_radix_tree_dirty_tag(struct page *page);
3770
+void f2fs_clear_page_cache_dirty_tag(struct page *page);
34873771 int f2fs_init_post_read_processing(void);
34883772 void f2fs_destroy_post_read_processing(void);
34893773 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
....@@ -3495,16 +3779,20 @@
34953779 int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
34963780 void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
34973781 block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
3498
-int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background,
3782
+int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background, bool force,
34993783 			unsigned int segno);
35003784 void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
35013785 int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count);
3786
+int __init f2fs_create_garbage_collection_cache(void);
3787
+void f2fs_destroy_garbage_collection_cache(void);
35023788 
35033789 /*
35043790  * recovery.c
35053791  */
35063792 int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
35073793 bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
3794
+int __init f2fs_create_recovery_cache(void);
3795
+void f2fs_destroy_recovery_cache(void);
35083796 
35093797 /*
35103798  * debug.c
....@@ -3515,9 +3803,19 @@
35153803 	struct f2fs_sb_info *sbi;
35163804 	int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
35173805 	int main_area_segs, main_area_sections, main_area_zones;
3518
-	unsigned long long hit_largest, hit_cached, hit_rbtree;
3519
-	unsigned long long hit_total, total_ext;
3520
-	int ext_tree, zombie_tree, ext_node;
3806
+	unsigned long long hit_cached[NR_EXTENT_CACHES];
3807
+	unsigned long long hit_rbtree[NR_EXTENT_CACHES];
3808
+	unsigned long long total_ext[NR_EXTENT_CACHES];
3809
+	unsigned long long hit_total[NR_EXTENT_CACHES];
3810
+	int ext_tree[NR_EXTENT_CACHES];
3811
+	int zombie_tree[NR_EXTENT_CACHES];
3812
+	int ext_node[NR_EXTENT_CACHES];
3813
+	/* to count memory footprint */
3814
+	unsigned long long ext_mem[NR_EXTENT_CACHES];
3815
+	/* for read extent cache */
3816
+	unsigned long long hit_largest;
3817
+	/* for block age extent cache */
3818
+	unsigned long long allocated_data_blocks;
35213819 	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
35223820 	int ndirty_data, ndirty_qdata;
35233821 	int inmem_pages;
....@@ -3533,14 +3831,18 @@
35333831 	int nr_discarding, nr_discarded;
35343832 	int nr_discard_cmd;
35353833 	unsigned int undiscard_blks;
3834
+	int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
3835
+	unsigned int cur_ckpt_time, peak_ckpt_time;
35363836 	int inline_xattr, inline_inode, inline_dir, append, update, orphans;
3537
-	int compr_inode, compr_blocks;
3837
+	int compr_inode;
3838
+	unsigned long long compr_blocks;
35383839 	int aw_cnt, max_aw_cnt, vw_cnt, max_vw_cnt;
35393840 	unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
35403841 	unsigned int bimodal, avg_vblocks;
35413842 	int util_free, util_valid, util_invalid;
35423843 	int rsvd_segs, overp_segs;
3543
-	int dirty_count, node_pages, meta_pages;
3844
+	int dirty_count, node_pages, meta_pages, compress_pages;
3845
+	int compress_page_hit;
35443846 	int prefree_count, call_count, cp_count, bg_cp_count;
35453847 	int tot_segs, node_segs, data_segs, free_segs, free_secs;
35463848 	int bg_node_segs, bg_data_segs;
....@@ -3550,6 +3852,9 @@
35503852 	int curseg[NR_CURSEG_TYPE];
35513853 	int cursec[NR_CURSEG_TYPE];
35523854 	int curzone[NR_CURSEG_TYPE];
3855
+	unsigned int dirty_seg[NR_CURSEG_TYPE];
3856
+	unsigned int full_seg[NR_CURSEG_TYPE];
3857
+	unsigned int valid_blks[NR_CURSEG_TYPE];
35533858 
35543859 	unsigned int meta_count[META_MAX];
35553860 	unsigned int segment_count[2];
....@@ -3571,10 +3876,10 @@
35713876 #define stat_other_skip_bggc_count(sbi)	((sbi)->other_skip_bggc++)
35723877 #define stat_inc_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]++)
35733878 #define stat_dec_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]--)
3574
-#define stat_inc_total_hit(sbi)		(atomic64_inc(&(sbi)->total_hit_ext))
3575
-#define stat_inc_rbtree_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_rbtree))
3879
+#define stat_inc_total_hit(sbi, type)		(atomic64_inc(&(sbi)->total_hit_ext[type]))
3880
+#define stat_inc_rbtree_node_hit(sbi, type)	(atomic64_inc(&(sbi)->read_hit_rbtree[type]))
35763881 #define stat_inc_largest_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_largest))
3577
-#define stat_inc_cached_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_cached))
3882
+#define stat_inc_cached_node_hit(sbi, type)	(atomic64_inc(&(sbi)->read_hit_cached[type]))
35783883 #define stat_inc_inline_xattr(inode)					\
35793884 	do {								\
35803885 		if (f2fs_has_inline_xattr(inode))			\
....@@ -3616,9 +3921,9 @@
36163921 			(atomic_dec(&F2FS_I_SB(inode)->compr_inode));	\
36173922 	} while (0)
36183923 #define stat_add_compr_blocks(inode, blocks)				\
3619
-		(atomic_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
3924
+		(atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
36203925 #define stat_sub_compr_blocks(inode, blocks)				\
3621
-		(atomic_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
3926
+		(atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
36223927 #define stat_inc_meta_count(sbi, blkaddr)				\
36233928 	do {								\
36243929 		if (blkaddr < SIT_I(sbi)->sit_base_addr)		\
....@@ -3700,10 +4005,10 @@
37004005 #define stat_other_skip_bggc_count(sbi)			do { } while (0)
37014006 #define stat_inc_dirty_inode(sbi, type)			do { } while (0)
37024007 #define stat_dec_dirty_inode(sbi, type)			do { } while (0)
3703
-#define stat_inc_total_hit(sbi)				do { } while (0)
3704
-#define stat_inc_rbtree_node_hit(sbi)			do { } while (0)
4008
+#define stat_inc_total_hit(sbi, type)			do { } while (0)
4009
+#define stat_inc_rbtree_node_hit(sbi, type)		do { } while (0)
37054010 #define stat_inc_largest_node_hit(sbi)			do { } while (0)
3706
-#define stat_inc_cached_node_hit(sbi)			do { } while (0)
4011
+#define stat_inc_cached_node_hit(sbi, type)		do { } while (0)
37074012 #define stat_inc_inline_xattr(inode)			do { } while (0)
37084013 #define stat_dec_inline_xattr(inode)			do { } while (0)
37094014 #define stat_inc_inline_inode(inode)			do { } while (0)
....@@ -3714,8 +4019,6 @@
37144019 #define stat_dec_compr_inode(inode)			do { } while (0)
37154020 #define stat_add_compr_blocks(inode, blocks)		do { } while (0)
37164021 #define stat_sub_compr_blocks(inode, blocks)		do { } while (0)
3717
-#define stat_inc_atomic_write(inode)			do { } while (0)
3718
-#define stat_dec_atomic_write(inode)			do { } while (0)
37194022 #define stat_update_max_atomic_write(inode)		do { } while (0)
37204023 #define stat_inc_volatile_write(inode)			do { } while (0)
37214024 #define stat_dec_volatile_write(inode)			do { } while (0)
....@@ -3752,6 +4055,7 @@
37524055  * inline.c
37534056  */
37544057 bool f2fs_may_inline_data(struct inode *inode);
4058
+bool f2fs_sanity_check_inline_data(struct inode *inode);
37554059 bool f2fs_may_inline_dentry(struct inode *inode);
37564060 void f2fs_do_read_inline_data(struct page *page, struct page *ipage);
37574061 void f2fs_truncate_inline_inode(struct inode *inode,
....@@ -3794,6 +4098,10 @@
37944098  */
37954099 struct rb_entry *f2fs_lookup_rb_tree(struct rb_root_cached *root,
37964100 				struct rb_entry *cached_re, unsigned int ofs);
4101
+struct rb_node **f2fs_lookup_rb_tree_ext(struct f2fs_sb_info *sbi,
4102
+				struct rb_root_cached *root,
4103
+				struct rb_node **parent,
4104
+				unsigned long long key, bool *left_most);
37974105 struct rb_node **f2fs_lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
37984106 				struct rb_root_cached *root,
37994107 				struct rb_node **parent,
....@@ -3804,20 +4112,34 @@
38044112 		struct rb_node ***insert_p, struct rb_node **insert_parent,
38054113 		bool force, bool *leftmost);
38064114 bool f2fs_check_rb_tree_consistence(struct f2fs_sb_info *sbi,
3807
-						struct rb_root_cached *root);
3808
-unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink);
3809
-bool f2fs_init_extent_tree(struct inode *inode, struct f2fs_extent *i_ext);
4115
+				struct rb_root_cached *root, bool check_key);
4116
+void f2fs_init_extent_tree(struct inode *inode);
38104117 void f2fs_drop_extent_tree(struct inode *inode);
3811
-unsigned int f2fs_destroy_extent_node(struct inode *inode);
4118
+void f2fs_destroy_extent_node(struct inode *inode);
38124119 void f2fs_destroy_extent_tree(struct inode *inode);
3813
-bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs,
3814
-			struct extent_info *ei);
3815
-void f2fs_update_extent_cache(struct dnode_of_data *dn);
3816
-void f2fs_update_extent_cache_range(struct dnode_of_data *dn,
3817
-			pgoff_t fofs, block_t blkaddr, unsigned int len);
38184120 void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
38194121 int __init f2fs_create_extent_cache(void);
38204122 void f2fs_destroy_extent_cache(void);
4123
+
4124
+/* read extent cache ops */
4125
+void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage);
4126
+bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs,
4127
+			struct extent_info *ei);
4128
+void f2fs_update_read_extent_cache(struct dnode_of_data *dn);
4129
+void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn,
4130
+			pgoff_t fofs, block_t blkaddr, unsigned int len);
4131
+unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi,
4132
+			int nr_shrink);
4133
+
4134
+/* block age extent cache ops */
4135
+void f2fs_init_age_extent_tree(struct inode *inode);
4136
+bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs,
4137
+			struct extent_info *ei);
4138
+void f2fs_update_age_extent_cache(struct dnode_of_data *dn);
4139
+void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn,
4140
+			pgoff_t fofs, unsigned int len);
4141
+unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi,
4142
+			int nr_shrink);
38214143 
38224144 /*
38234145  * sysfs.c
....@@ -3871,7 +4193,9 @@
38714193 bool f2fs_is_compress_backend_ready(struct inode *inode);
38724194 int f2fs_init_compress_mempool(void);
38734195 void f2fs_destroy_compress_mempool(void);
3874
-void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity);
4196
+void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task);
4197
+void f2fs_end_read_compressed_page(struct page *page, bool failed,
4198
+				block_t blkaddr, bool in_task);
38754199 bool f2fs_cluster_is_empty(struct compress_ctx *cc);
38764200 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
38774201 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page);
....@@ -3880,16 +4204,45 @@
38804204 						struct writeback_control *wbc,
38814205 						enum iostat_type io_type);
38824206 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
4207
+void f2fs_update_read_extent_tree_range_compressed(struct inode *inode,
4208
+				pgoff_t fofs, block_t blkaddr,
4209
+				unsigned int llen, unsigned int c_len);
38834210 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
38844211 				unsigned nr_pages, sector_t *last_block_in_bio,
38854212 				bool is_readahead, bool for_write);
38864213 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
3887
-void f2fs_free_dic(struct decompress_io_ctx *dic);
3888
-void f2fs_decompress_end_io(struct page **rpages,
3889
-			unsigned int cluster_size, bool err, bool verity);
4214
+void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
4215
+				bool in_task);
4216
+void f2fs_put_page_dic(struct page *page, bool in_task);
4217
+unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn);
38904218 int f2fs_init_compress_ctx(struct compress_ctx *cc);
3891
-void f2fs_destroy_compress_ctx(struct compress_ctx *cc);
4219
+void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
38924220 void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
4221
+int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
4222
+void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
4223
+int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
4224
+void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
4225
+int __init f2fs_init_compress_cache(void);
4226
+void f2fs_destroy_compress_cache(void);
4227
+struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
4228
+void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr);
4229
+void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4230
+						nid_t ino, block_t blkaddr);
4231
+bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4232
+								block_t blkaddr);
4233
+void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
4234
+#define inc_compr_inode_stat(inode)					\
4235
+	do {								\
4236
+		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);		\
4237
+		sbi->compr_new_inode++;					\
4238
+	} while (0)
4239
+#define add_compr_block_stat(inode, blocks)				\
4240
+	do {								\
4241
+		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);		\
4242
+		int diff = F2FS_I(inode)->i_cluster_size - blocks;	\
4243
+		sbi->compr_written_block += blocks;			\
4244
+		sbi->compr_saved_block += diff;				\
4245
+	} while (0)
38934246 #else
38944247 static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
38954248 static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
....@@ -3906,42 +4259,83 @@
39064259 }
39074260 static inline int f2fs_init_compress_mempool(void) { return 0; }
39084261 static inline void f2fs_destroy_compress_mempool(void) { }
4262
+static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic,
4263
+				bool in_task) { }
4264
+static inline void f2fs_end_read_compressed_page(struct page *page,
4265
+				bool failed, block_t blkaddr, bool in_task)
4266
+{
4267
+	WARN_ON_ONCE(1);
4268
+}
4269
+static inline void f2fs_put_page_dic(struct page *page, bool in_task)
4270
+{
4271
+	WARN_ON_ONCE(1);
4272
+}
4273
+static inline unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn) { return 0; }
4274
+static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
4275
+static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
4276
+static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
4277
+static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
4278
+static inline int __init f2fs_init_compress_cache(void) { return 0; }
4279
+static inline void f2fs_destroy_compress_cache(void) { }
4280
+static inline void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi,
4281
+				block_t blkaddr) { }
4282
+static inline void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
4283
+				struct page *page, nid_t ino, block_t blkaddr) { }
4284
+static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi,
4285
+				struct page *page, block_t blkaddr) { return false; }
4286
+static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
4287
+							nid_t ino) { }
4288
+#define inc_compr_inode_stat(inode)		do { } while (0)
4289
+static inline void f2fs_update_read_extent_tree_range_compressed(
4290
+				struct inode *inode,
4291
+				pgoff_t fofs, block_t blkaddr,
4292
+				unsigned int llen, unsigned int c_len) { }
39094293 #endif
39104294 
3911
-static inline void set_compress_context(struct inode *inode)
4295
+static inline int set_compress_context(struct inode *inode)
39124296 {
4297
+#ifdef CONFIG_F2FS_FS_COMPRESSION
39134298 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
39144299 
39154300 	F2FS_I(inode)->i_compress_algorithm =
39164301 			F2FS_OPTION(sbi).compress_algorithm;
39174302 	F2FS_I(inode)->i_log_cluster_size =
39184303 			F2FS_OPTION(sbi).compress_log_size;
4304
+	F2FS_I(inode)->i_compress_flag =
4305
+			F2FS_OPTION(sbi).compress_chksum ?
4306
+				1 << COMPRESS_CHKSUM : 0;
39194307 	F2FS_I(inode)->i_cluster_size =
39204308 			1 << F2FS_I(inode)->i_log_cluster_size;
4309
+	if (F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 &&
4310
+			F2FS_OPTION(sbi).compress_level)
4311
+		F2FS_I(inode)->i_compress_flag |=
4312
+				F2FS_OPTION(sbi).compress_level <<
4313
+				COMPRESS_LEVEL_OFFSET;
39214314 	F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
39224315 	set_inode_flag(inode, FI_COMPRESSED_FILE);
39234316 	stat_inc_compr_inode(inode);
4317
+	inc_compr_inode_stat(inode);
39244318 	f2fs_mark_inode_dirty_sync(inode, true);
4319
+	return 0;
4320
+#else
4321
+	return -EOPNOTSUPP;
4322
+#endif
39254323 }
39264324 
3927
-static inline u64 f2fs_disable_compressed_file(struct inode *inode)
4325
+static inline bool f2fs_disable_compressed_file(struct inode *inode)
39284326 {
39294327 	struct f2fs_inode_info *fi = F2FS_I(inode);
39304328 
39314329 	if (!f2fs_compressed_file(inode))
3932
-		return 0;
3933
-	if (S_ISREG(inode->i_mode)) {
3934
-		if (get_dirty_pages(inode))
3935
-			return 1;
3936
-		if (fi->i_compr_blocks)
3937
-			return fi->i_compr_blocks;
3938
-	}
4330
+		return true;
4331
+	if (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))
4332
+		return false;
39394333 
39404334 	fi->i_flags &= ~F2FS_COMPR_FL;
39414335 	stat_dec_compr_inode(inode);
39424336 	clear_inode_flag(inode, FI_COMPRESSED_FILE);
39434337 	f2fs_mark_inode_dirty_sync(inode, true);
3944
-	return 0;
4338
+	return true;
39454339 }
39464340 
39474341 #define F2FS_FEATURE_FUNCS(name, flagname) \
....@@ -3963,6 +4357,7 @@
39634357 F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
39644358 F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
39654359 F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4360
+F2FS_FEATURE_FUNCS(readonly, RO);
39664361 
39674362 #ifdef CONFIG_BLK_DEV_ZONED
39684363 static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
....@@ -4022,20 +4417,9 @@
40224417 	return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
40234418 }
40244419 
4025
-static inline bool f2fs_may_encrypt(struct inode *dir, struct inode *inode)
4420
+static inline bool f2fs_low_mem_mode(struct f2fs_sb_info *sbi)
40264421 {
4027
-#ifdef CONFIG_FS_ENCRYPTION
4028
-	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
4029
-	umode_t mode = inode->i_mode;
4030
-
4031
-	/*
4032
-	 * If the directory encrypted or dummy encryption enabled,
4033
-	 * then we should encrypt the inode.
4034
-	 */
4035
-	if (IS_ENCRYPTED(dir) || DUMMY_ENCRYPTION_ENABLED(sbi))
4036
-		return (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode));
4037
-#endif
4038
-	return false;
4422
+	return F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW;
40394423 }
40404424 
40414425 static inline bool f2fs_may_compress(struct inode *inode)
....@@ -4051,16 +4435,17 @@
40514435 						u64 blocks, bool add)
40524436 {
40534437 	int diff = F2FS_I(inode)->i_cluster_size - blocks;
4438
+	struct f2fs_inode_info *fi = F2FS_I(inode);
40544439 
40554440 	/* don't update i_compr_blocks if saved blocks were released */
4056
-	if (!add && !F2FS_I(inode)->i_compr_blocks)
4441
+	if (!add && !atomic_read(&fi->i_compr_blocks))
40574442 		return;
40584443 
40594444 	if (add) {
4060
-		F2FS_I(inode)->i_compr_blocks += diff;
4445
+		atomic_add(diff, &fi->i_compr_blocks);
40614446 		stat_add_compr_blocks(inode, diff);
40624447 	} else {
4063
-		F2FS_I(inode)->i_compr_blocks -= diff;
4448
+		atomic_sub(diff, &fi->i_compr_blocks);
40644449 		stat_sub_compr_blocks(inode, diff);
40654450 	}
40664451 	f2fs_mark_inode_dirty_sync(inode, true);
....@@ -4113,13 +4498,18 @@
41134498 		if (F2FS_IO_ALIGNED(sbi))
41144499 			return true;
41154500 	}
4116
-	if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED) &&
4117
-					!IS_SWAPFILE(inode))
4501
+	if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED))
41184502 		return true;
41194503 
41204504 	return false;
41214505 }
41224506 
4507
+static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4508
+{
4509
+	return fsverity_active(inode) &&
4510
+	       idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4511
+}
4512
+
41234513 #ifdef CONFIG_F2FS_FAULT_INJECTION
41244514 extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate,
41254515 							unsigned int type);