forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-10-12 a5969cabbb4660eab42b6ef0412cbbd1200cf14d 
 kernel/fs/f2fs/f2fs.h
....@@ -22,6 +22,7 @@
2222 #include <linux/bio.h>
2323 #include <linux/blkdev.h>
2424 #include <linux/quotaops.h>
25
+#include <linux/part_stat.h>
2526 #include <crypto/hash.h>
2627 
2728 #include <linux/fscrypt.h>
....@@ -32,10 +33,8 @@
3233 #else
3334 #define f2fs_bug_on(sbi, condition)					\
3435 	do {								\
35
-		if (unlikely(condition)) {				\
36
-			WARN_ON(1);					\
36
+		if (WARN_ON(condition))					\
3737 			set_sbi_flag(sbi, SBI_NEED_FSCK);		\
38
-		}							\
3938 	} while (0)
4039 #endif
4140 
....@@ -44,7 +43,6 @@
4443 	FAULT_KVMALLOC,
4544 	FAULT_PAGE_ALLOC,
4645 	FAULT_PAGE_GET,
47
-	FAULT_ALLOC_BIO,
4846 	FAULT_ALLOC_NID,
4947 	FAULT_ORPHAN,
5048 	FAULT_BLOCK,
....@@ -71,7 +69,7 @@
7169 #define IS_FAULT_SET(fi, type) ((fi)->inject_type & (1 << (type)))
7270 #endif
7371 
74
-#define MIN_ROOT_RESERVED_BLOCKS (128 * 1024 * 1024U)
72
+#define MIN_ROOT_RESERVED_BLOCKS (128 * 1024 * 1024)
7573 
7674 /*
7775  * For mount options
....@@ -88,7 +86,7 @@
8886 #define F2FS_MOUNT_FLUSH_MERGE		0x00000400
8987 #define F2FS_MOUNT_NOBARRIER		0x00000800
9088 #define F2FS_MOUNT_FASTBOOT		0x00001000
91
-#define F2FS_MOUNT_EXTENT_CACHE		0x00002000
89
+#define F2FS_MOUNT_READ_EXTENT_CACHE	0x00002000
9290 #define F2FS_MOUNT_DATA_FLUSH		0x00008000
9391 #define F2FS_MOUNT_FAULT_INJECTION	0x00010000
9492 #define F2FS_MOUNT_USRQUOTA		0x00080000
....@@ -99,6 +97,11 @@
9997 #define F2FS_MOUNT_RESERVE_ROOT		0x01000000
10098 #define F2FS_MOUNT_DISABLE_CHECKPOINT	0x02000000
10199 #define F2FS_MOUNT_NORECOVERY		0x04000000
100
+#define F2FS_MOUNT_ATGC			0x08000000
101
+#define F2FS_MOUNT_MERGE_CHECKPOINT	0x10000000
102
+#define	F2FS_MOUNT_GC_MERGE		0x20000000
103
+#define F2FS_MOUNT_COMPRESS_CACHE	0x40000000
104
+#define F2FS_MOUNT_AGE_EXTENT_CACHE	0x80000000
102105 
103106 #define F2FS_OPTION(sbi)	((sbi)->mount_opt)
104107 #define clear_opt(sbi, option)	(F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option)
....@@ -116,6 +119,18 @@
116119 typedef u32 nid_t;
117120 
118121 #define COMPRESS_EXT_NUM		16
122
+
123
+/*
124
+ * An implementation of an rwsem that is explicitly unfair to readers. This
125
+ * prevents priority inversion when a low-priority reader acquires the read lock
126
+ * while sleeping on the write lock but the write lock is needed by
127
+ * higher-priority clients.
128
+ */
129
+
130
+struct f2fs_rwsem {
131
+        struct rw_semaphore internal_rwsem;
132
+        wait_queue_head_t read_waiters;
133
+};
119134 
120135 struct f2fs_mount_info {
121136 	unsigned int opt;
....@@ -139,10 +154,8 @@
139154 	int fsync_mode;			/* fsync policy */
140155 	int fs_mode;			/* fs mode: LFS or ADAPTIVE */
141156 	int bggc_mode;			/* bggc mode: off, on or sync */
142
-	struct fscrypt_dummy_context dummy_enc_ctx; /* test dummy encryption */
143
-#ifdef CONFIG_FS_ENCRYPTION
144
-	bool inlinecrypt;		/* inline encryption enabled */
145
-#endif
157
+	int memory_mode;		/* memory mode */
158
+	struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
146159 	block_t unusable_cap_perc;	/* percentage for cap */
147160 	block_t unusable_cap;		/* Amount of space allowed to be
148161 					 * unusable when disabling checkpoint
....@@ -150,8 +163,11 @@
150163 
151164 	/* For compression */
152165 	unsigned char compress_algorithm;	/* algorithm type */
153
-	unsigned compress_log_size;		/* cluster log size */
166
+	unsigned char compress_log_size;	/* cluster log size */
167
+	unsigned char compress_level;		/* compress level */
168
+	bool compress_chksum;			/* compressed data chksum */
154169 	unsigned char compress_ext_cnt;		/* extension count */
170
+	int compress_mode;			/* compression mode */
155171 	unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN];	/* extensions */
156172 };
157173 
....@@ -169,6 +185,7 @@
169185 #define F2FS_FEATURE_SB_CHKSUM		0x0800
170186 #define F2FS_FEATURE_CASEFOLD		0x1000
171187 #define F2FS_FEATURE_COMPRESSION	0x2000
188
+#define F2FS_FEATURE_RO			0x4000
172189 
173190 #define __F2FS_HAS_FEATURE(raw_super, mask)				\
174191 	((raw_super->feature & cpu_to_le32(mask)) != 0)
....@@ -238,6 +255,10 @@
238255 					 * condition of read on truncated area
239256 					 * by extent_cache
240257 					 */
258
+	DATA_GENERIC_ENHANCE_UPDATE,	/*
259
+					 * strong check on range and segment
260
+					 * bitmap for update case
261
+					 */
241262 	META_GENERIC,
242263 };
243264 
....@@ -267,6 +288,26 @@
267288 	struct list_head list;	/* list head */
268289 	struct page *page;	/* warm node page pointer */
269290 	unsigned int seq_id;	/* sequence id */
291
+};
292
+
293
+struct ckpt_req {
294
+	struct completion wait;		/* completion for checkpoint done */
295
+	struct llist_node llnode;	/* llist_node to be linked in wait queue */
296
+	int ret;			/* return code of checkpoint */
297
+	ktime_t queue_time;		/* request queued time */
298
+};
299
+
300
+struct ckpt_req_control {
301
+	struct task_struct *f2fs_issue_ckpt;	/* checkpoint task */
302
+	int ckpt_thread_ioprio;			/* checkpoint merge thread ioprio */
303
+	wait_queue_head_t ckpt_wait_queue;	/* waiting queue for wake-up */
304
+	atomic_t issued_ckpt;		/* # of actually issued ckpts */
305
+	atomic_t total_ckpt;		/* # of total ckpts */
306
+	atomic_t queued_ckpt;		/* # of queued ckpts */
307
+	struct llist_head issue_list;	/* list for command issue */
308
+	spinlock_t stat_lock;		/* lock for below checkpoint time stats */
309
+	unsigned int cur_time;		/* cur wait time in msec for currently issued checkpoint */
310
+	unsigned int peak_time;		/* peak wait time in msec until now */
270311 };
271312 
272313 /* for the bitmap indicate blocks to be discarded */
....@@ -405,93 +446,6 @@
405446 	return size <= MAX_SIT_JENTRIES(journal);
406447 }
407448 
408
-/*
409
- * ioctl commands
410
- */
411
-#define F2FS_IOC_GETFLAGS		FS_IOC_GETFLAGS
412
-#define F2FS_IOC_SETFLAGS		FS_IOC_SETFLAGS
413
-#define F2FS_IOC_GETVERSION		FS_IOC_GETVERSION
414
-
415
-#define F2FS_IOCTL_MAGIC		0xf5
416
-#define F2FS_IOC_START_ATOMIC_WRITE	_IO(F2FS_IOCTL_MAGIC, 1)
417
-#define F2FS_IOC_COMMIT_ATOMIC_WRITE	_IO(F2FS_IOCTL_MAGIC, 2)
418
-#define F2FS_IOC_START_VOLATILE_WRITE	_IO(F2FS_IOCTL_MAGIC, 3)
419
-#define F2FS_IOC_RELEASE_VOLATILE_WRITE	_IO(F2FS_IOCTL_MAGIC, 4)
420
-#define F2FS_IOC_ABORT_VOLATILE_WRITE	_IO(F2FS_IOCTL_MAGIC, 5)
421
-#define F2FS_IOC_GARBAGE_COLLECT	_IOW(F2FS_IOCTL_MAGIC, 6, __u32)
422
-#define F2FS_IOC_WRITE_CHECKPOINT	_IO(F2FS_IOCTL_MAGIC, 7)
423
-#define F2FS_IOC_DEFRAGMENT		_IOWR(F2FS_IOCTL_MAGIC, 8,	\
424
-						struct f2fs_defragment)
425
-#define F2FS_IOC_MOVE_RANGE		_IOWR(F2FS_IOCTL_MAGIC, 9,	\
426
-						struct f2fs_move_range)
427
-#define F2FS_IOC_FLUSH_DEVICE		_IOW(F2FS_IOCTL_MAGIC, 10,	\
428
-						struct f2fs_flush_device)
429
-#define F2FS_IOC_GARBAGE_COLLECT_RANGE	_IOW(F2FS_IOCTL_MAGIC, 11,	\
430
-						struct f2fs_gc_range)
431
-#define F2FS_IOC_GET_FEATURES		_IOR(F2FS_IOCTL_MAGIC, 12, __u32)
432
-#define F2FS_IOC_SET_PIN_FILE		_IOW(F2FS_IOCTL_MAGIC, 13, __u32)
433
-#define F2FS_IOC_GET_PIN_FILE		_IOR(F2FS_IOCTL_MAGIC, 14, __u32)
434
-#define F2FS_IOC_PRECACHE_EXTENTS	_IO(F2FS_IOCTL_MAGIC, 15)
435
-#define F2FS_IOC_RESIZE_FS		_IOW(F2FS_IOCTL_MAGIC, 16, __u64)
436
-#define F2FS_IOC_GET_COMPRESS_BLOCKS	_IOR(F2FS_IOCTL_MAGIC, 17, __u64)
437
-#define F2FS_IOC_RELEASE_COMPRESS_BLOCKS				\
438
-					_IOR(F2FS_IOCTL_MAGIC, 18, __u64)
439
-#define F2FS_IOC_RESERVE_COMPRESS_BLOCKS				\
440
-					_IOR(F2FS_IOCTL_MAGIC, 19, __u64)
441
-
442
-#define F2FS_IOC_GET_VOLUME_NAME	FS_IOC_GETFSLABEL
443
-#define F2FS_IOC_SET_VOLUME_NAME	FS_IOC_SETFSLABEL
444
-
445
-#define F2FS_IOC_SET_ENCRYPTION_POLICY	FS_IOC_SET_ENCRYPTION_POLICY
446
-#define F2FS_IOC_GET_ENCRYPTION_POLICY	FS_IOC_GET_ENCRYPTION_POLICY
447
-#define F2FS_IOC_GET_ENCRYPTION_PWSALT	FS_IOC_GET_ENCRYPTION_PWSALT
448
-
449
-/*
450
- * should be same as XFS_IOC_GOINGDOWN.
451
- * Flags for going down operation used by FS_IOC_GOINGDOWN
452
- */
453
-#define F2FS_IOC_SHUTDOWN	_IOR('X', 125, __u32)	/* Shutdown */
454
-#define F2FS_GOING_DOWN_FULLSYNC	0x0	/* going down with full sync */
455
-#define F2FS_GOING_DOWN_METASYNC	0x1	/* going down with metadata */
456
-#define F2FS_GOING_DOWN_NOSYNC		0x2	/* going down */
457
-#define F2FS_GOING_DOWN_METAFLUSH	0x3	/* going down with meta flush */
458
-#define F2FS_GOING_DOWN_NEED_FSCK	0x4	/* going down to trigger fsck */
459
-
460
-#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
461
-/*
462
- * ioctl commands in 32 bit emulation
463
- */
464
-#define F2FS_IOC32_GETFLAGS		FS_IOC32_GETFLAGS
465
-#define F2FS_IOC32_SETFLAGS		FS_IOC32_SETFLAGS
466
-#define F2FS_IOC32_GETVERSION		FS_IOC32_GETVERSION
467
-#endif
468
-
469
-#define F2FS_IOC_FSGETXATTR		FS_IOC_FSGETXATTR
470
-#define F2FS_IOC_FSSETXATTR		FS_IOC_FSSETXATTR
471
-
472
-struct f2fs_gc_range {
473
-	u32 sync;
474
-	u64 start;
475
-	u64 len;
476
-};
477
-
478
-struct f2fs_defragment {
479
-	u64 start;
480
-	u64 len;
481
-};
482
-
483
-struct f2fs_move_range {
484
-	u32 dst_fd;		/* destination fd */
485
-	u64 pos_in;		/* start position in src_fd */
486
-	u64 pos_out;		/* start position in dst_fd */
487
-	u64 len;		/* size to move */
488
-};
489
-
490
-struct f2fs_flush_device {
491
-	u32 dev_num;		/* device number to flush */
492
-	u32 segments;		/* # of segments to flush */
493
-};
494
-
495449 /* for inline stuff */
496450 #define DEF_INLINE_RESERVED_SIZE	1
497451 static inline int get_extra_isize(struct inode *inode);
....@@ -544,11 +498,11 @@
544498 #ifdef CONFIG_UNICODE
545499 	/*
546500 	 * For casefolded directories: the casefolded name, but it's left NULL
547
-	 * if the original name is not valid Unicode, if the directory is both
548
-	 * casefolded and encrypted and its encryption key is unavailable, or if
549
-	 * the filesystem is doing an internal operation where usr_fname is also
550
-	 * NULL.  In all these cases we fall back to treating the name as an
551
-	 * opaque byte sequence.
501
+	 * if the original name is not valid Unicode, if the original name is
502
+	 * "." or "..", if the directory is both casefolded and encrypted and
503
+	 * its encryption key is unavailable, or if the filesystem is doing an
504
+	 * internal operation where usr_fname is also NULL.  In all these cases
505
+	 * we fall back to treating the name as an opaque byte sequence.
552506 	 */
553507 	struct fscrypt_str cf_name;
554508 #endif
....@@ -622,18 +576,59 @@
622576 #define F2FS_MIN_EXTENT_LEN	64	/* minimum extent length */
623577 
624578 /* number of extent info in extent cache we try to shrink */
625
-#define EXTENT_CACHE_SHRINK_NUMBER	128
579
+#define READ_EXTENT_CACHE_SHRINK_NUMBER	128
580
+
581
+/* number of age extent info in extent cache we try to shrink */
582
+#define AGE_EXTENT_CACHE_SHRINK_NUMBER	128
583
+#define LAST_AGE_WEIGHT			30
584
+#define SAME_AGE_REGION			1024
585
+
586
+/*
587
+ * Define data block with age less than 1GB as hot data
588
+ * define data block with age less than 10GB but more than 1GB as warm data
589
+ */
590
+#define DEF_HOT_DATA_AGE_THRESHOLD	262144
591
+#define DEF_WARM_DATA_AGE_THRESHOLD	2621440
592
+
593
+/* extent cache type */
594
+enum extent_type {
595
+	EX_READ,
596
+	EX_BLOCK_AGE,
597
+	NR_EXTENT_CACHES,
598
+};
626599 
627600 struct rb_entry {
628601 	struct rb_node rb_node;		/* rb node located in rb-tree */
629
-	unsigned int ofs;		/* start offset of the entry */
630
-	unsigned int len;		/* length of the entry */
602
+	union {
603
+		struct {
604
+			unsigned int ofs;	/* start offset of the entry */
605
+			unsigned int len;	/* length of the entry */
606
+		};
607
+		unsigned long long key;		/* 64-bits key */
608
+	} __packed;
631609 };
632610 
633611 struct extent_info {
634612 	unsigned int fofs;		/* start offset in a file */
635613 	unsigned int len;		/* length of the extent */
636
-	u32 blk;			/* start block address of the extent */
614
+	union {
615
+		/* read extent_cache */
616
+		struct {
617
+			/* start block address of the extent */
618
+			block_t blk;
619
+#ifdef CONFIG_F2FS_FS_COMPRESSION
620
+			/* physical extent length of compressed blocks */
621
+			unsigned int c_len;
622
+#endif
623
+		};
624
+		/* block age extent_cache */
625
+		struct {
626
+			/* block age of the extent */
627
+			unsigned long long age;
628
+			/* last total blocks allocated */
629
+			unsigned long long last_blocks;
630
+		};
631
+	};
637632 };
638633 
639634 struct extent_node {
....@@ -645,13 +640,25 @@
645640 
646641 struct extent_tree {
647642 	nid_t ino;			/* inode number */
643
+	enum extent_type type;		/* keep the extent tree type */
648644 	struct rb_root_cached root;	/* root of extent info rb-tree */
649645 	struct extent_node *cached_en;	/* recently accessed extent node */
650
-	struct extent_info largest;	/* largested extent info */
651646 	struct list_head list;		/* to be used by sbi->zombie_list */
652647 	rwlock_t lock;			/* protect extent info rb-tree */
653648 	atomic_t node_cnt;		/* # of extent node in rb-tree*/
654649 	bool largest_updated;		/* largest extent updated */
650
+	struct extent_info largest;	/* largest cached extent for EX_READ */
651
+};
652
+
653
+struct extent_tree_info {
654
+	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
655
+	struct mutex extent_tree_lock;	/* locking extent radix tree */
656
+	struct list_head extent_list;		/* lru list for shrinker */
657
+	spinlock_t extent_lock;			/* locking extent lru list */
658
+	atomic_t total_ext_tree;		/* extent tree count */
659
+	struct list_head zombie_list;		/* extent zombie tree list */
660
+	atomic_t total_zombie_tree;		/* extent zombie tree count */
661
+	atomic_t total_ext_node;		/* extent info count */
655662 };
656663 
657664 /*
....@@ -701,21 +708,26 @@
701708 #define FADVISE_MODIFIABLE_BITS	(FADVISE_COLD_BIT | FADVISE_HOT_BIT)
702709 
703710 #define file_is_cold(inode)	is_file(inode, FADVISE_COLD_BIT)
704
-#define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
705711 #define file_set_cold(inode)	set_file(inode, FADVISE_COLD_BIT)
706
-#define file_lost_pino(inode)	set_file(inode, FADVISE_LOST_PINO_BIT)
707712 #define file_clear_cold(inode)	clear_file(inode, FADVISE_COLD_BIT)
713
+
714
+#define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
715
+#define file_lost_pino(inode)	set_file(inode, FADVISE_LOST_PINO_BIT)
708716 #define file_got_pino(inode)	clear_file(inode, FADVISE_LOST_PINO_BIT)
717
+
709718 #define file_is_encrypt(inode)	is_file(inode, FADVISE_ENCRYPT_BIT)
710719 #define file_set_encrypt(inode)	set_file(inode, FADVISE_ENCRYPT_BIT)
711
-#define file_clear_encrypt(inode) clear_file(inode, FADVISE_ENCRYPT_BIT)
720
+
712721 #define file_enc_name(inode)	is_file(inode, FADVISE_ENC_NAME_BIT)
713722 #define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
723
+
714724 #define file_keep_isize(inode)	is_file(inode, FADVISE_KEEP_SIZE_BIT)
715725 #define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
726
+
716727 #define file_is_hot(inode)	is_file(inode, FADVISE_HOT_BIT)
717728 #define file_set_hot(inode)	set_file(inode, FADVISE_HOT_BIT)
718729 #define file_clear_hot(inode)	clear_file(inode, FADVISE_HOT_BIT)
730
+
719731 #define file_is_verity(inode)	is_file(inode, FADVISE_VERITY_BIT)
720732 #define file_set_verity(inode)	set_file(inode, FADVISE_VERITY_BIT)
721733 
....@@ -751,7 +763,8 @@
751763 	FI_DROP_CACHE,		/* drop dirty page cache */
752764 	FI_DATA_EXIST,		/* indicate data exists */
753765 	FI_INLINE_DOTS,		/* indicate inline dot dentries */
754
-	FI_DO_DEFRAG,		/* indicate defragment is running */
766
+	FI_SKIP_WRITES,		/* should skip data page writeback */
767
+	FI_OPU_WRITE,		/* used for opu per file */
755768 	FI_DIRTY_FILE,		/* indicate regular/symlink has dirty pages */
756769 	FI_NO_PREALLOC,		/* indicate skipped preallocated blocks */
757770 	FI_HOT_DATA,		/* indicate file is hot */
....@@ -761,7 +774,11 @@
761774 	FI_ATOMIC_REVOKE_REQUEST, /* request to drop atomic data */
762775 	FI_VERITY_IN_PROGRESS,	/* building fs-verity Merkle tree */
763776 	FI_COMPRESSED_FILE,	/* indicate file's data can be compressed */
777
+	FI_COMPRESS_CORRUPT,	/* indicate compressed cluster is corrupted */
764778 	FI_MMAP_FILE,		/* indicate file was mmapped */
779
+	FI_ENABLE_COMPRESS,	/* enable compression in "user" compression mode */
780
+	FI_COMPRESS_RELEASED,	/* compressed blocks were released */
781
+	FI_ALIGNED_WRITE,	/* enable aligned write */
765782 	FI_MAX,			/* max flag, never be used */
766783 };
767784 
....@@ -778,12 +795,13 @@
778795 
779796 	/* Use below internally in f2fs*/
780797 	unsigned long flags[BITS_TO_LONGS(FI_MAX)];	/* use to pass per-file flags */
781
-	struct rw_semaphore i_sem;	/* protect fi info */
798
+	struct f2fs_rwsem i_sem;	/* protect fi info */
782799 	atomic_t dirty_pages;		/* # of dirty pages */
783800 	f2fs_hash_t chash;		/* hash value of given file name */
784801 	unsigned int clevel;		/* maximum level of given file name */
785802 	struct task_struct *task;	/* lookup and create consistency */
786803 	struct task_struct *cp_task;	/* separate cp/wb IO stats*/
804
+	struct task_struct *wb_task;	/* indicate inode is in context of writeback */
787805 	nid_t i_xattr_nid;		/* node id that contains xattrs */
788806 	loff_t	last_disk_size;		/* lastly written file size */
789807 	spinlock_t i_size_lock;		/* protect last_disk_size */
....@@ -800,12 +818,13 @@
800818 	struct list_head inmem_pages;	/* inmemory pages managed by f2fs */
801819 	struct task_struct *inmem_task;	/* store inmemory task */
802820 	struct mutex inmem_lock;	/* lock for inmemory pages */
803
-	struct extent_tree *extent_tree;	/* cached extent_tree entry */
821
+	struct extent_tree *extent_tree[NR_EXTENT_CACHES];
822
+					/* cached extent_tree entry */
804823 
805824 	/* avoid racing between foreground op and gc */
806
-	struct rw_semaphore i_gc_rwsem[2];
807
-	struct rw_semaphore i_mmap_sem;
808
-	struct rw_semaphore i_xattr_sem; /* avoid racing between reading and changing EAs */
825
+	struct f2fs_rwsem i_gc_rwsem[2];
826
+	struct f2fs_rwsem i_mmap_sem;
827
+	struct f2fs_rwsem i_xattr_sem; /* avoid racing between reading and changing EAs */
809828 
810829 	int i_extra_isize;		/* size of extra space located in i_addr */
811830 	kprojid_t i_projid;		/* id for project quota */
....@@ -814,13 +833,15 @@
814833 	struct timespec64 i_disk_time[4];/* inode disk times */
815834 
816835 	/* for file compress */
817
-	u64 i_compr_blocks;			/* # of compressed blocks */
836
+	atomic_t i_compr_blocks;		/* # of compressed blocks */
818837 	unsigned char i_compress_algorithm;	/* algorithm type */
819838 	unsigned char i_log_cluster_size;	/* log of cluster size */
839
+	unsigned char i_compress_level;		/* compress level (lz4hc,zstd) */
840
+	unsigned short i_compress_flag;		/* compress flag */
820841 	unsigned int i_cluster_size;		/* cluster size */
821842 };
822843 
823
-static inline void get_extent_info(struct extent_info *ext,
844
+static inline void get_read_extent_info(struct extent_info *ext,
824845 					struct f2fs_extent *i_ext)
825846 {
826847 	ext->fofs = le32_to_cpu(i_ext->fofs);
....@@ -828,20 +849,12 @@
828849 	ext->len = le32_to_cpu(i_ext->len);
829850 }
830851 
831
-static inline void set_raw_extent(struct extent_info *ext,
852
+static inline void set_raw_read_extent(struct extent_info *ext,
832853 					struct f2fs_extent *i_ext)
833854 {
834855 	i_ext->fofs = cpu_to_le32(ext->fofs);
835856 	i_ext->blk = cpu_to_le32(ext->blk);
836857 	i_ext->len = cpu_to_le32(ext->len);
837
-}
838
-
839
-static inline void set_extent_info(struct extent_info *ei, unsigned int fofs,
840
-						u32 blk, unsigned int len)
841
-{
842
-	ei->fofs = fofs;
843
-	ei->blk = blk;
844
-	ei->len = len;
845858 }
846859 
847860 static inline bool __is_discard_mergeable(struct discard_info *back,
....@@ -863,35 +876,6 @@
863876 	return __is_discard_mergeable(cur, front, max_len);
864877 }
865878 
866
-static inline bool __is_extent_mergeable(struct extent_info *back,
867
-						struct extent_info *front)
868
-{
869
-	return (back->fofs + back->len == front->fofs &&
870
-			back->blk + back->len == front->blk);
871
-}
872
-
873
-static inline bool __is_back_mergeable(struct extent_info *cur,
874
-						struct extent_info *back)
875
-{
876
-	return __is_extent_mergeable(back, cur);
877
-}
878
-
879
-static inline bool __is_front_mergeable(struct extent_info *cur,
880
-						struct extent_info *front)
881
-{
882
-	return __is_extent_mergeable(cur, front);
883
-}
884
-
885
-extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
886
-static inline void __try_update_largest_extent(struct extent_tree *et,
887
-						struct extent_node *en)
888
-{
889
-	if (en->ei.len > et->largest.len) {
890
-		et->largest = en->ei;
891
-		et->largest_updated = true;
892
-	}
893
-}
894
-
895879 /*
896880  * For free nid management
897881  */
....@@ -899,6 +883,13 @@
899883 	FREE_NID,		/* newly added to free nid list */
900884 	PREALLOC_NID,		/* it is preallocated */
901885 	MAX_NID_STATE,
886
+};
887
+
888
+enum nat_state {
889
+	TOTAL_NAT,
890
+	DIRTY_NAT,
891
+	RECLAIMABLE_NAT,
892
+	MAX_NAT_STATE,
902893 };
903894 
904895 struct f2fs_nm_info {
....@@ -913,11 +904,10 @@
913904 	/* NAT cache management */
914905 	struct radix_tree_root nat_root;/* root of the nat entry cache */
915906 	struct radix_tree_root nat_set_root;/* root of the nat set cache */
916
-	struct rw_semaphore nat_tree_lock;	/* protect nat_tree_lock */
907
+	struct f2fs_rwsem nat_tree_lock;	/* protect nat entry tree */
917908 	struct list_head nat_entries;	/* cached nat entry list (clean) */
918909 	spinlock_t nat_list_lock;	/* protect clean nat entry list */
919
-	unsigned int nat_cnt;		/* the # of cached nat entries */
920
-	unsigned int dirty_nat_cnt;	/* total num of nat entries in set */
910
+	unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */
921911 	unsigned int nat_blocks;	/* # of nat blocks */
922912 
923913 	/* free node ids management */
....@@ -986,7 +976,10 @@
986976  */
987977 #define	NR_CURSEG_DATA_TYPE	(3)
988978 #define NR_CURSEG_NODE_TYPE	(3)
989
-#define NR_CURSEG_TYPE	(NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
979
+#define NR_CURSEG_INMEM_TYPE	(2)
980
+#define NR_CURSEG_RO_TYPE	(2)
981
+#define NR_CURSEG_PERSIST_TYPE	(NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE)
982
+#define NR_CURSEG_TYPE		(NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE)
990983 
991984 enum {
992985 	CURSEG_HOT_DATA	= 0,	/* directory entry blocks */
....@@ -995,8 +988,11 @@
995988 	CURSEG_HOT_NODE,	/* direct node blocks of directory files */
996989 	CURSEG_WARM_NODE,	/* direct node blocks of normal files */
997990 	CURSEG_COLD_NODE,	/* indirect node blocks */
998
-	NO_CHECK_TYPE,
999
-	CURSEG_COLD_DATA_PINNED,/* cold data for pinned file */
991
+	NR_PERSISTENT_LOG,	/* number of persistent log */
992
+	CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG,
993
+				/* pinned file that needs consecutive block address */
994
+	CURSEG_ALL_DATA_ATGC,	/* SSR alloctor in hot/warm/cold data area */
995
+	NO_CHECK_TYPE,		/* number of persistent & inmem log */
1000996 };
1001997 
1002998 struct flush_cmd {
....@@ -1021,7 +1017,7 @@
10211017 	struct dirty_seglist_info *dirty_info;	/* dirty segment information */
10221018 	struct curseg_info *curseg_array;	/* active segment information */
10231019 
1024
-	struct rw_semaphore curseg_lock;	/* for preventing curseg change */
1020
+	struct f2fs_rwsem curseg_lock;	/* for preventing curseg change */
10251021 
10261022 	block_t seg0_blkaddr;		/* block address of 0'th segment */
10271023 	block_t main_blkaddr;		/* start block address of main area */
....@@ -1030,6 +1026,7 @@
10301026 	unsigned int segment_count;	/* total # of segments */
10311027 	unsigned int main_segments;	/* # of segments in main area */
10321028 	unsigned int reserved_segments;	/* # of reserved segments */
1029
+	unsigned int additional_reserved_segments;/* reserved segs for IO align feature */
10331030 	unsigned int ovp_segments;	/* # of overprovision segments */
10341031 
10351032 	/* a threshold to reclaim prefree segments */
....@@ -1095,8 +1092,8 @@
10951092  */
10961093 #define PAGE_TYPE_OF_BIO(type)	((type) > META ? META : (type))
10971094 enum page_type {
1098
-	DATA,
1099
-	NODE,
1095
+	DATA = 0,
1096
+	NODE = 1,	/* should not change this */
11001097 	META,
11011098 	NR_PAGE_TYPE,
11021099 	META_FLUSH,
....@@ -1186,6 +1183,7 @@
11861183 	bool retry;		/* need to reallocate block address */
11871184 	int compr_blocks;	/* # of compressed block addresses */
11881185 	bool encrypted;		/* indicate file is encrypted */
1186
+	bool post_read;		/* require post read */
11891187 	enum iostat_type io_type;	/* io type */
11901188 	struct writeback_control *io_wbc; /* writeback control */
11911189 	struct bio **bio;		/* bio for ipu */
....@@ -1204,11 +1202,11 @@
12041202 	struct bio *bio;		/* bios to merge */
12051203 	sector_t last_block_in_bio;	/* last block number */
12061204 	struct f2fs_io_info fio;	/* store buffered io info. */
1207
-	struct rw_semaphore io_rwsem;	/* blocking op for bio */
1205
+	struct f2fs_rwsem io_rwsem;	/* blocking op for bio */
12081206 	spinlock_t io_lock;		/* serialize DATA/NODE IOs */
12091207 	struct list_head io_list;	/* track fios */
12101208 	struct list_head bio_list;	/* bio entry list head */
1211
-	struct rw_semaphore bio_list_lock;	/* lock to protect bio entry list */
1209
+	struct f2fs_rwsem bio_list_lock;	/* lock to protect bio entry list */
12121210 };
12131211 
12141212 #define FDEV(i)				(sbi->devs[i])
....@@ -1241,6 +1239,18 @@
12411239 	unsigned long ino_num;			/* number of entries */
12421240 };
12431241 
1242
+/* for GC_AT */
1243
+struct atgc_management {
1244
+	bool atgc_enabled;			/* ATGC is enabled or not */
1245
+	struct rb_root_cached root;		/* root of victim rb-tree */
1246
+	struct list_head victim_list;		/* linked with all victim entries */
1247
+	unsigned int victim_count;		/* victim count in rb-tree */
1248
+	unsigned int candidate_ratio;		/* candidate ratio */
1249
+	unsigned int max_candidate_count;	/* max candidate count */
1250
+	unsigned int age_weight;		/* age weight, vblock_weight = 100 - age_weight */
1251
+	unsigned long long age_threshold;	/* age threshold */
1252
+};
1253
+
12441254 /* For s_flag in struct f2fs_sb_info */
12451255 enum {
12461256 	SBI_IS_DIRTY,				/* dirty flag for checkpoint */
....@@ -1257,6 +1267,7 @@
12571267 	SBI_QUOTA_SKIP_FLUSH,			/* skip flushing quota in current CP */
12581268 	SBI_QUOTA_NEED_REPAIR,			/* quota file may be corrupted */
12591269 	SBI_IS_RESIZEFS,			/* resizefs is in process */
1270
+	SBI_IS_FREEZING,			/* freezefs is in process */
12601271 };
12611272 
12621273 enum {
....@@ -1273,7 +1284,11 @@
12731284 	GC_NORMAL,
12741285 	GC_IDLE_CB,
12751286 	GC_IDLE_GREEDY,
1276
-	GC_URGENT,
1287
+	GC_IDLE_AT,
1288
+	GC_URGENT_HIGH,
1289
+	GC_URGENT_LOW,
1290
+	GC_URGENT_MID,
1291
+	MAX_GC_MODE,
12771292 };
12781293 
12791294 enum {
....@@ -1307,36 +1322,158 @@
13071322 	FSYNC_MODE_NOBARRIER,	/* fsync behaves nobarrier based on posix */
13081323 };
13091324 
1325
+enum {
1326
+	COMPR_MODE_FS,		/*
1327
+				 * automatically compress compression
1328
+				 * enabled files
1329
+				 */
1330
+	COMPR_MODE_USER,	/*
1331
+				 * automatical compression is disabled.
1332
+				 * user can control the file compression
1333
+				 * using ioctls
1334
+				 */
1335
+};
1336
+
1337
+enum {
1338
+	MEMORY_MODE_NORMAL,	/* memory mode for normal devices */
1339
+	MEMORY_MODE_LOW,	/* memory mode for low memry devices */
1340
+};
1341
+
1342
+static inline int f2fs_test_bit(unsigned int nr, char *addr);
1343
+static inline void f2fs_set_bit(unsigned int nr, char *addr);
1344
+static inline void f2fs_clear_bit(unsigned int nr, char *addr);
1345
+
13101346 /*
1311
- * this value is set in page as a private data which indicate that
1312
- * the page is atomically written, and it is in inmem_pages list.
1347
+ * Layout of f2fs page.private:
1348
+ *
1349
+ * Layout A: lowest bit should be 1
1350
+ * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
1351
+ * bit 0	PAGE_PRIVATE_NOT_POINTER
1352
+ * bit 1	PAGE_PRIVATE_ATOMIC_WRITE
1353
+ * bit 2	PAGE_PRIVATE_DUMMY_WRITE
1354
+ * bit 3	PAGE_PRIVATE_ONGOING_MIGRATION
1355
+ * bit 4	PAGE_PRIVATE_INLINE_INODE
1356
+ * bit 5	PAGE_PRIVATE_REF_RESOURCE
1357
+ * bit 6-	f2fs private data
1358
+ *
1359
+ * Layout B: lowest bit should be 0
1360
+ * page.private is a wrapped pointer.
13131361  */
1314
-#define ATOMIC_WRITTEN_PAGE		((unsigned long)-1)
1315
-#define DUMMY_WRITTEN_PAGE		((unsigned long)-2)
1362
+enum {
1363
+	PAGE_PRIVATE_NOT_POINTER,		/* private contains non-pointer data */
1364
+	PAGE_PRIVATE_ATOMIC_WRITE,		/* data page from atomic write path */
1365
+	PAGE_PRIVATE_DUMMY_WRITE,		/* data page for padding aligned IO */
1366
+	PAGE_PRIVATE_ONGOING_MIGRATION,		/* data page which is on-going migrating */
1367
+	PAGE_PRIVATE_INLINE_INODE,		/* inode page contains inline data */
1368
+	PAGE_PRIVATE_REF_RESOURCE,		/* dirty page has referenced resources */
1369
+	PAGE_PRIVATE_MAX
1370
+};
13161371 
1317
-#define IS_ATOMIC_WRITTEN_PAGE(page)			\
1318
-		(page_private(page) == (unsigned long)ATOMIC_WRITTEN_PAGE)
1319
-#define IS_DUMMY_WRITTEN_PAGE(page)			\
1320
-		(page_private(page) == (unsigned long)DUMMY_WRITTEN_PAGE)
1372
+#define PAGE_PRIVATE_GET_FUNC(name, flagname) \
1373
+static inline bool page_private_##name(struct page *page) \
1374
+{ \
1375
+	return PagePrivate(page) && \
1376
+		test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
1377
+		test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1378
+}
13211379 
1322
-#ifdef CONFIG_FS_ENCRYPTION
1323
-#define DUMMY_ENCRYPTION_ENABLED(sbi) \
1324
-	(unlikely(F2FS_OPTION(sbi).dummy_enc_ctx.ctx != NULL))
1325
-#else
1326
-#define DUMMY_ENCRYPTION_ENABLED(sbi) (0)
1327
-#endif
1380
+#define PAGE_PRIVATE_SET_FUNC(name, flagname) \
1381
+static inline void set_page_private_##name(struct page *page) \
1382
+{ \
1383
+	if (!PagePrivate(page)) { \
1384
+		get_page(page); \
1385
+		SetPagePrivate(page); \
1386
+		set_page_private(page, 0); \
1387
+	} \
1388
+	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
1389
+	set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1390
+}
1391
+
1392
+#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
1393
+static inline void clear_page_private_##name(struct page *page) \
1394
+{ \
1395
+	clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
1396
+	if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) { \
1397
+		set_page_private(page, 0); \
1398
+		if (PagePrivate(page)) { \
1399
+			ClearPagePrivate(page); \
1400
+			put_page(page); \
1401
+		}\
1402
+	} \
1403
+}
1404
+
1405
+PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
1406
+PAGE_PRIVATE_GET_FUNC(reference, REF_RESOURCE);
1407
+PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
1408
+PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
1409
+PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
1410
+PAGE_PRIVATE_GET_FUNC(dummy, DUMMY_WRITE);
1411
+
1412
+PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
1413
+PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
1414
+PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
1415
+PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
1416
+PAGE_PRIVATE_SET_FUNC(dummy, DUMMY_WRITE);
1417
+
1418
+PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
1419
+PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
1420
+PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
1421
+PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
1422
+PAGE_PRIVATE_CLEAR_FUNC(dummy, DUMMY_WRITE);
1423
+
1424
+static inline unsigned long get_page_private_data(struct page *page)
1425
+{
1426
+	unsigned long data = page_private(page);
1427
+
1428
+	if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
1429
+		return 0;
1430
+	return data >> PAGE_PRIVATE_MAX;
1431
+}
1432
+
1433
+static inline void set_page_private_data(struct page *page, unsigned long data)
1434
+{
1435
+	if (!PagePrivate(page)) {
1436
+		get_page(page);
1437
+		SetPagePrivate(page);
1438
+		set_page_private(page, 0);
1439
+	}
1440
+	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page));
1441
+	page_private(page) |= data << PAGE_PRIVATE_MAX;
1442
+}
1443
+
1444
+static inline void clear_page_private_data(struct page *page)
1445
+{
1446
+	page_private(page) &= (1 << PAGE_PRIVATE_MAX) - 1;
1447
+	if (page_private(page) == 1 << PAGE_PRIVATE_NOT_POINTER) {
1448
+		set_page_private(page, 0);
1449
+		if (PagePrivate(page)) {
1450
+			ClearPagePrivate(page);
1451
+			put_page(page);
1452
+		}
1453
+	}
1454
+}
13281455 
13291456 /* For compression */
13301457 enum compress_algorithm_type {
13311458 	COMPRESS_LZO,
13321459 	COMPRESS_LZ4,
13331460 	COMPRESS_ZSTD,
1461
+	COMPRESS_LZORLE,
13341462 	COMPRESS_MAX,
13351463 };
13361464 
1337
-#define COMPRESS_DATA_RESERVED_SIZE		5
1465
+enum compress_flag {
1466
+	COMPRESS_CHKSUM,
1467
+	COMPRESS_MAX_FLAG,
1468
+};
1469
+
1470
+#define	COMPRESS_WATERMARK			20
1471
+#define	COMPRESS_PERCENT			20
1472
+
1473
+#define COMPRESS_DATA_RESERVED_SIZE		4
13381474 struct compress_data {
13391475 	__le32 clen;			/* compressed data size */
1476
+	__le32 chksum;			/* compressed data chksum */
13401477 	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
13411478 	u8 cdata[];			/* compressed data */
13421479 };
....@@ -1344,6 +1481,8 @@
13441481 #define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
13451482 
13461483 #define F2FS_COMPRESSED_PAGE_MAGIC	0xF5F2C000
1484
+
1485
+#define	COMPRESS_LEVEL_OFFSET	8
13471486 
13481487 /* compress context */
13491488 struct compress_ctx {
....@@ -1369,10 +1508,10 @@
13691508 	struct inode *inode;		/* inode the context belong to */
13701509 	struct page **rpages;		/* pages store raw data in cluster */
13711510 	unsigned int nr_rpages;		/* total page number in rpages */
1372
-	refcount_t ref;			/* referrence count of raw page */
1511
+	atomic_t pending_pages;		/* in-flight compressed page count */
13731512 };
13741513 
1375
-/* decompress io context for read IO path */
1514
+/* Context for decompressing one cluster on the read IO path */
13761515 struct decompress_io_ctx {
13771516 	u32 magic;			/* magic number to indicate page is compressed */
13781517 	struct inode *inode;		/* inode the context belong to */
....@@ -1388,22 +1527,50 @@
13881527 	struct compress_data *cbuf;	/* virtual mapped address on cpages */
13891528 	size_t rlen;			/* valid data length in rbuf */
13901529 	size_t clen;			/* valid data length in cbuf */
1391
-	refcount_t ref;			/* referrence count of compressed page */
1392
-	bool failed;			/* indicate IO error during decompression */
1530
+
1531
+	/*
1532
+	 * The number of compressed pages remaining to be read in this cluster.
1533
+	 * This is initially nr_cpages.  It is decremented by 1 each time a page
1534
+	 * has been read (or failed to be read).  When it reaches 0, the cluster
1535
+	 * is decompressed (or an error is reported).
1536
+	 *
1537
+	 * If an error occurs before all the pages have been submitted for I/O,
1538
+	 * then this will never reach 0.  In this case the I/O submitter is
1539
+	 * responsible for calling f2fs_decompress_end_io() instead.
1540
+	 */
1541
+	atomic_t remaining_pages;
1542
+
1543
+	/*
1544
+	 * Number of references to this decompress_io_ctx.
1545
+	 *
1546
+	 * One reference is held for I/O completion.  This reference is dropped
1547
+	 * after the pagecache pages are updated and unlocked -- either after
1548
+	 * decompression (and verity if enabled), or after an error.
1549
+	 *
1550
+	 * In addition, each compressed page holds a reference while it is in a
1551
+	 * bio.  These references are necessary prevent compressed pages from
1552
+	 * being freed while they are still in a bio.
1553
+	 */
1554
+	refcount_t refcnt;
1555
+
1556
+	bool failed;			/* IO error occurred before decompression? */
1557
+	bool need_verity;		/* need fs-verity verification after decompression? */
13931558 	void *private;			/* payload buffer for specified decompression algorithm */
13941559 	void *private2;			/* extra payload buffer */
1560
+	struct work_struct verity_work;	/* work to verify the decompressed pages */
1561
+	struct work_struct free_work;	/* work for late free this structure itself */
13951562 };
13961563 
13971564 #define NULL_CLUSTER			((unsigned int)(~0))
13981565 #define MIN_COMPRESS_LOG_SIZE		2
13991566 #define MAX_COMPRESS_LOG_SIZE		8
1400
-#define MAX_COMPRESS_WINDOW_SIZE	((PAGE_SIZE) << MAX_COMPRESS_LOG_SIZE)
1567
+#define MAX_COMPRESS_WINDOW_SIZE(log_size)	((PAGE_SIZE) << (log_size))
14011568 
14021569 struct f2fs_sb_info {
14031570 	struct super_block *sb;			/* pointer to VFS super block */
14041571 	struct proc_dir_entry *s_proc;		/* proc entry */
14051572 	struct f2fs_super_block *raw_super;	/* raw super block pointer */
1406
-	struct rw_semaphore sb_lock;		/* lock for raw super block */
1573
+	struct f2fs_rwsem sb_lock;		/* lock for raw super block */
14071574 	int valid_super_block;			/* valid super block no */
14081575 	unsigned long s_flag;				/* flags for sbi */
14091576 	struct mutex writepages;		/* mutex for writepages() */
....@@ -1423,7 +1590,7 @@
14231590 	/* for bio operations */
14241591 	struct f2fs_bio_info *write_io[NR_PAGE_TYPE];	/* for write bios */
14251592 	/* keep migration IO order for LFS mode */
1426
-	struct rw_semaphore io_order_lock;
1593
+	struct f2fs_rwsem io_order_lock;
14271594 	mempool_t *write_io_dummy;		/* Dummy pages */
14281595 
14291596 	/* for checkpoint */
....@@ -1431,15 +1598,16 @@
14311598 	int cur_cp_pack;			/* remain current cp pack */
14321599 	spinlock_t cp_lock;			/* for flag in ckpt */
14331600 	struct inode *meta_inode;		/* cache meta blocks */
1434
-	struct mutex cp_mutex;			/* checkpoint procedure lock */
1435
-	struct rw_semaphore cp_rwsem;		/* blocking FS operations */
1436
-	struct rw_semaphore node_write;		/* locking node writes */
1437
-	struct rw_semaphore node_change;	/* locking node change */
1601
+	struct f2fs_rwsem cp_global_sem;	/* checkpoint procedure lock */
1602
+	struct f2fs_rwsem cp_rwsem;		/* blocking FS operations */
1603
+	struct f2fs_rwsem node_write;		/* locking node writes */
1604
+	struct f2fs_rwsem node_change;	/* locking node change */
14381605 	wait_queue_head_t cp_wait;
14391606 	unsigned long last_time[MAX_TIME];	/* to store time in jiffies */
14401607 	long interval_time[MAX_TIME];		/* to store thresholds */
1608
+	struct ckpt_req_control cprc_info;	/* for checkpoint request control */
14411609 
1442
-	struct inode_management im[MAX_INO_ENTRY];      /* manage inode cache */
1610
+	struct inode_management im[MAX_INO_ENTRY];	/* manage inode cache */
14431611 
14441612 	spinlock_t fsync_node_lock;		/* for node entry lock */
14451613 	struct list_head fsync_node_list;	/* node list head */
....@@ -1455,14 +1623,13 @@
14551623 	struct mutex flush_lock;		/* for flush exclusion */
14561624 
14571625 	/* for extent tree cache */
1458
-	struct radix_tree_root extent_tree_root;/* cache extent cache entries */
1459
-	struct mutex extent_tree_lock;	/* locking extent radix tree */
1460
-	struct list_head extent_list;		/* lru list for shrinker */
1461
-	spinlock_t extent_lock;			/* locking extent lru list */
1462
-	atomic_t total_ext_tree;		/* extent tree count */
1463
-	struct list_head zombie_list;		/* extent zombie tree list */
1464
-	atomic_t total_zombie_tree;		/* extent zombie tree count */
1465
-	atomic_t total_ext_node;		/* extent info count */
1626
+	struct extent_tree_info extent_tree[NR_EXTENT_CACHES];
1627
+	atomic64_t allocated_data_blocks;	/* for block age extent_cache */
1628
+
1629
+	/* The threshold used for hot and warm data seperation*/
1630
+	unsigned int hot_data_age_threshold;
1631
+	unsigned int warm_data_age_threshold;
1632
+	unsigned int last_age_weight;
14661633 
14671634 	/* basic filesystem units */
14681635 	unsigned int log_sectors_per_block;	/* log2 sectors per block */
....@@ -1473,14 +1640,15 @@
14731640 	unsigned int meta_ino_num;		/* meta inode number*/
14741641 	unsigned int log_blocks_per_seg;	/* log2 blocks per segment */
14751642 	unsigned int blocks_per_seg;		/* blocks per segment */
1643
+	unsigned int unusable_blocks_per_sec;	/* unusable blocks per section */
14761644 	unsigned int segs_per_sec;		/* segments per section */
14771645 	unsigned int secs_per_zone;		/* sections per zone */
14781646 	unsigned int total_sections;		/* total section count */
14791647 	unsigned int total_node_count;		/* total node block count */
14801648 	unsigned int total_valid_node_count;	/* valid node block count */
1481
-	loff_t max_file_blocks;			/* max block index of file */
14821649 	int dir_level;				/* directory level */
14831650 	int readdir_ra;				/* readahead inode in readdir */
1651
+	u64 max_io_bytes;			/* max io bytes to merge IOs */
14841652 
14851653 	block_t user_block_count;		/* # of user blocks */
14861654 	block_t total_valid_block_count;	/* # of valid blocks */
....@@ -1493,7 +1661,7 @@
14931661 	block_t unusable_block_count;		/* # of blocks saved by last cp */
14941662 
14951663 	unsigned int nquota_files;		/* # of quota sysfile */
1496
-	struct rw_semaphore quota_sem;		/* blocking cp for flags */
1664
+	struct f2fs_rwsem quota_sem;		/* blocking cp for flags */
14971665 
14981666 	/* # of pages, see count_type */
14991667 	atomic_t nr_pages[NR_COUNT_TYPE];
....@@ -1509,22 +1677,24 @@
15091677 	struct f2fs_mount_info mount_opt;	/* mount options */
15101678 
15111679 	/* for cleaning operations */
1512
-	struct rw_semaphore gc_lock;		/*
1680
+	struct f2fs_rwsem gc_lock;		/*
15131681 						 * semaphore for GC, avoid
15141682 						 * race between GC and GC or CP
15151683 						 */
15161684 	struct f2fs_gc_kthread	*gc_thread;	/* GC thread */
1685
+	struct atgc_management am;		/* atgc management */
15171686 	unsigned int cur_victim_sec;		/* current victim section num */
15181687 	unsigned int gc_mode;			/* current GC state */
15191688 	unsigned int next_victim_seg[2];	/* next segment in victim section */
1689
+
15201690 	/* for skip statistic */
1521
-	unsigned int atomic_files;              /* # of opened atomic file */
1691
+	unsigned int atomic_files;		/* # of opened atomic file */
15221692 	unsigned long long skipped_atomic_files[2];	/* FG_GC and BG_GC */
15231693 	unsigned long long skipped_gc_rwsem;		/* FG_GC only */
15241694 
15251695 	/* threshold for gc trials on pinned files */
15261696 	u64 gc_pin_file_threshold;
1527
-	struct rw_semaphore pin_sem;
1697
+	struct f2fs_rwsem pin_sem;
15281698 
15291699 	/* maximum # of trials to find a victim segment for SSR and GC */
15301700 	unsigned int max_victim_search;
....@@ -1543,15 +1713,19 @@
15431713 	unsigned int segment_count[2];		/* # of allocated segments */
15441714 	unsigned int block_count[2];		/* # of allocated blocks */
15451715 	atomic_t inplace_count;		/* # of inplace update */
1546
-	atomic64_t total_hit_ext;		/* # of lookup extent cache */
1547
-	atomic64_t read_hit_rbtree;		/* # of hit rbtree extent node */
1548
-	atomic64_t read_hit_largest;		/* # of hit largest extent node */
1549
-	atomic64_t read_hit_cached;		/* # of hit cached extent node */
1716
+	/* # of lookup extent cache */
1717
+	atomic64_t total_hit_ext[NR_EXTENT_CACHES];
1718
+	/* # of hit rbtree extent node */
1719
+	atomic64_t read_hit_rbtree[NR_EXTENT_CACHES];
1720
+	/* # of hit cached extent node */
1721
+	atomic64_t read_hit_cached[NR_EXTENT_CACHES];
1722
+	/* # of hit largest extent node in read extent cache */
1723
+	atomic64_t read_hit_largest;
15501724 	atomic_t inline_xattr;			/* # of inline_xattr inodes */
15511725 	atomic_t inline_inode;			/* # of inline_data inodes */
15521726 	atomic_t inline_dir;			/* # of inline_dentry inodes */
15531727 	atomic_t compr_inode;			/* # of compressed inodes */
1554
-	atomic_t compr_blocks;			/* # of compressed blocks */
1728
+	atomic64_t compr_blocks;		/* # of compressed blocks */
15551729 	atomic_t vw_cnt;			/* # of volatile writes */
15561730 	atomic_t max_aw_cnt;			/* max # of atomic writes */
15571731 	atomic_t max_vw_cnt;			/* max # of volatile writes */
....@@ -1574,8 +1748,14 @@
15741748 	unsigned int node_io_flag;
15751749 
15761750 	/* For sysfs suppport */
1577
-	struct kobject s_kobj;
1751
+	struct kobject s_kobj;			/* /sys/fs/f2fs/<devname> */
15781752 	struct completion s_kobj_unregister;
1753
+
1754
+	struct kobject s_stat_kobj;		/* /sys/fs/f2fs/<devname>/stat */
1755
+	struct completion s_stat_kobj_unregister;
1756
+
1757
+	struct kobject s_feature_list_kobj;		/* /sys/fs/f2fs/<devname>/feature_list */
1758
+	struct completion s_feature_list_kobj_unregister;
15791759 
15801760 	/* For shrinker support */
15811761 	struct list_head s_list;
....@@ -1600,6 +1780,26 @@
16001780 
16011781 	struct kmem_cache *inline_xattr_slab;	/* inline xattr entry */
16021782 	unsigned int inline_xattr_slab_size;	/* default inline xattr slab size */
1783
+
1784
+	/* For reclaimed segs statistics per each GC mode */
1785
+	unsigned int gc_segment_mode;		/* GC state for reclaimed segments */
1786
+	unsigned int gc_reclaimed_segs[MAX_GC_MODE];	/* Reclaimed segs for each mode */
1787
+
1788
+#ifdef CONFIG_F2FS_FS_COMPRESSION
1789
+	struct kmem_cache *page_array_slab;	/* page array entry */
1790
+	unsigned int page_array_slab_size;	/* default page array slab size */
1791
+
1792
+	/* For runtime compression statistics */
1793
+	u64 compr_written_block;
1794
+	u64 compr_saved_block;
1795
+	u32 compr_new_inode;
1796
+
1797
+	/* For compressed block cache */
1798
+	struct inode *compress_inode;		/* cache compressed blocks */
1799
+	unsigned int compress_percent;		/* cache page percentage */
1800
+	unsigned int compress_watermark;	/* cache page watermark */
1801
+	atomic_t compress_page_hit;		/* cache hit count */
1802
+#endif
16031803 };
16041804 
16051805 struct f2fs_private_dio {
....@@ -1651,13 +1851,6 @@
16511851 	return sbi->s_ndevs > 1;
16521852 }
16531853 
1654
-/* For write statistics. Suppose sector size is 512 bytes,
1655
- * and the return value is in kbytes. s is of struct f2fs_sb_info.
1656
- */
1657
-#define BD_PART_WRITTEN(s)						 \
1658
-(((u64)part_stat_read((s)->sb->s_bdev->bd_part, sectors[STAT_WRITE]) -   \
1659
-		(s)->sectors_written_start) >> 1)
1660
-
16611854 static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
16621855 {
16631856 	unsigned long now = jiffies;
....@@ -1707,7 +1900,6 @@
17071900 	BUG_ON(crypto_shash_descsize(sbi->s_chksum_driver) != sizeof(desc.ctx));
17081901 
17091902 	desc.shash.tfm = sbi->s_chksum_driver;
1710
-	desc.shash.flags = 0;
17111903 	*(u32 *)desc.ctx = crc;
17121904 
17131905 	err = crypto_shash_update(&desc.shash, address, length);
....@@ -1925,29 +2117,93 @@
19252117 	return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set;
19262118 }
19272119 
2120
+#define init_f2fs_rwsem(sem)					\
2121
+do {								\
2122
+	static struct lock_class_key __key;			\
2123
+								\
2124
+	__init_f2fs_rwsem((sem), #sem, &__key);			\
2125
+} while (0)
2126
+
2127
+static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem,
2128
+		const char *sem_name, struct lock_class_key *key)
2129
+{
2130
+	__init_rwsem(&sem->internal_rwsem, sem_name, key);
2131
+	init_waitqueue_head(&sem->read_waiters);
2132
+}
2133
+
2134
+static inline int f2fs_rwsem_is_locked(struct f2fs_rwsem *sem)
2135
+{
2136
+	return rwsem_is_locked(&sem->internal_rwsem);
2137
+}
2138
+
2139
+static inline int f2fs_rwsem_is_contended(struct f2fs_rwsem *sem)
2140
+{
2141
+	return rwsem_is_contended(&sem->internal_rwsem);
2142
+}
2143
+
2144
+static inline void f2fs_down_read(struct f2fs_rwsem *sem)
2145
+{
2146
+	wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem));
2147
+}
2148
+
2149
+static inline int f2fs_down_read_trylock(struct f2fs_rwsem *sem)
2150
+{
2151
+	return down_read_trylock(&sem->internal_rwsem);
2152
+}
2153
+
2154
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
2155
+static inline void f2fs_down_read_nested(struct f2fs_rwsem *sem, int subclass)
2156
+{
2157
+	down_read_nested(&sem->internal_rwsem, subclass);
2158
+}
2159
+#else
2160
+#define f2fs_down_read_nested(sem, subclass) f2fs_down_read(sem)
2161
+#endif
2162
+
2163
+static inline void f2fs_up_read(struct f2fs_rwsem *sem)
2164
+{
2165
+	up_read(&sem->internal_rwsem);
2166
+}
2167
+
2168
+static inline void f2fs_down_write(struct f2fs_rwsem *sem)
2169
+{
2170
+	down_write(&sem->internal_rwsem);
2171
+}
2172
+
2173
+static inline int f2fs_down_write_trylock(struct f2fs_rwsem *sem)
2174
+{
2175
+	return down_write_trylock(&sem->internal_rwsem);
2176
+}
2177
+
2178
+static inline void f2fs_up_write(struct f2fs_rwsem *sem)
2179
+{
2180
+	up_write(&sem->internal_rwsem);
2181
+	wake_up_all(&sem->read_waiters);
2182
+}
2183
+
19282184 static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
19292185 {
1930
-	down_read(&sbi->cp_rwsem);
2186
+	f2fs_down_read(&sbi->cp_rwsem);
19312187 }
19322188 
19332189 static inline int f2fs_trylock_op(struct f2fs_sb_info *sbi)
19342190 {
1935
-	return down_read_trylock(&sbi->cp_rwsem);
2191
+	return f2fs_down_read_trylock(&sbi->cp_rwsem);
19362192 }
19372193 
19382194 static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi)
19392195 {
1940
-	up_read(&sbi->cp_rwsem);
2196
+	f2fs_up_read(&sbi->cp_rwsem);
19412197 }
19422198 
19432199 static inline void f2fs_lock_all(struct f2fs_sb_info *sbi)
19442200 {
1945
-	down_write(&sbi->cp_rwsem);
2201
+	f2fs_down_write(&sbi->cp_rwsem);
19462202 }
19472203 
19482204 static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi)
19492205 {
1950
-	up_write(&sbi->cp_rwsem);
2206
+	f2fs_up_write(&sbi->cp_rwsem);
19512207 }
19522208 
19532209 static inline int __get_cp_reason(struct f2fs_sb_info *sbi)
....@@ -2037,6 +2293,11 @@
20372293 
20382294 	if (!__allow_reserved_blocks(sbi, inode, true))
20392295 		avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
2296
+
2297
+	if (F2FS_IO_ALIGNED(sbi))
2298
+		avail_user_block_count -= sbi->blocks_per_seg *
2299
+				SM_I(sbi)->additional_reserved_segments;
2300
+
20402301 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
20412302 		if (avail_user_block_count > sbi->unusable_block_count)
20422303 			avail_user_block_count -= sbi->unusable_block_count;
....@@ -2199,6 +2460,7 @@
21992460 static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
22002461 {
22012462 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2463
+	void *tmp_ptr = &ckpt->sit_nat_version_bitmap;
22022464 	int offset;
22032465 
22042466 	if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) {
....@@ -2208,7 +2470,7 @@
22082470 		 * if large_nat_bitmap feature is enabled, leave checksum
22092471 		 * protection for all nat/sit bitmaps.
22102472 		 */
2211
-		return &ckpt->sit_nat_version_bitmap + offset + sizeof(__le32);
2473
+		return tmp_ptr + offset + sizeof(__le32);
22122474 	}
22132475 
22142476 	if (__cp_payload(sbi) > 0) {
....@@ -2219,7 +2481,7 @@
22192481 	} else {
22202482 		offset = (flag == NAT_BITMAP) ?
22212483 			le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
2222
-		return &ckpt->sit_nat_version_bitmap + offset;
2484
+		return tmp_ptr + offset;
22232485 	}
22242486 }
22252487 
....@@ -2251,6 +2513,7 @@
22512513 	return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
22522514 }
22532515 
2516
+extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync);
22542517 static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
22552518 					struct inode *inode, bool is_inode)
22562519 {
....@@ -2282,6 +2545,11 @@
22822545 
22832546 	if (!__allow_reserved_blocks(sbi, inode, false))
22842547 		valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
2548
+
2549
+	if (F2FS_IO_ALIGNED(sbi))
2550
+		valid_block_count += sbi->blocks_per_seg *
2551
+				SM_I(sbi)->additional_reserved_segments;
2552
+
22852553 	user_block_count = sbi->user_block_count;
22862554 	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
22872555 		user_block_count -= sbi->unusable_block_count;
....@@ -2326,11 +2594,17 @@
23262594 {
23272595 	spin_lock(&sbi->stat_lock);
23282596 
2329
-	f2fs_bug_on(sbi, !sbi->total_valid_block_count);
2330
-	f2fs_bug_on(sbi, !sbi->total_valid_node_count);
2597
+	if (unlikely(!sbi->total_valid_block_count ||
2598
+			!sbi->total_valid_node_count)) {
2599
+		f2fs_warn(sbi, "dec_valid_node_count: inconsistent block counts, total_valid_block:%u, total_valid_node:%u",
2600
+			  sbi->total_valid_block_count,
2601
+			  sbi->total_valid_node_count);
2602
+		set_sbi_flag(sbi, SBI_NEED_FSCK);
2603
+	} else {
2604
+		sbi->total_valid_block_count--;
2605
+		sbi->total_valid_node_count--;
2606
+	}
23312607 
2332
-	sbi->total_valid_node_count--;
2333
-	sbi->total_valid_block_count--;
23342608 	if (sbi->reserved_blocks &&
23352609 		sbi->current_reserved_blocks < sbi->reserved_blocks)
23362610 		sbi->current_reserved_blocks++;
....@@ -2458,25 +2732,39 @@
24582732 	return entry;
24592733 }
24602734 
2461
-static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2735
+static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
24622736 {
2463
-	if (sbi->gc_mode == GC_URGENT)
2464
-		return true;
2465
-
24662737 	if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
24672738 		get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
24682739 		get_pages(sbi, F2FS_WB_CP_DATA) ||
24692740 		get_pages(sbi, F2FS_DIO_READ) ||
24702741 		get_pages(sbi, F2FS_DIO_WRITE))
2471
-		return false;
2742
+		return true;
24722743 
24732744 	if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
24742745 			atomic_read(&SM_I(sbi)->dcc_info->queued_discard))
2475
-		return false;
2746
+		return true;
24762747 
24772748 	if (SM_I(sbi) && SM_I(sbi)->fcc_info &&
24782749 			atomic_read(&SM_I(sbi)->fcc_info->queued_flush))
2750
+		return true;
2751
+	return false;
2752
+}
2753
+
2754
+static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
2755
+{
2756
+	if (sbi->gc_mode == GC_URGENT_HIGH)
2757
+		return true;
2758
+
2759
+	if (is_inflight_io(sbi, type))
24792760 		return false;
2761
+
2762
+	if (sbi->gc_mode == GC_URGENT_MID)
2763
+		return true;
2764
+
2765
+	if (sbi->gc_mode == GC_URGENT_LOW &&
2766
+			(type == DISCARD_TIME || type == GC_TIME))
2767
+		return true;
24802768 
24812769 	return f2fs_time_over(sbi, type);
24822770 }
....@@ -2643,17 +2931,18 @@
26432931 	case FI_NEW_INODE:
26442932 		if (set)
26452933 			return;
2646
-		/* fall through */
2934
+		fallthrough;
26472935 	case FI_DATA_EXIST:
26482936 	case FI_INLINE_DOTS:
26492937 	case FI_PIN_FILE:
2938
+	case FI_COMPRESS_RELEASED:
26502939 		f2fs_mark_inode_dirty_sync(inode, true);
26512940 	}
26522941 }
26532942 
26542943 static inline void set_inode_flag(struct inode *inode, int flag)
26552944 {
2656
-	test_and_set_bit(flag, F2FS_I(inode)->flags);
2945
+	set_bit(flag, F2FS_I(inode)->flags);
26572946 	__mark_inode_dirty_flag(inode, flag, true);
26582947 }
26592948 
....@@ -2664,7 +2953,7 @@
26642953 
26652954 static inline void clear_inode_flag(struct inode *inode, int flag)
26662955 {
2667
-	test_and_clear_bit(flag, F2FS_I(inode)->flags);
2956
+	clear_bit(flag, F2FS_I(inode)->flags);
26682957 	__mark_inode_dirty_flag(inode, flag, false);
26692958 }
26702959 
....@@ -2768,6 +3057,8 @@
27683057 		set_bit(FI_EXTRA_ATTR, fi->flags);
27693058 	if (ri->i_inline & F2FS_PIN_FILE)
27703059 		set_bit(FI_PIN_FILE, fi->flags);
3060
+	if (ri->i_inline & F2FS_COMPRESS_RELEASED)
3061
+		set_bit(FI_COMPRESS_RELEASED, fi->flags);
27713062 }
27723063 
27733064 static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri)
....@@ -2788,6 +3079,8 @@
27883079 		ri->i_inline |= F2FS_EXTRA_ATTR;
27893080 	if (is_inode_flag_set(inode, FI_PIN_FILE))
27903081 		ri->i_inline |= F2FS_PIN_FILE;
3082
+	if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED))
3083
+		ri->i_inline |= F2FS_COMPRESS_RELEASED;
27913084 }
27923085 
27933086 static inline int f2fs_has_extra_attr(struct inode *inode)
....@@ -2804,6 +3097,22 @@
28043097 {
28053098 	return S_ISREG(inode->i_mode) &&
28063099 		is_inode_flag_set(inode, FI_COMPRESSED_FILE);
3100
+}
3101
+
3102
+static inline bool f2fs_need_compress_data(struct inode *inode)
3103
+{
3104
+	int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode;
3105
+
3106
+	if (!f2fs_compressed_file(inode))
3107
+		return false;
3108
+
3109
+	if (compress_mode == COMPR_MODE_FS)
3110
+		return true;
3111
+	else if (compress_mode == COMPR_MODE_USER &&
3112
+			is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
3113
+		return true;
3114
+
3115
+	return false;
28073116 }
28083117 
28093118 static inline unsigned int addrs_per_inode(struct inode *inode)
....@@ -2980,25 +3289,6 @@
29803289 	return false;
29813290 }
29823291 
2983
-static inline bool f2fs_may_extent_tree(struct inode *inode)
2984
-{
2985
-	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2986
-
2987
-	if (!test_opt(sbi, EXTENT_CACHE) ||
2988
-			is_inode_flag_set(inode, FI_NO_EXTENT) ||
2989
-			is_inode_flag_set(inode, FI_COMPRESSED_FILE))
2990
-		return false;
2991
-
2992
-	/*
2993
-	 * for recovered files during mount do not create extents
2994
-	 * if shrinker is not registered.
2995
-	 */
2996
-	if (list_empty(&sbi->s_list))
2997
-		return false;
2998
-
2999
-	return S_ISREG(inode->i_mode);
3000
-}
3001
-
30023292 static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi,
30033293 					size_t size, gfp_t flags)
30043294 {
....@@ -3122,27 +3412,6 @@
31223412 	return true;
31233413 }
31243414 
3125
-static inline void f2fs_set_page_private(struct page *page,
3126
-						unsigned long data)
3127
-{
3128
-	if (PagePrivate(page))
3129
-		return;
3130
-
3131
-	get_page(page);
3132
-	SetPagePrivate(page);
3133
-	set_page_private(page, data);
3134
-}
3135
-
3136
-static inline void f2fs_clear_page_private(struct page *page)
3137
-{
3138
-	if (!PagePrivate(page))
3139
-		return;
3140
-
3141
-	set_page_private(page, 0);
3142
-	ClearPagePrivate(page);
3143
-	f2fs_put_page(page, 0);
3144
-}
3145
-
31463415 /*
31473416  * file.c
31483417  */
....@@ -3248,7 +3517,9 @@
32483517 void f2fs_inode_synced(struct inode *inode);
32493518 int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
32503519 int f2fs_quota_sync(struct super_block *sb, int type);
3520
+loff_t max_file_blocks(struct inode *inode);
32513521 void f2fs_quota_off_umount(struct super_block *sb);
3522
+void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason);
32523523 int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
32533524 int f2fs_sync_fs(struct super_block *sb, int sync);
32543525 int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
....@@ -3261,7 +3532,6 @@
32613532 /*
32623533  * node.c
32633534  */
3264
-struct dnode_of_data;
32653535 struct node_info;
32663536 
32673537 int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
....@@ -3274,7 +3544,7 @@
32743544 bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
32753545 bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino);
32763546 int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
3277
-						struct node_info *ni);
3547
+				struct node_info *ni, bool checkpoint_context);
32783548 pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
32793549 int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
32803550 int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
....@@ -3288,7 +3558,7 @@
32883558 struct page *f2fs_get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid);
32893559 struct page *f2fs_get_node_page_ra(struct page *parent, int start);
32903560 int f2fs_move_node_page(struct page *node_page, int gc_type);
3291
-int f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
3561
+void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
32923562 int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
32933563 			struct writeback_control *wbc, bool atomic,
32943564 			unsigned int *seq_id);
....@@ -3338,9 +3608,16 @@
33383608 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
33393609 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
33403610 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
3341
-void allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3611
+bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
3612
+void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
3613
+void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi);
3614
+void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi);
3615
+void f2fs_get_new_segment(struct f2fs_sb_info *sbi,
3616
+			unsigned int *newseg, bool new_sec, int dir);
3617
+void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
33423618 					unsigned int start, unsigned int end);
3343
-void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi, int type);
3619
+void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force);
3620
+void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
33443621 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
33453622 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
33463623 					struct cp_control *cpc);
....@@ -3355,7 +3632,8 @@
33553632 int f2fs_inplace_write_data(struct f2fs_io_info *fio);
33563633 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
33573634 			block_t old_blkaddr, block_t new_blkaddr,
3358
-			bool recover_curseg, bool recover_newaddr);
3635
+			bool recover_curseg, bool recover_newaddr,
3636
+			bool from_gc);
33593637 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
33603638 			block_t old_addr, block_t new_addr,
33613639 			unsigned char version, bool recover_curseg,
....@@ -3363,7 +3641,7 @@
33633641 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
33643642 			block_t old_blkaddr, block_t *new_blkaddr,
33653643 			struct f2fs_summary *sum, int type,
3366
-			struct f2fs_io_info *fio, bool add_list);
3644
+			struct f2fs_io_info *fio);
33673645 void f2fs_wait_on_page_writeback(struct page *page,
33683646 			enum page_type type, bool ordered, bool locked);
33693647 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
....@@ -3374,6 +3652,8 @@
33743652 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
33753653 			unsigned int val, int alloc);
33763654 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
3655
+int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi);
3656
+int f2fs_check_write_pointer(struct f2fs_sb_info *sbi);
33773657 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi);
33783658 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi);
33793659 int __init f2fs_create_segment_manager_caches(void);
....@@ -3381,14 +3661,20 @@
33813661 int f2fs_rw_hint_to_seg_type(enum rw_hint hint);
33823662 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
33833663 			enum page_type type, enum temp_type temp);
3664
+unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
3665
+			unsigned int segno);
3666
+unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
3667
+			unsigned int segno);
33843668 
33853669 /*
33863670  * checkpoint.c
33873671  */
3388
-void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io);
3672
+void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
3673
+							unsigned char reason);
3674
+void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
33893675 struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
33903676 struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
3391
-struct page *f2fs_get_meta_page_nofail(struct f2fs_sb_info *sbi, pgoff_t index);
3677
+struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
33923678 struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index);
33933679 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
33943680 					block_t blkaddr, int type);
....@@ -3414,19 +3700,24 @@
34143700 int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
34153701 void f2fs_update_dirty_page(struct inode *inode, struct page *page);
34163702 void f2fs_remove_dirty_inode(struct inode *inode);
3417
-int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type);
3703
+int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
3704
+								bool from_cp);
34183705 void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
3706
+u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
34193707 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
34203708 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
34213709 int __init f2fs_create_checkpoint_caches(void);
34223710 void f2fs_destroy_checkpoint_caches(void);
3711
+int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi);
3712
+int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi);
3713
+void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi);
3714
+void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi);
34233715 
34243716 /*
34253717  * data.c
34263718  */
34273719 int __init f2fs_init_bioset(void);
34283720 void f2fs_destroy_bioset(void);
3429
-struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio);
34303721 int f2fs_init_bio_entry_cache(void);
34313722 void f2fs_destroy_bio_entry_cache(void);
34323723 void f2fs_submit_bio(struct f2fs_sb_info *sbi,
....@@ -3451,9 +3742,6 @@
34513742 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index);
34523743 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from);
34533744 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index);
3454
-int f2fs_mpage_readpages(struct address_space *mapping,
3455
-			struct list_head *pages, struct page *page,
3456
-			unsigned nr_pages, bool is_readahead);
34573745 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
34583746 			int op_flags, bool for_write);
34593747 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index);
....@@ -3462,7 +3750,7 @@
34623750 struct page *f2fs_get_new_data_page(struct inode *inode,
34633751 			struct page *ipage, pgoff_t index, bool new_i_size);
34643752 int f2fs_do_write_data_page(struct f2fs_io_info *fio);
3465
-void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
3753
+void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock);
34663754 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
34673755 			int create, int flag);
34683756 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
....@@ -3474,7 +3762,7 @@
34743762 				struct bio **bio, sector_t *last_block,
34753763 				struct writeback_control *wbc,
34763764 				enum iostat_type io_type,
3477
-				int compr_blocks);
3765
+				int compr_blocks, bool allow_balance);
34783766 void f2fs_invalidate_page(struct page *page, unsigned int offset,
34793767 			unsigned int length);
34803768 int f2fs_release_page(struct page *page, gfp_t wait);
....@@ -3483,7 +3771,7 @@
34833771 			struct page *page, enum migrate_mode mode);
34843772 #endif
34853773 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
3486
-void f2fs_clear_radix_tree_dirty_tag(struct page *page);
3774
+void f2fs_clear_page_cache_dirty_tag(struct page *page);
34873775 int f2fs_init_post_read_processing(void);
34883776 void f2fs_destroy_post_read_processing(void);
34893777 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi);
....@@ -3495,16 +3783,20 @@
34953783 int f2fs_start_gc_thread(struct f2fs_sb_info *sbi);
34963784 void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi);
34973785 block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode);
3498
-int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background,
3786
+int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background, bool force,
34993787 			unsigned int segno);
35003788 void f2fs_build_gc_manager(struct f2fs_sb_info *sbi);
3501
-int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count);
3789
+int f2fs_resize_fs(struct file *filp, __u64 block_count);
3790
+int __init f2fs_create_garbage_collection_cache(void);
3791
+void f2fs_destroy_garbage_collection_cache(void);
35023792 
35033793 /*
35043794  * recovery.c
35053795  */
35063796 int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only);
35073797 bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi);
3798
+int __init f2fs_create_recovery_cache(void);
3799
+void f2fs_destroy_recovery_cache(void);
35083800 
35093801 /*
35103802  * debug.c
....@@ -3515,9 +3807,19 @@
35153807 	struct f2fs_sb_info *sbi;
35163808 	int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs;
35173809 	int main_area_segs, main_area_sections, main_area_zones;
3518
-	unsigned long long hit_largest, hit_cached, hit_rbtree;
3519
-	unsigned long long hit_total, total_ext;
3520
-	int ext_tree, zombie_tree, ext_node;
3810
+	unsigned long long hit_cached[NR_EXTENT_CACHES];
3811
+	unsigned long long hit_rbtree[NR_EXTENT_CACHES];
3812
+	unsigned long long total_ext[NR_EXTENT_CACHES];
3813
+	unsigned long long hit_total[NR_EXTENT_CACHES];
3814
+	int ext_tree[NR_EXTENT_CACHES];
3815
+	int zombie_tree[NR_EXTENT_CACHES];
3816
+	int ext_node[NR_EXTENT_CACHES];
3817
+	/* to count memory footprint */
3818
+	unsigned long long ext_mem[NR_EXTENT_CACHES];
3819
+	/* for read extent cache */
3820
+	unsigned long long hit_largest;
3821
+	/* for block age extent cache */
3822
+	unsigned long long allocated_data_blocks;
35213823 	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta;
35223824 	int ndirty_data, ndirty_qdata;
35233825 	int inmem_pages;
....@@ -3533,14 +3835,18 @@
35333835 	int nr_discarding, nr_discarded;
35343836 	int nr_discard_cmd;
35353837 	unsigned int undiscard_blks;
3838
+	int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
3839
+	unsigned int cur_ckpt_time, peak_ckpt_time;
35363840 	int inline_xattr, inline_inode, inline_dir, append, update, orphans;
3537
-	int compr_inode, compr_blocks;
3841
+	int compr_inode;
3842
+	unsigned long long compr_blocks;
35383843 	int aw_cnt, max_aw_cnt, vw_cnt, max_vw_cnt;
35393844 	unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
35403845 	unsigned int bimodal, avg_vblocks;
35413846 	int util_free, util_valid, util_invalid;
35423847 	int rsvd_segs, overp_segs;
3543
-	int dirty_count, node_pages, meta_pages;
3848
+	int dirty_count, node_pages, meta_pages, compress_pages;
3849
+	int compress_page_hit;
35443850 	int prefree_count, call_count, cp_count, bg_cp_count;
35453851 	int tot_segs, node_segs, data_segs, free_segs, free_secs;
35463852 	int bg_node_segs, bg_data_segs;
....@@ -3550,6 +3856,9 @@
35503856 	int curseg[NR_CURSEG_TYPE];
35513857 	int cursec[NR_CURSEG_TYPE];
35523858 	int curzone[NR_CURSEG_TYPE];
3859
+	unsigned int dirty_seg[NR_CURSEG_TYPE];
3860
+	unsigned int full_seg[NR_CURSEG_TYPE];
3861
+	unsigned int valid_blks[NR_CURSEG_TYPE];
35533862 
35543863 	unsigned int meta_count[META_MAX];
35553864 	unsigned int segment_count[2];
....@@ -3571,10 +3880,10 @@
35713880 #define stat_other_skip_bggc_count(sbi)	((sbi)->other_skip_bggc++)
35723881 #define stat_inc_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]++)
35733882 #define stat_dec_dirty_inode(sbi, type)	((sbi)->ndirty_inode[type]--)
3574
-#define stat_inc_total_hit(sbi)		(atomic64_inc(&(sbi)->total_hit_ext))
3575
-#define stat_inc_rbtree_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_rbtree))
3883
+#define stat_inc_total_hit(sbi, type)		(atomic64_inc(&(sbi)->total_hit_ext[type]))
3884
+#define stat_inc_rbtree_node_hit(sbi, type)	(atomic64_inc(&(sbi)->read_hit_rbtree[type]))
35763885 #define stat_inc_largest_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_largest))
3577
-#define stat_inc_cached_node_hit(sbi)	(atomic64_inc(&(sbi)->read_hit_cached))
3886
+#define stat_inc_cached_node_hit(sbi, type)	(atomic64_inc(&(sbi)->read_hit_cached[type]))
35783887 #define stat_inc_inline_xattr(inode)					\
35793888 	do {								\
35803889 		if (f2fs_has_inline_xattr(inode))			\
....@@ -3616,9 +3925,9 @@
36163925 			(atomic_dec(&F2FS_I_SB(inode)->compr_inode));	\
36173926 	} while (0)
36183927 #define stat_add_compr_blocks(inode, blocks)				\
3619
-		(atomic_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
3928
+		(atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
36203929 #define stat_sub_compr_blocks(inode, blocks)				\
3621
-		(atomic_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
3930
+		(atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
36223931 #define stat_inc_meta_count(sbi, blkaddr)				\
36233932 	do {								\
36243933 		if (blkaddr < SIT_I(sbi)->sit_base_addr)		\
....@@ -3700,10 +4009,10 @@
37004009 #define stat_other_skip_bggc_count(sbi)			do { } while (0)
37014010 #define stat_inc_dirty_inode(sbi, type)			do { } while (0)
37024011 #define stat_dec_dirty_inode(sbi, type)			do { } while (0)
3703
-#define stat_inc_total_hit(sbi)				do { } while (0)
3704
-#define stat_inc_rbtree_node_hit(sbi)			do { } while (0)
4012
+#define stat_inc_total_hit(sbi, type)			do { } while (0)
4013
+#define stat_inc_rbtree_node_hit(sbi, type)		do { } while (0)
37054014 #define stat_inc_largest_node_hit(sbi)			do { } while (0)
3706
-#define stat_inc_cached_node_hit(sbi)			do { } while (0)
4015
+#define stat_inc_cached_node_hit(sbi, type)		do { } while (0)
37074016 #define stat_inc_inline_xattr(inode)			do { } while (0)
37084017 #define stat_dec_inline_xattr(inode)			do { } while (0)
37094018 #define stat_inc_inline_inode(inode)			do { } while (0)
....@@ -3714,8 +4023,6 @@
37144023 #define stat_dec_compr_inode(inode)			do { } while (0)
37154024 #define stat_add_compr_blocks(inode, blocks)		do { } while (0)
37164025 #define stat_sub_compr_blocks(inode, blocks)		do { } while (0)
3717
-#define stat_inc_atomic_write(inode)			do { } while (0)
3718
-#define stat_dec_atomic_write(inode)			do { } while (0)
37194026 #define stat_update_max_atomic_write(inode)		do { } while (0)
37204027 #define stat_inc_volatile_write(inode)			do { } while (0)
37214028 #define stat_dec_volatile_write(inode)			do { } while (0)
....@@ -3752,6 +4059,7 @@
37524059  * inline.c
37534060  */
37544061 bool f2fs_may_inline_data(struct inode *inode);
4062
+bool f2fs_sanity_check_inline_data(struct inode *inode);
37554063 bool f2fs_may_inline_dentry(struct inode *inode);
37564064 void f2fs_do_read_inline_data(struct page *page, struct page *ipage);
37574065 void f2fs_truncate_inline_inode(struct inode *inode,
....@@ -3794,6 +4102,10 @@
37944102  */
37954103 struct rb_entry *f2fs_lookup_rb_tree(struct rb_root_cached *root,
37964104 				struct rb_entry *cached_re, unsigned int ofs);
4105
+struct rb_node **f2fs_lookup_rb_tree_ext(struct f2fs_sb_info *sbi,
4106
+				struct rb_root_cached *root,
4107
+				struct rb_node **parent,
4108
+				unsigned long long key, bool *left_most);
37974109 struct rb_node **f2fs_lookup_rb_tree_for_insert(struct f2fs_sb_info *sbi,
37984110 				struct rb_root_cached *root,
37994111 				struct rb_node **parent,
....@@ -3804,20 +4116,34 @@
38044116 		struct rb_node ***insert_p, struct rb_node **insert_parent,
38054117 		bool force, bool *leftmost);
38064118 bool f2fs_check_rb_tree_consistence(struct f2fs_sb_info *sbi,
3807
-						struct rb_root_cached *root);
3808
-unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink);
3809
-bool f2fs_init_extent_tree(struct inode *inode, struct f2fs_extent *i_ext);
4119
+				struct rb_root_cached *root, bool check_key);
4120
+void f2fs_init_extent_tree(struct inode *inode);
38104121 void f2fs_drop_extent_tree(struct inode *inode);
3811
-unsigned int f2fs_destroy_extent_node(struct inode *inode);
4122
+void f2fs_destroy_extent_node(struct inode *inode);
38124123 void f2fs_destroy_extent_tree(struct inode *inode);
3813
-bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs,
3814
-			struct extent_info *ei);
3815
-void f2fs_update_extent_cache(struct dnode_of_data *dn);
3816
-void f2fs_update_extent_cache_range(struct dnode_of_data *dn,
3817
-			pgoff_t fofs, block_t blkaddr, unsigned int len);
38184124 void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi);
38194125 int __init f2fs_create_extent_cache(void);
38204126 void f2fs_destroy_extent_cache(void);
4127
+
4128
+/* read extent cache ops */
4129
+void f2fs_init_read_extent_tree(struct inode *inode, struct page *ipage);
4130
+bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs,
4131
+			struct extent_info *ei);
4132
+void f2fs_update_read_extent_cache(struct dnode_of_data *dn);
4133
+void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn,
4134
+			pgoff_t fofs, block_t blkaddr, unsigned int len);
4135
+unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi,
4136
+			int nr_shrink);
4137
+
4138
+/* block age extent cache ops */
4139
+void f2fs_init_age_extent_tree(struct inode *inode);
4140
+bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs,
4141
+			struct extent_info *ei);
4142
+void f2fs_update_age_extent_cache(struct dnode_of_data *dn);
4143
+void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn,
4144
+			pgoff_t fofs, unsigned int len);
4145
+unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi,
4146
+			int nr_shrink);
38214147 
38224148 /*
38234149  * sysfs.c
....@@ -3871,7 +4197,9 @@
38714197 bool f2fs_is_compress_backend_ready(struct inode *inode);
38724198 int f2fs_init_compress_mempool(void);
38734199 void f2fs_destroy_compress_mempool(void);
3874
-void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity);
4200
+void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task);
4201
+void f2fs_end_read_compressed_page(struct page *page, bool failed,
4202
+				block_t blkaddr, bool in_task);
38754203 bool f2fs_cluster_is_empty(struct compress_ctx *cc);
38764204 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index);
38774205 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page);
....@@ -3880,16 +4208,45 @@
38804208 						struct writeback_control *wbc,
38814209 						enum iostat_type io_type);
38824210 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index);
4211
+void f2fs_update_read_extent_tree_range_compressed(struct inode *inode,
4212
+				pgoff_t fofs, block_t blkaddr,
4213
+				unsigned int llen, unsigned int c_len);
38834214 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
38844215 				unsigned nr_pages, sector_t *last_block_in_bio,
38854216 				bool is_readahead, bool for_write);
38864217 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc);
3887
-void f2fs_free_dic(struct decompress_io_ctx *dic);
3888
-void f2fs_decompress_end_io(struct page **rpages,
3889
-			unsigned int cluster_size, bool err, bool verity);
4218
+void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
4219
+				bool in_task);
4220
+void f2fs_put_page_dic(struct page *page, bool in_task);
4221
+unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn);
38904222 int f2fs_init_compress_ctx(struct compress_ctx *cc);
3891
-void f2fs_destroy_compress_ctx(struct compress_ctx *cc);
4223
+void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
38924224 void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
4225
+int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
4226
+void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
4227
+int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
4228
+void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
4229
+int __init f2fs_init_compress_cache(void);
4230
+void f2fs_destroy_compress_cache(void);
4231
+struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
4232
+void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr);
4233
+void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4234
+						nid_t ino, block_t blkaddr);
4235
+bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
4236
+								block_t blkaddr);
4237
+void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino);
4238
+#define inc_compr_inode_stat(inode)					\
4239
+	do {								\
4240
+		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);		\
4241
+		sbi->compr_new_inode++;					\
4242
+	} while (0)
4243
+#define add_compr_block_stat(inode, blocks)				\
4244
+	do {								\
4245
+		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);		\
4246
+		int diff = F2FS_I(inode)->i_cluster_size - blocks;	\
4247
+		sbi->compr_written_block += blocks;			\
4248
+		sbi->compr_saved_block += diff;				\
4249
+	} while (0)
38934250 #else
38944251 static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
38954252 static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
....@@ -3906,42 +4263,83 @@
39064263 }
39074264 static inline int f2fs_init_compress_mempool(void) { return 0; }
39084265 static inline void f2fs_destroy_compress_mempool(void) { }
4266
+static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic,
4267
+				bool in_task) { }
4268
+static inline void f2fs_end_read_compressed_page(struct page *page,
4269
+				bool failed, block_t blkaddr, bool in_task)
4270
+{
4271
+	WARN_ON_ONCE(1);
4272
+}
4273
+static inline void f2fs_put_page_dic(struct page *page, bool in_task)
4274
+{
4275
+	WARN_ON_ONCE(1);
4276
+}
4277
+static inline unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn) { return 0; }
4278
+static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
4279
+static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
4280
+static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
4281
+static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
4282
+static inline int __init f2fs_init_compress_cache(void) { return 0; }
4283
+static inline void f2fs_destroy_compress_cache(void) { }
4284
+static inline void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi,
4285
+				block_t blkaddr) { }
4286
+static inline void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
4287
+				struct page *page, nid_t ino, block_t blkaddr) { }
4288
+static inline bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi,
4289
+				struct page *page, block_t blkaddr) { return false; }
4290
+static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi,
4291
+							nid_t ino) { }
4292
+#define inc_compr_inode_stat(inode)		do { } while (0)
4293
+static inline void f2fs_update_read_extent_tree_range_compressed(
4294
+				struct inode *inode,
4295
+				pgoff_t fofs, block_t blkaddr,
4296
+				unsigned int llen, unsigned int c_len) { }
39094297 #endif
39104298 
3911
-static inline void set_compress_context(struct inode *inode)
4299
+static inline int set_compress_context(struct inode *inode)
39124300 {
4301
+#ifdef CONFIG_F2FS_FS_COMPRESSION
39134302 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
39144303 
39154304 	F2FS_I(inode)->i_compress_algorithm =
39164305 			F2FS_OPTION(sbi).compress_algorithm;
39174306 	F2FS_I(inode)->i_log_cluster_size =
39184307 			F2FS_OPTION(sbi).compress_log_size;
4308
+	F2FS_I(inode)->i_compress_flag =
4309
+			F2FS_OPTION(sbi).compress_chksum ?
4310
+				1 << COMPRESS_CHKSUM : 0;
39194311 	F2FS_I(inode)->i_cluster_size =
39204312 			1 << F2FS_I(inode)->i_log_cluster_size;
4313
+	if (F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 &&
4314
+			F2FS_OPTION(sbi).compress_level)
4315
+		F2FS_I(inode)->i_compress_flag |=
4316
+				F2FS_OPTION(sbi).compress_level <<
4317
+				COMPRESS_LEVEL_OFFSET;
39214318 	F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
39224319 	set_inode_flag(inode, FI_COMPRESSED_FILE);
39234320 	stat_inc_compr_inode(inode);
4321
+	inc_compr_inode_stat(inode);
39244322 	f2fs_mark_inode_dirty_sync(inode, true);
4323
+	return 0;
4324
+#else
4325
+	return -EOPNOTSUPP;
4326
+#endif
39254327 }
39264328 
3927
-static inline u64 f2fs_disable_compressed_file(struct inode *inode)
4329
+static inline bool f2fs_disable_compressed_file(struct inode *inode)
39284330 {
39294331 	struct f2fs_inode_info *fi = F2FS_I(inode);
39304332 
39314333 	if (!f2fs_compressed_file(inode))
3932
-		return 0;
3933
-	if (S_ISREG(inode->i_mode)) {
3934
-		if (get_dirty_pages(inode))
3935
-			return 1;
3936
-		if (fi->i_compr_blocks)
3937
-			return fi->i_compr_blocks;
3938
-	}
4334
+		return true;
4335
+	if (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))
4336
+		return false;
39394337 
39404338 	fi->i_flags &= ~F2FS_COMPR_FL;
39414339 	stat_dec_compr_inode(inode);
39424340 	clear_inode_flag(inode, FI_COMPRESSED_FILE);
39434341 	f2fs_mark_inode_dirty_sync(inode, true);
3944
-	return 0;
4342
+	return true;
39454343 }
39464344 
39474345 #define F2FS_FEATURE_FUNCS(name, flagname) \
....@@ -3963,6 +4361,7 @@
39634361 F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM);
39644362 F2FS_FEATURE_FUNCS(casefold, CASEFOLD);
39654363 F2FS_FEATURE_FUNCS(compression, COMPRESSION);
4364
+F2FS_FEATURE_FUNCS(readonly, RO);
39664365 
39674366 #ifdef CONFIG_BLK_DEV_ZONED
39684367 static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi,
....@@ -4022,20 +4421,9 @@
40224421 	return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
40234422 }
40244423 
4025
-static inline bool f2fs_may_encrypt(struct inode *dir, struct inode *inode)
4424
+static inline bool f2fs_low_mem_mode(struct f2fs_sb_info *sbi)
40264425 {
4027
-#ifdef CONFIG_FS_ENCRYPTION
4028
-	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
4029
-	umode_t mode = inode->i_mode;
4030
-
4031
-	/*
4032
-	 * If the directory encrypted or dummy encryption enabled,
4033
-	 * then we should encrypt the inode.
4034
-	 */
4035
-	if (IS_ENCRYPTED(dir) || DUMMY_ENCRYPTION_ENABLED(sbi))
4036
-		return (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode));
4037
-#endif
4038
-	return false;
4426
+	return F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW;
40394427 }
40404428 
40414429 static inline bool f2fs_may_compress(struct inode *inode)
....@@ -4051,16 +4439,17 @@
40514439 						u64 blocks, bool add)
40524440 {
40534441 	int diff = F2FS_I(inode)->i_cluster_size - blocks;
4442
+	struct f2fs_inode_info *fi = F2FS_I(inode);
40544443 
40554444 	/* don't update i_compr_blocks if saved blocks were released */
4056
-	if (!add && !F2FS_I(inode)->i_compr_blocks)
4445
+	if (!add && !atomic_read(&fi->i_compr_blocks))
40574446 		return;
40584447 
40594448 	if (add) {
4060
-		F2FS_I(inode)->i_compr_blocks += diff;
4449
+		atomic_add(diff, &fi->i_compr_blocks);
40614450 		stat_add_compr_blocks(inode, diff);
40624451 	} else {
4063
-		F2FS_I(inode)->i_compr_blocks -= diff;
4452
+		atomic_sub(diff, &fi->i_compr_blocks);
40644453 		stat_sub_compr_blocks(inode, diff);
40654454 	}
40664455 	f2fs_mark_inode_dirty_sync(inode, true);
....@@ -4113,13 +4502,18 @@
41134502 		if (F2FS_IO_ALIGNED(sbi))
41144503 			return true;
41154504 	}
4116
-	if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED) &&
4117
-					!IS_SWAPFILE(inode))
4505
+	if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED))
41184506 		return true;
41194507 
41204508 	return false;
41214509 }
41224510 
4511
+static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
4512
+{
4513
+	return fsverity_active(inode) &&
4514
+	       idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
4515
+}
4516
+
41234517 #ifdef CONFIG_F2FS_FAULT_INJECTION
41244518 extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate,
41254519 							unsigned int type);