forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376 
 kernel/fs/xfs/xfs_log_cil.c
....@@ -10,10 +10,7 @@
1010 #include "xfs_shared.h"
1111 #include "xfs_trans_resv.h"
1212 #include "xfs_mount.h"
13
-#include "xfs_error.h"
14
-#include "xfs_alloc.h"
1513 #include "xfs_extent_busy.h"
16
-#include "xfs_discard.h"
1714 #include "xfs_trans.h"
1815 #include "xfs_trans_priv.h"
1916 #include "xfs_log.h"
....@@ -40,8 +37,7 @@
4037 {
4138 	struct xlog_ticket *tic;
4239 
43
-	tic = xlog_ticket_alloc(log, 0, 1, XFS_TRANSACTION, 0,
44
-				KM_SLEEP|KM_NOFS);
40
+	tic = xlog_ticket_alloc(log, 0, 1, XFS_TRANSACTION, 0);
4541 
4642 	/*
4743 	 * set the current reservation to zero so we know to steal the basic
....@@ -182,14 +178,14 @@
182178 
183179 			/*
184180 			 * We free and allocate here as a realloc would copy
185
-			 * unecessary data. We don't use kmem_zalloc() for the
181
+			 * unnecessary data. We don't use kmem_zalloc() for the
186182 			 * same reason - we don't need to zero the data area in
187183 			 * the buffer, only the log vector header and the iovec
188184 			 * storage.
189185 			 */
190186 			kmem_free(lip->li_lv_shadow);
191187 
192
-			lv = kmem_alloc_large(buf_size, KM_SLEEP | KM_NOFS);
188
+			lv = kmem_alloc_large(buf_size, KM_NOFS);
193189 			memset(lv, 0, xlog_cil_iovec_space(niovecs));
194190 
195191 			lv->lv_item = lip;
....@@ -243,10 +239,11 @@
243239 	 * this CIL context and so we need to pin it. If we are replacing the
244240 	 * old_lv, then remove the space it accounts for and make it the shadow
245241 	 * buffer for later freeing. In both cases we are now switching to the
246
-	 * shadow buffer, so update the the pointer to it appropriately.
242
+	 * shadow buffer, so update the pointer to it appropriately.
247243 	 */
248244 	if (!old_lv) {
249
-		lv->lv_item->li_ops->iop_pin(lv->lv_item);
245
+		if (lv->lv_item->li_ops->iop_pin)
246
+			lv->lv_item->li_ops->iop_pin(lv->lv_item);
250247 		lv->lv_item->li_lv_shadow = NULL;
251248 	} else if (old_lv != lv) {
252249 		ASSERT(lv->lv_buf_len != XFS_LOG_VEC_ORDERED);
....@@ -576,11 +573,23 @@
576573  */
577574 static void
578575 xlog_cil_committed(
579
-	void	*args,
580
-	int	abort)
576
+	struct xfs_cil_ctx	*ctx)
581577 {
582
-	struct xfs_cil_ctx	*ctx = args;
583578 	struct xfs_mount	*mp = ctx->cil->xc_log->l_mp;
579
+	bool			abort = XLOG_FORCED_SHUTDOWN(ctx->cil->xc_log);
580
+
581
+	/*
582
+	 * If the I/O failed, we're aborting the commit and already shutdown.
583
+	 * Wake any commit waiters before aborting the log items so we don't
584
+	 * block async log pushers on callbacks. Async log pushers explicitly do
585
+	 * not wait on log force completion because they may be holding locks
586
+	 * required to unpin items.
587
+	 */
588
+	if (abort) {
589
+		spin_lock(&ctx->cil->xc_push_lock);
590
+		wake_up_all(&ctx->cil->xc_commit_wait);
591
+		spin_unlock(&ctx->cil->xc_push_lock);
592
+	}
584593 
585594 	xfs_trans_committed_bulk(ctx->cil->xc_log->l_ailp, ctx->lv_chain,
586595 					ctx->start_lsn, abort);
....@@ -589,15 +598,7 @@
589598 	xfs_extent_busy_clear(mp, &ctx->busy_extents,
590599 			     (mp->m_flags & XFS_MOUNT_DISCARD) && !abort);
591600 
592
-	/*
593
-	 * If we are aborting the commit, wake up anyone waiting on the
594
-	 * committing list.  If we don't, then a shutdown we can leave processes
595
-	 * waiting in xlog_cil_force_lsn() waiting on a sequence commit that
596
-	 * will never happen because we aborted it.
597
-	 */
598601 	spin_lock(&ctx->cil->xc_push_lock);
599
-	if (abort)
600
-		wake_up_all(&ctx->cil->xc_commit_wait);
601602 	list_del(&ctx->committing);
602603 	spin_unlock(&ctx->cil->xc_push_lock);
603604 
....@@ -609,25 +610,40 @@
609610 		kmem_free(ctx);
610611 }
611612 
613
+void
614
+xlog_cil_process_committed(
615
+	struct list_head	*list)
616
+{
617
+	struct xfs_cil_ctx	*ctx;
618
+
619
+	while ((ctx = list_first_entry_or_null(list,
620
+			struct xfs_cil_ctx, iclog_entry))) {
621
+		list_del(&ctx->iclog_entry);
622
+		xlog_cil_committed(ctx);
623
+	}
624
+}
625
+
612626 /*
613
- * Push the Committed Item List to the log. If @push_seq flag is zero, then it
614
- * is a background flush and so we can chose to ignore it. Otherwise, if the
615
- * current sequence is the same as @push_seq we need to do a flush. If
616
- * @push_seq is less than the current sequence, then it has already been
627
+ * Push the Committed Item List to the log.
628
+ *
629
+ * If the current sequence is the same as xc_push_seq we need to do a flush. If
630
+ * xc_push_seq is less than the current sequence, then it has already been
617631  * flushed and we don't need to do anything - the caller will wait for it to
618632  * complete if necessary.
619633  *
620
- * @push_seq is a value rather than a flag because that allows us to do an
621
- * unlocked check of the sequence number for a match. Hence we can allows log
622
- * forces to run racily and not issue pushes for the same sequence twice. If we
623
- * get a race between multiple pushes for the same sequence they will block on
624
- * the first one and then abort, hence avoiding needless pushes.
634
+ * xc_push_seq is checked unlocked against the sequence number for a match.
635
+ * Hence we can allow log forces to run racily and not issue pushes for the
636
+ * same sequence twice.  If we get a race between multiple pushes for the same
637
+ * sequence they will block on the first one and then abort, hence avoiding
638
+ * needless pushes.
625639  */
626
-STATIC int
627
-xlog_cil_push(
628
-	struct xlog		*log)
640
+static void
641
+xlog_cil_push_work(
642
+	struct work_struct	*work)
629643 {
630
-	struct xfs_cil		*cil = log->l_cilp;
644
+	struct xfs_cil		*cil =
645
+		container_of(work, struct xfs_cil, xc_push_work);
646
+	struct xlog		*log = cil->xc_log;
631647 	struct xfs_log_vec	*lv;
632648 	struct xfs_cil_ctx	*ctx;
633649 	struct xfs_cil_ctx	*new_ctx;
....@@ -641,10 +657,7 @@
641657 	xfs_lsn_t		commit_lsn;
642658 	xfs_lsn_t		push_seq;
643659 
644
-	if (!cil)
645
-		return 0;
646
-
647
-	new_ctx = kmem_zalloc(sizeof(*new_ctx), KM_SLEEP|KM_NOFS);
660
+	new_ctx = kmem_zalloc(sizeof(*new_ctx), KM_NOFS);
648661 	new_ctx->ticket = xlog_cil_ticket_alloc(log);
649662 
650663 	down_write(&cil->xc_ctx_lock);
....@@ -653,6 +666,17 @@
653666 	spin_lock(&cil->xc_push_lock);
654667 	push_seq = cil->xc_push_seq;
655668 	ASSERT(push_seq <= ctx->sequence);
669
+
670
+	/*
671
+	 * As we are about to switch to a new, empty CIL context, we no longer
672
+	 * need to throttle tasks on CIL space overruns. Wake any waiters that
673
+	 * the hard push throttle may have caught so they can start committing
674
+	 * to the new context. The ctx->xc_push_lock provides the serialisation
675
+	 * necessary for safely using the lockless waitqueue_active() check in
676
+	 * this context.
677
+	 */
678
+	if (waitqueue_active(&cil->xc_push_wait))
679
+		wake_up_all(&cil->xc_push_wait);
656680 
657681 	/*
658682 	 * Check if we've anything to push. If there is nothing, then we don't
....@@ -666,7 +690,7 @@
666690 	}
667691 
668692 
669
-	/* check for a previously pushed seqeunce */
693
+	/* check for a previously pushed sequence */
670694 	if (push_seq < cil->xc_ctx->sequence) {
671695 		spin_unlock(&cil->xc_push_lock);
672696 		goto out_skip;
....@@ -724,7 +748,7 @@
724748 
725749 	/*
726750 	 * initialise the new context and attach it to the CIL. Then attach
727
-	 * the current context to the CIL committing lsit so it can be found
751
+	 * the current context to the CIL committing list so it can be found
728752 	 * during log forces to extract the commit lsn of the sequence that
729753 	 * needs to be forced.
730754 	 */
....@@ -753,7 +777,7 @@
753777 	 * that higher sequences will wait for us to write out a commit record
754778 	 * before they do.
755779 	 *
756
-	 * xfs_log_force_lsn requires us to mirror the new sequence into the cil
780
+	 * xfs_log_force_seq requires us to mirror the new sequence into the cil
757781 	 * structure atomically with the addition of this sequence to the
758782 	 * committing list. This also ensures that we can do unlocked checks
759783 	 * against the current sequence in log forces without risking
....@@ -787,7 +811,7 @@
787811 	lvhdr.lv_iovecp = &lhdr;
788812 	lvhdr.lv_next = ctx->lv_chain;
789813 
790
-	error = xlog_write(log, &lvhdr, tic, &ctx->start_lsn, NULL, 0);
814
+	error = xlog_write(log, &lvhdr, tic, &ctx->start_lsn, NULL, 0, true);
791815 	if (error)
792816 		goto out_abort_free_ticket;
793817 
....@@ -825,17 +849,21 @@
825849 	}
826850 	spin_unlock(&cil->xc_push_lock);
827851 
828
-	/* xfs_log_done always frees the ticket on error. */
829
-	commit_lsn = xfs_log_done(log->l_mp, tic, &commit_iclog, false);
830
-	if (commit_lsn == -1)
831
-		goto out_abort;
832
-
833
-	/* attach all the transactions w/ busy extents to iclog */
834
-	ctx->log_cb.cb_func = xlog_cil_committed;
835
-	ctx->log_cb.cb_arg = ctx;
836
-	error = xfs_log_notify(commit_iclog, &ctx->log_cb);
852
+	error = xlog_commit_record(log, tic, &commit_iclog, &commit_lsn);
837853 	if (error)
854
+		goto out_abort_free_ticket;
855
+
856
+	xfs_log_ticket_ungrant(log, tic);
857
+
858
+	spin_lock(&commit_iclog->ic_callback_lock);
859
+	if (commit_iclog->ic_state == XLOG_STATE_IOERROR) {
860
+		spin_unlock(&commit_iclog->ic_callback_lock);
838861 		goto out_abort;
862
+	}
863
+	ASSERT_ALWAYS(commit_iclog->ic_state == XLOG_STATE_ACTIVE ||
864
+		      commit_iclog->ic_state == XLOG_STATE_WANT_SYNC);
865
+	list_add_tail(&ctx->iclog_entry, &commit_iclog->ic_callbacks);
866
+	spin_unlock(&commit_iclog->ic_callback_lock);
839867 
840868 	/*
841869 	 * now the checkpoint commit is complete and we've attached the
....@@ -848,28 +876,20 @@
848876 	spin_unlock(&cil->xc_push_lock);
849877 
850878 	/* release the hounds! */
851
-	return xfs_log_release_iclog(log->l_mp, commit_iclog);
879
+	xfs_log_release_iclog(commit_iclog);
880
+	return;
852881 
853882 out_skip:
854883 	up_write(&cil->xc_ctx_lock);
855884 	xfs_log_ticket_put(new_ctx->ticket);
856885 	kmem_free(new_ctx);
857
-	return 0;
886
+	return;
858887 
859888 out_abort_free_ticket:
860
-	xfs_log_ticket_put(tic);
889
+	xfs_log_ticket_ungrant(log, tic);
861890 out_abort:
862
-	xlog_cil_committed(ctx, XFS_LI_ABORTED);
863
-	return -EIO;
864
-}
865
-
866
-static void
867
-xlog_cil_push_work(
868
-	struct work_struct	*work)
869
-{
870
-	struct xfs_cil		*cil = container_of(work, struct xfs_cil,
871
-							xc_push_work);
872
-	xlog_cil_push(cil->xc_log);
891
+	ASSERT(XLOG_FORCED_SHUTDOWN(log));
892
+	xlog_cil_committed(ctx);
873893 }
874894 
875895 /*
....@@ -881,7 +901,7 @@
881901  */
882902 static void
883903 xlog_cil_push_background(
884
-	struct xlog	*log)
904
+	struct xlog	*log) __releases(cil->xc_ctx_lock)
885905 {
886906 	struct xfs_cil	*cil = log->l_cilp;
887907 
....@@ -892,17 +912,46 @@
892912 	ASSERT(!list_empty(&cil->xc_cil));
893913 
894914 	/*
895
-	 * don't do a background push if we haven't used up all the
915
+	 * Don't do a background push if we haven't used up all the
896916 	 * space available yet.
897917 	 */
898
-	if (cil->xc_ctx->space_used < XLOG_CIL_SPACE_LIMIT(log))
918
+	if (cil->xc_ctx->space_used < XLOG_CIL_SPACE_LIMIT(log)) {
919
+		up_read(&cil->xc_ctx_lock);
899920 		return;
921
+	}
900922 
901923 	spin_lock(&cil->xc_push_lock);
902924 	if (cil->xc_push_seq < cil->xc_current_sequence) {
903925 		cil->xc_push_seq = cil->xc_current_sequence;
904926 		queue_work(log->l_mp->m_cil_workqueue, &cil->xc_push_work);
905927 	}
928
+
929
+	/*
930
+	 * Drop the context lock now, we can't hold that if we need to sleep
931
+	 * because we are over the blocking threshold. The push_lock is still
932
+	 * held, so blocking threshold sleep/wakeup is still correctly
933
+	 * serialised here.
934
+	 */
935
+	up_read(&cil->xc_ctx_lock);
936
+
937
+	/*
938
+	 * If we are well over the space limit, throttle the work that is being
939
+	 * done until the push work on this context has begun. Enforce the hard
940
+	 * throttle on all transaction commits once it has been activated, even
941
+	 * if the committing transactions have resulted in the space usage
942
+	 * dipping back down under the hard limit.
943
+	 *
944
+	 * The ctx->xc_push_lock provides the serialisation necessary for safely
945
+	 * using the lockless waitqueue_active() check in this context.
946
+	 */
947
+	if (cil->xc_ctx->space_used >= XLOG_CIL_BLOCKING_SPACE_LIMIT(log) ||
948
+	    waitqueue_active(&cil->xc_push_wait)) {
949
+		trace_xfs_log_cil_wait(log, cil->xc_ctx->ticket);
950
+		ASSERT(cil->xc_ctx->space_used < log->l_logsize);
951
+		xlog_wait(&cil->xc_push_wait, &cil->xc_push_lock);
952
+		return;
953
+	}
954
+
906955 	spin_unlock(&cil->xc_push_lock);
907956 
908957 }
....@@ -971,15 +1020,14 @@
9711020  * allowed again.
9721021  */
9731022 void
974
-xfs_log_commit_cil(
975
-	struct xfs_mount	*mp,
1023
+xlog_cil_commit(
1024
+	struct xlog		*log,
9761025 	struct xfs_trans	*tp,
977
-	xfs_lsn_t		*commit_lsn,
1026
+	xfs_csn_t		*commit_seq,
9781027 	bool			regrant)
9791028 {
980
-	struct xlog		*log = mp->m_log;
9811029 	struct xfs_cil		*cil = log->l_cilp;
982
-	xfs_lsn_t		xc_commit_lsn;
1030
+	struct xfs_log_item	*lip, *next;
9831031 
9841032 	/*
9851033 	 * Do all necessary memory allocation before we lock the CIL.
....@@ -993,17 +1041,16 @@
9931041 
9941042 	xlog_cil_insert_items(log, tp);
9951043 
996
-	xc_commit_lsn = cil->xc_ctx->sequence;
997
-	if (commit_lsn)
998
-		*commit_lsn = xc_commit_lsn;
999
-
1000
-	xfs_log_done(mp, tp->t_ticket, NULL, regrant);
1044
+	if (regrant && !XLOG_FORCED_SHUTDOWN(log))
1045
+		xfs_log_ticket_regrant(log, tp->t_ticket);
1046
+	else
1047
+		xfs_log_ticket_ungrant(log, tp->t_ticket);
10011048 	tp->t_ticket = NULL;
10021049 	xfs_trans_unreserve_and_mod_sb(tp);
10031050 
10041051 	/*
10051052 	 * Once all the items of the transaction have been copied to the CIL,
1006
-	 * the items can be unlocked and freed.
1053
+	 * the items can be unlocked and possibly freed.
10071054 	 *
10081055 	 * This needs to be done before we drop the CIL context lock because we
10091056 	 * have to update state in the log items and unlock them before they go
....@@ -1012,11 +1059,17 @@
10121059 	 * the log items. This affects (at least) processing of stale buffers,
10131060 	 * inodes and EFIs.
10141061 	 */
1015
-	xfs_trans_free_items(tp, xc_commit_lsn, false);
1062
+	trace_xfs_trans_commit_items(tp, _RET_IP_);
1063
+	list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
1064
+		xfs_trans_del_item(lip);
1065
+		if (lip->li_ops->iop_committing)
1066
+			lip->li_ops->iop_committing(lip, cil->xc_ctx->sequence);
1067
+	}
1068
+	if (commit_seq)
1069
+		*commit_seq = cil->xc_ctx->sequence;
10161070 
1071
+	/* xlog_cil_push_background() releases cil->xc_ctx_lock */
10171072 	xlog_cil_push_background(log);
1018
-
1019
-	up_read(&cil->xc_ctx_lock);
10201073 }
10211074 
10221075 /*
....@@ -1030,9 +1083,9 @@
10301083  * iclog flush is necessary following this call.
10311084  */
10321085 xfs_lsn_t
1033
-xlog_cil_force_lsn(
1086
+xlog_cil_force_seq(
10341087 	struct xlog	*log,
1035
-	xfs_lsn_t	sequence)
1088
+	xfs_csn_t	sequence)
10361089 {
10371090 	struct xfs_cil		*cil = log->l_cilp;
10381091 	struct xfs_cil_ctx	*ctx;
....@@ -1126,23 +1179,19 @@
11261179  */
11271180 bool
11281181 xfs_log_item_in_current_chkpt(
1129
-	struct xfs_log_item *lip)
1182
+	struct xfs_log_item	*lip)
11301183 {
1131
-	struct xfs_cil_ctx *ctx;
1184
+	struct xfs_cil		*cil = lip->li_mountp->m_log->l_cilp;
11321185 
11331186 	if (list_empty(&lip->li_cil))
11341187 		return false;
1135
-
1136
-	ctx = lip->li_mountp->m_log->l_cilp->xc_ctx;
11371188 
11381189 	/*
11391190 	 * li_seq is written on the first commit of a log item to record the
11401191 	 * first checkpoint it is written to. Hence if it is different to the
11411192 	 * current sequence, we're in a new checkpoint.
11421193 	 */
1143
-	if (XFS_LSN_CMP(lip->li_seq, ctx->sequence) != 0)
1144
-		return false;
1145
-	return true;
1194
+	return lip->li_seq == READ_ONCE(cil->xc_current_sequence);
11461195 }
11471196 
11481197 /*
....@@ -1155,11 +1204,11 @@
11551204 	struct xfs_cil	*cil;
11561205 	struct xfs_cil_ctx *ctx;
11571206 
1158
-	cil = kmem_zalloc(sizeof(*cil), KM_SLEEP|KM_MAYFAIL);
1207
+	cil = kmem_zalloc(sizeof(*cil), KM_MAYFAIL);
11591208 	if (!cil)
11601209 		return -ENOMEM;
11611210 
1162
-	ctx = kmem_zalloc(sizeof(*ctx), KM_SLEEP|KM_MAYFAIL);
1211
+	ctx = kmem_zalloc(sizeof(*ctx), KM_MAYFAIL);
11631212 	if (!ctx) {
11641213 		kmem_free(cil);
11651214 		return -ENOMEM;
....@@ -1170,6 +1219,7 @@
11701219 	INIT_LIST_HEAD(&cil->xc_committing);
11711220 	spin_lock_init(&cil->xc_cil_lock);
11721221 	spin_lock_init(&cil->xc_push_lock);
1222
+	init_waitqueue_head(&cil->xc_push_wait);
11731223 	init_rwsem(&cil->xc_ctx_lock);
11741224 	init_waitqueue_head(&cil->xc_commit_wait);
11751225