forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c
....@@ -7,7 +7,8 @@
77 #include <linux/gfp.h>
88 #include <linux/kernel.h>
99 #include <linux/list.h>
10
-#include <linux/rhashtable.h>
10
+#include <linux/mutex.h>
11
+#include <linux/objagg.h>
1112 #include <linux/rtnetlink.h>
1213 #include <linux/slab.h>
1314 
....@@ -24,11 +25,14 @@
2425 	unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
2526 	struct gen_pool *erp_tables;
2627 	struct mlxsw_sp *mlxsw_sp;
28
+	struct mlxsw_sp_acl_bf *bf;
2729 	unsigned int num_erp_banks;
2830 };
2931 
3032 struct mlxsw_sp_acl_erp_key {
3133 	char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
34
+#define __MASK_LEN 0x38
35
+#define __MASK_IDX(i) (__MASK_LEN - (i) - 1)
3236 	bool ctcam;
3337 };
3438 
....@@ -36,10 +40,8 @@
3640 	struct mlxsw_sp_acl_erp_key key;
3741 	u8 id;
3842 	u8 index;
39
-	refcount_t refcnt;
4043 	DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
4144 	struct list_head list;
42
-	struct rhash_head ht_node;
4345 	struct mlxsw_sp_acl_erp_table *erp_table;
4446 };
4547 
....@@ -53,7 +55,6 @@
5355 	DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
5456 	DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
5557 	struct list_head atcam_erps_list;
56
-	struct rhashtable erp_ht;
5758 	struct mlxsw_sp_acl_erp_core *erp_core;
5859 	struct mlxsw_sp_acl_atcam_region *aregion;
5960 	const struct mlxsw_sp_acl_erp_table_ops *ops;
....@@ -61,12 +62,9 @@
6162 	unsigned int num_atcam_erps;
6263 	unsigned int num_max_atcam_erps;
6364 	unsigned int num_ctcam_erps;
64
-};
65
-
66
-static const struct rhashtable_params mlxsw_sp_acl_erp_ht_params = {
67
-	.key_len = sizeof(struct mlxsw_sp_acl_erp_key),
68
-	.key_offset = offsetof(struct mlxsw_sp_acl_erp, key),
69
-	.head_offset = offsetof(struct mlxsw_sp_acl_erp, ht_node),
65
+	unsigned int num_deltas;
66
+	struct objagg *objagg;
67
+	struct mutex objagg_lock; /* guards objagg manipulation */
7068 };
7169 
7270 struct mlxsw_sp_acl_erp_table_ops {
....@@ -119,14 +117,17 @@
119117 	.erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
120118 };
121119 
122
-bool mlxsw_sp_acl_erp_is_ctcam_erp(const struct mlxsw_sp_acl_erp *erp)
120
+static bool
121
+mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)
123122 {
124
-	return erp->key.ctcam;
123
+	return erp_table->ops != &erp_single_mask_ops &&
124
+	       erp_table->ops != &erp_no_mask_ops;
125125 }
126126 
127
-u8 mlxsw_sp_acl_erp_id(const struct mlxsw_sp_acl_erp *erp)
127
+static unsigned int
128
+mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)
128129 {
129
-	return erp->id;
130
+	return erp->index % erp->erp_table->erp_core->num_erp_banks;
130131 }
131132 
132133 static unsigned int
....@@ -194,12 +195,15 @@
194195 
195196 static int
196197 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
197
-				 const struct mlxsw_sp_acl_erp *erp)
198
+				 struct mlxsw_sp_acl_erp_key *key)
198199 {
200
+	DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
199201 	unsigned long bit;
200202 	int err;
201203 
202
-	for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
204
+	bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
205
+			  MLXSW_SP_ACL_TCAM_MASK_LEN);
206
+	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
203207 		mlxsw_sp_acl_erp_master_mask_bit_set(bit,
204208 						     &erp_table->master_mask);
205209 
....@@ -210,7 +214,7 @@
210214 	return 0;
211215 
212216 err_master_mask_update:
213
-	for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
217
+	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
214218 		mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
215219 						       &erp_table->master_mask);
216220 	return err;
....@@ -218,12 +222,15 @@
218222 
219223 static int
220224 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
221
-				   const struct mlxsw_sp_acl_erp *erp)
225
+				   struct mlxsw_sp_acl_erp_key *key)
222226 {
227
+	DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
223228 	unsigned long bit;
224229 	int err;
225230 
226
-	for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
231
+	bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
232
+			  MLXSW_SP_ACL_TCAM_MASK_LEN);
233
+	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
227234 		mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
228235 						       &erp_table->master_mask);
229236 
....@@ -234,7 +241,7 @@
234241 	return 0;
235242 
236243 err_master_mask_update:
237
-	for_each_set_bit(bit, erp->mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
244
+	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
238245 		mlxsw_sp_acl_erp_master_mask_bit_set(bit,
239246 						     &erp_table->master_mask);
240247 	return err;
....@@ -256,26 +263,16 @@
256263 		goto err_erp_id_get;
257264 
258265 	memcpy(&erp->key, key, sizeof(*key));
259
-	bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
260
-			  MLXSW_SP_ACL_TCAM_MASK_LEN);
261266 	list_add(&erp->list, &erp_table->atcam_erps_list);
262
-	refcount_set(&erp->refcnt, 1);
263267 	erp_table->num_atcam_erps++;
264268 	erp->erp_table = erp_table;
265269 
266
-	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, erp);
270
+	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
267271 	if (err)
268272 		goto err_master_mask_set;
269273 
270
-	err = rhashtable_insert_fast(&erp_table->erp_ht, &erp->ht_node,
271
-				     mlxsw_sp_acl_erp_ht_params);
272
-	if (err)
273
-		goto err_rhashtable_insert;
274
-
275274 	return erp;
276275 
277
-err_rhashtable_insert:
278
-	mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
279276 err_master_mask_set:
280277 	erp_table->num_atcam_erps--;
281278 	list_del(&erp->list);
....@@ -290,9 +287,7 @@
290287 {
291288 	struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
292289 
293
-	rhashtable_remove_fast(&erp_table->erp_ht, &erp->ht_node,
294
-			       mlxsw_sp_acl_erp_ht_params);
295
-	mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
290
+	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
296291 	erp_table->num_atcam_erps--;
297292 	list_del(&erp->list);
298293 	mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
....@@ -525,6 +520,48 @@
525520 }
526521 
527522 static int
523
+mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,
524
+			   struct mlxsw_sp_acl_erp *erp)
525
+{
526
+	struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
527
+	unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
528
+	struct mlxsw_sp_acl_atcam_entry *aentry;
529
+	int err;
530
+
531
+	list_for_each_entry(aentry, &aregion->entries_list, list) {
532
+		err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,
533
+						erp_table->erp_core->bf,
534
+						aregion, erp_bank, aentry);
535
+		if (err)
536
+			goto bf_entry_add_err;
537
+	}
538
+
539
+	return 0;
540
+
541
+bf_entry_add_err:
542
+	list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,
543
+					     list)
544
+		mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
545
+					  erp_table->erp_core->bf,
546
+					  aregion, erp_bank, aentry);
547
+	return err;
548
+}
549
+
550
+static void
551
+mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,
552
+			   struct mlxsw_sp_acl_erp *erp)
553
+{
554
+	struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
555
+	unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
556
+	struct mlxsw_sp_acl_atcam_entry *aentry;
557
+
558
+	list_for_each_entry_reverse(aentry, &aregion->entries_list, list)
559
+		mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
560
+					  erp_table->erp_core->bf,
561
+					  aregion, erp_bank, aentry);
562
+}
563
+
564
+static int
528565 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
529566 {
530567 	struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
....@@ -548,15 +585,23 @@
548585 		goto err_table_master_rp;
549586 	}
550587 
551
-	/* Maintain the same eRP bank for the master RP, so that we
552
-	 * wouldn't need to update the bloom filter
588
+	/* Make sure the master RP is using a valid index, as
589
+	 * only a single eRP row is currently allocated.
553590 	 */
554
-	master_rp->index = master_rp->index % erp_core->num_erp_banks;
591
+	master_rp->index = 0;
555592 	__set_bit(master_rp->index, erp_table->erp_index_bitmap);
556593 
557594 	err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
558595 	if (err)
559596 		goto err_table_master_rp_add;
597
+
598
+	/* Update Bloom filter before enabling eRP table, as rules
599
+	 * on the master RP were not set to Bloom filter up to this
600
+	 * point.
601
+	 */
602
+	err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);
603
+	if (err)
604
+		goto err_table_bf_add;
560605 
561606 	err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
562607 	if (err)
....@@ -565,6 +610,8 @@
565610 	return 0;
566611 
567612 err_table_enable:
613
+	mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
614
+err_table_bf_add:
568615 	mlxsw_sp_acl_erp_table_erp_del(master_rp);
569616 err_table_master_rp_add:
570617 	__clear_bit(master_rp->index, erp_table->erp_index_bitmap);
....@@ -585,6 +632,7 @@
585632 	master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
586633 	if (!master_rp)
587634 		return;
635
+	mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
588636 	mlxsw_sp_acl_erp_table_erp_del(master_rp);
589637 	__clear_bit(master_rp->index, erp_table->erp_index_bitmap);
590638 	mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
....@@ -647,9 +695,55 @@
647695 	mlxsw_sp_acl_erp_table_enable(erp_table, false);
648696 }
649697 
650
-static void
651
-mlxsw_sp_acl_erp_ctcam_table_ops_set(struct mlxsw_sp_acl_erp_table *erp_table)
698
+static int
699
+__mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,
700
+				   unsigned int *inc_num)
652701 {
702
+	int err;
703
+
704
+	/* If there are C-TCAM eRP or deltas in use we need to transition
705
+	 * the region to use eRP table, if it is not already done
706
+	 */
707
+	if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {
708
+		err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
709
+		if (err)
710
+			return err;
711
+	}
712
+
713
+	/* When C-TCAM or deltas are used, the eRP table must be used */
714
+	if (erp_table->ops != &erp_multiple_masks_ops)
715
+		erp_table->ops = &erp_multiple_masks_ops;
716
+
717
+	(*inc_num)++;
718
+
719
+	return 0;
720
+}
721
+
722
+static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)
723
+{
724
+	return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
725
+						  &erp_table->num_ctcam_erps);
726
+}
727
+
728
+static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)
729
+{
730
+	return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
731
+						  &erp_table->num_deltas);
732
+}
733
+
734
+static void
735
+__mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,
736
+				   unsigned int *dec_num)
737
+{
738
+	(*dec_num)--;
739
+
740
+	/* If there are no C-TCAM eRP or deltas in use, the state we
741
+	 * transition to depends on the number of A-TCAM eRPs currently
742
+	 * in use.
743
+	 */
744
+	if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)
745
+		return;
746
+
653747 	switch (erp_table->num_atcam_erps) {
654748 	case 2:
655749 		/* Keep using the eRP table, but correctly set the
....@@ -683,9 +777,21 @@
683777 	}
684778 }
685779 
780
+static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)
781
+{
782
+	__mlxsw_sp_acl_erp_table_other_dec(erp_table,
783
+					   &erp_table->num_ctcam_erps);
784
+}
785
+
786
+static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)
787
+{
788
+	__mlxsw_sp_acl_erp_table_other_dec(erp_table,
789
+					   &erp_table->num_deltas);
790
+}
791
+
686792 static struct mlxsw_sp_acl_erp *
687
-__mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
688
-				     struct mlxsw_sp_acl_erp_key *key)
793
+mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
794
+				   struct mlxsw_sp_acl_erp_key *key)
689795 {
690796 	struct mlxsw_sp_acl_erp *erp;
691797 	int err;
....@@ -697,67 +803,29 @@
697803 	memcpy(&erp->key, key, sizeof(*key));
698804 	bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
699805 			  MLXSW_SP_ACL_TCAM_MASK_LEN);
700
-	refcount_set(&erp->refcnt, 1);
701
-	erp_table->num_ctcam_erps++;
806
+
807
+	err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);
808
+	if (err)
809
+		goto err_erp_ctcam_inc;
810
+
702811 	erp->erp_table = erp_table;
703812 
704
-	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, erp);
813
+	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
705814 	if (err)
706815 		goto err_master_mask_set;
707
-
708
-	err = rhashtable_insert_fast(&erp_table->erp_ht, &erp->ht_node,
709
-				     mlxsw_sp_acl_erp_ht_params);
710
-	if (err)
711
-		goto err_rhashtable_insert;
712816 
713817 	err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
714818 	if (err)
715819 		goto err_erp_region_ctcam_enable;
716820 
717
-	/* When C-TCAM is used, the eRP table must be used */
718
-	erp_table->ops = &erp_multiple_masks_ops;
719
-
720821 	return erp;
721822 
722823 err_erp_region_ctcam_enable:
723
-	rhashtable_remove_fast(&erp_table->erp_ht, &erp->ht_node,
724
-			       mlxsw_sp_acl_erp_ht_params);
725
-err_rhashtable_insert:
726
-	mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
824
+	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
727825 err_master_mask_set:
728
-	erp_table->num_ctcam_erps--;
826
+	mlxsw_sp_acl_erp_ctcam_dec(erp_table);
827
+err_erp_ctcam_inc:
729828 	kfree(erp);
730
-	return ERR_PTR(err);
731
-}
732
-
733
-static struct mlxsw_sp_acl_erp *
734
-mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
735
-				   struct mlxsw_sp_acl_erp_key *key)
736
-{
737
-	struct mlxsw_sp_acl_erp *erp;
738
-	int err;
739
-
740
-	/* There is a special situation where we need to spill rules
741
-	 * into the C-TCAM, yet the region is still using a master
742
-	 * mask and thus not performing a lookup in the C-TCAM. This
743
-	 * can happen when two rules that only differ in priority - and
744
-	 * thus sharing the same key - are programmed. In this case
745
-	 * we transition the region to use an eRP table
746
-	 */
747
-	err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
748
-	if (err)
749
-		return ERR_PTR(err);
750
-
751
-	erp = __mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
752
-	if (IS_ERR(erp)) {
753
-		err = PTR_ERR(erp);
754
-		goto err_erp_create;
755
-	}
756
-
757
-	return erp;
758
-
759
-err_erp_create:
760
-	mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
761829 	return ERR_PTR(err);
762830 }
763831 
....@@ -767,19 +835,9 @@
767835 	struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
768836 
769837 	mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
770
-	rhashtable_remove_fast(&erp_table->erp_ht, &erp->ht_node,
771
-			       mlxsw_sp_acl_erp_ht_params);
772
-	mlxsw_sp_acl_erp_master_mask_clear(erp_table, erp);
773
-	erp_table->num_ctcam_erps--;
838
+	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
839
+	mlxsw_sp_acl_erp_ctcam_dec(erp_table);
774840 	kfree(erp);
775
-
776
-	/* Once the last C-TCAM eRP was destroyed, the state we
777
-	 * transition to depends on the number of A-TCAM eRPs currently
778
-	 * in use
779
-	 */
780
-	if (erp_table->num_ctcam_erps > 0)
781
-		return;
782
-	mlxsw_sp_acl_erp_ctcam_table_ops_set(erp_table);
783841 }
784842 
785843 static struct mlxsw_sp_acl_erp *
....@@ -790,7 +848,7 @@
790848 	int err;
791849 
792850 	if (key->ctcam)
793
-		return __mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
851
+		return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
794852 
795853 	/* Expand the eRP table for the new eRP, if needed */
796854 	err = mlxsw_sp_acl_erp_table_expand(erp_table);
....@@ -838,7 +896,8 @@
838896 	mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
839897 	mlxsw_sp_acl_erp_generic_destroy(erp);
840898 
841
-	if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0)
899
+	if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&
900
+	    erp_table->num_deltas == 0)
842901 		erp_table->ops = &erp_two_masks_ops;
843902 }
844903 
....@@ -940,46 +999,323 @@
940999 	WARN_ON(1);
9411000 }
9421001 
943
-struct mlxsw_sp_acl_erp *
944
-mlxsw_sp_acl_erp_get(struct mlxsw_sp_acl_atcam_region *aregion,
945
-		     const char *mask, bool ctcam)
1002
+struct mlxsw_sp_acl_erp_mask *
1003
+mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,
1004
+			  const char *mask, bool ctcam)
9461005 {
9471006 	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
9481007 	struct mlxsw_sp_acl_erp_key key;
949
-	struct mlxsw_sp_acl_erp *erp;
950
-
951
-	/* eRPs are allocated from a shared resource, but currently all
952
-	 * allocations are done under RTNL.
953
-	 */
954
-	ASSERT_RTNL();
1008
+	struct objagg_obj *objagg_obj;
9551009 
9561010 	memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
9571011 	key.ctcam = ctcam;
958
-	erp = rhashtable_lookup_fast(&erp_table->erp_ht, &key,
959
-				     mlxsw_sp_acl_erp_ht_params);
960
-	if (erp) {
961
-		refcount_inc(&erp->refcnt);
962
-		return erp;
963
-	}
964
-
965
-	return erp_table->ops->erp_create(erp_table, &key);
1012
+	mutex_lock(&erp_table->objagg_lock);
1013
+	objagg_obj = objagg_obj_get(erp_table->objagg, &key);
1014
+	mutex_unlock(&erp_table->objagg_lock);
1015
+	if (IS_ERR(objagg_obj))
1016
+		return ERR_CAST(objagg_obj);
1017
+	return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;
9661018 }
9671019 
968
-void mlxsw_sp_acl_erp_put(struct mlxsw_sp_acl_atcam_region *aregion,
969
-			  struct mlxsw_sp_acl_erp *erp)
1020
+void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,
1021
+			       struct mlxsw_sp_acl_erp_mask *erp_mask)
9701022 {
1023
+	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
9711024 	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
9721025 
973
-	ASSERT_RTNL();
974
-
975
-	if (!refcount_dec_and_test(&erp->refcnt))
976
-		return;
977
-
978
-	erp_table->ops->erp_destroy(erp_table, erp);
1026
+	mutex_lock(&erp_table->objagg_lock);
1027
+	objagg_obj_put(erp_table->objagg, objagg_obj);
1028
+	mutex_unlock(&erp_table->objagg_lock);
9791029 }
9801030 
1031
+int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,
1032
+			       struct mlxsw_sp_acl_atcam_region *aregion,
1033
+			       struct mlxsw_sp_acl_erp_mask *erp_mask,
1034
+			       struct mlxsw_sp_acl_atcam_entry *aentry)
1035
+{
1036
+	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1037
+	const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1038
+	unsigned int erp_bank;
1039
+
1040
+	if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1041
+		return 0;
1042
+
1043
+	erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1044
+	return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,
1045
+					erp->erp_table->erp_core->bf,
1046
+					aregion, erp_bank, aentry);
1047
+}
1048
+
1049
+void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,
1050
+				struct mlxsw_sp_acl_atcam_region *aregion,
1051
+				struct mlxsw_sp_acl_erp_mask *erp_mask,
1052
+				struct mlxsw_sp_acl_atcam_entry *aentry)
1053
+{
1054
+	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1055
+	const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1056
+	unsigned int erp_bank;
1057
+
1058
+	if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1059
+		return;
1060
+
1061
+	erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1062
+	mlxsw_sp_acl_bf_entry_del(mlxsw_sp,
1063
+				  erp->erp_table->erp_core->bf,
1064
+				  aregion, erp_bank, aentry);
1065
+}
1066
+
1067
+bool
1068
+mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1069
+{
1070
+	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1071
+	const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);
1072
+
1073
+	return key->ctcam;
1074
+}
1075
+
1076
+u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1077
+{
1078
+	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1079
+	const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1080
+
1081
+	return erp->id;
1082
+}
1083
+
1084
+struct mlxsw_sp_acl_erp_delta {
1085
+	struct mlxsw_sp_acl_erp_key key;
1086
+	u16 start;
1087
+	u8 mask;
1088
+};
1089
+
1090
+u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)
1091
+{
1092
+	return delta->start;
1093
+}
1094
+
1095
+u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)
1096
+{
1097
+	return delta->mask;
1098
+}
1099
+
1100
+u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
1101
+				const char *enc_key)
1102
+{
1103
+	u16 start = delta->start;
1104
+	u8 mask = delta->mask;
1105
+	u16 tmp;
1106
+
1107
+	if (!mask)
1108
+		return 0;
1109
+
1110
+	tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];
1111
+	if (start / 8 + 1 < __MASK_LEN)
1112
+		tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;
1113
+	tmp >>= start % 8;
1114
+	tmp &= mask;
1115
+	return tmp;
1116
+}
1117
+
1118
+void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
1119
+				  const char *enc_key)
1120
+{
1121
+	u16 start = delta->start;
1122
+	u8 mask = delta->mask;
1123
+	unsigned char *byte;
1124
+	u16 tmp;
1125
+
1126
+	tmp = mask;
1127
+	tmp <<= start % 8;
1128
+	tmp = ~tmp;
1129
+
1130
+	byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];
1131
+	*byte &= tmp & 0xff;
1132
+	if (start / 8 + 1 < __MASK_LEN) {
1133
+		byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];
1134
+		*byte &= (tmp >> 8) & 0xff;
1135
+	}
1136
+}
1137
+
1138
+static const struct mlxsw_sp_acl_erp_delta
1139
+mlxsw_sp_acl_erp_delta_default = {};
1140
+
1141
+const struct mlxsw_sp_acl_erp_delta *
1142
+mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1143
+{
1144
+	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1145
+	const struct mlxsw_sp_acl_erp_delta *delta;
1146
+
1147
+	delta = objagg_obj_delta_priv(objagg_obj);
1148
+	if (!delta)
1149
+		delta = &mlxsw_sp_acl_erp_delta_default;
1150
+	return delta;
1151
+}
1152
+
1153
+static int
1154
+mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,
1155
+			    const struct mlxsw_sp_acl_erp_key *key,
1156
+			    u16 *delta_start, u8 *delta_mask)
1157
+{
1158
+	int offset = 0;
1159
+	int si = -1;
1160
+	u16 pmask;
1161
+	u16 mask;
1162
+	int i;
1163
+
1164
+	/* The difference between 2 masks can be up to 8 consecutive bits. */
1165
+	for (i = 0; i < __MASK_LEN; i++) {
1166
+		if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])
1167
+			continue;
1168
+		if (si == -1)
1169
+			si = i;
1170
+		else if (si != i - 1)
1171
+			return -EINVAL;
1172
+	}
1173
+	if (si == -1) {
1174
+		/* The masks are the same, this can happen in case eRPs with
1175
+		 * the same mask were created in both A-TCAM and C-TCAM.
1176
+		 * The only possible condition under which this can happen
1177
+		 * is identical rule insertion. Delta is not possible here.
1178
+		 */
1179
+		return -EINVAL;
1180
+	}
1181
+	pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];
1182
+	mask = (unsigned char) key->mask[__MASK_IDX(si)];
1183
+	if (si + 1 < __MASK_LEN) {
1184
+		pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;
1185
+		mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;
1186
+	}
1187
+
1188
+	if ((pmask ^ mask) & pmask)
1189
+		return -EINVAL;
1190
+	mask &= ~pmask;
1191
+	while (!(mask & (1 << offset)))
1192
+		offset++;
1193
+	while (!(mask & 1))
1194
+		mask >>= 1;
1195
+	if (mask & 0xff00)
1196
+		return -EINVAL;
1197
+
1198
+	*delta_start = si * 8 + offset;
1199
+	*delta_mask = mask;
1200
+
1201
+	return 0;
1202
+}
1203
+
1204
+static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,
1205
+					 const void *obj)
1206
+{
1207
+	const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1208
+	const struct mlxsw_sp_acl_erp_key *key = obj;
1209
+	u16 delta_start;
1210
+	u8 delta_mask;
1211
+	int err;
1212
+
1213
+	err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1214
+					  &delta_start, &delta_mask);
1215
+	return err ? false : true;
1216
+}
1217
+
1218
+static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2)
1219
+{
1220
+	const struct mlxsw_sp_acl_erp_key *key1 = obj1;
1221
+	const struct mlxsw_sp_acl_erp_key *key2 = obj2;
1222
+
1223
+	/* For hints purposes, two objects are considered equal
1224
+	 * in case the masks are the same. Does not matter what
1225
+	 * the "ctcam" value is.
1226
+	 */
1227
+	return memcmp(key1->mask, key2->mask, sizeof(key1->mask));
1228
+}
1229
+
1230
+static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,
1231
+					   void *obj)
1232
+{
1233
+	struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1234
+	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1235
+	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1236
+	struct mlxsw_sp_acl_erp_key *key = obj;
1237
+	struct mlxsw_sp_acl_erp_delta *delta;
1238
+	u16 delta_start;
1239
+	u8 delta_mask;
1240
+	int err;
1241
+
1242
+	if (parent_key->ctcam || key->ctcam)
1243
+		return ERR_PTR(-EINVAL);
1244
+	err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1245
+					  &delta_start, &delta_mask);
1246
+	if (err)
1247
+		return ERR_PTR(-EINVAL);
1248
+
1249
+	delta = kzalloc(sizeof(*delta), GFP_KERNEL);
1250
+	if (!delta)
1251
+		return ERR_PTR(-ENOMEM);
1252
+	delta->start = delta_start;
1253
+	delta->mask = delta_mask;
1254
+
1255
+	err = mlxsw_sp_acl_erp_delta_inc(erp_table);
1256
+	if (err)
1257
+		goto err_erp_delta_inc;
1258
+
1259
+	memcpy(&delta->key, key, sizeof(*key));
1260
+	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);
1261
+	if (err)
1262
+		goto err_master_mask_set;
1263
+
1264
+	return delta;
1265
+
1266
+err_master_mask_set:
1267
+	mlxsw_sp_acl_erp_delta_dec(erp_table);
1268
+err_erp_delta_inc:
1269
+	kfree(delta);
1270
+	return ERR_PTR(err);
1271
+}
1272
+
1273
+static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)
1274
+{
1275
+	struct mlxsw_sp_acl_erp_delta *delta = delta_priv;
1276
+	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1277
+	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1278
+
1279
+	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);
1280
+	mlxsw_sp_acl_erp_delta_dec(erp_table);
1281
+	kfree(delta);
1282
+}
1283
+
1284
+static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,
1285
+					  unsigned int root_id)
1286
+{
1287
+	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1288
+	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1289
+	struct mlxsw_sp_acl_erp_key *key = obj;
1290
+
1291
+	if (!key->ctcam &&
1292
+	    root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&
1293
+	    root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)
1294
+		return ERR_PTR(-ENOBUFS);
1295
+	return erp_table->ops->erp_create(erp_table, key);
1296
+}
1297
+
1298
+static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)
1299
+{
1300
+	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1301
+	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1302
+
1303
+	erp_table->ops->erp_destroy(erp_table, root_priv);
1304
+}
1305
+
1306
+static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {
1307
+	.obj_size = sizeof(struct mlxsw_sp_acl_erp_key),
1308
+	.delta_check = mlxsw_sp_acl_erp_delta_check,
1309
+	.hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp,
1310
+	.delta_create = mlxsw_sp_acl_erp_delta_create,
1311
+	.delta_destroy = mlxsw_sp_acl_erp_delta_destroy,
1312
+	.root_create = mlxsw_sp_acl_erp_root_create,
1313
+	.root_destroy = mlxsw_sp_acl_erp_root_destroy,
1314
+};
1315
+
9811316 static struct mlxsw_sp_acl_erp_table *
982
-mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion)
1317
+mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,
1318
+			      struct objagg_hints *hints)
9831319 {
9841320 	struct mlxsw_sp_acl_erp_table *erp_table;
9851321 	int err;
....@@ -988,18 +1324,22 @@
9881324 	if (!erp_table)
9891325 		return ERR_PTR(-ENOMEM);
9901326 
991
-	err = rhashtable_init(&erp_table->erp_ht, &mlxsw_sp_acl_erp_ht_params);
992
-	if (err)
993
-		goto err_rhashtable_init;
1327
+	erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,
1328
+					  hints, aregion);
1329
+	if (IS_ERR(erp_table->objagg)) {
1330
+		err = PTR_ERR(erp_table->objagg);
1331
+		goto err_objagg_create;
1332
+	}
9941333 
9951334 	erp_table->erp_core = aregion->atcam->erp_core;
9961335 	erp_table->ops = &erp_no_mask_ops;
9971336 	INIT_LIST_HEAD(&erp_table->atcam_erps_list);
9981337 	erp_table->aregion = aregion;
1338
+	mutex_init(&erp_table->objagg_lock);
9991339 
10001340 	return erp_table;
10011341 
1002
-err_rhashtable_init:
1342
+err_objagg_create:
10031343 	kfree(erp_table);
10041344 	return ERR_PTR(err);
10051345 }
....@@ -1008,7 +1348,8 @@
10081348 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
10091349 {
10101350 	WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1011
-	rhashtable_destroy(&erp_table->erp_ht);
1351
+	mutex_destroy(&erp_table->objagg_lock);
1352
+	objagg_destroy(erp_table->objagg);
10121353 	kfree(erp_table);
10131354 }
10141355 
....@@ -1033,12 +1374,93 @@
10331374 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
10341375 }
10351376 
1036
-int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion)
1377
+static int
1378
+mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,
1379
+			     struct mlxsw_sp_acl_atcam_region *aregion,
1380
+			     struct objagg_hints *hints, bool *p_rehash_needed)
10371381 {
1038
-	struct mlxsw_sp_acl_erp_table *erp_table;
1382
+	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1383
+	const struct objagg_stats *ostats;
1384
+	const struct objagg_stats *hstats;
10391385 	int err;
10401386 
1041
-	erp_table = mlxsw_sp_acl_erp_table_create(aregion);
1387
+	*p_rehash_needed = false;
1388
+
1389
+	mutex_lock(&erp_table->objagg_lock);
1390
+	ostats = objagg_stats_get(erp_table->objagg);
1391
+	mutex_unlock(&erp_table->objagg_lock);
1392
+	if (IS_ERR(ostats)) {
1393
+		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");
1394
+		return PTR_ERR(ostats);
1395
+	}
1396
+
1397
+	hstats = objagg_hints_stats_get(hints);
1398
+	if (IS_ERR(hstats)) {
1399
+		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");
1400
+		err = PTR_ERR(hstats);
1401
+		goto err_hints_stats_get;
1402
+	}
1403
+
1404
+	/* Very basic criterion for now. */
1405
+	if (hstats->root_count < ostats->root_count)
1406
+		*p_rehash_needed = true;
1407
+
1408
+	err = 0;
1409
+
1410
+	objagg_stats_put(hstats);
1411
+err_hints_stats_get:
1412
+	objagg_stats_put(ostats);
1413
+	return err;
1414
+}
1415
+
1416
+void *
1417
+mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
1418
+{
1419
+	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1420
+	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1421
+	struct objagg_hints *hints;
1422
+	bool rehash_needed;
1423
+	int err;
1424
+
1425
+	mutex_lock(&erp_table->objagg_lock);
1426
+	hints = objagg_hints_get(erp_table->objagg,
1427
+				 OBJAGG_OPT_ALGO_SIMPLE_GREEDY);
1428
+	mutex_unlock(&erp_table->objagg_lock);
1429
+	if (IS_ERR(hints)) {
1430
+		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");
1431
+		return ERR_CAST(hints);
1432
+	}
1433
+	err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,
1434
+					   &rehash_needed);
1435
+	if (err)
1436
+		goto errout;
1437
+
1438
+	if (!rehash_needed) {
1439
+		err = -EAGAIN;
1440
+		goto errout;
1441
+	}
1442
+	return hints;
1443
+
1444
+errout:
1445
+	objagg_hints_put(hints);
1446
+	return ERR_PTR(err);
1447
+}
1448
+
1449
+void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)
1450
+{
1451
+	struct objagg_hints *hints = hints_priv;
1452
+
1453
+	objagg_hints_put(hints);
1454
+}
1455
+
1456
+int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,
1457
+				 void *hints_priv)
1458
+{
1459
+	struct mlxsw_sp_acl_erp_table *erp_table;
1460
+	struct objagg_hints *hints = hints_priv;
1461
+	int err;
1462
+
1463
+	erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);
10421464 	if (IS_ERR(erp_table))
10431465 		return PTR_ERR(erp_table);
10441466 	aregion->erp_table = erp_table;
....@@ -1118,6 +1540,12 @@
11181540 	if (err)
11191541 		goto err_gen_pool_add;
11201542 
1543
+	erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);
1544
+	if (IS_ERR(erp_core->bf)) {
1545
+		err = PTR_ERR(erp_core->bf);
1546
+		goto err_bf_init;
1547
+	}
1548
+
11211549 	/* Different regions require masks of different sizes */
11221550 	err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
11231551 	if (err)
....@@ -1126,6 +1554,8 @@
11261554 	return 0;
11271555 
11281556 err_erp_tables_sizes_query:
1557
+	mlxsw_sp_acl_bf_fini(erp_core->bf);
1558
+err_bf_init:
11291559 err_gen_pool_add:
11301560 	gen_pool_destroy(erp_core->erp_tables);
11311561 	return err;
....@@ -1134,6 +1564,7 @@
11341564 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
11351565 					 struct mlxsw_sp_acl_erp_core *erp_core)
11361566 {
1567
+	mlxsw_sp_acl_bf_fini(erp_core->bf);
11371568 	gen_pool_destroy(erp_core->erp_tables);
11381569 }
11391570