forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 23fa18eaa71266feff7ba8d83022d9e1cc83c65a 
 kernel/net/openvswitch/meter.c
....@@ -1,9 +1,6 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (c) 2017 Nicira, Inc.
3
- *
4
- * This program is free software; you can redistribute it and/or
5
- * modify it under the terms of version 2 of the GNU General Public
6
- * License as published by the Free Software Foundation.
74  */
85 
96 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
....@@ -15,14 +12,13 @@
1512 #include <linux/openvswitch.h>
1613 #include <linux/netlink.h>
1714 #include <linux/rculist.h>
15
+#include <linux/swap.h>
1816 
1917 #include <net/netlink.h>
2018 #include <net/genetlink.h>
2119 
2220 #include "datapath.h"
2321 #include "meter.h"
24
-
25
-#define METER_HASH_BUCKETS 1024
2622 
2723 static const struct nla_policy meter_policy[OVS_METER_ATTR_MAX + 1] = {
2824 	[OVS_METER_ATTR_ID] = { .type = NLA_U32, },
....@@ -42,6 +38,11 @@
4238 	[OVS_BAND_ATTR_STATS] = { .len = sizeof(struct ovs_flow_stats) },
4339 };
4440 
41
+static u32 meter_hash(struct dp_meter_instance *ti, u32 id)
42
+{
43
+	return id % ti->n_meters;
44
+}
45
+
4546 static void ovs_meter_free(struct dp_meter *meter)
4647 {
4748 	if (!meter)
....@@ -50,39 +51,162 @@
5051 	kfree_rcu(meter, rcu);
5152 }
5253 
53
-static struct hlist_head *meter_hash_bucket(const struct datapath *dp,
54
-					    u32 meter_id)
55
-{
56
-	return &dp->meters[meter_id & (METER_HASH_BUCKETS - 1)];
57
-}
58
-
5954 /* Call with ovs_mutex or RCU read lock. */
60
-static struct dp_meter *lookup_meter(const struct datapath *dp,
55
+static struct dp_meter *lookup_meter(const struct dp_meter_table *tbl,
6156 				     u32 meter_id)
6257 {
58
+	struct dp_meter_instance *ti = rcu_dereference_ovsl(tbl->ti);
59
+	u32 hash = meter_hash(ti, meter_id);
6360 	struct dp_meter *meter;
64
-	struct hlist_head *head;
6561 
66
-	head = meter_hash_bucket(dp, meter_id);
67
-	hlist_for_each_entry_rcu(meter, head, dp_hash_node) {
68
-		if (meter->id == meter_id)
69
-			return meter;
70
-	}
62
+	meter = rcu_dereference_ovsl(ti->dp_meters[hash]);
63
+	if (meter && likely(meter->id == meter_id))
64
+		return meter;
65
+
7166 	return NULL;
7267 }
7368 
74
-static void attach_meter(struct datapath *dp, struct dp_meter *meter)
69
+static struct dp_meter_instance *dp_meter_instance_alloc(const u32 size)
7570 {
76
-	struct hlist_head *head = meter_hash_bucket(dp, meter->id);
71
+	struct dp_meter_instance *ti;
7772 
78
-	hlist_add_head_rcu(&meter->dp_hash_node, head);
73
+	ti = kvzalloc(sizeof(*ti) +
74
+		      sizeof(struct dp_meter *) * size,
75
+		      GFP_KERNEL);
76
+	if (!ti)
77
+		return NULL;
78
+
79
+	ti->n_meters = size;
80
+
81
+	return ti;
7982 }
8083 
81
-static void detach_meter(struct dp_meter *meter)
84
+static void dp_meter_instance_free(struct dp_meter_instance *ti)
8285 {
86
+	kvfree(ti);
87
+}
88
+
89
+static void dp_meter_instance_free_rcu(struct rcu_head *rcu)
90
+{
91
+	struct dp_meter_instance *ti;
92
+
93
+	ti = container_of(rcu, struct dp_meter_instance, rcu);
94
+	kvfree(ti);
95
+}
96
+
97
+static int
98
+dp_meter_instance_realloc(struct dp_meter_table *tbl, u32 size)
99
+{
100
+	struct dp_meter_instance *ti = rcu_dereference_ovsl(tbl->ti);
101
+	int n_meters = min(size, ti->n_meters);
102
+	struct dp_meter_instance *new_ti;
103
+	int i;
104
+
105
+	new_ti = dp_meter_instance_alloc(size);
106
+	if (!new_ti)
107
+		return -ENOMEM;
108
+
109
+	for (i = 0; i < n_meters; i++)
110
+		if (rcu_dereference_ovsl(ti->dp_meters[i]))
111
+			new_ti->dp_meters[i] = ti->dp_meters[i];
112
+
113
+	rcu_assign_pointer(tbl->ti, new_ti);
114
+	call_rcu(&ti->rcu, dp_meter_instance_free_rcu);
115
+
116
+	return 0;
117
+}
118
+
119
+static void dp_meter_instance_insert(struct dp_meter_instance *ti,
120
+				     struct dp_meter *meter)
121
+{
122
+	u32 hash;
123
+
124
+	hash = meter_hash(ti, meter->id);
125
+	rcu_assign_pointer(ti->dp_meters[hash], meter);
126
+}
127
+
128
+static void dp_meter_instance_remove(struct dp_meter_instance *ti,
129
+				     struct dp_meter *meter)
130
+{
131
+	u32 hash;
132
+
133
+	hash = meter_hash(ti, meter->id);
134
+	RCU_INIT_POINTER(ti->dp_meters[hash], NULL);
135
+}
136
+
137
+static int attach_meter(struct dp_meter_table *tbl, struct dp_meter *meter)
138
+{
139
+	struct dp_meter_instance *ti = rcu_dereference_ovsl(tbl->ti);
140
+	u32 hash = meter_hash(ti, meter->id);
141
+	int err;
142
+
143
+	/* In generally, slots selected should be empty, because
144
+	 * OvS uses id-pool to fetch a available id.
145
+	 */
146
+	if (unlikely(rcu_dereference_ovsl(ti->dp_meters[hash])))
147
+		return -EBUSY;
148
+
149
+	dp_meter_instance_insert(ti, meter);
150
+
151
+	/* That function is thread-safe. */
152
+	tbl->count++;
153
+	if (tbl->count >= tbl->max_meters_allowed) {
154
+		err = -EFBIG;
155
+		goto attach_err;
156
+	}
157
+
158
+	if (tbl->count >= ti->n_meters &&
159
+	    dp_meter_instance_realloc(tbl, ti->n_meters * 2)) {
160
+		err = -ENOMEM;
161
+		goto attach_err;
162
+	}
163
+
164
+	return 0;
165
+
166
+attach_err:
167
+	dp_meter_instance_remove(ti, meter);
168
+	tbl->count--;
169
+	return err;
170
+}
171
+
172
+static int detach_meter(struct dp_meter_table *tbl, struct dp_meter *meter)
173
+{
174
+	struct dp_meter_instance *ti;
175
+
83176 	ASSERT_OVSL();
84
-	if (meter)
85
-		hlist_del_rcu(&meter->dp_hash_node);
177
+	if (!meter)
178
+		return 0;
179
+
180
+	ti = rcu_dereference_ovsl(tbl->ti);
181
+	dp_meter_instance_remove(ti, meter);
182
+
183
+	tbl->count--;
184
+
185
+	/* Shrink the meter array if necessary. */
186
+	if (ti->n_meters > DP_METER_ARRAY_SIZE_MIN &&
187
+	    tbl->count <= (ti->n_meters / 4)) {
188
+		int half_size = ti->n_meters / 2;
189
+		int i;
190
+
191
+		/* Avoid hash collision, don't move slots to other place.
192
+		 * Make sure there are no references of meters in array
193
+		 * which will be released.
194
+		 */
195
+		for (i = half_size; i < ti->n_meters; i++)
196
+			if (rcu_dereference_ovsl(ti->dp_meters[i]))
197
+				goto out;
198
+
199
+		if (dp_meter_instance_realloc(tbl, half_size))
200
+			goto shrink_err;
201
+	}
202
+
203
+out:
204
+	return 0;
205
+
206
+shrink_err:
207
+	dp_meter_instance_insert(ti, meter);
208
+	tbl->count++;
209
+	return -ENOMEM;
86210 }
87211 
88212 static struct sk_buff *
....@@ -118,16 +242,15 @@
118242 	if (nla_put_u32(reply, OVS_METER_ATTR_ID, meter_id))
119243 		goto error;
120244 
121
-	if (!meter)
122
-		return 0;
123
-
124245 	if (nla_put(reply, OVS_METER_ATTR_STATS,
125
-		    sizeof(struct ovs_flow_stats), &meter->stats) ||
126
-	    nla_put_u64_64bit(reply, OVS_METER_ATTR_USED, meter->used,
246
+		    sizeof(struct ovs_flow_stats), &meter->stats))
247
+		goto error;
248
+
249
+	if (nla_put_u64_64bit(reply, OVS_METER_ATTR_USED, meter->used,
127250 			      OVS_METER_ATTR_PAD))
128251 		goto error;
129252 
130
-	nla = nla_nest_start(reply, OVS_METER_ATTR_BANDS);
253
+	nla = nla_nest_start_noflag(reply, OVS_METER_ATTR_BANDS);
131254 	if (!nla)
132255 		goto error;
133256 
....@@ -136,7 +259,7 @@
136259 	for (i = 0; i < meter->n_bands; ++i, ++band) {
137260 		struct nlattr *band_nla;
138261 
139
-		band_nla = nla_nest_start(reply, OVS_BAND_ATTR_UNSPEC);
262
+		band_nla = nla_nest_start_noflag(reply, OVS_BAND_ATTR_UNSPEC);
140263 		if (!band_nla || nla_put(reply, OVS_BAND_ATTR_STATS,
141264 					 sizeof(struct ovs_flow_stats),
142265 					 &band->stats))
....@@ -152,25 +275,39 @@
152275 
153276 static int ovs_meter_cmd_features(struct sk_buff *skb, struct genl_info *info)
154277 {
155
-	struct sk_buff *reply;
278
+	struct ovs_header *ovs_header = info->userhdr;
156279 	struct ovs_header *ovs_reply_header;
157280 	struct nlattr *nla, *band_nla;
158
-	int err;
281
+	struct sk_buff *reply;
282
+	struct datapath *dp;
283
+	int err = -EMSGSIZE;
159284 
160285 	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_FEATURES,
161286 					  &ovs_reply_header);
162287 	if (IS_ERR(reply))
163288 		return PTR_ERR(reply);
164289 
165
-	if (nla_put_u32(reply, OVS_METER_ATTR_MAX_METERS, U32_MAX) ||
166
-	    nla_put_u32(reply, OVS_METER_ATTR_MAX_BANDS, DP_MAX_BANDS))
290
+	ovs_lock();
291
+	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
292
+	if (!dp) {
293
+		err = -ENODEV;
294
+		goto exit_unlock;
295
+	}
296
+
297
+	if (nla_put_u32(reply, OVS_METER_ATTR_MAX_METERS,
298
+			dp->meter_tbl.max_meters_allowed))
299
+		goto exit_unlock;
300
+
301
+	ovs_unlock();
302
+
303
+	if (nla_put_u32(reply, OVS_METER_ATTR_MAX_BANDS, DP_MAX_BANDS))
167304 		goto nla_put_failure;
168305 
169
-	nla = nla_nest_start(reply, OVS_METER_ATTR_BANDS);
306
+	nla = nla_nest_start_noflag(reply, OVS_METER_ATTR_BANDS);
170307 	if (!nla)
171308 		goto nla_put_failure;
172309 
173
-	band_nla = nla_nest_start(reply, OVS_BAND_ATTR_UNSPEC);
310
+	band_nla = nla_nest_start_noflag(reply, OVS_BAND_ATTR_UNSPEC);
174311 	if (!band_nla)
175312 		goto nla_put_failure;
176313 	/* Currently only DROP band type is supported. */
....@@ -182,9 +319,10 @@
182319 	genlmsg_end(reply, ovs_reply_header);
183320 	return genlmsg_reply(reply, info);
184321 
322
+exit_unlock:
323
+	ovs_unlock();
185324 nla_put_failure:
186325 	nlmsg_free(reply);
187
-	err = -EMSGSIZE;
188326 	return err;
189327 }
190328 
....@@ -206,8 +344,7 @@
206344 			return ERR_PTR(-EINVAL);
207345 
208346 	/* Allocate and set up the meter before locking anything. */
209
-	meter = kzalloc(n_bands * sizeof(struct dp_meter_band) +
210
-			sizeof(*meter), GFP_KERNEL);
347
+	meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL);
211348 	if (!meter)
212349 		return ERR_PTR(-ENOMEM);
213350 
....@@ -228,9 +365,9 @@
228365 		struct nlattr *attr[OVS_BAND_ATTR_MAX + 1];
229366 		u32 band_max_delta_t;
230367 
231
-		err = nla_parse((struct nlattr **)&attr, OVS_BAND_ATTR_MAX,
232
-				nla_data(nla), nla_len(nla), band_policy,
233
-				NULL);
368
+		err = nla_parse_deprecated((struct nlattr **)&attr,
369
+					   OVS_BAND_ATTR_MAX, nla_data(nla),
370
+					   nla_len(nla), band_policy, NULL);
234371 		if (err)
235372 			goto exit_free_meter;
236373 
....@@ -276,14 +413,14 @@
276413 	struct sk_buff *reply;
277414 	struct ovs_header *ovs_reply_header;
278415 	struct ovs_header *ovs_header = info->userhdr;
416
+	struct dp_meter_table *meter_tbl;
279417 	struct datapath *dp;
280418 	int err;
281419 	u32 meter_id;
282420 	bool failed;
283421 
284
-	if (!a[OVS_METER_ATTR_ID]) {
285
-		return -ENODEV;
286
-	}
422
+	if (!a[OVS_METER_ATTR_ID])
423
+		return -EINVAL;
287424 
288425 	meter = dp_meter_create(a);
289426 	if (IS_ERR_OR_NULL(meter))
....@@ -303,12 +440,18 @@
303440 		goto exit_unlock;
304441 	}
305442 
443
+	meter_tbl = &dp->meter_tbl;
306444 	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);
307445 
308
-	/* Cannot fail after this. */
309
-	old_meter = lookup_meter(dp, meter_id);
310
-	detach_meter(old_meter);
311
-	attach_meter(dp, meter);
446
+	old_meter = lookup_meter(meter_tbl, meter_id);
447
+	err = detach_meter(meter_tbl, old_meter);
448
+	if (err)
449
+		goto exit_unlock;
450
+
451
+	err = attach_meter(meter_tbl, meter);
452
+	if (err)
453
+		goto exit_free_old_meter;
454
+
312455 	ovs_unlock();
313456 
314457 	/* Build response with the meter_id and stats from
....@@ -330,6 +473,8 @@
330473 	genlmsg_end(reply, ovs_reply_header);
331474 	return genlmsg_reply(reply, info);
332475 
476
+exit_free_old_meter:
477
+	ovs_meter_free(old_meter);
333478 exit_unlock:
334479 	ovs_unlock();
335480 	nlmsg_free(reply);
....@@ -340,14 +485,14 @@
340485 
341486 static int ovs_meter_cmd_get(struct sk_buff *skb, struct genl_info *info)
342487 {
343
-	struct nlattr **a = info->attrs;
344
-	u32 meter_id;
345488 	struct ovs_header *ovs_header = info->userhdr;
346489 	struct ovs_header *ovs_reply_header;
347
-	struct datapath *dp;
348
-	int err;
349
-	struct sk_buff *reply;
490
+	struct nlattr **a = info->attrs;
350491 	struct dp_meter *meter;
492
+	struct sk_buff *reply;
493
+	struct datapath *dp;
494
+	u32 meter_id;
495
+	int err;
351496 
352497 	if (!a[OVS_METER_ATTR_ID])
353498 		return -EINVAL;
....@@ -368,7 +513,7 @@
368513 	}
369514 
370515 	/* Locate meter, copy stats. */
371
-	meter = lookup_meter(dp, meter_id);
516
+	meter = lookup_meter(&dp->meter_tbl, meter_id);
372517 	if (!meter) {
373518 		err = -ENOENT;
374519 		goto exit_unlock;
....@@ -393,18 +538,17 @@
393538 
394539 static int ovs_meter_cmd_del(struct sk_buff *skb, struct genl_info *info)
395540 {
396
-	struct nlattr **a = info->attrs;
397
-	u32 meter_id;
398541 	struct ovs_header *ovs_header = info->userhdr;
399542 	struct ovs_header *ovs_reply_header;
400
-	struct datapath *dp;
401
-	int err;
402
-	struct sk_buff *reply;
543
+	struct nlattr **a = info->attrs;
403544 	struct dp_meter *old_meter;
545
+	struct sk_buff *reply;
546
+	struct datapath *dp;
547
+	u32 meter_id;
548
+	int err;
404549 
405550 	if (!a[OVS_METER_ATTR_ID])
406551 		return -EINVAL;
407
-	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);
408552 
409553 	reply = ovs_meter_cmd_reply_start(info, OVS_METER_CMD_DEL,
410554 					  &ovs_reply_header);
....@@ -419,14 +563,19 @@
419563 		goto exit_unlock;
420564 	}
421565 
422
-	old_meter = lookup_meter(dp, meter_id);
566
+	meter_id = nla_get_u32(a[OVS_METER_ATTR_ID]);
567
+	old_meter = lookup_meter(&dp->meter_tbl, meter_id);
423568 	if (old_meter) {
424569 		spin_lock_bh(&old_meter->lock);
425570 		err = ovs_meter_cmd_reply_stats(reply, meter_id, old_meter);
426571 		WARN_ON(err);
427572 		spin_unlock_bh(&old_meter->lock);
428
-		detach_meter(old_meter);
573
+
574
+		err = detach_meter(&dp->meter_tbl, old_meter);
575
+		if (err)
576
+			goto exit_unlock;
429577 	}
578
+
430579 	ovs_unlock();
431580 	ovs_meter_free(old_meter);
432581 	genlmsg_end(reply, ovs_reply_header);
....@@ -446,16 +595,16 @@
446595 bool ovs_meter_execute(struct datapath *dp, struct sk_buff *skb,
447596 		       struct sw_flow_key *key, u32 meter_id)
448597 {
449
-	struct dp_meter *meter;
450
-	struct dp_meter_band *band;
451598 	long long int now_ms = div_u64(ktime_get_ns(), 1000 * 1000);
452599 	long long int long_delta_ms;
453
-	u32 delta_ms;
454
-	u32 cost;
600
+	struct dp_meter_band *band;
601
+	struct dp_meter *meter;
455602 	int i, band_exceeded_max = -1;
456603 	u32 band_exceeded_rate = 0;
604
+	u32 delta_ms;
605
+	u32 cost;
457606 
458
-	meter = lookup_meter(dp, meter_id);
607
+	meter = lookup_meter(&dp->meter_tbl, meter_id);
459608 	/* Do not drop the packet when there is no meter. */
460609 	if (!meter)
461610 		return false;
....@@ -533,29 +682,29 @@
533682 	return false;
534683 }
535684 
536
-static struct genl_ops dp_meter_genl_ops[] = {
685
+static const struct genl_small_ops dp_meter_genl_ops[] = {
537686 	{ .cmd = OVS_METER_CMD_FEATURES,
687
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
538688 		.flags = 0,		  /* OK for unprivileged users. */
539
-		.policy = meter_policy,
540689 		.doit = ovs_meter_cmd_features
541690 	},
542691 	{ .cmd = OVS_METER_CMD_SET,
692
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
543693 		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
544694 					   *  privilege.
545695 					   */
546
-		.policy = meter_policy,
547696 		.doit = ovs_meter_cmd_set,
548697 	},
549698 	{ .cmd = OVS_METER_CMD_GET,
699
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
550700 		.flags = 0,		  /* OK for unprivileged users. */
551
-		.policy = meter_policy,
552701 		.doit = ovs_meter_cmd_get,
553702 	},
554703 	{ .cmd = OVS_METER_CMD_DEL,
704
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
555705 		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
556706 					   *  privilege.
557707 					   */
558
-		.policy = meter_policy,
559708 		.doit = ovs_meter_cmd_del
560709 	},
561710 };
....@@ -569,10 +718,11 @@
569718 	.name = OVS_METER_FAMILY,
570719 	.version = OVS_METER_VERSION,
571720 	.maxattr = OVS_METER_ATTR_MAX,
721
+	.policy = meter_policy,
572722 	.netnsok = true,
573723 	.parallel_ops = true,
574
-	.ops = dp_meter_genl_ops,
575
-	.n_ops = ARRAY_SIZE(dp_meter_genl_ops),
724
+	.small_ops = dp_meter_genl_ops,
725
+	.n_small_ops = ARRAY_SIZE(dp_meter_genl_ops),
576726 	.mcgrps = &ovs_meter_multicast_group,
577727 	.n_mcgrps = 1,
578728 	.module = THIS_MODULE,
....@@ -580,32 +730,39 @@
580730 
581731 int ovs_meters_init(struct datapath *dp)
582732 {
583
-	int i;
733
+	struct dp_meter_table *tbl = &dp->meter_tbl;
734
+	struct dp_meter_instance *ti;
735
+	unsigned long free_mem_bytes;
584736 
585
-	dp->meters = kmalloc_array(METER_HASH_BUCKETS,
586
-				   sizeof(struct hlist_head), GFP_KERNEL);
587
-
588
-	if (!dp->meters)
737
+	ti = dp_meter_instance_alloc(DP_METER_ARRAY_SIZE_MIN);
738
+	if (!ti)
589739 		return -ENOMEM;
590740 
591
-	for (i = 0; i < METER_HASH_BUCKETS; i++)
592
-		INIT_HLIST_HEAD(&dp->meters[i]);
741
+	/* Allow meters in a datapath to use ~3.12% of physical memory. */
742
+	free_mem_bytes = nr_free_buffer_pages() * (PAGE_SIZE >> 5);
743
+	tbl->max_meters_allowed = min(free_mem_bytes / sizeof(struct dp_meter),
744
+				      DP_METER_NUM_MAX);
745
+	if (!tbl->max_meters_allowed)
746
+		goto out_err;
747
+
748
+	rcu_assign_pointer(tbl->ti, ti);
749
+	tbl->count = 0;
593750 
594751 	return 0;
752
+
753
+out_err:
754
+	dp_meter_instance_free(ti);
755
+	return -ENOMEM;
595756 }
596757 
597758 void ovs_meters_exit(struct datapath *dp)
598759 {
760
+	struct dp_meter_table *tbl = &dp->meter_tbl;
761
+	struct dp_meter_instance *ti = rcu_dereference_raw(tbl->ti);
599762 	int i;
600763 
601
-	for (i = 0; i < METER_HASH_BUCKETS; i++) {
602
-		struct hlist_head *head = &dp->meters[i];
603
-		struct dp_meter *meter;
604
-		struct hlist_node *n;
764
+	for (i = 0; i < ti->n_meters; i++)
765
+		ovs_meter_free(rcu_dereference_raw(ti->dp_meters[i]));
605766 
606
-		hlist_for_each_entry_safe(meter, n, head, dp_hash_node)
607
-			kfree(meter);
608
-	}
609
-
610
-	kfree(dp->meters);
767
+	dp_meter_instance_free(ti);
611768 }