.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * net/sched/cls_matchll.c Match-all classifier |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> |
---|
5 | | - * |
---|
6 | | - * This program is free software; you can redistribute it and/or modify |
---|
7 | | - * it under the terms of the GNU General Public License as published by |
---|
8 | | - * the Free Software Foundation; either version 2 of the License, or |
---|
9 | | - * (at your option) any later version. |
---|
10 | 6 | */ |
---|
11 | 7 | |
---|
12 | 8 | #include <linux/kernel.h> |
---|
13 | 9 | #include <linux/init.h> |
---|
14 | 10 | #include <linux/module.h> |
---|
| 11 | +#include <linux/percpu.h> |
---|
15 | 12 | |
---|
16 | 13 | #include <net/sch_generic.h> |
---|
17 | 14 | #include <net/pkt_cls.h> |
---|
.. | .. |
---|
22 | 19 | u32 handle; |
---|
23 | 20 | u32 flags; |
---|
24 | 21 | unsigned int in_hw_count; |
---|
| 22 | + struct tc_matchall_pcnt __percpu *pf; |
---|
25 | 23 | struct rcu_work rwork; |
---|
| 24 | + bool deleting; |
---|
26 | 25 | }; |
---|
27 | 26 | |
---|
28 | 27 | static int mall_classify(struct sk_buff *skb, const struct tcf_proto *tp, |
---|
.. | .. |
---|
30 | 29 | { |
---|
31 | 30 | struct cls_mall_head *head = rcu_dereference_bh(tp->root); |
---|
32 | 31 | |
---|
| 32 | + if (unlikely(!head)) |
---|
| 33 | + return -1; |
---|
| 34 | + |
---|
33 | 35 | if (tc_skip_sw(head->flags)) |
---|
34 | 36 | return -1; |
---|
35 | 37 | |
---|
36 | 38 | *res = head->res; |
---|
| 39 | + __this_cpu_inc(head->pf->rhit); |
---|
37 | 40 | return tcf_exts_exec(skb, &head->exts, res); |
---|
38 | 41 | } |
---|
39 | 42 | |
---|
.. | .. |
---|
46 | 49 | { |
---|
47 | 50 | tcf_exts_destroy(&head->exts); |
---|
48 | 51 | tcf_exts_put_net(&head->exts); |
---|
| 52 | + free_percpu(head->pf); |
---|
49 | 53 | kfree(head); |
---|
50 | 54 | } |
---|
51 | 55 | |
---|
.. | .. |
---|
71 | 75 | cls_mall.command = TC_CLSMATCHALL_DESTROY; |
---|
72 | 76 | cls_mall.cookie = cookie; |
---|
73 | 77 | |
---|
74 | | - tc_setup_cb_call(block, NULL, TC_SETUP_CLSMATCHALL, &cls_mall, false); |
---|
75 | | - tcf_block_offload_dec(block, &head->flags); |
---|
| 78 | + tc_setup_cb_destroy(block, tp, TC_SETUP_CLSMATCHALL, &cls_mall, false, |
---|
| 79 | + &head->flags, &head->in_hw_count, true); |
---|
76 | 80 | } |
---|
77 | 81 | |
---|
78 | 82 | static int mall_replace_hw_filter(struct tcf_proto *tp, |
---|
.. | .. |
---|
85 | 89 | bool skip_sw = tc_skip_sw(head->flags); |
---|
86 | 90 | int err; |
---|
87 | 91 | |
---|
| 92 | + cls_mall.rule = flow_rule_alloc(tcf_exts_num_actions(&head->exts)); |
---|
| 93 | + if (!cls_mall.rule) |
---|
| 94 | + return -ENOMEM; |
---|
| 95 | + |
---|
88 | 96 | tc_cls_common_offload_init(&cls_mall.common, tp, head->flags, extack); |
---|
89 | 97 | cls_mall.command = TC_CLSMATCHALL_REPLACE; |
---|
90 | | - cls_mall.exts = &head->exts; |
---|
91 | 98 | cls_mall.cookie = cookie; |
---|
92 | 99 | |
---|
93 | | - err = tc_setup_cb_call(block, NULL, TC_SETUP_CLSMATCHALL, |
---|
94 | | - &cls_mall, skip_sw); |
---|
95 | | - if (err < 0) { |
---|
| 100 | + err = tc_setup_flow_action(&cls_mall.rule->action, &head->exts); |
---|
| 101 | + if (err) { |
---|
| 102 | + kfree(cls_mall.rule); |
---|
| 103 | + mall_destroy_hw_filter(tp, head, cookie, NULL); |
---|
| 104 | + if (skip_sw) |
---|
| 105 | + NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); |
---|
| 106 | + else |
---|
| 107 | + err = 0; |
---|
| 108 | + |
---|
| 109 | + return err; |
---|
| 110 | + } |
---|
| 111 | + |
---|
| 112 | + err = tc_setup_cb_add(block, tp, TC_SETUP_CLSMATCHALL, &cls_mall, |
---|
| 113 | + skip_sw, &head->flags, &head->in_hw_count, true); |
---|
| 114 | + tc_cleanup_flow_action(&cls_mall.rule->action); |
---|
| 115 | + kfree(cls_mall.rule); |
---|
| 116 | + |
---|
| 117 | + if (err) { |
---|
96 | 118 | mall_destroy_hw_filter(tp, head, cookie, NULL); |
---|
97 | 119 | return err; |
---|
98 | | - } else if (err > 0) { |
---|
99 | | - head->in_hw_count = err; |
---|
100 | | - tcf_block_offload_inc(block, &head->flags); |
---|
101 | 120 | } |
---|
102 | 121 | |
---|
103 | 122 | if (skip_sw && !(head->flags & TCA_CLS_FLAGS_IN_HW)) |
---|
.. | .. |
---|
106 | 125 | return 0; |
---|
107 | 126 | } |
---|
108 | 127 | |
---|
109 | | -static void mall_destroy(struct tcf_proto *tp, struct netlink_ext_ack *extack) |
---|
| 128 | +static void mall_destroy(struct tcf_proto *tp, bool rtnl_held, |
---|
| 129 | + struct netlink_ext_ack *extack) |
---|
110 | 130 | { |
---|
111 | 131 | struct cls_mall_head *head = rtnl_dereference(tp->root); |
---|
112 | 132 | |
---|
.. | .. |
---|
148 | 168 | { |
---|
149 | 169 | int err; |
---|
150 | 170 | |
---|
151 | | - err = tcf_exts_validate(net, tp, tb, est, &head->exts, ovr, extack); |
---|
| 171 | + err = tcf_exts_validate(net, tp, tb, est, &head->exts, ovr, true, |
---|
| 172 | + extack); |
---|
152 | 173 | if (err < 0) |
---|
153 | 174 | return err; |
---|
154 | 175 | |
---|
.. | .. |
---|
162 | 183 | static int mall_change(struct net *net, struct sk_buff *in_skb, |
---|
163 | 184 | struct tcf_proto *tp, unsigned long base, |
---|
164 | 185 | u32 handle, struct nlattr **tca, |
---|
165 | | - void **arg, bool ovr, struct netlink_ext_ack *extack) |
---|
| 186 | + void **arg, bool ovr, bool rtnl_held, |
---|
| 187 | + struct netlink_ext_ack *extack) |
---|
166 | 188 | { |
---|
167 | 189 | struct cls_mall_head *head = rtnl_dereference(tp->root); |
---|
168 | 190 | struct nlattr *tb[TCA_MATCHALL_MAX + 1]; |
---|
.. | .. |
---|
176 | 198 | if (head) |
---|
177 | 199 | return -EEXIST; |
---|
178 | 200 | |
---|
179 | | - err = nla_parse_nested(tb, TCA_MATCHALL_MAX, tca[TCA_OPTIONS], |
---|
180 | | - mall_policy, NULL); |
---|
| 201 | + err = nla_parse_nested_deprecated(tb, TCA_MATCHALL_MAX, |
---|
| 202 | + tca[TCA_OPTIONS], mall_policy, NULL); |
---|
181 | 203 | if (err < 0) |
---|
182 | 204 | return err; |
---|
183 | 205 | |
---|
.. | .. |
---|
191 | 213 | if (!new) |
---|
192 | 214 | return -ENOBUFS; |
---|
193 | 215 | |
---|
194 | | - err = tcf_exts_init(&new->exts, TCA_MATCHALL_ACT, 0); |
---|
| 216 | + err = tcf_exts_init(&new->exts, net, TCA_MATCHALL_ACT, 0); |
---|
195 | 217 | if (err) |
---|
196 | 218 | goto err_exts_init; |
---|
197 | 219 | |
---|
.. | .. |
---|
199 | 221 | handle = 1; |
---|
200 | 222 | new->handle = handle; |
---|
201 | 223 | new->flags = flags; |
---|
| 224 | + new->pf = alloc_percpu(struct tc_matchall_pcnt); |
---|
| 225 | + if (!new->pf) { |
---|
| 226 | + err = -ENOMEM; |
---|
| 227 | + goto err_alloc_percpu; |
---|
| 228 | + } |
---|
202 | 229 | |
---|
203 | 230 | err = mall_set_parms(net, tp, new, base, tb, tca[TCA_RATE], ovr, |
---|
204 | 231 | extack); |
---|
.. | .. |
---|
221 | 248 | |
---|
222 | 249 | err_replace_hw_filter: |
---|
223 | 250 | err_set_parms: |
---|
| 251 | + free_percpu(new->pf); |
---|
| 252 | +err_alloc_percpu: |
---|
224 | 253 | tcf_exts_destroy(&new->exts); |
---|
225 | 254 | err_exts_init: |
---|
226 | 255 | kfree(new); |
---|
.. | .. |
---|
228 | 257 | } |
---|
229 | 258 | |
---|
230 | 259 | static int mall_delete(struct tcf_proto *tp, void *arg, bool *last, |
---|
231 | | - struct netlink_ext_ack *extack) |
---|
| 260 | + bool rtnl_held, struct netlink_ext_ack *extack) |
---|
232 | 261 | { |
---|
233 | | - return -EOPNOTSUPP; |
---|
| 262 | + struct cls_mall_head *head = rtnl_dereference(tp->root); |
---|
| 263 | + |
---|
| 264 | + head->deleting = true; |
---|
| 265 | + *last = true; |
---|
| 266 | + return 0; |
---|
234 | 267 | } |
---|
235 | 268 | |
---|
236 | | -static void mall_walk(struct tcf_proto *tp, struct tcf_walker *arg) |
---|
| 269 | +static void mall_walk(struct tcf_proto *tp, struct tcf_walker *arg, |
---|
| 270 | + bool rtnl_held) |
---|
237 | 271 | { |
---|
238 | 272 | struct cls_mall_head *head = rtnl_dereference(tp->root); |
---|
239 | 273 | |
---|
240 | 274 | if (arg->count < arg->skip) |
---|
241 | 275 | goto skip; |
---|
| 276 | + |
---|
| 277 | + if (!head || head->deleting) |
---|
| 278 | + return; |
---|
242 | 279 | if (arg->fn(tp, head, arg) < 0) |
---|
243 | 280 | arg->stop = 1; |
---|
244 | 281 | skip: |
---|
245 | 282 | arg->count++; |
---|
246 | 283 | } |
---|
247 | 284 | |
---|
248 | | -static int mall_reoffload(struct tcf_proto *tp, bool add, tc_setup_cb_t *cb, |
---|
| 285 | +static int mall_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, |
---|
249 | 286 | void *cb_priv, struct netlink_ext_ack *extack) |
---|
250 | 287 | { |
---|
251 | 288 | struct cls_mall_head *head = rtnl_dereference(tp->root); |
---|
.. | .. |
---|
256 | 293 | if (tc_skip_hw(head->flags)) |
---|
257 | 294 | return 0; |
---|
258 | 295 | |
---|
| 296 | + cls_mall.rule = flow_rule_alloc(tcf_exts_num_actions(&head->exts)); |
---|
| 297 | + if (!cls_mall.rule) |
---|
| 298 | + return -ENOMEM; |
---|
| 299 | + |
---|
259 | 300 | tc_cls_common_offload_init(&cls_mall.common, tp, head->flags, extack); |
---|
260 | 301 | cls_mall.command = add ? |
---|
261 | 302 | TC_CLSMATCHALL_REPLACE : TC_CLSMATCHALL_DESTROY; |
---|
262 | | - cls_mall.exts = &head->exts; |
---|
263 | 303 | cls_mall.cookie = (unsigned long)head; |
---|
264 | 304 | |
---|
265 | | - err = cb(TC_SETUP_CLSMATCHALL, &cls_mall, cb_priv); |
---|
| 305 | + err = tc_setup_flow_action(&cls_mall.rule->action, &head->exts); |
---|
266 | 306 | if (err) { |
---|
267 | | - if (add && tc_skip_sw(head->flags)) |
---|
| 307 | + kfree(cls_mall.rule); |
---|
| 308 | + if (add && tc_skip_sw(head->flags)) { |
---|
| 309 | + NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); |
---|
268 | 310 | return err; |
---|
| 311 | + } |
---|
269 | 312 | return 0; |
---|
270 | 313 | } |
---|
271 | 314 | |
---|
272 | | - tc_cls_offload_cnt_update(block, &head->in_hw_count, &head->flags, add); |
---|
| 315 | + err = tc_setup_cb_reoffload(block, tp, add, cb, TC_SETUP_CLSMATCHALL, |
---|
| 316 | + &cls_mall, cb_priv, &head->flags, |
---|
| 317 | + &head->in_hw_count); |
---|
| 318 | + tc_cleanup_flow_action(&cls_mall.rule->action); |
---|
| 319 | + kfree(cls_mall.rule); |
---|
| 320 | + |
---|
| 321 | + if (err) |
---|
| 322 | + return err; |
---|
273 | 323 | |
---|
274 | 324 | return 0; |
---|
275 | 325 | } |
---|
276 | 326 | |
---|
277 | | -static int mall_dump(struct net *net, struct tcf_proto *tp, void *fh, |
---|
278 | | - struct sk_buff *skb, struct tcmsg *t) |
---|
| 327 | +static void mall_stats_hw_filter(struct tcf_proto *tp, |
---|
| 328 | + struct cls_mall_head *head, |
---|
| 329 | + unsigned long cookie) |
---|
279 | 330 | { |
---|
| 331 | + struct tc_cls_matchall_offload cls_mall = {}; |
---|
| 332 | + struct tcf_block *block = tp->chain->block; |
---|
| 333 | + |
---|
| 334 | + tc_cls_common_offload_init(&cls_mall.common, tp, head->flags, NULL); |
---|
| 335 | + cls_mall.command = TC_CLSMATCHALL_STATS; |
---|
| 336 | + cls_mall.cookie = cookie; |
---|
| 337 | + |
---|
| 338 | + tc_setup_cb_call(block, TC_SETUP_CLSMATCHALL, &cls_mall, false, true); |
---|
| 339 | + |
---|
| 340 | + tcf_exts_stats_update(&head->exts, cls_mall.stats.bytes, |
---|
| 341 | + cls_mall.stats.pkts, cls_mall.stats.drops, |
---|
| 342 | + cls_mall.stats.lastused, |
---|
| 343 | + cls_mall.stats.used_hw_stats, |
---|
| 344 | + cls_mall.stats.used_hw_stats_valid); |
---|
| 345 | +} |
---|
| 346 | + |
---|
| 347 | +static int mall_dump(struct net *net, struct tcf_proto *tp, void *fh, |
---|
| 348 | + struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) |
---|
| 349 | +{ |
---|
| 350 | + struct tc_matchall_pcnt gpf = {}; |
---|
280 | 351 | struct cls_mall_head *head = fh; |
---|
281 | 352 | struct nlattr *nest; |
---|
| 353 | + int cpu; |
---|
282 | 354 | |
---|
283 | 355 | if (!head) |
---|
284 | 356 | return skb->len; |
---|
285 | 357 | |
---|
| 358 | + if (!tc_skip_hw(head->flags)) |
---|
| 359 | + mall_stats_hw_filter(tp, head, (unsigned long)head); |
---|
| 360 | + |
---|
286 | 361 | t->tcm_handle = head->handle; |
---|
287 | 362 | |
---|
288 | | - nest = nla_nest_start(skb, TCA_OPTIONS); |
---|
| 363 | + nest = nla_nest_start_noflag(skb, TCA_OPTIONS); |
---|
289 | 364 | if (!nest) |
---|
290 | 365 | goto nla_put_failure; |
---|
291 | 366 | |
---|
.. | .. |
---|
296 | 371 | if (head->flags && nla_put_u32(skb, TCA_MATCHALL_FLAGS, head->flags)) |
---|
297 | 372 | goto nla_put_failure; |
---|
298 | 373 | |
---|
| 374 | + for_each_possible_cpu(cpu) { |
---|
| 375 | + struct tc_matchall_pcnt *pf = per_cpu_ptr(head->pf, cpu); |
---|
| 376 | + |
---|
| 377 | + gpf.rhit += pf->rhit; |
---|
| 378 | + } |
---|
| 379 | + |
---|
| 380 | + if (nla_put_64bit(skb, TCA_MATCHALL_PCNT, |
---|
| 381 | + sizeof(struct tc_matchall_pcnt), |
---|
| 382 | + &gpf, TCA_MATCHALL_PAD)) |
---|
| 383 | + goto nla_put_failure; |
---|
| 384 | + |
---|
299 | 385 | if (tcf_exts_dump(skb, &head->exts)) |
---|
300 | 386 | goto nla_put_failure; |
---|
301 | 387 | |
---|