| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * net/sched/cls_tcindex.c Packet classifier for skb->tc_index |
|---|
| 3 | 4 | * |
|---|
| .. | .. |
|---|
| 10 | 11 | #include <linux/skbuff.h> |
|---|
| 11 | 12 | #include <linux/errno.h> |
|---|
| 12 | 13 | #include <linux/slab.h> |
|---|
| 14 | +#include <linux/refcount.h> |
|---|
| 13 | 15 | #include <net/act_api.h> |
|---|
| 14 | 16 | #include <net/netlink.h> |
|---|
| 15 | 17 | #include <net/pkt_cls.h> |
|---|
| .. | .. |
|---|
| 25 | 27 | #define DEFAULT_HASH_SIZE 64 /* optimized for diffserv */ |
|---|
| 26 | 28 | |
|---|
| 27 | 29 | |
|---|
| 30 | +struct tcindex_data; |
|---|
| 31 | + |
|---|
| 28 | 32 | struct tcindex_filter_result { |
|---|
| 29 | 33 | struct tcf_exts exts; |
|---|
| 30 | 34 | struct tcf_result res; |
|---|
| 35 | + struct tcindex_data *p; |
|---|
| 31 | 36 | struct rcu_work rwork; |
|---|
| 32 | 37 | }; |
|---|
| 33 | 38 | |
|---|
| .. | .. |
|---|
| 48 | 53 | u32 hash; /* hash table size; 0 if undefined */ |
|---|
| 49 | 54 | u32 alloc_hash; /* allocated size */ |
|---|
| 50 | 55 | u32 fall_through; /* 0: only classify if explicit match */ |
|---|
| 56 | + refcount_t refcnt; /* a temporary refcnt for perfect hash */ |
|---|
| 51 | 57 | struct rcu_work rwork; |
|---|
| 52 | 58 | }; |
|---|
| 53 | 59 | |
|---|
| 54 | 60 | static inline int tcindex_filter_is_set(struct tcindex_filter_result *r) |
|---|
| 55 | 61 | { |
|---|
| 56 | 62 | return tcf_exts_has_actions(&r->exts) || r->res.classid; |
|---|
| 63 | +} |
|---|
| 64 | + |
|---|
| 65 | +static void tcindex_data_get(struct tcindex_data *p) |
|---|
| 66 | +{ |
|---|
| 67 | + refcount_inc(&p->refcnt); |
|---|
| 68 | +} |
|---|
| 69 | + |
|---|
| 70 | +static void tcindex_data_put(struct tcindex_data *p) |
|---|
| 71 | +{ |
|---|
| 72 | + if (refcount_dec_and_test(&p->refcnt)) { |
|---|
| 73 | + kfree(p->perfect); |
|---|
| 74 | + kfree(p->h); |
|---|
| 75 | + kfree(p); |
|---|
| 76 | + } |
|---|
| 57 | 77 | } |
|---|
| 58 | 78 | |
|---|
| 59 | 79 | static struct tcindex_filter_result *tcindex_lookup(struct tcindex_data *p, |
|---|
| .. | .. |
|---|
| 131 | 151 | p->mask = 0xffff; |
|---|
| 132 | 152 | p->hash = DEFAULT_HASH_SIZE; |
|---|
| 133 | 153 | p->fall_through = 1; |
|---|
| 154 | + refcount_set(&p->refcnt, 1); /* Paired with tcindex_destroy_work() */ |
|---|
| 134 | 155 | |
|---|
| 135 | 156 | rcu_assign_pointer(tp->root, p); |
|---|
| 136 | 157 | return 0; |
|---|
| .. | .. |
|---|
| 140 | 161 | { |
|---|
| 141 | 162 | tcf_exts_destroy(&r->exts); |
|---|
| 142 | 163 | tcf_exts_put_net(&r->exts); |
|---|
| 164 | + tcindex_data_put(r->p); |
|---|
| 143 | 165 | } |
|---|
| 144 | 166 | |
|---|
| 145 | 167 | static void tcindex_destroy_rexts_work(struct work_struct *work) |
|---|
| .. | .. |
|---|
| 173 | 195 | } |
|---|
| 174 | 196 | |
|---|
| 175 | 197 | static int tcindex_delete(struct tcf_proto *tp, void *arg, bool *last, |
|---|
| 176 | | - struct netlink_ext_ack *extack) |
|---|
| 198 | + bool rtnl_held, struct netlink_ext_ack *extack) |
|---|
| 177 | 199 | { |
|---|
| 178 | 200 | struct tcindex_data *p = rtnl_dereference(tp->root); |
|---|
| 179 | 201 | struct tcindex_filter_result *r = arg; |
|---|
| .. | .. |
|---|
| 211 | 233 | else |
|---|
| 212 | 234 | __tcindex_destroy_fexts(f); |
|---|
| 213 | 235 | } else { |
|---|
| 236 | + tcindex_data_get(p); |
|---|
| 237 | + |
|---|
| 214 | 238 | if (tcf_exts_get_net(&r->exts)) |
|---|
| 215 | 239 | tcf_queue_work(&r->rwork, tcindex_destroy_rexts_work); |
|---|
| 216 | 240 | else |
|---|
| .. | .. |
|---|
| 227 | 251 | struct tcindex_data, |
|---|
| 228 | 252 | rwork); |
|---|
| 229 | 253 | |
|---|
| 230 | | - kfree(p->perfect); |
|---|
| 231 | | - kfree(p->h); |
|---|
| 232 | | - kfree(p); |
|---|
| 254 | + tcindex_data_put(p); |
|---|
| 233 | 255 | } |
|---|
| 234 | 256 | |
|---|
| 235 | 257 | static inline int |
|---|
| .. | .. |
|---|
| 246 | 268 | [TCA_TCINDEX_CLASSID] = { .type = NLA_U32 }, |
|---|
| 247 | 269 | }; |
|---|
| 248 | 270 | |
|---|
| 249 | | -static int tcindex_filter_result_init(struct tcindex_filter_result *r) |
|---|
| 271 | +static int tcindex_filter_result_init(struct tcindex_filter_result *r, |
|---|
| 272 | + struct tcindex_data *p, |
|---|
| 273 | + struct net *net) |
|---|
| 250 | 274 | { |
|---|
| 251 | 275 | memset(r, 0, sizeof(*r)); |
|---|
| 252 | | - return tcf_exts_init(&r->exts, TCA_TCINDEX_ACT, TCA_TCINDEX_POLICE); |
|---|
| 276 | + r->p = p; |
|---|
| 277 | + return tcf_exts_init(&r->exts, net, TCA_TCINDEX_ACT, |
|---|
| 278 | + TCA_TCINDEX_POLICE); |
|---|
| 253 | 279 | } |
|---|
| 280 | + |
|---|
| 281 | +static void tcindex_free_perfect_hash(struct tcindex_data *cp); |
|---|
| 254 | 282 | |
|---|
| 255 | 283 | static void tcindex_partial_destroy_work(struct work_struct *work) |
|---|
| 256 | 284 | { |
|---|
| .. | .. |
|---|
| 258 | 286 | struct tcindex_data, |
|---|
| 259 | 287 | rwork); |
|---|
| 260 | 288 | |
|---|
| 261 | | - kfree(p->perfect); |
|---|
| 289 | + rtnl_lock(); |
|---|
| 290 | + if (p->perfect) |
|---|
| 291 | + tcindex_free_perfect_hash(p); |
|---|
| 262 | 292 | kfree(p); |
|---|
| 293 | + rtnl_unlock(); |
|---|
| 263 | 294 | } |
|---|
| 264 | 295 | |
|---|
| 265 | 296 | static void tcindex_free_perfect_hash(struct tcindex_data *cp) |
|---|
| .. | .. |
|---|
| 281 | 312 | return -ENOMEM; |
|---|
| 282 | 313 | |
|---|
| 283 | 314 | for (i = 0; i < cp->hash; i++) { |
|---|
| 284 | | - err = tcf_exts_init(&cp->perfect[i].exts, |
|---|
| 315 | + err = tcf_exts_init(&cp->perfect[i].exts, net, |
|---|
| 285 | 316 | TCA_TCINDEX_ACT, TCA_TCINDEX_POLICE); |
|---|
| 286 | 317 | if (err < 0) |
|---|
| 287 | 318 | goto errout; |
|---|
| 288 | | -#ifdef CONFIG_NET_CLS_ACT |
|---|
| 289 | | - cp->perfect[i].exts.net = net; |
|---|
| 290 | | -#endif |
|---|
| 319 | + cp->perfect[i].p = cp; |
|---|
| 291 | 320 | } |
|---|
| 292 | 321 | |
|---|
| 293 | 322 | return 0; |
|---|
| .. | .. |
|---|
| 310 | 339 | int err, balloc = 0; |
|---|
| 311 | 340 | struct tcf_exts e; |
|---|
| 312 | 341 | |
|---|
| 313 | | - err = tcf_exts_init(&e, TCA_TCINDEX_ACT, TCA_TCINDEX_POLICE); |
|---|
| 342 | + err = tcf_exts_init(&e, net, TCA_TCINDEX_ACT, TCA_TCINDEX_POLICE); |
|---|
| 314 | 343 | if (err < 0) |
|---|
| 315 | 344 | return err; |
|---|
| 316 | | - err = tcf_exts_validate(net, tp, tb, est, &e, ovr, extack); |
|---|
| 345 | + err = tcf_exts_validate(net, tp, tb, est, &e, ovr, true, extack); |
|---|
| 317 | 346 | if (err < 0) |
|---|
| 318 | 347 | goto errout; |
|---|
| 319 | 348 | |
|---|
| .. | .. |
|---|
| 332 | 361 | cp->alloc_hash = p->alloc_hash; |
|---|
| 333 | 362 | cp->fall_through = p->fall_through; |
|---|
| 334 | 363 | cp->tp = tp; |
|---|
| 364 | + refcount_set(&cp->refcnt, 1); /* Paired with tcindex_destroy_work() */ |
|---|
| 335 | 365 | |
|---|
| 336 | 366 | if (tb[TCA_TCINDEX_HASH]) |
|---|
| 337 | 367 | cp->hash = nla_get_u32(tb[TCA_TCINDEX_HASH]); |
|---|
| .. | .. |
|---|
| 368 | 398 | } |
|---|
| 369 | 399 | cp->h = p->h; |
|---|
| 370 | 400 | |
|---|
| 371 | | - err = tcindex_filter_result_init(&new_filter_result); |
|---|
| 401 | + err = tcindex_filter_result_init(&new_filter_result, cp, net); |
|---|
| 372 | 402 | if (err < 0) |
|---|
| 373 | 403 | goto errout_alloc; |
|---|
| 374 | 404 | if (old_r) |
|---|
| .. | .. |
|---|
| 436 | 466 | goto errout_alloc; |
|---|
| 437 | 467 | f->key = handle; |
|---|
| 438 | 468 | f->next = NULL; |
|---|
| 439 | | - err = tcindex_filter_result_init(&f->result); |
|---|
| 469 | + err = tcindex_filter_result_init(&f->result, cp, net); |
|---|
| 440 | 470 | if (err < 0) { |
|---|
| 441 | 471 | kfree(f); |
|---|
| 442 | 472 | goto errout_alloc; |
|---|
| .. | .. |
|---|
| 449 | 479 | } |
|---|
| 450 | 480 | |
|---|
| 451 | 481 | if (old_r && old_r != r) { |
|---|
| 452 | | - err = tcindex_filter_result_init(old_r); |
|---|
| 482 | + err = tcindex_filter_result_init(old_r, cp, net); |
|---|
| 453 | 483 | if (err < 0) { |
|---|
| 454 | 484 | kfree(f); |
|---|
| 455 | 485 | goto errout_alloc; |
|---|
| .. | .. |
|---|
| 500 | 530 | tcindex_change(struct net *net, struct sk_buff *in_skb, |
|---|
| 501 | 531 | struct tcf_proto *tp, unsigned long base, u32 handle, |
|---|
| 502 | 532 | struct nlattr **tca, void **arg, bool ovr, |
|---|
| 503 | | - struct netlink_ext_ack *extack) |
|---|
| 533 | + bool rtnl_held, struct netlink_ext_ack *extack) |
|---|
| 504 | 534 | { |
|---|
| 505 | 535 | struct nlattr *opt = tca[TCA_OPTIONS]; |
|---|
| 506 | 536 | struct nlattr *tb[TCA_TCINDEX_MAX + 1]; |
|---|
| .. | .. |
|---|
| 510 | 540 | |
|---|
| 511 | 541 | pr_debug("tcindex_change(tp %p,handle 0x%08x,tca %p,arg %p),opt %p," |
|---|
| 512 | 542 | "p %p,r %p,*arg %p\n", |
|---|
| 513 | | - tp, handle, tca, arg, opt, p, r, arg ? *arg : NULL); |
|---|
| 543 | + tp, handle, tca, arg, opt, p, r, *arg); |
|---|
| 514 | 544 | |
|---|
| 515 | 545 | if (!opt) |
|---|
| 516 | 546 | return 0; |
|---|
| 517 | 547 | |
|---|
| 518 | | - err = nla_parse_nested(tb, TCA_TCINDEX_MAX, opt, tcindex_policy, NULL); |
|---|
| 548 | + err = nla_parse_nested_deprecated(tb, TCA_TCINDEX_MAX, opt, |
|---|
| 549 | + tcindex_policy, NULL); |
|---|
| 519 | 550 | if (err < 0) |
|---|
| 520 | 551 | return err; |
|---|
| 521 | 552 | |
|---|
| .. | .. |
|---|
| 523 | 554 | tca[TCA_RATE], ovr, extack); |
|---|
| 524 | 555 | } |
|---|
| 525 | 556 | |
|---|
| 526 | | -static void tcindex_walk(struct tcf_proto *tp, struct tcf_walker *walker) |
|---|
| 557 | +static void tcindex_walk(struct tcf_proto *tp, struct tcf_walker *walker, |
|---|
| 558 | + bool rtnl_held) |
|---|
| 527 | 559 | { |
|---|
| 528 | 560 | struct tcindex_data *p = rtnl_dereference(tp->root); |
|---|
| 529 | 561 | struct tcindex_filter *f, *next; |
|---|
| .. | .. |
|---|
| 559 | 591 | } |
|---|
| 560 | 592 | } |
|---|
| 561 | 593 | |
|---|
| 562 | | -static void tcindex_destroy(struct tcf_proto *tp, |
|---|
| 594 | +static void tcindex_destroy(struct tcf_proto *tp, bool rtnl_held, |
|---|
| 563 | 595 | struct netlink_ext_ack *extack) |
|---|
| 564 | 596 | { |
|---|
| 565 | 597 | struct tcindex_data *p = rtnl_dereference(tp->root); |
|---|
| .. | .. |
|---|
| 570 | 602 | if (p->perfect) { |
|---|
| 571 | 603 | for (i = 0; i < p->hash; i++) { |
|---|
| 572 | 604 | struct tcindex_filter_result *r = p->perfect + i; |
|---|
| 605 | + |
|---|
| 606 | + /* tcf_queue_work() does not guarantee the ordering we |
|---|
| 607 | + * want, so we have to take this refcnt temporarily to |
|---|
| 608 | + * ensure 'p' is freed after all tcindex_filter_result |
|---|
| 609 | + * here. Imperfect hash does not need this, because it |
|---|
| 610 | + * uses linked lists rather than an array. |
|---|
| 611 | + */ |
|---|
| 612 | + tcindex_data_get(p); |
|---|
| 573 | 613 | |
|---|
| 574 | 614 | tcf_unbind_filter(tp, &r->res); |
|---|
| 575 | 615 | if (tcf_exts_get_net(&r->exts)) |
|---|
| .. | .. |
|---|
| 586 | 626 | |
|---|
| 587 | 627 | for (f = rtnl_dereference(p->h[i]); f; f = next) { |
|---|
| 588 | 628 | next = rtnl_dereference(f->next); |
|---|
| 589 | | - tcindex_delete(tp, &f->result, &last, NULL); |
|---|
| 629 | + tcindex_delete(tp, &f->result, &last, rtnl_held, NULL); |
|---|
| 590 | 630 | } |
|---|
| 591 | 631 | } |
|---|
| 592 | 632 | |
|---|
| .. | .. |
|---|
| 595 | 635 | |
|---|
| 596 | 636 | |
|---|
| 597 | 637 | static int tcindex_dump(struct net *net, struct tcf_proto *tp, void *fh, |
|---|
| 598 | | - struct sk_buff *skb, struct tcmsg *t) |
|---|
| 638 | + struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) |
|---|
| 599 | 639 | { |
|---|
| 600 | 640 | struct tcindex_data *p = rtnl_dereference(tp->root); |
|---|
| 601 | 641 | struct tcindex_filter_result *r = fh; |
|---|
| .. | .. |
|---|
| 605 | 645 | tp, fh, skb, t, p, r); |
|---|
| 606 | 646 | pr_debug("p->perfect %p p->h %p\n", p->perfect, p->h); |
|---|
| 607 | 647 | |
|---|
| 608 | | - nest = nla_nest_start(skb, TCA_OPTIONS); |
|---|
| 648 | + nest = nla_nest_start_noflag(skb, TCA_OPTIONS); |
|---|
| 609 | 649 | if (nest == NULL) |
|---|
| 610 | 650 | goto nla_put_failure; |
|---|
| 611 | 651 | |
|---|