.. | .. |
---|
13 | 13 | #include <net/netns/generic.h> |
---|
14 | 14 | |
---|
15 | 15 | struct tcf_idrinfo { |
---|
16 | | - spinlock_t lock; |
---|
| 16 | + struct mutex lock; |
---|
17 | 17 | struct idr action_idr; |
---|
18 | 18 | struct net *net; |
---|
19 | 19 | }; |
---|
.. | .. |
---|
23 | 23 | struct tc_action { |
---|
24 | 24 | const struct tc_action_ops *ops; |
---|
25 | 25 | __u32 type; /* for backward compat(TCA_OLD_COMPAT) */ |
---|
26 | | - __u32 order; |
---|
27 | 26 | struct tcf_idrinfo *idrinfo; |
---|
28 | 27 | |
---|
29 | 28 | u32 tcfa_index; |
---|
.. | .. |
---|
32 | 31 | int tcfa_action; |
---|
33 | 32 | struct tcf_t tcfa_tm; |
---|
34 | 33 | struct gnet_stats_basic_packed tcfa_bstats; |
---|
| 34 | + struct gnet_stats_basic_packed tcfa_bstats_hw; |
---|
35 | 35 | struct gnet_stats_queue tcfa_qstats; |
---|
36 | 36 | struct net_rate_estimator __rcu *tcfa_rate_est; |
---|
37 | 37 | spinlock_t tcfa_lock; |
---|
38 | 38 | struct gnet_stats_basic_cpu __percpu *cpu_bstats; |
---|
| 39 | + struct gnet_stats_basic_cpu __percpu *cpu_bstats_hw; |
---|
39 | 40 | struct gnet_stats_queue __percpu *cpu_qstats; |
---|
40 | 41 | struct tc_cookie __rcu *act_cookie; |
---|
41 | | - struct tcf_chain *goto_chain; |
---|
| 42 | + struct tcf_chain __rcu *goto_chain; |
---|
| 43 | + u32 tcfa_flags; |
---|
| 44 | + u8 hw_stats; |
---|
| 45 | + u8 used_hw_stats; |
---|
| 46 | + bool used_hw_stats_valid; |
---|
42 | 47 | }; |
---|
43 | 48 | #define tcf_index common.tcfa_index |
---|
44 | 49 | #define tcf_refcnt common.tcfa_refcnt |
---|
.. | .. |
---|
49 | 54 | #define tcf_qstats common.tcfa_qstats |
---|
50 | 55 | #define tcf_rate_est common.tcfa_rate_est |
---|
51 | 56 | #define tcf_lock common.tcfa_lock |
---|
| 57 | + |
---|
| 58 | +#define TCA_ACT_HW_STATS_ANY (TCA_ACT_HW_STATS_IMMEDIATE | \ |
---|
| 59 | + TCA_ACT_HW_STATS_DELAYED) |
---|
52 | 60 | |
---|
53 | 61 | /* Update lastuse only if needed, to avoid dirtying a cache line. |
---|
54 | 62 | * We use a temp variable to avoid fetching jiffies twice. |
---|
.. | .. |
---|
77 | 85 | #define ACT_P_CREATED 1 |
---|
78 | 86 | #define ACT_P_DELETED 1 |
---|
79 | 87 | |
---|
| 88 | +typedef void (*tc_action_priv_destructor)(void *priv); |
---|
| 89 | + |
---|
80 | 90 | struct tc_action_ops { |
---|
81 | 91 | struct list_head head; |
---|
82 | 92 | char kind[IFNAMSIZ]; |
---|
83 | | - __u32 type; /* TBD to match kind */ |
---|
| 93 | + enum tca_id id; /* identifier should match kind */ |
---|
84 | 94 | size_t size; |
---|
85 | 95 | struct module *owner; |
---|
86 | 96 | int (*act)(struct sk_buff *, const struct tc_action *, |
---|
87 | 97 | struct tcf_result *); /* called under RCU BH lock*/ |
---|
88 | 98 | int (*dump)(struct sk_buff *, struct tc_action *, int, int); |
---|
89 | 99 | void (*cleanup)(struct tc_action *); |
---|
90 | | - int (*lookup)(struct net *net, struct tc_action **a, u32 index, |
---|
91 | | - struct netlink_ext_ack *extack); |
---|
| 100 | + int (*lookup)(struct net *net, struct tc_action **a, u32 index); |
---|
92 | 101 | int (*init)(struct net *net, struct nlattr *nla, |
---|
93 | 102 | struct nlattr *est, struct tc_action **act, int ovr, |
---|
94 | | - int bind, bool rtnl_held, |
---|
95 | | - struct netlink_ext_ack *extack); |
---|
| 103 | + int bind, bool rtnl_held, struct tcf_proto *tp, |
---|
| 104 | + u32 flags, struct netlink_ext_ack *extack); |
---|
96 | 105 | int (*walk)(struct net *, struct sk_buff *, |
---|
97 | 106 | struct netlink_callback *, int, |
---|
98 | 107 | const struct tc_action_ops *, |
---|
99 | 108 | struct netlink_ext_ack *); |
---|
100 | | - void (*stats_update)(struct tc_action *, u64, u32, u64); |
---|
| 109 | + void (*stats_update)(struct tc_action *, u64, u64, u64, u64, bool); |
---|
101 | 110 | size_t (*get_fill_size)(const struct tc_action *act); |
---|
102 | | - struct net_device *(*get_dev)(const struct tc_action *a); |
---|
103 | | - void (*put_dev)(struct net_device *dev); |
---|
| 111 | + struct net_device *(*get_dev)(const struct tc_action *a, |
---|
| 112 | + tc_action_priv_destructor *destructor); |
---|
| 113 | + struct psample_group * |
---|
| 114 | + (*get_psample_group)(const struct tc_action *a, |
---|
| 115 | + tc_action_priv_destructor *destructor); |
---|
104 | 116 | }; |
---|
105 | 117 | |
---|
106 | 118 | struct tc_action_net { |
---|
.. | .. |
---|
119 | 131 | return -ENOMEM; |
---|
120 | 132 | tn->ops = ops; |
---|
121 | 133 | tn->idrinfo->net = net; |
---|
122 | | - spin_lock_init(&tn->idrinfo->lock); |
---|
| 134 | + mutex_init(&tn->idrinfo->lock); |
---|
123 | 135 | idr_init(&tn->idrinfo->action_idr); |
---|
124 | 136 | return err; |
---|
125 | 137 | } |
---|
.. | .. |
---|
149 | 161 | int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index); |
---|
150 | 162 | int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est, |
---|
151 | 163 | struct tc_action **a, const struct tc_action_ops *ops, |
---|
152 | | - int bind, bool cpustats); |
---|
153 | | -void tcf_idr_insert(struct tc_action_net *tn, struct tc_action *a); |
---|
154 | | - |
---|
| 164 | + int bind, bool cpustats, u32 flags); |
---|
| 165 | +int tcf_idr_create_from_flags(struct tc_action_net *tn, u32 index, |
---|
| 166 | + struct nlattr *est, struct tc_action **a, |
---|
| 167 | + const struct tc_action_ops *ops, int bind, |
---|
| 168 | + u32 flags); |
---|
| 169 | +void tcf_idr_insert_many(struct tc_action *actions[]); |
---|
155 | 170 | void tcf_idr_cleanup(struct tc_action_net *tn, u32 index); |
---|
156 | 171 | int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index, |
---|
157 | 172 | struct tc_action **a, int bind); |
---|
158 | | -int __tcf_idr_release(struct tc_action *a, bool bind, bool strict); |
---|
159 | | - |
---|
160 | | -static inline int tcf_idr_release(struct tc_action *a, bool bind) |
---|
161 | | -{ |
---|
162 | | - return __tcf_idr_release(a, bind, false); |
---|
163 | | -} |
---|
| 173 | +int tcf_idr_release(struct tc_action *a, bool bind); |
---|
164 | 174 | |
---|
165 | 175 | int tcf_register_action(struct tc_action_ops *a, struct pernet_operations *ops); |
---|
166 | 176 | int tcf_unregister_action(struct tc_action_ops *a, |
---|
.. | .. |
---|
170 | 180 | int nr_actions, struct tcf_result *res); |
---|
171 | 181 | int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, |
---|
172 | 182 | struct nlattr *est, char *name, int ovr, int bind, |
---|
173 | | - struct tc_action *actions[], size_t *attr_size, |
---|
| 183 | + struct tc_action *actions[], int init_res[], size_t *attr_size, |
---|
174 | 184 | bool rtnl_held, struct netlink_ext_ack *extack); |
---|
| 185 | +struct tc_action_ops *tc_action_load_ops(char *name, struct nlattr *nla, |
---|
| 186 | + bool rtnl_held, |
---|
| 187 | + struct netlink_ext_ack *extack); |
---|
175 | 188 | struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp, |
---|
176 | 189 | struct nlattr *nla, struct nlattr *est, |
---|
177 | 190 | char *name, int ovr, int bind, |
---|
| 191 | + struct tc_action_ops *a_o, int *init_res, |
---|
178 | 192 | bool rtnl_held, |
---|
179 | 193 | struct netlink_ext_ack *extack); |
---|
180 | 194 | int tcf_action_dump(struct sk_buff *skb, struct tc_action *actions[], int bind, |
---|
181 | | - int ref); |
---|
| 195 | + int ref, bool terse); |
---|
182 | 196 | int tcf_action_dump_old(struct sk_buff *skb, struct tc_action *a, int, int); |
---|
183 | 197 | int tcf_action_dump_1(struct sk_buff *skb, struct tc_action *a, int, int); |
---|
| 198 | + |
---|
| 199 | +static inline void tcf_action_update_bstats(struct tc_action *a, |
---|
| 200 | + struct sk_buff *skb) |
---|
| 201 | +{ |
---|
| 202 | + if (likely(a->cpu_bstats)) { |
---|
| 203 | + bstats_cpu_update(this_cpu_ptr(a->cpu_bstats), skb); |
---|
| 204 | + return; |
---|
| 205 | + } |
---|
| 206 | + spin_lock(&a->tcfa_lock); |
---|
| 207 | + bstats_update(&a->tcfa_bstats, skb); |
---|
| 208 | + spin_unlock(&a->tcfa_lock); |
---|
| 209 | +} |
---|
| 210 | + |
---|
| 211 | +static inline void tcf_action_inc_drop_qstats(struct tc_action *a) |
---|
| 212 | +{ |
---|
| 213 | + if (likely(a->cpu_qstats)) { |
---|
| 214 | + qstats_drop_inc(this_cpu_ptr(a->cpu_qstats)); |
---|
| 215 | + return; |
---|
| 216 | + } |
---|
| 217 | + spin_lock(&a->tcfa_lock); |
---|
| 218 | + qstats_drop_inc(&a->tcfa_qstats); |
---|
| 219 | + spin_unlock(&a->tcfa_lock); |
---|
| 220 | +} |
---|
| 221 | + |
---|
| 222 | +static inline void tcf_action_inc_overlimit_qstats(struct tc_action *a) |
---|
| 223 | +{ |
---|
| 224 | + if (likely(a->cpu_qstats)) { |
---|
| 225 | + qstats_overlimit_inc(this_cpu_ptr(a->cpu_qstats)); |
---|
| 226 | + return; |
---|
| 227 | + } |
---|
| 228 | + spin_lock(&a->tcfa_lock); |
---|
| 229 | + qstats_overlimit_inc(&a->tcfa_qstats); |
---|
| 230 | + spin_unlock(&a->tcfa_lock); |
---|
| 231 | +} |
---|
| 232 | + |
---|
| 233 | +void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets, |
---|
| 234 | + u64 drops, bool hw); |
---|
184 | 235 | int tcf_action_copy_stats(struct sk_buff *, struct tc_action *, int); |
---|
185 | 236 | |
---|
| 237 | +int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, |
---|
| 238 | + struct tcf_chain **handle, |
---|
| 239 | + struct netlink_ext_ack *newchain); |
---|
| 240 | +struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, |
---|
| 241 | + struct tcf_chain *newchain); |
---|
186 | 242 | #endif /* CONFIG_NET_CLS_ACT */ |
---|
187 | 243 | |
---|
188 | 244 | static inline void tcf_action_stats_update(struct tc_action *a, u64 bytes, |
---|
189 | | - u64 packets, u64 lastuse) |
---|
| 245 | + u64 packets, u64 drops, |
---|
| 246 | + u64 lastuse, bool hw) |
---|
190 | 247 | { |
---|
191 | 248 | #ifdef CONFIG_NET_CLS_ACT |
---|
192 | 249 | if (!a->ops->stats_update) |
---|
193 | 250 | return; |
---|
194 | 251 | |
---|
195 | | - a->ops->stats_update(a, bytes, packets, lastuse); |
---|
| 252 | + a->ops->stats_update(a, bytes, packets, drops, lastuse, hw); |
---|
196 | 253 | #endif |
---|
197 | 254 | } |
---|
198 | 255 | |
---|
199 | | -#ifdef CONFIG_NET_CLS_ACT |
---|
200 | | -int tc_setup_cb_egdev_register(const struct net_device *dev, |
---|
201 | | - tc_setup_cb_t *cb, void *cb_priv); |
---|
202 | | -void tc_setup_cb_egdev_unregister(const struct net_device *dev, |
---|
203 | | - tc_setup_cb_t *cb, void *cb_priv); |
---|
204 | | -int tc_setup_cb_egdev_call(const struct net_device *dev, |
---|
205 | | - enum tc_setup_type type, void *type_data, |
---|
206 | | - bool err_stop); |
---|
207 | | -#else |
---|
208 | | -static inline |
---|
209 | | -int tc_setup_cb_egdev_register(const struct net_device *dev, |
---|
210 | | - tc_setup_cb_t *cb, void *cb_priv) |
---|
211 | | -{ |
---|
212 | | - return 0; |
---|
213 | | -} |
---|
214 | | - |
---|
215 | | -static inline |
---|
216 | | -void tc_setup_cb_egdev_unregister(const struct net_device *dev, |
---|
217 | | - tc_setup_cb_t *cb, void *cb_priv) |
---|
218 | | -{ |
---|
219 | | -} |
---|
220 | | - |
---|
221 | | -static inline |
---|
222 | | -int tc_setup_cb_egdev_call(const struct net_device *dev, |
---|
223 | | - enum tc_setup_type type, void *type_data, |
---|
224 | | - bool err_stop) |
---|
225 | | -{ |
---|
226 | | - return 0; |
---|
227 | | -} |
---|
228 | | -#endif |
---|
229 | 256 | |
---|
230 | 257 | #endif |
---|