.. | .. |
---|
33 | 33 | static DECLARE_RWSEM(bpf_devs_lock); |
---|
34 | 34 | |
---|
35 | 35 | struct bpf_offload_dev { |
---|
| 36 | + const struct bpf_prog_offload_ops *ops; |
---|
36 | 37 | struct list_head netdevs; |
---|
| 38 | + void *priv; |
---|
37 | 39 | }; |
---|
38 | 40 | |
---|
39 | 41 | struct bpf_offload_netdev { |
---|
.. | .. |
---|
106 | 108 | err = -EINVAL; |
---|
107 | 109 | goto err_unlock; |
---|
108 | 110 | } |
---|
| 111 | + offload->offdev = ondev->offdev; |
---|
109 | 112 | prog->aux->offload = offload; |
---|
110 | 113 | list_add_tail(&offload->offloads, &ondev->progs); |
---|
111 | 114 | dev_put(offload->netdev); |
---|
.. | .. |
---|
121 | 124 | return err; |
---|
122 | 125 | } |
---|
123 | 126 | |
---|
124 | | -static int __bpf_offload_ndo(struct bpf_prog *prog, enum bpf_netdev_command cmd, |
---|
125 | | - struct netdev_bpf *data) |
---|
| 127 | +int bpf_prog_offload_verifier_prep(struct bpf_prog *prog) |
---|
126 | 128 | { |
---|
127 | | - struct bpf_prog_offload *offload = prog->aux->offload; |
---|
128 | | - struct net_device *netdev; |
---|
| 129 | + struct bpf_prog_offload *offload; |
---|
| 130 | + int ret = -ENODEV; |
---|
129 | 131 | |
---|
130 | | - ASSERT_RTNL(); |
---|
| 132 | + down_read(&bpf_devs_lock); |
---|
| 133 | + offload = prog->aux->offload; |
---|
| 134 | + if (offload) { |
---|
| 135 | + ret = offload->offdev->ops->prepare(prog); |
---|
| 136 | + offload->dev_state = !ret; |
---|
| 137 | + } |
---|
| 138 | + up_read(&bpf_devs_lock); |
---|
131 | 139 | |
---|
132 | | - if (!offload) |
---|
133 | | - return -ENODEV; |
---|
134 | | - netdev = offload->netdev; |
---|
135 | | - |
---|
136 | | - data->command = cmd; |
---|
137 | | - |
---|
138 | | - return netdev->netdev_ops->ndo_bpf(netdev, data); |
---|
139 | | -} |
---|
140 | | - |
---|
141 | | -int bpf_prog_offload_verifier_prep(struct bpf_verifier_env *env) |
---|
142 | | -{ |
---|
143 | | - struct netdev_bpf data = {}; |
---|
144 | | - int err; |
---|
145 | | - |
---|
146 | | - data.verifier.prog = env->prog; |
---|
147 | | - |
---|
148 | | - rtnl_lock(); |
---|
149 | | - err = __bpf_offload_ndo(env->prog, BPF_OFFLOAD_VERIFIER_PREP, &data); |
---|
150 | | - if (err) |
---|
151 | | - goto exit_unlock; |
---|
152 | | - |
---|
153 | | - env->prog->aux->offload->dev_ops = data.verifier.ops; |
---|
154 | | - env->prog->aux->offload->dev_state = true; |
---|
155 | | -exit_unlock: |
---|
156 | | - rtnl_unlock(); |
---|
157 | | - return err; |
---|
| 140 | + return ret; |
---|
158 | 141 | } |
---|
159 | 142 | |
---|
160 | 143 | int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, |
---|
.. | .. |
---|
166 | 149 | down_read(&bpf_devs_lock); |
---|
167 | 150 | offload = env->prog->aux->offload; |
---|
168 | 151 | if (offload) |
---|
169 | | - ret = offload->dev_ops->insn_hook(env, insn_idx, prev_insn_idx); |
---|
| 152 | + ret = offload->offdev->ops->insn_hook(env, insn_idx, |
---|
| 153 | + prev_insn_idx); |
---|
170 | 154 | up_read(&bpf_devs_lock); |
---|
171 | 155 | |
---|
172 | 156 | return ret; |
---|
173 | 157 | } |
---|
174 | 158 | |
---|
| 159 | +int bpf_prog_offload_finalize(struct bpf_verifier_env *env) |
---|
| 160 | +{ |
---|
| 161 | + struct bpf_prog_offload *offload; |
---|
| 162 | + int ret = -ENODEV; |
---|
| 163 | + |
---|
| 164 | + down_read(&bpf_devs_lock); |
---|
| 165 | + offload = env->prog->aux->offload; |
---|
| 166 | + if (offload) { |
---|
| 167 | + if (offload->offdev->ops->finalize) |
---|
| 168 | + ret = offload->offdev->ops->finalize(env); |
---|
| 169 | + else |
---|
| 170 | + ret = 0; |
---|
| 171 | + } |
---|
| 172 | + up_read(&bpf_devs_lock); |
---|
| 173 | + |
---|
| 174 | + return ret; |
---|
| 175 | +} |
---|
| 176 | + |
---|
| 177 | +void |
---|
| 178 | +bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, |
---|
| 179 | + struct bpf_insn *insn) |
---|
| 180 | +{ |
---|
| 181 | + const struct bpf_prog_offload_ops *ops; |
---|
| 182 | + struct bpf_prog_offload *offload; |
---|
| 183 | + int ret = -EOPNOTSUPP; |
---|
| 184 | + |
---|
| 185 | + down_read(&bpf_devs_lock); |
---|
| 186 | + offload = env->prog->aux->offload; |
---|
| 187 | + if (offload) { |
---|
| 188 | + ops = offload->offdev->ops; |
---|
| 189 | + if (!offload->opt_failed && ops->replace_insn) |
---|
| 190 | + ret = ops->replace_insn(env, off, insn); |
---|
| 191 | + offload->opt_failed |= ret; |
---|
| 192 | + } |
---|
| 193 | + up_read(&bpf_devs_lock); |
---|
| 194 | +} |
---|
| 195 | + |
---|
| 196 | +void |
---|
| 197 | +bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) |
---|
| 198 | +{ |
---|
| 199 | + struct bpf_prog_offload *offload; |
---|
| 200 | + int ret = -EOPNOTSUPP; |
---|
| 201 | + |
---|
| 202 | + down_read(&bpf_devs_lock); |
---|
| 203 | + offload = env->prog->aux->offload; |
---|
| 204 | + if (offload) { |
---|
| 205 | + if (!offload->opt_failed && offload->offdev->ops->remove_insns) |
---|
| 206 | + ret = offload->offdev->ops->remove_insns(env, off, cnt); |
---|
| 207 | + offload->opt_failed |= ret; |
---|
| 208 | + } |
---|
| 209 | + up_read(&bpf_devs_lock); |
---|
| 210 | +} |
---|
| 211 | + |
---|
175 | 212 | static void __bpf_prog_offload_destroy(struct bpf_prog *prog) |
---|
176 | 213 | { |
---|
177 | 214 | struct bpf_prog_offload *offload = prog->aux->offload; |
---|
178 | | - struct netdev_bpf data = {}; |
---|
179 | | - |
---|
180 | | - data.offload.prog = prog; |
---|
181 | 215 | |
---|
182 | 216 | if (offload->dev_state) |
---|
183 | | - WARN_ON(__bpf_offload_ndo(prog, BPF_OFFLOAD_DESTROY, &data)); |
---|
| 217 | + offload->offdev->ops->destroy(prog); |
---|
184 | 218 | |
---|
185 | 219 | /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ |
---|
186 | 220 | bpf_prog_free_id(prog, true); |
---|
.. | .. |
---|
192 | 226 | |
---|
193 | 227 | void bpf_prog_offload_destroy(struct bpf_prog *prog) |
---|
194 | 228 | { |
---|
195 | | - rtnl_lock(); |
---|
196 | 229 | down_write(&bpf_devs_lock); |
---|
197 | 230 | if (prog->aux->offload) |
---|
198 | 231 | __bpf_prog_offload_destroy(prog); |
---|
199 | 232 | up_write(&bpf_devs_lock); |
---|
200 | | - rtnl_unlock(); |
---|
201 | 233 | } |
---|
202 | 234 | |
---|
203 | 235 | static int bpf_prog_offload_translate(struct bpf_prog *prog) |
---|
204 | 236 | { |
---|
205 | | - struct netdev_bpf data = {}; |
---|
206 | | - int ret; |
---|
| 237 | + struct bpf_prog_offload *offload; |
---|
| 238 | + int ret = -ENODEV; |
---|
207 | 239 | |
---|
208 | | - data.offload.prog = prog; |
---|
209 | | - |
---|
210 | | - rtnl_lock(); |
---|
211 | | - ret = __bpf_offload_ndo(prog, BPF_OFFLOAD_TRANSLATE, &data); |
---|
212 | | - rtnl_unlock(); |
---|
| 240 | + down_read(&bpf_devs_lock); |
---|
| 241 | + offload = prog->aux->offload; |
---|
| 242 | + if (offload) |
---|
| 243 | + ret = offload->offdev->ops->translate(prog); |
---|
| 244 | + up_read(&bpf_devs_lock); |
---|
213 | 245 | |
---|
214 | 246 | return ret; |
---|
215 | 247 | } |
---|
.. | .. |
---|
270 | 302 | struct inode *ns_inode; |
---|
271 | 303 | struct path ns_path; |
---|
272 | 304 | char __user *uinsns; |
---|
273 | | - void *res; |
---|
| 305 | + int res; |
---|
274 | 306 | u32 ulen; |
---|
275 | 307 | |
---|
276 | 308 | res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); |
---|
277 | | - if (IS_ERR(res)) { |
---|
| 309 | + if (res) { |
---|
278 | 310 | if (!info->ifindex) |
---|
279 | 311 | return -ENODEV; |
---|
280 | | - return PTR_ERR(res); |
---|
| 312 | + return res; |
---|
281 | 313 | } |
---|
282 | 314 | |
---|
283 | 315 | down_read(&bpf_devs_lock); |
---|
.. | .. |
---|
494 | 526 | }; |
---|
495 | 527 | struct inode *ns_inode; |
---|
496 | 528 | struct path ns_path; |
---|
497 | | - void *res; |
---|
| 529 | + int res; |
---|
498 | 530 | |
---|
499 | 531 | res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args); |
---|
500 | | - if (IS_ERR(res)) { |
---|
| 532 | + if (res) { |
---|
501 | 533 | if (!info->ifindex) |
---|
502 | 534 | return -ENODEV; |
---|
503 | | - return PTR_ERR(res); |
---|
| 535 | + return res; |
---|
504 | 536 | } |
---|
505 | 537 | |
---|
506 | 538 | ns_inode = ns_path.dentry->d_inode; |
---|
.. | .. |
---|
637 | 669 | } |
---|
638 | 670 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister); |
---|
639 | 671 | |
---|
640 | | -struct bpf_offload_dev *bpf_offload_dev_create(void) |
---|
| 672 | +struct bpf_offload_dev * |
---|
| 673 | +bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv) |
---|
641 | 674 | { |
---|
642 | 675 | struct bpf_offload_dev *offdev; |
---|
643 | 676 | int err; |
---|
.. | .. |
---|
657 | 690 | if (!offdev) |
---|
658 | 691 | return ERR_PTR(-ENOMEM); |
---|
659 | 692 | |
---|
| 693 | + offdev->ops = ops; |
---|
| 694 | + offdev->priv = priv; |
---|
660 | 695 | INIT_LIST_HEAD(&offdev->netdevs); |
---|
661 | 696 | |
---|
662 | 697 | return offdev; |
---|
.. | .. |
---|
669 | 704 | kfree(offdev); |
---|
670 | 705 | } |
---|
671 | 706 | EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy); |
---|
| 707 | + |
---|
| 708 | +void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev) |
---|
| 709 | +{ |
---|
| 710 | + return offdev->priv; |
---|
| 711 | +} |
---|
| 712 | +EXPORT_SYMBOL_GPL(bpf_offload_dev_priv); |
---|