| .. | .. |
|---|
| 36 | 36 | #include <linux/refcount.h> |
|---|
| 37 | 37 | #include <linux/mlx5/fs.h> |
|---|
| 38 | 38 | #include <linux/rhashtable.h> |
|---|
| 39 | +#include <linux/llist.h> |
|---|
| 40 | +#include <steering/fs_dr.h> |
|---|
| 41 | + |
|---|
| 42 | +#define FDB_TC_MAX_CHAIN 3 |
|---|
| 43 | +#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) |
|---|
| 44 | +#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) |
|---|
| 45 | + |
|---|
| 46 | +/* The index of the last real chain (FT) + 1 as chain zero is valid as well */ |
|---|
| 47 | +#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) |
|---|
| 48 | + |
|---|
| 49 | +#define FDB_TC_MAX_PRIO 16 |
|---|
| 50 | +#define FDB_TC_LEVELS_PER_PRIO 2 |
|---|
| 51 | + |
|---|
| 52 | +struct mlx5_modify_hdr { |
|---|
| 53 | + enum mlx5_flow_namespace_type ns_type; |
|---|
| 54 | + union { |
|---|
| 55 | + struct mlx5_fs_dr_action action; |
|---|
| 56 | + u32 id; |
|---|
| 57 | + }; |
|---|
| 58 | +}; |
|---|
| 59 | + |
|---|
| 60 | +struct mlx5_pkt_reformat { |
|---|
| 61 | + enum mlx5_flow_namespace_type ns_type; |
|---|
| 62 | + int reformat_type; /* from mlx5_ifc */ |
|---|
| 63 | + union { |
|---|
| 64 | + struct mlx5_fs_dr_action action; |
|---|
| 65 | + u32 id; |
|---|
| 66 | + }; |
|---|
| 67 | +}; |
|---|
| 68 | + |
|---|
| 69 | +/* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only, |
|---|
| 70 | + * and those are in parallel to one another when going over them to connect |
|---|
| 71 | + * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one |
|---|
| 72 | + * parallel namespace will not automatically connect to the first flow table |
|---|
| 73 | + * found in any prio in any next namespace, but skip the entire containing |
|---|
| 74 | + * TYPE_PRIO_CHAINS prio. |
|---|
| 75 | + * |
|---|
| 76 | + * This is used to implement tc chains, each chain of prios is a different |
|---|
| 77 | + * namespace inside a containing TYPE_PRIO_CHAINS prio. |
|---|
| 78 | + */ |
|---|
| 39 | 79 | |
|---|
| 40 | 80 | enum fs_node_type { |
|---|
| 41 | 81 | FS_TYPE_NAMESPACE, |
|---|
| 42 | 82 | FS_TYPE_PRIO, |
|---|
| 83 | + FS_TYPE_PRIO_CHAINS, |
|---|
| 43 | 84 | FS_TYPE_FLOW_TABLE, |
|---|
| 44 | 85 | FS_TYPE_FLOW_GROUP, |
|---|
| 45 | 86 | FS_TYPE_FLOW_ENTRY, |
|---|
| .. | .. |
|---|
| 54 | 95 | FS_FT_FDB = 0X4, |
|---|
| 55 | 96 | FS_FT_SNIFFER_RX = 0X5, |
|---|
| 56 | 97 | FS_FT_SNIFFER_TX = 0X6, |
|---|
| 57 | | - FS_FT_MAX_TYPE = FS_FT_SNIFFER_TX, |
|---|
| 98 | + FS_FT_RDMA_RX = 0X7, |
|---|
| 99 | + FS_FT_RDMA_TX = 0X8, |
|---|
| 100 | + FS_FT_MAX_TYPE = FS_FT_RDMA_TX, |
|---|
| 58 | 101 | }; |
|---|
| 59 | 102 | |
|---|
| 60 | 103 | enum fs_flow_table_op_mod { |
|---|
| .. | .. |
|---|
| 66 | 109 | FS_FTE_STATUS_EXISTING = 1UL << 0, |
|---|
| 67 | 110 | }; |
|---|
| 68 | 111 | |
|---|
| 112 | +enum mlx5_flow_steering_mode { |
|---|
| 113 | + MLX5_FLOW_STEERING_MODE_DMFS, |
|---|
| 114 | + MLX5_FLOW_STEERING_MODE_SMFS |
|---|
| 115 | +}; |
|---|
| 116 | + |
|---|
| 69 | 117 | struct mlx5_flow_steering { |
|---|
| 70 | 118 | struct mlx5_core_dev *dev; |
|---|
| 71 | | - struct kmem_cache *fgs_cache; |
|---|
| 119 | + enum mlx5_flow_steering_mode mode; |
|---|
| 120 | + struct kmem_cache *fgs_cache; |
|---|
| 72 | 121 | struct kmem_cache *ftes_cache; |
|---|
| 73 | 122 | struct mlx5_flow_root_namespace *root_ns; |
|---|
| 74 | 123 | struct mlx5_flow_root_namespace *fdb_root_ns; |
|---|
| 124 | + struct mlx5_flow_namespace **fdb_sub_ns; |
|---|
| 75 | 125 | struct mlx5_flow_root_namespace **esw_egress_root_ns; |
|---|
| 76 | 126 | struct mlx5_flow_root_namespace **esw_ingress_root_ns; |
|---|
| 77 | 127 | struct mlx5_flow_root_namespace *sniffer_tx_root_ns; |
|---|
| 78 | 128 | struct mlx5_flow_root_namespace *sniffer_rx_root_ns; |
|---|
| 129 | + struct mlx5_flow_root_namespace *rdma_rx_root_ns; |
|---|
| 130 | + struct mlx5_flow_root_namespace *rdma_tx_root_ns; |
|---|
| 79 | 131 | struct mlx5_flow_root_namespace *egress_root_ns; |
|---|
| 80 | 132 | }; |
|---|
| 81 | 133 | |
|---|
| .. | .. |
|---|
| 96 | 148 | |
|---|
| 97 | 149 | struct mlx5_flow_rule { |
|---|
| 98 | 150 | struct fs_node node; |
|---|
| 151 | + struct mlx5_flow_table *ft; |
|---|
| 99 | 152 | struct mlx5_flow_destination dest_attr; |
|---|
| 100 | 153 | /* next_ft should be accessed under chain_lock and only of |
|---|
| 101 | 154 | * destination type is FWD_NEXT_fT. |
|---|
| .. | .. |
|---|
| 112 | 165 | /* Type of children is mlx5_flow_group */ |
|---|
| 113 | 166 | struct mlx5_flow_table { |
|---|
| 114 | 167 | struct fs_node node; |
|---|
| 168 | + struct mlx5_fs_dr_table fs_dr_table; |
|---|
| 115 | 169 | u32 id; |
|---|
| 116 | 170 | u16 vport; |
|---|
| 117 | 171 | unsigned int max_fte; |
|---|
| .. | .. |
|---|
| 123 | 177 | unsigned int required_groups; |
|---|
| 124 | 178 | unsigned int group_size; |
|---|
| 125 | 179 | unsigned int num_groups; |
|---|
| 180 | + unsigned int max_fte; |
|---|
| 126 | 181 | } autogroup; |
|---|
| 127 | 182 | /* Protect fwd_rules */ |
|---|
| 128 | 183 | struct mutex lock; |
|---|
| .. | .. |
|---|
| 130 | 185 | struct list_head fwd_rules; |
|---|
| 131 | 186 | u32 flags; |
|---|
| 132 | 187 | struct rhltable fgs_hash; |
|---|
| 133 | | -}; |
|---|
| 134 | | - |
|---|
| 135 | | -struct mlx5_fc_cache { |
|---|
| 136 | | - u64 packets; |
|---|
| 137 | | - u64 bytes; |
|---|
| 138 | | - u64 lastuse; |
|---|
| 139 | | -}; |
|---|
| 140 | | - |
|---|
| 141 | | -struct mlx5_fc { |
|---|
| 142 | | - struct rb_node node; |
|---|
| 143 | | - struct list_head list; |
|---|
| 144 | | - |
|---|
| 145 | | - /* last{packets,bytes} members are used when calculating the delta since |
|---|
| 146 | | - * last reading |
|---|
| 147 | | - */ |
|---|
| 148 | | - u64 lastpackets; |
|---|
| 149 | | - u64 lastbytes; |
|---|
| 150 | | - |
|---|
| 151 | | - u32 id; |
|---|
| 152 | | - bool deleted; |
|---|
| 153 | | - bool aging; |
|---|
| 154 | | - |
|---|
| 155 | | - struct mlx5_fc_cache cache ____cacheline_aligned_in_smp; |
|---|
| 188 | + enum mlx5_flow_table_miss_action def_miss_action; |
|---|
| 189 | + struct mlx5_flow_namespace *ns; |
|---|
| 156 | 190 | }; |
|---|
| 157 | 191 | |
|---|
| 158 | 192 | struct mlx5_ft_underlay_qp { |
|---|
| .. | .. |
|---|
| 160 | 194 | u32 qpn; |
|---|
| 161 | 195 | }; |
|---|
| 162 | 196 | |
|---|
| 163 | | -#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_800 |
|---|
| 197 | +#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_a00 |
|---|
| 164 | 198 | /* Calculate the fte_match_param length and without the reserved length. |
|---|
| 165 | 199 | * Make sure the reserved field is the last. |
|---|
| 166 | 200 | */ |
|---|
| .. | .. |
|---|
| 175 | 209 | /* Type of children is mlx5_flow_rule */ |
|---|
| 176 | 210 | struct fs_fte { |
|---|
| 177 | 211 | struct fs_node node; |
|---|
| 212 | + struct mlx5_fs_dr_rule fs_dr_rule; |
|---|
| 178 | 213 | u32 val[MLX5_ST_SZ_DW_MATCH_PARAM]; |
|---|
| 179 | 214 | u32 dests_size; |
|---|
| 180 | 215 | u32 index; |
|---|
| 216 | + struct mlx5_flow_context flow_context; |
|---|
| 181 | 217 | struct mlx5_flow_act action; |
|---|
| 182 | 218 | enum fs_fte_status status; |
|---|
| 183 | 219 | struct mlx5_fc *counter; |
|---|
| 184 | 220 | struct rhash_head hash; |
|---|
| 221 | + int modify_mask; |
|---|
| 185 | 222 | }; |
|---|
| 186 | 223 | |
|---|
| 187 | 224 | /* Type of children is mlx5_flow_table/namespace */ |
|---|
| .. | .. |
|---|
| 197 | 234 | struct mlx5_flow_namespace { |
|---|
| 198 | 235 | /* parent == NULL => root ns */ |
|---|
| 199 | 236 | struct fs_node node; |
|---|
| 237 | + enum mlx5_flow_table_miss_action def_miss_action; |
|---|
| 200 | 238 | }; |
|---|
| 201 | 239 | |
|---|
| 202 | 240 | struct mlx5_flow_group_mask { |
|---|
| .. | .. |
|---|
| 207 | 245 | /* Type of children is fs_fte */ |
|---|
| 208 | 246 | struct mlx5_flow_group { |
|---|
| 209 | 247 | struct fs_node node; |
|---|
| 248 | + struct mlx5_fs_dr_matcher fs_dr_matcher; |
|---|
| 210 | 249 | struct mlx5_flow_group_mask mask; |
|---|
| 211 | 250 | u32 start_index; |
|---|
| 212 | 251 | u32 max_ftes; |
|---|
| .. | .. |
|---|
| 218 | 257 | |
|---|
| 219 | 258 | struct mlx5_flow_root_namespace { |
|---|
| 220 | 259 | struct mlx5_flow_namespace ns; |
|---|
| 260 | + enum mlx5_flow_steering_mode mode; |
|---|
| 261 | + struct mlx5_fs_dr_domain fs_dr_domain; |
|---|
| 221 | 262 | enum fs_flow_table_type table_type; |
|---|
| 222 | 263 | struct mlx5_core_dev *dev; |
|---|
| 223 | 264 | struct mlx5_flow_table *root_ft; |
|---|
| .. | .. |
|---|
| 234 | 275 | unsigned long delay); |
|---|
| 235 | 276 | void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev, |
|---|
| 236 | 277 | unsigned long interval); |
|---|
| 278 | + |
|---|
| 279 | +const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void); |
|---|
| 280 | + |
|---|
| 281 | +int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns, |
|---|
| 282 | + struct mlx5_flow_root_namespace *peer_ns); |
|---|
| 283 | + |
|---|
| 284 | +int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns, |
|---|
| 285 | + enum mlx5_flow_steering_mode mode); |
|---|
| 237 | 286 | |
|---|
| 238 | 287 | int mlx5_init_fs(struct mlx5_core_dev *dev); |
|---|
| 239 | 288 | void mlx5_cleanup_fs(struct mlx5_core_dev *dev); |
|---|
| .. | .. |
|---|
| 280 | 329 | (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \ |
|---|
| 281 | 330 | (type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) : \ |
|---|
| 282 | 331 | (type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) : \ |
|---|
| 283 | | - (BUILD_BUG_ON_ZERO(FS_FT_SNIFFER_TX != FS_FT_MAX_TYPE))\ |
|---|
| 332 | + (type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) : \ |
|---|
| 333 | + (type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) : \ |
|---|
| 334 | + (BUILD_BUG_ON_ZERO(FS_FT_RDMA_TX != FS_FT_MAX_TYPE))\ |
|---|
| 284 | 335 | ) |
|---|
| 285 | 336 | |
|---|
| 286 | 337 | #endif |
|---|