.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
---|
1 | 2 | /* |
---|
2 | | - * Copyright 2015 Amazon.com, Inc. or its affiliates. |
---|
3 | | - * |
---|
4 | | - * This software is available to you under a choice of one of two |
---|
5 | | - * licenses. You may choose to be licensed under the terms of the GNU |
---|
6 | | - * General Public License (GPL) Version 2, available from the file |
---|
7 | | - * COPYING in the main directory of this source tree, or the |
---|
8 | | - * BSD license below: |
---|
9 | | - * |
---|
10 | | - * Redistribution and use in source and binary forms, with or |
---|
11 | | - * without modification, are permitted provided that the following |
---|
12 | | - * conditions are met: |
---|
13 | | - * |
---|
14 | | - * - Redistributions of source code must retain the above |
---|
15 | | - * copyright notice, this list of conditions and the following |
---|
16 | | - * disclaimer. |
---|
17 | | - * |
---|
18 | | - * - Redistributions in binary form must reproduce the above |
---|
19 | | - * copyright notice, this list of conditions and the following |
---|
20 | | - * disclaimer in the documentation and/or other materials |
---|
21 | | - * provided with the distribution. |
---|
22 | | - * |
---|
23 | | - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
---|
24 | | - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
---|
25 | | - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
---|
26 | | - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
---|
27 | | - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
---|
28 | | - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
---|
29 | | - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
---|
30 | | - * SOFTWARE. |
---|
| 3 | + * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. |
---|
31 | 4 | */ |
---|
32 | 5 | |
---|
33 | 6 | #ifndef ENA_ETH_COM_H_ |
---|
.. | .. |
---|
67 | 40 | enum ena_eth_io_l4_proto_index l4_proto; |
---|
68 | 41 | bool l3_csum_err; |
---|
69 | 42 | bool l4_csum_err; |
---|
| 43 | + u8 l4_csum_checked; |
---|
70 | 44 | /* fragmented packet */ |
---|
71 | 45 | bool frag; |
---|
72 | 46 | u32 hash; |
---|
73 | 47 | u16 descs; |
---|
74 | 48 | int max_bufs; |
---|
| 49 | + u8 pkt_offset; |
---|
75 | 50 | }; |
---|
76 | 51 | |
---|
77 | 52 | int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, |
---|
.. | .. |
---|
86 | 61 | struct ena_com_buf *ena_buf, |
---|
87 | 62 | u16 req_id); |
---|
88 | 63 | |
---|
89 | | -int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id); |
---|
90 | | - |
---|
91 | 64 | bool ena_com_cq_empty(struct ena_com_io_cq *io_cq); |
---|
92 | 65 | |
---|
93 | 66 | static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq, |
---|
.. | .. |
---|
96 | 69 | writel(intr_reg->intr_control, io_cq->unmask_reg); |
---|
97 | 70 | } |
---|
98 | 71 | |
---|
99 | | -static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq) |
---|
| 72 | +static inline int ena_com_free_q_entries(struct ena_com_io_sq *io_sq) |
---|
100 | 73 | { |
---|
101 | 74 | u16 tail, next_to_comp, cnt; |
---|
102 | 75 | |
---|
.. | .. |
---|
107 | 80 | return io_sq->q_depth - 1 - cnt; |
---|
108 | 81 | } |
---|
109 | 82 | |
---|
| 83 | +/* Check if the submission queue has enough space to hold required_buffers */ |
---|
| 84 | +static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq, |
---|
| 85 | + u16 required_buffers) |
---|
| 86 | +{ |
---|
| 87 | + int temp; |
---|
| 88 | + |
---|
| 89 | + if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) |
---|
| 90 | + return ena_com_free_q_entries(io_sq) >= required_buffers; |
---|
| 91 | + |
---|
| 92 | + /* This calculation doesn't need to be 100% accurate. So to reduce |
---|
| 93 | + * the calculation overhead just Subtract 2 lines from the free descs |
---|
| 94 | + * (one for the header line and one to compensate the devision |
---|
| 95 | + * down calculation. |
---|
| 96 | + */ |
---|
| 97 | + temp = required_buffers / io_sq->llq_info.descs_per_entry + 2; |
---|
| 98 | + |
---|
| 99 | + return ena_com_free_q_entries(io_sq) > temp; |
---|
| 100 | +} |
---|
| 101 | + |
---|
| 102 | +static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq, |
---|
| 103 | + struct ena_com_tx_ctx *ena_tx_ctx) |
---|
| 104 | +{ |
---|
| 105 | + if (!ena_tx_ctx->meta_valid) |
---|
| 106 | + return false; |
---|
| 107 | + |
---|
| 108 | + return !!memcmp(&io_sq->cached_tx_meta, |
---|
| 109 | + &ena_tx_ctx->ena_meta, |
---|
| 110 | + sizeof(struct ena_com_tx_meta)); |
---|
| 111 | +} |
---|
| 112 | + |
---|
| 113 | +static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq) |
---|
| 114 | +{ |
---|
| 115 | + return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) && |
---|
| 116 | + io_sq->llq_info.max_entries_in_tx_burst > 0; |
---|
| 117 | +} |
---|
| 118 | + |
---|
| 119 | +static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq, |
---|
| 120 | + struct ena_com_tx_ctx *ena_tx_ctx) |
---|
| 121 | +{ |
---|
| 122 | + struct ena_com_llq_info *llq_info; |
---|
| 123 | + int descs_after_first_entry; |
---|
| 124 | + int num_entries_needed = 1; |
---|
| 125 | + u16 num_descs; |
---|
| 126 | + |
---|
| 127 | + if (!is_llq_max_tx_burst_exists(io_sq)) |
---|
| 128 | + return false; |
---|
| 129 | + |
---|
| 130 | + llq_info = &io_sq->llq_info; |
---|
| 131 | + num_descs = ena_tx_ctx->num_bufs; |
---|
| 132 | + |
---|
| 133 | + if (llq_info->disable_meta_caching || |
---|
| 134 | + unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx))) |
---|
| 135 | + ++num_descs; |
---|
| 136 | + |
---|
| 137 | + if (num_descs > llq_info->descs_num_before_header) { |
---|
| 138 | + descs_after_first_entry = num_descs - llq_info->descs_num_before_header; |
---|
| 139 | + num_entries_needed += DIV_ROUND_UP(descs_after_first_entry, |
---|
| 140 | + llq_info->descs_per_entry); |
---|
| 141 | + } |
---|
| 142 | + |
---|
| 143 | + pr_debug("Queue: %d num_descs: %d num_entries_needed: %d\n", io_sq->qid, |
---|
| 144 | + num_descs, num_entries_needed); |
---|
| 145 | + |
---|
| 146 | + return num_entries_needed > io_sq->entries_in_tx_burst_left; |
---|
| 147 | +} |
---|
| 148 | + |
---|
110 | 149 | static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq) |
---|
111 | 150 | { |
---|
112 | | - u16 tail; |
---|
| 151 | + u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst; |
---|
| 152 | + u16 tail = io_sq->tail; |
---|
113 | 153 | |
---|
114 | | - tail = io_sq->tail; |
---|
115 | | - |
---|
116 | | - pr_debug("write submission queue doorbell for queue: %d tail: %d\n", |
---|
| 154 | + pr_debug("Write submission queue doorbell for queue: %d tail: %d\n", |
---|
117 | 155 | io_sq->qid, tail); |
---|
118 | 156 | |
---|
119 | 157 | writel(tail, io_sq->db_addr); |
---|
| 158 | + |
---|
| 159 | + if (is_llq_max_tx_burst_exists(io_sq)) { |
---|
| 160 | + pr_debug("Reset available entries in tx burst for queue %d to %d\n", |
---|
| 161 | + io_sq->qid, max_entries_in_tx_burst); |
---|
| 162 | + io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst; |
---|
| 163 | + } |
---|
120 | 164 | |
---|
121 | 165 | return 0; |
---|
122 | 166 | } |
---|
.. | .. |
---|
126 | 170 | u16 unreported_comp, head; |
---|
127 | 171 | bool need_update; |
---|
128 | 172 | |
---|
129 | | - head = io_cq->head; |
---|
130 | | - unreported_comp = head - io_cq->last_head_update; |
---|
131 | | - need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); |
---|
| 173 | + if (unlikely(io_cq->cq_head_db_reg)) { |
---|
| 174 | + head = io_cq->head; |
---|
| 175 | + unreported_comp = head - io_cq->last_head_update; |
---|
| 176 | + need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); |
---|
132 | 177 | |
---|
133 | | - if (io_cq->cq_head_db_reg && need_update) { |
---|
134 | | - pr_debug("Write completion queue doorbell for queue %d: head: %d\n", |
---|
135 | | - io_cq->qid, head); |
---|
136 | | - writel(head, io_cq->cq_head_db_reg); |
---|
137 | | - io_cq->last_head_update = head; |
---|
| 178 | + if (unlikely(need_update)) { |
---|
| 179 | + pr_debug("Write completion queue doorbell for queue %d: head: %d\n", |
---|
| 180 | + io_cq->qid, head); |
---|
| 181 | + writel(head, io_cq->cq_head_db_reg); |
---|
| 182 | + io_cq->last_head_update = head; |
---|
| 183 | + } |
---|
138 | 184 | } |
---|
139 | 185 | |
---|
140 | 186 | return 0; |
---|
.. | .. |
---|
159 | 205 | io_sq->next_to_comp += elem; |
---|
160 | 206 | } |
---|
161 | 207 | |
---|
| 208 | +static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq) |
---|
| 209 | +{ |
---|
| 210 | + io_cq->head++; |
---|
| 211 | + |
---|
| 212 | + /* Switch phase bit in case of wrap around */ |
---|
| 213 | + if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0)) |
---|
| 214 | + io_cq->phase ^= 1; |
---|
| 215 | +} |
---|
| 216 | + |
---|
| 217 | +static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, |
---|
| 218 | + u16 *req_id) |
---|
| 219 | +{ |
---|
| 220 | + u8 expected_phase, cdesc_phase; |
---|
| 221 | + struct ena_eth_io_tx_cdesc *cdesc; |
---|
| 222 | + u16 masked_head; |
---|
| 223 | + |
---|
| 224 | + masked_head = io_cq->head & (io_cq->q_depth - 1); |
---|
| 225 | + expected_phase = io_cq->phase; |
---|
| 226 | + |
---|
| 227 | + cdesc = (struct ena_eth_io_tx_cdesc *) |
---|
| 228 | + ((uintptr_t)io_cq->cdesc_addr.virt_addr + |
---|
| 229 | + (masked_head * io_cq->cdesc_entry_size_in_bytes)); |
---|
| 230 | + |
---|
| 231 | + /* When the current completion descriptor phase isn't the same as the |
---|
| 232 | + * expected, it mean that the device still didn't update |
---|
| 233 | + * this completion. |
---|
| 234 | + */ |
---|
| 235 | + cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK; |
---|
| 236 | + if (cdesc_phase != expected_phase) |
---|
| 237 | + return -EAGAIN; |
---|
| 238 | + |
---|
| 239 | + dma_rmb(); |
---|
| 240 | + |
---|
| 241 | + *req_id = READ_ONCE(cdesc->req_id); |
---|
| 242 | + if (unlikely(*req_id >= io_cq->q_depth)) { |
---|
| 243 | + pr_err("Invalid req id %d\n", cdesc->req_id); |
---|
| 244 | + return -EINVAL; |
---|
| 245 | + } |
---|
| 246 | + |
---|
| 247 | + ena_com_cq_inc_head(io_cq); |
---|
| 248 | + |
---|
| 249 | + return 0; |
---|
| 250 | +} |
---|
| 251 | + |
---|
162 | 252 | #endif /* ENA_ETH_COM_H_ */ |
---|