.. | .. |
---|
1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
---|
2 | 2 | #define _GNU_SOURCE |
---|
3 | 3 | #include <getopt.h> |
---|
| 4 | +#include <limits.h> |
---|
4 | 5 | #include <string.h> |
---|
5 | 6 | #include <poll.h> |
---|
6 | 7 | #include <sys/eventfd.h> |
---|
.. | .. |
---|
17 | 18 | #include <linux/virtio.h> |
---|
18 | 19 | #include <linux/virtio_ring.h> |
---|
19 | 20 | #include "../../drivers/vhost/test.h" |
---|
| 21 | + |
---|
| 22 | +#define RANDOM_BATCH -1 |
---|
20 | 23 | |
---|
21 | 24 | /* Unused */ |
---|
22 | 25 | void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; |
---|
.. | .. |
---|
42 | 45 | size_t buf_size; |
---|
43 | 46 | struct vhost_memory *mem; |
---|
44 | 47 | }; |
---|
| 48 | + |
---|
| 49 | +static const struct vhost_vring_file no_backend = { .fd = -1 }, |
---|
| 50 | + backend = { .fd = 1 }; |
---|
| 51 | +static const struct vhost_vring_state null_state = {}; |
---|
45 | 52 | |
---|
46 | 53 | bool vq_notify(struct virtqueue *vq) |
---|
47 | 54 | { |
---|
.. | .. |
---|
88 | 95 | assert(r >= 0); |
---|
89 | 96 | } |
---|
90 | 97 | |
---|
| 98 | +static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) |
---|
| 99 | +{ |
---|
| 100 | + if (info->vq) |
---|
| 101 | + vring_del_virtqueue(info->vq); |
---|
| 102 | + |
---|
| 103 | + memset(info->ring, 0, vring_size(num, 4096)); |
---|
| 104 | + vring_init(&info->vring, num, info->ring, 4096); |
---|
| 105 | + info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true, |
---|
| 106 | + false, vq_notify, vq_callback, "test"); |
---|
| 107 | + assert(info->vq); |
---|
| 108 | + info->vq->priv = info; |
---|
| 109 | +} |
---|
| 110 | + |
---|
91 | 111 | static void vq_info_add(struct vdev_info *dev, int num) |
---|
92 | 112 | { |
---|
93 | 113 | struct vq_info *info = &dev->vqs[dev->nvqs]; |
---|
.. | .. |
---|
97 | 117 | info->call = eventfd(0, EFD_NONBLOCK); |
---|
98 | 118 | r = posix_memalign(&info->ring, 4096, vring_size(num, 4096)); |
---|
99 | 119 | assert(r >= 0); |
---|
100 | | - memset(info->ring, 0, vring_size(num, 4096)); |
---|
101 | | - vring_init(&info->vring, num, info->ring, 4096); |
---|
102 | | - info->vq = vring_new_virtqueue(info->idx, |
---|
103 | | - info->vring.num, 4096, &dev->vdev, |
---|
104 | | - true, false, info->ring, |
---|
105 | | - vq_notify, vq_callback, "test"); |
---|
106 | | - assert(info->vq); |
---|
107 | | - info->vq->priv = info; |
---|
| 120 | + vq_reset(info, num, &dev->vdev); |
---|
108 | 121 | vhost_vq_setup(dev, info); |
---|
109 | 122 | dev->fds[info->idx].fd = info->call; |
---|
110 | 123 | dev->fds[info->idx].events = POLLIN; |
---|
.. | .. |
---|
116 | 129 | int r; |
---|
117 | 130 | memset(dev, 0, sizeof *dev); |
---|
118 | 131 | dev->vdev.features = features; |
---|
| 132 | + INIT_LIST_HEAD(&dev->vdev.vqs); |
---|
| 133 | + spin_lock_init(&dev->vdev.vqs_list_lock); |
---|
119 | 134 | dev->buf_size = 1024; |
---|
120 | 135 | dev->buf = malloc(dev->buf_size); |
---|
121 | 136 | assert(dev->buf); |
---|
.. | .. |
---|
152 | 167 | } |
---|
153 | 168 | |
---|
154 | 169 | static void run_test(struct vdev_info *dev, struct vq_info *vq, |
---|
155 | | - bool delayed, int bufs) |
---|
| 170 | + bool delayed, int batch, int reset_n, int bufs) |
---|
156 | 171 | { |
---|
157 | 172 | struct scatterlist sl; |
---|
158 | | - long started = 0, completed = 0; |
---|
159 | | - long completed_before; |
---|
| 173 | + long started = 0, completed = 0, next_reset = reset_n; |
---|
| 174 | + long completed_before, started_before; |
---|
160 | 175 | int r, test = 1; |
---|
161 | 176 | unsigned len; |
---|
162 | 177 | long long spurious = 0; |
---|
| 178 | + const bool random_batch = batch == RANDOM_BATCH; |
---|
| 179 | + |
---|
163 | 180 | r = ioctl(dev->control, VHOST_TEST_RUN, &test); |
---|
164 | 181 | assert(r >= 0); |
---|
| 182 | + if (!reset_n) { |
---|
| 183 | + next_reset = INT_MAX; |
---|
| 184 | + } |
---|
| 185 | + |
---|
165 | 186 | for (;;) { |
---|
166 | 187 | virtqueue_disable_cb(vq->vq); |
---|
167 | 188 | completed_before = completed; |
---|
| 189 | + started_before = started; |
---|
168 | 190 | do { |
---|
169 | | - if (started < bufs) { |
---|
| 191 | + const bool reset = completed > next_reset; |
---|
| 192 | + if (random_batch) |
---|
| 193 | + batch = (random() % vq->vring.num) + 1; |
---|
| 194 | + |
---|
| 195 | + while (started < bufs && |
---|
| 196 | + (started - completed) < batch) { |
---|
170 | 197 | sg_init_one(&sl, dev->buf, dev->buf_size); |
---|
171 | 198 | r = virtqueue_add_outbuf(vq->vq, &sl, 1, |
---|
172 | 199 | dev->buf + started, |
---|
173 | 200 | GFP_ATOMIC); |
---|
174 | | - if (likely(r == 0)) { |
---|
175 | | - ++started; |
---|
176 | | - if (unlikely(!virtqueue_kick(vq->vq))) |
---|
| 201 | + if (unlikely(r != 0)) { |
---|
| 202 | + if (r == -ENOSPC && |
---|
| 203 | + started > started_before) |
---|
| 204 | + r = 0; |
---|
| 205 | + else |
---|
177 | 206 | r = -1; |
---|
| 207 | + break; |
---|
178 | 208 | } |
---|
179 | | - } else |
---|
| 209 | + |
---|
| 210 | + ++started; |
---|
| 211 | + |
---|
| 212 | + if (unlikely(!virtqueue_kick(vq->vq))) { |
---|
| 213 | + r = -1; |
---|
| 214 | + break; |
---|
| 215 | + } |
---|
| 216 | + } |
---|
| 217 | + |
---|
| 218 | + if (started >= bufs) |
---|
180 | 219 | r = -1; |
---|
181 | 220 | |
---|
| 221 | + if (reset) { |
---|
| 222 | + r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, |
---|
| 223 | + &no_backend); |
---|
| 224 | + assert(!r); |
---|
| 225 | + } |
---|
| 226 | + |
---|
182 | 227 | /* Flush out completed bufs if any */ |
---|
183 | | - if (virtqueue_get_buf(vq->vq, &len)) { |
---|
| 228 | + while (virtqueue_get_buf(vq->vq, &len)) { |
---|
184 | 229 | ++completed; |
---|
185 | 230 | r = 0; |
---|
186 | 231 | } |
---|
187 | 232 | |
---|
| 233 | + if (reset) { |
---|
| 234 | + struct vhost_vring_state s = { .index = 0 }; |
---|
| 235 | + |
---|
| 236 | + vq_reset(vq, vq->vring.num, &dev->vdev); |
---|
| 237 | + |
---|
| 238 | + r = ioctl(dev->control, VHOST_GET_VRING_BASE, |
---|
| 239 | + &s); |
---|
| 240 | + assert(!r); |
---|
| 241 | + |
---|
| 242 | + s.num = 0; |
---|
| 243 | + r = ioctl(dev->control, VHOST_SET_VRING_BASE, |
---|
| 244 | + &null_state); |
---|
| 245 | + assert(!r); |
---|
| 246 | + |
---|
| 247 | + r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, |
---|
| 248 | + &backend); |
---|
| 249 | + assert(!r); |
---|
| 250 | + |
---|
| 251 | + started = completed; |
---|
| 252 | + while (completed > next_reset) |
---|
| 253 | + next_reset += completed; |
---|
| 254 | + } |
---|
188 | 255 | } while (r == 0); |
---|
189 | | - if (completed == completed_before) |
---|
| 256 | + if (completed == completed_before && started == started_before) |
---|
190 | 257 | ++spurious; |
---|
191 | 258 | assert(completed <= bufs); |
---|
192 | 259 | assert(started <= bufs); |
---|
.. | .. |
---|
203 | 270 | test = 0; |
---|
204 | 271 | r = ioctl(dev->control, VHOST_TEST_RUN, &test); |
---|
205 | 272 | assert(r >= 0); |
---|
206 | | - fprintf(stderr, "spurious wakeups: 0x%llx\n", spurious); |
---|
| 273 | + fprintf(stderr, |
---|
| 274 | + "spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n", |
---|
| 275 | + spurious, started, completed); |
---|
207 | 276 | } |
---|
208 | 277 | |
---|
209 | 278 | const char optstring[] = "h"; |
---|
.. | .. |
---|
245 | 314 | .val = 'd', |
---|
246 | 315 | }, |
---|
247 | 316 | { |
---|
| 317 | + .name = "batch", |
---|
| 318 | + .val = 'b', |
---|
| 319 | + .has_arg = required_argument, |
---|
| 320 | + }, |
---|
| 321 | + { |
---|
| 322 | + .name = "reset", |
---|
| 323 | + .val = 'r', |
---|
| 324 | + .has_arg = optional_argument, |
---|
| 325 | + }, |
---|
| 326 | + { |
---|
248 | 327 | } |
---|
249 | 328 | }; |
---|
250 | 329 | |
---|
.. | .. |
---|
255 | 334 | " [--no-event-idx]" |
---|
256 | 335 | " [--no-virtio-1]" |
---|
257 | 336 | " [--delayed-interrupt]" |
---|
| 337 | + " [--batch=random/N]" |
---|
| 338 | + " [--reset=N]" |
---|
258 | 339 | "\n"); |
---|
259 | 340 | } |
---|
260 | 341 | |
---|
.. | .. |
---|
263 | 344 | struct vdev_info dev; |
---|
264 | 345 | unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | |
---|
265 | 346 | (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1); |
---|
| 347 | + long batch = 1, reset = 0; |
---|
266 | 348 | int o; |
---|
267 | 349 | bool delayed = false; |
---|
268 | 350 | |
---|
.. | .. |
---|
289 | 371 | case 'D': |
---|
290 | 372 | delayed = true; |
---|
291 | 373 | break; |
---|
| 374 | + case 'b': |
---|
| 375 | + if (0 == strcmp(optarg, "random")) { |
---|
| 376 | + batch = RANDOM_BATCH; |
---|
| 377 | + } else { |
---|
| 378 | + batch = strtol(optarg, NULL, 10); |
---|
| 379 | + assert(batch > 0); |
---|
| 380 | + assert(batch < (long)INT_MAX + 1); |
---|
| 381 | + } |
---|
| 382 | + break; |
---|
| 383 | + case 'r': |
---|
| 384 | + if (!optarg) { |
---|
| 385 | + reset = 1; |
---|
| 386 | + } else { |
---|
| 387 | + reset = strtol(optarg, NULL, 10); |
---|
| 388 | + assert(reset > 0); |
---|
| 389 | + assert(reset < (long)INT_MAX + 1); |
---|
| 390 | + } |
---|
| 391 | + break; |
---|
292 | 392 | default: |
---|
293 | 393 | assert(0); |
---|
294 | 394 | break; |
---|
.. | .. |
---|
298 | 398 | done: |
---|
299 | 399 | vdev_info_init(&dev, features); |
---|
300 | 400 | vq_info_add(&dev, 256); |
---|
301 | | - run_test(&dev, &dev.vqs[0], delayed, 0x100000); |
---|
| 401 | + run_test(&dev, &dev.vqs[0], delayed, batch, reset, 0x100000); |
---|
302 | 402 | return 0; |
---|
303 | 403 | } |
---|