| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Hierarchical Budget Worst-case Fair Weighted Fair Queueing |
|---|
| 3 | 4 | * (B-WF2Q+): hierarchical scheduling algorithm by which the BFQ I/O |
|---|
| 4 | 5 | * scheduler schedules generic entities. The latter can represent |
|---|
| 5 | 6 | * either single bfq queues (associated with processes) or groups of |
|---|
| 6 | 7 | * bfq queues (associated with cgroups). |
|---|
| 7 | | - * |
|---|
| 8 | | - * This program is free software; you can redistribute it and/or |
|---|
| 9 | | - * modify it under the terms of the GNU General Public License as |
|---|
| 10 | | - * published by the Free Software Foundation; either version 2 of the |
|---|
| 11 | | - * License, or (at your option) any later version. |
|---|
| 12 | | - * |
|---|
| 13 | | - * This program is distributed in the hope that it will be useful, |
|---|
| 14 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 15 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
|---|
| 16 | | - * General Public License for more details. |
|---|
| 17 | 8 | */ |
|---|
| 18 | 9 | #include "bfq-iosched.h" |
|---|
| 19 | 10 | |
|---|
| .. | .. |
|---|
| 44 | 35 | BFQ_DEFAULT_GRP_CLASS - 1; |
|---|
| 45 | 36 | } |
|---|
| 46 | 37 | |
|---|
| 38 | +unsigned int bfq_tot_busy_queues(struct bfq_data *bfqd) |
|---|
| 39 | +{ |
|---|
| 40 | + return bfqd->busy_queues[0] + bfqd->busy_queues[1] + |
|---|
| 41 | + bfqd->busy_queues[2]; |
|---|
| 42 | +} |
|---|
| 43 | + |
|---|
| 47 | 44 | static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, |
|---|
| 48 | 45 | bool expiration); |
|---|
| 49 | 46 | |
|---|
| .. | .. |
|---|
| 53 | 50 | * bfq_update_next_in_service - update sd->next_in_service |
|---|
| 54 | 51 | * @sd: sched_data for which to perform the update. |
|---|
| 55 | 52 | * @new_entity: if not NULL, pointer to the entity whose activation, |
|---|
| 56 | | - * requeueing or repositionig triggered the invocation of |
|---|
| 53 | + * requeueing or repositioning triggered the invocation of |
|---|
| 57 | 54 | * this function. |
|---|
| 58 | 55 | * @expiration: id true, this function is being invoked after the |
|---|
| 59 | 56 | * expiration of the in-service entity |
|---|
| .. | .. |
|---|
| 84 | 81 | |
|---|
| 85 | 82 | /* |
|---|
| 86 | 83 | * If this update is triggered by the activation, requeueing |
|---|
| 87 | | - * or repositiong of an entity that does not coincide with |
|---|
| 84 | + * or repositioning of an entity that does not coincide with |
|---|
| 88 | 85 | * sd->next_in_service, then a full lookup in the active tree |
|---|
| 89 | 86 | * can be avoided. In fact, it is enough to check whether the |
|---|
| 90 | 87 | * just-modified entity has the same priority as |
|---|
| .. | .. |
|---|
| 280 | 277 | */ |
|---|
| 281 | 278 | static u64 bfq_delta(unsigned long service, unsigned long weight) |
|---|
| 282 | 279 | { |
|---|
| 283 | | - u64 d = (u64)service << WFQ_SERVICE_SHIFT; |
|---|
| 284 | | - |
|---|
| 285 | | - do_div(d, weight); |
|---|
| 286 | | - return d; |
|---|
| 280 | + return div64_ul((u64)service << WFQ_SERVICE_SHIFT, weight); |
|---|
| 287 | 281 | } |
|---|
| 288 | 282 | |
|---|
| 289 | 283 | /** |
|---|
| .. | .. |
|---|
| 651 | 645 | { |
|---|
| 652 | 646 | struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); |
|---|
| 653 | 647 | |
|---|
| 654 | | - entity->on_st = false; |
|---|
| 648 | + entity->on_st_or_in_serv = false; |
|---|
| 655 | 649 | st->wsum -= entity->weight; |
|---|
| 656 | 650 | if (bfqq && !is_in_service) |
|---|
| 657 | 651 | bfq_put_queue(bfqq); |
|---|
| .. | .. |
|---|
| 731 | 725 | struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); |
|---|
| 732 | 726 | unsigned int prev_weight, new_weight; |
|---|
| 733 | 727 | struct bfq_data *bfqd = NULL; |
|---|
| 734 | | - struct rb_root *root; |
|---|
| 728 | + struct rb_root_cached *root; |
|---|
| 735 | 729 | #ifdef CONFIG_BFQ_GROUP_IOSCHED |
|---|
| 736 | 730 | struct bfq_sched_data *sd; |
|---|
| 737 | 731 | struct bfq_group *bfqg; |
|---|
| .. | .. |
|---|
| 747 | 741 | } |
|---|
| 748 | 742 | #endif |
|---|
| 749 | 743 | |
|---|
| 744 | + /* Matches the smp_wmb() in bfq_group_set_weight. */ |
|---|
| 745 | + smp_rmb(); |
|---|
| 750 | 746 | old_st->wsum -= entity->weight; |
|---|
| 751 | 747 | |
|---|
| 752 | 748 | if (entity->new_weight != entity->orig_weight) { |
|---|
| .. | .. |
|---|
| 1003 | 999 | */ |
|---|
| 1004 | 1000 | bfq_get_entity(entity); |
|---|
| 1005 | 1001 | |
|---|
| 1006 | | - entity->on_st = true; |
|---|
| 1002 | + entity->on_st_or_in_serv = true; |
|---|
| 1007 | 1003 | } |
|---|
| 1008 | 1004 | |
|---|
| 1009 | | -#ifdef BFQ_GROUP_IOSCHED_ENABLED |
|---|
| 1005 | +#ifdef CONFIG_BFQ_GROUP_IOSCHED |
|---|
| 1010 | 1006 | if (!bfq_entity_to_bfqq(entity)) { /* bfq_group */ |
|---|
| 1011 | 1007 | struct bfq_group *bfqg = |
|---|
| 1012 | 1008 | container_of(entity, struct bfq_group, entity); |
|---|
| .. | .. |
|---|
| 1154 | 1150 | } |
|---|
| 1155 | 1151 | |
|---|
| 1156 | 1152 | /** |
|---|
| 1157 | | - * __bfq_deactivate_entity - deactivate an entity from its service tree. |
|---|
| 1158 | | - * @entity: the entity to deactivate. |
|---|
| 1153 | + * __bfq_deactivate_entity - update sched_data and service trees for |
|---|
| 1154 | + * entity, so as to represent entity as inactive |
|---|
| 1155 | + * @entity: the entity being deactivated. |
|---|
| 1159 | 1156 | * @ins_into_idle_tree: if false, the entity will not be put into the |
|---|
| 1160 | 1157 | * idle tree. |
|---|
| 1161 | 1158 | * |
|---|
| 1162 | | - * Deactivates an entity, independently of its previous state. Must |
|---|
| 1163 | | - * be invoked only if entity is on a service tree. Extracts the entity |
|---|
| 1164 | | - * from that tree, and if necessary and allowed, puts it into the idle |
|---|
| 1165 | | - * tree. |
|---|
| 1159 | + * If necessary and allowed, puts entity into the idle tree. NOTE: |
|---|
| 1160 | + * entity may be on no tree if in service. |
|---|
| 1166 | 1161 | */ |
|---|
| 1167 | 1162 | bool __bfq_deactivate_entity(struct bfq_entity *entity, bool ins_into_idle_tree) |
|---|
| 1168 | 1163 | { |
|---|
| .. | .. |
|---|
| 1170 | 1165 | struct bfq_service_tree *st; |
|---|
| 1171 | 1166 | bool is_in_service; |
|---|
| 1172 | 1167 | |
|---|
| 1173 | | - if (!entity->on_st) /* entity never activated, or already inactive */ |
|---|
| 1168 | + if (!entity->on_st_or_in_serv) /* |
|---|
| 1169 | + * entity never activated, or |
|---|
| 1170 | + * already inactive |
|---|
| 1171 | + */ |
|---|
| 1174 | 1172 | return false; |
|---|
| 1175 | 1173 | |
|---|
| 1176 | 1174 | /* |
|---|
| .. | .. |
|---|
| 1391 | 1389 | * In this first case, update the virtual time in @st too (see the |
|---|
| 1392 | 1390 | * comments on this update inside the function). |
|---|
| 1393 | 1391 | * |
|---|
| 1394 | | - * In constrast, if there is an in-service entity, then return the |
|---|
| 1392 | + * In contrast, if there is an in-service entity, then return the |
|---|
| 1395 | 1393 | * entity that would be set in service if not only the above |
|---|
| 1396 | 1394 | * conditions, but also the next one held true: the currently |
|---|
| 1397 | 1395 | * in-service entity, on expiration, |
|---|
| .. | .. |
|---|
| 1474 | 1472 | * is being invoked as a part of the expiration path |
|---|
| 1475 | 1473 | * of the in-service queue. In this case, even if |
|---|
| 1476 | 1474 | * sd->in_service_entity is not NULL, |
|---|
| 1477 | | - * sd->in_service_entiy at this point is actually not |
|---|
| 1475 | + * sd->in_service_entity at this point is actually not |
|---|
| 1478 | 1476 | * in service any more, and, if needed, has already |
|---|
| 1479 | 1477 | * been properly queued or requeued into the right |
|---|
| 1480 | 1478 | * tree. The reason why sd->in_service_entity is still |
|---|
| 1481 | 1479 | * not NULL here, even if expiration is true, is that |
|---|
| 1482 | | - * sd->in_service_entiy is reset as a last step in the |
|---|
| 1480 | + * sd->in_service_entity is reset as a last step in the |
|---|
| 1483 | 1481 | * expiration path. So, if expiration is true, tell |
|---|
| 1484 | 1482 | * __bfq_lookup_next_entity that there is no |
|---|
| 1485 | 1483 | * sd->in_service_entity. |
|---|
| .. | .. |
|---|
| 1514 | 1512 | struct bfq_sched_data *sd; |
|---|
| 1515 | 1513 | struct bfq_queue *bfqq; |
|---|
| 1516 | 1514 | |
|---|
| 1517 | | - if (bfqd->busy_queues == 0) |
|---|
| 1515 | + if (bfq_tot_busy_queues(bfqd) == 0) |
|---|
| 1518 | 1516 | return NULL; |
|---|
| 1519 | 1517 | |
|---|
| 1520 | 1518 | /* |
|---|
| .. | .. |
|---|
| 1625 | 1623 | * service tree either, then release the service reference to |
|---|
| 1626 | 1624 | * the queue it represents (taken with bfq_get_entity). |
|---|
| 1627 | 1625 | */ |
|---|
| 1628 | | - if (!in_serv_entity->on_st) { |
|---|
| 1626 | + if (!in_serv_entity->on_st_or_in_serv) { |
|---|
| 1629 | 1627 | /* |
|---|
| 1630 | 1628 | * If no process is referencing in_serv_bfqq any |
|---|
| 1631 | 1629 | * longer, then the service reference may be the only |
|---|
| .. | .. |
|---|
| 1679 | 1677 | |
|---|
| 1680 | 1678 | bfq_clear_bfqq_busy(bfqq); |
|---|
| 1681 | 1679 | |
|---|
| 1682 | | - bfqd->busy_queues--; |
|---|
| 1680 | + bfqd->busy_queues[bfqq->ioprio_class - 1]--; |
|---|
| 1683 | 1681 | |
|---|
| 1684 | 1682 | if (bfqq->wr_coeff > 1) |
|---|
| 1685 | 1683 | bfqd->wr_busy_queues--; |
|---|
| .. | .. |
|---|
| 1702 | 1700 | bfq_activate_bfqq(bfqd, bfqq); |
|---|
| 1703 | 1701 | |
|---|
| 1704 | 1702 | bfq_mark_bfqq_busy(bfqq); |
|---|
| 1705 | | - bfqd->busy_queues++; |
|---|
| 1703 | + bfqd->busy_queues[bfqq->ioprio_class - 1]++; |
|---|
| 1706 | 1704 | |
|---|
| 1707 | 1705 | if (!bfqq->dispatched) |
|---|
| 1708 | 1706 | if (bfqq->wr_coeff == 1) |
|---|