.. | .. |
---|
37 | 37 | #include <linux/slab.h> |
---|
38 | 38 | #include <uapi/linux/sched/types.h> |
---|
39 | 39 | #include <linux/sched/clock.h> |
---|
40 | | -#include <soc/rockchip/rockchip_system_monitor.h> |
---|
41 | 40 | |
---|
42 | 41 | #define CREATE_TRACE_POINTS |
---|
43 | 42 | #include <trace/events/cpufreq_interactive.h> |
---|
.. | .. |
---|
101 | 100 | int touchboostpulse_duration_val; |
---|
102 | 101 | /* End time of touchboost pulse in ktime converted to usecs */ |
---|
103 | 102 | u64 touchboostpulse_endtime; |
---|
104 | | - bool touchboost, is_touchboosted; |
---|
105 | 103 | #endif |
---|
106 | 104 | bool boosted; |
---|
107 | 105 | |
---|
.. | .. |
---|
128 | 126 | struct interactive_policy *ipolicy; |
---|
129 | 127 | |
---|
130 | 128 | struct irq_work irq_work; |
---|
131 | | - struct irq_work boost_irq_work; |
---|
132 | 129 | u64 last_sample_time; |
---|
133 | 130 | unsigned long next_sample_jiffies; |
---|
134 | 131 | bool work_in_progress; |
---|
.. | .. |
---|
151 | 148 | u64 pol_hispeed_val_time; /* policy hispeed_validate_time */ |
---|
152 | 149 | u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */ |
---|
153 | 150 | int cpu; |
---|
154 | | - unsigned int task_boost_freq; |
---|
155 | | - unsigned long task_boost_util; |
---|
156 | | - u64 task_boos_endtime; |
---|
157 | 151 | }; |
---|
158 | 152 | |
---|
159 | 153 | static DEFINE_PER_CPU(struct interactive_cpu, interactive_cpu); |
---|
.. | .. |
---|
429 | 423 | new_freq < tunables->touchboost_freq) { |
---|
430 | 424 | new_freq = tunables->touchboost_freq; |
---|
431 | 425 | } |
---|
432 | | - if ((now < icpu->task_boos_endtime) && (new_freq < icpu->task_boost_freq)) { |
---|
433 | | - new_freq = icpu->task_boost_freq; |
---|
434 | | - } |
---|
435 | 426 | #endif |
---|
436 | 427 | if (policy->cur >= tunables->hispeed_freq && |
---|
437 | 428 | new_freq > policy->cur && |
---|
.. | .. |
---|
522 | 513 | sampling_rate = icpu->ipolicy->tunables->sampling_rate; |
---|
523 | 514 | icpu->last_sample_time = local_clock(); |
---|
524 | 515 | icpu->next_sample_jiffies = usecs_to_jiffies(sampling_rate) + jiffies; |
---|
525 | | - icpu->work_in_progress = true; |
---|
526 | | - irq_work_queue_on(&icpu->irq_work, icpu->cpu); |
---|
| 516 | + cpufreq_interactive_update(icpu); |
---|
527 | 517 | } |
---|
528 | 518 | } |
---|
529 | 519 | |
---|
.. | .. |
---|
609 | 599 | for_each_cpu(cpu, &tmp_mask) { |
---|
610 | 600 | struct interactive_cpu *icpu = &per_cpu(interactive_cpu, cpu); |
---|
611 | 601 | struct cpufreq_policy *policy; |
---|
612 | | -#ifdef CONFIG_ARCH_ROCKCHIP |
---|
613 | | - struct interactive_tunables *tunables; |
---|
614 | | - bool update_policy = false; |
---|
615 | | - u64 now; |
---|
616 | 602 | |
---|
617 | | - now = ktime_to_us(ktime_get()); |
---|
618 | | - if (!down_read_trylock(&icpu->enable_sem)) |
---|
619 | | - continue; |
---|
620 | | - |
---|
621 | | - if (!icpu->ipolicy) { |
---|
622 | | - up_read(&icpu->enable_sem); |
---|
623 | | - continue; |
---|
624 | | - } |
---|
625 | | - |
---|
626 | | - tunables = icpu->ipolicy->tunables; |
---|
627 | | - if (!tunables) { |
---|
628 | | - up_read(&icpu->enable_sem); |
---|
629 | | - continue; |
---|
630 | | - } |
---|
631 | | - |
---|
632 | | - if (tunables->touchboost && |
---|
633 | | - now > tunables->touchboostpulse_endtime) { |
---|
634 | | - tunables->touchboost = false; |
---|
635 | | - rockchip_monitor_clear_boosted(); |
---|
636 | | - update_policy = true; |
---|
637 | | - } |
---|
638 | | - |
---|
639 | | - if (!tunables->is_touchboosted && tunables->touchboost) { |
---|
640 | | - rockchip_monitor_set_boosted(); |
---|
641 | | - update_policy = true; |
---|
642 | | - } |
---|
643 | | - |
---|
644 | | - tunables->is_touchboosted = tunables->touchboost; |
---|
645 | | - |
---|
646 | | - up_read(&icpu->enable_sem); |
---|
647 | | - if (update_policy) |
---|
648 | | - cpufreq_update_policy(cpu); |
---|
649 | | -#endif |
---|
650 | 603 | policy = cpufreq_cpu_get(cpu); |
---|
651 | 604 | if (!policy) |
---|
652 | 605 | continue; |
---|
.. | .. |
---|
716 | 669 | unsigned long val, void *data) |
---|
717 | 670 | { |
---|
718 | 671 | struct cpufreq_freqs *freq = data; |
---|
719 | | - struct interactive_cpu *icpu = &per_cpu(interactive_cpu, freq->cpu); |
---|
| 672 | + struct cpufreq_policy *policy = freq->policy; |
---|
| 673 | + struct interactive_cpu *icpu; |
---|
720 | 674 | unsigned long flags; |
---|
| 675 | + int cpu; |
---|
721 | 676 | |
---|
722 | 677 | if (val != CPUFREQ_POSTCHANGE) |
---|
723 | 678 | return 0; |
---|
724 | 679 | |
---|
725 | | - if (!down_read_trylock(&icpu->enable_sem)) |
---|
726 | | - return 0; |
---|
| 680 | + for_each_cpu(cpu, policy->cpus) { |
---|
| 681 | + icpu = &per_cpu(interactive_cpu, cpu); |
---|
727 | 682 | |
---|
728 | | - if (!icpu->ipolicy) { |
---|
| 683 | + if (!down_read_trylock(&icpu->enable_sem)) |
---|
| 684 | + continue; |
---|
| 685 | + |
---|
| 686 | + if (!icpu->ipolicy) { |
---|
| 687 | + up_read(&icpu->enable_sem); |
---|
| 688 | + continue; |
---|
| 689 | + } |
---|
| 690 | + |
---|
| 691 | + spin_lock_irqsave(&icpu->load_lock, flags); |
---|
| 692 | + update_load(icpu, cpu); |
---|
| 693 | + spin_unlock_irqrestore(&icpu->load_lock, flags); |
---|
| 694 | + |
---|
729 | 695 | up_read(&icpu->enable_sem); |
---|
730 | | - return 0; |
---|
731 | 696 | } |
---|
732 | | - |
---|
733 | | - spin_lock_irqsave(&icpu->load_lock, flags); |
---|
734 | | - update_load(icpu, freq->cpu); |
---|
735 | | - spin_unlock_irqrestore(&icpu->load_lock, flags); |
---|
736 | | - |
---|
737 | | - up_read(&icpu->enable_sem); |
---|
738 | 697 | |
---|
739 | 698 | return 0; |
---|
740 | 699 | } |
---|
.. | .. |
---|
1052 | 1011 | return count; |
---|
1053 | 1012 | } |
---|
1054 | 1013 | |
---|
1055 | | -static ssize_t store_touchboost_freq(struct gov_attr_set *attr_set, |
---|
1056 | | - const char *buf, size_t count) |
---|
1057 | | -{ |
---|
1058 | | - struct interactive_tunables *tunables = to_tunables(attr_set); |
---|
1059 | | - unsigned long val; |
---|
1060 | | - int ret; |
---|
1061 | | - |
---|
1062 | | - ret = kstrtoul(buf, 0, &val); |
---|
1063 | | - if (ret < 0) |
---|
1064 | | - return ret; |
---|
1065 | | - |
---|
1066 | | - tunables->touchboost_freq = val; |
---|
1067 | | - |
---|
1068 | | - return count; |
---|
1069 | | -} |
---|
1070 | | - |
---|
1071 | | -static ssize_t show_touchboost_duration(struct gov_attr_set *attr_set, char *buf) |
---|
1072 | | -{ |
---|
1073 | | - struct interactive_tunables *tunables = to_tunables(attr_set); |
---|
1074 | | - |
---|
1075 | | - return sprintf(buf, "%d\n", tunables->touchboostpulse_duration_val); |
---|
1076 | | -} |
---|
1077 | | - |
---|
1078 | | -static ssize_t store_touchboost_duration(struct gov_attr_set *attr_set, |
---|
1079 | | - const char *buf, size_t count) |
---|
1080 | | -{ |
---|
1081 | | - struct interactive_tunables *tunables = to_tunables(attr_set); |
---|
1082 | | - int val, ret; |
---|
1083 | | - |
---|
1084 | | - ret = kstrtoint(buf, 0, &val); |
---|
1085 | | - if (ret < 0) |
---|
1086 | | - return ret; |
---|
1087 | | - |
---|
1088 | | - tunables->touchboostpulse_duration_val = val; |
---|
1089 | | - |
---|
1090 | | - return count; |
---|
1091 | | -} |
---|
1092 | | - |
---|
1093 | 1014 | show_one(hispeed_freq, "%u"); |
---|
1094 | 1015 | show_one(go_hispeed_load, "%lu"); |
---|
1095 | 1016 | show_one(min_sample_time, "%lu"); |
---|
.. | .. |
---|
1097 | 1018 | show_one(boost, "%u"); |
---|
1098 | 1019 | show_one(boostpulse_duration, "%u"); |
---|
1099 | 1020 | show_one(io_is_busy, "%u"); |
---|
1100 | | -show_one(touchboost_freq, "%lu"); |
---|
1101 | 1021 | |
---|
1102 | 1022 | gov_attr_rw(target_loads); |
---|
1103 | 1023 | gov_attr_rw(above_hispeed_delay); |
---|
.. | .. |
---|
1110 | 1030 | gov_attr_wo(boostpulse); |
---|
1111 | 1031 | gov_attr_rw(boostpulse_duration); |
---|
1112 | 1032 | gov_attr_rw(io_is_busy); |
---|
1113 | | -gov_attr_rw(touchboost_freq); |
---|
1114 | | -gov_attr_rw(touchboost_duration); |
---|
1115 | 1033 | |
---|
1116 | 1034 | static struct attribute *interactive_attributes[] = { |
---|
1117 | 1035 | &target_loads.attr, |
---|
.. | .. |
---|
1125 | 1043 | &boostpulse.attr, |
---|
1126 | 1044 | &boostpulse_duration.attr, |
---|
1127 | 1045 | &io_is_busy.attr, |
---|
1128 | | - &touchboost_freq.attr, |
---|
1129 | | - &touchboost_duration.attr, |
---|
1130 | 1046 | NULL |
---|
1131 | 1047 | }; |
---|
1132 | 1048 | |
---|
.. | .. |
---|
1220 | 1136 | for_each_cpu(i, policy->cpus) |
---|
1221 | 1137 | cpufreq_remove_update_util_hook(i); |
---|
1222 | 1138 | |
---|
1223 | | - synchronize_sched(); |
---|
| 1139 | + synchronize_rcu(); |
---|
1224 | 1140 | } |
---|
1225 | 1141 | |
---|
1226 | 1142 | static void icpu_cancel_work(struct interactive_cpu *icpu) |
---|
1227 | 1143 | { |
---|
1228 | 1144 | irq_work_sync(&icpu->irq_work); |
---|
1229 | | -#ifdef CONFIG_ARCH_ROCKCHIP |
---|
1230 | | - irq_work_sync(&icpu->boost_irq_work); |
---|
1231 | | -#endif |
---|
1232 | 1145 | icpu->work_in_progress = false; |
---|
1233 | 1146 | del_timer_sync(&icpu->slack_timer); |
---|
1234 | 1147 | } |
---|
.. | .. |
---|
1328 | 1241 | cpumask_set_cpu(i, &speedchange_cpumask); |
---|
1329 | 1242 | pcpu->loc_hispeed_val_time = |
---|
1330 | 1243 | ktime_to_us(ktime_get()); |
---|
1331 | | - tunables->touchboost = true; |
---|
1332 | 1244 | anyboost = 1; |
---|
1333 | 1245 | } |
---|
1334 | 1246 | |
---|
.. | .. |
---|
1454 | 1366 | tunables->attr_set = attr_set; |
---|
1455 | 1367 | } |
---|
1456 | 1368 | } |
---|
1457 | | - |
---|
1458 | | -static unsigned int get_freq_for_util(struct cpufreq_policy *policy, unsigned long util) |
---|
1459 | | -{ |
---|
1460 | | - struct cpufreq_frequency_table *pos; |
---|
1461 | | - unsigned long max_cap, cur_cap; |
---|
1462 | | - unsigned int freq = 0; |
---|
1463 | | - |
---|
1464 | | - max_cap = arch_scale_cpu_capacity(NULL, policy->cpu); |
---|
1465 | | - cpufreq_for_each_valid_entry(pos, policy->freq_table) { |
---|
1466 | | - freq = pos->frequency; |
---|
1467 | | - |
---|
1468 | | - cur_cap = max_cap * freq / policy->max; |
---|
1469 | | - if (cur_cap > util) |
---|
1470 | | - break; |
---|
1471 | | - } |
---|
1472 | | - |
---|
1473 | | - return freq; |
---|
1474 | | -} |
---|
1475 | | - |
---|
1476 | | -static void task_boost_irq_work(struct irq_work *irq_work) |
---|
1477 | | -{ |
---|
1478 | | - struct interactive_cpu *pcpu; |
---|
1479 | | - struct interactive_policy *ipolicy; |
---|
1480 | | - unsigned long flags[2]; |
---|
1481 | | - u64 now, prev_boos_endtime; |
---|
1482 | | - unsigned int boost_freq; |
---|
1483 | | - |
---|
1484 | | - pcpu = container_of(irq_work, struct interactive_cpu, boost_irq_work); |
---|
1485 | | - if (!down_read_trylock(&pcpu->enable_sem)) |
---|
1486 | | - return; |
---|
1487 | | - |
---|
1488 | | - ipolicy = pcpu->ipolicy; |
---|
1489 | | - if (!ipolicy) |
---|
1490 | | - goto out; |
---|
1491 | | - |
---|
1492 | | - if (ipolicy->policy->cur == ipolicy->policy->max) |
---|
1493 | | - goto out; |
---|
1494 | | - |
---|
1495 | | - now = ktime_to_us(ktime_get()); |
---|
1496 | | - prev_boos_endtime = pcpu->task_boos_endtime;; |
---|
1497 | | - pcpu->task_boos_endtime = now + ipolicy->tunables->sampling_rate; |
---|
1498 | | - boost_freq = get_freq_for_util(ipolicy->policy, pcpu->task_boost_util); |
---|
1499 | | - if ((now < prev_boos_endtime) && (boost_freq <= pcpu->task_boost_freq)) |
---|
1500 | | - goto out; |
---|
1501 | | - pcpu->task_boost_freq = boost_freq; |
---|
1502 | | - |
---|
1503 | | - spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]); |
---|
1504 | | - spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]); |
---|
1505 | | - if (pcpu->target_freq < pcpu->task_boost_freq) { |
---|
1506 | | - pcpu->target_freq = pcpu->task_boost_freq; |
---|
1507 | | - cpumask_set_cpu(pcpu->cpu, &speedchange_cpumask); |
---|
1508 | | - wake_up_process(speedchange_task); |
---|
1509 | | - } |
---|
1510 | | - spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]); |
---|
1511 | | - spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]); |
---|
1512 | | - |
---|
1513 | | -out: |
---|
1514 | | - up_read(&pcpu->enable_sem); |
---|
1515 | | -} |
---|
1516 | | - |
---|
1517 | | -extern unsigned long capacity_curr_of(int cpu); |
---|
1518 | | - |
---|
1519 | | -void cpufreq_task_boost(int cpu, unsigned long util) |
---|
1520 | | -{ |
---|
1521 | | - struct interactive_cpu *pcpu = &per_cpu(interactive_cpu, cpu); |
---|
1522 | | - unsigned long cap, min_util; |
---|
1523 | | - |
---|
1524 | | - if (!speedchange_task) |
---|
1525 | | - return; |
---|
1526 | | - |
---|
1527 | | - min_util = util + (util >> 2); |
---|
1528 | | - cap = capacity_curr_of(cpu); |
---|
1529 | | - if (min_util > cap) { |
---|
1530 | | - pcpu->task_boost_util = min_util; |
---|
1531 | | - irq_work_queue(&pcpu->boost_irq_work); |
---|
1532 | | - } |
---|
1533 | | -} |
---|
1534 | 1369 | #endif |
---|
1535 | 1370 | |
---|
1536 | 1371 | int cpufreq_interactive_init(struct cpufreq_policy *policy) |
---|
.. | .. |
---|
1639 | 1474 | idle_notifier_unregister(&cpufreq_interactive_idle_nb); |
---|
1640 | 1475 | #ifdef CONFIG_ARCH_ROCKCHIP |
---|
1641 | 1476 | input_unregister_handler(&cpufreq_interactive_input_handler); |
---|
1642 | | - if (tunables->touchboost) { |
---|
1643 | | - tunables->touchboost = false; |
---|
1644 | | - rockchip_monitor_clear_boosted(); |
---|
1645 | | - } |
---|
1646 | 1477 | #endif |
---|
1647 | 1478 | } |
---|
1648 | 1479 | |
---|
.. | .. |
---|
1763 | 1594 | icpu = &per_cpu(interactive_cpu, cpu); |
---|
1764 | 1595 | |
---|
1765 | 1596 | init_irq_work(&icpu->irq_work, irq_work); |
---|
1766 | | -#ifdef CONFIG_ARCH_ROCKCHIP |
---|
1767 | | - init_irq_work(&icpu->boost_irq_work, task_boost_irq_work); |
---|
1768 | | -#endif |
---|
1769 | 1597 | spin_lock_init(&icpu->load_lock); |
---|
1770 | 1598 | spin_lock_init(&icpu->target_freq_lock); |
---|
1771 | 1599 | init_rwsem(&icpu->enable_sem); |
---|