| .. | .. |
|---|
| 37 | 37 | #include <linux/slab.h> |
|---|
| 38 | 38 | #include <uapi/linux/sched/types.h> |
|---|
| 39 | 39 | #include <linux/sched/clock.h> |
|---|
| 40 | | -#include <soc/rockchip/rockchip_system_monitor.h> |
|---|
| 41 | 40 | |
|---|
| 42 | 41 | #define CREATE_TRACE_POINTS |
|---|
| 43 | 42 | #include <trace/events/cpufreq_interactive.h> |
|---|
| .. | .. |
|---|
| 101 | 100 | int touchboostpulse_duration_val; |
|---|
| 102 | 101 | /* End time of touchboost pulse in ktime converted to usecs */ |
|---|
| 103 | 102 | u64 touchboostpulse_endtime; |
|---|
| 104 | | - bool touchboost, is_touchboosted; |
|---|
| 105 | 103 | #endif |
|---|
| 106 | 104 | bool boosted; |
|---|
| 107 | 105 | |
|---|
| .. | .. |
|---|
| 128 | 126 | struct interactive_policy *ipolicy; |
|---|
| 129 | 127 | |
|---|
| 130 | 128 | struct irq_work irq_work; |
|---|
| 131 | | - struct irq_work boost_irq_work; |
|---|
| 132 | 129 | u64 last_sample_time; |
|---|
| 133 | 130 | unsigned long next_sample_jiffies; |
|---|
| 134 | 131 | bool work_in_progress; |
|---|
| .. | .. |
|---|
| 151 | 148 | u64 pol_hispeed_val_time; /* policy hispeed_validate_time */ |
|---|
| 152 | 149 | u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */ |
|---|
| 153 | 150 | int cpu; |
|---|
| 154 | | - unsigned int task_boost_freq; |
|---|
| 155 | | - unsigned long task_boost_util; |
|---|
| 156 | | - u64 task_boos_endtime; |
|---|
| 157 | 151 | }; |
|---|
| 158 | 152 | |
|---|
| 159 | 153 | static DEFINE_PER_CPU(struct interactive_cpu, interactive_cpu); |
|---|
| .. | .. |
|---|
| 429 | 423 | new_freq < tunables->touchboost_freq) { |
|---|
| 430 | 424 | new_freq = tunables->touchboost_freq; |
|---|
| 431 | 425 | } |
|---|
| 432 | | - if ((now < icpu->task_boos_endtime) && (new_freq < icpu->task_boost_freq)) { |
|---|
| 433 | | - new_freq = icpu->task_boost_freq; |
|---|
| 434 | | - } |
|---|
| 435 | 426 | #endif |
|---|
| 436 | 427 | if (policy->cur >= tunables->hispeed_freq && |
|---|
| 437 | 428 | new_freq > policy->cur && |
|---|
| .. | .. |
|---|
| 522 | 513 | sampling_rate = icpu->ipolicy->tunables->sampling_rate; |
|---|
| 523 | 514 | icpu->last_sample_time = local_clock(); |
|---|
| 524 | 515 | icpu->next_sample_jiffies = usecs_to_jiffies(sampling_rate) + jiffies; |
|---|
| 525 | | - icpu->work_in_progress = true; |
|---|
| 526 | | - irq_work_queue_on(&icpu->irq_work, icpu->cpu); |
|---|
| 516 | + cpufreq_interactive_update(icpu); |
|---|
| 527 | 517 | } |
|---|
| 528 | 518 | } |
|---|
| 529 | 519 | |
|---|
| .. | .. |
|---|
| 609 | 599 | for_each_cpu(cpu, &tmp_mask) { |
|---|
| 610 | 600 | struct interactive_cpu *icpu = &per_cpu(interactive_cpu, cpu); |
|---|
| 611 | 601 | struct cpufreq_policy *policy; |
|---|
| 612 | | -#ifdef CONFIG_ARCH_ROCKCHIP |
|---|
| 613 | | - struct interactive_tunables *tunables; |
|---|
| 614 | | - bool update_policy = false; |
|---|
| 615 | | - u64 now; |
|---|
| 616 | 602 | |
|---|
| 617 | | - now = ktime_to_us(ktime_get()); |
|---|
| 618 | | - if (!down_read_trylock(&icpu->enable_sem)) |
|---|
| 619 | | - continue; |
|---|
| 620 | | - |
|---|
| 621 | | - if (!icpu->ipolicy) { |
|---|
| 622 | | - up_read(&icpu->enable_sem); |
|---|
| 623 | | - continue; |
|---|
| 624 | | - } |
|---|
| 625 | | - |
|---|
| 626 | | - tunables = icpu->ipolicy->tunables; |
|---|
| 627 | | - if (!tunables) { |
|---|
| 628 | | - up_read(&icpu->enable_sem); |
|---|
| 629 | | - continue; |
|---|
| 630 | | - } |
|---|
| 631 | | - |
|---|
| 632 | | - if (tunables->touchboost && |
|---|
| 633 | | - now > tunables->touchboostpulse_endtime) { |
|---|
| 634 | | - tunables->touchboost = false; |
|---|
| 635 | | - rockchip_monitor_clear_boosted(); |
|---|
| 636 | | - update_policy = true; |
|---|
| 637 | | - } |
|---|
| 638 | | - |
|---|
| 639 | | - if (!tunables->is_touchboosted && tunables->touchboost) { |
|---|
| 640 | | - rockchip_monitor_set_boosted(); |
|---|
| 641 | | - update_policy = true; |
|---|
| 642 | | - } |
|---|
| 643 | | - |
|---|
| 644 | | - tunables->is_touchboosted = tunables->touchboost; |
|---|
| 645 | | - |
|---|
| 646 | | - up_read(&icpu->enable_sem); |
|---|
| 647 | | - if (update_policy) |
|---|
| 648 | | - cpufreq_update_policy(cpu); |
|---|
| 649 | | -#endif |
|---|
| 650 | 603 | policy = cpufreq_cpu_get(cpu); |
|---|
| 651 | 604 | if (!policy) |
|---|
| 652 | 605 | continue; |
|---|
| .. | .. |
|---|
| 716 | 669 | unsigned long val, void *data) |
|---|
| 717 | 670 | { |
|---|
| 718 | 671 | struct cpufreq_freqs *freq = data; |
|---|
| 719 | | - struct interactive_cpu *icpu = &per_cpu(interactive_cpu, freq->cpu); |
|---|
| 672 | + struct cpufreq_policy *policy = freq->policy; |
|---|
| 673 | + struct interactive_cpu *icpu; |
|---|
| 720 | 674 | unsigned long flags; |
|---|
| 675 | + int cpu; |
|---|
| 721 | 676 | |
|---|
| 722 | 677 | if (val != CPUFREQ_POSTCHANGE) |
|---|
| 723 | 678 | return 0; |
|---|
| 724 | 679 | |
|---|
| 725 | | - if (!down_read_trylock(&icpu->enable_sem)) |
|---|
| 726 | | - return 0; |
|---|
| 680 | + for_each_cpu(cpu, policy->cpus) { |
|---|
| 681 | + icpu = &per_cpu(interactive_cpu, cpu); |
|---|
| 727 | 682 | |
|---|
| 728 | | - if (!icpu->ipolicy) { |
|---|
| 683 | + if (!down_read_trylock(&icpu->enable_sem)) |
|---|
| 684 | + continue; |
|---|
| 685 | + |
|---|
| 686 | + if (!icpu->ipolicy) { |
|---|
| 687 | + up_read(&icpu->enable_sem); |
|---|
| 688 | + continue; |
|---|
| 689 | + } |
|---|
| 690 | + |
|---|
| 691 | + spin_lock_irqsave(&icpu->load_lock, flags); |
|---|
| 692 | + update_load(icpu, cpu); |
|---|
| 693 | + spin_unlock_irqrestore(&icpu->load_lock, flags); |
|---|
| 694 | + |
|---|
| 729 | 695 | up_read(&icpu->enable_sem); |
|---|
| 730 | | - return 0; |
|---|
| 731 | 696 | } |
|---|
| 732 | | - |
|---|
| 733 | | - spin_lock_irqsave(&icpu->load_lock, flags); |
|---|
| 734 | | - update_load(icpu, freq->cpu); |
|---|
| 735 | | - spin_unlock_irqrestore(&icpu->load_lock, flags); |
|---|
| 736 | | - |
|---|
| 737 | | - up_read(&icpu->enable_sem); |
|---|
| 738 | 697 | |
|---|
| 739 | 698 | return 0; |
|---|
| 740 | 699 | } |
|---|
| .. | .. |
|---|
| 1052 | 1011 | return count; |
|---|
| 1053 | 1012 | } |
|---|
| 1054 | 1013 | |
|---|
| 1055 | | -static ssize_t store_touchboost_freq(struct gov_attr_set *attr_set, |
|---|
| 1056 | | - const char *buf, size_t count) |
|---|
| 1057 | | -{ |
|---|
| 1058 | | - struct interactive_tunables *tunables = to_tunables(attr_set); |
|---|
| 1059 | | - unsigned long val; |
|---|
| 1060 | | - int ret; |
|---|
| 1061 | | - |
|---|
| 1062 | | - ret = kstrtoul(buf, 0, &val); |
|---|
| 1063 | | - if (ret < 0) |
|---|
| 1064 | | - return ret; |
|---|
| 1065 | | - |
|---|
| 1066 | | - tunables->touchboost_freq = val; |
|---|
| 1067 | | - |
|---|
| 1068 | | - return count; |
|---|
| 1069 | | -} |
|---|
| 1070 | | - |
|---|
| 1071 | | -static ssize_t show_touchboost_duration(struct gov_attr_set *attr_set, char *buf) |
|---|
| 1072 | | -{ |
|---|
| 1073 | | - struct interactive_tunables *tunables = to_tunables(attr_set); |
|---|
| 1074 | | - |
|---|
| 1075 | | - return sprintf(buf, "%d\n", tunables->touchboostpulse_duration_val); |
|---|
| 1076 | | -} |
|---|
| 1077 | | - |
|---|
| 1078 | | -static ssize_t store_touchboost_duration(struct gov_attr_set *attr_set, |
|---|
| 1079 | | - const char *buf, size_t count) |
|---|
| 1080 | | -{ |
|---|
| 1081 | | - struct interactive_tunables *tunables = to_tunables(attr_set); |
|---|
| 1082 | | - int val, ret; |
|---|
| 1083 | | - |
|---|
| 1084 | | - ret = kstrtoint(buf, 0, &val); |
|---|
| 1085 | | - if (ret < 0) |
|---|
| 1086 | | - return ret; |
|---|
| 1087 | | - |
|---|
| 1088 | | - tunables->touchboostpulse_duration_val = val; |
|---|
| 1089 | | - |
|---|
| 1090 | | - return count; |
|---|
| 1091 | | -} |
|---|
| 1092 | | - |
|---|
| 1093 | 1014 | show_one(hispeed_freq, "%u"); |
|---|
| 1094 | 1015 | show_one(go_hispeed_load, "%lu"); |
|---|
| 1095 | 1016 | show_one(min_sample_time, "%lu"); |
|---|
| .. | .. |
|---|
| 1097 | 1018 | show_one(boost, "%u"); |
|---|
| 1098 | 1019 | show_one(boostpulse_duration, "%u"); |
|---|
| 1099 | 1020 | show_one(io_is_busy, "%u"); |
|---|
| 1100 | | -show_one(touchboost_freq, "%lu"); |
|---|
| 1101 | 1021 | |
|---|
| 1102 | 1022 | gov_attr_rw(target_loads); |
|---|
| 1103 | 1023 | gov_attr_rw(above_hispeed_delay); |
|---|
| .. | .. |
|---|
| 1110 | 1030 | gov_attr_wo(boostpulse); |
|---|
| 1111 | 1031 | gov_attr_rw(boostpulse_duration); |
|---|
| 1112 | 1032 | gov_attr_rw(io_is_busy); |
|---|
| 1113 | | -gov_attr_rw(touchboost_freq); |
|---|
| 1114 | | -gov_attr_rw(touchboost_duration); |
|---|
| 1115 | 1033 | |
|---|
| 1116 | 1034 | static struct attribute *interactive_attributes[] = { |
|---|
| 1117 | 1035 | &target_loads.attr, |
|---|
| .. | .. |
|---|
| 1125 | 1043 | &boostpulse.attr, |
|---|
| 1126 | 1044 | &boostpulse_duration.attr, |
|---|
| 1127 | 1045 | &io_is_busy.attr, |
|---|
| 1128 | | - &touchboost_freq.attr, |
|---|
| 1129 | | - &touchboost_duration.attr, |
|---|
| 1130 | 1046 | NULL |
|---|
| 1131 | 1047 | }; |
|---|
| 1132 | 1048 | |
|---|
| .. | .. |
|---|
| 1220 | 1136 | for_each_cpu(i, policy->cpus) |
|---|
| 1221 | 1137 | cpufreq_remove_update_util_hook(i); |
|---|
| 1222 | 1138 | |
|---|
| 1223 | | - synchronize_sched(); |
|---|
| 1139 | + synchronize_rcu(); |
|---|
| 1224 | 1140 | } |
|---|
| 1225 | 1141 | |
|---|
| 1226 | 1142 | static void icpu_cancel_work(struct interactive_cpu *icpu) |
|---|
| 1227 | 1143 | { |
|---|
| 1228 | 1144 | irq_work_sync(&icpu->irq_work); |
|---|
| 1229 | | -#ifdef CONFIG_ARCH_ROCKCHIP |
|---|
| 1230 | | - irq_work_sync(&icpu->boost_irq_work); |
|---|
| 1231 | | -#endif |
|---|
| 1232 | 1145 | icpu->work_in_progress = false; |
|---|
| 1233 | 1146 | del_timer_sync(&icpu->slack_timer); |
|---|
| 1234 | 1147 | } |
|---|
| .. | .. |
|---|
| 1328 | 1241 | cpumask_set_cpu(i, &speedchange_cpumask); |
|---|
| 1329 | 1242 | pcpu->loc_hispeed_val_time = |
|---|
| 1330 | 1243 | ktime_to_us(ktime_get()); |
|---|
| 1331 | | - tunables->touchboost = true; |
|---|
| 1332 | 1244 | anyboost = 1; |
|---|
| 1333 | 1245 | } |
|---|
| 1334 | 1246 | |
|---|
| .. | .. |
|---|
| 1454 | 1366 | tunables->attr_set = attr_set; |
|---|
| 1455 | 1367 | } |
|---|
| 1456 | 1368 | } |
|---|
| 1457 | | - |
|---|
| 1458 | | -static unsigned int get_freq_for_util(struct cpufreq_policy *policy, unsigned long util) |
|---|
| 1459 | | -{ |
|---|
| 1460 | | - struct cpufreq_frequency_table *pos; |
|---|
| 1461 | | - unsigned long max_cap, cur_cap; |
|---|
| 1462 | | - unsigned int freq = 0; |
|---|
| 1463 | | - |
|---|
| 1464 | | - max_cap = arch_scale_cpu_capacity(NULL, policy->cpu); |
|---|
| 1465 | | - cpufreq_for_each_valid_entry(pos, policy->freq_table) { |
|---|
| 1466 | | - freq = pos->frequency; |
|---|
| 1467 | | - |
|---|
| 1468 | | - cur_cap = max_cap * freq / policy->max; |
|---|
| 1469 | | - if (cur_cap > util) |
|---|
| 1470 | | - break; |
|---|
| 1471 | | - } |
|---|
| 1472 | | - |
|---|
| 1473 | | - return freq; |
|---|
| 1474 | | -} |
|---|
| 1475 | | - |
|---|
| 1476 | | -static void task_boost_irq_work(struct irq_work *irq_work) |
|---|
| 1477 | | -{ |
|---|
| 1478 | | - struct interactive_cpu *pcpu; |
|---|
| 1479 | | - struct interactive_policy *ipolicy; |
|---|
| 1480 | | - unsigned long flags[2]; |
|---|
| 1481 | | - u64 now, prev_boos_endtime; |
|---|
| 1482 | | - unsigned int boost_freq; |
|---|
| 1483 | | - |
|---|
| 1484 | | - pcpu = container_of(irq_work, struct interactive_cpu, boost_irq_work); |
|---|
| 1485 | | - if (!down_read_trylock(&pcpu->enable_sem)) |
|---|
| 1486 | | - return; |
|---|
| 1487 | | - |
|---|
| 1488 | | - ipolicy = pcpu->ipolicy; |
|---|
| 1489 | | - if (!ipolicy) |
|---|
| 1490 | | - goto out; |
|---|
| 1491 | | - |
|---|
| 1492 | | - if (ipolicy->policy->cur == ipolicy->policy->max) |
|---|
| 1493 | | - goto out; |
|---|
| 1494 | | - |
|---|
| 1495 | | - now = ktime_to_us(ktime_get()); |
|---|
| 1496 | | - prev_boos_endtime = pcpu->task_boos_endtime;; |
|---|
| 1497 | | - pcpu->task_boos_endtime = now + ipolicy->tunables->sampling_rate; |
|---|
| 1498 | | - boost_freq = get_freq_for_util(ipolicy->policy, pcpu->task_boost_util); |
|---|
| 1499 | | - if ((now < prev_boos_endtime) && (boost_freq <= pcpu->task_boost_freq)) |
|---|
| 1500 | | - goto out; |
|---|
| 1501 | | - pcpu->task_boost_freq = boost_freq; |
|---|
| 1502 | | - |
|---|
| 1503 | | - spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]); |
|---|
| 1504 | | - spin_lock_irqsave(&pcpu->target_freq_lock, flags[1]); |
|---|
| 1505 | | - if (pcpu->target_freq < pcpu->task_boost_freq) { |
|---|
| 1506 | | - pcpu->target_freq = pcpu->task_boost_freq; |
|---|
| 1507 | | - cpumask_set_cpu(pcpu->cpu, &speedchange_cpumask); |
|---|
| 1508 | | - wake_up_process(speedchange_task); |
|---|
| 1509 | | - } |
|---|
| 1510 | | - spin_unlock_irqrestore(&pcpu->target_freq_lock, flags[1]); |
|---|
| 1511 | | - spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]); |
|---|
| 1512 | | - |
|---|
| 1513 | | -out: |
|---|
| 1514 | | - up_read(&pcpu->enable_sem); |
|---|
| 1515 | | -} |
|---|
| 1516 | | - |
|---|
| 1517 | | -extern unsigned long capacity_curr_of(int cpu); |
|---|
| 1518 | | - |
|---|
| 1519 | | -void cpufreq_task_boost(int cpu, unsigned long util) |
|---|
| 1520 | | -{ |
|---|
| 1521 | | - struct interactive_cpu *pcpu = &per_cpu(interactive_cpu, cpu); |
|---|
| 1522 | | - unsigned long cap, min_util; |
|---|
| 1523 | | - |
|---|
| 1524 | | - if (!speedchange_task) |
|---|
| 1525 | | - return; |
|---|
| 1526 | | - |
|---|
| 1527 | | - min_util = util + (util >> 2); |
|---|
| 1528 | | - cap = capacity_curr_of(cpu); |
|---|
| 1529 | | - if (min_util > cap) { |
|---|
| 1530 | | - pcpu->task_boost_util = min_util; |
|---|
| 1531 | | - irq_work_queue(&pcpu->boost_irq_work); |
|---|
| 1532 | | - } |
|---|
| 1533 | | -} |
|---|
| 1534 | 1369 | #endif |
|---|
| 1535 | 1370 | |
|---|
| 1536 | 1371 | int cpufreq_interactive_init(struct cpufreq_policy *policy) |
|---|
| .. | .. |
|---|
| 1639 | 1474 | idle_notifier_unregister(&cpufreq_interactive_idle_nb); |
|---|
| 1640 | 1475 | #ifdef CONFIG_ARCH_ROCKCHIP |
|---|
| 1641 | 1476 | input_unregister_handler(&cpufreq_interactive_input_handler); |
|---|
| 1642 | | - if (tunables->touchboost) { |
|---|
| 1643 | | - tunables->touchboost = false; |
|---|
| 1644 | | - rockchip_monitor_clear_boosted(); |
|---|
| 1645 | | - } |
|---|
| 1646 | 1477 | #endif |
|---|
| 1647 | 1478 | } |
|---|
| 1648 | 1479 | |
|---|
| .. | .. |
|---|
| 1763 | 1594 | icpu = &per_cpu(interactive_cpu, cpu); |
|---|
| 1764 | 1595 | |
|---|
| 1765 | 1596 | init_irq_work(&icpu->irq_work, irq_work); |
|---|
| 1766 | | -#ifdef CONFIG_ARCH_ROCKCHIP |
|---|
| 1767 | | - init_irq_work(&icpu->boost_irq_work, task_boost_irq_work); |
|---|
| 1768 | | -#endif |
|---|
| 1769 | 1597 | spin_lock_init(&icpu->load_lock); |
|---|
| 1770 | 1598 | spin_lock_init(&icpu->target_freq_lock); |
|---|
| 1771 | 1599 | init_rwsem(&icpu->enable_sem); |
|---|