|
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
| 2 | + |
| 3 | +#include <linux/jiffies.h> |
| 4 | +#include <linux/module.h> |
| 5 | +#include <linux/percpu.h> |
| 6 | +#include <linux/preempt.h> |
| 7 | +#include <linux/time.h> |
| 8 | +#include <linux/spinlock.h> |
| 9 | + |
| 10 | +#include "eytzinger.h" |
| 11 | +#include "time_stats.h" |
| 12 | + |
| 13 | +static const struct time_unit time_units[] = { |
| 14 | + { "ns", 1 }, |
| 15 | + { "us", NSEC_PER_USEC }, |
| 16 | + { "ms", NSEC_PER_MSEC }, |
| 17 | + { "s", NSEC_PER_SEC }, |
| 18 | + { "m", (u64) NSEC_PER_SEC * 60}, |
| 19 | + { "h", (u64) NSEC_PER_SEC * 3600}, |
| 20 | + { "eon", U64_MAX }, |
| 21 | +}; |
| 22 | + |
| 23 | +const struct time_unit *bch2_pick_time_units(u64 ns) |
| 24 | +{ |
| 25 | + const struct time_unit *u; |
| 26 | + |
| 27 | + for (u = time_units; |
| 28 | + u + 1 < time_units + ARRAY_SIZE(time_units) && |
| 29 | + ns >= u[1].nsecs << 1; |
| 30 | + u++) |
| 31 | + ; |
| 32 | + |
| 33 | + return u; |
| 34 | +} |
| 35 | + |
| 36 | +static void quantiles_update(struct quantiles *q, u64 v) |
| 37 | +{ |
| 38 | + unsigned i = 0; |
| 39 | + |
| 40 | + while (i < ARRAY_SIZE(q->entries)) { |
| 41 | + struct quantile_entry *e = q->entries + i; |
| 42 | + |
| 43 | + if (unlikely(!e->step)) { |
| 44 | + e->m = v; |
| 45 | + e->step = max_t(unsigned, v / 2, 1024); |
| 46 | + } else if (e->m > v) { |
| 47 | + e->m = e->m >= e->step |
| 48 | + ? e->m - e->step |
| 49 | + : 0; |
| 50 | + } else if (e->m < v) { |
| 51 | + e->m = e->m + e->step > e->m |
| 52 | + ? e->m + e->step |
| 53 | + : U32_MAX; |
| 54 | + } |
| 55 | + |
| 56 | + if ((e->m > v ? e->m - v : v - e->m) < e->step) |
| 57 | + e->step = max_t(unsigned, e->step / 2, 1); |
| 58 | + |
| 59 | + if (v >= e->m) |
| 60 | + break; |
| 61 | + |
| 62 | + i = eytzinger0_child(i, v > e->m); |
| 63 | + } |
| 64 | +} |
| 65 | + |
| 66 | +static inline void time_stats_update_one(struct bch2_time_stats *stats, |
| 67 | + u64 start, u64 end) |
| 68 | +{ |
| 69 | + u64 duration, freq; |
| 70 | + |
| 71 | + if (time_after64(end, start)) { |
| 72 | + duration = end - start; |
| 73 | + mean_and_variance_update(&stats->duration_stats, duration); |
| 74 | + mean_and_variance_weighted_update(&stats->duration_stats_weighted, duration); |
| 75 | + stats->max_duration = max(stats->max_duration, duration); |
| 76 | + stats->min_duration = min(stats->min_duration, duration); |
| 77 | + stats->total_duration += duration; |
| 78 | + |
| 79 | + if (stats->quantiles_enabled) |
| 80 | + quantiles_update(&stats->quantiles, duration); |
| 81 | + } |
| 82 | + |
| 83 | + if (stats->last_event && time_after64(end, stats->last_event)) { |
| 84 | + freq = end - stats->last_event; |
| 85 | + mean_and_variance_update(&stats->freq_stats, freq); |
| 86 | + mean_and_variance_weighted_update(&stats->freq_stats_weighted, freq); |
| 87 | + stats->max_freq = max(stats->max_freq, freq); |
| 88 | + stats->min_freq = min(stats->min_freq, freq); |
| 89 | + } |
| 90 | + |
| 91 | + stats->last_event = end; |
| 92 | +} |
| 93 | + |
| 94 | +void __bch2_time_stats_clear_buffer(struct bch2_time_stats *stats, |
| 95 | + struct time_stat_buffer *b) |
| 96 | +{ |
| 97 | + for (struct time_stat_buffer_entry *i = b->entries; |
| 98 | + i < b->entries + ARRAY_SIZE(b->entries); |
| 99 | + i++) |
| 100 | + time_stats_update_one(stats, i->start, i->end); |
| 101 | + b->nr = 0; |
| 102 | +} |
| 103 | + |
| 104 | +static noinline void time_stats_clear_buffer(struct bch2_time_stats *stats, |
| 105 | + struct time_stat_buffer *b) |
| 106 | +{ |
| 107 | + unsigned long flags; |
| 108 | + |
| 109 | + spin_lock_irqsave(&stats->lock, flags); |
| 110 | + __bch2_time_stats_clear_buffer(stats, b); |
| 111 | + spin_unlock_irqrestore(&stats->lock, flags); |
| 112 | +} |
| 113 | + |
| 114 | +void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end) |
| 115 | +{ |
| 116 | + unsigned long flags; |
| 117 | + |
| 118 | + WARN_ONCE(!stats->duration_stats_weighted.weight || |
| 119 | + !stats->freq_stats_weighted.weight, |
| 120 | + "uninitialized bch2_time_stats"); |
| 121 | + |
| 122 | + if (!stats->buffer) { |
| 123 | + spin_lock_irqsave(&stats->lock, flags); |
| 124 | + time_stats_update_one(stats, start, end); |
| 125 | + |
| 126 | + if (mean_and_variance_weighted_get_mean(stats->freq_stats_weighted) < 32 && |
| 127 | + stats->duration_stats.n > 1024) |
| 128 | + stats->buffer = |
| 129 | + alloc_percpu_gfp(struct time_stat_buffer, |
| 130 | + GFP_ATOMIC); |
| 131 | + spin_unlock_irqrestore(&stats->lock, flags); |
| 132 | + } else { |
| 133 | + struct time_stat_buffer *b; |
| 134 | + |
| 135 | + preempt_disable(); |
| 136 | + b = this_cpu_ptr(stats->buffer); |
| 137 | + |
| 138 | + BUG_ON(b->nr >= ARRAY_SIZE(b->entries)); |
| 139 | + b->entries[b->nr++] = (struct time_stat_buffer_entry) { |
| 140 | + .start = start, |
| 141 | + .end = end |
| 142 | + }; |
| 143 | + |
| 144 | + if (unlikely(b->nr == ARRAY_SIZE(b->entries))) |
| 145 | + time_stats_clear_buffer(stats, b); |
| 146 | + preempt_enable(); |
| 147 | + } |
| 148 | +} |
| 149 | + |
| 150 | +void bch2_time_stats_exit(struct bch2_time_stats *stats) |
| 151 | +{ |
| 152 | + free_percpu(stats->buffer); |
| 153 | +} |
| 154 | + |
| 155 | +void bch2_time_stats_init(struct bch2_time_stats *stats) |
| 156 | +{ |
| 157 | + memset(stats, 0, sizeof(*stats)); |
| 158 | + stats->duration_stats_weighted.weight = 8; |
| 159 | + stats->freq_stats_weighted.weight = 8; |
| 160 | + stats->min_duration = U64_MAX; |
| 161 | + stats->min_freq = U64_MAX; |
| 162 | + spin_lock_init(&stats->lock); |
| 163 | +} |
0 commit comments