001package org.javasimon.callback.quantiles;
002
003import static org.javasimon.callback.logging.LogTemplates.disabled;
004import static org.javasimon.utils.SimonUtils.presentNanoTime;
005
006import org.javasimon.Split;
007import org.javasimon.callback.logging.LogMessageSource;
008import org.javasimon.callback.logging.LogTemplate;
009
010import java.util.Arrays;
011import java.util.Collection;
012import java.util.Collections;
013import java.util.List;
014
015/**
016 * List of buckets and quantiles computer.
017 * Samples are not kept in buckets only the counter indicates their presence.
018 * <br/>
019 * Some details impact quantiles computation precision:
020 * <ul><li><em>Not enough samples</em>: The more samples you have, the more precise interpolation are</li>
021 * <li><em>Not enough buckets</em>: the more buckets are used, the more regular the distribution is and the more memory you'll need as well!</li>
022 * <li><em>All samples in one bucket</em>: samples should be evenly distributed on buckets. If all samples go into the same bucket, you should consider changing the min/max/number settings</li>
023 * </ul>
024 *
025 * @author Gerald Quintana
026 * @since 3.2
027 */
028public abstract class Buckets implements LogMessageSource<Split> {
029
030        /**
031         * Array of buckets, sorted by ranges.
032         * The first and last buckets are special:
033         * The first bucket is range -infinity to min,
034         * The last bucket is range max to +infinity.
035         * Other buckets are regular ones with constant width
036         */
037        protected final Bucket[] buckets;
038
039        /** Number of real buckets (=buckets.length-2). */
040        protected final int bucketNb;
041        /** Lower bound of all real buckets. */
042        protected final long min;
043        /** Upper bound of all real buckets. */
044        protected final long max;
045        /** Log template used to log quantiles. */
046        private LogTemplate<Split> logTemplate = disabled();
047
048        /**
049         * Constructor, initializes buckets.
050         *
051         * @param min Min of all values
052         * @param max Max of all values
053         * @param bucketNb Number of buckets
054         */
055        public Buckets(long min, long max, int bucketNb) {
056                // Check arguments
057                if (bucketNb < 3) {
058                        throw new IllegalArgumentException("Expected at least 3 buckets: " + bucketNb);
059                }
060                if (min >= max) {
061                        throw new IllegalArgumentException("Expected min<max: " + min + "/" + max);
062                }
063                // Initialize attributes
064                this.min = min;
065                this.max = max;
066                this.bucketNb = bucketNb;
067                // Initialize bucket array
068                this.buckets = new Bucket[bucketNb + 2];
069                buckets[0] = new Bucket(Long.MIN_VALUE, min);
070                buckets[bucketNb + 1] = new Bucket(max, Long.MAX_VALUE);
071        }
072
073        /** Computes expected count and check used buckets number. */
074        private int checkAndGetTotalCount() throws IllegalStateException {
075                int usedBuckets = 0;
076                int totalCount = buckets[0].getCount();
077                for (int i = 1; i <= bucketNb; i++) {
078                        int bucketCount = buckets[i].getCount();
079                        totalCount += bucketCount;
080                        if (bucketCount > 0) {
081                                usedBuckets++;
082                        }
083                }
084                totalCount += buckets[bucketNb + 1].getCount();
085                if (usedBuckets < 3) {
086                        throw new IllegalStateException("Only " + usedBuckets + " buckets used, not enough for interpolation, consider reconfiguring min/max/nb");
087                }
088                return totalCount;
089        }
090
091        /**
092         * Computes given quantile.
093         *
094         * @param ration Nth quantile: 0.5 is median
095         * @param totalCount Total count over all buckets
096         * @return Quantile
097         * @throws IllegalStateException Buckets are poorly configured and
098         * quantile can not be computed
099         * @throws IllegalArgumentException
100         */
101        private double computeQuantile(double ration, int totalCount) throws IllegalStateException, IllegalArgumentException {
102                if (ration <= 0.0D || ration >= 1.0D) {
103                        throw new IllegalArgumentException("Expected ratio between 0 and 1 excluded: " + ration);
104                }
105                final double expectedCount = ration * totalCount;
106                // Search bucket corresponding to expected count
107                double lastCount = 0D, newCount;
108                int bucketIndex = 0;
109                for (int i = 0; i < buckets.length; i++) {
110                        newCount = lastCount + buckets[i].getCount();
111                        if (expectedCount >= lastCount && expectedCount < newCount) {
112                                bucketIndex = i;
113                                break;
114                        }
115                        lastCount = newCount;
116                }
117                // Check that bucket index is in bounds
118                if (bucketIndex == 0) {
119                        throw new IllegalStateException("Quantile out of bounds: decrease min");
120                } else if (bucketIndex == bucketNb + 1) {
121                        throw new IllegalStateException("Quantile out of bounds: increase max");
122                }
123                // Interpolation of value
124                final Bucket bucket = buckets[bucketIndex];
125                return estimateQuantile(bucket, expectedCount, lastCount);
126        }
127
128        /**
129         * Interpolate quantile located in given Bucket using linear regression.
130         * <ul>
131         * <li>Quantile is between {@link Bucket#min} and {@link Bucket#max}</li>
132         * <li>Expected count is between last count and last count+{@link Bucket#count}</li>
133         * </ul>
134         *
135         * @param bucket Current bucket containing the quantile
136         * @param expectedCount Searched value
137         * @param lastCount Value of the bucket lower bound
138         * @return Compute quantile
139         */
140        protected double estimateQuantile(Bucket bucket, double expectedCount, double lastCount) {
141                return bucket.getMin() + (expectedCount - lastCount) * (bucket.getMax() - bucket.getMin()) / bucket.getCount();
142        }
143
144        /**
145         * Get the bucket containing the given value.
146         * Bucket should be sorted, the bucket whose min/max bounds are around the value is returned.
147         *
148         * @param value Value
149         * @return Bucket containing given value
150         */
151        protected Bucket getBucketForValue(long value) {
152                for (Bucket bucket : buckets) {
153                        if (bucket.contains(value)) {
154                                return bucket;
155                        }
156                }
157                throw new IllegalStateException("Non continuous buckets.");
158        }
159
160        /** Searches the appropriate bucket and add the value in it. */
161        public void addValue(long value) {
162                synchronized (buckets) {
163                        getBucketForValue(value).incrementCount();
164                }
165        }
166
167        /** For each value, search the appropriate bucket and add the value in it. */
168        public void addValues(Collection<Long> values) {
169                synchronized (buckets) {
170                        for (Long value : values) {
171                                addValue(value);
172                        }
173                }
174        }
175
176        /**
177         * Computes quantile.
178         *
179         * @param ratio Nth quantile, 0.5 is median. Expects values between 0 and 1.
180         * @return quantile
181         */
182        public double getQuantile(double ratio) {
183                synchronized (buckets) {
184                        int totalCount = checkAndGetTotalCount();
185                        return computeQuantile(ratio, totalCount);
186                }
187        }
188
189        /**
190         * Computes median.
191         *
192         * @return Median
193         */
194        public double getMedian() {
195                return getQuantile(0.5D);
196        }
197
198        /** Computes first (=0.25), second (=median=0.5) and third (=0.75) quartiles. */
199        public Double[] getQuartiles() {
200                return getQuantiles(0.25D, 0.50D, 0.75D);
201        }
202
203        /**
204         * Computes many quantiles.
205         *
206         * @param ratios Nth quantiles, 0.5 is median. Expects values between 0 and 1.
207         * @return quantiles or {@code null}, if computation failed
208         */
209        @SuppressWarnings("EmptyCatchBlock")
210        public Double[] getQuantiles(double... ratios) {
211                synchronized (buckets) {
212                        final Double[] quantiles = new Double[ratios.length];
213                        try {
214                                final int totalCount = checkAndGetTotalCount();
215                                for (int i = 0; i < ratios.length; i++) {
216                                        try {
217                                                quantiles[i] = computeQuantile(ratios[i], totalCount);
218                                        } catch (IllegalStateException e) {
219                                        }
220                                }
221                        } catch (IllegalStateException e) {
222                        }
223                        return quantiles;
224                }
225        }
226
227        public LogTemplate<Split> getLogTemplate() {
228                return logTemplate;
229        }
230
231        public void setLogTemplate(LogTemplate<Split> logTemplate) {
232                this.logTemplate = logTemplate;
233        }
234
235        /** Sample buckets and quantiles state. */
236        public BucketsSample sample() {
237                synchronized (buckets) {
238                        BucketSample[] bucketSamples = new BucketSample[buckets.length];
239                        for (int i = 0; i < buckets.length; i++) {
240                                bucketSamples[i] = buckets[i].sample();
241                        }
242                        Double[] quantiles = getQuantiles(0.50D, 0.90D);
243                        return new BucketsSample(bucketSamples, quantiles[0], quantiles[1]);
244                }
245        }
246
247        /**
248         * String containing: min/max/number configuration and 50%, 75% and 90% quantiles if available.
249         * Warning this method can be expensive as it is performing computation.
250         */
251        @Override
252        public String toString() {
253                return toString(false);
254        }
255
256        private String toString(boolean bars) {
257                StringBuilder stringBuilder = new StringBuilder("Buckets[");
258                stringBuilder.append("min=").append(presentNanoTime(min))
259                        .append(",max=").append(presentNanoTime(max))
260                        .append(",nb=").append(bucketNb)
261//                      .append(",width=").append(presentNanoTime(width)) // i don't know how important this information in that String.
262                        .append("] Quantiles[");
263                final String eol = System.getProperty("line.separator");
264                final String eoc = "\t";
265                BucketsSample bucketsSample = sample();
266                if (bucketsSample.getMedian() != null) {
267                        stringBuilder.append("median=").append(presentNanoTime(bucketsSample.getMedian()));
268                }
269                if (bucketsSample.getPercentile90() != null) {
270                        stringBuilder.append(",90%=").append(presentNanoTime(bucketsSample.getPercentile90()));
271                }
272                stringBuilder.append("]");
273                if (bars) {
274                        stringBuilder.append(eol);
275                        int maxCount = 0;
276                        final int barMax = 10;
277                        for (BucketSample bucketSample : bucketsSample.getBuckets()) {
278                                maxCount = Math.max(maxCount, bucketSample.getCount());
279                        }
280                        for (BucketSample bucketSample : bucketsSample.getBuckets()) {
281                                if (bucketSample.getMin() != Long.MIN_VALUE) {
282                                        stringBuilder.append(presentNanoTime(bucketSample.getMin()));
283                                }
284                                stringBuilder.append(eoc);
285                                if (bucketSample.getMax() != Long.MAX_VALUE) {
286                                        stringBuilder.append(presentNanoTime(bucketSample.getMax()));
287                                }
288                                stringBuilder.append(eoc)
289                                        .append(bucketSample.getCount()).append(eoc);
290                                if (maxCount > 0) {
291                                        final int barSize = bucketSample.getCount() * barMax / maxCount;
292                                        for (int i = 0; i < barSize; i++) {
293                                                stringBuilder.append('#');
294                                        }
295                                }
296                                stringBuilder.append(eol);
297                        }
298                }
299                return stringBuilder.toString();
300        }
301
302        /** Clears all buckets. */
303        public void clear() {
304                synchronized (buckets) {
305                        for (Bucket bucket : buckets) {
306                                bucket.clear();
307                        }
308                }
309        }
310
311        /**
312         * Returns the bucket list.
313         *
314         * @return list of buckets
315         */
316        public List<Bucket> getBuckets() {
317                return Collections.unmodifiableList(Arrays.asList(buckets));
318        }
319
320        /** Transforms buckets and quantiles into a loggable message. */
321        public String getLogMessage(Split lastSplit) {
322                return lastSplit.getStopwatch().getName() + " " + toString(true);
323        }
324
325        /** Logs eventually buckets config and quantiles. */
326        public void log(Split lastSplit) {
327                logTemplate.log(lastSplit, this);
328        }
329
330        public int getBucketNb() {
331                return bucketNb;
332        }
333
334        public long getMin() {
335                return min;
336        }
337
338        public long getMax() {
339                return max;
340        }
341}