node/benchmark/_node-bench-analysis.js

'use strict';

const { createHistogram } = require('node:perf_hooks');
const { inspect } = require('node:util');

function createRateHistogram(rates, scale, figures) {
  const histogram = createHistogram({ figures });
  for (const rate of rates) {
    const value = Math.max(1, Math.round(rate * scale));
    if (!Number.isSafeInteger(value)) {
      throw new RangeError('Benchmark rate is too large for the histogram scale');
    }
    histogram.record(value);
  }
  return histogram;
}

function holmAdjust(pValues) {
  const order = pValues
    .map((p, index) => ({ index, p }))
    .sort((a, b) => a.p - b.p);
  const adjusted = new Array(order.length);
  let running = 0;
  for (let index = 0; index < order.length; index++) {
    running = Math.max(
      running,
      Math.min(1, (order.length - index) * order[index].p),
    );
    adjusted[order[index].index] = running;
  }
  return adjusted;
}

function thresholdPValue(oldRates, newHistogram, scale, maxRegression) {
  const factor = 1 - maxRegression / 100;
  if (factor <= 0) return 1;
  const thresholdHistogram = createRateHistogram(
    oldRates.map((rate) => rate * factor), scale, 3);
  const result = thresholdHistogram.welchTest(newHistogram);
  if (Number.isNaN(result.pValue)) return 1;
  return result.tStatistic > 0 ?
    result.pValue / 2 : 1 - result.pValue / 2;
}

function isRegressionFailure(row, maxRegression) {
  return row.pThresholdAdjusted < 0.05 &&
         row.improvement + row.ci95 < -maxRegression;
}

function analyzeCompare(samples, scale, maxRegression) {
  const groups = new Map();
  for (const sample of samples) {
    let group = groups.get(sample.identity);
    if (group === undefined) {
      const suffix = sample.configuration === '' ?
        '' : ` ${sample.configuration}`;
      group = {
        name: `${sample.name}${suffix}`,
        new: [],
        old: [],
      };
      groups.set(sample.identity, group);
    }
    group[sample.binary].push(sample.rate);
  }

  const rows = [];
  let skipped = 0;
  for (const { name, old: oldRates, new: newRates } of groups.values()) {
    if (oldRates.length < 2 || newRates.length < 2) {
      skipped++;
      continue;
    }

    const oldHistogram = createRateHistogram(oldRates, scale, 3);
    const newHistogram = createRateHistogram(newRates, scale, 3);
    const oldMean = oldRates.reduce((sum, rate) => sum + rate, 0) /
      oldRates.length;
    const newMean = newRates.reduce((sum, rate) => sum + rate, 0) /
      newRates.length;
    const improvement = ((newMean - oldMean) / oldMean) * 100;
    const w95 = oldHistogram.welchTest(newHistogram, { confidence: 0.95 });
    const w99 = oldHistogram.welchTest(newHistogram, { confidence: 0.99 });
    const w999 = oldHistogram.welchTest(newHistogram, { confidence: 0.999 });
    let stars = '';
    if (w95.pValue < 0.001) stars = '***';
    else if (w95.pValue < 0.01) stars = ' **';
    else if (w95.pValue < 0.05) stars = '  *';
    const ciPercent = (result) => {
      const half = (result.confidenceInterval.upper -
                    result.confidenceInterval.lower) / 2;
      return (half / (oldMean * scale)) * 100;
    };
    const row = {
      ci95: ciPercent(w95),
      ci99: ciPercent(w99),
      ci999: ciPercent(w999),
      improvement,
      name,
      pValue: Number.isNaN(w95.pValue) ? 1 : w95.pValue,
      stars,
    };
    if (maxRegression !== undefined) {
      row.pThreshold = thresholdPValue(
        oldRates, newHistogram, scale, maxRegression);
    }
    rows.push(row);
  }

  const adjusted = holmAdjust(rows.map(({ pValue }) => pValue));
  const thresholdAdjusted = maxRegression === undefined ? null :
    holmAdjust(rows.map(({ pThreshold }) => pThreshold));
  let underpowered = 0;
  for (let index = 0; index < rows.length; index++) {
    const row = rows[index];
    row.pAdjusted = adjusted[index];
    if (thresholdAdjusted !== null) {
      row.pThresholdAdjusted = thresholdAdjusted[index];
    }
    row.inconclusive = maxRegression > 0 &&
                       row.stars.trim() === '' &&
                       row.ci95 > maxRegression;
    if (row.inconclusive) underpowered++;
  }

  const output = [];
  const maxNameLength = rows.reduce(
    (maximum, { name }) => Math.max(maximum, name.length), 0);
  const pad = (value, length) =>
    value + ' '.repeat(Math.max(0, length - value.length));
  const padStart = (value, length) =>
    ' '.repeat(Math.max(0, length - value.length)) + value;
  output.push(`${pad('', maxNameLength)}  confidence` +
              '   improvement   accuracy (*)    (**)   (***)');
  for (const row of rows) {
    const improvement =
      `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
    output.push(
      `${pad(row.name, maxNameLength)}  ${pad(row.stars, 10)}` +
      `  ${padStart(improvement, 11)}` +
      `   ±${row.ci95.toFixed(2)}%` +
      `  ±${row.ci99.toFixed(2)}%` +
      `  ±${row.ci999.toFixed(2)}%` +
      `${row.inconclusive ? '  (inconclusive)' : ''}`,
    );
  }

  if (skipped > 0) {
    output.push('');
    output.push(
      `Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` +
      ' skipped because Welch\'s t-test requires at least 2 samples per' +
      ' binary. Use --runs 2 or higher.',
    );
  }
  printCompareChart(output, rows, maxNameLength);

  output.push('');
  output.push(
    `Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).`,
    'Use --scale to adjust precision if needed.',
    '',
  );
  const significant = rows.filter(({ pAdjusted }) => pAdjusted < 0.05).length;
  output.push(
    'The confidence markers above are per-benchmark and uncorrected. ' +
    `After Holm-Bonferroni correction across ${rows.length} comparison` +
    `${rows.length === 1 ? '' : 's'}, ${significant} remain` +
    `${significant === 1 ? 's' : ''} significant at 5%.`,
  );
  if (maxRegression !== undefined) {
    output.push(
      `For --max-regression, one-sided p-values against the ` +
      `${maxRegression}% threshold were corrected separately.`,
    );
  }

  if (maxRegression > 0 && underpowered > 0) {
    output.push('');
    output.push(
      `Note: ${underpowered} of ${rows.length} comparison` +
      `${rows.length === 1 ? '' : 's'} could not resolve an effect as small ` +
      `as ${maxRegression}% and are marked (inconclusive). Raise --runs to ` +
      'narrow their confidence intervals.',
    );
  }

  const failures = maxRegression !== undefined ?
    rows.filter((row) => isRegressionFailure(row, maxRegression)) : [];
  if (failures.length > 0) {
    output.push('');
    output.push(
      `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` +
      ` regressed by more than ${maxRegression}% (the 95% interval excludes ` +
      `the threshold and its one-sided test is family-wise corrected across ` +
      `${rows.length} comparisons):`,
    );
    for (const failure of failures) {
      output.push(
        `  ${failure.name}  ${failure.improvement.toFixed(2)}% ` +
        `(95% CI up to ${(failure.improvement + failure.ci95).toFixed(2)}%, ` +
        `adjusted threshold p=` +
        `${failure.pThresholdAdjusted.toExponential(2)})`,
      );
    }
  }

  return {
    failed: failures.length > 0,
    output: `${output.join('\n')}\n`,
    rows,
  };
}

function printCompareChart(output, rows, maxNameLength) {
  if (rows.length === 0) return;
  const width = 40;
  const halfWidth = width / 2;
  let maximum = 0;
  for (const row of rows) {
    maximum = Math.max(maximum, Math.abs(row.improvement) + row.ci95);
  }
  if (maximum === 0) maximum = 1;
  const left = `-${maximum.toFixed(1)}%`;
  const right = `+${maximum.toFixed(1)}%`;
  const centerLabel = '0%';
  const labelPadding = maxNameLength + 5;
  output.push('');
  output.push(
    ' '.repeat(labelPadding) + left +
    ' '.repeat(Math.max(
      0, halfWidth - left.length - Math.floor(centerLabel.length / 2))) +
    centerLabel +
    ' '.repeat(Math.max(
      0, halfWidth - Math.ceil(centerLabel.length / 2) - right.length)) +
    right,
  );
  for (const row of rows) {
    const center = halfWidth;
    const result = center + (row.improvement / maximum) * halfWidth;
    const lower = center +
      ((row.improvement - row.ci95) / maximum) * halfWidth;
    const upper = center +
      ((row.improvement + row.ci95) / maximum) * halfWidth;
    let bar = '';
    for (let index = 0; index < width; index++) {
      const position = index + 0.5;
      if (index === Math.floor(center)) {
        bar += '|';
      } else if ((row.improvement >= 0 &&
                  position > center && position <= result) ||
                 (row.improvement < 0 &&
                  position < center && position >= result)) {
        bar += row.stars === '' ? '▓' : '█';
      } else if (position >= lower && position <= upper) {
        bar += '░';
      } else {
        bar += ' ';
      }
    }
    const label = `${row.improvement >= 0 ? '+' : ''}` +
      `${row.improvement.toFixed(2)}%`;
    output.push(
      `${row.name.padEnd(maxNameLength)}  ${bar}  ${label} ${row.stars.trim()}`,
    );
  }
}

function histogramScale(rates) {
  let minimum = Infinity;
  let maximum = 0;
  for (const rate of rates) {
    if (rate > 0 && rate < minimum) minimum = rate;
    if (rate > maximum) maximum = rate;
  }
  if (!Number.isFinite(minimum) || maximum === 0) return 1;
  let scale = 1;
  while (minimum * scale < 1e6 && maximum * scale < 1e15) scale *= 10;
  return scale;
}

function validateScatterParameters(samples, xAxis, category) {
  if (category !== undefined && category === xAxis) {
    throw new Error('--xaxis and --category must name different parameters');
  }
  for (const key of [xAxis, category]) {
    if (key === undefined) continue;
    if (samples.some(({ params }) =>
      !Object.hasOwn(params, key))) {
      const available = [...new Set(samples.flatMap(
        ({ params }) => Object.keys(params)))].sort();
      throw new Error(
        `The variable '${key}' is not present in every configuration. ` +
        `Available variables: ${available.join(', ')}`,
      );
    }
  }
}

function analyzeScatter(samples, xAxis, category, showChart) {
  validateScatterParameters(samples, xAxis, category);

  const parameterNames = [...new Set(samples.flatMap(
    ({ params }) => Object.keys(params)))];
  const aggregated = parameterNames.filter((name) => {
    if (name === xAxis || name === category) return false;
    const first = samples[0].params[name];
    return samples.some(({ params }) => params[name] !== first);
  });
  const groups = new Map();
  for (const sample of samples) {
    const xValue = sample.params[xAxis];
    const categoryValue = category === undefined ?
      undefined : sample.params[category];
    const key = valueKey([xValue, categoryValue]);
    let group = groups.get(key);
    if (group === undefined) {
      group = {
        categoryValue,
        members: [],
        observations: new Map(),
        xValue,
      };
      groups.set(key, group);
    }
    group.members.push(sample);
    let rates = group.observations.get(sample.observation);
    if (rates === undefined) {
      rates = [];
      group.observations.set(sample.observation, rates);
    }
    rates.push(sample.rate);
  }
  for (const group of groups.values()) {
    group.processRates = [...group.observations].map(([observation, rates]) => ({
      observation,
      rate: rates.reduce((sum, rate) => sum + rate, 0) / rates.length,
    }));
    group.rates = group.processRates.map(({ rate }) => rate);
  }

  const scale = histogramScale(samples.map(({ rate }) => rate));
  const compareValues = (a, b) => {
    if (typeof a === 'number' && typeof b === 'number') return a - b;
    return String(a).localeCompare(String(b));
  };
  const rows = [...groups.values()]
    .sort((a, b) => compareValues(a.xValue, b.xValue) ||
                    compareValues(a.categoryValue, b.categoryValue))
    .map((group) => {
      const histogram = createRateHistogram(group.rates, scale, 5);
      const count = group.rates.length;
      const mean = group.rates.reduce((sum, rate) => sum + rate, 0) / count;
      const meanInterval = histogram.meanCI();
      const confidenceInterval = count > 1 ?
        (meanInterval.upper - meanInterval.lower) / (2 * scale) : NaN;
      const medianInterval = histogram.percentileCI(50);
      const median = rawMedian(group.rates);
      const skewed = count > 1 &&
        (Math.abs(histogram.skewness) > 1 ||
         Math.abs(median - mean) > confidenceInterval);
      return {
        ...group,
        confidenceInterval,
        count,
        histogram,
        mean,
        median,
        medianLower: medianInterval.lower / scale,
        medianUpper: medianInterval.upper / scale,
        skewed,
      };
    });

  const legend = assignLabels(rows, 'xValue', 'xLabel');
  if (category !== undefined) {
    legend.push(...assignLabels(rows, 'categoryValue', 'categoryLabel'));
  }
  const output = [];
  const contamination = new Map();
  for (const variable of aggregated) {
    const share = varianceShare([...groups.values()], variable);
    contamination.set(variable, share);
    const percent = share === 1 ?
      '100' : (share >= 0.995 ? '>99' : (share * 100).toFixed(0));
    const suffix = Number.isNaN(share) ?
      '' : ` (explains ${percent}% of within-group variance)`;
    output.push(`aggregating variable: ${variable}${suffix}`);
  }
  const dominant = aggregated.filter(
    (variable) => contamination.get(variable) > 0.5);
  if (dominant.length > 0) {
    output.push('');
    wrapOutput(
      output,
      `${dominant.join(', ')} ${dominant.length === 1 ? 'explains' : 'explain'} ` +
      'most of the spread within each group. Pin the parameter or use it as ' +
      '--category; increasing --runs will not remove this source of variance.',
    );
  }
  if (aggregated.length > 0) output.push('');
  printScatterTable(output, rows, xAxis, category);
  if (showChart) printScatterChart(output, rows, xAxis, category);
  printScatterComparisons(output, rows, xAxis, category);
  if (legend.length > 0) {
    output.push('', 'Abbreviated values:');
    for (const { full, label } of legend) {
      output.push(`  ${label}`, `    = ${full}`);
    }
  }
  const singleSample = rows.filter(({ count }) => count < 2).length;
  if (singleSample > 0) {
    output.push('');
    wrapOutput(
      output,
      `Note: ${singleSample} group${singleSample === 1 ? ' has' : 's have'} ` +
      'only one sample, so no confidence interval could be estimated. Use ' +
      '--runs 2 or higher.',
    );
  }
  if (rows.some(({ skewed }) => skewed)) {
    output.push('');
    wrapOutput(
      output,
      '(!) marks groups where the median falls outside the mean confidence ' +
      'interval or the sample is strongly skewed. The median and its interval ' +
      'describe the typical run better for those groups.',
    );
  }
  return `${output.join('\n')}\n`;
}

function rawMedian(rates) {
  const sorted = [...rates].sort((a, b) => a - b);
  const middle = Math.floor(sorted.length / 2);
  return sorted.length % 2 === 0 ?
    (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
}

function valueKey(value) {
  return JSON.stringify(value, (_, item) => {
    if (typeof item === 'bigint') return { bigint: String(item) };
    return item;
  });
}

function varianceShare(groups, variable) {
  let between = 0;
  let total = 0;
  for (const group of groups) {
    if (group.members.length < 2) continue;
    const mean = group.members.reduce(
      (sum, sample) => sum + sample.rate, 0) / group.members.length;
    const levels = new Map();
    for (const sample of group.members) {
      const key = valueKey(sample.params[variable]);
      let level = levels.get(key);
      if (level === undefined) {
        level = { count: 0, sum: 0 };
        levels.set(key, level);
      }
      level.count++;
      level.sum += sample.rate;
    }
    for (const level of levels.values()) {
      between += level.count * ((level.sum / level.count) - mean) ** 2;
    }
    for (const sample of group.members) total += (sample.rate - mean) ** 2;
  }
  return total === 0 ? NaN : Math.min(1, between / total);
}

function effectSizeLabel(delta) {
  const absolute = Math.abs(delta);
  if (absolute < 0.147) return 'negligible';
  if (absolute < 0.33) return 'small';
  if (absolute < 0.474) return 'medium';
  return 'large';
}

const mannWhitneyFloors = new Map();
function mannWhitneyFloor(firstCount, secondCount) {
  const key = `${firstCount},${secondCount}`;
  let floor = mannWhitneyFloors.get(key);
  if (floor !== undefined) return floor;
  const low = createHistogram({ figures: 5 });
  const high = createHistogram({ figures: 5 });
  for (let index = 0; index < firstCount; index++) low.record(10000 + index);
  for (let index = 0; index < secondCount; index++) {
    high.record(10000 + firstCount + index);
  }
  floor = high.mannWhitneyTest(low).pValue;
  mannWhitneyFloors.set(key, floor);
  return floor;
}

function printScatterComparisons(output, rows, xAxis, category) {
  const usable = rows.filter(({ count }) => count > 1);
  if (usable.length < 2) return;
  const series = new Map();
  for (const row of usable) {
    const key = valueKey(row.categoryValue);
    if (!series.has(key)) series.set(key, []);
    series.get(key).push(row);
  }
  const sections = [];
  let floor = 0;
  for (const group of series.values()) {
    if (group.length < 2) continue;
    const entries = [];
    for (let index = 1; index < group.length; index++) {
      const previous = group[index - 1];
      const current = group[index];
      // Configurations in one file share a process. Split consecutive groups
      // across disjoint outer-process sets so the unpaired test does not treat
      // correlated observations as independent.
      const parity = index % 2;
      const previousRates = previous.processRates
        .filter(({ observation }) => observation % 2 === parity)
        .map(({ rate }) => rate);
      const currentRates = current.processRates
        .filter(({ observation }) => observation % 2 !== parity)
        .map(({ rate }) => rate);
      if (previousRates.length === 0 || currentRates.length === 0) continue;
      const comparisonScale = histogramScale([
        ...previousRates,
        ...currentRates,
      ]);
      const previousHistogram =
        createRateHistogram(previousRates, comparisonScale, 5);
      const currentHistogram =
        createRateHistogram(currentRates, comparisonScale, 5);
      const { pValue } = currentHistogram.mannWhitneyTest(previousHistogram);
      const delta = currentHistogram.cliffsD(previousHistogram);
      const previousMean = previousRates.reduce(
        (sum, rate) => sum + rate, 0) / previousRates.length;
      const currentMean = currentRates.reduce(
        (sum, rate) => sum + rate, 0) / currentRates.length;
      const change = ((currentMean - previousMean) / previousMean) * 100;
      floor = Math.max(
        floor, mannWhitneyFloor(previousRates.length, currentRates.length));
      let ratio = '';
      let exponent = '';
      if (typeof previous.xValue === 'number' &&
          typeof current.xValue === 'number' &&
          previous.xValue > 0 && current.xValue > 0 &&
          previous.xValue !== current.xValue &&
          previousMean > 0 && currentMean > 0) {
        const xRatio = current.xValue / previous.xValue;
        const value = Math.log(currentMean / previousMean) / Math.log(xRatio);
        ratio = `${xRatio.toFixed(1)}x`;
        exponent = `${value >= 0 ? '+' : ''}${value.toFixed(2)}`;
      }
      entries.push(
        `    ${previous.xLabel} -> ${current.xLabel}` +
        `  ${change >= 0 ? '+' : ''}${change.toFixed(2)}%` +
        (exponent === '' ? '' : `  ${ratio}  exponent=${exponent}`) +
        `  p=${pValue < 1e-4 ? pValue.toExponential(1) : pValue.toFixed(4)}` +
        `  delta=${delta >= 0 ? '+' : ''}${delta.toFixed(3)}` +
        ` (${effectSizeLabel(delta)})`,
      );
    }
    if (entries.length > 0) {
      sections.push({
        entries,
        heading: category === undefined ?
          undefined : `${category}=${group[0].categoryLabel}`,
      });
    }
  }
  if (sections.length === 0) return;
  output.push('', `Change between consecutive ${xAxis} values ` +
                  `(Mann-Whitney U on disjoint process sets, Cliff's delta):`);
  for (const section of sections) {
    output.push('');
    if (section.heading !== undefined) output.push(`  ${section.heading}`);
    output.push(...section.entries);
  }
  if (floor >= 0.05) {
    output.push('');
    wrapOutput(
      output,
      `Warning: at this sample size the smallest p-value this test can ` +
      `produce is ${floor.toFixed(4)}, so no comparison above can reach ` +
      'significance. Raise --runs.',
    );
  } else if (floor >= 0.005) {
    output.push('');
    wrapOutput(
      output,
      `Note: at this sample size the smallest p-value this test can produce ` +
      `is ${floor.toFixed(4)}. Raise --runs to strengthen non-significant ` +
      'results.',
    );
  }
}

function formatRate(rate) {
  return rate.toLocaleString('en-US', {
    maximumFractionDigits: 1,
    minimumFractionDigits: 1,
  });
}

function displayWidth(value) {
  return [...value].length;
}

function pad(value, width, right) {
  const padding = ' '.repeat(Math.max(0, width - displayWidth(value)));
  return right ? padding + value : value + padding;
}

function truncateMiddle(value, maximum = 24) {
  const characters = [...value];
  if (characters.length <= maximum) return value;
  const retained = maximum - 3;
  const head = Math.ceil(retained / 2);
  const tail = Math.floor(retained / 2);
  return `${characters.slice(0, head).join('')}...` +
         characters.slice(-tail).join('');
}

function assignLabels(rows, valueName, labelName) {
  const assigned = new Map();
  const used = new Map();
  const legend = [];
  for (const row of rows) {
    const key = valueKey(row[valueName]);
    let label = assigned.get(key);
    if (label === undefined) {
      const full = typeof row[valueName] === 'string' ?
        inspect(row[valueName]) : String(row[valueName]);
      const abbreviated = truncateMiddle(full);
      const collisions = used.get(abbreviated) ?? 0;
      used.set(abbreviated, collisions + 1);
      label = collisions === 0 ?
        abbreviated : `${abbreviated}~${collisions + 1}`;
      assigned.set(key, label);
      if (label !== full) legend.push({ full, label });
    }
    row[labelName] = label;
  }
  return legend;
}

function printScatterTable(output, rows, xAxis, category) {
  const header = [xAxis];
  if (category !== undefined) header.push(category);
  header.push(
    'samples', 'rate', 'confidence.interval', 'median', 'median.interval', '');
  const body = rows.map((row) => {
    const values = [row.xLabel];
    if (category !== undefined) values.push(row.categoryLabel);
    const medianInterval = row.count > 1 ?
      `[${(((row.medianLower - row.median) / row.median) * 100).toFixed(2)}%, ` +
      `+${(((row.medianUpper - row.median) / row.median) * 100).toFixed(2)}%]` :
      'NA';
    values.push(
      String(row.count),
      formatRate(row.mean),
      Number.isNaN(row.confidenceInterval) ?
        'NA' :
        `${formatRate(row.confidenceInterval)} ` +
        `(±${((row.confidenceInterval / row.mean) * 100).toFixed(2)}%)`,
      formatRate(row.median),
      medianInterval,
      row.skewed ? '(!)' : '',
    );
    return values;
  });
  const widths = header.map((value, index) => Math.max(
    displayWidth(value),
    ...body.map((values) => displayWidth(values[index])),
  ));
  const right = [typeof rows[0].xValue === 'number'];
  if (category !== undefined) {
    right.push(typeof rows[0].categoryValue === 'number');
  }
  right.push(true, true, true, true, true, false);
  const format = (values) => values.map(
    (value, index) => pad(value, widths[index], right[index])).join('  ').trimEnd();
  output.push(format(header));
  for (const values of body) output.push(format(values));
}

function printScatterChart(output, rows, xAxis, category) {
  if (rows.length === 0) return;
  const width = 40;
  let maximum = 0;
  for (const row of rows) {
    maximum = Math.max(
      maximum,
      row.mean + (Number.isNaN(row.confidenceInterval) ?
        0 : row.confidenceInterval),
    );
  }
  if (maximum === 0) return;
  const labels = rows.map((row) => {
    let label = `${xAxis}=${row.xLabel}`;
    if (category !== undefined) label += ` ${category}=${row.categoryLabel}`;
    return truncateMiddle(label, 44);
  });
  const labelWidth = Math.max(...labels.map(displayWidth));
  const rateWidth = Math.max(...rows.map(({ mean }) =>
    displayWidth(formatRate(mean))));
  const axis = formatRate(maximum);
  const indent = ' '.repeat(labelWidth + 2);
  output.push(
    '',
    'Rate in operations/second; longer is faster. │ marks the mean and the',
    'shaded band (░) is its 95% confidence interval.',
    '',
    `${indent}0${' '.repeat(Math.max(1, width - 1 - displayWidth(axis)))}${axis}`,
    `${indent}+${'-'.repeat(width - 2)}+`,
  );
  let previous;
  for (let index = 0; index < rows.length; index++) {
    const row = rows[index];
    if (previous !== undefined && previous !== row.xValue) output.push('');
    previous = row.xValue;
    const interval = Number.isNaN(row.confidenceInterval) ?
      0 : row.confidenceInterval;
    const end = (row.mean / maximum) * width;
    const lower = ((row.mean - interval) / maximum) * width;
    const upper = ((row.mean + interval) / maximum) * width;
    const meanCell = Math.min(width - 1, Math.floor(end));
    let bar = '';
    for (let cell = 0; cell < width; cell++) {
      const position = cell + 0.5;
      if (cell === meanCell) bar += '│';
      else if (position >= lower && position <= upper) bar += '░';
      else if (position <= end) bar += '█';
      else bar += ' ';
    }
    output.push(
      `${pad(labels[index], labelWidth, false)}  ${bar}  ` +
      pad(formatRate(row.mean), rateWidth, true),
    );
  }
}

function wrapOutput(output, text, width = 76) {
  let line = '';
  for (const word of text.split(/\s+/)) {
    if (line === '') line = word;
    else if (displayWidth(line) + displayWidth(word) + 1 <= width) {
      line += ` ${word}`;
    } else {
      output.push(line);
      line = word;
    }
  }
  if (line !== '') output.push(line);
}

module.exports = {
  analyzeCompare,
  analyzeScatter,
  holmAdjust,
  isRegressionFailure,
  validateScatterParameters,
};