fix: precision problem in extended in v0.3.x (#158)

* fix: precision problem in extended

* refactor: declare a fixed-length array for extended

* chore: updte version

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antv/scale",
-  "version": "0.3.10",
+  "version": "0.3.11",
   "description": "The scale module for G2",
   "author": "https://github.com/orgs/antvis/people",
   "license": "MIT",

src/util/extended.ts

@@ -1,5 +1,4 @@
 import { head, indexOf, size, last } from '@antv/util';
-import { precisionAdd } from './math';
 
 export const DEFAULT_Q = [1, 5, 2, 2.5, 4, 3];
 
@@ -60,6 +59,11 @@ function legibility() {
   return 1;
 }
 
+// 解决 js 计算精度问题
+function pretty(n: number) {
+  return n < 1e-15 ? n : parseFloat(n.toFixed(15));
+}
+
 /**
  * An Extension of Wilkinson's Algorithm for Position Tick Labels on Axes
  * https://www.yuque.com/preview/yuque/0/2019/pdf/185317/1546999150858-45c3b9c2-4e86-4223-bf1a-8a732e8195ed.pdf
@@ -76,7 +80,7 @@ export default function extended(
   m: number = 5,
   onlyLoose: boolean = true,
   Q: number[] = DEFAULT_Q,
-  w: [number, number, number, number] = [0.25, 0.2, 0.5, 0.05],
+  w: [number, number, number, number] = [0.25, 0.2, 0.5, 0.05]
 ): { min: number; max: number; ticks: number[] } {
   // nan 也会导致异常
   if (Number.isNaN(dMin) || Number.isNaN(dMax) || typeof dMin !== 'number' || typeof dMax !== 'number' || !m) {
@@ -164,18 +168,18 @@ export default function extended(
     j += 1;
   }
 
+  const size = Math.floor((best.lmax - best.lmin) / best.lstep);
+  const ticks = new Array(size);
   let i = 0;
 
-  // 步长为浮点数时处理精度
-  const range = new Array(Math.floor((best.lmax - best.lmin) / best.lstep));
-
-  for (let tick = best.lmin; tick <= best.lmax; tick = precisionAdd(tick, best.lstep)) {
-    range[i] = tick;
+  for (let tick = best.lmin; tick <= best.lmax; tick += best.lstep) {
+    ticks[i] = pretty(tick);
     i += 1;
   }
+
   return {
-    min: Math.min(dMin, head(range)),
-    max: Math.max(dMax, last(range)),
-    ticks: range,
+    min: Math.min(dMin, head(ticks)),
+    max: Math.max(dMax, last(ticks)),
+    ticks,
   };
-};
+}

tests/unit/tick-method/quantile-spec.ts

@@ -65,7 +65,7 @@ describe('test quantile test', () => {
       tickCount: 11
     });
     // 从概率上说，随机分布会均匀分布在 0，100 之间，所以每个 十分位的数值都正好是 10 的倍数
-    expect(ticks[5]).toBeGreaterThan(49);
+    expect(ticks[5]).toBeGreaterThan(48);
     expect(ticks[5]).toBeLessThan(52);
   });
 });

tests/unit/util/extended-spec.ts

@@ -40,7 +40,7 @@ describe('extended ticks', function () {
 
   it('extended for c(0, 0.0000267519)', () => {
     const res = extended(0, 0.0000267519);
-    expect(res.ticks).toEqual([0, 0.00001, 0.00002]);
+    expect(res.ticks).toEqual([0, 0.00001, 0.00002, 0.00003]);
   });
 
   it('extended for c(0.0000237464, 0.0000586372)', () => {
@@ -52,7 +52,7 @@ describe('extended ticks', function () {
   it('extended for c(0.243, 0.584, 0.987, 0.153, 0.433)', () => {
     const scale1 = extended(0.153, 0.987);
     // d3 version == [0.2, 0.4, 0.6000000000000001, 0.8, 1]
-    expect(scale1.ticks).toEqual([0, 0.25, 0.5, 0.75, 1]);
+    expect(scale1.ticks).toEqual([ 0, 0.25, 0.5, 0.75, 1 ]);
     const scale2 = extended(0.153, 0.987, 10);
     // same as d3
     expect(scale2.ticks).toEqual([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]);
@@ -64,8 +64,25 @@ describe('extended ticks', function () {
     const scale2 = extended(-5, 605, 6, false);
     expect(scale2.ticks).toEqual([0, 100, 200, 300, 400, 500, 600]);
   });
+
   it('extended for 0.5 0.5000000000000001', () => {
     const scale = extended(0.5, 0.5000000000000001);
     expect(scale.ticks).toEqual([0.5]);
   });
+
+  it('tiny number', () => {
+    expect(extended(0, 0.1, 5).ticks).toStrictEqual([0, 0.025, 0.05, 0.075, 0.1]);
+    expect(extended(0, 0.01, 5).ticks).toStrictEqual([0, 0.0025, 0.005, 0.0075, 0.01]);
+    expect(extended(0, 0.001, 5).ticks).toStrictEqual([0, 0.00025, 0.0005, 0.00075, 0.001]);
+    expect(extended(0, 0.0001, 6).ticks).toStrictEqual([0, 0.00002, 0.00004, 0.00006, 0.00008, 0.0001, 0.00012]);
+    expect(extended(0, 0.00001, 6).ticks).toStrictEqual([
+      0, 0.000002, 0.000004, 0.000006, 0.000008, 0.00001, 0.000012,
+    ]);
+    expect(extended(0, 0.000001, 6).ticks).toStrictEqual([0, 0.0000002, 0.0000004, 0.0000006, 0.0000008, 0.000001]);
+    expect(extended(0, 1e-15, 6).ticks).toStrictEqual([0, 2e-16, 4e-16, 6e-16, 8e-16, 1e-15]);
+  });
+
+  it('precision', () => {
+    expect(extended(0, 1.2, 5).ticks).toStrictEqual([0, 0.3, 0.6, 0.9, 1.2]);
+  });
 });