fix(Intersection): bug causing selection edge case

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [next]
 
+- fix(Intersection): bug causing selection edge case [#8735](https://github.com/fabricjs/fabric.js/pull/8735)
 - lint(): fix eslint errors [#8729](https://github.com/fabricjs/fabric.js/pull/8729)
 - fix(TS): `this.constructor` types [#8675](https://github.com/fabricjs/fabric.js/issues/8675)
 - fix(DraggableText): drag image blur [#8712](https://github.com/fabricjs/fabric.js/pull/8712)

src/Intersection.ts

@@ -1,25 +1,10 @@
 import { Point } from './Point';
+import { calcAngleBetweenVectors, createVector } from './util/misc/vectors';
 
 /* Adaptation of work of Kevin Lindsey ([email protected]) */
 
 export type IntersectionType = 'Intersection' | 'Coincident' | 'Parallel';
 
-/**
- * **Assuming `T`, `A`, `B` are points on the same line**,
- * check if `T` is contained in `[A, B]` by comparing the direction of the vectors from `T` to `A` and `B`
- * @param T
- * @param A
- * @param B
- * @returns true if `T` is contained
- */
-const isContainedInInterval = (T: Point, A: Point, B: Point) => {
-  const TA = new Point(T).subtract(A);
-  const TB = new Point(T).subtract(B);
-  return (
-    Math.sign(TA.x) !== Math.sign(TB.x) || Math.sign(TA.y) !== Math.sign(TB.y)
-  );
-};
-
 export class Intersection {
   declare points: Point[];
 
@@ -54,8 +39,28 @@ export class Intersection {
     return this;
   }
 
+  /**
+   * check if `T` is contained in `[A, B]` by comparing the direction of the vectors from `T` to `A` and `B`
+   * @param T
+   * @param A
+   * @param B
+   * @returns true if `T` is contained
+   */
+  static isContainedInInterval(T: Point, A: Point, B: Point) {
+    const AB = createVector(A, B);
+    return (
+      T.eq(A) ||
+      T.eq(B) ||
+      (!AB.eq(Point.ZERO) &&
+        calcAngleBetweenVectors(AB, createVector(T, B)) === 0 &&
+        calcAngleBetweenVectors(AB, createVector(A, T)) === 0)
+    );
+  }
+
   /**
    * Checks if a line intersects another
+   * @see {@link https://en.wikipedia.org/wiki/Line%E2%80%93line_intersection line intersection}
+   * @see {@link https://en.wikipedia.org/wiki/Cramer%27s_rule Cramer's rule}
    * @static
    * @param {Point} a1
    * @param {Point} a2
@@ -73,12 +78,12 @@ export class Intersection {
     aInfinite = true,
     bInfinite = true
   ): Intersection {
-    const b2xb1x = b2.x - b1.x,
-      a1yb1y = a1.y - b1.y,
+    const a2xa1x = a2.x - a1.x,
+      a2ya1y = a2.y - a1.y,
+      b2xb1x = b2.x - b1.x,
       b2yb1y = b2.y - b1.y,
       a1xb1x = a1.x - b1.x,
-      a2ya1y = a2.y - a1.y,
-      a2xa1x = a2.x - a1.x,
+      a1yb1y = a1.y - b1.y,
       uaT = b2xb1x * a1yb1y - b2yb1y * a1xb1x,
       ubT = a2xa1x * a1yb1y - a2ya1y * a1xb1x,
       uB = b2yb1y * a2xa1x - b2xb1x * a2ya1y;
@@ -100,10 +105,10 @@ export class Intersection {
         const segmentsCoincide =
           aInfinite ||
           bInfinite ||
-          isContainedInInterval(a1, b1, b2) ||
-          isContainedInInterval(a2, b1, b2) ||
-          isContainedInInterval(b1, a1, a2) ||
-          isContainedInInterval(b2, a1, a2);
+          Intersection.isContainedInInterval(a1, b1, b2) ||
+          Intersection.isContainedInInterval(a2, b1, b2) ||
+          Intersection.isContainedInInterval(b1, a1, a2) ||
+          Intersection.isContainedInInterval(b2, a1, a2);
         return new Intersection(segmentsCoincide ? 'Coincident' : undefined);
       } else {
         return new Intersection('Parallel');

src/Point.ts

@@ -12,9 +12,10 @@ export interface IPoint {
  */
 export class Point implements IPoint {
   declare x: number;
-
   declare y: number;
 
+  static ZERO = new Point();
+
   constructor();
   constructor(x: number, y: number);
   constructor(point: IPoint);
@@ -356,7 +357,7 @@ export class Point implements IPoint {
    * @param {TRadian} radians The radians of the angle for the rotation
    * @return {Point} The new rotated point
    */
-  rotate(radians: TRadian, origin: IPoint = originZero): Point {
+  rotate(radians: TRadian, origin: IPoint = Point.ZERO): Point {
     // TODO benchmark and verify the add and subtract how much cost
     // and then in case early return if no origin is passed
     const sinus = sin(radians),
@@ -384,5 +385,3 @@ export class Point implements IPoint {
     );
   }
 }
-
-const originZero = new Point(0, 0);

test/unit/intersection.js

@@ -83,12 +83,25 @@
   });
 
   QUnit.test('intersectSegmentSegment coincident but different', function(assert) {
-    var p1 = new fabric.Point(0, 0), p2 = new fabric.Point(0, 10),
-        p3 = new fabric.Point(0, 1), p4 = new fabric.Point(0, 9),
-        intersection = fabric.Intersection.intersectSegmentSegment(p1, p2, p3, p4);
-    assert.ok(intersection instanceof fabric.Intersection, 'returns a fabric.Intersection');
-    assert.equal(intersection.status, 'Coincident', 'it return a Coincident result');
-    assert.deepEqual(intersection.points, [], 'no point of intersections');
+    var a = new fabric.Point(0, 0),
+      b = new fabric.Point(0, 1),
+      c = new fabric.Point(0, 9),
+      d = new fabric.Point(0, 10);
+    [
+      fabric.Intersection.intersectSegmentSegment(a, d, b, c),
+      fabric.Intersection.intersectSegmentSegment(a, d, c, b),
+      fabric.Intersection.intersectSegmentSegment(d, a, b, c),
+      fabric.Intersection.intersectSegmentSegment(d, a, c, b),
+
+      fabric.Intersection.intersectSegmentSegment(a, c, b, d),
+      fabric.Intersection.intersectSegmentSegment(a, c, d, b),
+      fabric.Intersection.intersectSegmentSegment(c, a, b, d),
+      fabric.Intersection.intersectSegmentSegment(c, a, d, b),
+    ].forEach(intersection => {
+      assert.ok(intersection instanceof fabric.Intersection, 'returns a fabric.Intersection');
+      assert.equal(intersection.status, 'Coincident', 'it return a Coincident result');
+      assert.deepEqual(intersection.points, [], 'no point of intersections');
+    });
   });
 
   QUnit.test('intersectSegmentSegment no coincident, intersectLineLine coincident', function (assert) {
@@ -249,6 +262,31 @@
     assert.equal(intersection.points.length, 0, '0 points of intersections');
   });
 
+  QUnit.test('intersectPolygonRectangle line edge case', function (assert) {
+    const points = [
+      new fabric.Point(2, 2),
+      new fabric.Point(4, 2),
+      new fabric.Point(4, 4),
+      new fabric.Point(2, 4)
+    ];
+    [
+      [
+        new fabric.Point(10, 3),
+        new fabric.Point(30, 3)
+      ],
+      [
+        new fabric.Point(3, 10),
+        new fabric.Point(3, 30)
+      ]
+    ].forEach(([a, b]) => {
+      const intersection = fabric.Intersection.intersectPolygonRectangle(points, a, b);
+      assert.ok(typeof fabric.Intersection.intersectPolygonRectangle === 'function', 'has intersectPolygonPolygon function');
+      assert.ok(intersection instanceof fabric.Intersection, 'returns a fabric.Intersection');
+      assert.equal(intersection.status, undefined, `no intersection between { ${a} ${b} } and { ${points.join(' ')} }`);
+      assert.equal(intersection.points.length, 0, '0 points of intersections');
+    });
+  });
+
   QUnit.test('intersectPolygonPolygon coincident', function (assert) {
     const points = [
       new fabric.Point(0, 0),