fix(lexer): set default Data to undefined, fix #33

src/lexer/action/action.ts

@@ -58,7 +58,7 @@ export type AcceptedActionDecorator<
   ctx: AcceptedActionDecoratorContext<Data, ActionState, ErrorType>,
 ) => ActionOutput<NewData, NewErrorType>;
 
-export class Action<Data = never, ActionState = never, ErrorType = never> {
+export class Action<Data = undefined, ActionState = never, ErrorType = never> {
   readonly exec: WrappedActionExec<Data, ActionState, ErrorType>;
   /**
    * This flag is to indicate whether this action's output might be muted.
@@ -99,7 +99,7 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
    * Create an `Action` from `ActionExec`.
    * This should be treat as the public constructor of `Action`.
    */
-  static exec<Data = never, ActionState = never, ErrorType = never>(
+  static exec<Data = undefined, ActionState = never, ErrorType = never>(
     exec: ActionExec<Data, ActionState, ErrorType>,
     options?: Partial<Pick<Action<Data, ActionState, ErrorType>, "maybeMuted">>,
   ): Action<Data, ActionState, ErrorType> {
@@ -116,7 +116,7 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
   /**
    * Create an `Action` from `SimpleActionExec`.
    */
-  static simple<Data = never, ActionState = never, ErrorType = never>(
+  static simple<Data = undefined, ActionState = never, ErrorType = never>(
     f: SimpleActionExec<Data, ActionState, ErrorType>,
   ): Action<Data, ActionState, ErrorType> {
     return Action.exec((input) => {
@@ -132,7 +132,7 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
           digested: res,
           content: input.buffer.slice(input.start, input.start + res),
           data: undefined,
-        } as AcceptedActionExecOutput<never, ErrorType>;
+        } as AcceptedActionExecOutput<Data, ErrorType>;
       }
       if (typeof res === "string") {
         if (res.length <= 0) return rejectedActionOutput;
@@ -142,7 +142,7 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
           digested: res.length,
           content: res,
           data: undefined,
-        } as AcceptedActionExecOutput<never, ErrorType>;
+        } as AcceptedActionExecOutput<Data, ErrorType>;
       }
       // else, res is SimpleAcceptedActionOutput
       res.digested ??= res.content!.length; // if digested is undefined, content must be defined
@@ -215,7 +215,7 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
   /**
    * Create an `Action` from `RegExp/Action/SimpleActionExec`.
    */
-  static from<Data = never, ActionState = never, ErrorType = never>(
+  static from<Data = undefined, ActionState = never, ErrorType = never>(
     r: IntoAction<Data, ActionState, ErrorType>,
   ): Action<Data, ActionState, ErrorType> {
     return r instanceof RegExp
@@ -355,8 +355,8 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
    * Set data to `undefined` if `accept` is `true`.
    * Return a new action.
    */
-  clearData(): Action<never, ActionState, ErrorType> {
-    return this.data(() => undefined as never);
+  clearData(): Action<undefined, ActionState, ErrorType> {
+    return this.data(() => undefined);
   }
 
   /**
@@ -422,7 +422,7 @@ export class Action<Data = never, ActionState = never, ErrorType = never> {
    * This will reduce the lexer loop times to optimize the performance.
    * Return a new action.
    */
-  static reduce<Data = never, ActionState = never, ErrorType = never>(
+  static reduce<Data = undefined, ActionState = never, ErrorType = never>(
     ...actions: IntoAction<Data, ActionState, ErrorType>[]
   ): Action<Data, ActionState, ErrorType> {
     return actions.map((a) => Action.from(a)).reduce((a, b) => a.or(b));

src/lexer/action/builder.ts

@@ -7,7 +7,7 @@ export class ActionBuilder<ActionState, ErrorType> {
   /**
    * @alias {@link Action.exec}
    */
-  exec<Data = never>(
+  exec<Data = undefined>(
     ...props: Parameters<typeof Action.exec<Data, ActionState, ErrorType>>
   ) {
     return Action.exec<Data, ActionState, ErrorType>(...props);
@@ -16,7 +16,7 @@ export class ActionBuilder<ActionState, ErrorType> {
   /**
    * @alias {@link Action.simple}
    */
-  simple<Data = never>(
+  simple<Data = undefined>(
     ...props: Parameters<typeof Action.simple<Data, ActionState, ErrorType>>
   ) {
     return Action.simple<Data, ActionState, ErrorType>(...props);
@@ -41,7 +41,7 @@ export class ActionBuilder<ActionState, ErrorType> {
   /**
    * @alias {@link Action.from}
    */
-  from<Data = never>(
+  from<Data = undefined>(
     ...props: Parameters<typeof Action.from<Data, ActionState, ErrorType>>
   ) {
     return Action.from<Data, ActionState, ErrorType>(...props);
@@ -50,7 +50,7 @@ export class ActionBuilder<ActionState, ErrorType> {
   /**
    * @alias {@link Action.reduce}
    */
-  reduce<Data = never>(
+  reduce<Data = undefined>(
     ...props: Parameters<typeof Action.reduce<Data, ActionState, ErrorType>>
   ) {
     return Action.reduce<Data, ActionState, ErrorType>(...props);

src/lexer/builder.ts

@@ -123,7 +123,7 @@ export class Builder<
    * })
    */
   // TODO: different kinds map different data?
-  define<AppendKinds extends string, AppendData = never>(defs: {
+  define<AppendKinds extends string, AppendData = undefined>(defs: {
     [kind in AppendKinds]:
       | ActionSource<AppendData, ActionState, ErrorType>
       | ActionSource<AppendData, ActionState, ErrorType>[];

src/lexer/utils/common.ts

@@ -13,11 +13,11 @@ export function esc4regex(str: string) {
  * Use regex `\s+` instead of `\s` to reduce token emitted, to accelerate the lexing process.
  */
 export function whitespaces<ActionState = never, ErrorType = never>(): Action<
-  never,
+  undefined,
   ActionState,
   ErrorType
 > {
-  return Action.from<never, ActionState, ErrorType>(/\s+/);
+  return Action.from<undefined, ActionState, ErrorType>(/\s+/);
 }
 
 /**
@@ -39,7 +39,7 @@ export function fromTo<ActionState = never, ErrorType = never>(
      */
     autoGlobal?: boolean;
   },
-): Action<never, ActionState, ErrorType> {
+): Action<undefined, ActionState, ErrorType> {
   // make sure regex has the flag 'y/g' so we can use `regex.lastIndex` to reset state.
   if (from instanceof RegExp && (options.autoSticky ?? true))
     from = makeRegexAutoSticky(from);
@@ -112,7 +112,7 @@ export function comment<ActionState = never, ErrorType = never>(
     /** Default: `true`. */
     acceptEof?: boolean;
   },
-): Action<never, ActionState, ErrorType> {
+): Action<undefined, ActionState, ErrorType> {
   return fromTo<ActionState, ErrorType>(start, end, {
     ...options,
     acceptEof: options?.acceptEof ?? true,

src/lexer/utils/javascript/common.ts

@@ -5,7 +5,7 @@ import { comment as commonComment } from "../common";
  * Return an action that matches JavaScript comments (single line and multi line).
  */
 export function comment<ActionState = never, ErrorType = never>(): Action<
-  never,
+  undefined,
   ActionState,
   ErrorType
 > {

src/lexer/utils/javascript/numeric.ts

@@ -64,7 +64,7 @@ export function numericLiteral<
   const boundary = options?.boundary ?? true;
   const acceptInvalid = options?.acceptInvalid ?? true;
 
-  const valid = Action.from<never, ActionState, ErrorType>(
+  const valid = Action.from<undefined, ActionState, ErrorType>(
     enableSeparator
       ? new RegExp(
           `(?:0x[\\da-f]+|0o[0-7]+|\\d+(?:${separator}\\d+)*(?:\\.\\d+(?:${separator}\\d+)*)?(?:[eE][-+]?\\d+(?:${separator}\\d+)*)?)${

src/lexer/utils/javascript/regex.ts

@@ -26,10 +26,10 @@ export function regexLiteral<ActionState = never, ErrorType = never>(options?: {
 }): Action<{ invalid: boolean }, ActionState, ErrorType> {
   const action = (
     options?.boundary ?? true
-      ? Action.from<never, ActionState, ErrorType>(
+      ? Action.from<undefined, ActionState, ErrorType>(
           /\/(?:[^/\\]|\\.)+\/(?:[gimuy]*)(?=\W|$)/,
         )
-      : Action.from<never, ActionState, ErrorType>(
+      : Action.from<undefined, ActionState, ErrorType>(
           /\/(?:[^/\\]|\\.)+\/(?:[gimuy]*)/,
         )
   ).data(() => ({ invalid: false }));

src/lexer/utils/string.ts

@@ -54,7 +54,7 @@ export function stringLiteral<ActionState = never, ErrorType = never>(
   const acceptUnclosed = options?.acceptUnclosed ?? true;
   const lineContinuation = options?.lineContinuation ?? true;
 
-  const action = Action.from<never, ActionState, ErrorType>(
+  const action = Action.from<undefined, ActionState, ErrorType>(
     new RegExp(
       // open quote
       `(?:${esc4regex(open)})` +

src/lexer/utils/word.ts

@@ -5,7 +5,7 @@ import { Action } from "../action";
  */
 export function exact<ActionState = never, ErrorType = never>(
   ...ss: readonly string[]
-): Action<never, ActionState, ErrorType> {
+): Action<undefined, ActionState, ErrorType> {
   return Action.from((input) => {
     for (const s of ss) if (input.buffer.startsWith(s, input.start)) return s;
     return 0;
@@ -17,7 +17,7 @@ export function exact<ActionState = never, ErrorType = never>(
  */
 export function exactArray<ActionState = never, ErrorType = never>(
   ...ss: readonly string[]
-): Action<never, ActionState, ErrorType>[] {
+): Action<undefined, ActionState, ErrorType>[] {
   return ss.map((s) => exact(s));
 }
 
@@ -31,14 +31,15 @@ export function exactKind<
 >(
   ...ss: readonly Kinds[]
 ): {
-  [kind in Kinds]: Action<never, ActionState, ErrorType>;
+  [kind in Kinds]: Action<undefined, ActionState, ErrorType>;
 } {
-  const result: { [kind: string]: Action<never, ActionState, ErrorType> } = {};
+  const result: { [kind: string]: Action<undefined, ActionState, ErrorType> } =
+    {};
   for (const s of ss) {
     result[s] = exact(s);
   }
   return result as {
-    [kind in Kinds]: Action<never, ActionState, ErrorType>;
+    [kind in Kinds]: Action<undefined, ActionState, ErrorType>;
   };
 }
 
@@ -47,7 +48,7 @@ export function exactKind<
  */
 export function word<ActionState = never, ErrorType = never>(
   ...words: readonly string[]
-): Action<never, ActionState, ErrorType> {
+): Action<undefined, ActionState, ErrorType> {
   return Action.from((input) => {
     for (const word of words)
       if (
@@ -65,7 +66,7 @@ export function word<ActionState = never, ErrorType = never>(
  */
 export function wordArray<ActionState = never, ErrorType = never>(
   ...words: readonly string[]
-): Action<never, ActionState, ErrorType>[] {
+): Action<undefined, ActionState, ErrorType>[] {
   return words.map((s) => word(s));
 }
 
@@ -79,13 +80,14 @@ export function wordKind<
 >(
   ...words: readonly Kinds[]
 ): {
-  [kind in Kinds]: Action<never, ActionState, ErrorType>;
+  [kind in Kinds]: Action<undefined, ActionState, ErrorType>;
 } {
-  const result: { [kind: string]: Action<never, ActionState, ErrorType> } = {};
+  const result: { [kind: string]: Action<undefined, ActionState, ErrorType> } =
+    {};
   for (const w of words) {
     result[w] = word(w);
   }
   return result as {
-    [kind in Kinds]: Action<never, ActionState, ErrorType>;
+    [kind in Kinds]: Action<undefined, ActionState, ErrorType>;
   };
 }

src/parser/ELR/advanced/utils/grammar-expander.ts

@@ -21,11 +21,11 @@ export class GrammarExpander<
     never,
     | {
         kind: "";
-        data: never;
+        data: undefined;
       }
     | {
         kind: "grammar" | "rename";
-        data: never;
+        data: undefined;
       }
     | {
         kind: "literal";
@@ -35,7 +35,7 @@ export class GrammarExpander<
       }
     | {
         kind: "";
-        data: never;
+        data: undefined;
       },
     never,
     never

src/parser/ELR/advanced/utils/serialized-grammar-parser-data.ts

@@ -9,11 +9,11 @@ export const data: SerializableParserData<
   "gr",
   | {
       kind: "";
-      data: never;
+      data: undefined;
     }
   | {
       kind: "rename" | "grammar";
-      data: never;
+      data: undefined;
     }
   | {
       kind: "literal";
@@ -23,7 +23,7 @@ export const data: SerializableParserData<
     }
   | {
       kind: "";
-      data: never;
+      data: undefined;
     }
 > = {
   hash: 554900317,

tests/lexer/utils/javascript.test.ts

@@ -34,19 +34,19 @@ test("lexer utils javascript.numericLiteral", () => {
   expect(lexer.reset().lex(`  1e6_000`)?.content).toBe(`1e6_000`);
 
   // invalid
-  expect(lexer.reset().lex(`0o79`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`0xyz`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`1.2.`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`1..2`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`1e1e1`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`1e`)?.data.invalid).toBe(true);
+  expect(lexer.reset().lex(`0o79`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`0xyz`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`1.2.`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`1..2`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`1e1e1`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`1e`)?.data!.invalid).toBe(true);
   // additional test for #6
-  expect(lexer.reset().lex(`  0o79`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`  0xyz`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`  1.2.`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`  1..2`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`  1e1e1`)?.data.invalid).toBe(true);
-  expect(lexer.reset().lex(`  1e`)?.data.invalid).toBe(true);
+  expect(lexer.reset().lex(`  0o79`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`  0xyz`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`  1.2.`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`  1..2`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`  1e1e1`)?.data!.invalid).toBe(true);
+  expect(lexer.reset().lex(`  1e`)?.data!.invalid).toBe(true);
 
   // boundary
   expect(lexer.reset().lex(`123abc`)).toBe(null); // no boundary

utils/generate-serialized-grammar-parser.ts

@@ -25,18 +25,18 @@ const content = [
   "",
   `export const data: SerializableParserData<"gr", {
     kind: "";
-    data: never;
+    data: undefined;
 } | {
     kind: "rename" | "grammar";
-    data: never;
+    data: undefined;
 } | {
     kind: "literal";
     data: {
         unclosed: boolean;
     };
 } | {
     kind: "";
-    data: never;
+    data: undefined;
 }> = ${util.inspect(serializable, {
     // depth should be infinity, set it to a limited number in case of circular references
     depth: 10,