feat(Parser): just-in-time YTNode generation (#310)

* refactor: merge NavigatableText into Text

* fix(Text): data might not be object

* refactor: remove GetParserByName from map

* feat(Parser): just-in-time YTNode generation

* refactor: cleanup YTNodeGenerator

* fix: YTNode map imports

* feat(YTNodeGenerator): primative types

Add support for inferring primatives types

* fix(YTNodeGenerator): NavigationEndpoint detection

* fix(YTNodeGenerator): fix generated typescript

Correct types and linting for generated typescript class

* chore: update parsers after merge

* feat: add support for object type inference

* fix: object type def

* docs: basic YTNodeGenerator explanation

* docs: tsdoc for YTNodeGenerator

* docs: update parser updating guide

* fix: apply suggested changes

* docs: accessing generated nodes

docs/updating-the-parser.md

@@ -5,16 +5,18 @@ YouTube is constantly changing, so it is not rare to see YouTube crawlers/scrape
 Our parser, on the other hand, was written so that it behaves similarly to an official client, parsing and mapping renderers (a.k.a YTNodes) dynamically without hard-coding their path in the response. This way, whenever a new renderer pops up (e.g; YouTube adds a new feature / minor UI changes) the library will print a warning similar to this:
 
 ```
-InnertubeError: SomeRenderer not found!
+SomeRenderer not found!
 This is a bug, want to help us fix it? Follow the instructions at https://github.com/LuanRT/YouTube.js/blob/main/docs/updating-the-parser.md or report it at https://github.com/LuanRT/YouTube.js/issues!
-    at Parser.printError (...)
-    at Parser.parseItem (...)
-    at Parser.parseArray (...) {
-  info: {
-    // renderer data, can be used as a reference to implement the renderer parser
-  },
-  date: 2022-05-22T22:16:06.831Z,
-  version: '2.2.3'
+Introspected and JIT generated this class in the meantime:
+class SomeRenderer extends YTNode {
+  static type = 'SomeRenderer';
+
+  // ...
+
+  constructor(data: RawNode) {
+    super();
+    // ...
+  }
 }
 ```
 
@@ -24,7 +26,7 @@ This warning **does not** throw an error. The parser itself will continue workin
 
 Thanks to the modularity of the parser, a renderer can be implemented by simply adding a new file anywhere in the [classes directory](../src/parser/classes)!
 
-For example, say we found a new renderer named `verticalListRenderer`, to let the parser know it exists we would have to create a file with the following structure:
+For example, say we found a new renderer named `verticalListRenderer`, to let the parser know it exists at compile-time we would have to create a file with the following structure:
 
 > `../classes/VerticalList.ts`
 
@@ -49,6 +51,8 @@ class VerticalList extends YTNode {
 export default VerticalList;
 ```
 
+You may use the parser's generated class for the new renderer as a starting point for your own implementation.
+
 Then update the parser map:
 
 ```bash

scripts/build-parser-map.cjs

@@ -3,9 +3,7 @@ const fs = require('fs');
 const path = require('path');
 
 const import_list = [];
-
-const json = [];
-const misc_exports = [];
+const misc_imports = [];
 
 glob.sync('../src/parser/classes/**/*.{js,ts}', { cwd: __dirname })
   .forEach((file) => {
@@ -16,44 +14,26 @@ glob.sync('../src/parser/classes/**/*.{js,ts}', { cwd: __dirname })
 
     if (is_misc) {
       const class_name = file.split('/').pop().replace('.js', '').replace('.ts', '');
-      import_list.push(`import { default as ${class_name} } from './classes/${file}.js';`);
-      misc_exports.push(class_name);
+      misc_imports.push(`export { default as ${class_name} } from './classes/${file}.js';`);
     } else {
-      import_list.push(`import { default as ${import_name} } from './classes/${file}.js';
-export { ${import_name} };`);
-      json.push(import_name);
+      import_list.push(`export { default as ${import_name} } from './classes/${file}.js';`);
     }
   });
 
 fs.writeFileSync(
-  path.resolve(__dirname, '../src/parser/map.ts'),
+  path.resolve(__dirname, '../src/parser/nodes.ts'),
   `// This file was auto generated, do not edit.
 // See ./scripts/build-parser-map.js
-import { YTNodeConstructor } from './helpers.js';
 
 ${import_list.join('\n')}
+`
+);
 
-const map: Record<string, YTNodeConstructor> = {
-  ${json.join(',\n  ')}
-};
-
-export const Misc = {
-  ${misc_exports.join(',\n  ')}
-};
-
-/**
- * @param name - Name of the node to be parsed
- */
-export default function GetParserByName(name: string) {
-  const ParserConstructor = map[name];
-
-  if (!ParserConstructor) {
-    const error = new Error(\`Module not found: \${name}\`);
-    (error as any).code = 'MODULE_NOT_FOUND';
-    throw error;
-  }
+fs.writeFileSync(
+  path.resolve(__dirname, '../src/parser/misc.ts'),
+  `// This file was auto generated, do not edit.
+// See ./scripts/build-parser-map.js
 
-  return ParserConstructor;
-}
+${misc_imports.join('\n')}
 `
 );

src/parser/README.md

@@ -256,6 +256,65 @@ const videos = response.contents_memo.getType(Video);
 ## Adding new nodes
 Instructions can be found [here](https://github.com/LuanRT/YouTube.js/blob/main/docs/updating-the-parser.md).
 
+## Generating nodes at runtime
+YouTube constantly updates their client, and sometimes they add new nodes to the response. The parser needs to know about these new nodes in order to parse them correctly. Once a new node is dicovered by the parser, it will attempt to generate a new node class for it.
+
+Using the existing `YTNode` class, you may interact with these new nodes in a type-safe way. However, you will not be able to cast them to the node's specific type, as this requires the node to be defined at compile-time.
+
+The current implementation recognises the following values:
+- Renderers
+- Renderer arrays
+- Text
+- Navigation endpoints
+- Author (does not currently detect the author thumbnails)
+- Thumbnails
+- Objects (key-value pairs)
+- Primatives (string, number, boolean, etc.)
+
+This may be expanded in the future.
+
+At runtime, these JIT-generated nodes will revalidate themselves when constructed so that when the types change, the node will be re-generated.
+
+To access these nodes that have been generated at runtime, you may use the `Parser.getParserByName(name: string)` method. You may also check if a parser has been generated for a node by using the `Parser.hasParser(name: string)` method.
+
+```ts
+import { Parser } from "youtubei.js";
+
+// We may check if we have a parser for a node.
+if (Parser.hasParser('Example')) {
+  // Then retrieve it.
+  const Example = Parser.getParserByName('Example');
+  // We may then use the parser as normal.
+  const example = new Example(data);
+}
+```
+
+You may also generate your own nodes ahead of time, given you have an example of one of the nodes.
+
+```ts
+import { Generator } from "youtubei.js";
+
+// Provided you have an example of the node `Example`
+const example_data = {
+  "title": {
+    "runs": [
+      {
+        "text": "Example"
+      }
+    ]
+  }
+}
+
+// The first argument is the name of the class, the second is the data you have for the node.
+// It will return a class that extends YTNode.
+const Example = Generator.YTNodeGenerator.generateRuntimeClass('Example', example_data);
+
+// You may now use this class as you would any other node.
+const example = new Example(example_data);
+
+const title = example.key('title').instanceof(Text).toString();
+```
+
 ## How it works
 
 If you decompile a YouTube client and analyze it, it becomes apparent that it uses classes such as `../youtube/api/innertube/MusicItemRenderer` and `../youtube/api/innertube/SectionListRenderer` to parse objects from the response, map them into models, and generate the UI. The website operates similarly, but instead uses plain JSON. You can think of renderers as components in a web framework.

src/parser/classes/GridPlaylist.ts

@@ -3,7 +3,6 @@ import Parser from '../index.js';
 import Thumbnail from './misc/Thumbnail.js';
 import PlaylistAuthor from './misc/PlaylistAuthor.js';
 import NavigationEndpoint from './NavigationEndpoint.js';
-import NavigatableText from './misc/NavigatableText.js';
 import { YTNode } from '../helpers.js';
 
 class GridPlaylist extends YTNode {
@@ -14,7 +13,7 @@ class GridPlaylist extends YTNode {
   author?: PlaylistAuthor;
   badges;
   endpoint: NavigationEndpoint;
-  view_playlist: NavigatableText;
+  view_playlist: Text;
   thumbnails: Thumbnail[];
   thumbnail_renderer;
   sidebar_thumbnails: Thumbnail[] | null;
@@ -32,7 +31,7 @@ class GridPlaylist extends YTNode {
 
     this.badges = Parser.parse(data.ownerBadges);
     this.endpoint = new NavigationEndpoint(data.navigationEndpoint);
-    this.view_playlist = new NavigatableText(data.viewPlaylistText);
+    this.view_playlist = new Text(data.viewPlaylistText);
     this.thumbnails = Thumbnail.fromResponse(data.thumbnail);
     this.thumbnail_renderer = Parser.parse(data.thumbnailRenderer);
     this.sidebar_thumbnails = [].concat(...data.sidebarThumbnails?.map((thumbnail: any) => Thumbnail.fromResponse(thumbnail)) || []) || null;

src/parser/classes/Movie.ts

@@ -43,8 +43,8 @@ class Movie extends YTNode {
     this.author = new Author(data.longBylineText, data.ownerBadges, data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail);
 
     this.duration = {
-      text: data.lengthText ? new Text(data.lengthText).text : new Text(overlay_time_status).text,
-      seconds: timeToSeconds(data.lengthText ? new Text(data.lengthText).text : new Text(overlay_time_status).text)
+      text: data.lengthText ? new Text(data.lengthText).toString() : new Text(overlay_time_status).toString(),
+      seconds: timeToSeconds(data.lengthText ? new Text(data.lengthText).toString() : new Text(overlay_time_status).toString())
     };
 
     this.endpoint = new NavigationEndpoint(data.navigationEndpoint);

src/parser/classes/MusicSortFilterButton.ts

@@ -14,7 +14,7 @@ class MusicSortFilterButton extends YTNode {
   constructor(data: any) {
     super();
 
-    this.title = new Text(data.title).text;
+    this.title = new Text(data.title).toString();
     this.icon_type = data.icon?.icon_type || null;
     this.menu = Parser.parseItem(data.menu, MusicMultiSelectMenu);
   }

src/parser/classes/Playlist.ts

@@ -4,7 +4,6 @@ import Thumbnail from './misc/Thumbnail.js';
 import NavigationEndpoint from './NavigationEndpoint.js';
 import PlaylistAuthor from './misc/PlaylistAuthor.js';
 import { YTNode } from '../helpers.js';
-import NavigatableText from './misc/NavigatableText.js';
 
 class Playlist extends YTNode {
   static type = 'Playlist';
@@ -21,7 +20,7 @@ class Playlist extends YTNode {
   badges;
   endpoint: NavigationEndpoint;
   thumbnail_overlays;
-  view_playlist?: NavigatableText;
+  view_playlist?: Text;
 
   constructor(data: any) {
     super();
@@ -43,7 +42,7 @@ class Playlist extends YTNode {
     this.thumbnail_overlays = Parser.parseArray(data.thumbnailOverlays);
 
     if (data.viewPlaylistText) {
-      this.view_playlist = new NavigatableText(data.viewPlaylistText);
+      this.view_playlist = new Text(data.viewPlaylistText);
     }
   }
 }

src/parser/classes/PlaylistVideo.ts

@@ -47,7 +47,7 @@ class PlaylistVideo extends YTNode {
     }
 
     this.duration = {
-      text: new Text(data.lengthText).text,
+      text: new Text(data.lengthText).toString(),
       seconds: parseInt(data.lengthSeconds)
     };
   }

src/parser/classes/Video.ts

@@ -77,8 +77,8 @@ class Video extends YTNode {
     }
 
     this.duration = {
-      text: data.lengthText ? new Text(data.lengthText).text : new Text(overlay_time_status).text,
-      seconds: timeToSeconds(data.lengthText ? new Text(data.lengthText).text : new Text(overlay_time_status).text)
+      text: data.lengthText ? new Text(data.lengthText).toString() : new Text(overlay_time_status).toString(),
+      seconds: timeToSeconds(data.lengthText ? new Text(data.lengthText).toString() : new Text(overlay_time_status).toString())
     };
 
     this.show_action_menu = data.showActionMenu;

src/parser/classes/menus/MusicMultiSelectMenu.ts

@@ -13,7 +13,7 @@ class MusicMultiSelectMenu extends YTNode {
   constructor(data: RawNode) {
     super();
 
-    this.title = new Text(data.title.musicMenuTitleRenderer?.primaryText).text;
+    this.title = new Text(data.title.musicMenuTitleRenderer?.primaryText).toString();
     this.options = Parser.parseArray(data.options, [ MusicMultiSelectMenuItem, MusicMenuItemDivider ]);
   }
 }

src/parser/classes/menus/MusicMultiSelectMenuItem.ts

@@ -14,7 +14,7 @@ class MusicMultiSelectMenuItem extends YTNode {
   constructor(data: RawNode) {
     super();
 
-    this.title = new Text(data.title).text;
+    this.title = new Text(data.title).toString();
     this.form_item_entity_key = data.formItemEntityKey;
     this.selected_icon_type = data.selectedIcon?.iconType || null;
     this.endpoint = data.selectedCommand ? new NavigationEndpoint(data.selectedCommand) : null;

src/parser/classes/misc/Author.ts

@@ -1,24 +1,24 @@
 import Parser from '../../index.js';
-import NavigatableText from './NavigatableText.js';
 import NavigationEndpoint from '../NavigationEndpoint.js';
 import TextRun from './TextRun.js';
 import Thumbnail from './Thumbnail.js';
 import Constants from '../../../utils/Constants.js';
+import Text from './Text.js';
 
 class Author {
   #nav_text;
 
   id: string;
   name: string;
   thumbnails: Thumbnail[];
-  endpoint: NavigationEndpoint | null;
+  endpoint?: NavigationEndpoint;
   badges?: any;
   is_verified?: boolean | null;
   is_verified_artist?: boolean | null;
   url: string | null;
 
   constructor(item: any, badges?: any, thumbs?: any) {
-    this.#nav_text = new NavigatableText(item);
+    this.#nav_text = new Text(item);
 
     this.id =
       (this.#nav_text.runs?.[0] as TextRun)?.endpoint?.payload?.browseId ||

src/parser/classes/misc/Text.ts

@@ -1,5 +1,6 @@
 import TextRun from './TextRun.js';
 import EmojiRun from './EmojiRun.js';
+import NavigationEndpoint from '../NavigationEndpoint.js';
 import type { RawNode } from '../../index.js';
 
 export interface Run {
@@ -18,8 +19,9 @@ export function escape(text: string) {
 }
 
 class Text {
-  text: string;
+  text?: string;
   runs;
+  endpoint?: NavigationEndpoint;
 
   constructor(data: RawNode) {
     if (data?.hasOwnProperty('runs') && Array.isArray(data.runs)) {
@@ -29,16 +31,28 @@ class Text {
       );
       this.text = this.runs.map((run) => run.text).join('');
     } else {
-      this.text = data?.simpleText || 'N/A';
+      this.text = data?.simpleText;
+    }
+    if (typeof data === 'object' && data !== null && Reflect.has(data, 'navigationEndpoint')) {
+      this.endpoint = new NavigationEndpoint(data.navigationEndpoint);
+    }
+    if (typeof data === 'object' && data !== null && Reflect.has(data, 'titleNavigationEndpoint')) {
+      this.endpoint = new NavigationEndpoint(data.titleNavigationEndpoint);
     }
+    if (!this.endpoint)
+      this.endpoint = (this.runs?.[0] as TextRun)?.endpoint;
   }
 
   toHTML() {
     return this.runs ? this.runs.map((run) => run.toHTML()).join('') : this.text;
   }
 
+  isEmpty() {
+    return this.text === undefined;
+  }
+
   toString() {
-    return this.text;
+    return this.text || 'N/A';
   }
 }