diff --git a/.eslintrc.json b/.eslintrc.json index 427c7a16d6..501d452054 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -12,22 +12,7 @@ "src/headless/tsconfig.json", "test/api/tsconfig.json", "test/benchmark/tsconfig.json", - "addons/xterm-addon-attach/src/tsconfig.json", - "addons/xterm-addon-attach/test/tsconfig.json", - "addons/xterm-addon-fit/src/tsconfig.json", - "addons/xterm-addon-fit/test/tsconfig.json", - "addons/xterm-addon-ligatures/src/tsconfig.json", - "addons/xterm-addon-search/src/tsconfig.json", - "addons/xterm-addon-search/test/tsconfig.json", - "addons/xterm-addon-serialize/src/tsconfig.json", - "addons/xterm-addon-serialize/test/tsconfig.json", - "addons/xterm-addon-serialize/benchmark/tsconfig.json", - "addons/xterm-addon-unicode11/src/tsconfig.json", - "addons/xterm-addon-unicode11/test/tsconfig.json", - "addons/xterm-addon-web-links/src/tsconfig.json", - "addons/xterm-addon-web-links/test/tsconfig.json", - "addons/xterm-addon-webgl/src/tsconfig.json", - "addons/xterm-addon-webgl/test/tsconfig.json" + "addons/tsconfig.eslint.addons.json" ], "sourceType": "module" }, diff --git a/.gitignore b/.gitignore index 8aea04b2e9..5ad42012d4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules/ lib/ out/ out-test/ +out-worker/ .nyc_output/ Makefile.gyp *.Makefile @@ -21,6 +22,7 @@ package-lock.json # Keep bundled code out of Git dist/ demo/dist/ +demo/workers/ # dont commit benchmark folders .benchmark/ diff --git a/addons/tsconfig.eslint.addons.json b/addons/tsconfig.eslint.addons.json new file mode 100644 index 0000000000..bebfc5445a --- /dev/null +++ b/addons/tsconfig.eslint.addons.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2017", + "rootDir": ".", + "outDir": "../out-test", + "sourceMap": true, + "removeComments": true, + "strict": true, + "baseUrl": ".", + "paths": { + "browser/*": [ "../../../src/browser/*" ], + "common/*": [ "../../../src/common/*" ] + }, + "types": [ + "../node_modules/@types/mocha", + "../node_modules/@types/node", + "../out-test/api/TestUtils" + ] + }, + "include": [ + "../typings/xterm.d.ts", + "./**/src/*", + "./**/src-worker/*", + "./**/test/*", + "./**/benchmark/*" + ], + "references": [ + { "path": "../src/browser" }, + { "path": "../src/common" } + ] +} diff --git a/addons/xterm-addon-image/.gitignore b/addons/xterm-addon-image/.gitignore new file mode 100644 index 0000000000..3063f07d55 --- /dev/null +++ b/addons/xterm-addon-image/.gitignore @@ -0,0 +1,2 @@ +lib +node_modules diff --git a/addons/xterm-addon-image/.npmignore b/addons/xterm-addon-image/.npmignore new file mode 100644 index 0000000000..429b6062bf --- /dev/null +++ b/addons/xterm-addon-image/.npmignore @@ -0,0 +1,11 @@ +**/*.api.js +**/*.api.ts +tsconfig.json +.yarnrc +webpack.config.js + +fixture +src +src-worker +test +out-test diff --git a/addons/xterm-addon-image/LICENSE b/addons/xterm-addon-image/LICENSE new file mode 100644 index 0000000000..8f17892587 --- /dev/null +++ b/addons/xterm-addon-image/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019, The xterm.js authors (https://github.com/xtermjs/xterm.js) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/addons/xterm-addon-image/README.md b/addons/xterm-addon-image/README.md new file mode 100644 index 0000000000..73f1f30ac9 --- /dev/null +++ b/addons/xterm-addon-image/README.md @@ -0,0 +1,177 @@ +## xterm-addon-image + +Image output in xterm.js. + + +![](fixture/example.png) + + +⚠️ This is an experimental addon, that is still under construction. ⚠️ + + +### Install + +```bash +npm install --save xterm-addon-image +``` + +### Usage + +```ts +import { Terminal } from 'xterm'; +import { ImageAddon, IImageAddonOptions } from 'xterm-addon-image'; + +const WORKER_PATH = '/path/to/xterm-addon-image-worker.js'; + +// customize as needed +const customSettings: IImageAddonOptions = { + sixelSupport: true, + ... +} + +// initialization +const terminal = new Terminal(); +const imageAddon = new ImageAddon(WORKER_PATH, customSettings); +terminal.loadAddon(imageAddon); +``` + +### General Notes + +- The image decoding is done with a worker, therefore the addon will only work, if you expose the worker file as well. The worker is distributed under `lib/xterm-addon-image-worker.js`. Place the exported worker path as the first argument of the addon constructor, e.g. `new ImageAddon('/your/path/to/image/worker')`. Additionally make sure, that your main integration has proper read and execution permissions on the worker file, otherwise the addon will log a worker error and disable itself on the first image decoding attempt (lazy evaluated). + +- By default the addon will activate these `windowOptions` reports on the terminal: + - getWinSizePixels (CSI 14 t) + - getCellSizePixels (CSI 16 t) + - getWinSizeChars (CSI 18 t) + + to help applications getting useful terminal metrics for their image preparations. Set `enableSizeReports` in the constructor options to `false`, if you dont want the addon to alter these terminal settings. This is especially useful, if you have very strict security needs not allowing any terminal reports, or deal with `windowOptions` by other means. + + +### Operation Modes + +- **SIXEL Support** + Set by default, change it with `{sixelSupport: true}`. + +- **Scrolling On | Off** + By default scrolling is on, thus an image will advance the cursor at the bottom if needed. + The cursor will move with the image and be placed either right to the image or in the next line + (see cursor positioning). + + If scrolling is off, the image gets painted from the top left of the current viewport + and might be truncated if the image exceeds the viewport size. + The cursor position does not change. + + You can customize this behavior with the constructor option `{sixelScrolling: false}` + or with `DECSET 80` (off, binary: `\x1b [ ? 80 h`) and + `DECRST 80` (on, binary: `\x1b [ ? 80 l`) during runtime. + +- **Cursor Positioning** + If scrolling is set, the cursor will be placed at the beginning of the next row by default. + You can change this behavior with the following terminal sequences: + - `DECSET 8452` (binary: `\x1b [ ? 8452 h`) + For images not overflowing to the right, the cursor will move to the next right cell of the last image cell. + Images overflowing to the right, move the cursor to the next line. + Same as the constructor option `{cursorRight: true}`. + + - `DECRST 8452` (binary: `\x1b [ ? 8452 l`) + Always moves the cursor to the next line (default). Same as the constructor option `{cursorRight: false}`. + + - `DECRST 7730` (binary: `\x1b [ ? 7730 l`) + Move the cursor on the next line to the image start offset instead of the beginning. + This setting only applies if the cursor will wrap to the next line (thus never for scrolling off, + for `DECSET 8452` only after overflowing to the right). Same as the constructor option `{cursorBelow: true}`. + + - `DECSET 7730` (binary: `\x1b [ ? 7730 h`) + Keep the cursor on the next line at the beginning (default). Same as the constructor option `{cursorBelow: false}`. + +- **SIXEL Palette Handling** + By default the addon limits the palette size to 256 registers (as demanded by the DEC specification). + The limit can be increased to a maximum of 4096 registers (via `sixelPaletteLimit`). + + If `sixelPrivatePalette` is set (default), images are initialized with their own private palette derived from the default palette (`'VT340-COLOR'`). If `sixelPrivatePalette` is not set, the palette of the previous image will be used as initial palette. + + Note that the underlying SIXEL library currently applies colors immediately to pixels (*printer mode*), + thus it is technically possible to use more colors in one image than the palette has color slots. + This feature is called *high-color* in libsixel. + + In contrast older terminals were always bound to the palette due hardware limitations. + This limitation is mimicked by xterm's shared palette mode, which will re-color previous images from palette changes + treating all sixel images as indexed images. This true shared-palette *terminal mode* is currently not supported by + xterm.js, as it always operates in *printer mode*. + +- **SIXEL Raster Attributes Handling** + If raster attributes were found in the SIXEL data (level 2), the image will always be limited to the given height/width extend. We deviate here from the specification on purpose, as it allows several processing optimizations. For level 1 SIXEL data without any raster attributes the image can freely grow in width and height up to the last data byte, which has a much higher processing penalty. In general encoding libraries should not create level 1 data anymore and should not produce pixel information beyond the announced height/width extend. Both is discouraged by the >30 years old specification. + + Currently the SIXEL implementation of the addon does not take custom pixel sizes into account, a SIXEL pixel will map 1:1 to a screen pixel. + +### Storage and Drawing Settings + +The internal storage holds images up to `storageLimit` (in MB, calculated as 4-channel RBGA unpacked, default 100 MB). Once hit images get evicted by FIFO rules. Furthermore images on the alternate buffer will always be erased on buffer changes. + +The addon exposes two properties to interact with the storage limits at runtime: +- `storageLimit` + Change the value to your needs at runtime. This is especially useful, if you have multiple terminal + instances running, that all add to one upper memory limit. +- `storageUsage` + Inspect the current memory usage of the image storage. + +By default the addon will show a placeholder pattern for evicted images that are still part +of the terminal (e.g. in the scrollback). The pattern can be deactivated by toggling `showPlaceholder`. + +### Image Data Retrieval + +The addon provides the following API endpoints to retrieve raw image data as canvas: + +- `getImageAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined` + Returns the canvas containing the original image data (not resized) at the given buffer position. + The buffer position is the 0-based absolute index (including scrollback at top). + +- `extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined` + Returns a canvas containing the actual single tile image data (maybe resized) at the given buffer position. + The buffer position is the 0-based absolute index (including scrollback at top). + Note that the canvas gets created and data copied over for every call, thus it is not suitable for performance critical actions. + +### Memory Usage + +The addon does most image processing in Javascript and therefore can occupy a rather big amount of memory. To get an idea where the memory gets eaten, lets look at the data flow and processing steps: +- Incomming image data chunk at `term.write` (terminal) + `term.write` might stock up incoming chunks. To circumvent this, you will need proper flow control (see xterm.js docs). Note that with image output it is more likely to run into this issue, as it can create lots of MBs in very short time. +- Sequence Parser (terminal) + The parser operates on a buffer containing up to 2^17 codepoints (~0.5 MB). +- Sequence Handler - Chunk Processing (addon / mainthread) + Image data chunks are copied over and sent to the decoder worker as transferables with `postMessage`. To avoid a data congestion at the message port, allowed SIXEL data is hard limited by `sixelSizeLimit` (default 25 MB). The transport chunks are pooled, the pool cache may hold up to ~6 MB during active data transmission. +- Image Decoder (addon / worker) + The decoder works chunkwise allocating memory as needed. The allowed image size gets restricted by `pixelLimit` (default 16M pixels), the decoder holds 2 pixel buffers at maximum during decoding (RGBA, ~128 MB for 16M pixels). + After decoding the final pixel buffer is transferred back to the sequence handler. +- Sequence Handler - Image Finalization (addon / mainthread) + The pixel data gets written to a canvas of the same size (~64 MB for 16M pixels) and added to the storage. The pixel buffer is sent back to the worker to be used for the next image. +- Image Storage (addon / mainthread) + The image storage implements a FIFO cache, that will remove old images, if a new one arrives and `storageLimit` is hit (default 128 MB). The storage holds a canvas with the original image, and may additionally hold resized versions of images after a font rescaling. Both are accounted in `storageUsage` as a rough estimation of _pixels x 4 channels_. + +Following the steps above, a rough estimation of maximum memory usage by the addon can be calculated with these formulas (in bytes): +```typescript +// storage alone +const storageBytes = storageUsage * storageLimit * 1024 * 1024; +// decoding alone +const decodingBytes = sixelSizeLimit + 2 * (pixelLimit * 4); + +// totals +// inactive decoding +const totalInactive = storageBytes; +// active decoding +const totalActive = storageBytes + decodingBytes; +``` + +Note that browsers have offloading tricks for rarely touched memory segments, esp. `storageBytes` might not directly translate into real memory usage. Usage peaks will happen during active decoding of multiple big images due to the need of 2 full pixel buffers at the same time, which cannot be offloaded. Thus you may want to keep an eye on `pixelLimit` under limited memory conditions. +Further note that the formulas above do not respect the Javascript object's overhead. Compared to the raw buffer needs the book keeping by these objects is rather small (<<5%). + +_Why should I care about memory usage at all?_ +Well you don't have to, and it probably will just work fine with the addon defaults. But for bigger integrations, where much more data is held in the Javascript context (like multiple terminals on one page), it is likely to hit the engine's memory limit sooner or later under decoding and/or storage pressure. + +_How can I adjust the memory usage?_ +- `pixelLimit` + A constructor settings, thus you would have to anticipate, whether (multiple) terminals in your page gonna do lots of concurrent decoding. Since this is normally not the case and the memory usage is only temporarily peaking, a rather high value should work even with multiple terminals in one page. +- `storageLimit` + A constructor and a runtime setting. In conjunction with `storageUsage` you can do runtime checks and adjust the limit to your needs. If you have to lower the limit below the current usage, images will be removed and may turn into a placeholder in the terminal's scrollback (if `showPlaceholder` is set). When adjusting keep in mind to leave enough room for memory peaking for decoding. +- `sixelSizeLimit` + A constructor setting. This has only a small direct impact on peaking memory during decoding. It still will avoid processing of overly big or broken sequences at an earlier phase, thus may stop the decoder from entering its memory intensive task for potentially invalid data. \ No newline at end of file diff --git a/addons/xterm-addon-image/fixture/example.png b/addons/xterm-addon-image/fixture/example.png new file mode 100644 index 0000000000..3e04d920d2 Binary files /dev/null and b/addons/xterm-addon-image/fixture/example.png differ diff --git a/addons/xterm-addon-image/fixture/growing_rect.js b/addons/xterm-addon-image/fixture/growing_rect.js new file mode 100644 index 0000000000..a06f75c226 --- /dev/null +++ b/addons/xterm-addon-image/fixture/growing_rect.js @@ -0,0 +1,44 @@ +const sixelEncode = require('../node_modules/sixel/lib/SixelEncoder').image2sixel; +const toRGBA8888 = require('../node_modules/sixel/lib/Colors').toRGBA8888; + +function createRect(size, color) { + const pixels = new Uint32Array(size * size); + pixels.fill(toRGBA8888(...color)); + return sixelEncode(new Uint8ClampedArray(pixels.buffer), size, size); +} + +function createRectMinusOne(size, color) { + const pixels = new Uint32Array(size * size); + if (size - 1) { + const sub = new Uint32Array(size - 1); + sub.fill(toRGBA8888(...color)); + const last = size * (size - 1); + for (let y = 0; y < last; y += size) { + pixels.set(sub, y); + } + } + return sixelEncode(new Uint8ClampedArray(pixels.buffer), size, size); +} + +async function main() { + // clear + cursor and sixelScrolling off + process.stdout.write('\x1b[2J\x1b[?25;80l'); + + for (let i = 1; i < 300; ++i) { + await new Promise(res => setTimeout(() => { + process.stdout.write(createRect(i, [0, 255, 0])); + res(); + }, 5)); + } + for (let i = 299; i >= 1; --i) { + await new Promise(res => setTimeout(() => { + process.stdout.write(createRectMinusOne(i, [0, 255, 0])); + res(); + }, 5)); + } + + // re-enable cursor and sixel scrolling + process.stdout.write('\x1b[2J\x1b[?25;80h'); +} + +main(); diff --git a/addons/xterm-addon-image/fixture/palette.png b/addons/xterm-addon-image/fixture/palette.png new file mode 100644 index 0000000000..d58237b97c Binary files /dev/null and b/addons/xterm-addon-image/fixture/palette.png differ diff --git a/addons/xterm-addon-image/fixture/palette.sixel b/addons/xterm-addon-image/fixture/palette.sixel new file mode 100644 index 0000000000..f12ce885e5 --- /dev/null +++ b/addons/xterm-addon-image/fixture/palette.sixel @@ -0,0 +1,14 @@ +P0;0;q"1;1;640;80#0;2;0;0;0#1;2;0;13;0#2;2;0;25;0#3;2;0;38;0#4;2;0;50;0#5;2;0;63;0#6;2;0;75;0#7;2;0;88;0#8;2;13;0;0#9;2;13;13;0#10;2;13;25;0#11;2;13;38;0#12;2;13;50;0#13;2;13;63;0#14;2;13;75;0#15;2;13;88;0#16;2;25;0;0#17;2;25;13;0#18;2;25;25;0#19;2;25;38;0#20;2;25;50;0#21;2;25;63;0#22;2;25;75;0#23;2;25;88;0#24;2;38;0;0#25;2;38;13;0#26;2;38;25;0#27;2;38;38;0#28;2;38;50;0#29;2;38;63;0#30;2;38;75;0#31;2;38;88;0#32;2;50;0;0#33;2;50;13;0#34;2;50;25;0#35;2;50;38;0#36;2;50;50;0#37;2;50;63;0#38;2;50;75;0#39;2;50;88;0#40;2;63;0;0#41;2;63;13;0#42;2;63;25;0#43;2;63;38;0#44;2;63;50;0#45;2;63;63;0#46;2;63;75;0#47;2;63;88;0#48;2;75;0;0#49;2;75;13;0#50;2;75;25;0#51;2;75;38;0#52;2;75;50;0#53;2;75;63;0#54;2;75;75;0#55;2;75;88;0#56;2;88;0;0#57;2;88;13;0#58;2;88;25;0#59;2;88;38;0#60;2;88;50;0#61;2;88;63;0#62;2;88;75;0#63;2;88;88;0#64;2;0;0;13#65;2;0;13;13#66;2;0;25;13#67;2;0;38;13#68;2;0;50;13#69;2;0;63;13#70;2;0;75;13#71;2;0;88;13#72;2;13;0;13#73;2;13;13;13#74;2;13;25;13#75;2;13;38;13#76;2;13;50;13#77;2;13;63;13#78;2;13;75;13#79;2;13;88;13#80;2;25;0;13#81;2;25;13;13#82;2;25;25;13#83;2;25;38;13#84;2;25;50;13#85;2;25;63;13#86;2;25;75;13#87;2;25;88;13#88;2;38;0;13#89;2;38;13;13#90;2;38;25;13#91;2;38;38;13#92;2;38;50;13#93;2;38;63;13#94;2;38;75;13#95;2;38;88;13#96;2;50;0;13#97;2;50;13;13#98;2;50;25;13#99;2;50;38;13#100;2;50;50;13#101;2;50;63;13#102;2;50;75;13#103;2;50;88;13#104;2;63;0;13#105;2;63;13;13#106;2;63;25;13#107;2;63;38;13#108;2;63;50;13#109;2;63;63;13#110;2;63;75;13#111;2;63;88;13#112;2;75;0;13#113;2;75;13;13#114;2;75;25;13#115;2;75;38;13#116;2;75;50;13#117;2;75;63;13#118;2;75;75;13#119;2;75;88;13#120;2;88;0;13#121;2;88;13;13#122;2;88;25;13#123;2;88;38;13#124;2;88;50;13#125;2;88;63;13#126;2;88;75;13#127;2;88;88;13#128;2;0;0;25#129;2;0;13;25#130;2;0;25;25#131;2;0;38;25#132;2;0;50;25#133;2;0;63;25#134;2;0;75;25#135;2;0;88;25#136;2;13;0;25#137;2;13;13;25#138;2;13;25;25#139;2;13;38;25#140;2;13;50;25#141;2;13;63;25#142;2;13;75;25#143;2;13;88;25#144;2;25;0;25#145;2;25;13;25#146;2;25;25;25#147;2;25;38;25#148;2;25;50;25#149;2;25;63;25#150;2;25;75;25#151;2;25;88;25#152;2;38;0;25#153;2;38;13;25#154;2;38;25;25#155;2;38;38;25#156;2;38;50;25#157;2;38;63;25#158;2;38;75;25#159;2;38;88;25#160;2;50;0;25#161;2;50;13;25#162;2;50;25;25#163;2;50;38;25#164;2;50;50;25#165;2;50;63;25#166;2;50;75;25#167;2;50;88;25#168;2;63;0;25#169;2;63;13;25#170;2;63;25;25#171;2;63;38;25#172;2;63;50;25#173;2;63;63;25#174;2;63;75;25#175;2;63;88;25#176;2;75;0;25#177;2;75;13;25#178;2;75;25;25#179;2;75;38;25#180;2;75;50;25#181;2;75;63;25#182;2;75;75;25#183;2;75;88;25#184;2;88;0;25#185;2;88;13;25#186;2;88;25;25#187;2;88;38;25#188;2;88;50;25#189;2;88;63;25#190;2;88;75;25#191;2;88;88;25#192;2;0;0;38#193;2;0;13;38#194;2;0;25;38#195;2;0;38;38#196;2;0;50;38#197;2;0;63;38#198;2;0;75;38#199;2;0;88;38#200;2;13;0;38#201;2;13;13;38#202;2;13;25;38#203;2;13;38;38#204;2;13;50;38#205;2;13;63;38#206;2;13;75;38#207;2;13;88;38#208;2;25;0;38#209;2;25;13;38#210;2;25;25;38#211;2;25;38;38#212;2;25;50;38#213;2;25;63;38#214;2;25;75;38#215;2;25;88;38#216;2;38;0;38#217;2;38;13;38#218;2;38;25;38#219;2;38;38;38#220;2;38;50;38#221;2;38;63;38#222;2;38;75;38#223;2;38;88;38#224;2;50;0;38#225;2;50;13;38#226;2;50;25;38#227;2;50;38;38#228;2;50;50;38#229;2;50;63;38#230;2;50;75;38#231;2;50;88;38#232;2;63;0;38#233;2;63;13;38#234;2;63;25;38#235;2;63;38;38#236;2;63;50;38#237;2;63;63;38#238;2;63;75;38#239;2;63;88;38#240;2;75;0;38#241;2;75;13;38#242;2;75;25;38#243;2;75;38;38#244;2;75;50;38#245;2;75;63;38#246;2;75;75;38#247;2;75;88;38#248;2;88;0;38#249;2;88;13;38#250;2;88;25;38#251;2;88;38;38#252;2;88;50;38#253;2;88;63;38#254;2;88;75;38#255;2;88;88;38#256;2;0;0;50#257;2;0;13;50#258;2;0;25;50#259;2;0;38;50#260;2;0;50;50#261;2;0;63;50#262;2;0;75;50#263;2;0;88;50#264;2;13;0;50#265;2;13;13;50#266;2;13;25;50#267;2;13;38;50#268;2;13;50;50#269;2;13;63;50#270;2;13;75;50#271;2;13;88;50#272;2;25;0;50#273;2;25;13;50#274;2;25;25;50#275;2;25;38;50#276;2;25;50;50#277;2;25;63;50#278;2;25;75;50#279;2;25;88;50#280;2;38;0;50#281;2;38;13;50#282;2;38;25;50#283;2;38;38;50#284;2;38;50;50#285;2;38;63;50#286;2;38;75;50#287;2;38;88;50#288;2;50;0;50#289;2;50;13;50#290;2;50;25;50#291;2;50;38;50#292;2;50;50;50#293;2;50;63;50#294;2;50;75;50#295;2;50;88;50#296;2;63;0;50#297;2;63;13;50#298;2;63;25;50#299;2;63;38;50#300;2;63;50;50#301;2;63;63;50#302;2;63;75;50#303;2;63;88;50#304;2;75;0;50#305;2;75;13;50#306;2;75;25;50#307;2;75;38;50#308;2;75;50;50#309;2;75;63;50#310;2;75;75;50#311;2;75;88;50#312;2;88;0;50#313;2;88;13;50#314;2;88;25;50#315;2;88;38;50#316;2;88;50;50#317;2;88;63;50#318;2;88;75;50#319;2;88;88;50#320;2;0;0;63#321;2;0;13;63#322;2;0;25;63#323;2;0;38;63#324;2;0;50;63#325;2;0;63;63#326;2;0;75;63#327;2;0;88;63#328;2;13;0;63#329;2;13;13;63#330;2;13;25;63#331;2;13;38;63#332;2;13;50;63#333;2;13;63;63#334;2;13;75;63#335;2;13;88;63#336;2;25;0;63#337;2;25;13;63#338;2;25;25;63#339;2;25;38;63#340;2;25;50;63#341;2;25;63;63#342;2;25;75;63#343;2;25;88;63#344;2;38;0;63#345;2;38;13;63#346;2;38;25;63#347;2;38;38;63#348;2;38;50;63#349;2;38;63;63#350;2;38;75;63#351;2;38;88;63#352;2;50;0;63#353;2;50;13;63#354;2;50;25;63#355;2;50;38;63#356;2;50;50;63#357;2;50;63;63#358;2;50;75;63#359;2;50;88;63#360;2;63;0;63#361;2;63;13;63#362;2;63;25;63#363;2;63;38;63#364;2;63;50;63#365;2;63;63;63#366;2;63;75;63#367;2;63;88;63#368;2;75;0;63#369;2;75;13;63#370;2;75;25;63#371;2;75;38;63#372;2;75;50;63#373;2;75;63;63#374;2;75;75;63#375;2;75;88;63#376;2;88;0;63#377;2;88;13;63#378;2;88;25;63#379;2;88;38;63#380;2;88;50;63#381;2;88;63;63#382;2;88;75;63#383;2;88;88;63#384;2;0;0;75#385;2;0;13;75#386;2;0;25;75#387;2;0;38;75#388;2;0;50;75#389;2;0;63;75#390;2;0;75;75#391;2;0;88;75#392;2;13;0;75#393;2;13;13;75#394;2;13;25;75#395;2;13;38;75#396;2;13;50;75#397;2;13;63;75#398;2;13;75;75#399;2;13;88;75#400;2;25;0;75#401;2;25;13;75#402;2;25;25;75#403;2;25;38;75#404;2;25;50;75#405;2;25;63;75#406;2;25;75;75#407;2;25;88;75#408;2;38;0;75#409;2;38;13;75#410;2;38;25;75#411;2;38;38;75#412;2;38;50;75#413;2;38;63;75#414;2;38;75;75#415;2;38;88;75#416;2;50;0;75#417;2;50;13;75#418;2;50;25;75#419;2;50;38;75#420;2;50;50;75#421;2;50;63;75#422;2;50;75;75#423;2;50;88;75#424;2;63;0;75#425;2;63;13;75#426;2;63;25;75#427;2;63;38;75#428;2;63;50;75#429;2;63;63;75#430;2;63;75;75#431;2;63;88;75#432;2;75;0;75#433;2;75;13;75#434;2;75;25;75#435;2;75;38;75#436;2;75;50;75#437;2;75;63;75#438;2;75;75;75#439;2;75;88;75#440;2;88;0;75#441;2;88;13;75#442;2;88;25;75#443;2;88;38;75#444;2;88;50;75#445;2;88;63;75#446;2;88;75;75#447;2;88;88;75#448;2;0;0;88#449;2;0;13;88#450;2;0;25;88#451;2;0;38;88#452;2;0;50;88#453;2;0;63;88#454;2;0;75;88#455;2;0;88;88#456;2;13;0;88#457;2;13;13;88#458;2;13;25;88#459;2;13;38;88#460;2;13;50;88#461;2;13;63;88#462;2;13;75;88#463;2;13;88;88#464;2;25;0;88#465;2;25;13;88#466;2;25;25;88#467;2;25;38;88#468;2;25;50;88#469;2;25;63;88#470;2;25;75;88#471;2;25;88;88#472;2;38;0;88#473;2;38;13;88#474;2;38;25;88#475;2;38;38;88#476;2;38;50;88#477;2;38;63;88#478;2;38;75;88#479;2;38;88;88#480;2;50;0;88#481;2;50;13;88#482;2;50;25;88#483;2;50;38;88#484;2;50;50;88#485;2;50;63;88#486;2;50;75;88#487;2;50;88;88#488;2;63;0;88#489;2;63;13;88#490;2;63;25;88#491;2;63;38;88#492;2;63;50;88#493;2;63;63;88#494;2;63;75;88#495;2;63;88;88#496;2;75;0;88#497;2;75;13;88#498;2;75;25;88#499;2;75;38;88#500;2;75;50;88#501;2;75;63;88#502;2;75;75;88#503;2;75;88;88#504;2;88;0;88#505;2;88;13;88#506;2;88;25;88#507;2;88;38;88#508;2;88;50;88#509;2;88;63;88#510;2;88;75;88#511;2;88;88;88#0!10~$#1!10?!10~$#2!20?!10~$#3!30?!10~$#4!40?!10~$#5!50?!10~$#6!60?!10~$#7!70?!10~$#8!80?!10~$#9!90?!10~$#10!100?!10~$#11!110?!10~$#12!120?!10~$#13!130?!10~$#14!140?!10~$#15!150?!10~$#16!160?!10~$#17!170?!10~$#18!180?!10~$#19!190?!10~$#20!200?!10~$#21!210?!10~$#22!220?!10~$#23!230?!10~$#24!240?!10~$#25!250?!10~$#26!260?!10~$#27!270?!10~$#28!280?!10~$#29!290?!10~$#30!300?!10~$#31!310?!10~$#32!320?!10~$#33!330?!10~$#34!340?!10~$#35!350?!10~$#36!360?!10~$#37!370?!10~$#38!380?!10~$#39!390?!10~$#40!400?!10~$#41!410?!10~$#42!420?!10~$#43!430?!10~$#44!440?!10~$#45!450?!10~$#46!460?!10~$#47!470?!10~$#48!480?!10~$#49!490?!10~$#50!500?!10~$#51!510?!10~$#52!520?!10~$#53!530?!10~$#54!540?!10~$#55!550?!10~$#56!560?!10~$#57!570?!10~$#58!580?!10~$#59!590?!10~$#60!600?!10~$#61!610?!10~$#62!620?!10~$#63!630?!10~$- +#0!10N$#64!10o$#1!10?!10N$#65!10?!10o$#2!20?!10N$#66!20?!10o$#3!30?!10N$#67!30?!10o$#4!40?!10N$#68!40?!10o$#5!50?!10N$#69!50?!10o$#6!60?!10N$#70!60?!10o$#7!70?!10N$#71!70?!10o$#8!80?!10N$#72!80?!10o$#9!90?!10N$#73!90?!10o$#10!100?!10N$#74!100?!10o$#11!110?!10N$#75!110?!10o$#12!120?!10N$#76!120?!10o$#13!130?!10N$#77!130?!10o$#14!140?!10N$#78!140?!10o$#15!150?!10N$#79!150?!10o$#16!160?!10N$#80!160?!10o$#17!170?!10N$#81!170?!10o$#18!180?!10N$#82!180?!10o$#19!190?!10N$#83!190?!10o$#20!200?!10N$#84!200?!10o$#21!210?!10N$#85!210?!10o$#22!220?!10N$#86!220?!10o$#23!230?!10N$#87!230?!10o$#24!240?!10N$#88!240?!10o$#25!250?!10N$#89!250?!10o$#26!260?!10N$#90!260?!10o$#27!270?!10N$#91!270?!10o$#28!280?!10N$#92!280?!10o$#29!290?!10N$#93!290?!10o$#30!300?!10N$#94!300?!10o$#31!310?!10N$#95!310?!10o$#32!320?!10N$#96!320?!10o$#33!330?!10N$#97!330?!10o$#34!340?!10N$#98!340?!10o$#35!350?!10N$#99!350?!10o$#36!360?!10N$#100!360?!10o$#37!370?!10N$#101!370?!10o$#38!380?!10N$#102!380?!10o$#39!390?!10N$#103!390?!10o$#40!400?!10N$#104!400?!10o$#41!410?!10N$#105!410?!10o$#42!420?!10N$#106!420?!10o$#43!430?!10N$#107!430?!10o$#44!440?!10N$#108!440?!10o$#45!450?!10N$#109!450?!10o$#46!460?!10N$#110!460?!10o$#47!470?!10N$#111!470?!10o$#48!480?!10N$#112!480?!10o$#49!490?!10N$#113!490?!10o$#50!500?!10N$#114!500?!10o$#51!510?!10N$#115!510?!10o$#52!520?!10N$#116!520?!10o$#53!530?!10N$#117!530?!10o$#54!540?!10N$#118!540?!10o$#55!550?!10N$#119!550?!10o$#56!560?!10N$#120!560?!10o$#57!570?!10N$#121!570?!10o$#58!580?!10N$#122!580?!10o$#59!590?!10N$#123!590?!10o$#60!600?!10N$#124!600?!10o$#61!610?!10N$#125!610?!10o$#62!620?!10N$#126!620?!10o$#63!630?!10N$#127!630?!10o$- +#64!10~$#65!10?!10~$#66!20?!10~$#67!30?!10~$#68!40?!10~$#69!50?!10~$#70!60?!10~$#71!70?!10~$#72!80?!10~$#73!90?!10~$#74!100?!10~$#75!110?!10~$#76!120?!10~$#77!130?!10~$#78!140?!10~$#79!150?!10~$#80!160?!10~$#81!170?!10~$#82!180?!10~$#83!190?!10~$#84!200?!10~$#85!210?!10~$#86!220?!10~$#87!230?!10~$#88!240?!10~$#89!250?!10~$#90!260?!10~$#91!270?!10~$#92!280?!10~$#93!290?!10~$#94!300?!10~$#95!310?!10~$#96!320?!10~$#97!330?!10~$#98!340?!10~$#99!350?!10~$#100!360?!10~$#101!370?!10~$#102!380?!10~$#103!390?!10~$#104!400?!10~$#105!410?!10~$#106!420?!10~$#107!430?!10~$#108!440?!10~$#109!450?!10~$#110!460?!10~$#111!470?!10~$#112!480?!10~$#113!490?!10~$#114!500?!10~$#115!510?!10~$#116!520?!10~$#117!530?!10~$#118!540?!10~$#119!550?!10~$#120!560?!10~$#121!570?!10~$#122!580?!10~$#123!590?!10~$#124!600?!10~$#125!610?!10~$#126!620?!10~$#127!630?!10~$- +#64!10B$#128!10{$#65!10?!10B$#129!10?!10{$#66!20?!10B$#130!20?!10{$#67!30?!10B$#131!30?!10{$#68!40?!10B$#132!40?!10{$#69!50?!10B$#133!50?!10{$#70!60?!10B$#134!60?!10{$#71!70?!10B$#135!70?!10{$#72!80?!10B$#136!80?!10{$#73!90?!10B$#137!90?!10{$#74!100?!10B$#138!100?!10{$#75!110?!10B$#139!110?!10{$#76!120?!10B$#140!120?!10{$#77!130?!10B$#141!130?!10{$#78!140?!10B$#142!140?!10{$#79!150?!10B$#143!150?!10{$#80!160?!10B$#144!160?!10{$#81!170?!10B$#145!170?!10{$#82!180?!10B$#146!180?!10{$#83!190?!10B$#147!190?!10{$#84!200?!10B$#148!200?!10{$#85!210?!10B$#149!210?!10{$#86!220?!10B$#150!220?!10{$#87!230?!10B$#151!230?!10{$#88!240?!10B$#152!240?!10{$#89!250?!10B$#153!250?!10{$#90!260?!10B$#154!260?!10{$#91!270?!10B$#155!270?!10{$#92!280?!10B$#156!280?!10{$#93!290?!10B$#157!290?!10{$#94!300?!10B$#158!300?!10{$#95!310?!10B$#159!310?!10{$#96!320?!10B$#160!320?!10{$#97!330?!10B$#161!330?!10{$#98!340?!10B$#162!340?!10{$#99!350?!10B$#163!350?!10{$#100!360?!10B$#164!360?!10{$#101!370?!10B$#165!370?!10{$#102!380?!10B$#166!380?!10{$#103!390?!10B$#167!390?!10{$#104!400?!10B$#168!400?!10{$#105!410?!10B$#169!410?!10{$#106!420?!10B$#170!420?!10{$#107!430?!10B$#171!430?!10{$#108!440?!10B$#172!440?!10{$#109!450?!10B$#173!450?!10{$#110!460?!10B$#174!460?!10{$#111!470?!10B$#175!470?!10{$#112!480?!10B$#176!480?!10{$#113!490?!10B$#177!490?!10{$#114!500?!10B$#178!500?!10{$#115!510?!10B$#179!510?!10{$#116!520?!10B$#180!520?!10{$#117!530?!10B$#181!530?!10{$#118!540?!10B$#182!540?!10{$#119!550?!10B$#183!550?!10{$#120!560?!10B$#184!560?!10{$#121!570?!10B$#185!570?!10{$#122!580?!10B$#186!580?!10{$#123!590?!10B$#187!590?!10{$#124!600?!10B$#188!600?!10{$#125!610?!10B$#189!610?!10{$#126!620?!10B$#190!620?!10{$#127!630?!10B$#191!630?!10{$- +#128!10~$#129!10?!10~$#130!20?!10~$#131!30?!10~$#132!40?!10~$#133!50?!10~$#134!60?!10~$#135!70?!10~$#136!80?!10~$#137!90?!10~$#138!100?!10~$#139!110?!10~$#140!120?!10~$#141!130?!10~$#142!140?!10~$#143!150?!10~$#144!160?!10~$#145!170?!10~$#146!180?!10~$#147!190?!10~$#148!200?!10~$#149!210?!10~$#150!220?!10~$#151!230?!10~$#152!240?!10~$#153!250?!10~$#154!260?!10~$#155!270?!10~$#156!280?!10~$#157!290?!10~$#158!300?!10~$#159!310?!10~$#160!320?!10~$#161!330?!10~$#162!340?!10~$#163!350?!10~$#164!360?!10~$#165!370?!10~$#166!380?!10~$#167!390?!10~$#168!400?!10~$#169!410?!10~$#170!420?!10~$#171!430?!10~$#172!440?!10~$#173!450?!10~$#174!460?!10~$#175!470?!10~$#176!480?!10~$#177!490?!10~$#178!500?!10~$#179!510?!10~$#180!520?!10~$#181!530?!10~$#182!540?!10~$#183!550?!10~$#184!560?!10~$#185!570?!10~$#186!580?!10~$#187!590?!10~$#188!600?!10~$#189!610?!10~$#190!620?!10~$#191!630?!10~$- +#192!10~$#193!10?!10~$#194!20?!10~$#195!30?!10~$#196!40?!10~$#197!50?!10~$#198!60?!10~$#199!70?!10~$#200!80?!10~$#201!90?!10~$#202!100?!10~$#203!110?!10~$#204!120?!10~$#205!130?!10~$#206!140?!10~$#207!150?!10~$#208!160?!10~$#209!170?!10~$#210!180?!10~$#211!190?!10~$#212!200?!10~$#213!210?!10~$#214!220?!10~$#215!230?!10~$#216!240?!10~$#217!250?!10~$#218!260?!10~$#219!270?!10~$#220!280?!10~$#221!290?!10~$#222!300?!10~$#223!310?!10~$#224!320?!10~$#225!330?!10~$#226!340?!10~$#227!350?!10~$#228!360?!10~$#229!370?!10~$#230!380?!10~$#231!390?!10~$#232!400?!10~$#233!410?!10~$#234!420?!10~$#235!430?!10~$#236!440?!10~$#237!450?!10~$#238!460?!10~$#239!470?!10~$#240!480?!10~$#241!490?!10~$#242!500?!10~$#243!510?!10~$#244!520?!10~$#245!530?!10~$#246!540?!10~$#247!550?!10~$#248!560?!10~$#249!570?!10~$#250!580?!10~$#251!590?!10~$#252!600?!10~$#253!610?!10~$#254!620?!10~$#255!630?!10~$- +#192!10N$#256!10o$#193!10?!10N$#257!10?!10o$#194!20?!10N$#258!20?!10o$#195!30?!10N$#259!30?!10o$#196!40?!10N$#260!40?!10o$#197!50?!10N$#261!50?!10o$#198!60?!10N$#262!60?!10o$#199!70?!10N$#263!70?!10o$#200!80?!10N$#264!80?!10o$#201!90?!10N$#265!90?!10o$#202!100?!10N$#266!100?!10o$#203!110?!10N$#267!110?!10o$#204!120?!10N$#268!120?!10o$#205!130?!10N$#269!130?!10o$#206!140?!10N$#270!140?!10o$#207!150?!10N$#271!150?!10o$#208!160?!10N$#272!160?!10o$#209!170?!10N$#273!170?!10o$#210!180?!10N$#274!180?!10o$#211!190?!10N$#275!190?!10o$#212!200?!10N$#276!200?!10o$#213!210?!10N$#277!210?!10o$#214!220?!10N$#278!220?!10o$#215!230?!10N$#279!230?!10o$#216!240?!10N$#280!240?!10o$#217!250?!10N$#281!250?!10o$#218!260?!10N$#282!260?!10o$#219!270?!10N$#283!270?!10o$#220!280?!10N$#284!280?!10o$#221!290?!10N$#285!290?!10o$#222!300?!10N$#286!300?!10o$#223!310?!10N$#287!310?!10o$#224!320?!10N$#288!320?!10o$#225!330?!10N$#289!330?!10o$#226!340?!10N$#290!340?!10o$#227!350?!10N$#291!350?!10o$#228!360?!10N$#292!360?!10o$#229!370?!10N$#293!370?!10o$#230!380?!10N$#294!380?!10o$#231!390?!10N$#295!390?!10o$#232!400?!10N$#296!400?!10o$#233!410?!10N$#297!410?!10o$#234!420?!10N$#298!420?!10o$#235!430?!10N$#299!430?!10o$#236!440?!10N$#300!440?!10o$#237!450?!10N$#301!450?!10o$#238!460?!10N$#302!460?!10o$#239!470?!10N$#303!470?!10o$#240!480?!10N$#304!480?!10o$#241!490?!10N$#305!490?!10o$#242!500?!10N$#306!500?!10o$#243!510?!10N$#307!510?!10o$#244!520?!10N$#308!520?!10o$#245!530?!10N$#309!530?!10o$#246!540?!10N$#310!540?!10o$#247!550?!10N$#311!550?!10o$#248!560?!10N$#312!560?!10o$#249!570?!10N$#313!570?!10o$#250!580?!10N$#314!580?!10o$#251!590?!10N$#315!590?!10o$#252!600?!10N$#316!600?!10o$#253!610?!10N$#317!610?!10o$#254!620?!10N$#318!620?!10o$#255!630?!10N$#319!630?!10o$- +#256!10~$#257!10?!10~$#258!20?!10~$#259!30?!10~$#260!40?!10~$#261!50?!10~$#262!60?!10~$#263!70?!10~$#264!80?!10~$#265!90?!10~$#266!100?!10~$#267!110?!10~$#268!120?!10~$#269!130?!10~$#270!140?!10~$#271!150?!10~$#272!160?!10~$#273!170?!10~$#274!180?!10~$#275!190?!10~$#276!200?!10~$#277!210?!10~$#278!220?!10~$#279!230?!10~$#280!240?!10~$#281!250?!10~$#282!260?!10~$#283!270?!10~$#284!280?!10~$#285!290?!10~$#286!300?!10~$#287!310?!10~$#288!320?!10~$#289!330?!10~$#290!340?!10~$#291!350?!10~$#292!360?!10~$#293!370?!10~$#294!380?!10~$#295!390?!10~$#296!400?!10~$#297!410?!10~$#298!420?!10~$#299!430?!10~$#300!440?!10~$#301!450?!10~$#302!460?!10~$#303!470?!10~$#304!480?!10~$#305!490?!10~$#306!500?!10~$#307!510?!10~$#308!520?!10~$#309!530?!10~$#310!540?!10~$#311!550?!10~$#312!560?!10~$#313!570?!10~$#314!580?!10~$#315!590?!10~$#316!600?!10~$#317!610?!10~$#318!620?!10~$#319!630?!10~$- +#256!10B$#320!10{$#257!10?!10B$#321!10?!10{$#258!20?!10B$#322!20?!10{$#259!30?!10B$#323!30?!10{$#260!40?!10B$#324!40?!10{$#261!50?!10B$#325!50?!10{$#262!60?!10B$#326!60?!10{$#263!70?!10B$#327!70?!10{$#264!80?!10B$#328!80?!10{$#265!90?!10B$#329!90?!10{$#266!100?!10B$#330!100?!10{$#267!110?!10B$#331!110?!10{$#268!120?!10B$#332!120?!10{$#269!130?!10B$#333!130?!10{$#270!140?!10B$#334!140?!10{$#271!150?!10B$#335!150?!10{$#272!160?!10B$#336!160?!10{$#273!170?!10B$#337!170?!10{$#274!180?!10B$#338!180?!10{$#275!190?!10B$#339!190?!10{$#276!200?!10B$#340!200?!10{$#277!210?!10B$#341!210?!10{$#278!220?!10B$#342!220?!10{$#279!230?!10B$#343!230?!10{$#280!240?!10B$#344!240?!10{$#281!250?!10B$#345!250?!10{$#282!260?!10B$#346!260?!10{$#283!270?!10B$#347!270?!10{$#284!280?!10B$#348!280?!10{$#285!290?!10B$#349!290?!10{$#286!300?!10B$#350!300?!10{$#287!310?!10B$#351!310?!10{$#288!320?!10B$#352!320?!10{$#289!330?!10B$#353!330?!10{$#290!340?!10B$#354!340?!10{$#291!350?!10B$#355!350?!10{$#292!360?!10B$#356!360?!10{$#293!370?!10B$#357!370?!10{$#294!380?!10B$#358!380?!10{$#295!390?!10B$#359!390?!10{$#296!400?!10B$#360!400?!10{$#297!410?!10B$#361!410?!10{$#298!420?!10B$#362!420?!10{$#299!430?!10B$#363!430?!10{$#300!440?!10B$#364!440?!10{$#301!450?!10B$#365!450?!10{$#302!460?!10B$#366!460?!10{$#303!470?!10B$#367!470?!10{$#304!480?!10B$#368!480?!10{$#305!490?!10B$#369!490?!10{$#306!500?!10B$#370!500?!10{$#307!510?!10B$#371!510?!10{$#308!520?!10B$#372!520?!10{$#309!530?!10B$#373!530?!10{$#310!540?!10B$#374!540?!10{$#311!550?!10B$#375!550?!10{$#312!560?!10B$#376!560?!10{$#313!570?!10B$#377!570?!10{$#314!580?!10B$#378!580?!10{$#315!590?!10B$#379!590?!10{$#316!600?!10B$#380!600?!10{$#317!610?!10B$#381!610?!10{$#318!620?!10B$#382!620?!10{$#319!630?!10B$#383!630?!10{$- +#320!10~$#321!10?!10~$#322!20?!10~$#323!30?!10~$#324!40?!10~$#325!50?!10~$#326!60?!10~$#327!70?!10~$#328!80?!10~$#329!90?!10~$#330!100?!10~$#331!110?!10~$#332!120?!10~$#333!130?!10~$#334!140?!10~$#335!150?!10~$#336!160?!10~$#337!170?!10~$#338!180?!10~$#339!190?!10~$#340!200?!10~$#341!210?!10~$#342!220?!10~$#343!230?!10~$#344!240?!10~$#345!250?!10~$#346!260?!10~$#347!270?!10~$#348!280?!10~$#349!290?!10~$#350!300?!10~$#351!310?!10~$#352!320?!10~$#353!330?!10~$#354!340?!10~$#355!350?!10~$#356!360?!10~$#357!370?!10~$#358!380?!10~$#359!390?!10~$#360!400?!10~$#361!410?!10~$#362!420?!10~$#363!430?!10~$#364!440?!10~$#365!450?!10~$#366!460?!10~$#367!470?!10~$#368!480?!10~$#369!490?!10~$#370!500?!10~$#371!510?!10~$#372!520?!10~$#373!530?!10~$#374!540?!10~$#375!550?!10~$#376!560?!10~$#377!570?!10~$#378!580?!10~$#379!590?!10~$#380!600?!10~$#381!610?!10~$#382!620?!10~$#383!630?!10~$- +#384!10~$#385!10?!10~$#386!20?!10~$#387!30?!10~$#388!40?!10~$#389!50?!10~$#390!60?!10~$#391!70?!10~$#392!80?!10~$#393!90?!10~$#394!100?!10~$#395!110?!10~$#396!120?!10~$#397!130?!10~$#398!140?!10~$#399!150?!10~$#400!160?!10~$#401!170?!10~$#402!180?!10~$#403!190?!10~$#404!200?!10~$#405!210?!10~$#406!220?!10~$#407!230?!10~$#408!240?!10~$#409!250?!10~$#410!260?!10~$#411!270?!10~$#412!280?!10~$#413!290?!10~$#414!300?!10~$#415!310?!10~$#416!320?!10~$#417!330?!10~$#418!340?!10~$#419!350?!10~$#420!360?!10~$#421!370?!10~$#422!380?!10~$#423!390?!10~$#424!400?!10~$#425!410?!10~$#426!420?!10~$#427!430?!10~$#428!440?!10~$#429!450?!10~$#430!460?!10~$#431!470?!10~$#432!480?!10~$#433!490?!10~$#434!500?!10~$#435!510?!10~$#436!520?!10~$#437!530?!10~$#438!540?!10~$#439!550?!10~$#440!560?!10~$#441!570?!10~$#442!580?!10~$#443!590?!10~$#444!600?!10~$#445!610?!10~$#446!620?!10~$#447!630?!10~$- +#384!10N$#448!10o$#385!10?!10N$#449!10?!10o$#386!20?!10N$#450!20?!10o$#387!30?!10N$#451!30?!10o$#388!40?!10N$#452!40?!10o$#389!50?!10N$#453!50?!10o$#390!60?!10N$#454!60?!10o$#391!70?!10N$#455!70?!10o$#392!80?!10N$#456!80?!10o$#393!90?!10N$#457!90?!10o$#394!100?!10N$#458!100?!10o$#395!110?!10N$#459!110?!10o$#396!120?!10N$#460!120?!10o$#397!130?!10N$#461!130?!10o$#398!140?!10N$#462!140?!10o$#399!150?!10N$#463!150?!10o$#400!160?!10N$#464!160?!10o$#401!170?!10N$#465!170?!10o$#402!180?!10N$#466!180?!10o$#403!190?!10N$#467!190?!10o$#404!200?!10N$#468!200?!10o$#405!210?!10N$#469!210?!10o$#406!220?!10N$#470!220?!10o$#407!230?!10N$#471!230?!10o$#408!240?!10N$#472!240?!10o$#409!250?!10N$#473!250?!10o$#410!260?!10N$#474!260?!10o$#411!270?!10N$#475!270?!10o$#412!280?!10N$#476!280?!10o$#413!290?!10N$#477!290?!10o$#414!300?!10N$#478!300?!10o$#415!310?!10N$#479!310?!10o$#416!320?!10N$#480!320?!10o$#417!330?!10N$#481!330?!10o$#418!340?!10N$#482!340?!10o$#419!350?!10N$#483!350?!10o$#420!360?!10N$#484!360?!10o$#421!370?!10N$#485!370?!10o$#422!380?!10N$#486!380?!10o$#423!390?!10N$#487!390?!10o$#424!400?!10N$#488!400?!10o$#425!410?!10N$#489!410?!10o$#426!420?!10N$#490!420?!10o$#427!430?!10N$#491!430?!10o$#428!440?!10N$#492!440?!10o$#429!450?!10N$#493!450?!10o$#430!460?!10N$#494!460?!10o$#431!470?!10N$#495!470?!10o$#432!480?!10N$#496!480?!10o$#433!490?!10N$#497!490?!10o$#434!500?!10N$#498!500?!10o$#435!510?!10N$#499!510?!10o$#436!520?!10N$#500!520?!10o$#437!530?!10N$#501!530?!10o$#438!540?!10N$#502!540?!10o$#439!550?!10N$#503!550?!10o$#440!560?!10N$#504!560?!10o$#441!570?!10N$#505!570?!10o$#442!580?!10N$#506!580?!10o$#443!590?!10N$#507!590?!10o$#444!600?!10N$#508!600?!10o$#445!610?!10N$#509!610?!10o$#446!620?!10N$#510!620?!10o$#447!630?!10N$#511!630?!10o$- +#448!10~$#449!10?!10~$#450!20?!10~$#451!30?!10~$#452!40?!10~$#453!50?!10~$#454!60?!10~$#455!70?!10~$#456!80?!10~$#457!90?!10~$#458!100?!10~$#459!110?!10~$#460!120?!10~$#461!130?!10~$#462!140?!10~$#463!150?!10~$#464!160?!10~$#465!170?!10~$#466!180?!10~$#467!190?!10~$#468!200?!10~$#469!210?!10~$#470!220?!10~$#471!230?!10~$#472!240?!10~$#473!250?!10~$#474!260?!10~$#475!270?!10~$#476!280?!10~$#477!290?!10~$#478!300?!10~$#479!310?!10~$#480!320?!10~$#481!330?!10~$#482!340?!10~$#483!350?!10~$#484!360?!10~$#485!370?!10~$#486!380?!10~$#487!390?!10~$#488!400?!10~$#489!410?!10~$#490!420?!10~$#491!430?!10~$#492!440?!10~$#493!450?!10~$#494!460?!10~$#495!470?!10~$#496!480?!10~$#497!490?!10~$#498!500?!10~$#499!510?!10~$#500!520?!10~$#501!530?!10~$#502!540?!10~$#503!550?!10~$#504!560?!10~$#505!570?!10~$#506!580?!10~$#507!590?!10~$#508!600?!10~$#509!610?!10~$#510!620?!10~$#511!630?!10~$- +#448!10B$#449!10?!10B$#450!20?!10B$#451!30?!10B$#452!40?!10B$#453!50?!10B$#454!60?!10B$#455!70?!10B$#456!80?!10B$#457!90?!10B$#458!100?!10B$#459!110?!10B$#460!120?!10B$#461!130?!10B$#462!140?!10B$#463!150?!10B$#464!160?!10B$#465!170?!10B$#466!180?!10B$#467!190?!10B$#468!200?!10B$#469!210?!10B$#470!220?!10B$#471!230?!10B$#472!240?!10B$#473!250?!10B$#474!260?!10B$#475!270?!10B$#476!280?!10B$#477!290?!10B$#478!300?!10B$#479!310?!10B$#480!320?!10B$#481!330?!10B$#482!340?!10B$#483!350?!10B$#484!360?!10B$#485!370?!10B$#486!380?!10B$#487!390?!10B$#488!400?!10B$#489!410?!10B$#490!420?!10B$#491!430?!10B$#492!440?!10B$#493!450?!10B$#494!460?!10B$#495!470?!10B$#496!480?!10B$#497!490?!10B$#498!500?!10B$#499!510?!10B$#500!520?!10B$#501!530?!10B$#502!540?!10B$#503!550?!10B$#504!560?!10B$#505!570?!10B$#506!580?!10B$#507!590?!10B$#508!600?!10B$#509!610?!10B$#510!620?!10B$#511!630?!10B$\ \ No newline at end of file diff --git a/addons/xterm-addon-image/package.json b/addons/xterm-addon-image/package.json new file mode 100644 index 0000000000..a2f7e9c4d1 --- /dev/null +++ b/addons/xterm-addon-image/package.json @@ -0,0 +1,27 @@ +{ + "name": "xterm-addon-image", + "version": "0.0.1", + "author": { + "name": "The xterm.js authors", + "url": "https://xtermjs.org/" + }, + "main": "lib/xterm-addon-image.js", + "types": "typings/xterm-addon-image.d.ts", + "repository": "https://github.com/xtermjs/xterm.js", + "license": "MIT", + "scripts": { + "build": "../../node_modules/.bin/tsc -p src", + "prepackage": "npm run build", + "package": "../../node_modules/.bin/webpack", + "prepublishOnly": "npm run package" + }, + "peerDependencies": { + "xterm": "^4.14.0" + }, + "dependencies": { + "sixel": "^0.15.0" + }, + "devDependencies": { + "png-ts": "^0.0.3" + } +} diff --git a/addons/xterm-addon-image/src-worker/main.ts b/addons/xterm-addon-image/src-worker/main.ts new file mode 100644 index 0000000000..8d76f9d129 --- /dev/null +++ b/addons/xterm-addon-image/src-worker/main.ts @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { AckPayload, IImageWorkerMessage, IPostMessage, MessageType, PaletteType } from '../src/WorkerTypes'; + +import { Decoder } from 'sixel/lib/Decoder'; +import { PALETTE_VT340_COLOR, PALETTE_VT340_GREY, PALETTE_ANSI_256 } from 'sixel/lib/Colors'; + + +// narrow types for postMessage to our protocol +declare const postMessage: IPostMessage; + + +let imageBuffer: ArrayBuffer | undefined; +let sizeExceeded = false; +let dec: Decoder; + +// setup options loaded from ACK +let pixelLimit = 0; + +// always free decoder ressources after decoding if it exceeds this limit +const MEM_PERMA_LIMIT = 4194304; // 1024 pixels * 1024 pixels * 4 channels = 4MB + + +function messageHandler(event: MessageEvent): void { + const data = event.data; + switch (data.type) { + case MessageType.SIXEL_PUT: + if (!sizeExceeded) { + dec.decode(new Uint8Array(data.payload.buffer, 0, data.payload.length)); + if (dec.height * dec.width > pixelLimit) { + sizeExceeded = true; + dec.release(); + console.warn('image worker: pixelLimit exceeded, aborting'); + postMessage({ type: MessageType.SIZE_EXCEEDED }); + } + } + postMessage({ type: MessageType.CHUNK_TRANSFER, payload: data.payload.buffer }, [data.payload.buffer]); + break; + case MessageType.SIXEL_END: + const success = data.payload; + if (success) { + if (!dec || !dec.width || !dec.height || sizeExceeded) { + postMessage({ type: MessageType.SIXEL_IMAGE, payload: null }); + } else { + const width = dec.width; + const height = dec.height; + const bytes = width * height * 4; + if (!imageBuffer || imageBuffer.byteLength < bytes) { + imageBuffer = new ArrayBuffer(bytes); + } + new Uint32Array(imageBuffer, 0, width * height).set(dec.data32); + postMessage({ + type: MessageType.SIXEL_IMAGE, + payload: { + buffer: imageBuffer, + width, + height + } + }, [imageBuffer]); + imageBuffer = undefined; + if (dec.memoryUsage > MEM_PERMA_LIMIT) { + dec.release(); + } + } + } + sizeExceeded = false; + break; + case MessageType.CHUNK_TRANSFER: + if (!imageBuffer) { + imageBuffer = data.payload; + } + break; + case MessageType.SIXEL_INIT: + sizeExceeded = false; + const { fillColor, paletteType, limit } = data.payload; + const palette = paletteType === PaletteType.SHARED + ? null + : paletteType === PaletteType.VT340_COLOR + ? PALETTE_VT340_COLOR + : paletteType === PaletteType.VT340_GREY + ? PALETTE_VT340_GREY + : PALETTE_ANSI_256; + dec.init(fillColor, palette, limit); + break; + case MessageType.ACK: + pixelLimit = data.options?.pixelLimit || 0; + dec = new Decoder({ memoryLimit: pixelLimit * 4 }); + postMessage({ type: MessageType.ACK, payload: AckPayload.ALIVE, options: null }); + break; + } +} +self.addEventListener('message', messageHandler, false); diff --git a/addons/xterm-addon-image/src-worker/tsconfig.json b/addons/xterm-addon-image/src-worker/tsconfig.json new file mode 100644 index 0000000000..be4fe229d4 --- /dev/null +++ b/addons/xterm-addon-image/src-worker/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES6", + "lib": ["webworker", "ES6"], + "module": "commonjs", + "sourceMap": true, + "outDir": "../out-worker", + "rootDir": ".", + "strict": true, + "noUnusedLocals": true, + "preserveWatchOutput": true, + "types": [ + "../../../node_modules/@types/mocha" + ] + }, + "include": [ + "./**/*" + ] +} \ No newline at end of file diff --git a/addons/xterm-addon-image/src/ImageAddon.ts b/addons/xterm-addon-image/src/ImageAddon.ts new file mode 100644 index 0000000000..5fce3b0753 --- /dev/null +++ b/addons/xterm-addon-image/src/ImageAddon.ts @@ -0,0 +1,323 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { ITerminalAddon, IDisposable } from 'xterm'; +import { ImageRenderer } from './ImageRenderer'; +import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage'; +import { SixelHandler } from './SixelHandler'; +import { ITerminalExt, IImageAddonOptions } from './Types'; +import { WorkerManager } from './WorkerManager'; + + +// default values of addon ctor options +const DEFAULT_OPTIONS: IImageAddonOptions = { + enableSizeReports: true, + pixelLimit: 16777216, // limit to 4096 * 4096 pixels + cursorRight: false, + cursorBelow: false, + sixelSupport: true, + sixelScrolling: true, + sixelPaletteLimit: 256, + sixelSizeLimit: 25000000, + sixelPrivatePalette: true, + sixelDefaultPalette: 'VT340-COLOR', + storageLimit: 128, + showPlaceholder: true +}; + +// max palette size supported by the sixel lib (compile time setting) +const MAX_SIXEL_PALETTE_SIZE = 4096; + +// definitions for _xtermGraphicsAttributes sequence +const enum GaItem { + COLORS = 1, + SIXEL_GEO = 2, + REGIS_GEO = 3 +} +const enum GaAction { + READ = 1, + SET_DEFAULT = 2, + SET = 3, + READ_MAX = 4 +} +const enum GaStatus { + SUCCESS = 0, + ITEM_ERROR = 1, + ACTION_ERROR = 2, + FAILURE = 3 +} + + +export class ImageAddon implements ITerminalAddon { + private _opts: IImageAddonOptions; + private _defaultOpts: IImageAddonOptions; + private _storage: ImageStorage | undefined; + private _renderer: ImageRenderer | undefined; + private _disposables: IDisposable[] = []; + private _terminal: ITerminalExt | undefined; + private _workerManager: WorkerManager; + + constructor(workerPath: string, opts: Partial) { + this._opts = Object.assign({}, DEFAULT_OPTIONS, opts); + this._defaultOpts = Object.assign({}, DEFAULT_OPTIONS, opts); + this._workerManager = new WorkerManager(workerPath, this._opts); + this._disposeLater(this._workerManager); + } + + public dispose(): void { + for (const obj of this._disposables) { + obj.dispose(); + } + this._disposables.length = 0; + } + + private _disposeLater(...args: IDisposable[]): void { + for (const obj of args) { + this._disposables.push(obj); + } + } + + public activate(terminal: ITerminalExt): void { + this._terminal = terminal; + + // internal data structures + this._renderer = new ImageRenderer(terminal, this._opts.showPlaceholder); + this._storage = new ImageStorage(terminal, this._renderer, this._opts); + + // enable size reports + if (this._opts.enableSizeReports) { + const windowOptions = terminal.getOption('windowOptions'); + windowOptions.getWinSizePixels = true; + windowOptions.getCellSizePixels = true; + windowOptions.getWinSizeChars = true; + terminal.setOption('windowOptions', windowOptions); + } + + this._disposeLater( + this._renderer, + this._storage, + + // DECSET/DECRST/DA1/XTSMGRAPHICS handlers + terminal.parser.registerCsiHandler({ prefix: '?', final: 'h' }, params => this._decset(params)), + terminal.parser.registerCsiHandler({ prefix: '?', final: 'l' }, params => this._decrst(params)), + terminal.parser.registerCsiHandler({ final: 'c' }, params => this._da1(params)), + terminal.parser.registerCsiHandler({ prefix: '?', final: 'S' }, params => this._xtermGraphicsAttributes(params)), + + // render hook + terminal.onRender(range => this._storage?.render(range)), + + /** + * reset handlers covered: + * - DECSTR + * - RIS + * - Terminal.reset() + */ + terminal.parser.registerCsiHandler({ intermediates: '!', final: 'p' }, () => this.reset()), + terminal.parser.registerEscHandler({ final: 'c' }, () => this.reset()), + terminal._core._inputHandler.onRequestReset(() => this.reset()), + + // wipe canvas and delete alternate images on buffer switch + terminal.buffer.onBufferChange(() => this._storage?.wipeAlternate()), + + // extend images to the right on resize + terminal.onResize(metrics => this._storage?.viewportResize(metrics)) + ); + + // SIXEL handler + if (this._opts.sixelSupport) { + this._disposeLater( + terminal._core._inputHandler._parser.registerDcsHandler( + { final: 'q' }, new SixelHandler(this._opts, this._storage, terminal, this._workerManager)) + ); + } + } + + // Note: storageLimit is skipped here to not intoduce a surprising side effect. + public reset(): boolean { + // reset options customizable by sequences to defaults + this._opts.sixelScrolling = this._defaultOpts.sixelScrolling; + this._opts.cursorRight = this._defaultOpts.cursorRight; + this._opts.cursorBelow = this._defaultOpts.cursorBelow; + this._opts.sixelPrivatePalette = this._defaultOpts.sixelPrivatePalette; + this._opts.sixelPaletteLimit = this._defaultOpts.sixelPaletteLimit; + // also clear image storage + this._storage?.reset(); + return false; + } + + public get storageLimit(): number { + return this._storage?.getLimit() || -1; + } + + public set storageLimit(limit: number) { + this._storage?.setLimit(limit); + this._opts.storageLimit = limit; + } + + public get storageUsage(): number { + if (this._storage) { + return this._storage.getUsage(); + } + return -1; + } + + public get showPlaceholder(): boolean { + return this._opts.showPlaceholder; + } + + public set showPlaceholder(value: boolean) { + this._opts.showPlaceholder = value; + this._renderer?.showPlaceholder(value); + } + + public getImageAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined { + return this._storage?.getImageAtBufferCell(x, y); + } + + public extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined { + return this._storage?.extractTileAtBufferCell(x, y); + } + + private _report(s: string): void { + this._terminal?._core.coreService.triggerDataEvent(s); + } + + private _decset(params: (number | number[])[]): boolean { + for (let i = 0; i < params.length; ++i) { + switch (params[i]) { + case 80: + this._opts.sixelScrolling = false; + break; + case 1070: + this._opts.sixelPrivatePalette = true; + break; + case 8452: + this._opts.cursorRight = true; + break; + case 7730: + this._opts.cursorBelow = false; + break; + } + } + return false; + } + + private _decrst(params: (number | number[])[]): boolean { + for (let i = 0; i < params.length; ++i) { + switch (params[i]) { + case 80: + this._opts.sixelScrolling = true; + break; + case 1070: + this._opts.sixelPrivatePalette = false; + break; + case 8452: + this._opts.cursorRight = false; + break; + case 7730: + this._opts.cursorBelow = true; + break; + } + } + return false; + } + + // overload DA to return something more appropriate + private _da1(params: (number | number[])[]): boolean { + if (params[0] > 0) { + return true; + } + // reported features: + // 62 - VT220 + // 4 - SIXEL support + // 9 - charsets + // 22 - ANSI colors + if (this._opts.sixelSupport && !this._workerManager.failed) { + this._report(`\x1b[?62;4;9;22c`); + return true; + } + return false; + } + + /** + * Implementation of xterm's graphics attribute sequence. + * + * Supported features: + * - read/change palette limits (max 4096 by sixel lib) + * - read SIXEL canvas geometry (reports current window canvas or + * squared pixelLimit if canvas > pixel limit) + * + * Everything else is deactivated. + */ + private _xtermGraphicsAttributes(params: (number | number[])[]): boolean { + if (params.length < 2) { + return true; + } + if (this._workerManager.failed) { + // on worker error report graphics caps as not supported + this._report(`\x1b[?${params[0]};${GaStatus.ITEM_ERROR}S`); + return true; + } + if (params[0] === GaItem.COLORS) { + switch (params[1]) { + case GaAction.READ: + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${this._opts.sixelPaletteLimit}S`); + return true; + case GaAction.SET_DEFAULT: + this._opts.sixelPaletteLimit = this._defaultOpts.sixelPaletteLimit; + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${this._opts.sixelPaletteLimit}S`); + return true; + case GaAction.SET: + if (params.length > 2 && !(params[2] instanceof Array) && params[2] <= MAX_SIXEL_PALETTE_SIZE) { + this._opts.sixelPaletteLimit = params[2]; + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${this._opts.sixelPaletteLimit}S`); + } else { + this._report(`\x1b[?${params[0]};${GaStatus.ACTION_ERROR}S`); + } + return true; + case GaAction.READ_MAX: + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${MAX_SIXEL_PALETTE_SIZE}S`); + return true; + default: + this._report(`\x1b[?${params[0]};${GaStatus.ACTION_ERROR}S`); + return true; + } + } + if (params[0] === GaItem.SIXEL_GEO) { + switch (params[1]) { + // we only implement read and read_max here + case GaAction.READ: + let width = this._renderer?.dimensions?.canvasWidth; + let height = this._renderer?.dimensions?.canvasHeight; + if (!width || !height) { + // for some reason we have no working image renderer + // --> fallback to default cell size + const cellSize = CELL_SIZE_DEFAULT; + width = (this._terminal?.cols || 80) * cellSize.width; + height = (this._terminal?.rows || 24) * cellSize.height; + } + if (width * height < this._opts.pixelLimit) { + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${width.toFixed(0)};${height.toFixed(0)}S`); + } else { + // if we overflow pixelLimit report that squared instead + const x = Math.floor(Math.sqrt(this._opts.pixelLimit)); + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${x};${x}S`); + } + return true; + case GaAction.READ_MAX: + // read_max returns pixelLimit as square area + const x = Math.floor(Math.sqrt(this._opts.pixelLimit)); + this._report(`\x1b[?${params[0]};${GaStatus.SUCCESS};${x};${x}S`); + return true; + default: + this._report(`\x1b[?${params[0]};${GaStatus.ACTION_ERROR}S`); + return true; + } + } + // exit with error on ReGIS or any other requests + this._report(`\x1b[?${params[0]};${GaStatus.ITEM_ERROR}S`); + return true; + } +} diff --git a/addons/xterm-addon-image/src/ImageRenderer.ts b/addons/xterm-addon-image/src/ImageRenderer.ts new file mode 100644 index 0000000000..8699b35380 --- /dev/null +++ b/addons/xterm-addon-image/src/ImageRenderer.ts @@ -0,0 +1,352 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { toRGBA8888 } from 'sixel/lib/Colors'; +import { IDisposable } from 'xterm'; +import { ICellSize, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types'; + + +const PLACEHOLDER_LENGTH = 4096; +const PLACEHOLDER_HEIGHT = 24; + +/** + * ImageRenderer - terminal frontend extension: + * - provide primitives for canvas, ImageData, Bitmap (static) + * - add canvas layer to DOM (browser only for now) + * - draw image tiles onRender + */ +export class ImageRenderer implements IDisposable { + public canvas: HTMLCanvasElement | undefined; + private _ctx: CanvasRenderingContext2D | null | undefined; + private _placeholder: HTMLCanvasElement | undefined; + private _placeholderBitmap: ImageBitmap | undefined; + private _optionsRefresh: IDisposable | undefined; + private _oldOpen: ((parent: HTMLElement) => void) | undefined; + private _renderService: IRenderService | undefined; + private _oldSetRenderer: ((renderer: any) => void) | undefined; + + // drawing primitive - canvas + public static createCanvas(width: number, height: number): HTMLCanvasElement { + const canvas = document.createElement('canvas'); + canvas.width = width | 0; + canvas.height = height | 0; + return canvas; + } + + // drawing primitive - ImageData with optional buffer + public static createImageData(ctx: CanvasRenderingContext2D, width: number, height: number, buffer?: ArrayBuffer): ImageData { + if (typeof ImageData !== 'function') { + const imgData = ctx.createImageData(width, height); + if (buffer) { + imgData.data.set(new Uint8ClampedArray(buffer, 0, width * height * 4)); + } + return imgData; + } + return buffer + ? new ImageData(new Uint8ClampedArray(buffer, 0, width * height * 4), width, height) + : new ImageData(width, height); + } + + // drawing primitive - ImageBitmap + public static createImageBitmap(img: ImageBitmapSource): Promise { + if (typeof createImageBitmap !== 'function') { + return Promise.resolve(undefined); + } + return createImageBitmap(img); + } + + + constructor(private _terminal: ITerminalExt, private _showPlaceholder: boolean) { + this._oldOpen = this._terminal._core.open; + this._terminal._core.open = (parent: HTMLElement): void => { + this._oldOpen?.call(this._terminal._core, parent); + this._open(); + }; + if (this._terminal._core.screenElement) { + this._open(); + } + // hack to spot fontSize changes + this._optionsRefresh = this._terminal._core.optionsService.onOptionChange(option => { + if (option === 'fontSize') { + this.rescaleCanvas(); + this._renderService?.refreshRows(0, this._terminal.rows); + } + }); + } + + + public dispose(): void { + this._optionsRefresh?.dispose(); + this._removeLayerFromDom(); + if (this._terminal._core && this._oldOpen) { + this._terminal._core.open = this._oldOpen; + this._oldOpen = undefined; + } + if (this._renderService && this._oldSetRenderer) { + this._renderService.setRenderer = this._oldSetRenderer; + this._oldSetRenderer = undefined; + } + this._renderService = undefined; + this.canvas = undefined; + this._ctx = undefined; + this._placeholderBitmap?.close(); + this._placeholderBitmap = undefined; + this._placeholder = undefined; + } + + /** + * Enable the placeholder. + */ + public showPlaceholder(value: boolean): void { + if (value) { + if (!this._placeholder && this.cellSize.height !== -1) { + this._createPlaceHolder(Math.max(this.cellSize.height + 1, PLACEHOLDER_HEIGHT)); + } + } else { + this._placeholderBitmap?.close(); + this._placeholderBitmap = undefined; + this._placeholder = undefined; + } + this._renderService?.refreshRows(0, this._terminal.rows); + } + + /** + * Dimensions of the terminal. + * Forwarded from internal render service. + */ + public get dimensions(): IRenderDimensions | undefined { + return this._renderService?.dimensions; + } + + /** + * Current cell size (float). + */ + public get cellSize(): ICellSize { + return { + width: this.dimensions?.actualCellWidth || -1, + height: this.dimensions?.actualCellHeight || -1 + }; + } + + /** + * Clear a region of the image layer canvas. + */ + public clearLines(start: number, end: number): void { + this._ctx?.clearRect( + 0, + start * (this.dimensions?.actualCellHeight || 0), + this.dimensions?.canvasWidth || 0, + (++end - start) * (this.dimensions?.actualCellHeight || 0) + ); + } + + /** + * Clear whole image canvas. + */ + public clearAll(): void { + this._ctx?.clearRect(0, 0, this.canvas?.width || 0, this.canvas?.height || 0); + } + + /** + * Draw neighboring tiles on the image layer canvas. + */ + public draw(imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void { + if (!this._ctx) { + return; + } + const { width, height } = this.cellSize; + + // Don't try to draw anything, if we cannot get valid renderer metrics. + if (width === -1 || height === -1) { + return; + } + + this._rescaleImage(imgSpec, width, height); + const img = imgSpec.actual!; + const cols = Math.ceil(img.width / width); + + const sx = (tileId % cols) * width; + const sy = Math.floor(tileId / cols) * height; + const dx = col * width; + const dy = row * height; + + // safari bug: never access image source out of bounds + const finalWidth = count * width + sx > img.width ? img.width - sx : count * width; + const finalHeight = sy + height > img.height ? img.height - sy : height; + + // Floor all pixel offsets to get stable tile mapping without any overflows. + // Note: For not pixel perfect aligned cells like in the DOM renderer + // this will move a tile slightly to the top/left (subpixel range, thus ignore it). + this._ctx.drawImage( + img, + Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight), + Math.floor(dx), Math.floor(dy), Math.floor(finalWidth), Math.floor(finalHeight) + ); + } + + /** + * Extract a single tile from an image. + */ + public extractTile(imgSpec: IImageSpec, tileId: number): HTMLCanvasElement | undefined { + const { width, height } = this.cellSize; + // Don't try to draw anything, if we cannot get valid renderer metrics. + if (width === -1 || height === -1) { + return; + } + this._rescaleImage(imgSpec, width, height); + const img = imgSpec.actual!; + const cols = Math.ceil(img.width / width); + const sx = (tileId % cols) * width; + const sy = Math.floor(tileId / cols) * height; + const finalWidth = width + sx > img.width ? img.width - sx : width; + const finalHeight = sy + height > img.height ? img.height - sy : height; + + const canvas = ImageRenderer.createCanvas(finalWidth, finalHeight); + const ctx = canvas.getContext('2d'); + if (ctx) { + ctx.drawImage( + img, + Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight), + 0, 0, Math.floor(finalWidth), Math.floor(finalHeight) + ); + return canvas; + } + } + + /** + * Draw a line with placeholder on the image layer canvas. + */ + public drawPlaceholder(col: number, row: number, count: number = 1): void { + if ((this._placeholderBitmap || this._placeholder) && this._ctx) { + const { width, height } = this.cellSize; + + // Don't try to draw anything, if we cannot get valid renderer metrics. + if (width === -1 || height === -1) { + return; + } + + if (height >= this._placeholder!.height) { + this._createPlaceHolder(height + 1); + } + this._ctx.drawImage( + this._placeholderBitmap || this._placeholder!, + col * width, + (row * height) % 2 ? 0 : 1, // needs %2 offset correction + width * count, + height, + col * width, + row * height, + width * count, + height + ); + } + } + + /** + * Rescale image layer canvas if needed. + * Checked once from `ImageStorage.render`. + */ + public rescaleCanvas(): void { + if (!this.canvas) { + return; + } + if (this.canvas.width !== this.dimensions?.canvasWidth || this.canvas.height !== this.dimensions.canvasHeight) { + this.canvas.width = this.dimensions?.canvasWidth || 0; + this.canvas.height = this.dimensions?.canvasHeight || 0; + } + } + + /** + * Rescale image in storage if needed. + */ + private _rescaleImage(spec: IImageSpec, currentWidth: number, currentHeight: number): void { + if (currentWidth === spec.actualCellSize.width && currentHeight === spec.actualCellSize.height) { + return; + } + const { width: originalWidth, height: originalHeight } = spec.origCellSize; + if (currentWidth === originalWidth && currentHeight === originalHeight) { + spec.actual = spec.orig; + spec.actualCellSize.width = originalWidth; + spec.actualCellSize.height = originalHeight; + return; + } + const canvas = ImageRenderer.createCanvas( + Math.ceil(spec.orig!.width * currentWidth / originalWidth), + Math.ceil(spec.orig!.height * currentHeight / originalHeight) + ); + const ctx = canvas.getContext('2d'); + if (ctx) { + ctx.drawImage(spec.orig!, 0, 0, canvas.width, canvas.height); + spec.actual = canvas; + spec.actualCellSize.width = currentWidth; + spec.actualCellSize.height = currentHeight; + } + } + + /** + * Lazy init for the renderer. + */ + private _open(): void { + this._renderService = this._terminal._core._renderService; + this._oldSetRenderer = this._renderService.setRenderer.bind(this._renderService); + this._renderService.setRenderer = (renderer: any) => { + this._removeLayerFromDom(); + this._oldSetRenderer?.call(this._renderService, renderer); + this._insertLayerToDom(); + }; + this._insertLayerToDom(); + if (this._showPlaceholder) { + this._createPlaceHolder(); + } + } + + private _insertLayerToDom(): void { + this.canvas = ImageRenderer.createCanvas(this.dimensions?.canvasWidth || 0, this.dimensions?.canvasHeight || 0); + this.canvas.classList.add('xterm-image-layer'); + this._terminal._core.screenElement?.appendChild(this.canvas); + this._ctx = this.canvas.getContext('2d', { alpha: true, desynchronized: true }); + } + + private _removeLayerFromDom(): void { + this.canvas?.parentNode?.removeChild(this.canvas); + } + + private _createPlaceHolder(height: number = PLACEHOLDER_HEIGHT): void { + this._placeholderBitmap?.close(); + this._placeholderBitmap = undefined; + + // create blueprint to fill placeholder with + const bWidth = 32; // must be 2^n + const blueprint = ImageRenderer.createCanvas(bWidth, height); + const ctx = blueprint.getContext('2d', { alpha: false }); + if (!ctx) return; + const imgData = ImageRenderer.createImageData(ctx, bWidth, height); + const d32 = new Uint32Array(imgData.data.buffer); + const black = toRGBA8888(0, 0, 0); + const white = toRGBA8888(255, 255, 255); + d32.fill(black); + for (let y = 0; y < height; ++y) { + const shift = y % 2; + const offset = y * bWidth; + for (let x = 0; x < bWidth; x += 2) { + d32[offset + x + shift] = white; + } + } + ctx.putImageData(imgData, 0, 0); + + // create placeholder line, width aligned to blueprint width + const width = (screen.width + bWidth - 1) & ~(bWidth - 1) || PLACEHOLDER_LENGTH; + this._placeholder = ImageRenderer.createCanvas(width, height); + const ctx2 = this._placeholder.getContext('2d', { alpha: false }); + if (!ctx2) { + this._placeholder = undefined; + return; + } + for (let i = 0; i < width; i += bWidth) { + ctx2.drawImage(blueprint, i, 0); + } + ImageRenderer.createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap); + } +} diff --git a/addons/xterm-addon-image/src/ImageStorage.ts b/addons/xterm-addon-image/src/ImageStorage.ts new file mode 100644 index 0000000000..59192338c6 --- /dev/null +++ b/addons/xterm-addon-image/src/ImageStorage.ts @@ -0,0 +1,489 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ +import { IDisposable } from 'xterm'; +import { ImageRenderer } from './ImageRenderer'; +import { ITerminalExt, IExtendedAttrsImage, IImageAddonOptions, IImageSpec, IBufferLineExt, BgFlags, Cell, Content, ICellSize } from './Types'; + + +// fallback default cell size +export const CELL_SIZE_DEFAULT: ICellSize = { + width: 7, + height: 14 +}; + +/** + * Extend extended attribute to also hold image tile information. + */ +export class ExtendedAttrsImage implements IExtendedAttrsImage { + constructor( + public underlineStyle = 0, + public underlineColor: number = -1, + public imageId = -1, + public tileId = -1 + ) { } + public clone(): ExtendedAttrsImage { + return new ExtendedAttrsImage(this.underlineStyle, this.underlineColor, this.imageId, this.tileId); + } + public isEmpty(): boolean { + return this.underlineStyle === 0 && this.imageId === -1; + } +} +const EMPTY_ATTRS = new ExtendedAttrsImage(); + + +/** + * ImageStorage - extension of CoreTerminal: + * - hold image data + * - write/read image data to/from buffer + * + * TODO: image composition for overwrites + */ +export class ImageStorage implements IDisposable { + // storage + private _images: Map = new Map(); + // last used id + private _lastId = 0; + // last evicted id + private _lowestId = 0; + // whether last render call has drawn anything + private _hasDrawn = false; + // hard limit of stored pixels (fallback limit of 10 MB) + private _pixelLimit: number = 2500000; + + private _viewportMetrics: { cols: number, rows: number }; + + constructor( + private _terminal: ITerminalExt, + private _renderer: ImageRenderer, + private _opts: IImageAddonOptions + ) { + try { + this.setLimit(this._opts.storageLimit); + } catch (e: any) { + console.error(e.message); + console.warn(`storageLimit is set to ${this.getLimit()} MB`); + } + this._viewportMetrics = { + cols: this._terminal.cols, + rows: this._terminal.rows + }; + } + + public dispose(): void { + this.reset(); + } + + public reset(): void { + for (const spec of this._images.values()) { + spec.marker?.dispose(); + } + this._images.clear(); + this._renderer.clearAll(); + } + + public getLimit(): number { + return this._pixelLimit * 4 / 1000000; + } + + public setLimit(value: number): void { + if (value < 1 || value > 1000) { + throw RangeError('invalid storageLimit, should be at least 1 MB and not exceed 1G'); + } + this._pixelLimit = (value / 4 * 1000000) >>> 0; + this._evictOldest(0); + } + + public getUsage(): number { + return this._getStoredPixels() * 4 / 1000000; + } + + private _getStoredPixels(): number { + let storedPixels = 0; + for (const spec of this._images.values()) { + if (spec.orig) { + storedPixels += spec.orig.width * spec.orig.height; + if (spec.actual && spec.actual !== spec.orig) { + storedPixels += spec.actual.width * spec.actual.height; + } + } + } + return storedPixels; + } + + /** + * Wipe canvas and images on alternate buffer. + */ + public wipeAlternate(): void { + // remove all alternate tagged images + const zero = []; + for (const [id, spec] of this._images.entries()) { + if (spec.bufferType === 'alternate') { + spec.marker?.dispose(); + zero.push(id); + } + } + for (const id of zero) { + this._images.delete(id); + } + // mark canvas to be wiped on next render + this._hasDrawn = true; + } + + /** + * Method to add an image to the storage. + */ + public addImage(img: HTMLCanvasElement): void { + // never allow storage to exceed memory limit + this._evictOldest(img.width * img.height); + + // calc rows x cols needed to display the image + let cellSize = this._renderer.cellSize; + if (cellSize.width === -1 || cellSize.height === -1) { + cellSize = CELL_SIZE_DEFAULT; + } + const cols = Math.ceil(img.width / cellSize.width); + const rows = Math.ceil(img.height / cellSize.height); + + const imageId = ++this._lastId; + + const buffer = this._terminal._core.buffer; + const termCols = this._terminal.cols; + const termRows = this._terminal.rows; + const originX = buffer.x; + const originY = buffer.y; + let offset = originX; + let tileCount = 0; + + if (!this._opts.sixelScrolling) { + this._terminal._core._dirtyRowService.markAllDirty(); + buffer.x = 0; + buffer.y = 0; + offset = 0; + } + + // TODO: how to go with origin mode / scroll margins here? + for (let row = 0; row < rows; ++row) { + const line = buffer.lines.get(buffer.y + buffer.ybase); + for (let col = 0; col < cols; ++col) { + if (offset + col >= termCols) break; + this._writeToCell(line as IBufferLineExt, offset + col, imageId, row * cols + col); + tileCount++; + } + if (this._opts.sixelScrolling) { + if (row < rows - 1) this._terminal._core._inputHandler.lineFeed(); + } else { + if (++buffer.y >= termRows) break; + } + buffer.x = offset; + } + + // cursor positioning modes + if (this._opts.sixelScrolling) { + if (this._opts.cursorRight) { + buffer.x = offset + cols; + if (buffer.x >= termCols) { + this._terminal._core._inputHandler.lineFeed(); + buffer.x = (this._opts.cursorBelow) ? offset : 0; + } + } else { + this._terminal._core._inputHandler.lineFeed(); + buffer.x = (this._opts.cursorBelow) ? offset : 0; + } + } else { + buffer.x = originX; + buffer.y = originY; + } + + // deleted images with zero tile count + const zero = []; + for (const [id, spec] of this._images.entries()) { + if (spec.tileCount < 1) { + spec.marker?.dispose(); + zero.push(id); + } + } + for (const id of zero) { + this._images.delete(id); + } + + // eviction marker: + // delete the image when the marker gets disposed + const endMarker = this._terminal.registerMarker(0); + endMarker?.onDispose(() => { + const spec = this._images.get(imageId); + if (spec) { + this._images.delete(imageId); + } + }); + + // since markers do not work on alternate for some reason, + // we evict images here manually + if (this._terminal.buffer.active.type === 'alternate') { + this._evictOnAlternate(); + } + + // create storage entry + const imgSpec: IImageSpec = { + orig: img, + origCellSize: cellSize, + actual: img, + actualCellSize: { ...cellSize }, // clone needed, since later modified + marker: endMarker || undefined, + tileCount, + bufferType: this._terminal.buffer.active.type + }; + + // finally add the image + this._images.set(imageId, imgSpec); + } + + + /** + * Render method. Collects buffer information and triggers + * canvas updates. + */ + // TODO: Should we move this to the ImageRenderer? + public render(range: { start: number, end: number }): void { + // exit early if we dont have any images to test for + // FIXME: leaves garbage on screen for IL/DL + if (!this._images.size || !this._renderer.canvas) { + if (this._hasDrawn) { + this._renderer.clearAll(); + this._hasDrawn = false; + } + return; + } + + const { start, end } = range; + const buffer = this._terminal._core.buffer; + const cols = this._terminal._core.cols; + this._hasDrawn = false; + + // clear drawing area + this._renderer.clearLines(start, end); + // rescale if needed + this._renderer.rescaleCanvas(); + + // walk all cells in viewport and draw tiles found + for (let row = start; row <= end; ++row) { + const line = buffer.lines.get(row + buffer.ydisp) as IBufferLineExt; + if (!line) return; + for (let col = 0; col < cols; ++col) { + if (line.getBg(col) & BgFlags.HAS_EXTENDED) { + let e: IExtendedAttrsImage = line._extendedAttrs[col] || EMPTY_ATTRS; + const imageId = e.imageId; + if (imageId === undefined || imageId === -1) { + continue; + } + const imgSpec = this._images.get(imageId); + if (e.tileId !== -1) { + const startTile = e.tileId; + const startCol = col; + let count = 1; + /** + * merge tiles to the right into a single draw call, if: + * - not at end of line + * - cell has same image id + * - cell has consecutive tile id + */ + while ( + ++col < cols + && (line.getBg(col) & BgFlags.HAS_EXTENDED) + && (e = line._extendedAttrs[col] || EMPTY_ATTRS) + && (e.imageId === imageId) + && (e.tileId === startTile + count) + ) { + count++; + } + col--; + if (imgSpec) { + if (imgSpec.actual) { + this._renderer.draw(imgSpec, startTile, startCol, row, count); + } + } else if (this._opts.showPlaceholder) { + this._renderer.drawPlaceholder(startCol, row, count); + } + this._hasDrawn = true; + } + } + } + } + } + + public viewportResize(metrics: { cols: number, rows: number }): void { + // exit early if we have nothing in storage + if (!this._images.size) { + this._viewportMetrics = metrics; + return; + } + + // handle only viewport width enlargements, exit all other cases + // TODO: needs patch for tile counter + if (this._viewportMetrics.cols >= metrics.cols) { + this._viewportMetrics = metrics; + return; + } + + // walk scrollbuffer at old col width to find all possible expansion matches + const buffer = this._terminal._core.buffer; + const rows = buffer.lines.length; + const oldCol = this._viewportMetrics.cols - 1; + for (let row = 0; row < rows; ++row) { + const line = buffer.lines.get(row) as IBufferLineExt; + if (line.getBg(oldCol) & BgFlags.HAS_EXTENDED) { + const e: IExtendedAttrsImage = line._extendedAttrs[oldCol] || EMPTY_ATTRS; + const imageId = e.imageId; + if (imageId === undefined || imageId === -1) { + continue; + } + const imgSpec = this._images.get(imageId); + if (!imgSpec) { + continue; + } + // found an image tile at oldCol, check if it qualifies for right exapansion + const tilesPerRow = Math.ceil((imgSpec.actual?.width || 0) / imgSpec.actualCellSize.width); + if ((e.tileId % tilesPerRow) + 1 >= tilesPerRow) { + continue; + } + // expand only if right side is empty (nothing got wrapped from below) + let hasData = false; + for (let rightCol = oldCol + 1; rightCol > metrics.cols; ++rightCol) { + if (line._data[rightCol * Cell.SIZE + Cell.CONTENT] & Content.HAS_CONTENT_MASK) { + hasData = true; + break; + } + } + if (hasData) { + continue; + } + // do right expansion on terminal buffer + const end = Math.min(metrics.cols, tilesPerRow - (e.tileId % tilesPerRow) + oldCol); + let lastTile = e.tileId; + for (let expandCol = oldCol + 1; expandCol < end; ++expandCol) { + this._writeToCell(line as IBufferLineExt, expandCol, imageId, ++lastTile); + imgSpec.tileCount++; + } + } + } + // store new viewport metrics + this._viewportMetrics = metrics; + } + + /** + * Retrieve original canvas at buffer position. + */ + public getImageAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined { + const buffer = this._terminal._core.buffer; + const line = buffer.lines.get(y) as IBufferLineExt; + if (line && line.getBg(x) & BgFlags.HAS_EXTENDED) { + const e: IExtendedAttrsImage = line._extendedAttrs[x] || EMPTY_ATTRS; + if (e.imageId && e.imageId !== -1) { + return this._images.get(e.imageId)?.orig; + } + } + } + + /** + * Extract active single tile at buffer position. + */ + public extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined { + const buffer = this._terminal._core.buffer; + const line = buffer.lines.get(y) as IBufferLineExt; + if (line && line.getBg(x) & BgFlags.HAS_EXTENDED) { + const e: IExtendedAttrsImage = line._extendedAttrs[x] || EMPTY_ATTRS; + if (e.imageId && e.imageId !== -1 && e.tileId !== -1) { + const spec = this._images.get(e.imageId); + if (spec) { + return this._renderer.extractTile(spec, e.tileId); + } + } + } + } + + // TODO: Do we need some blob offloading tricks here to avoid early eviction? + // also see https://stackoverflow.com/questions/28307789/is-there-any-limitation-on-javascript-max-blob-size + private _evictOldest(room: number): number { + const used = this._getStoredPixels(); + let current = used; + while (this._pixelLimit < current + room && this._images.size) { + const spec = this._images.get(++this._lowestId); + if (spec && spec.orig) { + current -= spec.orig.width * spec.orig.height; + if (spec.actual && spec.orig !== spec.actual) { + current -= spec.actual.width * spec.actual.height; + } + spec.marker?.dispose(); + this._images.delete(this._lowestId); + } + } + return used - current; + } + + private _writeToCell(line: IBufferLineExt, x: number, imageId: number, tileId: number): void { + if (line._data[x * Cell.SIZE + Cell.BG] & BgFlags.HAS_EXTENDED) { + const old = line._extendedAttrs[x]; + if (old) { + if (old.imageId !== undefined) { + // found an old ExtendedAttrsImage, since we know that + // they are always isolated instances (single cell usage), + // we can re-use it and just update their id entries + const oldSpec = this._images.get(old.imageId); + if (oldSpec) { + // early eviction for in-viewport overwrites + oldSpec.tileCount--; + } + old.imageId = imageId; + old.tileId = tileId; + return; + } + // found a plain ExtendedAttrs instance, clone it to new entry + line._extendedAttrs[x] = new ExtendedAttrsImage(old.underlineStyle, old.underlineColor, imageId, tileId); + return; + } + } + // fall-through: always create new ExtendedAttrsImage entry + line._data[x * Cell.SIZE + Cell.BG] |= BgFlags.HAS_EXTENDED; + line._extendedAttrs[x] = new ExtendedAttrsImage(0, -1, imageId, tileId); + } + + private _evictOnAlternate(): void { + // nullify tile count of all images on alternate buffer + for (const spec of this._images.values()) { + if (spec.bufferType === 'alternate') { + spec.tileCount = 0; + } + } + // re-count tiles on whole buffer + const buffer = this._terminal._core.buffer; + for (let y = 0; y < this._terminal.rows; ++y) { + const line = buffer.lines.get(y) as IBufferLineExt; + if (!line) { + continue; + } + for (let x = 0; x < this._terminal.cols; ++x) { + if (line._data[x * Cell.SIZE + Cell.BG] & BgFlags.HAS_EXTENDED) { + const imgId = line._extendedAttrs[x]?.imageId; + if (imgId) { + const spec = this._images.get(imgId); + if (spec) { + spec.tileCount++; + } + } + } + } + } + // deleted images with zero tile count + const zero = []; + for (const [id, spec] of this._images.entries()) { + if (spec.bufferType === 'alternate' && !spec.tileCount) { + spec.marker?.dispose(); + zero.push(id); + } + } + for (const id of zero) { + this._images.delete(id); + } + } +} diff --git a/addons/xterm-addon-image/src/SixelHandler.ts b/addons/xterm-addon-image/src/SixelHandler.ts new file mode 100644 index 0000000000..44c54845a8 --- /dev/null +++ b/addons/xterm-addon-image/src/SixelHandler.ts @@ -0,0 +1,158 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { ImageStorage } from './ImageStorage'; +import { IDcsHandler, IParams, IImageAddonOptions, ITerminalExt, AttributeData, IColorManager } from './Types'; +import { toRGBA8888, BIG_ENDIAN } from 'sixel/lib/Colors'; +import { RGBA8888 } from 'sixel/lib/Types'; +import { WorkerManager } from './WorkerManager'; +import { ImageRenderer } from './ImageRenderer'; +import { PaletteType } from 'WorkerTypes'; + + +export class SixelHandler implements IDcsHandler { + private _size = 0; + private _fillColor = 0; + private _aborted = false; + + constructor( + private readonly _opts: IImageAddonOptions, + private readonly _storage: ImageStorage, + private readonly _coreTerminal: ITerminalExt, + private readonly _workerManager: WorkerManager + ) {} + + // called on new SIXEL DCS sequence + public hook(params: IParams): void { + // NOOP fall-through for all actions if worker is in non-working condition + this._aborted = this._workerManager.failed; + if (this._aborted) { + return; + } + this._fillColor = params.params[1] === 1 ? 0 : extractActiveBg( + this._coreTerminal._core._inputHandler._curAttrData, + this._coreTerminal._core._colorManager.colors); + // image palette is either shared (using previous one), or one of + // 'VT340-COLOR' | 'VT340-GREY' | 'ANSI256' (ANSI256 as fallthrough) + const palette = this._opts.sixelPrivatePalette === false + ? PaletteType.SHARED + : this._opts.sixelDefaultPalette === 'VT340-COLOR' + ? PaletteType.VT340_COLOR + : this._opts.sixelDefaultPalette === 'VT340-GREY' + ? PaletteType.VT340_GREY + : PaletteType.ANSI_256; + this._size = 0; + this._workerManager.sixelInit(this._fillColor, palette, this._opts.sixelPaletteLimit); + } + + // called for any SIXEL data chunk + public put(data: Uint32Array, start: number, end: number): void { + if (this._aborted || this._workerManager.failed) { + return; + } + if (this._workerManager.sizeExceeded) { + this._workerManager.sixelEnd(false); + this._aborted = true; + return; + } + this._size += end - start; + if (this._size > this._opts.sixelSizeLimit) { + console.warn(`SIXEL: too much data, aborting`); + this._workerManager.sixelEnd(false); + this._aborted = true; + return; + } + /** + * copy data over to worker: + * - narrow data from uint32 to uint8 (high codepoints are not valid for SIXELs) + * - push multiple buffer chunks until all data got written + * + * We cannot limit data flow at the PUT stage as async pausing is + * only implemented for UNHOOK in the parser. To avoid OOM from message flooding + * we have `sixelSizeLimit` above in place. + */ + let p = start; + while (p < end) { + const chunk = new Uint8Array(this._workerManager.getChunk()); + const length = Math.min(end - p, chunk.length); + chunk.set(data.subarray(p, p += length)); + this._workerManager.sixelPut(chunk, length); + } + } + + /** + * Called on finalizing the SIXEL DCS sequence. + * Some notes on control flow and return values: + * - worker is in non-working condition: NOOP with sync return + * - `sixelSizeLimit` exceeded: NOOP with sync return + * - `sixelEnd(false)`: NOOP with sync return + * - `sixelEnd(true)`: + * async path waiting for `Promise` + * from worker depending on decoding success, + * a valid image definition will be added + * to the terminal before finally returning + */ + public unhook(success: boolean): boolean | Promise { + if (this._aborted || this._workerManager.failed) { + return true; + } + const imgPromise = this._workerManager.sixelEnd(success); + if (!imgPromise) { + return true; + } + + return imgPromise.then(data => { + if (!data) { + return true; + } + const canvas = ImageRenderer.createCanvas(data.width, data.height); + const ctx = canvas.getContext('2d'); + if (ctx) { + const imageData = ImageRenderer.createImageData(ctx, data.width, data.height, data.buffer); + ctx.putImageData(imageData, 0, 0); // still taking pretty long for big images + this._storage.addImage(canvas); + } + this._workerManager.sixelSendBuffer(data.buffer); + return true; + }); + } +} + + +/** + * Some helpers to extract current terminal colors. + */ + +// get currently active background color from terminal +// also respect INVERSE setting +function extractActiveBg(attr: AttributeData, colors: IColorManager['colors']): RGBA8888 { + let bg = 0; + if (attr.isInverse()) { + if (attr.isFgDefault()) { + bg = convertLe(colors.foreground.rgba); + } else if (attr.isFgRGB()) { + const t = (attr.constructor as typeof AttributeData).toColorRGB(attr.getFgColor()); + bg = toRGBA8888(...t); + } else { + bg = convertLe(colors.ansi[attr.getFgColor()].rgba); + } + } else { + if (attr.isBgDefault()) { + bg = convertLe(colors.background.rgba); + } else if (attr.isBgRGB()) { + const t = (attr.constructor as typeof AttributeData).toColorRGB(attr.getBgColor()); + bg = toRGBA8888(...t); + } else { + bg = convertLe(colors.ansi[attr.getBgColor()].rgba); + } + } + return bg; +} + +// rgba values on the color managers are always in BE, thus convert to LE +function convertLe(color: number): RGBA8888 { + if (BIG_ENDIAN) return color; + return (color & 0xFF) << 24 | (color >>> 8 & 0xFF) << 16 | (color >>> 16 & 0xFF) << 8 | color >>> 24 & 0xFF; +} diff --git a/addons/xterm-addon-image/src/Types.d.ts b/addons/xterm-addon-image/src/Types.d.ts new file mode 100644 index 0000000000..f4edd20948 --- /dev/null +++ b/addons/xterm-addon-image/src/Types.d.ts @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { IDisposable, IMarker, Terminal } from 'xterm'; + +// private imports from base repo we build against +import { Cell, BgFlags, Content } from 'common/buffer/Constants'; +import type { AttributeData } from 'common/buffer/AttributeData'; +import type { IParams, IDcsHandler, IEscapeSequenceParser } from 'common/parser/Types'; +import type { IBuffer } from 'common/buffer/Types'; +import type { IBufferLine, IExtendedAttrs, IInputHandler } from 'common/Types'; +import type { IOptionsService, IDirtyRowService, ICoreService } from 'common/services/Services'; +import type { IColorManager, ITerminal } from 'browser/Types'; +import type { IRenderDimensions } from 'browser/renderer/Types'; +import type { IRenderService } from 'browser/services/Services'; + +// export some privates for local usage +export { AttributeData, IParams, IDcsHandler, BgFlags, IRenderDimensions, IRenderService, IColorManager, Cell, Content }; + +/** + * Plugin ctor options. + */ +export interface IImageAddonOptions { + enableSizeReports: boolean; + pixelLimit: number; + storageLimit: number; + showPlaceholder: boolean; + cursorRight: boolean; + cursorBelow: boolean; + sixelSupport: boolean; + sixelScrolling: boolean; + sixelPaletteLimit: number; + sixelSizeLimit: number; + sixelPrivatePalette: boolean; + sixelDefaultPalette: 'VT340-COLOR' | 'VT340-GREY' | 'ANSI256'; +} + +/** + * Stub into private interfaces. + * This should be kept in line with common libs. + * Any change made here should be replayed in the accessors test case to + * have a somewhat reliable testing against code changes in the core repo. + */ + +// overloaded IExtendedAttrs to hold image refs +export interface IExtendedAttrsImage extends IExtendedAttrs { + imageId: number; + tileId: number; +} + +/* eslint-disable */ +export interface IBufferLineExt extends IBufferLine { + _extendedAttrs: {[index: number]: IExtendedAttrsImage | undefined}; + _data: Uint32Array; +} + +interface IInputHandlerExt extends IInputHandler { + _parser: IEscapeSequenceParser; + _curAttrData: AttributeData; + onRequestReset(handler: () => void): IDisposable; +} + +export interface ICoreTerminalExt extends ITerminal { + _dirtyRowService: IDirtyRowService; + _colorManager: IColorManager; + _inputHandler: IInputHandlerExt; + _renderService: IRenderService; +} + +export interface ITerminalExt extends Terminal { + _core: ICoreTerminalExt; +} +/* eslint-enable */ + + +/** + * Some storage definitions. + */ +export interface ICellSize { + width: number; + height: number; +} + +export interface IImageSpec { + orig: HTMLCanvasElement | undefined; + origCellSize: ICellSize; + actual: HTMLCanvasElement | undefined; + actualCellSize: ICellSize; + marker: IMarker | undefined; + tileCount: number; + bufferType: 'alternate' | 'normal'; +} diff --git a/addons/xterm-addon-image/src/WorkerManager.ts b/addons/xterm-addon-image/src/WorkerManager.ts new file mode 100644 index 0000000000..e12f9cb39a --- /dev/null +++ b/addons/xterm-addon-image/src/WorkerManager.ts @@ -0,0 +1,155 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { IImageAddonOptions } from './Types'; +import { IDisposable } from 'xterm'; +import { IImageWorkerMessage, IImagePixel, IImageWorker, MessageType, PaletteType, AckPayload } from './WorkerTypes'; + + + +// pool cleanup interval in ms +const CLEANUP_INTERVAL = 20000; + + +/** + * Manager to encapsulate certain worker aspects: + * - lazy worker loading + * - low level communication protocol with worker + * - promise based image dispatcher + * - mem pooling + */ +export class WorkerManager implements IDisposable { + private _worker: IImageWorker | undefined; + private _memPool: ArrayBuffer[] = []; + private _sixelResolver: ((img: IImagePixel | null) => void) | undefined; + private _failedToLoad = false; + private _poolCheckerInterval: number | undefined; + private _lastActive = 0; + public sizeExceeded = false; + + constructor( + public url: string, + private _opts: IImageAddonOptions, + public chunkSize: number = 65536 * 2, + public maxPoolSize: number = 50 + ) {} + + private _startupError: () => void = () => { + console.warn('ImageAddon worker failed to load, image output is disabled.'); + this._failedToLoad = true; + this.dispose(); + }; + + private _message: (msg: MessageEvent) => void = event => { + const data = event.data; + switch (data.type) { + case MessageType.CHUNK_TRANSFER: + this.storeChunk(data.payload); + break; + case MessageType.SIXEL_IMAGE: + if (this._sixelResolver) { + this._sixelResolver(data.payload); + this._sixelResolver = undefined; + } + break; + case MessageType.ACK: + this._worker?.removeEventListener('error', this._startupError); + break; + case MessageType.SIZE_EXCEEDED: + this.sizeExceeded = true; + break; + } + }; + + private _setSixelResolver(resolver?: (img: IImagePixel | null) => void): void { + if (this._sixelResolver) { + this._sixelResolver(null); + } + this._sixelResolver = resolver; + } + + public dispose(): void { + this._worker?.terminate(); + this._worker = undefined; + this._setSixelResolver(); + this.flushPool(); + if (this._poolCheckerInterval) { + clearInterval(this._poolCheckerInterval); + this._poolCheckerInterval = undefined; + } + } + + public get failed(): boolean { + return this._failedToLoad; + } + + public get worker(): IImageWorker | undefined { + if (!this._worker && !this._failedToLoad) { + this._worker = new Worker(this.url); + this._worker.addEventListener('message', this._message, false); + this._worker.addEventListener('error', this._startupError, false); + this._worker.postMessage({ + type: MessageType.ACK, + payload: AckPayload.PING, + options: { pixelLimit: this._opts.pixelLimit } + }); + } + return this._worker; + } + + public getChunk(): ArrayBuffer { + this._lastActive = Date.now(); + return this._memPool.pop() || new ArrayBuffer(this.chunkSize); + } + + public storeChunk(chunk: ArrayBuffer): void { + if (!this._poolCheckerInterval) { + this._poolCheckerInterval = setInterval(() => { + if (Date.now() - this._lastActive > CLEANUP_INTERVAL) { + this.flushPool(); + clearInterval(this._poolCheckerInterval); + this._poolCheckerInterval = undefined; + } + }, CLEANUP_INTERVAL); + } + if (this._memPool.length < this.maxPoolSize) { + this._memPool.push(chunk); + } + } + + public flushPool(): void { + this._memPool.length = 0; + } + + // SIXEL message interface + public sixelInit(fillColor: number, paletteType: PaletteType, limit: number): void { + this._setSixelResolver(); + this.sizeExceeded = false; + this.worker?.postMessage({ + type: MessageType.SIXEL_INIT, + payload: { fillColor, paletteType, limit } + }); + } + public sixelPut(data: Uint8Array, length: number): void { + this.worker?.postMessage({ + type: MessageType.SIXEL_PUT, + payload: { + buffer: data.buffer, + length + } + }, [data.buffer]); + } + public sixelEnd(success: boolean): Promise | void { + let result: Promise | undefined; + if (success && this.worker) { + result = new Promise(resolve => this._setSixelResolver(resolve)); + } + this.worker?.postMessage({ type: MessageType.SIXEL_END, payload: success }); + return result; + } + public sixelSendBuffer(buffer: ArrayBuffer): void { + this.worker?.postMessage({ type: MessageType.CHUNK_TRANSFER, payload: buffer }, [buffer]); + } +} diff --git a/addons/xterm-addon-image/src/WorkerTypes.d.ts b/addons/xterm-addon-image/src/WorkerTypes.d.ts new file mode 100644 index 0000000000..679230718f --- /dev/null +++ b/addons/xterm-addon-image/src/WorkerTypes.d.ts @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +// setup options +export interface ISetupOptions { + pixelLimit: number; +} + +// pixel data from worker +export interface IImagePixel { + buffer: ArrayBuffer; + width: number; + height: number; +} + +// message types +export const enum MessageType { + ACK = 1, + SIXEL_INIT = 2, + SIXEL_PUT = 3, + SIXEL_END = 4, + SIXEL_IMAGE = 5, + CHUNK_TRANSFER = 6, + SIZE_EXCEEDED = 7 +} + +// palette types +export const enum PaletteType { + SHARED = 0, + VT340_COLOR = 1, + VT340_GREY = 2, + ANSI_256 = 3 +} + +// ACK payload +export const enum AckPayload { + PING = 0, + ALIVE = 1 +} + +/** + * Worker message protocol types (used on both ends). + */ +export interface IAckMessage { + type: MessageType.ACK; + payload: AckPayload; + options: ISetupOptions | null; +} +// outgoing +export interface ISixelInitMessage { + type: MessageType.SIXEL_INIT; + payload: { + fillColor: number; + paletteType: PaletteType; + limit: number; + }; +} +export interface ISixelPutMessage { + type: MessageType.SIXEL_PUT; + payload: { + buffer: ArrayBuffer; + length: number; + }; +} +export interface ISixelEndMessage { + type: MessageType.SIXEL_END; + payload: boolean; +} +// incoming +export interface ISixelImageMessage { + type: MessageType.SIXEL_IMAGE; + payload: IImagePixel | null; +} +export interface IChunkTransferMessage { + type: MessageType.CHUNK_TRANSFER; + payload: ArrayBuffer; +} +export interface ISizeExceededMessage { + type: MessageType.SIZE_EXCEEDED; +} + +export type IImageWorkerMessage = ( + IAckMessage | ISixelInitMessage | ISixelPutMessage | ISixelEndMessage | + ISixelImageMessage | IChunkTransferMessage | ISizeExceededMessage +); + +export interface IPostMessage { + (message: T, transfer: Transferable[]): void; + (message: T, options?: PostMessageOptions): void; +} + +export interface IImageWorker extends Worker { + postMessage: IPostMessage; +} diff --git a/addons/xterm-addon-image/src/tsconfig.json b/addons/xterm-addon-image/src/tsconfig.json new file mode 100644 index 0000000000..da519e17d9 --- /dev/null +++ b/addons/xterm-addon-image/src/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "es2017", + "module": "commonjs", + "sourceMap": true, + "outDir": "../out", + "rootDir": ".", + "strict": true, + "noUnusedLocals": true, + "preserveWatchOutput": true, + "types": [ + "../../../node_modules/@types/mocha" + ], + "baseUrl": ".", + "paths": { + "browser/*": [ "../../../src/browser/*" ], + "common/*": [ "../../../src/common/*" ] + } + }, + "include": [ + "./**/*", + "../../../typings/xterm.d.ts" + ], + "references": [ + { "path": "../../../src/browser" }, + { "path": "../../../src/common" } + ] +} diff --git a/addons/xterm-addon-image/test/ImageAddon.api.ts b/addons/xterm-addon-image/test/ImageAddon.api.ts new file mode 100644 index 0000000000..32c21a90bc --- /dev/null +++ b/addons/xterm-addon-image/test/ImageAddon.api.ts @@ -0,0 +1,397 @@ +/** + * Copyright (c) 2020 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { assert } from 'chai'; +import { openTerminal, launchBrowser } from '../../../out-test/api/TestUtils'; +import { Browser, Page } from 'playwright'; +import { IImageAddonOptions } from '../src/Types'; +import { FINALIZER, introducer, sixelEncode } from 'sixel'; +import { readFileSync } from 'fs'; +import PNG from 'png-ts'; + +const APP = 'http://127.0.0.1:3001/test'; + +let browser: Browser; +let page: Page; +const width = 800; +const height = 600; + +// eslint-disable-next-line +declare const ImageAddon: { + new(workerPath: string, options?: Partial): any; +}; + +interface ITestData { + width: number; + height: number; + bytes: Uint8Array; + palette: number[]; + sixel: string; +} + +interface IDimensions { + cellWidth: number; + cellHeight: number; + width: number; + height: number; +} + +const IMAGE_WORKER_PATH = '/workers/xterm-addon-image-worker.js'; + +// image: 640 x 80, 512 color +const TESTDATA: ITestData = (() => { + const pngImage = PNG.load(readFileSync('./addons/xterm-addon-image/fixture/palette.png')); + const data8 = pngImage.decode(); + const data32 = new Uint32Array(data8.buffer); + const palette = new Set(); + for (let i = 0; i < data32.length; ++i) palette.add(data32[i]); + const sixel = sixelEncode(data8, pngImage.width, pngImage.height, [...palette]); + return { + width: pngImage.width, + height: pngImage.height, + bytes: data8, + palette: [...palette], + sixel + }; +})(); +const SIXEL_SEQ_0 = introducer(0) + TESTDATA.sixel + FINALIZER; +// const SIXEL_SEQ_1 = introducer(1) + TESTDATA.sixel + FINALIZER; +// const SIXEL_SEQ_2 = introducer(2) + TESTDATA.sixel + FINALIZER; + + +describe('ImageAddon', () => { + before(async () => { + browser = await launchBrowser(); + page = await (await browser.newContext()).newPage(); + await page.setViewportSize({ width, height }); + }); + + after(async () => { + await browser.close(); + }); + + beforeEach(async () => { + await page.goto(APP); + await openTerminal(page); + await page.evaluate(opts => { + (window as any).imageAddon = new ImageAddon(opts.workerPath, opts.opts); + (window as any).term.loadAddon((window as any).imageAddon); + }, { workerPath: IMAGE_WORKER_PATH, opts: { sixelPaletteLimit: 512 } }); + }); + + it('test for private accessors', async () => { + // terminal privates + const accessors = [ + '_core', + '_core._dirtyRowService', + '_core._renderService', + '_core._inputHandler', + '_core._inputHandler._parser', + '_core._inputHandler._curAttrData', + '_core._colorManager' + ]; + for (const prop of accessors) { + assert.equal( + await page.evaluate('(() => { const v = window.term.' + prop + '; return v !== undefined && v !== null; })()'), + true, `problem at ${prop}` + ); + } + // bufferline privates + assert.equal(await page.evaluate('window.term._core.buffer.lines.get(0)._data instanceof Uint32Array'), true); + assert.equal(await page.evaluate('window.term._core.buffer.lines.get(0)._extendedAttrs instanceof Object'), true); + // inputhandler privates + assert.equal(await page.evaluate('window.term._core._inputHandler._curAttrData.constructor.name'), 'AttributeData'); + assert.equal(await page.evaluate('window.term._core._inputHandler._parser.constructor.name'), 'EscapeSequenceParser'); + }); + + describe('ctor options', () => { + it('empty settings should load defaults', async () => { + const DEFAULT_OPTIONS: IImageAddonOptions = { + enableSizeReports: true, + pixelLimit: 16777216, + cursorRight: false, + cursorBelow: false, + sixelSupport: true, + sixelScrolling: true, + sixelPaletteLimit: 512, // set to 512 to get example image working + sixelSizeLimit: 25000000, + sixelPrivatePalette: true, + sixelDefaultPalette: 'VT340-COLOR', + storageLimit: 128, + showPlaceholder: true + }; + assert.deepEqual(await page.evaluate(`window.imageAddon._opts`), DEFAULT_OPTIONS); + }); + it('custom settings should overload defaults', async () => { + const customSettings: IImageAddonOptions = { + enableSizeReports: false, + pixelLimit: 5, + cursorRight: true, + cursorBelow: true, + sixelSupport: false, + sixelScrolling: false, + sixelPaletteLimit: 1024, + sixelSizeLimit: 1000, + sixelPrivatePalette: false, + sixelDefaultPalette: 'VT340-GREY', + storageLimit: 10, + showPlaceholder: false + }; + await page.evaluate(opts => { + (window as any).imageAddonCustom = new ImageAddon(opts.workerPath, opts.opts); + (window as any).term.loadAddon((window as any).imageAddonCustom); + }, { workerPath: IMAGE_WORKER_PATH, opts: customSettings }); + assert.deepEqual(await page.evaluate(`window.imageAddonCustom._opts`), customSettings); + }); + }); + + describe('scrolling & cursor modes', () => { + it('testdata default (scrolling, cursor next line, beginning)', async () => { + const dim = await getDimensions(); + await writeToTerminal(SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [0, Math.ceil(TESTDATA.height/dim.cellHeight)]); + // moved to right by 10 cells + await writeToTerminal('#'.repeat(10) + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [0, Math.ceil(TESTDATA.height/dim.cellHeight) * 2]); + // await new Promise(res => setTimeout(res, 1000)); + }); + it('write testdata noScrolling', async () => { + await writeToTerminal('\x1b[?80h' + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [0, 0]); + // second draw does not change anything + await writeToTerminal(SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [0, 0]); + }); + it.skip('testdata cursor right', async () => { + const dim = await getDimensions(); + await writeToTerminal('\x1b[?8452h' + SIXEL_SEQ_0); + // currently failing on OSX firefox with AssertionError: expected [ 72, 4 ] to deeply equal [ 72, 5 ] + assert.deepEqual(await getCursor(), [Math.ceil(TESTDATA.width/dim.cellWidth), Math.floor(TESTDATA.height/dim.cellHeight)]); + }); + it('testdata cursor right with overflow beginning', async () => { + const dim = await getDimensions(); + await writeToTerminal('\x1b[?8452h' + '#'.repeat(30) + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [0, Math.ceil(TESTDATA.height/dim.cellHeight)]); + }); + it('testdata cursor right with overflow below', async () => { + const dim = await getDimensions(); + await writeToTerminal('\x1b[?8452h\x1b[?7730l' + '#'.repeat(30) + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [30, Math.ceil(TESTDATA.height/dim.cellHeight)]); + }); + it('testdata cursor always below', async () => { + const dim = await getDimensions(); + // offset 0 + await writeToTerminal('\x1b[?7730l' + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [0, Math.ceil(TESTDATA.height/dim.cellHeight)]); + // moved to right by 10 cells + await writeToTerminal('#'.repeat(10) + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [10, Math.ceil(TESTDATA.height/dim.cellHeight) * 2]); + // moved by 30 cells (+10 prev) + await writeToTerminal('#'.repeat(30) + SIXEL_SEQ_0); + assert.deepEqual(await getCursor(), [10 + 30, Math.ceil(TESTDATA.height/dim.cellHeight) * 3]); + }); + }); + + describe('image lifecycle & eviction', () => { + it('delete image once scrolled off', async () => { + await writeToTerminal(SIXEL_SEQ_0); + assert.equal(await getImageStorageLength(), 1); + // scroll to scrollback + rows - 1 + await page.evaluate( + scrollback => new Promise(res => (window as any).term.write('\n'.repeat(scrollback), res)), + (await getScrollbackPlusRows() - 1) + ); + assert.equal(await getImageStorageLength(), 1); + // scroll one further should delete the image + await page.evaluate(() => new Promise(res => (window as any).term.write('\n', res))); + assert.equal(await getImageStorageLength(), 0); + }); + it('get storageUsage', async () => { + assert.equal(await page.evaluate('imageAddon.storageUsage'), 0); + await writeToTerminal(SIXEL_SEQ_0); + assert.closeTo(await page.evaluate('imageAddon.storageUsage'), 640 * 80 * 4 / 1000000, 0.05); + }); + it('get/set storageLimit', async () => { + assert.equal(await page.evaluate('imageAddon.storageLimit'), 128); + assert.equal(await page.evaluate('imageAddon.storageLimit = 1'), 1); + assert.equal(await page.evaluate('imageAddon.storageLimit'), 1); + }); + it('remove images by storage limit pressure', async () => { + assert.equal(await page.evaluate('imageAddon.storageLimit = 1'), 1); + // never go beyond storage limit + await writeToTerminal(SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0); + const usage = await page.evaluate('imageAddon.storageUsage'); + await writeToTerminal(SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0); + assert.equal(await page.evaluate('imageAddon.storageUsage'), usage); + assert.equal(usage as number < 1, true); + }); + it('set storageLimit removes images synchronously', async () => { + await writeToTerminal(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); + const usage: number = await page.evaluate('imageAddon.storageUsage'); + const newUsage: number = await page.evaluate('imageAddon.storageLimit = 1; imageAddon.storageUsage'); + assert.equal(newUsage < usage, true); + assert.equal(newUsage < 1, true); + }); + it('clear alternate images on buffer change', async () => { + assert.equal(await page.evaluate('imageAddon.storageUsage'), 0); + await writeToTerminal('\x1b[?1049h' + SIXEL_SEQ_0); + assert.closeTo(await page.evaluate('imageAddon.storageUsage'), 640 * 80 * 4 / 1000000, 0.05); + await writeToTerminal('\x1b[?1049l'); + assert.equal(await page.evaluate('imageAddon.storageUsage'), 0); + }); + it('evict tiles by in-place overwrites (only full overwrite tested)', async () => { + await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H'); + const usage = await page.evaluate('imageAddon.storageUsage'); + await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H'); + await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H'); + await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H'); + assert.equal(await page.evaluate('imageAddon.storageUsage'), usage); + }); + it('manual eviction on alternate buffer must not miss images', async () => { + await writeToTerminal('\x1b[?1049h'); + await writeToTerminal(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); + const usage: number = await page.evaluate('imageAddon.storageUsage'); + await writeToTerminal(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); + await writeToTerminal(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); + const newUsage: number = await page.evaluate('imageAddon.storageUsage'); + assert.equal(newUsage, usage); + }); + }); + + describe('worker integration & manager', () => { + async function execOnManager(prop?: string): Promise { + if (prop) { + return page.evaluate('window.imageAddon._workerManager.' + prop); + } + return page.evaluate('window.imageAddon._workerManager'); + } + it('gets URL from addon settings', async () => { + // hard coded default + assert.equal(await execOnManager('url'), '/workers/xterm-addon-image-worker.js'); + // custom + await page.evaluate(opts => { + (window as any).imageAddonCustom = new ImageAddon('xyz.js', opts); + (window as any).term.loadAddon((window as any).imageAddonCustom); + }, {}); + assert.equal(await page.evaluate(`window.imageAddonCustom._workerManager.url`), 'xyz.js'); + }); + it('timed chunk pooling', async () =>{ + // image fits into one chunk + await writeToTerminal(SIXEL_SEQ_0); + assert.equal(await execOnManager('_memPool.length'), 1); + assert.notEqual(await execOnManager('_poolCheckerInterval'), undefined); + const lastActive = await execOnManager('_lastActive'); + assert.notEqual(lastActive, 0); + }); + it.skip('max chunks with cleanup after 20s', async function (): Promise { + // Note: by default this test is skipped as it takes really long + this.timeout(30000); + // more than max chunks created (exceeding pooling) + const count = 100; // MAX_CHUNKS is 50 + const chunkLength = Math.ceil(SIXEL_SEQ_0.length/count); + for (let i = 0; i < count; ++i) { + const offset = i * chunkLength; + page.evaluate(data => (window as any).term.write(data), SIXEL_SEQ_0.slice(offset, offset + chunkLength)); + } + await writeToTerminal(''); // wait until all got consumed + assert.equal(await execOnManager('_memPool.length'), 50); + assert.notEqual(await execOnManager('_poolCheckerInterval'), undefined); + const lastActive = await execOnManager('_lastActive'); + assert.notEqual(lastActive, 0); + // should drop back to 0 after 20000 + await new Promise(res => setTimeout(async () => { + assert.equal(await execOnManager('_memPool.length'), 0); + assert.equal(await execOnManager('_poolCheckerInterval'), undefined); + res(); + }, 20000)); + }); + it('dispose should stop everything', async () => { + await writeToTerminal(SIXEL_SEQ_0); + const mustResolveWithDispose = execOnManager('sixelEnd(true)').then(() => 'yeah'); + await execOnManager('dispose()'); + // worker gone + assert.equal(await execOnManager('_worker'), undefined); + // pending resolver cleared + assert.equal(await mustResolveWithDispose, 'yeah'); + assert.equal(await execOnManager('_sixelResolver'), undefined); + // pool and checker cleared + assert.equal(await execOnManager('_memPool.length'), 0); + assert.equal(await execOnManager('_poolCheckerInterval'), undefined); + }); + describe('handle worker loading error gracefully', () => { + beforeEach(async () => { + await page.evaluate(opts => { + (window as any).imageAddonCustom = new ImageAddon('xyz.js', opts); + (window as any).term.loadAddon((window as any).imageAddonCustom); + }, {}); + }); + it('failed is set upon first worker access', async () => { + assert.equal(await page.evaluate(`window.imageAddonCustom._workerManager.failed`), false); + // We have to test it here with .endSixel as it is the only promised method + // we have implemented. This is needed to wait here for the full request-response + // cycling of the initial ACK message after the lazy worker loading. + assert.equal(await page.evaluate(`window.imageAddonCustom._workerManager.sixelEnd(true)`), null); + // Alternatively we could have waited for some time after the first `worker` access. + // await page.evaluate(`window.imageAddonCustom._workerManager.worker`); + // await new Promise(res => setTimeout(res, 50)); + assert.equal(await page.evaluate(`window.imageAddonCustom._workerManager.failed`), true); + // Note: For the sixel handler this means that early `sixelInit` and `sixelPut` API calls + // are still not a NOOP, as the worker instance in the manager still looks healthy. + // This is not really a problem, as those calls are only sending and not waiting for response. + // A minor optimization in the handler tests for the failed state on every action to spot it as + // early as possible. + }); + it('sequence turns into NOOP, handler does not block forever', async () => { + // dispose normal image addon + await page.evaluate(`window.imageAddon.dispose()`); + // proper SIXEL sequence + await writeToTerminal('#' + SIXEL_SEQ_0 + '#'); + assert.deepEqual(await getCursor(), [2, 0]); + // sequence with color definition but missing SIXEL bytes (0 pixel image) + await writeToTerminal('#' + '\x1bPq#14;2;0;100;100\x1b\\' + '#'); + assert.deepEqual(await getCursor(), [4, 0]); + // shortest possible sequence (no data bytes at all) + await writeToTerminal('#' + '\x1bPq\x1b\\' + '#'); + assert.deepEqual(await getCursor(), [6, 0]); + }); + }); + }); + +}); + +/** + * terminal access helpers. + */ +async function getDimensions(): Promise { + const dimensions: any = await page.evaluate(`term._core._renderService.dimensions`); + return { + cellWidth: Math.round(dimensions.actualCellWidth), + cellHeight: Math.round(dimensions.actualCellHeight), + width: Math.round(dimensions.canvasWidth), + height: Math.round(dimensions.canvasHeight) + }; +} + +async function getCursor(): Promise<[number, number]> { + return page.evaluate('[window.term.buffer.active.cursorX, window.term.buffer.active.cursorY]'); +} + +async function getImageStorageLength(): Promise { + return page.evaluate('window.imageAddon._storage._images.size'); +} + +async function getScrollbackPlusRows(): Promise { + return page.evaluate('window.term.getOption(\'scrollback\') + window.term.rows'); +} + +async function writeToTerminal(d: string): Promise { + return page.evaluate(data => new Promise(res => (window as any).term.write(data, res)), d); +} diff --git a/addons/xterm-addon-image/test/tsconfig.json b/addons/xterm-addon-image/test/tsconfig.json new file mode 100644 index 0000000000..a8c25b5cae --- /dev/null +++ b/addons/xterm-addon-image/test/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2017", + "rootDir": ".", + "outDir": "../out-test", + "sourceMap": true, + "removeComments": true, + "strict": true, + "baseUrl": ".", + "paths": { + "browser/*": [ "../../../src/browser/*" ], + "common/*": [ "../../../src/common/*" ] + }, + "types": [ + "../../../node_modules/@types/mocha", + "../../../node_modules/@types/node", + "../../../out-test/api/TestUtils" + ] + }, + "include": [ + "./**/*", + "../../../typings/xterm.d.ts" + ], + "references": [ + { "path": "../../../src/browser" }, + { "path": "../../../src/common" } + ] +} diff --git a/addons/xterm-addon-image/tsconfig.json b/addons/xterm-addon-image/tsconfig.json new file mode 100644 index 0000000000..34882739b0 --- /dev/null +++ b/addons/xterm-addon-image/tsconfig.json @@ -0,0 +1,9 @@ +{ + "files": [], + "include": [], + "references": [ + { "path": "./src" }, + { "path": "./test" }, + { "path": "./src-worker" } + ] +} diff --git a/addons/xterm-addon-image/typings/xterm-addon-image.d.ts b/addons/xterm-addon-image/typings/xterm-addon-image.d.ts new file mode 100644 index 0000000000..b1e64f5577 --- /dev/null +++ b/addons/xterm-addon-image/typings/xterm-addon-image.d.ts @@ -0,0 +1,131 @@ +/** + * Copyright (c) 2017 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { Terminal, ITerminalAddon } from 'xterm'; + +declare module 'xterm-addon-image' { + export interface IImageAddonOptions { + /** + * Enable size reports in windowOptions: + * - getWinSizePixels (CSI 14 t) + * - getCellSizePixels (CSI 16 t) + * - getWinSizeChars (CSI 18 t) + * + * If `true` (default), the reports will be activated during addon loading. + * If `false`, no settings will be touched. Use `false`, if you have high + * security constraints and/or deal with windowOptions by other means. + * On addon disposal, the settings will not change. + */ + enableSizeReports?: boolean; + + /** + * Maximum pixels a single image may hold. Images exceeding this number will + * be discarded during processing with no changes to the terminal buffer + * (no cursor advance, no placeholder). + * This setting is mainly used to restrict images sizes during initial decoding + * including the final canvas creation. + * + * Note: The image worker decoder may hold additional memory up to + * `pixelLimit` * 4 bytes permanently, plus the same amount on top temporarily + * for pixel transfers, which should be taken into account under memory pressure conditions. + * + * Note: Browsers restrict allowed canvas dimensions further. We dont reflect those + * limits here, thus the construction of an oddly shaped image having most pixels + * in one dimension still can fail. + * + * Note: `storageLimit` bytes are calculated from images by multiplying the pixels with 4 + * (4 channels with one byte, images are stored as RGBA8888). + * + * Default is 2^16 (4096 x 4096 pixels). + */ + pixelLimit?: number; + + /** + * Storage limit in MB. + * The storage implements a FIFO cache removing old images, when the limit gets hit. + * Also exposed as addon property for runtime adjustments. + * Default is 128 MB. + */ + storageLimit?: number; + + /** + * Whether to show a placeholder for images removed from cache, default is true. + */ + showPlaceholder?: boolean; + + /** + * Leave cursor to right of image. + * This has no effect, if an image covers all cells to the right. + * Same as DECSET 8452, default is false. + */ + cursorRight?: boolean; + + /** + * Leave cursor below the first row of an image, scrolling if needed. + * If disabled, the cursor is left at the beginning of the next line. + * This settings is partially overwritten by `cursorRight`, if an image + * does not cover all cells to the right. + * Same as DECSET 7730, default is false. + */ + cursorBelow?: boolean; + + /** + * SIXEL settings + */ + + /** Whether SIXEL is enabled (default is true). */ + sixelSupport?: boolean; + /** Whether SIXEL scrolling is enabled (default is true). Same as DECSET 80. */ + sixelScrolling?: boolean; + /** Palette color limit (default 256). */ + sixelPaletteLimit?: number; + /** SIXEL image size limit in bytes (default 25000000 bytes). */ + sixelSizeLimit?: number; + /** Whether to use private palettes for SIXEL sequences (default is true). Same as DECSET 1070. */ + sixelPrivatePalette?: boolean; + /** Default start palette (default is 'VT340-COLOR'). */ + sixelDefaultPalette?: 'VT340-COLOR' | 'VT340-GREY' | 'ANSI256'; + } + + export class ImageAddon implements ITerminalAddon { + constructor(workerPath: string, options?: IImageAddonOptions); + public activate(terminal: Terminal): void; + public dispose(): void; + + /** + * Reset the image addon. + * + * This resets all runtime options to default values (as given to the ctor) + * and resets the image storage. + */ + public reset(): void; + + /** + * Getter/Setter for the storageLimit in MB. + * Synchronously deletes images if the stored data exceeds the new value. + */ + public storageLimit: number; + + /** + * Current memory usage of the stored images in MB. + */ + public readonly storageUsage: number; + + /** + * Getter/Setter whether the placeholder should be shown. + */ + public showPlaceholder: boolean; + + /** + * Get original image canvas at buffer position. + */ + public getImageAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined; + + /** + * Extract single tile canvas at buffer position. + */ + public extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined; + } +} diff --git a/addons/xterm-addon-image/webpack.config.js b/addons/xterm-addon-image/webpack.config.js new file mode 100644 index 0000000000..d2b3c2604b --- /dev/null +++ b/addons/xterm-addon-image/webpack.config.js @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2019 The xterm.js authors. All rights reserved. + * @license MIT + */ + +const path = require('path'); + +const addonName = 'ImageAddon'; +const mainFile = 'xterm-addon-image.js'; +const mainFileWorker = 'xterm-addon-image-worker.js'; +const workerName = 'main'; + +const addon = { + entry: `./out/${addonName}.js`, + devtool: 'source-map', + module: { + rules: [ + { + test: /\.js$/, + use: ["source-map-loader"], + enforce: "pre", + exclude: /node_modules/ + } + ] + }, + output: { + filename: mainFile, + path: path.resolve('./lib'), + library: addonName, + libraryTarget: 'umd' + }, + mode: 'production' +}; + +// worker target bundled as ./lib/xterm-addon-image-worker.js +const worker = { + entry: `./out-worker/${workerName}.js`, + devtool: 'source-map', + module: { + rules: [ + { + test: /\.js$/, + use: ["source-map-loader"], + enforce: "pre", + exclude: /node_modules/ + } + ] + }, + output: { + filename: mainFileWorker, + path: path.resolve('./lib'), + libraryTarget: 'umd' + }, + mode: 'production' +}; + +module.exports = [addon, worker]; diff --git a/addons/xterm-addon-image/yarn.lock b/addons/xterm-addon-image/yarn.lock new file mode 100644 index 0000000000..2e399ad42e --- /dev/null +++ b/addons/xterm-addon-image/yarn.lock @@ -0,0 +1,20 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +pako@^1.0.6: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + +png-ts@^0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/png-ts/-/png-ts-0.0.3.tgz#497fa90f13b9a2cdcd5a457cb1c28ab3c68ec145" + integrity sha512-Qwn3yMfbrbaN86QjrDAqD1UVJc4AV4hvBCx5Dv9libLd6D20xKtgOFs/UcvD0nnjxWlgS12kEVWCDFd9ZtwB+g== + dependencies: + pako "^1.0.6" + +sixel@^0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/sixel/-/sixel-0.15.0.tgz#621e4a4ca50ad1c8d2fccdf93317bf25370b0d3a" + integrity sha512-osNcrv21tyW1IwYM36zugnW5zD2Pm/4N04jofhjq4XBcseVGLpDXJIqkXz0DFuGW1UtLYjkdcojJpvtLtw3CaA== diff --git a/demo/client.ts b/demo/client.ts index cd0657a0b6..57b14942e3 100644 --- a/demo/client.ts +++ b/demo/client.ts @@ -15,6 +15,7 @@ import { SearchAddon, ISearchOptions } from '../addons/xterm-addon-search/out/Se import { SerializeAddon } from '../addons/xterm-addon-serialize/out/SerializeAddon'; import { WebLinksAddon } from '../addons/xterm-addon-web-links/out/WebLinksAddon'; import { WebglAddon } from '../addons/xterm-addon-webgl/out/WebglAddon'; +import { ImageAddon, IImageAddonOptions } from '../addons/xterm-addon-image/out/ImageAddon'; import { Unicode11Addon } from '../addons/xterm-addon-unicode11/out/Unicode11Addon'; import { LigaturesAddon } from '../addons/xterm-addon-ligatures/out/LigaturesAddon'; @@ -38,6 +39,7 @@ export interface IWindowWithTerminal extends Window { Terminal?: typeof TerminalType; AttachAddon?: typeof AttachAddon; FitAddon?: typeof FitAddon; + ImageAddon?: typeof ImageAddon; SearchAddon?: typeof SearchAddon; SerializeAddon?: typeof SerializeAddon; WebLinksAddon?: typeof WebLinksAddon; @@ -53,7 +55,7 @@ let socketURL; let socket; let pid; -type AddonType = 'attach' | 'fit' | 'search' | 'serialize' | 'unicode11' | 'web-links' | 'webgl' | 'ligatures'; +type AddonType = 'attach' | 'fit' | 'image' | 'ligatures' | 'search' | 'serialize' | 'unicode11' | 'web-links' | 'webgl'; interface IDemoAddon { name: T; @@ -61,41 +63,46 @@ interface IDemoAddon { ctor: T extends 'attach' ? typeof AttachAddon : T extends 'fit' ? typeof FitAddon : + T extends 'image' ? typeof ImageAddon : T extends 'search' ? typeof SearchAddon : T extends 'serialize' ? typeof SerializeAddon : T extends 'web-links' ? typeof WebLinksAddon : T extends 'unicode11' ? typeof Unicode11Addon : T extends 'ligatures' ? typeof LigaturesAddon : typeof WebglAddon; - instance?: + instance?: T extends 'attach' ? AttachAddon : T extends 'fit' ? FitAddon : + T extends 'image' ? ImageAddon : T extends 'search' ? SearchAddon : T extends 'serialize' ? SerializeAddon : T extends 'web-links' ? WebLinksAddon : T extends 'webgl' ? WebglAddon : - T extends 'unicode11' ? typeof Unicode11Addon : + T extends 'unicode11' ? Unicode11Addon : T extends 'ligatures' ? typeof LigaturesAddon : never; } +const IMAGE_WORKER_PATH = '/workers/xterm-addon-image-worker.js'; + const addons: { [T in AddonType]: IDemoAddon} = { attach: { name: 'attach', ctor: AttachAddon, canChange: false }, fit: { name: 'fit', ctor: FitAddon, canChange: false }, + image: { name: 'image', ctor: ImageAddon, canChange: true }, + ligatures: { name: 'ligatures', ctor: LigaturesAddon, canChange: true }, search: { name: 'search', ctor: SearchAddon, canChange: true }, serialize: { name: 'serialize', ctor: SerializeAddon, canChange: true }, 'web-links': { name: 'web-links', ctor: WebLinksAddon, canChange: true }, webgl: { name: 'webgl', ctor: WebglAddon, canChange: true }, - unicode11: { name: 'unicode11', ctor: Unicode11Addon, canChange: true }, - ligatures: { name: 'ligatures', ctor: LigaturesAddon, canChange: true } + unicode11: { name: 'unicode11', ctor: Unicode11Addon, canChange: true } }; const terminalContainer = document.getElementById('terminal-container'); const actionElements = { - findNext: document.querySelector('#find-next'), - findPrevious: document.querySelector('#find-previous') + findNext: document.querySelector('#find-next'), + findPrevious: document.querySelector('#find-previous') }; -const paddingElement = document.getElementById('padding'); +const paddingElement = document.querySelector('#padding'); function setPadding(): void { term.element.style.padding = parseInt(paddingElement.value, 10).toString() + 'px'; @@ -104,9 +111,9 @@ function setPadding(): void { function getSearchOptions(e: KeyboardEvent): ISearchOptions { return { - regex: (document.getElementById('regex') as HTMLInputElement).checked, - wholeWord: (document.getElementById('whole-word') as HTMLInputElement).checked, - caseSensitive: (document.getElementById('case-sensitive') as HTMLInputElement).checked, + regex: document.querySelector('#regex').checked, + wholeWord: document.querySelector('#whole-word').checked, + caseSensitive: document.querySelector('#case-sensitive').checked, incremental: e.key !== `Enter` }; } @@ -120,6 +127,7 @@ const disposeRecreateButtonHandler = () => { socket = null; addons.attach.instance = undefined; addons.fit.instance = undefined; + addons.image.instance = undefined; addons.search.instance = undefined; addons.serialize.instance = undefined; addons.unicode11.instance = undefined; @@ -137,6 +145,7 @@ if (document.location.pathname === '/test') { window.Terminal = Terminal; window.AttachAddon = AttachAddon; window.FitAddon = FitAddon; + window.ImageAddon = ImageAddon; window.SearchAddon = SearchAddon; window.SerializeAddon = SerializeAddon; window.Unicode11Addon = Unicode11Addon; @@ -148,6 +157,7 @@ if (document.location.pathname === '/test') { document.getElementById('dispose').addEventListener('click', disposeRecreateButtonHandler); document.getElementById('serialize').addEventListener('click', serializeButtonHandler); document.getElementById('custom-glyph').addEventListener('click', writeCustomGlyphHandler); + initImageAddonExposed(); document.getElementById('load-test').addEventListener('click', loadTest); } @@ -169,12 +179,14 @@ function createTerminal(): void { addons.serialize.instance = new SerializeAddon(); addons.fit.instance = new FitAddon(); addons.unicode11.instance = new Unicode11Addon(); + addons.image.instance = new ImageAddon(IMAGE_WORKER_PATH); // TODO: Remove arguments when link provider API is the default addons['web-links'].instance = new WebLinksAddon(undefined, undefined, true); typedTerm.loadAddon(addons.fit.instance); typedTerm.loadAddon(addons.search.instance); typedTerm.loadAddon(addons.serialize.instance); typedTerm.loadAddon(addons.unicode11.instance); + typedTerm.loadAddon(addons.image.instance); typedTerm.loadAddon(addons['web-links'].instance); window.term = term; // Expose `term` to window for debugging purposes @@ -209,8 +221,8 @@ function createTerminal(): void { setTimeout(() => { initOptions(term); // TODO: Clean this up, opt-cols/rows doesn't exist anymore - (document.getElementById(`opt-cols`)).value = term.cols; - (document.getElementById(`opt-rows`)).value = term.rows; + document.querySelector('#opt-cols').valueAsNumber = term.cols; + document.querySelector('#opt-rows').valueAsNumber = term.rows; paddingElement.value = '0'; // Set terminal size again to set the specific dimensions on the demo @@ -335,14 +347,14 @@ function initOptions(term: TerminalType): void { // Attach listeners booleanOptions.forEach(o => { - const input = document.getElementById(`opt-${o}`); + const input = document.querySelector(`#opt-${o}`); addDomListener(input, 'change', () => { console.log('change', o, input.checked); term.options[o] = input.checked; }); }); numberOptions.forEach(o => { - const input = document.getElementById(`opt-${o}`); + const input = document.querySelector(`#opt-${o}`); addDomListener(input, 'change', () => { console.log('change', o, input.value); if (o === 'cols' || o === 'rows') { @@ -362,7 +374,7 @@ function initOptions(term: TerminalType): void { }); }); Object.keys(stringOptions).forEach(o => { - const input = document.getElementById(`opt-${o}`); + const input = document.querySelector(`#opt-${o}`); addDomListener(input, 'change', () => { console.log('change', o, input.value); term.options[o] = input.value; @@ -384,6 +396,19 @@ function initAddons(term: TerminalType): void { term.unicode.activeVersion = '11'; } addDomListener(checkbox, 'change', () => { + if (name === 'image') { + if (checkbox.checked) { + const ctorOptionsJson = document.querySelector('#image-options').value; + addon.instance = ctorOptionsJson + ? new addon.ctor(IMAGE_WORKER_PATH, JSON.parse(ctorOptionsJson)) + : new addon.ctor(IMAGE_WORKER_PATH); + term.loadAddon(addon.instance); + } else { + addon.instance!.dispose(); + addon.instance = undefined; + } + return; + } if (checkbox.checked) { addon.instance = new addon.ctor(); term.loadAddon(addon.instance); @@ -427,8 +452,8 @@ function addDomListener(element: HTMLElement, type: string, handler: (...args: a } function updateTerminalSize(): void { - const cols = parseInt((document.getElementById(`opt-cols`)).value, 10); - const rows = parseInt((document.getElementById(`opt-rows`)).value, 10); + const cols = document.querySelector('#opt-cols').valueAsNumber; + const rows = document.querySelector('#opt-rows').valueAsNumber; const width = (cols * term._core._renderService.dimensions.actualCellWidth + term._core.viewport.scrollBarWidth).toString() + 'px'; const height = (rows * term._core._renderService.dimensions.actualCellHeight).toString() + 'px'; terminalContainer.style.width = width; @@ -440,14 +465,13 @@ function serializeButtonHandler(): void { const output = addons.serialize.instance.serialize(); const outputString = JSON.stringify(output); - document.getElementById('serialize-output').innerText = outputString; - if ((document.getElementById('write-to-terminal') as HTMLInputElement).checked) { + document.querySelector('#serialize-output').innerText = outputString; + if (document.querySelector('#write-to-terminal').checked) { term.reset(); term.write(output); } } - function writeCustomGlyphHandler() { term.write('\n\r'); term.write('\n\r'); @@ -493,6 +517,56 @@ function writeCustomGlyphHandler() { window.scrollTo(0, 0); } +function initImageAddonExposed(): void { + const DEFAULT_OPTIONS: IImageAddonOptions = (addons.image.instance as any)._defaultOpts; + const limitStorageElement = document.querySelector('#image-storagelimit'); + limitStorageElement.valueAsNumber = addons.image.instance.storageLimit; + addDomListener(limitStorageElement, 'change', () => { + try { + addons.image.instance.storageLimit = limitStorageElement.valueAsNumber; + limitStorageElement.valueAsNumber = addons.image.instance.storageLimit; + console.log('changed storageLimit to', addons.image.instance.storageLimit); + } catch (e) { + limitStorageElement.valueAsNumber = addons.image.instance.storageLimit; + console.log('storageLimit at', addons.image.instance.storageLimit); + throw e; + } + }); + const showPlaceholderElement = document.querySelector('#image-showplaceholder'); + showPlaceholderElement.checked = addons.image.instance.showPlaceholder; + addDomListener(showPlaceholderElement, 'change', () => { + addons.image.instance.showPlaceholder = showPlaceholderElement.checked; + }); + const ctorOptionsElement = document.querySelector('#image-options'); + ctorOptionsElement.value = JSON.stringify(DEFAULT_OPTIONS, null, 2); + + const sixel_demo = (url: string) => () => fetch(url) + .then(resp => resp.arrayBuffer()) + .then(buffer => { + term.write('\r\n'); + term.write(new Uint8Array(buffer)) + }); + + document.getElementById('image-demo1').addEventListener('click', + sixel_demo('https://raw.githubusercontent.com/saitoha/libsixel/master/images/snake.six')); + document.getElementById('image-demo2').addEventListener('click', + sixel_demo('https://raw.githubusercontent.com/jerch/node-sixel/master/testfiles/biplane.six')); + + // demo for image retrieval API + term.element.addEventListener('click', (ev: MouseEvent) => { + if (!ev.ctrlKey || !addons.image.instance) return; + const pos = term._core._mouseService!.getCoords(ev, term._core.screenElement!, term.cols, term.rows); + const x = pos[0] - 1; + const y = pos[1] - 1; + const canvas = ev.shiftKey + // ctrl+shift+click: get single tile + ? addons.image.instance.extractTileAtBufferCell(x, term.buffer.active.viewportY + y) + // ctrl+click: get original image + : addons.image.instance.getImageAtBufferCell(x, term.buffer.active.viewportY + y); + canvas?.toBlob(data => window.open(URL.createObjectURL(data), '_blank')); + }); +} + function loadTest() { const isWebglEnabled = !!addons.webgl.instance; const testData = []; diff --git a/demo/index.html b/demo/index.html index 9c86783b52..85abf034a2 100644 --- a/demo/index.html +++ b/demo/index.html @@ -50,6 +50,21 @@

SerializeAddon


           
+          
Image Addon

+          

+            image addon settings
+            

+              

+              
+              


+              
+            

+          

+          
+          
         
         

           
Style

diff --git a/demo/server.js b/demo/server.js
index c0d5e1f67d..417fb14b4a 100644
--- a/demo/server.js
+++ b/demo/server.js
@@ -38,6 +38,7 @@ function startServer() {
 
   app.use('/dist', express.static(__dirname + '/dist'));
   app.use('/src', express.static(__dirname + '/src'));
+  app.use('/workers', express.static(__dirname + '/workers'));
 
   app.post('/terminals', (req, res) => {
     const env = Object.assign({}, process.env);
diff --git a/demo/start.js b/demo/start.js
index b40b9bc3df..e77e5a50c5 100644
--- a/demo/start.js
+++ b/demo/start.js
@@ -66,7 +66,55 @@ const clientConfig = {
   mode: 'development',
   watch: true
 };
-const compiler = webpack(clientConfig);
+
+/**
+ * Blueprint to bundle addon workers.
+ * Expects entry under `./addons/xterm-addon-${addonName}/src-worker/main.ts`.
+ */
+function generateAddonWorker(addonName) {
+  return {
+    entry: `./addons/xterm-addon-${addonName}/src-worker/main.ts`,
+    devtool: 'inline-source-map',
+    module: {
+      rules: [
+        {
+          test: /\.tsx?$/,
+          use: 'ts-loader',
+          exclude: /node_modules/
+        },
+        {
+          test: /\.js$/,
+          use: ["source-map-loader"],
+          enforce: "pre",
+          exclude: /node_modules/
+        }
+      ]
+    },
+    resolve: {
+      modules: [
+        'node_modules',
+        path.resolve(__dirname, '..'),
+        path.resolve(__dirname, '../addons')
+      ],
+      extensions: [ '.tsx', '.ts', '.js' ],
+      alias: {
+        common: path.resolve('./out/common'),
+        browser: path.resolve('./out/browser')
+      }
+    },
+    output: {
+      filename: `xterm-addon-${addonName}-worker.js`,
+      path: path.resolve(__dirname, 'workers')
+    },
+    mode: 'development',
+    watch: true
+  }
+}
+
+const compiler = webpack([
+  clientConfig,
+  generateAddonWorker('image')
+]);
 
 compiler.watch({
   // Example watchOptions
diff --git a/demo/tsconfig.json b/demo/tsconfig.json
index 4e1b503593..018ddb42e5 100644
--- a/demo/tsconfig.json
+++ b/demo/tsconfig.json
@@ -8,6 +8,7 @@
     "paths": {
       "xterm-addon-attach": ["../addons/xterm-addon-attach"],
       "xterm-addon-fit": ["../addons/xterm-addon-fit"],
+      "xterm-addon-image": ["../addons/xterm-addon-image"],
       "xterm-addon-search": ["../addons/xterm-addon-search"],
       "xterm-addon-serialize": ["../addons/xterm-addon-serialize"],
       "xterm-addon-web-links": ["../addons/xterm-addon-web-links"],
diff --git a/package.json b/package.json
index 10aa9e6cab..86d0b60010 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
     "benchmark": "NODE_PATH=./out xterm-benchmark -r 5 -c test/benchmark/benchmark.json",
     "benchmark-baseline": "NODE_PATH=./out xterm-benchmark -r 5 -c test/benchmark/benchmark.json --baseline out-test/benchmark/test/benchmark/*benchmark.js",
     "benchmark-eval": "NODE_PATH=./out xterm-benchmark -r 5 -c test/benchmark/benchmark.json --eval out-test/benchmark/test/benchmark/*benchmark.js",
-    "clean": "rm -rf lib out addons/*/lib addons/*/out",
+    "clean": "rm -rf lib out addons/*/lib addons/*/out addons/*/out-worker",
     "vtfeatures": "node bin/extract_vtfeatures.js src/**/*.ts src/*.ts"
   },
   "devDependencies": {
diff --git a/src/browser/Terminal.ts b/src/browser/Terminal.ts
index 23122f3fa1..cd0eb21ba4 100644
--- a/src/browser/Terminal.ts
+++ b/src/browser/Terminal.ts
@@ -1310,13 +1310,13 @@ export class Terminal extends CoreTerminal implements ITerminal {
 
     switch (type) {
       case WindowsOptionsReportType.GET_WIN_SIZE_PIXELS:
-        const canvasWidth = this._renderService.dimensions.scaledCanvasWidth.toFixed(0);
-        const canvasHeight = this._renderService.dimensions.scaledCanvasHeight.toFixed(0);
+        const canvasWidth = this._renderService.dimensions.canvasWidth.toFixed(0);
+        const canvasHeight = this._renderService.dimensions.canvasHeight.toFixed(0);
         this.coreService.triggerDataEvent(`${C0.ESC}[4;${canvasHeight};${canvasWidth}t`);
         break;
       case WindowsOptionsReportType.GET_CELL_SIZE_PIXELS:
-        const cellWidth = this._renderService.dimensions.scaledCellWidth.toFixed(0);
-        const cellHeight = this._renderService.dimensions.scaledCellHeight.toFixed(0);
+        const cellWidth = this._renderService.dimensions.actualCellWidth.toFixed(0);
+        const cellHeight = this._renderService.dimensions.actualCellHeight.toFixed(0);
         this.coreService.triggerDataEvent(`${C0.ESC}[6;${cellHeight};${cellWidth}t`);
         break;
     }
diff --git a/src/common/buffer/BufferLine.ts b/src/common/buffer/BufferLine.ts
index f0bf4fcb67..2ff3db445c 100644
--- a/src/common/buffer/BufferLine.ts
+++ b/src/common/buffer/BufferLine.ts
@@ -5,36 +5,10 @@
 
 import { CharData, IBufferLine, ICellData, IAttributeData, IExtendedAttrs } from 'common/Types';
 import { stringFromCodePoint } from 'common/input/TextDecoder';
-import { CHAR_DATA_CHAR_INDEX, CHAR_DATA_WIDTH_INDEX, CHAR_DATA_ATTR_INDEX, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE, WHITESPACE_CELL_CHAR, Content, BgFlags } from 'common/buffer/Constants';
+import { CHAR_DATA_CHAR_INDEX, CHAR_DATA_WIDTH_INDEX, CHAR_DATA_ATTR_INDEX, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE, WHITESPACE_CELL_CHAR, Content, BgFlags, Cell } from 'common/buffer/Constants';
 import { CellData } from 'common/buffer/CellData';
 import { AttributeData, ExtendedAttrs } from 'common/buffer/AttributeData';
 
-/**
- * buffer memory layout:
- *
- *   |             uint32_t             |        uint32_t         |        uint32_t         |
- *   |             `content`            |          `FG`           |          `BG`           |
- *   | wcwidth(2) comb(1) codepoint(21) | flags(8) R(8) G(8) B(8) | flags(8) R(8) G(8) B(8) |
- */
-
-
-/** typed array slots taken by one cell */
-const CELL_SIZE = 3;
-
-/**
- * Cell member indices.
- *
- * Direct access:
- *    `content = data[column * CELL_SIZE + Cell.CONTENT];`
- *    `fg = data[column * CELL_SIZE + Cell.FG];`
- *    `bg = data[column * CELL_SIZE + Cell.BG];`
- */
-const enum Cell {
-  CONTENT = 0,
-  FG = 1, // currently simply holds all known attrs
-  BG = 2  // currently unused
-}
-
 export const DEFAULT_ATTR_DATA = Object.freeze(new AttributeData());
 
 /**
@@ -59,7 +33,7 @@ export class BufferLine implements IBufferLine {
   public length: number;
 
   constructor(cols: number, fillCellData?: ICellData, public isWrapped: boolean = false) {
-    this._data = new Uint32Array(cols * CELL_SIZE);
+    this._data = new Uint32Array(cols * Cell.SIZE);
     const cell = fillCellData || CellData.fromCharData([0, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]);
     for (let i = 0; i < cols; ++i) {
       this.setCell(i, cell);
@@ -72,10 +46,10 @@ export class BufferLine implements IBufferLine {
    * @deprecated
    */
   public get(index: number): CharData {
-    const content = this._data[index * CELL_SIZE + Cell.CONTENT];
+    const content = this._data[index * Cell.SIZE + Cell.CONTENT];
     const cp = content & Content.CODEPOINT_MASK;
     return [
-      this._data[index * CELL_SIZE + Cell.FG],
+      this._data[index * Cell.SIZE + Cell.FG],
       (content & Content.IS_COMBINED_MASK)
         ? this._combined[index]
         : (cp) ? stringFromCodePoint(cp) : '',
@@ -91,12 +65,12 @@ export class BufferLine implements IBufferLine {
    * @deprecated
    */
   public set(index: number, value: CharData): void {
-    this._data[index * CELL_SIZE + Cell.FG] = value[CHAR_DATA_ATTR_INDEX];
+    this._data[index * Cell.SIZE + Cell.FG] = value[CHAR_DATA_ATTR_INDEX];
     if (value[CHAR_DATA_CHAR_INDEX].length > 1) {
       this._combined[index] = value[1];
-      this._data[index * CELL_SIZE + Cell.CONTENT] = index | Content.IS_COMBINED_MASK | (value[CHAR_DATA_WIDTH_INDEX] << Content.WIDTH_SHIFT);
+      this._data[index * Cell.SIZE + Cell.CONTENT] = index | Content.IS_COMBINED_MASK | (value[CHAR_DATA_WIDTH_INDEX] << Content.WIDTH_SHIFT);
     } else {
-      this._data[index * CELL_SIZE + Cell.CONTENT] = value[CHAR_DATA_CHAR_INDEX].charCodeAt(0) | (value[CHAR_DATA_WIDTH_INDEX] << Content.WIDTH_SHIFT);
+      this._data[index * Cell.SIZE + Cell.CONTENT] = value[CHAR_DATA_CHAR_INDEX].charCodeAt(0) | (value[CHAR_DATA_WIDTH_INDEX] << Content.WIDTH_SHIFT);
     }
   }
 
@@ -105,22 +79,22 @@ export class BufferLine implements IBufferLine {
    * use these when only one value is needed, otherwise use `loadCell`
    */
   public getWidth(index: number): number {
-    return this._data[index * CELL_SIZE + Cell.CONTENT] >> Content.WIDTH_SHIFT;
+    return this._data[index * Cell.SIZE + Cell.CONTENT] >> Content.WIDTH_SHIFT;
   }
 
   /** Test whether content has width. */
   public hasWidth(index: number): number {
-    return this._data[index * CELL_SIZE + Cell.CONTENT] & Content.WIDTH_MASK;
+    return this._data[index * Cell.SIZE + Cell.CONTENT] & Content.WIDTH_MASK;
   }
 
   /** Get FG cell component. */
   public getFg(index: number): number {
-    return this._data[index * CELL_SIZE + Cell.FG];
+    return this._data[index * Cell.SIZE + Cell.FG];
   }
 
   /** Get BG cell component. */
   public getBg(index: number): number {
-    return this._data[index * CELL_SIZE + Cell.BG];
+    return this._data[index * Cell.SIZE + Cell.BG];
   }
 
   /**
@@ -129,7 +103,7 @@ export class BufferLine implements IBufferLine {
    * from real empty cells.
    * */
   public hasContent(index: number): number {
-    return this._data[index * CELL_SIZE + Cell.CONTENT] & Content.HAS_CONTENT_MASK;
+    return this._data[index * Cell.SIZE + Cell.CONTENT] & Content.HAS_CONTENT_MASK;
   }
 
   /**
@@ -138,7 +112,7 @@ export class BufferLine implements IBufferLine {
    * a single UTF32 codepoint or the last codepoint of a combined string.
    */
   public getCodePoint(index: number): number {
-    const content = this._data[index * CELL_SIZE + Cell.CONTENT];
+    const content = this._data[index * Cell.SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED_MASK) {
       return this._combined[index].charCodeAt(this._combined[index].length - 1);
     }
@@ -147,12 +121,12 @@ export class BufferLine implements IBufferLine {
 
   /** Test whether the cell contains a combined string. */
   public isCombined(index: number): number {
-    return this._data[index * CELL_SIZE + Cell.CONTENT] & Content.IS_COMBINED_MASK;
+    return this._data[index * Cell.SIZE + Cell.CONTENT] & Content.IS_COMBINED_MASK;
   }
 
   /** Returns the string content of the cell. */
   public getString(index: number): string {
-    const content = this._data[index * CELL_SIZE + Cell.CONTENT];
+    const content = this._data[index * Cell.SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED_MASK) {
       return this._combined[index];
     }
@@ -168,7 +142,7 @@ export class BufferLine implements IBufferLine {
    * to GC as it significantly reduced the amount of new objects/references needed.
    */
   public loadCell(index: number, cell: ICellData): ICellData {
-    const startIndex = index * CELL_SIZE;
+    const startIndex = index * Cell.SIZE;
     cell.content = this._data[startIndex + Cell.CONTENT];
     cell.fg = this._data[startIndex + Cell.FG];
     cell.bg = this._data[startIndex + Cell.BG];
@@ -191,9 +165,9 @@ export class BufferLine implements IBufferLine {
     if (cell.bg & BgFlags.HAS_EXTENDED) {
       this._extendedAttrs[index] = cell.extended;
     }
-    this._data[index * CELL_SIZE + Cell.CONTENT] = cell.content;
-    this._data[index * CELL_SIZE + Cell.FG] = cell.fg;
-    this._data[index * CELL_SIZE + Cell.BG] = cell.bg;
+    this._data[index * Cell.SIZE + Cell.CONTENT] = cell.content;
+    this._data[index * Cell.SIZE + Cell.FG] = cell.fg;
+    this._data[index * Cell.SIZE + Cell.BG] = cell.bg;
   }
 
   /**
@@ -205,9 +179,9 @@ export class BufferLine implements IBufferLine {
     if (bg & BgFlags.HAS_EXTENDED) {
       this._extendedAttrs[index] = eAttrs;
     }
-    this._data[index * CELL_SIZE + Cell.CONTENT] = codePoint | (width << Content.WIDTH_SHIFT);
-    this._data[index * CELL_SIZE + Cell.FG] = fg;
-    this._data[index * CELL_SIZE + Cell.BG] = bg;
+    this._data[index * Cell.SIZE + Cell.CONTENT] = codePoint | (width << Content.WIDTH_SHIFT);
+    this._data[index * Cell.SIZE + Cell.FG] = fg;
+    this._data[index * Cell.SIZE + Cell.BG] = bg;
   }
 
   /**
@@ -217,7 +191,7 @@ export class BufferLine implements IBufferLine {
    * by the previous `setDataFromCodePoint` call, we can omit it here.
    */
   public addCodepointToCell(index: number, codePoint: number): void {
-    let content = this._data[index * CELL_SIZE + Cell.CONTENT];
+    let content = this._data[index * Cell.SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED_MASK) {
       // we already have a combined string, simply add
       this._combined[index] += stringFromCodePoint(codePoint);
@@ -234,7 +208,7 @@ export class BufferLine implements IBufferLine {
         // simply set the data in the cell buffer with a width of 1
         content = codePoint | (1 << Content.WIDTH_SHIFT);
       }
-      this._data[index * CELL_SIZE + Cell.CONTENT] = content;
+      this._data[index * Cell.SIZE + Cell.CONTENT] = content;
     }
   }
 
@@ -313,10 +287,10 @@ export class BufferLine implements IBufferLine {
       return;
     }
     if (cols > this.length) {
-      const data = new Uint32Array(cols * CELL_SIZE);
+      const data = new Uint32Array(cols * Cell.SIZE);
       if (this.length) {
-        if (cols * CELL_SIZE < this._data.length) {
-          data.set(this._data.subarray(0, cols * CELL_SIZE));
+        if (cols * Cell.SIZE < this._data.length) {
+          data.set(this._data.subarray(0, cols * Cell.SIZE));
         } else {
           data.set(this._data);
         }
@@ -327,8 +301,8 @@ export class BufferLine implements IBufferLine {
       }
     } else {
       if (cols) {
-        const data = new Uint32Array(cols * CELL_SIZE);
-        data.set(this._data.subarray(0, cols * CELL_SIZE));
+        const data = new Uint32Array(cols * Cell.SIZE);
+        data.set(this._data.subarray(0, cols * Cell.SIZE));
         this._data = data;
         // Remove any cut off combined data, FIXME: repeat this for extended attrs
         const keys = Object.keys(this._combined);
@@ -392,8 +366,8 @@ export class BufferLine implements IBufferLine {
 
   public getTrimmedLength(): number {
     for (let i = this.length - 1; i >= 0; --i) {
-      if ((this._data[i * CELL_SIZE + Cell.CONTENT] & Content.HAS_CONTENT_MASK)) {
-        return i + (this._data[i * CELL_SIZE + Cell.CONTENT] >> Content.WIDTH_SHIFT);
+      if ((this._data[i * Cell.SIZE + Cell.CONTENT] & Content.HAS_CONTENT_MASK)) {
+        return i + (this._data[i * Cell.SIZE + Cell.CONTENT] >> Content.WIDTH_SHIFT);
       }
     }
     return 0;
@@ -403,14 +377,14 @@ export class BufferLine implements IBufferLine {
     const srcData = src._data;
     if (applyInReverse) {
       for (let cell = length - 1; cell >= 0; cell--) {
-        for (let i = 0; i < CELL_SIZE; i++) {
-          this._data[(destCol + cell) * CELL_SIZE + i] = srcData[(srcCol + cell) * CELL_SIZE + i];
+        for (let i = 0; i < Cell.SIZE; i++) {
+          this._data[(destCol + cell) * Cell.SIZE + i] = srcData[(srcCol + cell) * Cell.SIZE + i];
         }
       }
     } else {
       for (let cell = 0; cell < length; cell++) {
-        for (let i = 0; i < CELL_SIZE; i++) {
-          this._data[(destCol + cell) * CELL_SIZE + i] = srcData[(srcCol + cell) * CELL_SIZE + i];
+        for (let i = 0; i < Cell.SIZE; i++) {
+          this._data[(destCol + cell) * Cell.SIZE + i] = srcData[(srcCol + cell) * Cell.SIZE + i];
         }
       }
     }
@@ -431,7 +405,7 @@ export class BufferLine implements IBufferLine {
     }
     let result = '';
     while (startCol < endCol) {
-      const content = this._data[startCol * CELL_SIZE + Cell.CONTENT];
+      const content = this._data[startCol * Cell.SIZE + Cell.CONTENT];
       const cp = content & Content.CODEPOINT_MASK;
       result += (content & Content.IS_COMBINED_MASK) ? this._combined[startCol] : (cp) ? stringFromCodePoint(cp) : WHITESPACE_CELL_CHAR;
       startCol += (content >> Content.WIDTH_SHIFT) || 1; // always advance by 1
diff --git a/src/common/buffer/Constants.ts b/src/common/buffer/Constants.ts
index a2c1b884c8..ee1f200516 100644
--- a/src/common/buffer/Constants.ts
+++ b/src/common/buffer/Constants.ts
@@ -29,6 +29,32 @@ export const WHITESPACE_CELL_CHAR = ' ';
 export const WHITESPACE_CELL_WIDTH = 1;
 export const WHITESPACE_CELL_CODE = 32;
 
+
+/**
+ * buffer memory layout:
+ *
+ *   |             uint32_t             |        uint32_t         |        uint32_t         |
+ *   |             `content`            |          `FG`           |          `BG`           |
+ *   | wcwidth(2) comb(1) codepoint(21) | flags(8) R(8) G(8) B(8) | flags(8) R(8) G(8) B(8) |
+ */
+
+
+/**
+ * Cell member indices and size.
+ *
+ * Direct access:
+ *    `content = data[column * Cell.SIZE + Cell.CONTENT];`
+ *    `fg = data[column * Cell.SIZE + Cell.FG];`
+ *    `bg = data[column * Cell.SIZE + Cell.BG];`
+ */
+export const enum Cell {
+  CONTENT = 0,  // codepoint and wcwidth information (enum Content)
+  FG = 1,       // foreground color in lower 3 bytes (rgb), attrs in 4th byte (enum FgFlags)
+  BG = 2,       // background color in lower 3 bytes (rgb), attrs in 4th byte (enum BgFlags)
+  SIZE = 3      // size of single cell on buffer array
+}
+
+
 /**
  * Bitmasks for accessing data in `content`.
  */
diff --git a/tsconfig.all.json b/tsconfig.all.json
index 6fa02446a2..74020069d1 100644
--- a/tsconfig.all.json
+++ b/tsconfig.all.json
@@ -8,6 +8,7 @@
     { "path": "./test/benchmark" },
     { "path": "./addons/xterm-addon-attach" },
     { "path": "./addons/xterm-addon-fit" },
+    { "path": "./addons/xterm-addon-image" },
     { "path": "./addons/xterm-addon-ligatures" },
     { "path": "./addons/xterm-addon-search" },
     { "path": "./addons/xterm-addon-serialize" },