Add postprocessor::Sequence

bindings/node/lib/bindings/decoders.d.ts

@@ -49,3 +49,9 @@ export function ctcDecoder(
   word_delimiter_token?: string,
   cleanup?: boolean
 ): Decoder;
+
+/**
+ * Instantiate a new Sequence Decoder
+ * @param [decoders] The decoders to chain
+ */
+export function sequenceDecoder(decoders: Decoder[]): Decoder;

bindings/node/lib/bindings/decoders.js

@@ -6,4 +6,5 @@ module.exports = {
   metaspaceDecoder: native.decoders_Metaspace,
   bpeDecoder: native.decoders_BPEDecoder,
   ctcDecoder: native.decoders_CTC,
+  sequenceDecoder: native.decoders_Sequence,
 };

bindings/node/lib/bindings/decoders.test.ts

@@ -1,4 +1,10 @@
-import { bpeDecoder, ctcDecoder, metaspaceDecoder, wordPieceDecoder } from "./decoders";
+import {
+  bpeDecoder,
+  ctcDecoder,
+  metaspaceDecoder,
+  sequenceDecoder,
+  wordPieceDecoder,
+} from "./decoders";
 
 describe("wordPieceDecoder", () => {
   it("accepts `undefined` as first parameter", () => {
@@ -42,3 +48,25 @@ describe("ctcDecoder", () => {
     ).toEqual("hello");
   });
 });
+
+describe("sequenceDecoder", () => {
+  it("accepts `empty list` as parameter", () => {
+    expect(sequenceDecoder([])).toBeDefined();
+  });
+  it("encodes correctly", () => {
+    expect(
+      sequenceDecoder([ctcDecoder(), metaspaceDecoder()]).decode([
+        "▁",
+        "▁",
+        "H",
+        "H",
+        "i",
+        "i",
+        "▁",
+        "y",
+        "o",
+        "u",
+      ])
+    ).toEqual("Hi you");
+  });
+});

bindings/node/native/src/decoders.rs

@@ -14,11 +14,11 @@ pub struct Decoder {
 }
 
 impl tk::Decoder for Decoder {
-    fn decode(&self, tokens: Vec<String>) -> tk::Result<String> {
+    fn decode_chain(&self, tokens: Vec<String>) -> tk::Result<Vec<String>> {
         self.decoder
             .as_ref()
             .ok_or("Uninitialized Decoder")?
-            .decode(tokens)
+            .decode_chain(tokens)
     }
 }
 
@@ -42,6 +42,7 @@ declare_types! {
                 .map_err(|e| Error(format!("{}", e)))?;
 
             Ok(cx.string(output).upcast())
+
         }
     }
 }
@@ -115,12 +116,40 @@ fn ctc_decoder(mut cx: FunctionContext) -> JsResult<JsDecoder> {
     Ok(decoder)
 }
 
+/// sequence()
+fn sequence(mut cx: FunctionContext) -> JsResult<JsDecoder> {
+    let decoders = cx.argument::<JsArray>(0)?.to_vec(&mut cx)?;
+    let mut sequence = Vec::with_capacity(decoders.len());
+
+    decoders.into_iter().try_for_each(|decoder| {
+        match decoder.downcast::<JsDecoder>().or_throw(&mut cx) {
+            Ok(decoder) => {
+                let guard = cx.lock();
+                if let Some(decoder_arc) = &decoder.borrow(&guard).decoder {
+                    let decoder: DecoderWrapper = (**decoder_arc).clone();
+                    sequence.push(decoder);
+                }
+                Ok(())
+            }
+            Err(e) => Err(e),
+        }
+    })?;
+
+    let mut pretok = JsDecoder::new::<_, JsDecoder, _>(&mut cx, vec![])?;
+    let guard = cx.lock();
+    pretok.borrow_mut(&guard).decoder = Some(Arc::new(tk::DecoderWrapper::Sequence(
+        tk::decoders::sequence::Sequence::new(sequence),
+    )));
+    Ok(pretok)
+}
+
 /// Register everything here
 pub fn register(m: &mut ModuleContext, prefix: &str) -> NeonResult<()> {
     m.export_function(&format!("{}_ByteLevel", prefix), byte_level)?;
     m.export_function(&format!("{}_WordPiece", prefix), wordpiece)?;
     m.export_function(&format!("{}_Metaspace", prefix), metaspace)?;
     m.export_function(&format!("{}_BPEDecoder", prefix), bpe_decoder)?;
     m.export_function(&format!("{}_CTC", prefix), ctc_decoder)?;
+    m.export_function(&format!("{}_Sequence", prefix), sequence)?;
     Ok(())
 }

bindings/node/native/src/pre_tokenizers.rs

@@ -31,7 +31,7 @@ impl FromJsValue for JsSplitDelimiterBehavior {
     }
 }
 
-impl<'s> From<JsSplitDelimiterBehavior> for SplitDelimiterBehavior {
+impl From<JsSplitDelimiterBehavior> for SplitDelimiterBehavior {
     fn from(v: JsSplitDelimiterBehavior) -> Self {
         v.0
     }

bindings/node/native/src/processors.rs

@@ -33,6 +33,17 @@ impl tk::PostProcessor for Processor {
             .ok_or("Uninitialized PostProcessor")?
             .process(encoding, pair_encoding, add_special_tokens)
     }
+
+    fn process_chain(
+        &self,
+        encodings: Vec<Encoding>,
+        add_special_tokens: bool,
+    ) -> tk::Result<Vec<Encoding>> {
+        self.processor
+            .as_ref()
+            .ok_or("Uninitialized PostProcessor")?
+            .process_chain(encodings, add_special_tokens)
+    }
 }
 
 declare_types! {