diff --git a/src/encoder.ts b/src/encoder.ts index 7fe99ee..ce3e202 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -126,7 +126,7 @@ function bpe(token: string, cache: Map = new Map()): string { return result } -function* encodeGenerator( +export function* encodeGenerator( text: string, cache: Map = new Map(), ): Generator {