Add SplitDelimiterBehavior to Punctuation constructor

Resolves: #642

tokenizers/src/pre_tokenizers/punctuation.rs

@@ -1,17 +1,33 @@
+use serde::{Deserialize, Serialize};
+
 use crate::tokenizer::{PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior};
 use unicode_categories::UnicodeCategories;
 
 fn is_punc(x: char) -> bool {
     char::is_ascii_punctuation(&x) || x.is_punctuation()
 }
 
-#[derive(Copy, Clone, Debug)]
-pub struct Punctuation;
-impl_serde_unit_struct!(PunctuationVisitor, Punctuation);
+#[derive(Serialize, Deserialize, Copy, Clone, Debug)]
+#[serde(tag = "type")]
+pub struct Punctuation {
+    behavior: SplitDelimiterBehavior,
+}
+
+impl Punctuation {
+    pub fn new(behavior: SplitDelimiterBehavior) -> Self {
+        Self { behavior }
+    }
+}
+
+impl Default for Punctuation {
+    fn default() -> Self {
+        Self::new(SplitDelimiterBehavior::Isolated)
+    }
+}
 
 impl PreTokenizer for Punctuation {
     fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
-        pretokenized.split(|_, s| s.split(is_punc, SplitDelimiterBehavior::Isolated))
+        pretokenized.split(|_, s| s.split(is_punc, self.behavior))
     }
 }
 
@@ -22,7 +38,7 @@ mod tests {
 
     #[test]
     fn punctuation_basic() {
-        let pretok = Punctuation;
+        let pretok = Punctuation::default();
         let mut pretokenized: PreTokenizedString = "Hey friend!     How are you?!?".into();
         pretok.pre_tokenize(&mut pretokenized).unwrap();
         assert_eq!(

tokenizers/src/pre_tokenizers/sequence.rs

@@ -33,7 +33,7 @@ mod tests {
     fn sequence_basic() {
         let pretokenizers = vec![
             PreTokenizerWrapper::WhitespaceSplit(WhitespaceSplit),
-            PreTokenizerWrapper::Punctuation(Punctuation),
+            PreTokenizerWrapper::Punctuation(Punctuation::default()),
         ];
         let pretok = Sequence::new(pretokenizers);
         let mut pretokenized: PreTokenizedString = "Hey friend!     How are you?!?".into();