Avoid the use of Cow in token

mitsuhiko · Nov 19, 2022 · 6e60023 · 6e60023
1 parent 8b13553
commit 6e60023
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 12 deletions.
diff --git a/minijinja/src/compiler/lexer.rs b/minijinja/src/compiler/lexer.rs
@@ -1,5 +1,3 @@
-use std::borrow::Cow;
-
 use crate::compiler::tokens::{Span, Token};
 use crate::error::{Error, ErrorKind};
 use crate::utils::{memchr, memstr, unescape};
@@ -223,17 +221,14 @@ impl<'s> TokenizerState<'s> {
         let s = self.advance(str_len + 2);
         Ok(if has_escapes {
             (
-                Token::Str(Cow::Owned(match unescape(&s[1..s.len() - 1]) {
+                Token::String(match unescape(&s[1..s.len() - 1]) {
                     Ok(unescaped) => unescaped,
                     Err(err) => return Err(err),
-                })),
+                }),
                 self.span(old_loc),
             )
         } else {
-            (
-                Token::Str(Cow::Borrowed(&s[1..s.len() - 1])),
-                self.span(old_loc),
-            )
+            (Token::Str(&s[1..s.len() - 1]), self.span(old_loc))
         })
     }
 }

diff --git a/minijinja/src/compiler/parser.rs b/minijinja/src/compiler/parser.rs
@@ -526,6 +526,7 @@ impl<'a> Parser<'a> {
             Token::Ident("none" | "None") => Ok(const_val!(())),
             Token::Ident(name) => Ok(ast::Expr::Var(Spanned::new(ast::Var { id: name }, span))),
             Token::Str(val) => Ok(const_val!(val)),
+            Token::String(val) => Ok(const_val!(val)),
             Token::Int(val) => Ok(const_val!(val)),
             Token::Float(val) => Ok(const_val!(val)),
             Token::ParenOpen => self.parse_tuple_or_expression(span),

diff --git a/minijinja/src/compiler/tokens.rs b/minijinja/src/compiler/tokens.rs
@@ -1,4 +1,3 @@
-use std::borrow::Cow;
 use std::fmt;
 
 /// Represents a token in the stream.
@@ -16,8 +15,10 @@ pub enum Token<'a> {
     BlockEnd(bool),
     /// An identifier.
     Ident(&'a str),
-    /// A string.
-    Str(Cow<'a, str>),
+    /// A borrowed string.
+    Str(&'a str),
+    /// An allocated string.
+    String(String),
     /// An integer (limited to i64)
     Int(i64),
     /// A float
@@ -85,7 +86,7 @@ impl<'a> fmt::Display for Token<'a> {
             Token::BlockStart(_) => write!(f, "start of block"),
             Token::BlockEnd(_) => write!(f, "end of block"),
             Token::Ident(_) => write!(f, "identifier"),
-            Token::Str(_) => write!(f, "string"),
+            Token::Str(_) | Token::String(_) => write!(f, "string"),
             Token::Int(_) => write!(f, "integer"),
             Token::Float(_) => write!(f, "float"),
             Token::Plus => write!(f, "`+`"),