update encoding to match decoding

uses heredocs with pipe

encode/encode.go

@@ -53,24 +53,57 @@ func (enc *Encoder) Encode(v any) (err error) {
 	return
 }
 
-// true if written as a heredoc
-func (enc *Encoder) writeHere(str string) (okay bool) {
-	if okay = len(str) > 23 && strings.ContainsRune(str, runes.Newline); okay {
-		lines := strings.FieldsFunc(str, func(q rune) bool {
-			return q == runes.Newline
-		})
-		tab := &enc.Tabs
-		tab.WriteString(`"""`)
-		tab.Softline()
-		for _, el := range lines {
-			tab.Escape(el)
-			tab.Softline()
-			tab.newLines++ // not needed if using backtick literals... hrm.
+// fix? determine quote style based on some sort of heuristic....
+func (enc *Encoder) encodeQuotes(str string) {
+	if !strings.ContainsRune(str, runes.Newline) {
+		enc.Tabs.Quote(str)
+	} else {
+		// note: using strings.FieldsFunc isnt enough
+		// by creating left and right parts; it eats trailing newlines
+		var lines []string
+		var prev int
+		for i, q := range str {
+			if q == runes.Newline {
+				lines = append(lines, str[prev:i])
+				prev = i + 1
+			}
 		}
-		tab.newLines--
+		lines = append(lines, str[prev:])
+		writeHere(&enc.Tabs, lines)
+	}
+}
+
+func writeHere(tab *TabWriter, lines []string) {
+	if len(lines) == 0 {
+		panic("heredocs should have lines")
+	}
+	tab.WriteString(`|`)
+	tab.Indent(true)
+	var escaped, emptyLine bool
+	for _, el := range lines {
+		tab.Nextline()
+		if tab.Escape(el) {
+			escaped = true
+		}
+		emptyLine = len(el) == 0
+	}
+	if !emptyLine {
+		// there was content in the final line,
+		// so the heredoc should trim the final line.
+		// if we're escaping, we have to do that with backslash.
+		if escaped {
+			tab.WriteRune('\\')
+		}
+		tab.Nextline()
+	}
+	// if we're escaping we have to write the double quotes
+	// if we're not escaping we can choose to write it if the final line was empty
+	if escaped || emptyLine {
 		tab.WriteString(`"""`)
+	} else {
+		tab.WriteString(`'''`)
 	}
-	return
+	tab.Indent(false)
 }
 
 // writes a single value to the stream wrapped by tab writer
@@ -115,11 +148,7 @@ func (enc *Encoder) WriteValue(v r.Value, wasMaps bool) (err error) {
 				}
 
 			case r.String:
-				// fix: determine wrapping based on settings?
-				// select raw strings based on the presence of escapes?
-				if str := v.String(); !enc.writeHere(str) {
-					tab.Quote(str)
-				}
+				enc.encodeQuotes(v.String())
 
 			case r.Array, r.Slice:
 				// tbd: look at tag for "want array"?
@@ -200,7 +229,9 @@ func (enc *Encoder) writeCollection(it Iterator, cmts Commenting, wasMaps, maps
 		}
 		return // early out.
 	}
-	tab.OptionalLine(wasMaps)
+	if wasMaps {
+		tab.Softline()
+	}
 	//
 	for hasNext {
 		key, val := it.GetKey(), getValue(it)

encode/encodeComments_test.go

@@ -0,0 +1,45 @@
+package encode_test
+
+import (
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"io/fs"
+	"strings"
+	"testing"
+
+	"github.com/ionous/tell/collect/orderedmap"
+	"github.com/ionous/tell/encode"
+	"github.com/ionous/tell/testdata"
+)
+
+func TestCommentEncoding(t *testing.T) {
+	filePath := "smallCatalogComments"
+	if e := func() (err error) {
+		if want, e := fs.ReadFile(testdata.Tell, filePath+".tell"); e != nil {
+			err = e
+		} else if b, e := fs.ReadFile(testdata.Json, filePath+".json"); e != nil {
+			err = e
+		} else {
+			var m orderedmap.OrderedMap
+			if e := json.Unmarshal(b, &m); e != nil {
+				err = e
+			} else if src, ok := m.Get("content"); !ok {
+				err = errors.New("missing content")
+			} else {
+				var buf strings.Builder
+				enc := encode.MakeCommentEncoder(&buf)
+				if e := enc.Encode(&src); e != nil {
+					err = e
+				} else if have, want := buf.String(), string(want); have != want {
+					err = errors.New("mismatched")
+					t.Log(want)
+					t.Log(have)
+				}
+			}
+		}
+		return
+	}(); e != nil {
+		t.Fatal(e)
+	}
+}

encode/encodeTabs.go

@@ -10,7 +10,7 @@ import (
 type TabWriter struct {
 	depth    int // requested leading spaces on each new line
 	spaces   int // trailing spaces
-	newLines int
+	newLines int // requested newlines, not written until pad()
 	Writer   io.Writer
 	xpos     int
 }
@@ -30,10 +30,9 @@ func (tab *TabWriter) Softline() {
 	tab.spaces = 0
 }
 
-func (tab *TabWriter) OptionalLine(b bool) {
-	if b {
-		tab.Softline()
-	}
+func (tab *TabWriter) Nextline() {
+	tab.newLines++
+	tab.spaces = 0
 }
 
 // inc: increases the current indent
@@ -77,10 +76,12 @@ func (tab *TabWriter) Quote(s string) {
 	tab.WriteString(str)
 }
 
-// write a non-quoted escaped string
-func (tab *TabWriter) Escape(s string) {
+// escape the contents of a string
+// returns true if there were any escapes
+func (tab *TabWriter) Escape(s string) bool {
 	str := strconv.Quote(s) // fix? strconv doesnt have a writer api
-	tab.WriteString(str[1 : len(str)-1])
+	cnt, _ := tab.WriteString(str[1 : len(str)-1])
+	return cnt > len(s)
 }
 
 func (tab *TabWriter) WriteString(s string) (int, error) {

encode/encode_test.go

@@ -2,23 +2,18 @@ package encode_test
 
 import (
 	_ "embed"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
 	"strings"
 	"testing"
 
-	"github.com/ionous/tell/collect/orderedmap"
 	"github.com/ionous/tell/encode"
-	"github.com/ionous/tell/testdata"
 )
 
-//go:embed encodedTest.tell
-var encodedTest []byte
-
+// write various kinds of values
 func TestEncoding(t *testing.T) {
-	if e := testEncoding(t,
+	// -----------------------------
+	//  #  | input  | encoded results
+	// ------------------------------
+	testEncoding(t,
 		/* 0 */ true, line(`true`),
 		/* 1 */ false, line(`false`),
 		/* 2 */ "hello", line(`"hello"`),
@@ -28,69 +23,79 @@ func TestEncoding(t *testing.T) {
 		/* 6 */ int64(4294967296), line(`4294967296`),
 		/* 7 */ 0.1, line(`0.1`),
 		/* 8 */ []any{}, line(`[]`),
-		/* 9 */ map[string]any{
-			"hello": "there",
-			"empty": []any{},
-			"value": 23,
-			"bool":  true,
-			"nil":   nil,
+		/* 9 */
+		lines("hello", "there"), chomp(`
+|
+  hello
+  there
+  '''`),
+		/* 10 */
+		lines("hello", "    indents"), chomp(`
+|
+  hello
+      indents
+  '''`),
+		/* 11 */
+		"trailing newlines\n   in heredocs\ndon't collapse\n", chomp(`
+|
+  trailing newlines
+     in heredocs
+  don't collapse
+  """`),
+		/* 12 */
+		lines(
+			`this implementation`,
+			`prefers \ escaping`), chomp(`
+|
+  this implementation
+  prefers \\ escaping\
+  """`),
+	)
+}
+
+// match "encodedTest.tell"
+func TestEncodingMap(t *testing.T) {
+	testEncoding(t,
+		map[string]any{
+			"bool":    true,
+			"empty":   []any{},
+			"hello":   "there",
+			"heredoc": lines("a string", "with several lines", "becomes a heredoc."),
 			"map": map[string]any{
 				"bool":  true,
 				"hello": "world",
 				"value": 11,
 			},
+			"nil": nil,
 			"slice": []any{
 				"5",
 				5,
 				false,
 			},
-			"heredoc": `a string
-with several lines
-becomes a heredoc.`,
+			"value": 23,
 		},
 		string(encodedTest),
-	); e != nil {
-		t.Fatal(e)
-	}
+	)
 }
 
-func TestCommentEncoding(t *testing.T) {
-	filePath := "smallCatalogComments"
-	if e := func() (err error) {
-		if want, e := fs.ReadFile(testdata.Tell, filePath+".tell"); e != nil {
-			err = e
-		} else if b, e := fs.ReadFile(testdata.Json, filePath+".json"); e != nil {
-			err = e
-		} else {
-			var m orderedmap.OrderedMap
-			if e := json.Unmarshal(b, &m); e != nil {
-				err = e
-			} else if src, ok := m.Get("content"); !ok {
-				err = errors.New("missing content")
-			} else {
-				var buf strings.Builder
-				enc := encode.MakeCommentEncoder(&buf)
-				if e := enc.Encode(&src); e != nil {
-					err = e
-				} else if have, want := buf.String(), string(want); have != want {
-					err = errors.New("mismatched")
-					t.Log(want)
-					t.Log(have)
-				}
-			}
-		}
-		return
-	}(); e != nil {
-		t.Fatal(e)
-	}
-}
+//go:embed encodedTest.tell
+var encodedTest []byte
 
 func line(s string) string {
 	return s + "\n"
 }
 
+func lines(s ...string) string {
+	return strings.Join(s, "\n")
+}
+
+// gofmt is problematic for strings ( and comments! )
+func chomp(s string) string {
+	return s[1:] + "\n"
+}
+
 // tests without comments
-func testEncoding(t *testing.T, pairs ...any) (err error) {
+func testEncoding(t *testing.T, pairs ...any) {
 	cnt := len(pairs)
 	if cnt&1 != 0 {
 		panic("mismatched tests")
@@ -100,15 +105,15 @@ func testEncoding(t *testing.T, pairs ...any) (err error) {
 	for i := 0; i < cnt; i += 2 {
 		src, expect := pairs[i], pairs[i+1].(string)
 		if e := enc.Encode(src); e != nil {
-			err = fmt.Errorf("failed to marshal test #%d because %v", i/2, e)
-			break
+			t.Errorf("failed to marshal test #%d because %v", i/2, e)
 		} else {
 			if got := buf.String(); got != expect {
 				t.Logf("have\n%s", got)
 				t.Logf("want\n%s", expect)
+				t.Logf("have\n%v", []byte(got))
+				t.Logf("want\n%v", []byte(expect))
 				//
-				err = fmt.Errorf("failed test #%d", i/2)
-				break
+				t.Errorf("failed test #%d", i/2)
 			}
 			buf.Reset()
 		}

encode/encodedTest.tell

@@ -1,13 +1,11 @@
 bool: true
 empty: []
 hello: "there"
-heredoc: """
-  a string
-
-  with several lines
-
-  becomes a heredoc.
-  """
+heredoc: |
+    a string
+    with several lines
+    becomes a heredoc.
+    '''
 map:
   bool: true
   hello: "world"

testdata/smallCatalogComments.json

@@ -21,7 +21,7 @@
         "options:": [
           "\r\n# A non-inline prefix",
           "Big",
-          "Big description for a big\nentry.\n"
+          "Big description for a big\nentry."
         ],
         "attributes:": [
           "",

testdata/smallCatalogComments.tell

@@ -17,10 +17,10 @@ catalog:
       -
         # A non-inline prefix
         "Big"
-      - """
-        Big description for a big
-        entry.
-        """
+      - |
+          Big description for a big
+          entry.
+          '''
     attributes:
       - 1
       - 2