From 314e2b8788bd34f6f97342359cf1d4368e60b576 Mon Sep 17 00:00:00 2001 From: Michael B Date: Wed, 14 Jun 2023 14:33:55 +0200 Subject: [PATCH 1/3] options: add "-o base=2" Add experimental support for base 2 (bits). For example: $ fq -o base=2 '.frames[0].header' example.mp3 Also, add various comments to make sense of the dump.go code. --- internal/binwriter/binwriter.go | 57 ++++++++++++++++ pkg/interp/dump.go | 111 +++++++++++++++++++++++++------- pkg/interp/interp.go | 1 + pkg/interp/options.jq | 24 ++++++- 4 files changed, 167 insertions(+), 26 deletions(-) create mode 100644 internal/binwriter/binwriter.go diff --git a/internal/binwriter/binwriter.go b/internal/binwriter/binwriter.go new file mode 100644 index 000000000..583441c63 --- /dev/null +++ b/internal/binwriter/binwriter.go @@ -0,0 +1,57 @@ +package binwriter + +import ( + "io" + + "github.com/wader/fq/pkg/bitio" +) + +type Writer struct { + w io.Writer + width int + startLineOffset int + offset int + fn func(b byte) string +} + +func New(w io.Writer, width int, startLineOffset int, fn func(b byte) string) *Writer { + return &Writer{ + w: w, + width: width, + startLineOffset: startLineOffset, + offset: 0, + fn: fn, + } +} + +func (w *Writer) WriteBits(p []byte, nBits int64) (n int64, err error) { + for w.offset < w.startLineOffset { + b := []byte(" ") + if w.offset%w.width == w.width-1 { + b = []byte(" \n") + } + if _, err := w.w.Write(b); err != nil { + return 0, err + } + w.offset++ + } + + for i := int64(0); i < nBits; i++ { + var v byte + if bitio.Read64(p, i, 1) == 1 { + v = 1 + } + if _, err := w.w.Write([]byte(w.fn(v))); err != nil { + return 0, err + } + + w.offset++ + if w.offset%w.width == 0 { + if _, err := w.w.Write([]byte("\n")); err != nil { + return 0, err + } + } + } + + return nBits, nil +} diff --git a/pkg/interp/dump.go b/pkg/interp/dump.go index ef78167c3..efa2e4d5f 100644 --- a/pkg/interp/dump.go +++ b/pkg/interp/dump.go @@ -9,6 +9,7 @@ import ( "github.com/wader/fq/internal/ansi" "github.com/wader/fq/internal/asciiwriter" + "github.com/wader/fq/internal/binwriter" "github.com/wader/fq/internal/bitioex" "github.com/wader/fq/internal/columnwriter" "github.com/wader/fq/internal/hexpairwriter" @@ -72,6 +73,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD cprint := func(c int, a ...any) { fmt.Fprint(cw.Columns[c], a...) } + // cfmt: column i fmt.fprintf cfmt := func(c int, format string, a ...any) { fmt.Fprintf(cw.Columns[c], format, a...) } @@ -118,7 +120,9 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD // show address bar on root, nested root and format change if depth == 0 || v.IsRoot || v.Format != nil { + // write header: 00 01 02 03 04 cfmt(colHex, "%s", deco.DumpHeader.F(ctx.hexHeader)) + // write header: 012345 cfmt(colASCII, "%s", deco.DumpHeader.F(ctx.asciiHeader)) if willDisplayData { @@ -188,6 +192,10 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD cprint(colField, "\n") + // -------------------------------------------------- + // Error handling + // -------------------------------------------------- + if valueErr != nil { var printErrs func(depth int, err error) printErrs = func(depth int, err error) { @@ -223,24 +231,33 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD printErrs(depth, valueErr) } + // -------------------------------------------------- + // For a given field, compute various helper variables + // -------------------------------------------------- + rootBitLen, err := bitioex.Len(rootV.RootReader) if err != nil { return err } bufferLastBit := rootBitLen - 1 - startBit := innerRange.Start - stopBit := innerRange.Stop() - 1 - sizeBits := innerRange.Len - lastDisplayBit := stopBit + startBit := innerRange.Start // field's start bit index (for entire file) + stopBit := innerRange.Stop() - 1 // field's end bit index (for entire file); inclusive + sizeBits := innerRange.Len // field's bit length (1, 8, 16, 32, ...) - if opts.DisplayBytes > 0 && sizeBits > int64(opts.DisplayBytes)*8 { - lastDisplayBit = startBit + (int64(opts.DisplayBytes)*8 - 1) - if lastDisplayBit%(int64(opts.LineBytes)*8) != 0 { - lastDisplayBit += (int64(opts.LineBytes) * 8) - lastDisplayBit%(int64(opts.LineBytes)*8) - 1 + // determine lastDisplayBit: + // sometimes the field's bit length overflows the max width of a line; + // cut off the overflow in such cases. + lastDisplayBit := stopBit + displayBits := int64(opts.DisplayBytes) * 8 + lineBits := int64(opts.LineBytes) * 8 + if opts.DisplayBytes > 0 && sizeBits > displayBits { + lastDisplayBit = startBit + (displayBits - 1) + if lastDisplayBit%lineBits != 0 { + lastDisplayBit += lineBits - lastDisplayBit%lineBits - 1 } - if lastDisplayBit > stopBit || stopBit-lastDisplayBit <= int64(opts.LineBytes)*8 { + if lastDisplayBit > stopBit || stopBit-lastDisplayBit <= lineBits { lastDisplayBit = stopBit } } @@ -258,38 +275,71 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD if displaySizeBits > maxDisplaySizeBits { displaySizeBits = maxDisplaySizeBits } + if displaySizeBits > stopBit-startBit { + displaySizeBits = stopBit - startBit + 1 // TODO: -1 hmm + } startLine := startByte / int64(opts.LineBytes) startLineByteOffset := startByte % int64(opts.LineBytes) + + startLineBitOffset := startBit % int64(opts.LineBytes*8) + startLineByte := startLine * int64(opts.LineBytes) lastDisplayLine := lastDisplayByte / int64(opts.LineBytes) + // -------------------------------------------------- + // Output Data + // -------------------------------------------------- + // has length and is not compound or a collapsed struct/array (max depth) if willDisplayData { + // write address: 0x00012 (example) cfmt(colAddr, "%s%s\n", rootIndent, deco.DumpAddr.F(mathex.PadFormatInt(startLineByte, opts.Addrbase, true, addrWidth))) - vBR, err := bitioex.Range(rootV.RootReader, startByte*8, displaySizeBits) + vBR1, err := bitioex.Range(rootV.RootReader, startByte*8, displaySizeBits) + if err != nil { + return err + } + + vBR2, err := bitioex.Range(rootV.RootReader, startBit, displaySizeBits) if err != nil { return err } addrLines := lastDisplayLine - startLine + 1 hexpairFn := func(b byte) string { return deco.ByteColor(b).Wrap(hexpairwriter.Pair(b)) } + binFn := func(b byte) string { return deco.ByteColor(b).Wrap(string("01"[int(b)])) } asciiFn := func(b byte) string { return deco.ByteColor(b).Wrap(asciiwriter.SafeASCII(b)) } - hexBR, err := bitio.CloneReadSeeker(vBR) - if err != nil { - return err - } - if _, err := bitioex.CopyBitsBuffer( - hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn), - hexBR, - buf); err != nil { - return err + switch opts.Base { + case 16: + // write hex: 89 50 4e 47 0d 0a 1a 0a ... + hexBR, err := bitio.CloneReadSeeker(vBR1) + if err != nil { + return err + } + if _, err := bitioex.CopyBitsBuffer( + hexpairwriter.New(cw.Columns[colHex], opts.LineBytes, int(startLineByteOffset), hexpairFn), + hexBR, + buf); err != nil { + return err + } + case 2: + // write bits: 100010010101000... + hexBR, err := bitio.CloneReadSeeker(vBR2) + if err != nil { + return err + } + if _, err := bitio.CopyBuffer( + binwriter.New(cw.Columns[colHex], opts.LineBytes*8, int(startLineBitOffset), binFn), + hexBR, + buf); err != nil { + return err + } } - - asciiBR, err := bitio.CloneReadSeeker(vBR) + // write ascii: .PNG.........IHDR... + asciiBR, err := bitio.CloneReadSeeker(vBR1) if err != nil { return err } @@ -364,7 +414,13 @@ func dump(v *decode.Value, w io.Writer, opts *Options) error { } addrColumnWidth := maxAddrIndentWidth - hexColumnWidth := opts.LineBytes*3 - 1 + var hexColumnWidth int + switch opts.Base { + case 16: + hexColumnWidth = opts.LineBytes*3 - 1 + case 2: + hexColumnWidth = opts.LineBytes * 8 + } asciiColumnWidth := opts.LineBytes treeColumnWidth := -1 // TODO: set with and truncate/wrap properly @@ -387,11 +443,18 @@ func dump(v *decode.Value, w io.Writer, opts *Options) error { var hexHeader string var asciiHeader string + var spaceLength int + switch opts.Base { + case 16: + spaceLength = 1 + case 2: + spaceLength = 8 - 2 // TODO: adapt for wider screens + } for i := 0; i < opts.LineBytes; i++ { s := mathex.PadFormatInt(int64(i), opts.Addrbase, false, 2) hexHeader += s - if i < opts.LineBytes-1 { - hexHeader += " " + if spaceLength > 1 || i < opts.LineBytes-1 { + hexHeader += strings.Repeat(" ", spaceLength) } asciiHeader += s[len(s)-1:] } diff --git a/pkg/interp/interp.go b/pkg/interp/interp.go index 700674ec6..1acf8a974 100644 --- a/pkg/interp/interp.go +++ b/pkg/interp/interp.go @@ -1044,6 +1044,7 @@ type Options struct { BitsFormat string LineBytes int DisplayBytes int + Base int Addrbase int Sizebase int SkipGaps bool diff --git a/pkg/interp/options.jq b/pkg/interp/options.jq index 35f14a15a..4b9ad1db0 100644 --- a/pkg/interp/options.jq +++ b/pkg/interp/options.jq @@ -78,6 +78,7 @@ def _opt_options: argdecode: "array_string_pair", argjson: "array_string_pair", array_truncate: "number", + base: "number", bits_format: "string", byte_colors: "csv_ranges_array", color: "boolean", @@ -528,12 +529,31 @@ def options($opts): + [$opts] ) | add - | ( if .width != 0 then [_intdiv(_intdiv(.width; 8); 2) * 2, 4] | max - else 16 + | ( if .width != 0 then + if .base == 2 then + # set input data bits to ~44.4% of width + .5 * (8 / (8 + 1)) * .width | floor + else + # set input data hex to ~37.5% of width + .5 * (3 / (3 + 1)) * .width | floor + end + else + 16 * 3 + end + ) as $input_data_width + | ( if .base == 2 then + # 100010010101000 + # show at least 1 byte + [_intdiv($input_data_width; 8), 1] | max + else + # 89 50 4e 47 0d 0a 1a 0a + # show an even amount of bytes; and at least 4 + [_intdiv(_intdiv($input_data_width; 3); 2) * 2, 4] | max end ) as $display_bytes # default if not set | .display_bytes |= (. // $display_bytes) | .line_bytes |= (. // $display_bytes) + | .base |= (. // 16) ); def options: options({}); From deeafe7f5a0cd616d754bc16d3e1ce30f7424303 Mon Sep 17 00:00:00 2001 From: Michael B Date: Wed, 14 Jun 2023 22:49:22 +0200 Subject: [PATCH 2/3] fix: only adjust `displaySizeBits` for base 2 --- pkg/interp/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/interp/dump.go b/pkg/interp/dump.go index efa2e4d5f..913643314 100644 --- a/pkg/interp/dump.go +++ b/pkg/interp/dump.go @@ -275,7 +275,7 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD if displaySizeBits > maxDisplaySizeBits { displaySizeBits = maxDisplaySizeBits } - if displaySizeBits > stopBit-startBit { + if opts.Base == 2 && displaySizeBits > stopBit-startBit { displaySizeBits = stopBit - startBit + 1 // TODO: -1 hmm } From 88b65631e06f9196e006f23dcad48145b566a981 Mon Sep 17 00:00:00 2001 From: Michael B Date: Thu, 15 Jun 2023 15:46:04 +0200 Subject: [PATCH 3/3] fix: move bitioex.Range; add "base" option in tests --- pkg/interp/dump.go | 10 ++++------ pkg/interp/testdata/args.fqtest | 1 + pkg/interp/testdata/options.fqtest | 1 + 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/interp/dump.go b/pkg/interp/dump.go index 913643314..941abe054 100644 --- a/pkg/interp/dump.go +++ b/pkg/interp/dump.go @@ -281,7 +281,6 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD startLine := startByte / int64(opts.LineBytes) startLineByteOffset := startByte % int64(opts.LineBytes) - startLineBitOffset := startBit % int64(opts.LineBytes*8) startLineByte := startLine * int64(opts.LineBytes) @@ -302,11 +301,6 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD return err } - vBR2, err := bitioex.Range(rootV.RootReader, startBit, displaySizeBits) - if err != nil { - return err - } - addrLines := lastDisplayLine - startLine + 1 hexpairFn := func(b byte) string { return deco.ByteColor(b).Wrap(hexpairwriter.Pair(b)) } binFn := func(b byte) string { return deco.ByteColor(b).Wrap(string("01"[int(b)])) } @@ -327,6 +321,10 @@ func dumpEx(v *decode.Value, ctx *dumpCtx, depth int, rootV *decode.Value, rootD } case 2: // write bits: 100010010101000... + vBR2, err := bitioex.Range(rootV.RootReader, startBit, displaySizeBits) + if err != nil { + return err + } hexBR, err := bitio.CloneReadSeeker(vBR2) if err != nil { return err diff --git a/pkg/interp/testdata/args.fqtest b/pkg/interp/testdata/args.fqtest index 61795a5a6..a0074ea97 100644 --- a/pkg/interp/testdata/args.fqtest +++ b/pkg/interp/testdata/args.fqtest @@ -72,6 +72,7 @@ arg [] argdecode [] argjson [] array_truncate 50 +base 16 bits_format string byte_colors 0-255=brightwhite,0=brightblack,32-126:9-13=white color false diff --git a/pkg/interp/testdata/options.fqtest b/pkg/interp/testdata/options.fqtest index 2b07b35da..1b1aa75a8 100644 --- a/pkg/interp/testdata/options.fqtest +++ b/pkg/interp/testdata/options.fqtest @@ -7,6 +7,7 @@ $ fq -n options "argdecode": [], "argjson": [], "array_truncate": 50, + "base": 16, "bits_format": "string", "byte_colors": [ {