Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Completely support full-width characters in differential rendering #654

Merged
merged 2 commits into from
Apr 29, 2024
Merged

Completely support full-width characters in differential rendering #654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eae3e4c
Add a cut variation of Reline::Unicode.take_range method take_mbchar_…
tompng Dec 31, 2022
a7c4064
Consider fullwidth take_range in differential rendering
tompng Nov 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions lib/reline/line_editor.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,8 +414,13 @@ def render_line_differential(old_items, new_items)
@output.write "#{Reline::IOGate::RESET_COLOR}#{' ' * width}"
else
x, w, content = new_items[level]
content = Reline::Unicode.take_range(content, base_x - x, width) unless x == base_x && w == width
Reline::IOGate.move_cursor_column base_x
cover_begin = base_x != 0 && new_levels[base_x - 1] == level
cover_end = new_levels[base_x + width] == level
pos = 0
unless x == base_x && w == width
content, pos = Reline::Unicode.take_mbchar_range(content, base_x - x, width, cover_begin: cover_begin, cover_end: cover_end, padding: true)
end
Reline::IOGate.move_cursor_column x + pos
@output.write "#{Reline::IOGate::RESET_COLOR}#{content}#{Reline::IOGate::RESET_COLOR}"
end
base_x += width
Expand Down Expand Up @@ -699,13 +704,6 @@ def add_dialog_proc(name, p, context = nil)

DIALOG_DEFAULT_HEIGHT = 20

private def padding_space_with_escape_sequences(str, width)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm curious, why did you combine calculating padding with take_mbchar_range?

Copy link
Member Author

@tompng tompng Apr 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Padding sometimes have background colors.
The last testcase I added is a good example.

assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう\e[0m", 1, 4, padding: true)

colored padding

So we can't add non-colored space as padding. Padding logic cannot be separated from take_mbchar_range

padding_width = width - calculate_width(str, true)
# padding_width should be only positive value. But macOS and Alacritty returns negative value.
padding_width = 0 if padding_width < 0
str + (' ' * padding_width)
end

private def dialog_range(dialog, dialog_y)
x_range = dialog.column...dialog.column + dialog.width
y_range = dialog_y + dialog.vertical_offset...dialog_y + dialog.vertical_offset + dialog.contents.size
Expand Down Expand Up @@ -778,7 +776,7 @@ def add_dialog_proc(name, p, context = nil)
dialog.contents = contents.map.with_index do |item, i|
line_sgr = i == pointer ? enhanced_sgr : default_sgr
str_width = dialog.width - (scrollbar_pos.nil? ? 0 : @block_elem_width)
str = padding_space_with_escape_sequences(Reline::Unicode.take_range(item, 0, str_width), str_width)
str, = Reline::Unicode.take_mbchar_range(item, 0, str_width, padding: true)
colored_content = "#{line_sgr}#{str}"
if scrollbar_pos
if scrollbar_pos <= (i * 2) and (i * 2 + 1) < (scrollbar_pos + bar_height)
Expand Down
56 changes: 51 additions & 5 deletions lib/reline/unicode.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,32 +179,78 @@ def self.split_by_width(str, max_width, encoding = str.encoding, offset: 0)

# Take a chunk of a String cut by width with escape sequences.
def self.take_range(str, start_col, max_width)
take_mbchar_range(str, start_col, max_width).first
end

def self.take_mbchar_range(str, start_col, width, cover_begin: false, cover_end: false, padding: false)
chunk = String.new(encoding: str.encoding)

end_col = start_col + width
total_width = 0
rest = str.encode(Encoding::UTF_8)
in_zero_width = false
chunk_start_col = nil
chunk_end_col = nil
has_csi = false
rest.scan(WIDTH_SCANNER) do |non_printing_start, non_printing_end, csi, osc, gc|
case
when non_printing_start
in_zero_width = true
chunk << NON_PRINTING_START
when non_printing_end
in_zero_width = false
chunk << NON_PRINTING_END
when csi
has_csi = true
chunk << csi
when osc
chunk << osc
when gc
if in_zero_width
chunk << gc
next
end

mbchar_width = get_mbchar_width(gc)
prev_width = total_width
total_width += mbchar_width

if (cover_begin || padding ? total_width <= start_col : prev_width < start_col)
# Current character haven't reached start_col yet
next
elsif padding && !cover_begin && prev_width < start_col && start_col < total_width
# Add preceding padding. This padding might have background color.
chunk << ' '
chunk_start_col ||= start_col
chunk_end_col = total_width
next
elsif (cover_end ? prev_width < end_col : total_width <= end_col)
# Current character is in the range
chunk << gc
chunk_start_col ||= prev_width
chunk_end_col = total_width
break if total_width >= end_col
else
mbchar_width = get_mbchar_width(gc)
total_width += mbchar_width
break if (start_col + max_width) < total_width
chunk << gc if start_col < total_width
# Current character exceeds end_col
if padding && end_col < total_width
# Add succeeding padding. This padding might have background color.
chunk << ' '
chunk_start_col ||= prev_width
chunk_end_col = end_col
end
break
end
end
end
chunk
chunk_start_col ||= start_col
chunk_end_col ||= start_col
if padding && chunk_end_col < end_col
# Append padding. This padding should not include background color.
chunk << "\e[0m" if has_csi
chunk << ' ' * (end_col - chunk_end_col)
chunk_end_col = end_col
end
[chunk, chunk_start_col, chunk_end_col - chunk_start_col]
end

def self.get_next_mbchar_size(line, byte_pointer)
Expand Down
30 changes: 30 additions & 0 deletions test/reline/test_line_editor.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,36 @@ def test_dialog_move
end
end

def test_multibyte
base = [0, 12, '一二三一二三']
left = [0, 3, 'LLL']
right = [9, 3, 'RRR']
front = [3, 6, 'FFFFFF']
# 一 FFFFFF 三
# 一二三一二三
assert_output '[COL_2]二三一二' do
@line_editor.render_line_differential([base, front], [base, nil])
end

# LLLFFFFFF 三
# LLL 三一二三
assert_output '[COL_3] 三一二' do
@line_editor.render_line_differential([base, left, front], [base, left, nil])
end

# 一 FFFFFFRRR
# 一二三一 RRR
assert_output '[COL_2]二三一 ' do
@line_editor.render_line_differential([base, right, front], [base, right, nil])
end

# LLLFFFFFFRRR
# LLL 三一 RRR
assert_output '[COL_3] 三一 ' do
@line_editor.render_line_differential([base, left, right, front], [base, left, right, nil])
end
end

def test_complicated
state_a = [nil, [19, 7, 'bbbbbbb'], [15, 8, 'cccccccc'], [10, 5, 'ddddd'], [18, 4, 'eeee'], [1, 3, 'fff'], [17, 2, 'gg'], [7, 1, 'h']]
state_b = [[5, 9, 'aaaaaaaaa'], nil, [15, 8, 'cccccccc'], nil, [18, 4, 'EEEE'], [25, 4, 'ffff'], [17, 2, 'gg'], [2, 2, 'hh']]
Expand Down
26 changes: 24 additions & 2 deletions test/reline/test_unicode.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ def test_split_by_width_csi_reset_sgr_optimization
def test_take_range
assert_equal 'cdef', Reline::Unicode.take_range('abcdefghi', 2, 4)
assert_equal 'あde', Reline::Unicode.take_range('abあdef', 2, 4)
assert_equal 'zerocdef', Reline::Unicode.take_range("ab\1zero\2cdef", 2, 4)
assert_equal 'bzerocde', Reline::Unicode.take_range("ab\1zero\2cdef", 1, 4)
assert_equal "\1zero\2cdef", Reline::Unicode.take_range("ab\1zero\2cdef", 2, 4)
assert_equal "b\1zero\2cde", Reline::Unicode.take_range("ab\1zero\2cdef", 1, 4)
assert_equal "\e[31mcd\e[42mef", Reline::Unicode.take_range("\e[31mabcd\e[42mefg", 2, 4)
assert_equal "\e]0;1\acd", Reline::Unicode.take_range("ab\e]0;1\acd", 2, 3)
assert_equal 'いう', Reline::Unicode.take_range('あいうえお', 2, 4)
Expand All @@ -67,4 +67,26 @@ def test_calculate_width
assert_equal 10, Reline::Unicode.calculate_width('あいうえお')
assert_equal 10, Reline::Unicode.calculate_width('あいうえお', true)
end

def test_take_mbchar_range
assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4)
assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, padding: true)
assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, cover_begin: true)
assert_equal ['cdef', 2, 4], Reline::Unicode.take_mbchar_range('abcdefghi', 2, 4, cover_end: true)
assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4)
assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, padding: true)
assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, cover_begin: true)
assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 2, 4, cover_end: true)
assert_equal ['う', 4, 2], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4)
assert_equal [' う ', 3, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, padding: true)
assert_equal ['いう', 2, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_begin: true)
assert_equal ['うえ', 4, 4], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_end: true)
assert_equal ['いう ', 2, 5], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_begin: true, padding: true)
assert_equal [' うえ', 3, 5], Reline::Unicode.take_mbchar_range('あいうえお', 3, 4, cover_end: true, padding: true)
assert_equal [' うえお ', 3, 10], Reline::Unicode.take_mbchar_range('あいうえお', 3, 10, padding: true)
assert_equal [" \e[41mうえお\e[0m ", 3, 10], Reline::Unicode.take_mbchar_range("あい\e[41mうえお", 3, 10, padding: true)
assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true)
assert_equal ["\e[31mc\1ABC\2d\e[0mef", 2, 4], Reline::Unicode.take_mbchar_range("\e[31mabc\1ABC\2d\e[0mefghi", 2, 4)
assert_equal ["\e[41m \e[42mい\e[43m ", 1, 4], Reline::Unicode.take_mbchar_range("\e[41mあ\e[42mい\e[43mう", 1, 4, padding: true)
end
end
Loading