From ed1a01a3179b629eabddf83a9e140d19d70c3a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Gro=C3=9Fe?= Date: Thu, 23 Feb 2023 10:18:15 +0000 Subject: [PATCH] Image previews The `sixel` flag enables sixel support in `ratatu-image`, which then needs libsixel to build. Additionally to enable the preview feature, one must add it to config.json: ```json "image_preview": { "size": { "width": 60, "height": 10, } } ``` There is a `backend: "halfblocks"` option in `image_preview` to force a specific backend rather than guessing it, which might not be 100% reliable in all terminals. --- Cargo.lock | 48 +++++++++ Cargo.toml | 10 ++ flake.nix | 14 ++- src/base.rs | 14 ++- src/config.rs | 25 +++++ src/main.rs | 10 ++ src/message/mod.rs | 76 +++++++++++++- src/preview.rs | 183 +++++++++++++++++++++++++++++++++ src/tests.rs | 1 + src/windows/room/scrollback.rs | 32 +++++- src/worker.rs | 64 +++++++++++- 11 files changed, 463 insertions(+), 14 deletions(-) create mode 100644 src/preview.rs diff --git a/Cargo.lock b/Cargo.lock index dd63a0e..866a674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -940,6 +940,12 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "dyn-clone" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" + [[package]] name = "ed25519" version = "1.5.3" @@ -1570,6 +1576,7 @@ dependencies = [ "modalkit", "open", "pretty_assertions", + "ratatui-image", "regex", "rpassword", "serde", @@ -1885,6 +1892,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "make-cmd" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8ca8afbe8af1785e09636acb5a41e08a765f5f0340568716c18a8700ba3c0d3" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -2823,6 +2836,22 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "ratatui-image" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7bb914100a51af18c080ce9aa146259371b940bf22c8c21f5d8e5b75dc5805" +dependencies = [ + "base64 0.21.4", + "dyn-clone", + "image", + "rand 0.8.5", + "ratatui", + "rustix 0.38.17", + "serde", + "sixel-bytes", +] + [[package]] name = "rayon" version = "1.8.0" @@ -3347,6 +3376,25 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "sixel-bytes" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45cad296a72571e80953823496e9a55caf893e264de9a7c5cfd29427fca720fc" +dependencies = [ + "sixel-sys-static", +] + +[[package]] +name = "sixel-sys-static" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2988846c5099382a880a7dd385d38b203a60430710a9c22e538d500e6908f4f9" +dependencies = [ + "make-cmd", + "pkg-config", +] + [[package]] name = "slab" version = "0.4.9" diff --git a/Cargo.toml b/Cargo.toml index 297d5ce..1625820 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,11 @@ features = ["e2e-encryption", "sled", "rustls-tls"] version = "1.24.1" features = ["macros", "net", "rt-multi-thread", "sync", "time"] +[dependencies.ratatui-image] +version = "0.3.1" +default-features = false +features = ["serde", "rustix"] + [dev-dependencies] lazy_static = "1.4.0" pretty_assertions = "1.4.0" @@ -74,3 +79,8 @@ pretty_assertions = "1.4.0" [profile.release] lto = true incremental = false + +[features] +default = [] +sixel = ["ratatui-image/sixel"] + diff --git a/flake.nix b/flake.nix index 39e8628..5e4a3e6 100644 --- a/flake.nix +++ b/flake.nix @@ -22,18 +22,24 @@ pname = "iamb"; version = "0.0.7"; src = ./.; - cargoLock.lockFile = ./Cargo.lock; + cargoLock = { + lockFile = ./Cargo.lock; + outputHashes = { + "modalkit-0.0.16" = "sha256-mjAD1v0r2+SzPdoB2wZ/5iJ1NZK+3OSvCYcUZ5Ef38Y="; + }; + }; nativeBuildInputs = [ pkgs.pkgconfig ]; buildInputs = [ pkgs.openssl ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin.apple_sdk.frameworks; [ AppKit Security ]); }; devShell = mkShell { buildInputs = [ - (rustNightly.override { extensions = [ "rust-src" ]; }) + (rustNightly.override { + extensions = [ "rust-src" "rust-analyzer-preview" "rustfmt" "clippy" ]; + }) pkg-config cargo-tarpaulin - rust-analyzer - rustfmt + cargo-watch ]; }; }); diff --git a/src/base.rs b/src/base.rs index 7089bb4..c1efd87 100644 --- a/src/base.rs +++ b/src/base.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use emojis::Emoji; +use ratatui_image::picker::Picker; use serde::{ de::Error as SerdeError, de::Visitor, @@ -490,6 +491,12 @@ pub enum IambError { /// A failure to access the system's clipboard. #[error("Could not use system clipboard data")] Clipboard, + + #[error("IO error: {0}")] + IOError(#[from] std::io::Error), + + #[error("Preview error: {0}")] + Preview(String), } impl From for UIError { @@ -603,6 +610,10 @@ impl RoomInfo { self.messages.get(self.get_message_key(event_id)?) } + pub fn get_event_mut(&mut self, event_id: &EventId) -> Option<&mut Message> { + self.messages.get_mut(self.keys.get(event_id)?.to_message_key()?) + } + /// Insert a reaction to a message. pub fn insert_reaction(&mut self, react: ReactionEvent) { match react { @@ -825,6 +836,7 @@ pub struct ChatStore { /// Information gathered by the background thread. pub sync_info: SyncInfo, + pub picker: Option, } impl ChatStore { @@ -833,7 +845,7 @@ impl ChatStore { ChatStore { worker, settings, - + picker: None, cmds: crate::commands::setup_commands(), emojis: emoji_map(), diff --git a/src/config.rs b/src/config.rs index 86ddec5..61793ba 100644 --- a/src/config.rs +++ b/src/config.rs @@ -11,6 +11,7 @@ use std::process; use clap::Parser; use matrix_sdk::ruma::{OwnedRoomAliasId, OwnedRoomId, OwnedUserId, UserId}; +use ratatui_image::picker::ProtocolType; use serde::{de::Error as SerdeError, de::Visitor, Deserialize, Deserializer}; use tracing::Level; use url::Url; @@ -246,6 +247,26 @@ pub enum UserDisplayStyle { DisplayName, } +#[derive(Clone, Deserialize)] +pub struct ImagePreviewValues { + pub backend: Option, + pub size: ImagePreviewSize, +} +impl Default for ImagePreviewValues { + fn default() -> Self { + ImagePreviewValues { + backend: None, + size: ImagePreviewSize { width: 66, height: 10 }, + } + } +} + +#[derive(Clone, Deserialize)] +pub struct ImagePreviewSize { + pub width: usize, + pub height: usize, +} + #[derive(Clone)] pub struct TunableValues { pub log_level: Level, @@ -260,6 +281,7 @@ pub struct TunableValues { pub username_display: UserDisplayStyle, pub default_room: Option, pub open_command: Option>, + pub image_preview: Option, } #[derive(Clone, Default, Deserialize)] @@ -276,6 +298,7 @@ pub struct Tunables { pub username_display: Option, pub default_room: Option, pub open_command: Option>, + pub image_preview: Option, } impl Tunables { @@ -295,6 +318,7 @@ impl Tunables { username_display: self.username_display.or(other.username_display), default_room: self.default_room.or(other.default_room), open_command: self.open_command.or(other.open_command), + image_preview: self.image_preview.or(other.image_preview), } } @@ -312,6 +336,7 @@ impl Tunables { username_display: self.username_display.unwrap_or_default(), default_room: self.default_room, open_command: self.open_command, + image_preview: self.image_preview, } } } diff --git a/src/main.rs b/src/main.rs index dfb3997..2985f6d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,7 @@ use std::sync::Arc; use std::time::Duration; use clap::Parser; +use ratatui_image::picker::Picker; use tokio::sync::Mutex as AsyncMutex; use tracing_subscriber::FmtSubscriber; @@ -68,6 +69,7 @@ mod commands; mod config; mod keybindings; mod message; +mod preview; mod util; mod windows; mod worker; @@ -264,9 +266,17 @@ impl Application { let bindings = KeyManager::new(bindings); let mut locked = store.lock().await; + + #[cfg(not(target = "windows"))] + if settings.tunables.image_preview.is_some() { + let picker = Picker::from_termios(None).unwrap(); + locked.application.picker = Some(picker); + } + let screen = setup_screen(settings, locked.deref_mut())?; let worker = locked.application.worker.clone(); + drop(locked); let actstack = VecDeque::new(); diff --git a/src/message/mod.rs b/src/message/mod.rs index de76bfa..37ba6af 100644 --- a/src/message/mod.rs +++ b/src/message/mod.rs @@ -51,7 +51,9 @@ use modalkit::tui::{ }; use modalkit::editing::{base::ViewportContext, cursor::Cursor}; +use ratatui_image::protocol::Protocol; +use crate::config::ImagePreviewSize; use crate::{ base::{IambResult, RoomInfo}, config::ApplicationSettings, @@ -584,12 +586,21 @@ impl<'a> MessageFormatter<'a> { } } +pub enum ImageBackend { + None, + Downloading(ImagePreviewSize), + Preparing(ImagePreviewSize), + Loaded(Box), + Error(String), +} + pub struct Message { pub event: MessageEvent, pub sender: OwnedUserId, pub timestamp: MessageTimeStamp, pub downloaded: bool, pub html: Option, + pub image_backend: ImageBackend, } impl Message { @@ -597,7 +608,14 @@ impl Message { let html = event.html(); let downloaded = false; - Message { event, sender, timestamp, downloaded, html } + Message { + event, + sender, + timestamp, + downloaded, + html, + image_backend: ImageBackend::None, + } } pub fn reply_to(&self) -> Option { @@ -683,6 +701,29 @@ impl Message { } } + pub fn line_preview( + &self, + prev: Option<&Message>, + vwctx: &ViewportContext, + ) -> Option<(&dyn Protocol, u16, u16)> { + if let ImageBackend::Loaded(backend) = &self.image_backend { + let width = vwctx.get_width(); + // The x position where get_render_format would render the text. + let x = (if USER_GUTTER + MIN_MSG_LEN <= width { + USER_GUTTER + } else { + 0 + } + 1) as u16; + // See get_render_format; account for possible "date" line. + let date_y = match &prev { + Some(prev) if !prev.timestamp.same_day(&self.timestamp) => 1, + _ => 0, + }; + return Some((backend.as_ref(), x, date_y)); + } + None + } + pub fn show<'a>( &'a self, prev: Option<&Message>, @@ -793,6 +834,22 @@ impl Message { msg.to_mut().push_str(" \u{2705}"); } + if let Some(placeholder) = match &self.image_backend { + ImageBackend::None => None, + ImageBackend::Downloading(image_preview_size) => { + Some(Message::placeholder_frame(Some("Downloading..."), image_preview_size)) + }, + ImageBackend::Preparing(image_preview_size) => { + Some(Message::placeholder_frame(Some("Preparing..."), image_preview_size)) + }, + ImageBackend::Loaded(backend) => { + Some(Message::placeholder_frame(None, &backend.rect().into())) + }, + ImageBackend::Error(err) => Some(format!("[Image error: {err}]")), + } { + msg.to_mut().insert_str(0, &placeholder); + } + wrapped_text(msg, width, style) } } @@ -805,6 +862,23 @@ impl Message { settings.get_user_span(self.sender.as_ref(), info) } + fn placeholder_frame(text: Option<&str>, image_preview_size: &ImagePreviewSize) -> String { + let ImagePreviewSize { width, height } = image_preview_size; + let mut placeholder = "\u{230c}".to_string(); + placeholder.push_str(&" ".repeat(width - 2)); + placeholder.push_str("\u{230d}\n"); + placeholder.push(' '); + if let Some(text) = text { + placeholder.push_str(text); + } + + placeholder.push_str(&"\n".repeat(height - 2)); + placeholder.push('\u{230e}'); + placeholder.push_str(&" ".repeat(width - 2)); + placeholder.push_str("\u{230f}\n"); + placeholder + } + fn show_sender<'a>( &'a self, prev: Option<&Message>, diff --git a/src/preview.rs b/src/preview.rs new file mode 100644 index 0000000..d14ea10 --- /dev/null +++ b/src/preview.rs @@ -0,0 +1,183 @@ +use std::{ + convert::TryFrom, + fs::File, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +use matrix_sdk::{ + media::{MediaFormat, MediaRequest}, + ruma::{ + events::{ + room::{ + message::{MessageType, RoomMessageEventContent}, + MediaSource, + }, + MessageLikeEvent, + }, + OwnedEventId, + OwnedRoomId, + }, + Media, +}; +use modalkit::tui::layout::Rect; +use ratatui_image::Resize; + +use crate::{ + base::{AsyncProgramStore, ChatStore, IambError}, + config::ImagePreviewSize, + message::ImageBackend, +}; + +pub struct PreviewSource { + pub source: MediaSource, + pub event_id: OwnedEventId, +} + +impl TryFrom<&MessageLikeEvent> for PreviewSource { + type Error = &'static str; + fn try_from(ev: &MessageLikeEvent) -> Result { + if let MessageLikeEvent::Original(ev) = &ev { + if let MessageType::Image(c) = &ev.content.msgtype { + Ok(PreviewSource { + source: c.source.clone(), + event_id: ev.event_id.clone(), + }) + } else { + Err("content message type is not image") + } + } else { + Err("event is not original event") + } + } +} + +impl From for Rect { + fn from(value: ImagePreviewSize) -> Self { + Rect::new(0, 0, value.width as _, value.height as _) + } +} +impl From for ImagePreviewSize { + fn from(rect: Rect) -> Self { + ImagePreviewSize { width: rect.width as _, height: rect.height as _ } + } +} + +pub fn spawn_insert_preview( + store: AsyncProgramStore, + room_id: OwnedRoomId, + source: PreviewSource, + media: Media, + cache_dir: PathBuf, +) { + tokio::spawn(async move { + let event_id = source.event_id.clone(); + let img = download_or_cache(source, media, cache_dir) + .await + .map(std::io::Cursor::new) + .map(image::io::Reader::new) + .map_err(IambError::Matrix) + .and_then(|reader| reader.with_guessed_format().map_err(IambError::IOError)) + .and_then(|reader| reader.decode().map_err(IambError::Image)); + match img { + Err(err) => { + try_set_msg_preview_error( + &mut store.lock().await.application, + room_id, + event_id, + err, + ); + }, + Ok(img) => { + let mut locked = store.lock().await; + let ChatStore { rooms, picker, settings, .. } = &mut locked.application; + + match picker + .as_mut() + .ok_or_else(|| IambError::Preview("Picker is empty".to_string())) + .and_then(|picker| { + Ok(( + picker, + rooms + .get_or_default(room_id.clone()) + .get_event_mut(&event_id) + .ok_or_else(|| { + IambError::Preview("Message not found".to_string()) + })?, + settings.tunables.image_preview.clone().ok_or_else(|| { + IambError::Preview("image_preview settings not found".to_string()) + })?, + )) + }) + .and_then(|(picker, msg, image_preview)| { + msg.image_backend = ImageBackend::Preparing(image_preview.size.clone()); + picker + .new_static_fit(img, image_preview.size.into(), Resize::Fit) + .map_err(|err| IambError::Preview(format!("{err:?}"))) + .map(|backend| (backend, msg)) + }) { + Err(err) => { + try_set_msg_preview_error(&mut locked.application, room_id, event_id, err); + }, + Ok((backend, msg)) => { + msg.image_backend = ImageBackend::Loaded(backend); + }, + } + }, + } + }); +} + +fn try_set_msg_preview_error( + application: &mut ChatStore, + room_id: OwnedRoomId, + event_id: OwnedEventId, + err: IambError, +) { + let rooms = &mut application.rooms; + + match rooms + .get_or_default(room_id) + .get_event_mut(&event_id) + .ok_or_else(|| IambError::Preview("Message not found".to_string())) + { + Ok(msg) => msg.image_backend = ImageBackend::Error(format!("{err:?}")), + Err(err) => eprintln!("{err:?}"), + } +} + +async fn download_or_cache( + source: PreviewSource, + media: Media, + mut cache_path: PathBuf, +) -> Result, matrix_sdk::Error> { + cache_path.push(Path::new("image_preview_downloads")); + cache_path.push(Path::new(source.event_id.localpart())); + + match File::open(&cache_path) { + Ok(mut f) => { + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer)?; + Ok(buffer) + }, + Err(_) => { + match media + .get_media_content( + &MediaRequest { source: source.source, format: MediaFormat::File }, + true, + ) + .await + { + Ok(buffer) => { + if let Err(err) = + File::create(&cache_path).and_then(|mut f| f.write_all(&buffer)) + { + eprintln!("cache file write error ({:?}): {}", cache_path, err); + } + Ok(buffer) + }, + Err(err) => Err(err), + } + }, + } +} diff --git a/src/tests.rs b/src/tests.rs index 94d120f..8c0e2d7 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -192,6 +192,7 @@ pub fn mock_tunables() -> TunableValues { .collect::>(), open_command: None, username_display: UserDisplayStyle::Username, + image_preview: None, } } diff --git a/src/windows/room/scrollback.rs b/src/windows/room/scrollback.rs index d3e8549..f3e9f55 100644 --- a/src/windows/room/scrollback.rs +++ b/src/windows/room/scrollback.rs @@ -1,6 +1,7 @@ //! Message scrollback use std::collections::HashSet; +use ratatui_image::FixedImage; use regex::Regex; use matrix_sdk::ruma::OwnedRoomId; @@ -1264,6 +1265,7 @@ impl<'a> StatefulWidget for Scrollback<'a> { fn render(self, area: Rect, buf: &mut Buffer, state: &mut Self::State) { let info = self.store.application.rooms.get_or_default(state.room_id.clone()); let settings = &self.store.application.settings; + let picker = &self.store.application.picker; let area = if state.cursor.timestamp.is_some() { render_jump_to_recent(area, buf, self.focused) } else { @@ -1307,7 +1309,11 @@ impl<'a> StatefulWidget for Scrollback<'a> { let sel = key == cursor_key; let txt = item.show(prev, foc && sel, &state.viewctx, info, settings); - prev = Some(item); + let mut msg_preview = if picker.is_some() { + item.line_preview(prev, &state.viewctx) + } else { + None + }; let incomplete_ok = !full || !sel; @@ -1323,9 +1329,17 @@ impl<'a> StatefulWidget for Scrollback<'a> { continue; } - lines.push((key, row, line)); + let line_preview = match msg_preview { + // Only take the preview into the matching row number. + Some((_, _, y)) if y as usize == row => msg_preview.take(), + _ => None, + }; + + lines.push((key, row, line, line_preview)); sawit |= sel; } + + prev = Some(item); } if lines.len() > height { @@ -1333,7 +1347,7 @@ impl<'a> StatefulWidget for Scrollback<'a> { let _ = lines.drain(..n); } - if let Some(((ts, event_id), row, _)) = lines.first() { + if let Some(((ts, event_id), row, _, _)) = lines.first() { state.viewctx.corner.timestamp = Some((*ts, event_id.clone())); state.viewctx.corner.text_row = *row; } @@ -1341,8 +1355,18 @@ impl<'a> StatefulWidget for Scrollback<'a> { let mut y = area.top(); let x = area.left(); - for (_, _, txt) in lines.into_iter() { + for ((_, _), _, txt, line_preview) in lines.into_iter() { let _ = buf.set_line(x, y, &txt, area.width); + if let Some((backend, msg_x, _)) = line_preview { + let image_widget = FixedImage::new(backend); + let mut rect = backend.rect(); + rect.x = msg_x; + rect.y = y; + // Don't render outside of scrollback area + if rect.bottom() <= area.bottom() { + image_widget.render(rect, buf); + } + } y += 1; } diff --git a/src/worker.rs b/src/worker.rs index 646f010..6efb1a4 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -75,6 +75,8 @@ use matrix_sdk::{ use modalkit::editing::action::{EditInfo, InfoMessage, UIError}; +use crate::message::ImageBackend; +use crate::preview::{spawn_insert_preview, PreviewSource}; use crate::{ base::{ AsyncProgramStore, @@ -230,9 +232,18 @@ async fn load_older_one( async fn load_insert(room_id: OwnedRoomId, res: MessageFetchResult, store: AsyncProgramStore) { let mut locked = store.lock().await; - let ChatStore { need_load, presences, rooms, .. } = &mut locked.application; + let ChatStore { + need_load, + presences, + rooms, + worker, + picker, + settings, + .. + } = &mut locked.application; let info = rooms.get_or_default(room_id.clone()); info.fetching = false; + let client = &worker.client; match res { Ok((fetch_id, msgs)) => { @@ -245,7 +256,28 @@ async fn load_insert(room_id: OwnedRoomId, res: MessageFetchResult, store: Async info.insert_encrypted(msg); }, AnyMessageLikeEvent::RoomMessage(msg) => { - info.insert(msg); + if picker.is_some() { + let source = PreviewSource::try_from(&msg); + info.insert(msg); + if let Ok(source) = source { + if let (Some(msg), Some(image_preview)) = ( + info.get_event_mut(&source.event_id), + &settings.tunables.image_preview, + ) { + msg.image_backend = + ImageBackend::Downloading(image_preview.size.clone()); + } + spawn_insert_preview( + store.clone(), + room_id.clone(), + source, + client.media(), + settings.dirs.cache.clone(), + ) + } + } else { + info.insert(msg); + } }, AnyMessageLikeEvent::Reaction(ev) => { info.insert_reaction(ev); @@ -786,8 +818,32 @@ impl ClientWorker { let sender = ev.sender().to_owned(); let _ = locked.application.presences.get_or_default(sender); - let info = locked.application.get_room_info(room_id.to_owned()); - info.insert(ev.into_full_event(room_id.to_owned())); + let ChatStore { rooms, picker, settings, .. } = &mut locked.application; + let info = rooms.get_or_default(room_id.to_owned()); + + if picker.is_some() { + let full_ev = ev.into_full_event(room_id.to_owned()); + let source = PreviewSource::try_from(&full_ev); + info.insert(full_ev); + if let Ok(source) = source { + if let (Some(msg), Some(image_preview)) = ( + info.get_event_mut(&source.event_id), + &settings.tunables.image_preview, + ) { + msg.image_backend = + ImageBackend::Downloading(image_preview.size.clone()); + } + spawn_insert_preview( + store.clone(), + room_id.to_owned(), + source, + client.media(), + settings.dirs.cache.clone(), + ) + }; + } else { + info.insert(ev.into_full_event(room_id.to_owned())); + } } }, );