Let's add images

Cargo.toml

@@ -46,6 +46,7 @@ bytemuck = { version = "1.12.1", features = ["derive"] }
 smallvec = "1.8.0"
 moscato = { git = "https://github.com/dfrg/pinot", rev = "59db153" }
 peniko = { git = "https://github.com/linebender/peniko", rev = "cafdac9a211a0fb2fec5656bd663d1ac770bcc81" }
+guillotiere = "0.6.2"
 
 [workspace.dependencies]
 wgpu = "0.15"

examples/headless/src/main.rs

@@ -6,7 +6,7 @@ use std::{
 
 use anyhow::{anyhow, bail, Context, Result};
 use clap::{CommandFactory, Parser};
-use scenes::{SceneParams, SceneSet, SimpleText};
+use scenes::{ImageCache, SceneParams, SceneSet, SimpleText};
 use vello::{
     block_on_wgpu,
     kurbo::{Affine, Vec2},
@@ -97,9 +97,11 @@ async fn render(mut scenes: SceneSet, index: usize, args: &Args) -> Result<()> {
     let mut builder = SceneBuilder::for_fragment(&mut fragment);
     let example_scene = &mut scenes.scenes[index];
     let mut text = SimpleText::new();
+    let mut images = ImageCache::new();
     let mut scene_params = SceneParams {
         time: args.time.unwrap_or(0.),
         text: &mut text,
+        images: &mut images,
         resolution: None,
         base_color: None,
     };

examples/scenes/Cargo.toml

@@ -15,6 +15,7 @@ vello = { path = "../../" }
 vello_svg = { path = "../../integrations/vello_svg" }
 anyhow = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
+image = "0.24.5"
 
 # Used for the `download` command
 byte-unit = "4.0"

examples/scenes/src/images.rs

@@ -0,0 +1,51 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use vello::peniko::{Blob, Format, Image};
+
+/// Simple hack to support loading images for examples.
+#[derive(Default)]
+pub struct ImageCache {
+    files: HashMap<PathBuf, Image>,
+    bytes: HashMap<usize, Image>,
+}
+
+impl ImageCache {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn from_file(&mut self, path: impl AsRef<Path>) -> anyhow::Result<Image> {
+        let path = path.as_ref();
+        if let Some(image) = self.files.get(path) {
+            Ok(image.clone())
+        } else {
+            let data = std::fs::read(path)?;
+            let image = decode_image(&data)?;
+            self.files.insert(path.to_owned(), image.clone());
+            Ok(image)
+        }
+    }
+
+    pub fn from_bytes(&mut self, key: usize, bytes: &[u8]) -> anyhow::Result<Image> {
+        if let Some(image) = self.bytes.get(&key) {
+            Ok(image.clone())
+        } else {
+            let image = decode_image(bytes)?;
+            self.bytes.insert(key, image.clone());
+            Ok(image)
+        }
+    }
+}
+
+fn decode_image(data: &[u8]) -> anyhow::Result<Image> {
+    let image = image::io::Reader::new(std::io::Cursor::new(data))
+        .with_guessed_format()?
+        .decode()?;
+    let width = image.width();
+    let height = image.height();
+    let data = Arc::new(image.into_rgba8().into_vec());
+    let blob = Blob::new(data);
+    Ok(Image::new(blob, Format::Rgba8, width, height))
+}

examples/scenes/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod download;
+mod images;
 mod simple_text;
 mod svg;
 mod test_scenes;
@@ -7,6 +8,7 @@ use std::path::PathBuf;
 use anyhow::{anyhow, Result};
 use clap::{Args, Subcommand};
 use download::Download;
+pub use images::ImageCache;
 pub use simple_text::SimpleText;
 pub use svg::{default_scene, scene_from_files};
 pub use test_scenes::test_scenes;
@@ -16,6 +18,7 @@ use vello::{kurbo::Vec2, peniko::Color, SceneBuilder};
 pub struct SceneParams<'a> {
     pub time: f64,
     pub text: &'a mut SimpleText,
+    pub images: &'a mut ImageCache,
     pub resolution: Option<Vec2>,
     pub base_color: Option<vello::peniko::Color>,
 }

examples/scenes/src/test_scenes.rs

@@ -3,6 +3,8 @@ use vello::kurbo::{Affine, BezPath, Ellipse, PathEl, Point, Rect};
 use vello::peniko::*;
 use vello::*;
 
+const PIET_LOGO_IMAGE: &[u8] = include_bytes!("../../assets/piet-logo.png");
+
 macro_rules! scene {
     ($name: ident) => {
         scene!($name: false)
@@ -97,6 +99,13 @@ fn cardioid_and_friends(sb: &mut SceneBuilder, _: &mut SceneParams) {
 }
 
 fn animated_text(sb: &mut SceneBuilder, params: &mut SceneParams) {
+    // Uses the static array address as a cache key for expedience. Real code
+    // should use a better strategy.
+    let piet_logo = params
+        .images
+        .from_bytes(PIET_LOGO_IMAGE.as_ptr() as usize, PIET_LOGO_IMAGE)
+        .unwrap();
+
     use PathEl::*;
     let rect = Rect::from_origin_size(Point::new(0.0, 0.0), (1000.0, 1000.0));
     let star = [
@@ -184,6 +193,10 @@ fn animated_text(sb: &mut SceneBuilder, params: &mut SceneParams) {
         None,
         &star,
     );
+    sb.draw_image(
+        &piet_logo,
+        Affine::translate((550.0, 250.0)) * Affine::skew(-20f64.to_radians().tan(), 0.0),
+    );
 }
 
 fn brush_transform(sb: &mut SceneBuilder, params: &mut SceneParams) {

examples/with_winit/src/lib.rs

@@ -19,7 +19,7 @@ use std::time::Instant;
 
 use anyhow::Result;
 use clap::{CommandFactory, Parser};
-use scenes::{SceneParams, SceneSet, SimpleText};
+use scenes::{ImageCache, SceneParams, SceneSet, SimpleText};
 use vello::peniko::Color;
 use vello::util::RenderSurface;
 use vello::{
@@ -96,6 +96,7 @@ fn run(
     let mut scene = Scene::new();
     let mut fragment = SceneFragment::new();
     let mut simple_text = SimpleText::new();
+    let mut images = ImageCache::new();
     let start = Instant::now();
 
     let mut touch_state = multi_touch::TouchState::new();
@@ -264,6 +265,7 @@ fn run(
             let mut scene_params = SceneParams {
                 time: start.elapsed().as_secs_f64(),
                 text: &mut simple_text,
+                images: &mut images,
                 resolution: None,
                 base_color: None,
             };

shader/coarse.wgsl

@@ -126,6 +126,13 @@ fn write_grad(ty: u32, index: u32, info_offset: u32) {
     cmd_offset += 3u;
 }
 
+fn write_image(info_offset: u32) {
+    alloc_cmd(2u);
+    ptcl[cmd_offset] = CMD_IMAGE;
+    ptcl[cmd_offset + 1u] = info_offset;
+    cmd_offset += 2u;
+}
+
 fn write_begin_clip() {
     alloc_cmd(1u);
     ptcl[cmd_offset] = CMD_BEGIN_CLIP;
@@ -377,6 +384,13 @@ fn main(
                             write_grad(CMD_RAD_GRAD, index, info_offset);
                         }
                     }
+                    // DRAWTAG_FILL_IMAGE
+                    case 0x248u: {
+                        let linewidth = bitcast<f32>(info_bin_data[di]);
+                        if write_path(tile, linewidth) {                            
+                            write_image(di + 1u);
+                        }
+                    }
                     // DRAWTAG_BEGIN_CLIP
                     case 0x9u: {
                         if tile.segments == 0u && tile.backdrop == 0 {

shader/draw_leaf.wgsl

@@ -113,7 +113,9 @@ fn main(
         var matrx: vec4<f32>;
         var translate: vec2<f32>;
         var linewidth = bbox.linewidth;
-        if linewidth >= 0.0 || tag_word == DRAWTAG_FILL_LIN_GRADIENT || tag_word == DRAWTAG_FILL_RAD_GRADIENT {
+        if linewidth >= 0.0 || tag_word == DRAWTAG_FILL_LIN_GRADIENT || tag_word == DRAWTAG_FILL_RAD_GRADIENT ||
+            tag_word == DRAWTAG_FILL_IMAGE 
+        {
             let transform = read_transform(config.transform_base, bbox.trans_ix);
             matrx = transform.matrx;
             translate = transform.translate;
@@ -123,8 +125,8 @@ fn main(
             linewidth *= sqrt(abs(matrx.x * matrx.w - matrx.y * matrx.z));
         }
         switch tag_word {
-            // DRAWTAG_FILL_COLOR, DRAWTAG_FILL_IMAGE
-            case 0x44u, 0x48u: {
+            // DRAWTAG_FILL_COLOR
+            case 0x44u: {
                 info[di] = bitcast<u32>(linewidth);
             }
             // DRAWTAG_FILL_LIN_GRADIENT
@@ -169,6 +171,21 @@ fn main(
                 info[di + 9u] = bitcast<u32>(ra);
                 info[di + 10u] = bitcast<u32>(roff);
             }
+            // DRAWTAG_FILL_IMAGE
+            case 0x248u: {
+                info[di] = bitcast<u32>(linewidth);
+                let inv_det = 1.0 / (matrx.x * matrx.w - matrx.y * matrx.z);
+                let inv_mat = inv_det * vec4(matrx.w, -matrx.y, -matrx.z, matrx.x);
+                let inv_tr = mat2x2(inv_mat.xy, inv_mat.zw) * -translate;
+                info[di + 1u] = bitcast<u32>(inv_mat.x);
+                info[di + 2u] = bitcast<u32>(inv_mat.y);
+                info[di + 3u] = bitcast<u32>(inv_mat.z);
+                info[di + 4u] = bitcast<u32>(inv_mat.w);
+                info[di + 5u] = bitcast<u32>(inv_tr.x);
+                info[di + 6u] = bitcast<u32>(inv_tr.y);
+                info[di + 7u] = scene[dd];
+                info[di + 8u] = scene[dd + 1u];
+            }
             default: {}
         }
     }

shader/fine.wgsl

@@ -40,6 +40,9 @@ var gradients: texture_2d<f32>;
 @group(0) @binding(6)
 var<storage> info: array<u32>;
 
+@group(0) @binding(7)
+var image_atlas: texture_2d<f32>;
+
 fn read_fill(cmd_ix: u32) -> CmdFill {
     let tile = ptcl[cmd_ix + 1u];
     let backdrop = i32(ptcl[cmd_ix + 2u]);
@@ -81,6 +84,24 @@ fn read_rad_grad(cmd_ix: u32) -> CmdRadGrad {
     return CmdRadGrad(index, matrx, xlat, c1, ra, roff);
 }
 
+fn read_image(cmd_ix: u32) -> CmdImage {
+    let info_offset = ptcl[cmd_ix + 1u];
+    let m0 = bitcast<f32>(info[info_offset]);
+    let m1 = bitcast<f32>(info[info_offset + 1u]);
+    let m2 = bitcast<f32>(info[info_offset + 2u]);
+    let m3 = bitcast<f32>(info[info_offset + 3u]);
+    let matrx = vec4(m0, m1, m2, m3);
+    let xlat = vec2(bitcast<f32>(info[info_offset + 4u]), bitcast<f32>(info[info_offset + 5u]));
+    let xy = info[info_offset + 6u];
+    let width_height = info[info_offset + 7u];
+    // The following are not intended to be bitcasts
+    let x = f32(xy >> 16u);
+    let y = f32(xy & 0xffffu);
+    let width = f32(width_height >> 16u);
+    let height = f32(width_height & 0xffffu);
+    return CmdImage(matrx, xlat, vec2(x, y), vec2(width, height));
+}
+
 fn read_end_clip(cmd_ix: u32) -> CmdEndClip {
     let blend = ptcl[cmd_ix + 1u];
     let alpha = bitcast<f32>(ptcl[cmd_ix + 2u]);
@@ -265,6 +286,28 @@ fn main(
                 }
                 cmd_ix += 3u;
             }
+            // CMD_IMAGE
+            case 8u: {
+                let image = read_image(cmd_ix);
+                let atlas_extents = image.atlas_offset + image.extents;
+                for (var i = 0u; i < PIXELS_PER_THREAD; i += 1u) {
+                    let my_xy = vec2(xy.x + f32(i), xy.y);
+                    let atlas_uv = image.matrx.xy * my_xy.x + image.matrx.zw * my_xy.y + image.xlat + image.atlas_offset;
+                    // This currently clips to the image bounds. TODO: extend modes
+                    if all(atlas_uv < atlas_extents) && area[i] != 0.0 {
+                        let uv_quad = vec4(max(floor(atlas_uv), image.atlas_offset), min(ceil(atlas_uv), atlas_extents));
+                        let uv_frac = fract(atlas_uv);
+                        let a = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.xy), 0));
+                        let b = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.xw), 0));
+                        let c = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.zy), 0));
+                        let d = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.zw), 0));
+                        let fg_rgba = mix(mix(a, b, uv_frac.y), mix(c, d, uv_frac.y), uv_frac.x);
+                        let fg_i = fg_rgba * area[i];
+                        rgba[i] = rgba[i] * (1.0 - fg_i.a) + fg_i;
+                    }
+                }
+                cmd_ix += 2u;
+            }
             // CMD_BEGIN_CLIP
             case 9u: {
                 if clip_depth < BLEND_STACK_SPLIT {
@@ -326,3 +369,7 @@ fn main(
     }
 #endif
 }
+
+fn premul_alpha(rgba: vec4<f32>) -> vec4<f32> {
+    return vec4(rgba.rgb * rgba.a, rgba.a);
+}

shader/shared/drawtag.wgsl

@@ -19,7 +19,7 @@ let DRAWTAG_NOP = 0u;
 let DRAWTAG_FILL_COLOR = 0x44u;
 let DRAWTAG_FILL_LIN_GRADIENT = 0x114u;
 let DRAWTAG_FILL_RAD_GRADIENT = 0x2dcu;
-let DRAWTAG_FILL_IMAGE = 0x48u;
+let DRAWTAG_FILL_IMAGE = 0x248u;
 let DRAWTAG_BEGIN_CLIP = 0x9u;
 let DRAWTAG_END_CLIP = 0x21u;
 

shader/shared/ptcl.wgsl

@@ -16,6 +16,7 @@ let CMD_SOLID = 3u;
 let CMD_COLOR = 5u;
 let CMD_LIN_GRAD = 6u;
 let CMD_RAD_GRAD = 7u;
+let CMD_IMAGE = 8u;
 let CMD_BEGIN_CLIP = 9u;
 let CMD_END_CLIP = 10u;
 let CMD_JUMP = 11u;
@@ -57,6 +58,13 @@ struct CmdRadGrad {
     roff: f32,
 }
 
+struct CmdImage {
+    matrx: vec4<f32>,
+    xlat: vec2<f32>,
+    atlas_offset: vec2<f32>,
+    extents: vec2<f32>,
+}
+
 struct CmdEndClip {
     blend: u32,
     alpha: f32,

src/encoding.rs

@@ -20,6 +20,7 @@ mod draw;
 mod encoding;
 mod glyph;
 mod glyph_cache;
+mod image_cache;
 mod math;
 mod monoid;
 mod path;

src/encoding/draw.rs

@@ -38,7 +38,7 @@ impl DrawTag {
     pub const RADIAL_GRADIENT: Self = Self(0x2dc);
 
     /// Image fill.
-    pub const IMAGE: Self = Self(0x48);
+    pub const IMAGE: Self = Self(0x248);
 
     /// Begin layer/clip.
     pub const BEGIN_CLIP: Self = Self(0x9);
@@ -104,10 +104,10 @@ pub struct DrawRadialGradient {
 #[derive(Clone, Copy, Debug, Default, Zeroable, Pod)]
 #[repr(C)]
 pub struct DrawImage {
-    /// Image index.
-    pub index: u32,
-    /// Packed image offset.
-    pub offset: u32,
+    /// Packed atlas coordinates.
+    pub xy: u32,
+    /// Packed image dimensions.
+    pub width_height: u32,
 }
 
 /// Draw data for a clip or layer.