Skip to content

Commit

Permalink
Merge pull request #406 from martinRenou/image_embedding_refactor
Browse files Browse the repository at this point in the history 
Refactor image embedding logic
  • Loading branch information
@martinRenou
martinRenou authored Dec 15, 2021
2 parents 84bf0aa + 9aad6fc commit 480a236
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 35 deletions.
28 changes: 19 additions & 9 deletions ipympl/backend_nbagg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
Float,
Instance,
List,
Tuple,
Unicode,
default,
)
Expand Down Expand Up @@ -184,8 +185,7 @@ class Canvas(DOMWidget, FigureCanvasWebAggCore):
# This will still be used by ipywidgets in the case of embedding.
_data_url = Any(None).tag(sync=True)

_width = CInt().tag(sync=True)
_height = CInt().tag(sync=True)
_size = Tuple([0, 0]).tag(sync=True)

_figure_label = Unicode('Figure').tag(sync=True)
_message = Unicode().tag(sync=True)
Expand Down Expand Up @@ -265,9 +265,11 @@ def send_json(self, content):
self._figure_label = content['label']

elif content['type'] == 'resize':
self._width = content['size'][0]
self._height = content['size'][1]
# Send resize message anyway
self._size = content['size']
# Send resize message anyway:
# We absolutely need this instead of a `_size` trait change listening
# on the front-end, otherwise ipywidgets might squash multiple changes
# and the resizing protocol is not respected anymore
self.send({'data': json.dumps(content)})

elif content['type'] == 'image_mode':
Expand Down Expand Up @@ -306,28 +308,36 @@ def _repr_mimebundle_(self, **kwargs):

buf = io.BytesIO()
self.figure.savefig(buf, format='png', dpi='figure')
self._data_url = b64encode(buf.getvalue()).decode('utf-8')
# Figure width in pixels

base64_image = b64encode(buf.getvalue()).decode('utf-8')
self._data_url = f'data:image/png;base64,{base64_image}'
# Figure size in pixels
pwidth = self.figure.get_figwidth() * self.figure.get_dpi()
pheight = self.figure.get_figheight() * self.figure.get_dpi()
# Scale size to match widget on HiDPI monitors.
if hasattr(self, 'device_pixel_ratio'): # Matplotlib 3.5+
width = pwidth / self.device_pixel_ratio
height = pheight / self.device_pixel_ratio
else:
width = pwidth / self._dpi_ratio
height = pheight / self._dpi_ratio
html = """
<div style="display: inline-block;">
<div class="jupyter-widgets widget-label" style="text-align: center;">
{}
</div>
<img src='data:image/png;base64,{}' width={}/>
<img src='{}' width={}/>
</div>
""".format(
self._figure_label, self._data_url, width
)

# Update the widget model properly for HTML embedding
self._size = (width, height)

data = {
'text/plain': plaintext,
'image/png': self._data_url,
'image/png': base64_image,
'text/html': html,
'application/vnd.jupyter.widget-view+json': {
'version_major': 2,
Expand Down
109 changes: 83 additions & 26 deletions src/mpl_widget.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
capture_scroll: false,
pan_zoom_throttle: 33,
_data_url: null,
_width: 0,
_height: 0,
_size: [0, 0],
_figure_label: 'Figure',
_message: '',
_cursor: 'pointer',
Expand Down Expand Up @@ -78,7 +77,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
this.resize_requested = false;
this.ratio = (window.devicePixelRatio || 1) / backingStore;

this.resize_canvas(this.get('_width'), this.get('_height'));
this.resize_canvas();

this._init_image();

Expand All @@ -88,13 +87,21 @@ export class MPLCanvasModel extends DOMWidgetModel {
view.update_canvas();
});
});
this.on('change:_size', () => {
this.resize_canvas();
this.offscreen_context.drawImage(this.image, 0, 0);
});
this.on('comm_live_update', this.update_disabled.bind(this));

this.update_disabled();

this.send_initialization_message();
}

get size(): [number, number] {
return this.get('_size');
}

get disabled(): boolean {
return !this.comm_live;
}
Expand Down Expand Up @@ -149,13 +156,12 @@ export class MPLCanvasModel extends DOMWidgetModel {
}

handle_resize(msg: { [index: string]: any }) {
const size = msg['size'];
this.resize_canvas(size[0], size[1]);
this.resize_canvas();
this.offscreen_context.drawImage(this.image, 0, 0);

if (!this.resize_requested) {
this._for_each_view((view: MPLCanvasView) => {
view.resize_canvas(size[0], size[1]);
view.resize_and_update_canvas(this.size);
});
}

Expand All @@ -169,6 +175,9 @@ export class MPLCanvasModel extends DOMWidgetModel {
}
}

/*
* Request a resize to the backend
*/
resize(width: number, height: number) {
// Do not request a super small size, as it seems to break the back-end
if (width <= 5 || height <= 5) {
Expand All @@ -177,7 +186,7 @@ export class MPLCanvasModel extends DOMWidgetModel {

this._for_each_view((view: MPLCanvasView) => {
// Do an initial resize of each view, stretching the old canvas.
view.resize_canvas(width, height);
view.resize_and_update_canvas([width, height]);
});

if (this.resize_requested) {
Expand All @@ -189,9 +198,12 @@ export class MPLCanvasModel extends DOMWidgetModel {
}
}

resize_canvas(width: number, height: number) {
this.offscreen_canvas.width = width * this.ratio;
this.offscreen_canvas.height = height * this.ratio;
/*
* Resize the offscreen canvas
*/
resize_canvas() {
this.offscreen_canvas.width = this.size[0] * this.ratio;
this.offscreen_canvas.height = this.size[1] * this.ratio;
}

handle_rubberband(msg: any) {
Expand Down Expand Up @@ -275,17 +287,49 @@ export class MPLCanvasModel extends DOMWidgetModel {
this.image = new Image();

this.image.onload = () => {
// In case of an embedded widget, the initial size is not correct
// and we are not receiving any resize event from the server
if (this.disabled) {
this.offscreen_canvas.width = this.image.width;
this.offscreen_canvas.height = this.image.height;

this.offscreen_context.drawImage(this.image, 0, 0);

this._for_each_view((view: MPLCanvasView) => {
// TODO Make this part of the CanvasView API?
// It feels out of place in the model
view.canvas.width = this.image.width / this.ratio;
view.canvas.height = this.image.height / this.ratio;
view.canvas.style.width = view.canvas.width + 'px';
view.canvas.style.height = view.canvas.height + 'px';

view.top_canvas.width = this.image.width / this.ratio;
view.top_canvas.height = this.image.height / this.ratio;
view.top_canvas.style.width = view.top_canvas.width + 'px';
view.top_canvas.style.height =
view.top_canvas.height + 'px';

view.canvas_div.style.width = view.canvas.width + 'px';
view.canvas_div.style.height = view.canvas.height + 'px';

view.update_canvas(true);
});

return;
}

// Full images could contain transparency (where diff images
// almost always do), so we need to clear the canvas so that
// there is no ghosting.
if (this.get('_image_mode') === 'full') {
// Full images could contain transparency (where diff images
// almost always do), so we need to clear the canvas so that
// there is no ghosting.
this.offscreen_context.clearRect(
0,
0,
this.offscreen_canvas.width,
this.offscreen_canvas.height
);
}

this.offscreen_context.drawImage(this.image, 0, 0);

this._for_each_view((view: MPLCanvasView) => {
Expand Down Expand Up @@ -556,19 +600,32 @@ export class MPLCanvasView extends DOMWidgetView {
return false;
});

this.resize_canvas(this.model.get('_width'), this.model.get('_height'));
this.update_canvas();
this.resize_and_update_canvas(this.model.size);
}

update_canvas() {
/*
* Update the canvas view
*/
update_canvas(stretch = false) {
if (this.canvas.width === 0 || this.canvas.height === 0) {
return;
}

this.top_context.save();

this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.context.drawImage(this.model.offscreen_canvas, 0, 0);

if (stretch) {
this.context.drawImage(
this.model.offscreen_canvas,
0,
0,
this.canvas.width,
this.canvas.height
);
} else {
this.context.drawImage(this.model.offscreen_canvas, 0, 0);
}

this.top_context.clearRect(
0,
Expand Down Expand Up @@ -651,18 +708,18 @@ export class MPLCanvasView extends DOMWidgetView {
this.footer.textContent = this.model.get('_message');
}

resize_canvas(width: number, height: number) {
resize_and_update_canvas(size: [number, number]) {
// Keep the size of the canvas, and rubber band canvas in sync.
this.canvas.setAttribute('width', `${width * this.model.ratio}`);
this.canvas.setAttribute('height', `${height * this.model.ratio}`);
this.canvas.style.width = width + 'px';
this.canvas.style.height = height + 'px';
this.canvas.setAttribute('width', `${size[0] * this.model.ratio}`);
this.canvas.setAttribute('height', `${size[1] * this.model.ratio}`);
this.canvas.style.width = size[0] + 'px';
this.canvas.style.height = size[1] + 'px';

this.top_canvas.setAttribute('width', String(width));
this.top_canvas.setAttribute('height', String(height));
this.top_canvas.setAttribute('width', String(size[0]));
this.top_canvas.setAttribute('height', String(size[1]));

this.canvas_div.style.width = width + 'px';
this.canvas_div.style.height = height + 'px';
this.canvas_div.style.width = size[0] + 'px';
this.canvas_div.style.height = size[1] + 'px';

this.update_canvas();
}
Expand Down

0 comments on commit 480a236

Please sign in to comment.