Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor image embedding logic #406

Merged
merged 1 commit into from
Dec 15, 2021
Merged

Refactor image embedding logic #406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions ipympl/backend_nbagg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
Float,
Instance,
List,
Tuple,
Unicode,
default,
)
Expand Down Expand Up @@ -184,8 +185,7 @@ class Canvas(DOMWidget, FigureCanvasWebAggCore):
# This will still be used by ipywidgets in the case of embedding.
_data_url = Any(None).tag(sync=True)

_width = CInt().tag(sync=True)
_height = CInt().tag(sync=True)
_size = Tuple([0, 0]).tag(sync=True)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Replacing the two private traits _width and _height into _size so that the resize happens in one comm message

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like it


_figure_label = Unicode('Figure').tag(sync=True)
_message = Unicode().tag(sync=True)
Expand Down Expand Up @@ -265,9 +265,11 @@ def send_json(self, content):
self._figure_label = content['label']

elif content['type'] == 'resize':
self._width = content['size'][0]
self._height = content['size'][1]
# Send resize message anyway
self._size = content['size']
# Send resize message anyway:
# We absolutely need this instead of a `_size` trait change listening
# on the front-end, otherwise ipywidgets might squash multiple changes
# and the resizing protocol is not respected anymore
self.send({'data': json.dumps(content)})

elif content['type'] == 'image_mode':
Expand Down Expand Up @@ -306,28 +308,36 @@ def _repr_mimebundle_(self, **kwargs):

buf = io.BytesIO()
self.figure.savefig(buf, format='png', dpi='figure')
self._data_url = b64encode(buf.getvalue()).decode('utf-8')
# Figure width in pixels

base64_image = b64encode(buf.getvalue()).decode('utf-8')
self._data_url = f'data:image/png;base64,{base64_image}'
# Figure size in pixels
pwidth = self.figure.get_figwidth() * self.figure.get_dpi()
pheight = self.figure.get_figheight() * self.figure.get_dpi()
# Scale size to match widget on HiDPI monitors.
if hasattr(self, 'device_pixel_ratio'): # Matplotlib 3.5+
width = pwidth / self.device_pixel_ratio
height = pheight / self.device_pixel_ratio
else:
width = pwidth / self._dpi_ratio
height = pheight / self._dpi_ratio
html = """
<div style="display: inline-block;">
<div class="jupyter-widgets widget-label" style="text-align: center;">
{}
</div>
<img src='data:image/png;base64,{}' width={}/>
<img src='{}' width={}/>
</div>
""".format(
self._figure_label, self._data_url, width
)

# Update the widget model properly for HTML embedding
self._size = (width, height)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is one important change here.

We need to make the plot size part of the widget model in the case of nbconvert --to html otherwise there is no registered size and the canvas would be 0 sized.


data = {
'text/plain': plaintext,
'image/png': self._data_url,
'image/png': base64_image,
'text/html': html,
'application/vnd.jupyter.widget-view+json': {
'version_major': 2,
Expand Down
109 changes: 83 additions & 26 deletions src/mpl_widget.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
capture_scroll: false,
pan_zoom_throttle: 33,
_data_url: null,
_width: 0,
_height: 0,
_size: [0, 0],
_figure_label: 'Figure',
_message: '',
_cursor: 'pointer',
Expand Down Expand Up @@ -78,7 +77,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
this.resize_requested = false;
this.ratio = (window.devicePixelRatio || 1) / backingStore;

this.resize_canvas(this.get('_width'), this.get('_height'));
this.resize_canvas();

this._init_image();

Expand All @@ -88,13 +87,21 @@ export class MPLCanvasModel extends DOMWidgetModel {
view.update_canvas();
});
});
this.on('change:_size', () => {
this.resize_canvas();
this.offscreen_context.drawImage(this.image, 0, 0);
});
this.on('comm_live_update', this.update_disabled.bind(this));

this.update_disabled();

this.send_initialization_message();
}

get size(): [number, number] {
return this.get('_size');
}

get disabled(): boolean {
return !this.comm_live;
}
Expand Down Expand Up @@ -149,13 +156,12 @@ export class MPLCanvasModel extends DOMWidgetModel {
}

handle_resize(msg: { [index: string]: any }) {
const size = msg['size'];
this.resize_canvas(size[0], size[1]);
this.resize_canvas();
this.offscreen_context.drawImage(this.image, 0, 0);

if (!this.resize_requested) {
this._for_each_view((view: MPLCanvasView) => {
view.resize_canvas(size[0], size[1]);
view.resize_and_update_canvas(this.size);
});
}

Expand All @@ -169,6 +175,9 @@ export class MPLCanvasModel extends DOMWidgetModel {
}
}

/*
* Request a resize to the backend
*/
resize(width: number, height: number) {
// Do not request a super small size, as it seems to break the back-end
if (width <= 5 || height <= 5) {
Expand All @@ -177,7 +186,7 @@ export class MPLCanvasModel extends DOMWidgetModel {

this._for_each_view((view: MPLCanvasView) => {
// Do an initial resize of each view, stretching the old canvas.
view.resize_canvas(width, height);
view.resize_and_update_canvas([width, height]);
});

if (this.resize_requested) {
Expand All @@ -189,9 +198,12 @@ export class MPLCanvasModel extends DOMWidgetModel {
}
}

resize_canvas(width: number, height: number) {
this.offscreen_canvas.width = width * this.ratio;
this.offscreen_canvas.height = height * this.ratio;
/*
* Resize the offscreen canvas
*/
resize_canvas() {
this.offscreen_canvas.width = this.size[0] * this.ratio;
this.offscreen_canvas.height = this.size[1] * this.ratio;
}

handle_rubberband(msg: any) {
Expand Down Expand Up @@ -275,17 +287,49 @@ export class MPLCanvasModel extends DOMWidgetModel {
this.image = new Image();

this.image.onload = () => {
// In case of an embedded widget, the initial size is not correct
// and we are not receiving any resize event from the server
if (this.disabled) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the second important change in this PR.

Here, we need a special case for when the widget is disabled (no counterpart in the kernel, or no kernel at all), where we display the image that is part of the model.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Am I right that this will come up in many situations, not just nbconvert, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. You can trigger this situation by saving the widget model in the page, shutdown the kernel, then refresh the page.

this.offscreen_canvas.width = this.image.width;
this.offscreen_canvas.height = this.image.height;

this.offscreen_context.drawImage(this.image, 0, 0);

this._for_each_view((view: MPLCanvasView) => {
// TODO Make this part of the CanvasView API?
// It feels out of place in the model
view.canvas.width = this.image.width / this.ratio;
view.canvas.height = this.image.height / this.ratio;
view.canvas.style.width = view.canvas.width + 'px';
view.canvas.style.height = view.canvas.height + 'px';

view.top_canvas.width = this.image.width / this.ratio;
view.top_canvas.height = this.image.height / this.ratio;
view.top_canvas.style.width = view.top_canvas.width + 'px';
view.top_canvas.style.height =
view.top_canvas.height + 'px';

view.canvas_div.style.width = view.canvas.width + 'px';
view.canvas_div.style.height = view.canvas.height + 'px';

view.update_canvas(true);
});

return;
}

// Full images could contain transparency (where diff images
// almost always do), so we need to clear the canvas so that
// there is no ghosting.
if (this.get('_image_mode') === 'full') {
// Full images could contain transparency (where diff images
// almost always do), so we need to clear the canvas so that
// there is no ghosting.
this.offscreen_context.clearRect(
0,
0,
this.offscreen_canvas.width,
this.offscreen_canvas.height
);
}

this.offscreen_context.drawImage(this.image, 0, 0);

this._for_each_view((view: MPLCanvasView) => {
Expand Down Expand Up @@ -556,19 +600,32 @@ export class MPLCanvasView extends DOMWidgetView {
return false;
});

this.resize_canvas(this.model.get('_width'), this.model.get('_height'));
this.update_canvas();
this.resize_and_update_canvas(this.model.size);
}

update_canvas() {
/*
* Update the canvas view
*/
update_canvas(stretch = false) {
if (this.canvas.width === 0 || this.canvas.height === 0) {
return;
}

this.top_context.save();

this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.context.drawImage(this.model.offscreen_canvas, 0, 0);

if (stretch) {
this.context.drawImage(
this.model.offscreen_canvas,
0,
0,
this.canvas.width,
this.canvas.height
);
} else {
this.context.drawImage(this.model.offscreen_canvas, 0, 0);
}

this.top_context.clearRect(
0,
Expand Down Expand Up @@ -651,18 +708,18 @@ export class MPLCanvasView extends DOMWidgetView {
this.footer.textContent = this.model.get('_message');
}

resize_canvas(width: number, height: number) {
resize_and_update_canvas(size: [number, number]) {
// Keep the size of the canvas, and rubber band canvas in sync.
this.canvas.setAttribute('width', `${width * this.model.ratio}`);
this.canvas.setAttribute('height', `${height * this.model.ratio}`);
this.canvas.style.width = width + 'px';
this.canvas.style.height = height + 'px';
this.canvas.setAttribute('width', `${size[0] * this.model.ratio}`);
this.canvas.setAttribute('height', `${size[1] * this.model.ratio}`);
this.canvas.style.width = size[0] + 'px';
this.canvas.style.height = size[1] + 'px';

this.top_canvas.setAttribute('width', String(width));
this.top_canvas.setAttribute('height', String(height));
this.top_canvas.setAttribute('width', String(size[0]));
this.top_canvas.setAttribute('height', String(size[1]));

this.canvas_div.style.width = width + 'px';
this.canvas_div.style.height = height + 'px';
this.canvas_div.style.width = size[0] + 'px';
this.canvas_div.style.height = size[1] + 'px';

this.update_canvas();
}
Expand Down