Introduce file_format_version #36

nbrmd/__init__.py

@@ -10,18 +10,21 @@
 """
 
 from .nbrmd import readf, writef, writes, reads, NOTEBOOK_EXTENSIONS
+from .file_format_version import FILE_FORMAT_VERSION
 
 try:
     from .rmarkdownexporter import RMarkdownExporter
     from .srcexporter import PyNotebookExporter
     from .srcexporter import RNotebookExporter
 except ImportError as err:
-    RMarkdownExporter = str(err)
+    RMarkdownExporter = PyNotebookExporter = RNotebookExporter = str(err)
 
 try:
     from .contentsmanager import RmdFileContentsManager
 except ImportError as err:
     RmdFileContentsManager = str(err)
 
-__all__ = ['readf', 'writef', 'writes', 'reads', 'NOTEBOOK_EXTENSIONS',
-           'RMarkdownExporter', 'RmdFileContentsManager']
+__all__ = ['readf', 'writef', 'writes', 'reads',
+           'NOTEBOOK_EXTENSIONS', 'FILE_FORMAT_VERSION',
+           'RMarkdownExporter', 'PyNotebookExporter', 'RNotebookExporter',
+           'RmdFileContentsManager']

nbrmd/cli.py

@@ -9,51 +9,7 @@
 from nbrmd import writes
 from .languages import get_default_language
 from .combine import combine_inputs_with_outputs
-
-
-def convert_nbrmd(nb_files, in_place=True, combine=True):
-    """
-    Export R markdown notebooks, or Jupyter notebooks, to the opposite format
-    :param nb_files: one or more notebooks
-    :param in_place: should result of conversion be stored in file
-    with opposite extension?
-    :param combine: should the current outputs of .ipynb file be preserved,
-    when a cell with corresponding input is found in .Rmd?
-    :return:
-    """
-    for nb_file in nb_files:
-        file, ext = os.path.splitext(nb_file)
-        if ext not in ['.ipynb', '.Rmd']:
-            raise TypeError(
-                'File {} is neither a Jupyter (.ipynb) nor a '
-                'R Markdown (.Rmd) notebook'.format(nb_file))
-
-        nb = readf(nb_file)
-
-        if in_place:
-            if ext == '.ipynb':
-                nb_dest = file + '.Rmd'
-                print('Jupyter notebook {} being converted to '
-                      'R Markdown {}'.format(nb_file, nb_dest))
-            else:
-                msg = ''
-                nb_dest = file + '.ipynb'
-                if combine and os.path.isfile(nb_dest):
-                    try:
-                        nb_outputs = readf(nb_dest)
-                        combine_inputs_with_outputs(nb, nb_outputs)
-                        msg = '(outputs were preserved)'
-                    except (IOError, NotJSONError) as err:
-                        msg = '(outputs were not preserved: {})'.format(err)
-                print('R Markdown {} being converted to '
-                      'Jupyter notebook {} {}'
-                      .format(nb_file, nb_dest, msg))
-            writef(nb, nb_dest)
-        else:
-            if ext == '.ipynb':
-                print(writes(nb))
-            else:
-                print(ipynb_writes(nb))
+from .file_format_version import check_file_version
 
 
 def cli_nbrmd(args=None):
@@ -83,48 +39,60 @@ def nbrmd():
     :return:
     """
     args = cli_nbrmd()
-    convert_nbrmd(args.notebooks, args.in_place, args.preserve_outputs)
+    convert(args.notebooks, args.in_place, args.preserve_outputs, True)
 
 
-def convert_nbsrc(nb_files, in_place=True, combine=True):
+def convert(nb_files, in_place=True, combine=True, markdown=False):
     """
-    Export python or R scripts, or Jupyter notebooks, to the opposite format
+    Export R markdown notebooks, python or R scripts, or Jupyter notebooks,
+    to the opposite format
     :param nb_files: one or more notebooks
+    :param markdown: R markdown to Jupyter, or scripts to Jupyter?
     :param in_place: should result of conversion be stored in file
     with opposite extension?
     :param combine: should the current outputs of .ipynb file be preserved,
-    when a cell with corresponding input is found in .py or .R file?
+    when a cell with corresponding input is found in .Rmd/.py or .R file?
     :return:
     """
     for nb_file in nb_files:
         file, ext = os.path.splitext(nb_file)
-        if ext not in ['.ipynb', '.py', '.R']:
-            raise TypeError(
-                'File {} is neither a Jupyter (.ipynb) nor a '
-                'python script (.py), nor a R script (.R)'.format(nb_file))
+        if markdown:
+            format = 'R Markdown'
+            if ext not in ['.ipynb', '.Rmd']:
+                raise TypeError(
+                    'File {} is neither a Jupyter (.ipynb) nor a '
+                    'R Markdown (.Rmd) notebook'.format(nb_file))
+        else:
+            format = 'source'
+            if ext not in ['.ipynb', '.py', '.R']:
+                raise TypeError(
+                    'File {} is neither a Jupyter (.ipynb) nor a '
+                    'python script (.py), nor a R script (.R)'.format(nb_file))
 
         nb = readf(nb_file)
         main_language = get_default_language(nb)
-        ext_dest = '.R' if main_language == 'R' else '.py'
+        ext_dest = '.Rmd' if markdown else '.R' \
+            if main_language == 'R' else '.py'
 
         if in_place:
             if ext == '.ipynb':
                 nb_dest = file + ext_dest
                 print('Jupyter notebook {} being converted to '
-                      'source {}'.format(nb_file, nb_dest))
+                      '{} {}'.format(nb_file, format, nb_dest))
             else:
                 msg = ''
                 nb_dest = file + '.ipynb'
                 if combine and os.path.isfile(nb_dest):
+                    check_file_version(nb, nb_file, nb_dest)
                     try:
                         nb_outputs = readf(nb_dest)
                         combine_inputs_with_outputs(nb, nb_outputs)
                         msg = '(outputs were preserved)'
                     except (IOError, NotJSONError) as error:
                         msg = '(outputs were not preserved: {})'.format(error)
-                print('R Markdown {} being converted to '
+                print('{} {} being converted to '
                       'Jupyter notebook {} {}'
-                      .format(nb_file, nb_dest, msg))
+                      .format(format, nb_file, nb_dest, msg))
             writef(nb, nb_dest)
         else:
             if ext == '.ipynb':
@@ -160,4 +128,4 @@ def nbsrc():
     :return:
     """
     args = cli_nbsrc()
-    convert_nbsrc(args.notebooks, args.in_place, args.preserve_outputs)
+    convert(args.notebooks, args.in_place, args.preserve_outputs, False)

nbrmd/contentsmanager.py

@@ -10,11 +10,13 @@
 except ImportError:
     pass
 from notebook.services.contents.filemanager import FileContentsManager
+from tornado.web import HTTPError
 from traitlets import Unicode
 from traitlets.config import Configurable
 import nbrmd
 
 from . import combine
+from .file_format_version import check_file_version
 
 
 def _nbrmd_writes(ext):
@@ -44,7 +46,7 @@ def check_formats(formats):
         formats = [group.split(',') for group in formats.split(';')]
 
     expected_format = ("Notebook metadata 'nbrmd_formats' should "
-                       "be a list of extension groups, like 'ipynb,Rmd'. "
+                       "be a list of extension groups, like 'ipynb,Rmd'.\n"
                        "Groups can be separated with colon, for instance: "
                        "'ipynb,nb.py;script.ipynb,py'")
 
@@ -120,7 +122,10 @@ def format_group(self, fmt, nb=None):
         nbrmd_formats = ((nb.metadata.get('nbrmd_formats') if nb else None)
                          or self.default_nbrmd_formats)
 
-        nbrmd_formats = check_formats(nbrmd_formats)
+        try:
+            nbrmd_formats = check_formats(nbrmd_formats)
+        except ValueError as err:
+            raise HTTPError(400, str(err))
 
         # Find group that contains the current format
         for group in nbrmd_formats:
@@ -184,6 +189,11 @@ def _read_notebook(self, os_path, as_version=4,
         else:
             nb_outputs = None
 
+        try:
+            check_file_version(nb, file + source_format, file + outputs_format)
+        except ValueError as err:
+            raise HTTPError(400, str(err))
+
         if nb_outputs:
             combine.combine_inputs_with_outputs(nb, nb_outputs)
             if self.notary.check_signature(nb_outputs):

nbrmd/file_format_version.py

@@ -0,0 +1,52 @@
+"""Current file format for the various text only notebook extensions"""
+
+import os
+
+FILE_FORMAT_VERSION = {
+    # R markdown format
+    '.Rmd': '1.0',
+    # Version 1.0 on 2018-08-22 - nbrmd v0.5.2 : Initial version
+
+    # Python scripts
+    '.py': '1.0',
+    # Version 1.0 on 2018-08-22 - nbrmd v0.5.2 : Initial version
+
+    # Python scripts
+    '.R': '1.0',
+    # Version 1.0 on 2018-08-22 - nbrmd v0.5.2 : Initial version
+}
+
+
+def file_format_version(ext):
+    """Return file format version for given ext"""
+    return FILE_FORMAT_VERSION.get(ext)
+
+
+def check_file_version(nb, source_path, outputs_path):
+    """Raise if file version in source file would override outputs"""
+    _, ext = os.path.splitext(source_path)
+    current = file_format_version(ext)
+    version = nb.metadata.get('nbrmd_file_format_version')
+    if version:
+        del nb.metadata['nbrmd_file_format_version']
+
+    # Missing version, still generated by nbrmd?
+    if nb.metadata and not version:
+        version = current
+
+    # Same version? OK
+    if version == current:
+        return
+
+    # Not merging? OK
+    if source_path == outputs_path:
+        return
+
+    raise ValueError("File {} has nbrmd_file_format_version={}, but "
+                     "current version for that extension is {}.\n"
+                     "It would not be safe to override the contents of {} "
+                     "with that file.\n"
+                     "Please remove one or the other file"
+                     .format(os.path.basename(source_path),
+                             version, current,
+                             os.path.basename(outputs_path)))

nbrmd/header.py

@@ -5,6 +5,7 @@
 import yaml
 import nbformat
 from nbformat.v4.nbbase import new_raw_cell
+from .file_format_version import file_format_version
 
 _HEADER_RE = re.compile(r"^---\s*$")
 _BLANK_RE = re.compile(r"^\s*$")
@@ -70,6 +71,9 @@ def metadata_and_cell_to_header(self, nb):
 
     metadata = _as_dict(nb.get('metadata', {}))
 
+    if file_format_version(self.ext):
+        metadata['nbrmd_format_version'] = file_format_version(self.ext)
+
     if metadata:
         header.extend(yaml.safe_dump({'jupyter': metadata},
                                      default_flow_style=False).splitlines())

notebook.ipynb

@@ -0,0 +1,8 @@
+{
+ "cells": [],
+ "metadata": {
+  "nbrmd_formats": "ipynb,py"
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebook.nb.py

@@ -0,0 +1,7 @@
+---
+jupyter:
+  nbrmd_formats: ipynb,py
+  nbrmd_file_format_version: '1.0'
+---
+
+# New cell

tests/test_active_cells.py

@@ -3,6 +3,8 @@
 import nbrmd
 from testfixtures import compare
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
 ACTIVE_ALL = {'.py': """# + {"active": "ipynb,py,R,Rmd"}
 # This cell is active in all extensions
 """,

tests/test_cli.py

@@ -2,9 +2,15 @@
 from shutil import copyfile
 import pytest
 import nbrmd
-from nbrmd.cli import convert_nbrmd as convert, cli_nbrmd as cli
+from nbrmd.cli import convert as convert_, cli_nbrmd as cli
 from .utils import list_all_notebooks, remove_outputs
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
+
+def convert(*args):
+    return convert_(*args, markdown=True)
+
 
 @pytest.mark.parametrize('nb_file',
                          list_all_notebooks('.ipynb') +

tests/test_cli_nbsrc.py

@@ -2,9 +2,15 @@
 from shutil import copyfile
 import pytest
 import nbrmd
-from nbrmd.cli import convert_nbsrc as convert, cli_nbsrc as cli
+from nbrmd.cli import convert as convert_, cli_nbsrc as cli
 from .utils import list_all_notebooks, remove_outputs
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
+
+def convert(*args):
+    return convert_(*args, markdown=False)
+
 
 @pytest.mark.parametrize('nb_file',
                          list_all_notebooks('.ipynb') +

tests/test_contentsmanager.py

@@ -1,9 +1,12 @@
 import os
 import sys
 import pytest
+import nbrmd
 from nbrmd import RmdFileContentsManager, readf
 from .utils import list_all_notebooks, remove_outputs
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
 
 @pytest.mark.skipif(isinstance(RmdFileContentsManager, str),
                     reason=RmdFileContentsManager)

tests/test_ipynb_to_R.py

@@ -4,6 +4,8 @@
 import nbrmd
 from .utils import list_r_notebooks, remove_outputs
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
 
 @pytest.mark.parametrize('nb_file', list_r_notebooks('.ipynb'))
 def test_identity_source_write_read(nb_file):

tests/test_ipynb_to_py.py

@@ -4,6 +4,8 @@
 import nbrmd
 from .utils import list_all_notebooks, remove_outputs
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
 
 @pytest.mark.parametrize('nb_file', list_all_notebooks('.ipynb'))
 def test_identity_source_write_read(nb_file):

tests/test_ipynb_to_rmd.py

@@ -5,6 +5,8 @@
 from .utils import list_all_notebooks, remove_outputs, \
     remove_outputs_and_header
 
+nbrmd.file_format_version.FILE_FORMAT_VERSION = {}
+
 
 @pytest.mark.parametrize('nb_file', list_all_notebooks('.ipynb'))
 def test_identity_source_write_read(nb_file):