Cannot load file from package directory using pathlib.Path()

[michaelweinold]

The pint package is used for physical units conversion. The underlying data is stored in .txt files that ship with the package, eg.: pint/default_en.txt.

This works fine in Pyodide, but using the xeus-python-kernel:

from pint import DimensionalityError, Quantity, UndefinedUnitError, UnitRegistry
UnitRegistry()

I get an error:

FileNotFoundError: [Errno 44] No such file or directory: '/lib/python3.10/site-packages/pint/default_en.txt'

Full Error Message

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[12], line 1
----> 1 UnitRegistry()

File /lib/python3.10/site-packages/pint/facets/plain/registry.py:138, in RegistryMeta.__call__(self, *args, **kwargs)
    136 def __call__(self, *args, **kwargs):
    137     obj = super().__call__(*args, **kwargs)
--> 138     obj._after_init()
    139     return obj

File /lib/python3.10/site-packages/pint/facets/system/registry.py:78, in GenericSystemRegistry._after_init(self)
     72 def _after_init(self) -> None:
     73     """Invoked at the end of ``__init__``.
     74 
     75     - Create default group and add all orphan units to it
     76     - Set default system
     77     """
---> 78     super()._after_init()
     80     #: System name to be used by default.
     81     self._default_system_name = self._default_system_name or self._defaults.get(
     82         "system", None
     83     )

File /lib/python3.10/site-packages/pint/facets/group/registry.py:64, in GenericGroupRegistry._after_init(self)
     58 def _after_init(self) -> None:
     59     """Invoked at the end of ``__init__``.
     60 
     61     - Create default group and add all orphan units to it
     62     - Set default system
     63     """
---> 64     super()._after_init()
     66     #: Copy units not defined in any group to the default group
     67     if "group" in self._defaults:

File /lib/python3.10/site-packages/pint/facets/plain/registry.py:307, in GenericPlainRegistry._after_init(self)
    305 if self._filename == "":
    306     path = pathlib.Path(__file__).parent.parent.parent / "default_en.txt"
--> 307     loaded_files = self.load_definitions(path, True)
    308 elif self._filename is not None:
    309     loaded_files = self.load_definitions(self._filename)

File /lib/python3.10/site-packages/pint/facets/plain/registry.py:550, in GenericPlainRegistry.load_definitions(self, file, is_resource)
    548     parsed_project = self._def_parser.parse_string("\n".join(file))
    549 else:
--> 550     parsed_project = self._def_parser.parse_file(file)
    552 for definition in self._def_parser.iter_parsed_project(parsed_project):
    553     self._helper_dispatch_adder(definition)

File /lib/python3.10/site-packages/pint/delegates/txt_defparser/defparser.py:137, in DefParser.parse_file(self, filename, cfg)
    134 def parse_file(
    135     self, filename: Union[pathlib.Path, str], cfg: Optional[ParserConfig] = None
    136 ):
--> 137     return fp.parse(
    138         filename,
    139         _PintParser,
    140         cfg or self._default_config,
    141         diskcache=self._diskcache,
    142     )

File /lib/python3.10/site-packages/pint/_vendor/flexparser.py:1393, in parse(entry_point, spec, config, strip_spaces, delimiters, locator, prefer_resource_as_file, **extra_parser_kwargs)
   1385 elif not (isinstance(entry_point, tuple) and len(entry_point) == 2):
   1386     raise TypeError(
   1387         f"Cannot handle type {type(entry_point)}, "
   1388         "use str or pathlib.Path for files or "
   1389         "(package: str, resource_name: str) tuple "
   1390         "for a resource."
   1391     )
-> 1393 pp[None] = parsed = parser.parse(entry_point)
   1394 pending.extend(
   1395     (parsed.location, el.target)
   1396     for el in parsed.parsed_source.filter_by(IncludeStatement)
   1397 )
   1399 while pending:

File /lib/python3.10/site-packages/pint/_vendor/flexparser.py:1010, in Parser.parse(self, source_location)
   1007     return self.parse_file(pathlib.Path(source_location))
   1009 if isinstance(source_location, pathlib.Path):
-> 1010     return self.parse_file(source_location)
   1012 raise TypeError(
   1013     f"Unknown type {type(source_location)}, "
   1014     "use str or pathlib.Path for files or "
   1015     "(package: str, resource_name: str) tuple "
   1016     "for a resource."
   1017 )

File /lib/python3.10/site-packages/pint/delegates/txt_defparser/defparser.py:77, in _PintParser.parse_file(self, path)
     75 def parse_file(self, path: pathlib.Path) -> PintSource:
     76     if self._diskcache is None:
---> 77         return super().parse_file(path)
     78     content, basename = self._diskcache.load(path, super().parse_file)
     79     return content

File /lib/python3.10/site-packages/pint/_vendor/flexparser.py:1042, in Parser.parse_file(self, path)
   1034 def parse_file(self, path: pathlib.Path) -> ParsedSource[RBT, CT]:
   1035     """Parse a file into a ParsedSourceFile.
   1036 
   1037     Parameters
   (...)
   1040         path of the file.
   1041     """
-> 1042     with path.open(mode="rb") as fi:
   1043         content = fi.read()
   1045     bos = BOF(
   1046         Hash.from_bytes(self._hasher, content), path, path.stat().st_mtime
   1047     ).set_simple_position(0, 0, 0)

File /lib/python3.10/pathlib.py:1117, in Path.open(self, mode, buffering, encoding, errors, newline)
   1115 if "b" not in mode:
   1116     encoding = io.text_encoding(encoding)
-> 1117 return self._accessor.open(self, mode, buffering, encoding, errors,
   1118                            newline)

FileNotFoundError: [Errno 44] No such file or directory: '/lib/python3.10/site-packages/pint/default_en.txt'

This is because the package attempts to load the file using pathlib.Path(__file__).parent.parent.parent / "default_en.txt":

def _after_init(self) -> None:
        """This should be called after all __init__"""

        if self._filename == "":
            path = pathlib.Path(__file__).parent.parent.parent / "default_en.txt"
            loaded_files = self.load_definitions(path, True)
        elif self._filename is not None:
            loaded_files = self.load_definitions(self._filename)
        else:
            loaded_files = None

        self._build_cache(loaded_files)
        self._initialized = True

Why is this working fine in Pyodide, but not in the xeus-python-kernel?

See also:

brightway-lca/brightway-live#52

[martinRenou]

So this is definitely empack filtering out the txt files. You can workaround it by trying the approach suggested here: jupyterlite/xeus-python-kernel#79 (comment)

Probably with something like:

  pint:
    include_patterns:
      - pattern: '*.so'
      - pattern: '*.py'
      - pattern: '*.txt'
    exclude_patterns:
      - pattern: '**/tests/**/*.py'
      - pattern: '**/tests/**/*.so'

[martinRenou]

In the long run, I think empack should not filter out too much by default, and only filter out things that it knows for sure should be filtered (tests, library-specific well known useless files)

[michaelweinold]

Ok, thank you - I'll try the workaround. In the meantime, we've made pint optional in our dependency list. I believe it is best to close the issue. You can always reference it when you change the way empack filters package files.