#!/usr/bin/env python3
-
""" Base module for most of the DASF Datasets. """
import json
@@ -104,14 +103,14 @@ Source code for dasf.datasets.base
# This is just to enable Xarray Cupy capabilities
import cupy_xarray as cx # noqa
import dask_cudf as dcudf
-except ImportError: # pragma: no cover
+except ImportError: # pragma: no cover
pass
try:
- import numcodecs
+ import numcodecs # noqa
from kvikio.nvcomp_codec import NvCompBatchCodec
from kvikio.zarr import GDSStore
-except ImportError: # pragma: no cover
+except ImportError: # pragma: no cover
pass
from pathlib import Path
@@ -131,7 +130,8 @@ Source code for dasf.datasets.base
[docs]
class Dataset(TargeteredTransform):
- """Class representing a generic dataset based on a TargeteredTransform
+ """
+ Class representing a generic dataset based on a TargeteredTransform
object.
Parameters
@@ -154,7 +154,10 @@ Source code for dasf.datasets.base
root: str = None,
*args,
**kwargs):
- """ Constructor of the object Dataset. """
+ """
+ Constructor of the object Dataset.
+
+ """
super().__init__(*args, **kwargs)
# Dataset internals
@@ -170,7 +173,8 @@ Source code for dasf.datasets.base
self.download()
def __set_dataset_cache_dir(self):
- """Generate cached directory in $HOME to store dataset(s).
+ """
+ Generate cached directory in $HOME to store dataset(s).
"""
self._cache_dir = os.path.abspath(str(Path.home()) + "/.cache/dasf/datasets/")
@@ -182,7 +186,8 @@ Source code for dasf.datasets.base
[docs]
def download(self):
- """Skeleton of the download method.
+ """
+ Skeleton of the download method.
"""
if self._download:
@@ -192,7 +197,8 @@ Source code for dasf.datasets.base
[docs]
def __len__(self) -> int:
- """Return internal data length.
+ """
+ Return internal data length.
"""
if self._data is None:
@@ -204,7 +210,8 @@ Source code for dasf.datasets.base
[docs]
def __getitem__(self, idx):
- """Generic __getitem__() function based on internal data.
+ """
+ Generic __getitem__() function based on internal data.
Parameters
----------
@@ -223,7 +230,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetArray(Dataset):
- """Class representing an dataset wich is defined as an array of a defined
+ """
+ Class representing an dataset wich is defined as an array of a defined
shape.
Parameters
@@ -243,7 +251,9 @@ Source code for dasf.datasets.base
download: bool = False,
root: str = None,
chunks="auto"):
- """ Constructor of the object DatasetArray. """
+ """
+ Constructor of the object DatasetArray.
+ """
Dataset.__init__(self, name, download, root)
self._chunks = chunks
@@ -259,8 +269,9 @@ Source code for dasf.datasets.base
[docs]
def __operator_check__(self, other):
- """Check what type of the data we are handling
-
+ """
+ Check what type of the data we are handling
+
Examples:
DatasetArray with array-like; or
DatasetArray with DatasetArray
@@ -284,8 +295,8 @@ Source code for dasf.datasets.base
[docs]
def __repr__(self):
- """Return a class representation based on internal array.
-
+ """
+ Return a class representation based on internal array.
"""
return repr(self._data)
@@ -293,7 +304,8 @@ Source code for dasf.datasets.base
[docs]
def __array__(self, dtype=None):
- """Array interface is required to support most of the array functions.
+ """
+ Array interface is required to support most of the array functions.
Parameters
----------
@@ -312,7 +324,34 @@ Source code for dasf.datasets.base
[docs]
def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
-
+ """
+ Any class, array subclass or not, can define this method or set
+ it to None in order to override the behavior of Arrays ufuncs.
+
+ Parameters
+ ----------
+ ufunc : Callable
+ The ufunc object that was called.
+
+ method : Str
+ A string indicating which Ufunc method was called (one of
+ "__call__", "reduce", "reduceat", "accumulate", "outer", "inner").
+
+ inputs : Any
+ A tuple of the input arguments to the ufunc.
+
+ kwargs : Any
+ A dictionary containing the optional input arguments of the ufunc.
+ If given, any out arguments, both positional and keyword, are
+ passed as a tuple in kwargs. See the discussion in Universal
+ functions (ufunc) for details.
+
+ Returns
+ -------
+ array : array-like
+ The return either the result of the operation.
+
+ """
assert self._data is not None, "Data is not loaded yet."
if method == '__call__':
scalars = []
@@ -332,7 +371,8 @@ Source code for dasf.datasets.base
def __check_op_input(self, in_data):
- """Return the proper type of data for operation
+ """
+ Return the proper type of data for operation
>>> Result = DatasetArray + Numpy; or
>>> Result = DatasetArray + DatasetArray
@@ -357,7 +397,8 @@ Source code for dasf.datasets.base
[docs]
def __add__(self, other):
- """Internal function of adding two array datasets.
+ """
+ Internal function of adding two array datasets.
Parameters
----------
@@ -366,7 +407,7 @@ Source code for dasf.datasets.base
Returns
-------
- DatasetArry
+ DatasetArray
A sum with two arrays.
"""
@@ -378,7 +419,8 @@ Source code for dasf.datasets.base
[docs]
def __sub__(self, other):
- """Internal function of subtracting two array datasets.
+ """
+ Internal function of subtracting two array datasets.
Parameters
----------
@@ -399,7 +441,8 @@ Source code for dasf.datasets.base
[docs]
def __mul__(self, other):
- """Internal function of multiplication two array datasets.
+ """
+ Internal function of multiplication two array datasets.
Parameters
----------
@@ -420,7 +463,8 @@ Source code for dasf.datasets.base
[docs]
def __div__(self, other):
- """Internal function of division two array datasets.
+ """
+ Internal function of division two array datasets.
Parameters
----------
@@ -439,7 +483,8 @@ Source code for dasf.datasets.base
def __copy_attrs_from_data(self):
- """Extends metadata to new transformed object (after operations).
+ """
+ Extends metadata to new transformed object (after operations).
"""
self._metadata["type"] = type(self._data)
@@ -451,7 +496,8 @@ Source code for dasf.datasets.base
self.__dict__[attr] = getattr(self._data, attr)
def __npy_header(self):
- """Read an array header from a filelike object.
+ """
+ Read an array header from a filelike object.
"""
with open(self._root_file, 'rb') as fobj:
@@ -463,7 +509,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load(self, xp, **kwargs):
- """Lazy load the dataset using an CPU dask container.
+ """
+ Lazy load the dataset using an CPU dask container.
Parameters
----------
@@ -482,7 +529,8 @@ Source code for dasf.datasets.base
local_data = dask.delayed(xp.load)(self._root_file, **kwargs)
- local_data = da.from_delayed(local_data, shape=npy_shape, dtype=xp.float32, meta=xp.array(()))
+ local_data = da.from_delayed(local_data, shape=npy_shape,
+ dtype=xp.float32, meta=xp.array(()))
if isinstance(self._chunks, tuple):
local_data = local_data.rechunk(self._chunks)
@@ -492,7 +540,8 @@ Source code for dasf.datasets.base
[docs]
def _load(self, xp, **kwargs):
- """Load data using CPU container.
+ """
+ Load data using CPU container.
Parameters
----------
@@ -509,7 +558,8 @@ Source code for dasf.datasets.base
[docs]
def _load_meta(self) -> dict:
- """Load metadata to inspect.
+ """
+ Load metadata to inspect.
Returns
-------
@@ -517,18 +567,19 @@ Source code for dasf.datasets.base
A dictionary with metadata information.
"""
- assert self._root_file is not None, ("There is no temporary file to "
- "inspect")
- assert os.path.isfile(self._root_file), ("The root variable should "
- "be a NPY file")
+ assert self._data is not None or self._root_file is not None, \
+ ("There is no temporary file to inspect")
+ assert self._data is not None or os.path.isfile(self._root_file), \
+ ("The root variable should be a NPY file")
- return self.inspect_metadata()
+ return self.metadata()
[docs]
def _lazy_load_gpu(self):
- """Load data with GPU container + DASK. (It does not load immediattly)
+ """
+ Load data with GPU container + DASK. (It does not load immediattly)
"""
self._metadata = self._load_meta()
@@ -539,7 +590,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_cpu(self):
- """Load data with CPU container + DASK. (It does not load immediattly)
+ """
+ Load data with CPU container + DASK. (It does not load immediattly)
"""
self._metadata = self._load_meta()
@@ -550,7 +602,8 @@ Source code for dasf.datasets.base
[docs]
def _load_gpu(self):
- """Load data with GPU container (e.g. cupy).
+ """
+ Load data with GPU container (e.g. cupy).
"""
self._metadata = self._load_meta()
@@ -561,7 +614,8 @@ Source code for dasf.datasets.base
[docs]
def _load_cpu(self):
- """Load data with CPU container (e.g. numpy).
+ """
+ Load data with CPU container (e.g. numpy).
"""
self._metadata = self._load_meta()
@@ -572,7 +626,8 @@ Source code for dasf.datasets.base
[docs]
def from_array(self, array):
- """Load data from an existing array.
+ """
+ Load data from an existing array.
Parameters
----------
@@ -587,7 +642,8 @@ Source code for dasf.datasets.base
[docs]
@task_handler
def load(self):
- """Placeholder for load function.
+ """
+ Placeholder for load function.
"""
...
@@ -595,7 +651,8 @@ Source code for dasf.datasets.base
@property
def shape(self) -> tuple:
- """Returns the shape of an array.
+ """
+ Returns the shape of an array.
Returns
-------
@@ -603,15 +660,16 @@ Source code for dasf.datasets.base
A tuple with the shape.
"""
- if self._data:
+ if self._data is not None:
return self._data.shape
return self.__npy_header()[0]
-
-[docs]
- def inspect_metadata(self) -> dict:
- """Return a dictionary with all metadata information from data.
+
+[docs]
+ def metadata(self) -> dict:
+ """
+ Return a dictionary with all metadata information from data.
Returns
-------
@@ -619,10 +677,14 @@ Source code for dasf.datasets.base
A dictionary with metadata information.
"""
- array_file_size = human_readable_size(
- os.path.getsize(self._root_file),
- decimal=2
- )
+ if self._root_file is not None:
+ size = os.path.getsize(self._root_file)
+ elif self._data is not None:
+ size = self._data.size
+ else:
+ size = 0
+
+ array_file_size = human_readable_size(size, decimal=2)
npy_shape = self.shape
@@ -639,7 +701,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetZarr(Dataset):
- """Class representing an dataset wich is defined as a Zarr array of a
+ """
+ Class representing an dataset wich is defined as a Zarr array of a
defined shape.
Parameters
@@ -677,7 +740,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load(self, xp, **kwargs):
- """Lazy load the dataset using an CPU dask container.
+ """
+ Lazy load the dataset using an CPU dask container.
Parameters
----------
@@ -693,8 +757,8 @@ Source code for dasf.datasets.base
"""
if (self._backend == "kvikio" and is_kvikio_supported() and
- (is_gds_supported() or is_kvikio_compat_mode())
- and is_nvcomp_codec_supported()):
+ (is_gds_supported() or is_kvikio_compat_mode()) and
+ is_nvcomp_codec_supported()):
store = GDSStore(self._root_file)
meta = json.loads(store[".zarray"])
meta["compressor"] = NvCompBatchCodec("lz4").get_config()
@@ -709,7 +773,8 @@ Source code for dasf.datasets.base
[docs]
def _load(self, xp, **kwargs):
- """Load data using CPU container.
+ """
+ Load data using CPU container.
Parameters
----------
@@ -793,7 +858,7 @@ Source code for dasf.datasets.base
"""
assert self._root_file is not None, "There is no temporary file to inspect"
- return self.inspect_metadata()
+ return self.metadata()
def __read_zarray(self, key):
@@ -813,7 +878,8 @@ Source code for dasf.datasets.base
@property
def shape(self) -> tuple:
- """Returns the shape of an array.
+ """
+ Returns the shape of an array.
Returns
-------
@@ -831,7 +897,8 @@ Source code for dasf.datasets.base
@property
def chunksize(self):
- """Returns the chunksize of an array.
+ """
+ Returns the chunksize of an array.
Returns
-------
@@ -847,10 +914,11 @@ Source code for dasf.datasets.base
return self._data.chunksize
-
-[docs]
- def inspect_metadata(self) -> dict:
- """Return a dictionary with all metadata information from data.
+
+[docs]
+ def metadata(self) -> dict:
+ """
+ Return a dictionary with all metadata information from data.
Returns
-------
@@ -885,14 +953,16 @@ Source code for dasf.datasets.base
[docs]
def __repr__(self):
- """Return a class representation based on internal array.
+ """
+ Return a class representation based on internal array.
"""
return repr(self._data)
def __check_op_input(self, in_data):
- """Return the proper type of data for operation
+ """
+ Return the proper type of data for operation
>>> Result = DatasetZarr + Numpy; or
>>> Result = DatasetZarr + DatasetZarr
@@ -917,7 +987,8 @@ Source code for dasf.datasets.base
[docs]
def __add__(self, other):
- """Internal function of adding two array datasets.
+ """
+ Internal function of adding two array datasets.
Parameters
----------
@@ -938,7 +1009,8 @@ Source code for dasf.datasets.base
[docs]
def __sub__(self, other):
- """Internal function of subtracting two array datasets.
+ """
+ Internal function of subtracting two array datasets.
Parameters
----------
@@ -959,7 +1031,8 @@ Source code for dasf.datasets.base
[docs]
def __mul__(self, other):
- """Internal function of multiplication two array datasets.
+ """
+ Internal function of multiplication two array datasets.
Parameters
----------
@@ -980,7 +1053,8 @@ Source code for dasf.datasets.base
[docs]
def __div__(self, other):
- """Internal function of division two array datasets.
+ """
+ Internal function of division two array datasets.
Parameters
----------
@@ -999,7 +1073,8 @@ Source code for dasf.datasets.base
def __copy_attrs_from_data(self):
- """Extends metadata to new transformed object (after operations).
+ """
+ Extends metadata to new transformed object (after operations).
"""
self._metadata["type"] = type(self._data)
@@ -1015,7 +1090,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetHDF5(Dataset):
- """Class representing an dataset wich is defined as a HDF5 dataset of a
+ """
+ Class representing an dataset wich is defined as a HDF5 dataset of a
defined shape.
Parameters
@@ -1038,7 +1114,10 @@ Source code for dasf.datasets.base
root: str = None,
chunks="auto",
dataset_path: str = None):
- """ Constructor of the object DatasetHDF5. """
+ """
+ Constructor of the object DatasetHDF5.
+
+ """
Dataset.__init__(self, name, download, root)
self._chunks = chunks
@@ -1059,7 +1138,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load(self, xp, **kwargs):
- """Lazy load the dataset using an CPU dask container.
+ """
+ Lazy load the dataset using an CPU dask container.
Parameters
----------
@@ -1082,7 +1162,8 @@ Source code for dasf.datasets.base
[docs]
def _load(self, xp=None, **kwargs):
- """Load data using CPU container.
+ """
+ Load data using CPU container.
Parameters
----------
@@ -1099,7 +1180,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_cpu(self):
- """Load data with CPU container + DASK. (It does not load immediattly)
+ """
+ Load data with CPU container + DASK. (It does not load immediattly)
"""
self._metadata = self._load_meta()
@@ -1110,7 +1192,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_gpu(self):
- """Load data with GPU container + DASK. (It does not load immediattly)
+ """
+ Load data with GPU container + DASK. (It does not load immediattly)
"""
self._metadata = self._load_meta()
@@ -1121,7 +1204,8 @@ Source code for dasf.datasets.base
[docs]
def _load_cpu(self):
- """Load data with CPU container (e.g. numpy).
+ """
+ Load data with CPU container (e.g. numpy).
"""
self._metadata = self._load_meta()
@@ -1132,7 +1216,8 @@ Source code for dasf.datasets.base
[docs]
def _load_gpu(self):
- """Load data with GPU container (e.g. cupy).
+ """
+ Load data with GPU container (e.g. cupy).
"""
self._metadata = self._load_meta()
@@ -1144,7 +1229,8 @@ Source code for dasf.datasets.base
[docs]
@task_handler
def load(self):
- """Placeholder for load function.
+ """
+ Placeholder for load function.
"""
...
@@ -1153,7 +1239,8 @@ Source code for dasf.datasets.base
[docs]
def _load_meta(self) -> dict:
- """Load metadata to inspect.
+ """
+ Load metadata to inspect.
Returns
-------
@@ -1164,13 +1251,14 @@ Source code for dasf.datasets.base
assert self._root_file is not None, "There is no temporary file to inspect"
assert self._dataset_path is not None, "There is no path to fetch data"
- return self.inspect_metadata()
+ return self.metadata()
-
-[docs]
- def inspect_metadata(self) -> dict:
- """Return a dictionary with all metadata information from data.
+
+[docs]
+ def metadata(self) -> dict:
+ """
+ Return a dictionary with all metadata information from data.
Returns
-------
@@ -1198,7 +1286,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetXarray(Dataset):
- """Class representing an dataset wich is defined as a Xarray dataset of a
+ """
+ Class representing an dataset wich is defined as a Xarray dataset of a
defined shape.
Parameters
@@ -1221,7 +1310,10 @@ Source code for dasf.datasets.base
root: str = None,
chunks=None,
data_var=None):
- """ Constructor of the object DatasetXarray. """
+ """
+ Constructor of the object DatasetXarray.
+
+ """
Dataset.__init__(self, name, download, root)
self._chunks = chunks
@@ -1242,7 +1334,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_cpu(self):
- """Load data with CPU container + DASK. (It does not load immediattly)
+ """
+ Load data with CPU container + DASK. (It does not load immediattly)
"""
assert self._chunks is not None, "Lazy operations require chunks"
@@ -1259,7 +1352,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_gpu(self):
- """Load data with GPU container + DASK. (It does not load immediattly)
+ """
+ Load data with GPU container + DASK. (It does not load immediattly)
"""
assert self._chunks is not None, "Lazy operations require chunks"
@@ -1276,7 +1370,8 @@ Source code for dasf.datasets.base
[docs]
def _load_cpu(self):
- """Load data with CPU container (e.g. numpy).
+ """
+ Load data with CPU container (e.g. numpy).
"""
if self._data_var:
@@ -1290,7 +1385,8 @@ Source code for dasf.datasets.base
[docs]
def _load_gpu(self):
- """Load data with GPU container (e.g. cupy).
+ """
+ Load data with GPU container (e.g. cupy).
"""
if self._data_var:
@@ -1305,7 +1401,8 @@ Source code for dasf.datasets.base
[docs]
@task_handler
def load(self):
- """Placeholder for load function.
+ """
+ Placeholder for load function.
"""
...
@@ -1314,7 +1411,8 @@ Source code for dasf.datasets.base
[docs]
def _load_meta(self) -> dict:
- """Load metadata to inspect.
+ """
+ Load metadata to inspect.
Returns
-------
@@ -1324,13 +1422,14 @@ Source code for dasf.datasets.base
"""
assert self._root_file is not None, "There is no temporary file to inspect"
- return self.inspect_metadata()
+ return self.metadata()
-
-[docs]
- def inspect_metadata(self) -> dict:
- """Return a dictionary with all metadata information from data.
+
+[docs]
+ def metadata(self) -> dict:
+ """
+ Return a dictionary with all metadata information from data.
Returns
-------
@@ -1354,7 +1453,8 @@ Source code for dasf.datasets.base
[docs]
def __len__(self) -> int:
- """Return internal data length.
+ """
+ Return internal data length.
"""
if self._data is None:
@@ -1369,7 +1469,8 @@ Source code for dasf.datasets.base
[docs]
def __getitem__(self, idx):
- """A __getitem__() function based on internal Xarray data.
+ """
+ A __getitem__() function based on internal Xarray data.
Parameters
----------
@@ -1392,7 +1493,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetLabeled(Dataset):
- """A class representing a labeled dataset. Each item is a 2-element tuple,
+ """
+ A class representing a labeled dataset. Each item is a 2-element tuple,
where the first element is a array of data and the second element is the
respective label. The items can be accessed from `dataset[x]`.
@@ -1418,7 +1520,10 @@ Source code for dasf.datasets.base
download: bool = False,
root: str = None,
chunks="auto"):
- """ Constructor of the object DatasetLabeled. """
+ """
+ Constructor of the object DatasetLabeled.
+
+ """
Dataset.__init__(self, name, download, root)
self._chunks = chunks
@@ -1426,7 +1531,8 @@ Source code for dasf.datasets.base
[docs]
def download(self):
- """Download the dataset.
+ """
+ Download the dataset.
"""
if hasattr(self, "_train") and hasattr(self._train, "download"):
@@ -1436,10 +1542,11 @@ Source code for dasf.datasets.base
self._val.download()
-
-[docs]
- def inspect_metadata(self) -> dict:
- """Return a dictionary with all metadata information from data
+
+[docs]
+ def metadata(self) -> dict:
+ """
+ Return a dictionary with all metadata information from data
(train and labels).
Returns
@@ -1448,8 +1555,8 @@ Source code for dasf.datasets.base
A dictionary with metadata information.
"""
- metadata_train = self._train.inspect_metadata()
- metadata_val = self._val.inspect_metadata()
+ metadata_train = self._train.metadata()
+ metadata_val = self._val.metadata()
assert (
metadata_train["shape"] == metadata_val["shape"]
@@ -1465,7 +1572,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load(self, xp, **kwargs) -> tuple:
- """Lazy load the dataset using an CPU dask container.
+ """
+ Lazy load the dataset using an CPU dask container.
Parameters
----------
@@ -1489,7 +1597,8 @@ Source code for dasf.datasets.base
[docs]
def _load(self, xp, **kwargs) -> tuple:
- """Load data using CPU container.
+ """
+ Load data using CPU container.
Parameters
----------
@@ -1513,7 +1622,8 @@ Source code for dasf.datasets.base
[docs]
def _load_meta(self) -> dict:
- """Load metadata to inspect.
+ """
+ Load metadata to inspect.
Returns
-------
@@ -1534,13 +1644,14 @@ Source code for dasf.datasets.base
"The root variable should be a file"
)
- return self.inspect_metadata()
+ return self.metadata()
[docs]
def _lazy_load_gpu(self):
- """Load data with GPU container + DASK. (It does not load immediattly)
+ """
+ Load data with GPU container + DASK. (It does not load immediattly)
"""
self._metadata = self._load_meta()
@@ -1550,7 +1661,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_cpu(self):
- """Load data with CPU container + DASK. (It does not load immediattly)
+ """
+ Load data with CPU container + DASK. (It does not load immediattly)
"""
self._metadata = self._load_meta()
@@ -1560,7 +1672,8 @@ Source code for dasf.datasets.base
[docs]
def _load_gpu(self):
- """Load data with GPU container (e.g. cupy).
+ """
+ Load data with GPU container (e.g. cupy).
"""
self._metadata = self._load_meta()
@@ -1570,7 +1683,8 @@ Source code for dasf.datasets.base
[docs]
def _load_cpu(self):
- """Load data with CPU container (e.g. numpy).
+ """
+ Load data with CPU container (e.g. numpy).
"""
self._metadata = self._load_meta()
@@ -1581,7 +1695,8 @@ Source code for dasf.datasets.base
[docs]
@task_handler
def load(self):
- """Placeholder for load function.
+ """
+ Placeholder for load function.
"""
...
@@ -1590,7 +1705,8 @@ Source code for dasf.datasets.base
[docs]
def __getitem__(self, idx):
- """A __getitem__() function for data and labeled data together.
+ """
+ A __getitem__() function for data and labeled data together.
Parameters
----------
@@ -1609,7 +1725,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetDataFrame(Dataset):
- """Class representing an dataset wich is defined as a dataframe.
+ """
+ Class representing an dataset wich is defined as a dataframe.
Parameters
----------
@@ -1628,7 +1745,10 @@ Source code for dasf.datasets.base
download: bool = True,
root: str = None,
chunks="auto"):
- """ Constructor of the object DatasetDataFrame. """
+ """
+ Constructor of the object DatasetDataFrame.
+
+ """
Dataset.__init__(self, name, download, root)
self._chunks = chunks
@@ -1644,7 +1764,8 @@ Source code for dasf.datasets.base
[docs]
def _load_meta(self) -> dict:
- """Load metadata to inspect.
+ """
+ Load metadata to inspect.
Returns
-------
@@ -1656,13 +1777,14 @@ Source code for dasf.datasets.base
"There is no temporary file to inspect"
)
- return self.inspect_metadata()
+ return self.metadata()
-
-[docs]
- def inspect_metadata(self) -> dict:
- """Return a dictionary with all metadata information from data.
+
+[docs]
+ def metadata(self) -> dict:
+ """
+ Return a dictionary with all metadata information from data.
Returns
-------
@@ -1687,7 +1809,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_gpu(self):
- """Load data with GPU container + DASK. (It does not load immediattly)
+ """
+ Load data with GPU container + DASK. (It does not load immediattly)
"""
self._data = dcudf.read_csv(self._root_file)
@@ -1698,7 +1821,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_cpu(self):
- """Load data with CPU container + DASK. (It does not load immediattly)
+ """
+ Load data with CPU container + DASK. (It does not load immediattly)
"""
self._data = ddf.read_csv(self._root_file)
@@ -1709,7 +1833,8 @@ Source code for dasf.datasets.base
[docs]
def _load_gpu(self):
- """Load data with GPU container (e.g. CuDF).
+ """
+ Load data with GPU container (e.g. CuDF).
"""
self._data = cudf.read_csv(self._root_file)
@@ -1720,7 +1845,8 @@ Source code for dasf.datasets.base
[docs]
def _load_cpu(self):
- """Load data with CPU container (e.g. pandas).
+ """
+ Load data with CPU container (e.g. pandas).
"""
self._data = pd.read_csv(self._root_file)
@@ -1732,7 +1858,8 @@ Source code for dasf.datasets.base
[docs]
@task_handler
def load(self):
- """Placeholder for load function.
+ """
+ Placeholder for load function.
"""
...
@@ -1740,7 +1867,8 @@ Source code for dasf.datasets.base
@property
def shape(self) -> tuple:
- """Returns the shape of an array.
+ """
+ Returns the shape of an array.
Returns
-------
@@ -1756,7 +1884,8 @@ Source code for dasf.datasets.base
[docs]
def __len__(self) -> int:
- """Return internal data length.
+ """
+ Return internal data length.
"""
if self._data is None:
raise Exception("Data is not loaded yet")
@@ -1767,7 +1896,8 @@ Source code for dasf.datasets.base
[docs]
def __getitem__(self, idx):
- """A __getitem__() function based on internal dataframe.
+ """
+ A __getitem__() function based on internal dataframe.
Parameters
----------
@@ -1786,7 +1916,8 @@ Source code for dasf.datasets.base
[docs]
class DatasetParquet(DatasetDataFrame):
- """Class representing an dataset wich is defined as a Parquet.
+ """
+ Class representing an dataset wich is defined as a Parquet.
Parameters
----------
@@ -1805,13 +1936,17 @@ Source code for dasf.datasets.base
download: bool = True,
root: str = None,
chunks="auto"):
- """ Constructor of the object DatasetParquet. """
+ """
+ Constructor of the object DatasetParquet.
+
+ """
DatasetDataFrame.__init__(self, name, download, root, chunks)
[docs]
def _lazy_load_gpu(self):
- """Load data with GPU container + DASK. (It does not load immediattly)
+ """
+ Load data with GPU container + DASK. (It does not load immediattly)
"""
self._data = dcudf.read_parquet(self._root_file)
@@ -1822,7 +1957,8 @@ Source code for dasf.datasets.base
[docs]
def _lazy_load_cpu(self):
- """Load data with CPU container + DASK. (It does not load immediattly)
+ """
+ Load data with CPU container + DASK. (It does not load immediattly)
"""
self._data = ddf.read_parquet(self._root_file)
@@ -1833,7 +1969,8 @@ Source code for dasf.datasets.base
[docs]
def _load_gpu(self):
- """Load data with GPU container (e.g. CuDF).
+ """
+ Load data with GPU container (e.g. CuDF).
"""
self._data = cudf.read_parquet(self._root_file)
@@ -1844,7 +1981,8 @@ Source code for dasf.datasets.base
[docs]
def _load_cpu(self):
- """Load data with CPU container (e.g. pandas).
+ """
+ Load data with CPU container (e.g. pandas).
"""
self._data = pd.read_parquet(self._root_file)
diff --git a/docs/_modules/dasf/datasets/datasets.html b/docs/_modules/dasf/datasets/datasets.html
index bdc4693..bc10c02 100644
--- a/docs/_modules/dasf/datasets/datasets.html
+++ b/docs/_modules/dasf/datasets/datasets.html
@@ -90,7 +90,7 @@ Source code for dasf.datasets.datasets