From c1d89fe9a1b06e610f32f823359771557b1ca12a Mon Sep 17 00:00:00 2001 From: Erjia Guan Date: Thu, 21 Apr 2022 14:10:06 -0700 Subject: [PATCH] Fix data-related public API (#368) Summary: Pull Request resolved: https://github.com/pytorch/data/pull/368 This is PR aims to expose the right data-relate API. There are two more changes made in this PR to convert public api to private api `check_lambda_fn` -> `_check_lambda_fn` `deprecation_warning` -> `_deprecation_warning` X-link: https://github.com/pytorch/pytorch/pull/76143 Reviewed By: albanD, NivekT Differential Revision: D35798311 Pulled By: ejguan fbshipit-source-id: b13fded5c88a533c706702fb2070c918c839dca4 --- torchdata/datapipes/iter/transform/callable.py | 6 +++--- torchdata/datapipes/iter/util/cacheholder.py | 4 ++-- torchdata/datapipes/iter/util/combining.py | 12 ++++++------ torchdata/datapipes/iter/util/paragraphaggregator.py | 4 ++-- torchdata/datapipes/iter/util/tararchiveloader.py | 4 ++-- torchdata/datapipes/iter/util/xzfileloader.py | 4 ++-- torchdata/datapipes/iter/util/ziparchiveloader.py | 4 ++-- torchdata/datapipes/map/util/utils.py | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/torchdata/datapipes/iter/transform/callable.py b/torchdata/datapipes/iter/transform/callable.py index 779f95bb0..c1187c451 100644 --- a/torchdata/datapipes/iter/transform/callable.py +++ b/torchdata/datapipes/iter/transform/callable.py @@ -7,7 +7,7 @@ from typing import Callable, Iterator, List, TypeVar from torch.utils.data import functional_datapipe, IterDataPipe -from torch.utils.data.datapipes.utils.common import check_lambda_fn +from torch.utils.data.datapipes.utils.common import _check_lambda_fn T_co = TypeVar("T_co", covariant=True) @@ -59,7 +59,7 @@ def __init__( ) -> None: self.datapipe = datapipe - check_lambda_fn(fn) + _check_lambda_fn(fn) self.fn = fn # type: ignore[assignment] assert batch_size > 0, "Batch size is required to be larger than 0!" @@ -118,7 +118,7 @@ class FlatMapperIterDataPipe(IterDataPipe[T_co]): def __init__(self, datapipe: IterDataPipe, fn: Callable, input_col=None) -> None: self.datapipe = datapipe - check_lambda_fn(fn) + _check_lambda_fn(fn) self.fn = fn # type: ignore[assignment] self.input_col = input_col diff --git a/torchdata/datapipes/iter/util/cacheholder.py b/torchdata/datapipes/iter/util/cacheholder.py index ea8eb7e4b..49b33056e 100644 --- a/torchdata/datapipes/iter/util/cacheholder.py +++ b/torchdata/datapipes/iter/util/cacheholder.py @@ -13,7 +13,7 @@ from functools import partial from typing import Callable, Deque, Dict, Iterator, Optional, TypeVar -from torch.utils.data.datapipes.utils.common import check_lambda_fn, DILL_AVAILABLE +from torch.utils.data.datapipes.utils.common import _check_lambda_fn, DILL_AVAILABLE from torch.utils.data.graph import traverse from torchdata.datapipes import functional_datapipe @@ -160,7 +160,7 @@ def __init__( ): self.source_datapipe = source_datapipe - check_lambda_fn(filepath_fn) + _check_lambda_fn(filepath_fn) filepath_fn = _generator_to_list(filepath_fn) if inspect.isgeneratorfunction(filepath_fn) else filepath_fn if hash_dict is not None and hash_type not in ("sha256", "md5"): diff --git a/torchdata/datapipes/iter/util/combining.py b/torchdata/datapipes/iter/util/combining.py index 3c77d7ace..8d1b435b3 100644 --- a/torchdata/datapipes/iter/util/combining.py +++ b/torchdata/datapipes/iter/util/combining.py @@ -9,7 +9,7 @@ from typing import Callable, Iterator, Optional, TypeVar from torch.utils.data import functional_datapipe, IterDataPipe, MapDataPipe -from torch.utils.data.datapipes.utils.common import check_lambda_fn +from torch.utils.data.datapipes.utils.common import _check_lambda_fn T_co = TypeVar("T_co", covariant=True) @@ -64,14 +64,14 @@ def __init__( raise TypeError(f"ref_datapipe must be a IterDataPipe, but its type is {type(ref_datapipe)} instead.") self.source_datapipe = source_datapipe self.ref_datapipe = ref_datapipe - check_lambda_fn(key_fn) + _check_lambda_fn(key_fn) self.key_fn = key_fn if ref_key_fn is not None: - check_lambda_fn(ref_key_fn) + _check_lambda_fn(ref_key_fn) self.ref_key_fn = key_fn if ref_key_fn is None else ref_key_fn self.keep_key = keep_key if merge_fn is not None: - check_lambda_fn(merge_fn) + _check_lambda_fn(merge_fn) self.merge_fn = merge_fn if buffer_size is not None and buffer_size <= 0: raise ValueError("'buffer_size' is required to be either None or a positive integer.") @@ -153,10 +153,10 @@ def __init__( raise TypeError(f"map_datapipe must be a MapDataPipe, but its type is {type(map_datapipe)} instead.") self.source_iterdatapipe: IterDataPipe = source_iterdatapipe self.map_datapipe: MapDataPipe = map_datapipe - check_lambda_fn(key_fn) + _check_lambda_fn(key_fn) self.key_fn: Callable = key_fn if merge_fn is not None: - check_lambda_fn(merge_fn) + _check_lambda_fn(merge_fn) self.merge_fn: Optional[Callable] = merge_fn self.length: int = -1 diff --git a/torchdata/datapipes/iter/util/paragraphaggregator.py b/torchdata/datapipes/iter/util/paragraphaggregator.py index bb166634a..f258c0ee8 100644 --- a/torchdata/datapipes/iter/util/paragraphaggregator.py +++ b/torchdata/datapipes/iter/util/paragraphaggregator.py @@ -6,7 +6,7 @@ from typing import Callable, Iterator, List, Tuple, TypeVar -from torch.utils.data.datapipes.utils.common import check_lambda_fn +from torch.utils.data.datapipes.utils.common import _check_lambda_fn from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe @@ -44,7 +44,7 @@ class ParagraphAggregatorIterDataPipe(IterDataPipe[Tuple[str, str]]): def __init__(self, source_datapipe: IterDataPipe[Tuple[str, T_co]], joiner: Callable = _default_line_join) -> None: self.source_datapipe: IterDataPipe[Tuple[str, T_co]] = source_datapipe - check_lambda_fn(joiner) + _check_lambda_fn(joiner) self.joiner: Callable = joiner def __iter__(self) -> Iterator[Tuple[str, str]]: diff --git a/torchdata/datapipes/iter/util/tararchiveloader.py b/torchdata/datapipes/iter/util/tararchiveloader.py index d27e38b85..ccb9e0c0f 100644 --- a/torchdata/datapipes/iter/util/tararchiveloader.py +++ b/torchdata/datapipes/iter/util/tararchiveloader.py @@ -10,7 +10,7 @@ from io import BufferedIOBase from typing import cast, IO, Iterable, Iterator, Optional, Tuple -from torch.utils.data.datapipes.utils.common import deprecation_warning +from torch.utils.data.datapipes.utils.common import _deprecation_warning from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe @@ -85,7 +85,7 @@ class TarArchiveReaderIterDataPipe(IterDataPipe[Tuple[str, BufferedIOBase]]): """ def __new__(cls, datapipe: Iterable[Tuple[str, BufferedIOBase]], mode: str = "r:*", length: int = -1): - deprecation_warning( + _deprecation_warning( cls.__name__, deprecation_version="0.4", removal_version="0.6", diff --git a/torchdata/datapipes/iter/util/xzfileloader.py b/torchdata/datapipes/iter/util/xzfileloader.py index 9577bf1f0..ac5c46beb 100644 --- a/torchdata/datapipes/iter/util/xzfileloader.py +++ b/torchdata/datapipes/iter/util/xzfileloader.py @@ -9,7 +9,7 @@ from io import BufferedIOBase from typing import Iterable, Iterator, Tuple -from torch.utils.data.datapipes.utils.common import deprecation_warning +from torch.utils.data.datapipes.utils.common import _deprecation_warning from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe @@ -74,7 +74,7 @@ class XzFileReaderIterDataPipe(IterDataPipe[Tuple[str, BufferedIOBase]]): """ def __new__(cls, datapipe: Iterable[Tuple[str, BufferedIOBase]], length: int = -1): - deprecation_warning( + _deprecation_warning( cls.__name__, deprecation_version="0.4", removal_version="0.6", diff --git a/torchdata/datapipes/iter/util/ziparchiveloader.py b/torchdata/datapipes/iter/util/ziparchiveloader.py index 5da1423a4..ae0c8f836 100644 --- a/torchdata/datapipes/iter/util/ziparchiveloader.py +++ b/torchdata/datapipes/iter/util/ziparchiveloader.py @@ -11,7 +11,7 @@ from io import BufferedIOBase from typing import cast, IO, Iterable, Iterator, Tuple -from torch.utils.data.datapipes.utils.common import deprecation_warning +from torch.utils.data.datapipes.utils.common import _deprecation_warning from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe @@ -86,7 +86,7 @@ class ZipArchiveReaderIterDataPipe(IterDataPipe[Tuple[str, BufferedIOBase]]): """ def __new__(cls, datapipe: Iterable[Tuple[str, BufferedIOBase]], length: int = -1): - deprecation_warning( + _deprecation_warning( cls.__name__, deprecation_version="0.4", removal_version="0.6", diff --git a/torchdata/datapipes/map/util/utils.py b/torchdata/datapipes/map/util/utils.py index 746e09601..0df063415 100644 --- a/torchdata/datapipes/map/util/utils.py +++ b/torchdata/datapipes/map/util/utils.py @@ -9,7 +9,7 @@ from typing import Callable, Dict, Optional from torch.utils.data import IterDataPipe, MapDataPipe -from torch.utils.data.datapipes.utils.common import check_lambda_fn, DILL_AVAILABLE +from torch.utils.data.datapipes.utils.common import _check_lambda_fn, DILL_AVAILABLE if DILL_AVAILABLE: import dill @@ -42,7 +42,7 @@ def __init__(self, datapipe: IterDataPipe, key_value_fn: Optional[Callable] = No if not isinstance(datapipe, IterDataPipe): raise TypeError(f"IterToMapConverter can only apply on IterDataPipe, but found {type(datapipe)}") self.datapipe = datapipe - check_lambda_fn(key_value_fn) + _check_lambda_fn(key_value_fn) self.key_value_fn = key_value_fn # type: ignore[assignment] self._map = None self._length = -1