From 7f24b9bfd6a4ab87c7c5d3e7daf4bd45bf77776e Mon Sep 17 00:00:00 2001 From: Joel Lamy-Poirier Date: Thu, 16 Jan 2025 18:24:23 -0500 Subject: [PATCH] fix --- fast_llm/data/dataset/gpt/indexed.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/fast_llm/data/dataset/gpt/indexed.py b/fast_llm/data/dataset/gpt/indexed.py index 833c569..85219e5 100644 --- a/fast_llm/data/dataset/gpt/indexed.py +++ b/fast_llm/data/dataset/gpt/indexed.py @@ -11,20 +11,6 @@ class GPTIndexedDataset(IndexedDataset): - """ - A GPT dataset containing a list of samples. - """ - - # def get(self, index: int, offset: int = 0, length: int | None = None): - # pass - - # def __len__(self) -> int: - # """ - # Number of documents in the dataset. - # Can be calculated from document sizes but may be overridden if there is a better method. - # """ - # return len(self.get_document_sizes()) - @abc.abstractmethod def get_document_sizes(self) -> np.ndarray: """