Skip to content

Commit

Permalink
Fix azure test (#902)
Browse files Browse the repository at this point in the history
Summary:
Fix bug introduced by #849

- Replace `skipIfNoFSSpecS3` by `skipIfNoFSSpecAZ` for tests to access Azure Blob
- Add `adlfs` to conda test requirements.

Pull Request resolved: #902

Reviewed By: NivekT

Differential Revision: D41378561

Pulled By: ejguan

fbshipit-source-id: be99d1371379e1c87c2f2ac062b009f88575aa0a
  • Loading branch information
ejguan authored and facebook-github-bot committed Nov 17, 2022
1 parent 594b30e commit e79ff42
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 2 deletions.
1 change: 1 addition & 0 deletions test/requirements.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ dependencies:
- s3fs
- numpy
- datasets
- adlfs
18 changes: 16 additions & 2 deletions test/test_remote_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,26 @@

try:
import fsspec

HAS_FSSPEC = True
except ImportError:
HAS_FSSPEC = False

try:
import s3fs

HAS_FSSPEC_S3 = True
except ImportError:
HAS_FSSPEC_S3 = False
skipIfNoFSSpecS3 = unittest.skipIf(not HAS_FSSPEC_S3, "no FSSpec with S3fs")
skipIfNoFSSpecS3 = unittest.skipIf(not (HAS_FSSPEC and HAS_FSSPEC_S3), "no FSSpec with S3fs")

try:
import adlfs

HAS_FSSPEC_AZ = True
except ImportError:
HAS_FSSPEC_AZ = False
skipIfNoFSSpecAZ = unittest.skipIf(not (HAS_FSSPEC and HAS_FSSPEC_AZ), "no FSSpec with adlfs")

try:
from torchdata._torchdata import S3Handler
Expand Down Expand Up @@ -213,7 +227,7 @@ def test_fsspec_io_iterdatapipe(self):
res = list(fsspec_loader_dp)
self.assertEqual(len(res), 18, f"{input} failed")

@skipIfNoFSSpecS3
@skipIfNoFSSpecAZ
def test_fsspec_azure_blob(self):
url = "public/curated/covid-19/ecdc_cases/latest/ecdc_cases.csv"
account_name = "pandemicdatalake"
Expand Down

0 comments on commit e79ff42

Please sign in to comment.