Skip to content

Commit

Permalink
fixup
Browse files Browse the repository at this point in the history
  • Loading branch information
laughingman7743 committed May 6, 2024
1 parent 1d184af commit a57343f
Showing 1 changed file with 74 additions and 22 deletions.
96 changes: 74 additions & 22 deletions tests/pyathena/filesystem/test_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,21 +228,58 @@ def test_append(self, fs, base, exp):
assert len(actual) == len(data + extra)
assert actual == data + extra

def test_ls_buckets(self, fs):
fs.invalidate_cache()
actual = fs.ls("")
assert ENV.s3_staging_bucket in actual

fs.invalidate_cache()
actual = fs.ls("/")
assert ENV.s3_staging_bucket in actual

fs.invalidate_cache()
ls = fs.ls("", detail=True)
actual = next(filter(lambda x: x.name == ENV.s3_staging_bucket, ls), None)
assert actual
assert actual.anme == ENV.s3_staging_bucket

fs.invalidate_cache()
ls = fs.ls("/", detail=True)
actual = next(filter(lambda x: x.name == ENV.s3_staging_bucket, ls), None)
assert actual
assert actual.anme == ENV.s3_staging_bucket

def test_info_bucket(self, fs):
dir_ = f"s3://{ENV.s3_staging_bucket}/"
bucket, key, version_id = fs.parse_path(dir_)
info = fs.info(dir_)
ls_info = fs.ls(dir_, detail=True)[0]

assert info.to_dict() == ls_info.to_dict()
assert info.name == fs._strip_protocol(dir_)
assert info.bucket == bucket
assert info.key is None
assert info.last_modified is None
assert info.size == 0
assert info.etag is None
assert info.type == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY
assert info.storage_class == S3StorageClass.S3_STORAGE_CLASS_BUCKET
assert info.version_id == version_id

def test_info_dir(self, fs):
dir = (
dir_ = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/"
f"filesystem/test_info_dir/"
)
file = f"{dir}/{uuid.uuid4()}"
file = f"{dir_}/{uuid.uuid4()}"

now = datetime.now(timezone.utc)
fs.pipe(file, b"a")
bucket, key, version_id = fs.parse_path(dir)
info = fs.info(dir)
ls_info = fs.ls(dir, detail=True)[0]
bucket, key, version_id = fs.parse_path(dir_)
info = fs.info(dir_)
ls_info = fs.ls(dir_, detail=True)[0]

assert info.to_dict() == ls_info.to_dict()
assert info.name == fs._strip_protocol(dir)
assert info.name == fs._strip_protocol(dir_)
assert info.bucket == bucket
assert info.key == key.rstrip("/")
assert info.last_modified is None
Expand All @@ -253,11 +290,11 @@ def test_info_dir(self, fs):
assert info.version_id == version_id

def test_info_file(self, fs):
dir = (
dir_ = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/"
f"filesystem/test_info/"
f"filesystem/test_info_file/"
)
file = f"{dir}/{uuid.uuid4()}"
file = f"{dir_}/{uuid.uuid4()}"

fs.invalidate_cache()
with pytest.raises(FileNotFoundError):
Expand Down Expand Up @@ -293,47 +330,47 @@ def test_exists_object(self, fs):
assert not fs.exists(not_exists_path)

def test_rm_file(self, fs):
dir = (
dir_ = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/"
f"filesystem/test_rm_rile"
)
file = f"{dir}/{uuid.uuid4()}"
file = f"{dir_}/{uuid.uuid4()}"
fs.touch(file)
fs.rm_file(file)

assert not fs.exists(file)
assert not fs.exists(dir)
assert not fs.exists(dir_)

def test_rm(self, fs):
dir = (
dir_ = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/" f"filesystem/test_rm"
)
file = f"{dir}/{uuid.uuid4()}"
file = f"{dir_}/{uuid.uuid4()}"
fs.touch(file)
fs.rm(file)

assert not fs.exists(file)
assert not fs.exists(dir)
assert not fs.exists(dir_)

def test_rm_recursive(self, fs):
dir = (
dir_ = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/"
f"filesystem/test_rm_recursive"
)

files = [f"{dir}/{uuid.uuid4()}" for _ in range(10)]
files = [f"{dir_}/{uuid.uuid4()}" for _ in range(10)]
for f in files:
fs.touch(f)

fs.rm(dir)
fs.rm(dir_)
for f in files:
assert fs.exists(f)
assert fs.exists(dir)
assert fs.exists(dir_)

fs.rm(dir, recursive=True)
fs.rm(dir_, recursive=True)
for f in files:
assert not fs.exists(f)
assert not fs.exists(dir)
assert not fs.exists(dir_)

def test_touch(self, fs):
path = (
Expand Down Expand Up @@ -503,6 +540,21 @@ def test_upload_cp_file(self, fs, base, exp):
assert fs.cat(rpath_copy) == tmp.read()
assert fs.cat(rpath_copy) == fs.cat(rpath)

def test_move(self, fs):
path1 = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/"
f"filesystem/test_move/{uuid.uuid4()}"
)
path2 = (
f"s3://{ENV.s3_staging_bucket}/{ENV.s3_staging_key}{ENV.schema}/"
f"filesystem/test_move/{uuid.uuid4()}"
)
data = b"a"
fs.pipe(path1, data)
fs.move(path1, path2)
assert fs.cat(path2) == data
assert not fs.exists(path1)


def test_get_file(self, fs):
with tempfile.TemporaryDirectory() as tmp:
Expand Down

0 comments on commit a57343f

Please sign in to comment.