Skip to content

Commit

Permalink
stop
Browse files Browse the repository at this point in the history
  • Loading branch information
martindurant committed Sep 4, 2024
1 parent 2de7c7b commit dcf3cfc
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 2 deletions.
7 changes: 5 additions & 2 deletions fsspec/implementations/reference.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,11 @@ def open_refs(field, record):
"""cached parquet file loader"""
path = self.url.format(field=field, record=record)
data = io.BytesIO(self.fs.cat_file(path))
df = self.pd.read_parquet(data, engine="fastparquet")
refs = {c: df[c].values for c in df.columns}
try:
df = self.pd.read_parquet(data, engine="fastparquet")
refs = {c: df[c].values for c in df.columns}
except IOError:
refs = None
return refs

self.open_refs = open_refs
Expand Down
21 changes: 21 additions & 0 deletions fsspec/implementations/tests/test_reference.py
Original file line number Diff line number Diff line change
Expand Up @@ -783,3 +783,24 @@ def test_deep_parq(m):
"instant/one/.zarray",
"instant/one/0",
]


def test_parquet_no_data(m):
zarr = pytest.importorskip("zarr")
lz = fsspec.implementations.reference.LazyReferenceMapper.create(
"memory://out.parq", fs=m
)

g = zarr.open_group(lz, mode="w")
arr = g.create_dataset(
name="one",
dtype="int32",
shape=(10,),
chunks=(5,),
compression=None,
fill_value=1,
)
lz.flush()

breakpoint()
assert (arr[:] == 1).all()

0 comments on commit dcf3cfc

Please sign in to comment.