From 97d93fc62c0419b8825d2ecc2f354a5aded06d68 Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Thu, 15 Jan 2015 07:37:37 -0800 Subject: [PATCH] rename chunks -> iterable --- into/__init__.py | 2 +- into/backends/bcolz.py | 10 +++---- into/backends/csv.py | 20 ++++++------- into/backends/h5py.py | 10 +++---- into/backends/hdfstore.py | 10 +++---- into/backends/pytables.py | 8 +++--- into/backends/sql.py | 4 +-- into/backends/tests/test_bcolz.py | 8 +++--- into/backends/tests/test_h5py.py | 6 ++-- into/backends/tests/test_hdfstore.py | 8 +++--- into/chunks.py | 22 +++++++-------- into/convert.py | 42 ++++++++++++++-------------- into/tests/test_chunks.py | 24 ++++++++-------- into/tests/test_convert.py | 18 ++++++------ 14 files changed, 96 insertions(+), 96 deletions(-) diff --git a/into/__init__.py b/into/__init__.py index 553f28b7..bc22e2d6 100644 --- a/into/__init__.py +++ b/into/__init__.py @@ -9,7 +9,7 @@ from .resource import resource from .into import into from .drop import drop -from .chunks import chunks, Chunks +from .chunks import iterable, IterableOf from datashape import discover, dshape from collections import Iterator import numpy as np diff --git a/into/backends/bcolz.py b/into/backends/bcolz.py index b03bc50a..744861ca 100644 --- a/into/backends/bcolz.py +++ b/into/backends/bcolz.py @@ -11,7 +11,7 @@ from ..convert import convert, ooc_types from ..resource import resource from ..drop import drop -from ..chunks import chunks +from ..chunks import iterable keywords = ['cparams', 'dflt', 'expectedlen', 'chunklen', 'rootdir'] @@ -30,7 +30,7 @@ def numpy_append_to_bcolz(a, b, **kwargs): @append.register((ctable, carray), object) def numpy_append_to_bcolz(a, b, **kwargs): - return append(a, convert(chunks(np.ndarray), b, **kwargs), **kwargs) + return append(a, convert(iterable(np.ndarray), b, **kwargs), **kwargs) @convert.register(ctable, np.ndarray, cost=2.0) @@ -48,7 +48,7 @@ def convert_bcolz_to_numpy(x, **kwargs): return x[:] -@append.register((carray, ctable), chunks(np.ndarray)) +@append.register((carray, ctable), iterable(np.ndarray)) def append_carray_with_chunks(a, c, **kwargs): for chunk in c: append(a, chunk) @@ -56,7 +56,7 @@ def append_carray_with_chunks(a, c, **kwargs): return a -@convert.register(chunks(np.ndarray), (ctable, carray), cost=1.2) +@convert.register(iterable(np.ndarray), (ctable, carray), cost=1.2) def bcolz_to_numpy_chunks(x, chunksize=2**20, **kwargs): def load(): first_n = min(1000, chunksize) @@ -64,7 +64,7 @@ def load(): yield first for i in range(first_n, x.shape[0], chunksize): yield x[i: i + chunksize] - return chunks(np.ndarray)(load) + return iterable(np.ndarray)(load) @resource.register('.*\.bcolz/?') diff --git a/into/backends/csv.py b/into/backends/csv.py index 837f3394..0e11f37e 100644 --- a/into/backends/csv.py +++ b/into/backends/csv.py @@ -19,7 +19,7 @@ from ..append import append from ..convert import convert, ooc_types from ..resource import resource -from ..chunks import chunks +from ..chunks import iterable from ..numpy_dtype import dshape_to_pandas from .pandas import coerce_datetimes @@ -57,7 +57,7 @@ def __init__(self, path, has_header='no-input', encoding='utf-8', **kwargs): @append.register(CSV, object) def append_object_to_csv(c, seq, **kwargs): - append(c, convert(chunks(pd.DataFrame), seq, **kwargs), **kwargs) + append(c, convert(iterable(pd.DataFrame), seq, **kwargs), **kwargs) return c @@ -92,7 +92,7 @@ def append_dataframe_to_csv(c, df, dshape=None, **kwargs): return c -@append.register(CSV, chunks(pd.DataFrame)) +@append.register(CSV, iterable(pd.DataFrame)) def append_iterator_to_csv(c, cs, **kwargs): for chunk in cs: append(c, chunk, **kwargs) @@ -174,7 +174,7 @@ def _csv_to_DataFrame(c, dshape=None, chunksize=None, **kwargs): **kwargs2) -@convert.register(chunks(pd.DataFrame), CSV, cost=10.0) +@convert.register(iterable(pd.DataFrame), CSV, cost=10.0) def CSV_to_chunks_of_dataframes(c, chunksize=2**20, **kwargs): # Load a small 1000 line DF to start # This helps with rapid viewing of a large CSV file @@ -187,7 +187,7 @@ def _(): yield first for df in rest: yield df - return chunks(pd.DataFrame)(_) + return iterable(pd.DataFrame)(_) @discover.register(CSV) @@ -226,19 +226,19 @@ def resource_csv(uri, **kwargs): def resource_glob(uri, **kwargs): filenames = sorted(glob(uri)) r = resource(filenames[0], **kwargs) - return chunks(type(r))([resource(u, **kwargs) for u in sorted(glob(uri))]) + return iterable(type(r))([resource(u, **kwargs) for u in sorted(glob(uri))]) # Alternatively check each time we iterate? def _(): return (resource(u, **kwargs) for u in glob(uri)) - return chunks(type(r))(_) + return iterable(type(r))(_) -@convert.register(chunks(pd.DataFrame), chunks(CSV), cost=10.0) +@convert.register(iterable(pd.DataFrame), iterable(CSV), cost=10.0) def convert_glob_of_csvs_to_chunks_of_dataframes(csvs, **kwargs): def _(): - return concat(convert(chunks(pd.DataFrame), csv, **kwargs) for csv in csvs) - return chunks(pd.DataFrame)(_) + return concat(convert(iterable(pd.DataFrame), csv, **kwargs) for csv in csvs) + return iterable(pd.DataFrame)(_) @dispatch(CSV) diff --git a/into/backends/h5py.py b/into/backends/h5py.py index 9352a308..5b7b846f 100644 --- a/into/backends/h5py.py +++ b/into/backends/h5py.py @@ -13,7 +13,7 @@ from ..convert import convert, ooc_types from ..create import create from ..resource import resource -from ..chunks import chunks, Chunks +from ..chunks import iterable, IterableOf from ..compatibility import unicode h5py_attributes = ['chunks', 'compression', 'compression_opts', 'dtype', @@ -95,7 +95,7 @@ def append_h5py(dset, x, **kwargs): return dset -@append.register(h5py.Dataset, chunks(np.ndarray)) +@append.register(h5py.Dataset, iterable(np.ndarray)) def append_h5py(dset, c, **kwargs): for chunk in c: append(dset, chunk) @@ -104,7 +104,7 @@ def append_h5py(dset, c, **kwargs): @append.register(h5py.Dataset, object) def append_h5py(dset, x, **kwargs): - return append(dset, convert(chunks(np.ndarray), x, **kwargs), **kwargs) + return append(dset, convert(iterable(np.ndarray), x, **kwargs), **kwargs) @convert.register(np.ndarray, h5py.Dataset, cost=3.0) @@ -116,12 +116,12 @@ def h5py_to_numpy(dset, force=False, **kwargs): return dset[:] -@convert.register(chunks(np.ndarray), h5py.Dataset, cost=3.0) +@convert.register(iterable(np.ndarray), h5py.Dataset, cost=3.0) def h5py_to_numpy_chunks(dset, chunksize=2**20, **kwargs): def load(): for i in range(0, dset.shape[0], chunksize): yield dset[i: i + chunksize] - return chunks(np.ndarray)(load) + return iterable(np.ndarray)(load) @resource.register('h5py://.+', priority=11) diff --git a/into/backends/hdfstore.py b/into/backends/hdfstore.py index f96e9688..d3420cfe 100644 --- a/into/backends/hdfstore.py +++ b/into/backends/hdfstore.py @@ -6,7 +6,7 @@ from datashape import discover from ..append import append from ..convert import convert, ooc_types -from ..chunks import chunks, Chunks +from ..chunks import iterable, IterableOf from ..resource import resource @@ -37,9 +37,9 @@ def discover_hdfstore_storer(storer): return n * measure -@convert.register(chunks(pd.DataFrame), pd.io.pytables.AppendableFrameTable) +@convert.register(iterable(pd.DataFrame), pd.io.pytables.AppendableFrameTable) def hdfstore_to_chunks_dataframes(data, chunksize=1000000, **kwargs): - return chunks(pd.DataFrame)(data.parent.select(data.pathname, chunksize=chunksize)) + return iterable(pd.DataFrame)(data.parent.select(data.pathname, chunksize=chunksize)) @convert.register(pd.DataFrame, (pd.io.pytables.AppendableFrameTable, @@ -81,7 +81,7 @@ def append_dataframe_to_hdfstore(store, df, **kwargs): @append.register((pd.io.pytables.Fixed, EmptyHDFStoreDataset), - chunks(pd.DataFrame)) + iterable(pd.DataFrame)) def append_chunks_dataframe_to_hdfstore(store, c, **kwargs): parent = store.parent for chunk in c: @@ -91,7 +91,7 @@ def append_chunks_dataframe_to_hdfstore(store, c, **kwargs): @append.register((pd.io.pytables.Fixed, EmptyHDFStoreDataset), object) def append_object_to_hdfstore(store, o, **kwargs): - return append(store, convert(chunks(pd.DataFrame), o, **kwargs), **kwargs) + return append(store, convert(iterable(pd.DataFrame), o, **kwargs), **kwargs) ooc_types |= set(HDFDataset) diff --git a/into/backends/pytables.py b/into/backends/pytables.py index d227848e..9b16e384 100644 --- a/into/backends/pytables.py +++ b/into/backends/pytables.py @@ -5,7 +5,7 @@ from ..append import append from ..convert import convert, ooc_types from ..resource import resource -from ..chunks import chunks, Chunks +from ..chunks import iterable from ..utils import tmpfile import os @@ -45,7 +45,7 @@ def numpy_to_pytables(t, x, **kwargs): @append.register((tables.Array, tables.Table), object) def append_h5py(dset, x, **kwargs): - return append(dset, convert(chunks(np.ndarray), x, **kwargs), **kwargs) + return append(dset, convert(iterable(np.ndarray), x, **kwargs), **kwargs) @convert.register(np.ndarray, tables.Table, cost=3.0) @@ -53,12 +53,12 @@ def pytables_to_numpy(t, **kwargs): return t[:] -@convert.register(chunks(np.ndarray), tables.Table, cost=3.0) +@convert.register(iterable(np.ndarray), tables.Table, cost=3.0) def pytables_to_numpy_chunks(t, chunksize=2**20, **kwargs): def load(): for i in range(0, t.shape[0], chunksize): yield t[i: i + chunksize] - return chunks(np.ndarray)(load) + return iterable(np.ndarray)(load) def dtype_to_pytables(dtype): diff --git a/into/backends/sql.py b/into/backends/sql.py index dd51e76a..1dd2f571 100644 --- a/into/backends/sql.py +++ b/into/backends/sql.py @@ -14,7 +14,7 @@ from ..convert import convert, ooc_types from ..append import append from ..resource import resource -from ..chunks import Chunks +from ..chunks import IterableOf # http://docs.sqlalchemy.org/en/latest/core/types.html @@ -243,7 +243,7 @@ def append_iterator_to_table(t, rows, dshape=None, **kwargs): return t -@append.register(sa.Table, Chunks) +@append.register(sa.Table, IterableOf) def append_anything_to_sql_Table(t, c, **kwargs): for item in c: append(t, item, **kwargs) diff --git a/into/backends/tests/test_bcolz.py b/into/backends/tests/test_bcolz.py index 665fc2db..41a5d7d7 100644 --- a/into/backends/tests/test_bcolz.py +++ b/into/backends/tests/test_bcolz.py @@ -2,7 +2,7 @@ from into.backends.bcolz import (append, convert, ctable, carray, resource, discover, drop) -from into.chunks import chunks +from into.chunks import iterable from into import append, convert, discover, into import numpy as np from into.utils import tmpfile @@ -31,8 +31,8 @@ def test_convert(): def test_chunks(): - c = convert(chunks(np.ndarray), a, chunksize=2) - assert isinstance(c, chunks(np.ndarray)) + c = convert(iterable(np.ndarray), a, chunksize=2) + assert isinstance(c, iterable(np.ndarray)) assert len(list(c)) == 2 assert eq(list(c)[1], [3, 4]) @@ -41,7 +41,7 @@ def test_chunks(): def test_append_chunks(): b = carray(x) - append(b, chunks(np.ndarray)([x, x])) + append(b, iterable(np.ndarray)([x, x])) assert len(b) == len(x) * 3 diff --git a/into/backends/tests/test_h5py.py b/into/backends/tests/test_h5py.py index 2c87b802..b7215ef7 100644 --- a/into/backends/tests/test_h5py.py +++ b/into/backends/tests/test_h5py.py @@ -4,7 +4,7 @@ resource_h5py) from contextlib import contextmanager from into.utils import tmpfile -from into.chunks import chunks +from into.chunks import iterable from into import into, append, convert, discover import datashape import h5py @@ -61,13 +61,13 @@ def test_numpy(): def test_chunks(): with file(x) as (fn, f, dset): - c = convert(chunks(np.ndarray), dset) + c = convert(iterable(np.ndarray), dset) assert eq(convert(np.ndarray, c), x) def test_append_chunks(): with file(x) as (fn, f, dset): - append(dset, chunks(np.ndarray)([x, x])) + append(dset, iterable(np.ndarray)([x, x])) assert len(dset) == len(x) * 3 diff --git a/into/backends/tests/test_hdfstore.py b/into/backends/tests/test_hdfstore.py index 5c4cb987..3528358f 100644 --- a/into/backends/tests/test_hdfstore.py +++ b/into/backends/tests/test_hdfstore.py @@ -2,7 +2,7 @@ from into.backends.hdfstore import discover from contextlib import contextmanager from into.utils import tmpfile -from into.chunks import chunks +from into.chunks import iterable from into import into, append, convert, resource, discover import datashape import pandas as pd @@ -60,7 +60,7 @@ def eq(a, b): def test_chunks(): with file(df) as (fn, f, dset): - c = convert(chunks(pd.DataFrame), dset) + c = convert(iterable(pd.DataFrame), dset) assert eq(convert(np.ndarray, c), df) @@ -97,14 +97,14 @@ def test_convert_pandas(): def test_convert_chunks(): with file(df) as (fn, f, dset): - c = convert(chunks(pd.DataFrame), dset, chunksize=len(df) / 2) + c = convert(iterable(pd.DataFrame), dset, chunksize=len(df) / 2) assert len(list(c)) == 2 assert eq(convert(pd.DataFrame, c), df) def test_append_chunks(): with file(df) as (fn, f, dset): - append(dset, chunks(pd.DataFrame)([df, df])) + append(dset, iterable(pd.DataFrame)([df, df])) assert discover(dset).shape[0] == len(df) * 3 diff --git a/into/chunks.py b/into/chunks.py index 5fbb612e..55fa579e 100644 --- a/into/chunks.py +++ b/into/chunks.py @@ -5,20 +5,20 @@ from .utils import cls_name -class Chunks(object): +class IterableOf(object): """ An Iterable of chunked data Iterates over chunks of in-memory data. Contains an iterable or a function that returns an iterator. - >>> c = Chunks([[1, 2, 3], [4, 5, 6]]) + >>> c = IterableOf([[1, 2, 3], [4, 5, 6]]) >>> next(iter(c)) [1, 2, 3] - For typed containers see the ``chunks`` function which generates - parametrized Chunks classes. + For typed containers see the ``iterable`` function which generates + parametrized ``IterableOf`` classes. - >>> c = chunks(list)([[1, 2, 3], [4, 5, 6]]) + >>> c = iterable(list)([[1, 2, 3], [4, 5, 6]]) >>> next(iter(c)) [1, 2, 3] @@ -36,15 +36,15 @@ def __iter__(self): return iter(self.data) -def chunks(cls): - """ Parametrized Chunks Class """ - return type('Chunks_' + cls_name(cls).replace('.', '_'), (Chunks,), {'container': cls}) +def iterable(cls): + """ Parametrized IterableOf Class """ + return type('iterable(%s)' % cls_name(cls), (IterableOf,), {'container': cls}) -chunks.__doc__ = Chunks.__doc__ +iterable.__doc__ = IterableOf.__doc__ -chunks = memoize(chunks) +iterable = memoize(iterable) -@discover.register(Chunks) +@discover.register(IterableOf) def discover_chunks(c, **kwargs): return var * discover(first(c)).subshape[0] diff --git a/into/convert.py b/into/convert.py index 38eb2a72..dce51bc7 100644 --- a/into/convert.py +++ b/into/convert.py @@ -7,7 +7,7 @@ from collections import Iterator, Iterable import datashape from .core import NetworkDispatcher, ooc_types -from .chunks import chunks, Chunks +from .chunks import iterable, IterableOf from .numpy_dtype import dshape_to_numpy from .utils import records_to_tuples @@ -88,25 +88,25 @@ def numpy_to_list(x, **kwargs): return x.tolist() -@convert.register(np.ndarray, chunks(np.ndarray), cost=1.0) +@convert.register(np.ndarray, iterable(np.ndarray), cost=1.0) def numpy_chunks_to_numpy(c, **kwargs): return np.concatenate(list(c)) -@convert.register(chunks(np.ndarray), np.ndarray, cost=0.5) +@convert.register(iterable(np.ndarray), np.ndarray, cost=0.5) def numpy_to_chunks_numpy(x, chunksize=2**20, **kwargs): - return chunks(np.ndarray)( + return iterable(np.ndarray)( lambda: (x[i:i+chunksize] for i in range(0, x.shape[0], chunksize))) -@convert.register(pd.DataFrame, chunks(pd.DataFrame), cost=1.0) +@convert.register(pd.DataFrame, iterable(pd.DataFrame), cost=1.0) def chunks_dataframe_to_dataframe(c, **kwargs): return pd.concat(list(c), axis=0, ignore_index=True) -@convert.register(chunks(pd.DataFrame), pd.DataFrame, cost=0.5) +@convert.register(iterable(pd.DataFrame), pd.DataFrame, cost=0.5) def dataframe_to_chunks_dataframe(x, chunksize=2**20, **kwargs): - return chunks(pd.DataFrame)( + return iterable(pd.DataFrame)( lambda: (x.iloc[i:i+chunksize] for i in range(0, x.shape[0], chunksize))) def ishashable(x): @@ -168,12 +168,12 @@ def iterator_to_list(seq, **kwargs): return list(seq) -@convert.register(Iterator, (chunks(pd.DataFrame), chunks(np.ndarray)), cost=10.0) +@convert.register(Iterator, (iterable(pd.DataFrame), iterable(np.ndarray)), cost=10.0) def numpy_chunks_to_iterator(c, **kwargs): return concat(convert(Iterator, chunk, **kwargs) for chunk in c) -@convert.register(chunks(np.ndarray), Iterator, cost=10.0) +@convert.register(iterable(np.ndarray), Iterator, cost=10.0) def iterator_to_numpy_chunks(seq, chunksize=1024, **kwargs): seq2 = partition_all(chunksize, seq) first, rest = next(seq2), seq2 @@ -182,10 +182,10 @@ def _(): yield x for i in rest: yield convert(np.ndarray, i, **kwargs) - return chunks(np.ndarray)(_) + return iterable(np.ndarray)(_) -@convert.register(chunks(pd.DataFrame), Iterator, cost=10.0) +@convert.register(iterable(pd.DataFrame), Iterator, cost=10.0) def iterator_to_DataFrame_chunks(seq, chunksize=1024, **kwargs): seq2 = partition_all(chunksize, seq) first, rest = next(seq2), seq2 @@ -194,7 +194,7 @@ def _(): yield df for i in rest: yield convert(pd.DataFrame, i, **kwargs) - return chunks(pd.DataFrame)(_) + return iterable(pd.DataFrame)(_) @convert.register(tuple, np.record) @@ -202,22 +202,22 @@ def numpy_record_to_tuple(rec, **kwargs): return rec.tolist() -@convert.register(chunks(np.ndarray), chunks(pd.DataFrame), cost=0.5) +@convert.register(iterable(np.ndarray), iterable(pd.DataFrame), cost=0.5) def chunked_pandas_to_chunked_numpy(c, **kwargs): - return chunks(np.ndarray)(lambda: (convert(np.ndarray, chunk, **kwargs) for chunk in c)) + return iterable(np.ndarray)(lambda: (convert(np.ndarray, chunk, **kwargs) for chunk in c)) -@convert.register(chunks(pd.DataFrame), chunks(np.ndarray), cost=0.5) +@convert.register(iterable(pd.DataFrame), iterable(np.ndarray), cost=0.5) def chunked_numpy_to_chunked_pandas(c, **kwargs): - return chunks(pd.DataFrame)(lambda: (convert(pd.DataFrame, chunk, **kwargs) for chunk in c)) + return iterable(pd.DataFrame)(lambda: (convert(pd.DataFrame, chunk, **kwargs) for chunk in c)) -@convert.register(chunks(np.ndarray), chunks(list), cost=10.0) +@convert.register(iterable(np.ndarray), iterable(list), cost=10.0) def chunked_list_to_chunked_numpy(c, **kwargs): - return chunks(np.ndarray)(lambda: (convert(np.ndarray, chunk, **kwargs) for chunk in c)) + return iterable(np.ndarray)(lambda: (convert(np.ndarray, chunk, **kwargs) for chunk in c)) -@convert.register(chunks(list), chunks(np.ndarray), cost=10.0) +@convert.register(iterable(list), iterable(np.ndarray), cost=10.0) def chunked_numpy_to_chunked_list(c, **kwargs): - return chunks(list)(lambda: (convert(list, chunk, **kwargs) for chunk in c)) + return iterable(list)(lambda: (convert(list, chunk, **kwargs) for chunk in c)) -ooc_types |= set([Iterator, Chunks]) +ooc_types |= set([Iterator, IterableOf]) diff --git a/into/tests/test_chunks.py b/into/tests/test_chunks.py index 203d69ed..b6cf9997 100644 --- a/into/tests/test_chunks.py +++ b/into/tests/test_chunks.py @@ -2,32 +2,32 @@ from toolz import first -CL = chunks(list) +IL = iterable(list) def test_chunks_basics(): - assert isinstance(CL, type) - assert issubclass(CL, Chunks) + assert isinstance(IL, type) + assert issubclass(IL, IterableOf) def test_chunks_isnt_consumable(): - cl = CL([[1, 2, 3], [4, 5, 6]]) + il = IL([[1, 2, 3], [4, 5, 6]]) - assert next(iter(cl)) == [1, 2, 3] - assert next(iter(cl)) == [1, 2, 3] + assert next(iter(il)) == [1, 2, 3] + assert next(iter(il)) == [1, 2, 3] def test_chunks_is_memoized(): - assert chunks(list) is chunks(list) + assert iterable(list) is iterable(list) def test_callables(): - cl = CL(lambda: (list(range(3)) for i in range(3))) + il = IL(lambda: (list(range(3)) for i in range(3))) - assert first(cl) == [0, 1, 2] - assert first(cl) == [0, 1, 2] + assert first(il) == [0, 1, 2] + assert first(il) == [0, 1, 2] def test_discover(): - cl = CL([[1, 2, 3], [4, 5, 6]]) - assert discover(cl).measure == discover(1).measure + il = IL([[1, 2, 3], [4, 5, 6]]) + assert discover(il).measure == discover(1).measure diff --git a/into/tests/test_convert.py b/into/tests/test_convert.py index 6ebf18db..2fb93622 100644 --- a/into/tests/test_convert.py +++ b/into/tests/test_convert.py @@ -1,7 +1,7 @@ from into.convert import (convert, list_to_numpy, iterator_to_numpy_chunks, numpy_to_chunks_numpy, dataframe_to_chunks_dataframe, chunks_dataframe_to_dataframe) -from into.chunks import chunks +from into.chunks import iterable from datashape import discover from toolz import first from collections import Iterator @@ -70,7 +70,7 @@ def test_dataframe_and_series(): def test_iterator_and_numpy_chunks(): c = iterator_to_numpy_chunks([1, 2, 3], chunksize=2) - assert isinstance(c, chunks(np.ndarray)) + assert isinstance(c, iterable(np.ndarray)) assert all(isinstance(chunk, np.ndarray) for chunk in c) c = iterator_to_numpy_chunks([1, 2, 3], chunksize=2) @@ -108,15 +108,15 @@ def test_list_to_numpy_on_dicts(): def test_chunks_numpy_pandas(): x = np.array([('Alice', 100), ('Bob', 200)], dtype=[('name', 'S7'), ('amount', 'i4')]) - n = chunks(np.ndarray)([x, x]) + n = iterable(np.ndarray)([x, x]) - pan = convert(chunks(pd.DataFrame), n) - num = convert(chunks(np.ndarray), pan) + pan = convert(iterable(pd.DataFrame), n) + num = convert(iterable(np.ndarray), pan) - assert isinstance(pan, chunks(pd.DataFrame)) + assert isinstance(pan, iterable(pd.DataFrame)) assert all(isinstance(chunk, pd.DataFrame) for chunk in pan) - assert isinstance(num, chunks(np.ndarray)) + assert isinstance(num, iterable(np.ndarray)) assert all(isinstance(chunk, np.ndarray) for chunk in num) @@ -175,7 +175,7 @@ def test_numpy_to_list_preserves_ns_datetimes(): def test_numpy_to_chunks_numpy(): x = np.arange(100) c = numpy_to_chunks_numpy(x, chunksize=10) - assert isinstance(c, chunks(np.ndarray)) + assert isinstance(c, iterable(np.ndarray)) assert len(list(c)) == 10 assert eq(list(c)[0], x[:10]) @@ -184,7 +184,7 @@ def test_pandas_and_chunks_pandas(): df = pd.DataFrame({'a': [1, 2, 3, 4], 'b': [1., 2., 3., 4.]}) c = dataframe_to_chunks_dataframe(df, chunksize=2) - assert isinstance(c, chunks(pd.DataFrame)) + assert isinstance(c, iterable(pd.DataFrame)) assert len(list(c)) == 2 df2 = chunks_dataframe_to_dataframe(c)