In [1]:

from bcolz_array_iterator2 import BcolzArrayIterator2 

In [2]:

from bcolz import carray

In [3]:

import numpy as np

In [4]:

x = np.arange(14); x

Out[4]:

array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13])

In [5]:

y = np.arange(14); y

Out[5]:

array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13])

In [16]:

x = carray(x, chunklen=3)
y = carray(y, chunklen=3)

In [17]:

b = BcolzArrayIterator2(x, y, shuffle=True, batch_size=3)

In [18]:

b.N

Out[18]:

In [19]:

nit = len(x)//b.batch_size+1; nit

Out[19]:

In [20]:

for j in range(10000):
    bx,by = list(zip(*[next(b) for i in range(nit)]))
    nx = np.concatenate(bx)
    ny = np.concatenate(by)
    assert(np.allclose(nx,ny))
    assert(len(np.unique(nx))==len(nx))

In [21]:

[next(b) for i in range(20)]

Out[21]:

[(array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([0, 1, 2]), array([0, 1, 2])),
 (array([12, 13]), array([12, 13])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([0, 1, 2]), array([0, 1, 2])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([12, 13]), array([12, 13])),
 (array([12, 13]), array([12, 13])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([0, 1, 2]), array([0, 1, 2])),
 (array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([12, 13]), array([12, 13])),
 (array([0, 1, 2]), array([0, 1, 2]))]

In [ ]: