Notebook

In [1]:

import sys
!{sys.executable} -m pip list

Package          Version
---------------- -------
appnope          0.1.0  
backcall         0.1.0  
cloudpickle      0.5.6  
decorator        4.3.0  
ipykernel        5.0.0  
ipyparallel      6.2.2  
ipython          7.0.1  
ipython-genutils 0.2.0  
jedi             0.12.1 
jupyter-client   5.2.3  
jupyter-core     4.4.0  
parso            0.3.1  
pexpect          4.6.0  
pickleshare      0.7.5  
pip              18.0   
prompt-toolkit   2.0.4  
ptyprocess       0.6.0  
Pygments         2.2.0  
python-dateutil  2.7.3  
pyzmq            17.1.2 
setuptools       40.4.3 
simplegeneric    0.8.1  
six              1.11.0 
tornado          5.1.1  
traitlets        4.3.2  
wcwidth          0.1.7  
wheel            0.31.1

In [2]:

import ipyparallel as ipp
rc = ipp.Client()

In [3]:

view = rc.load_balanced_view()

In [4]:

import os
import glob

directory_location = os.path.join(os.getcwd(), "*")

class inputobject(object):
    filepath = None

inputlist = []

x = 0
for image in glob.glob(directory_location):
    obj = inputobject()
    obj.filepath = image
    inputlist.append(obj)

In [5]:

view.map_async(lambda obj: obj.filepath, inputlist[x:x+100], ordered=False).get()

[7:apply]: 
---------------------------------------------------------------------------AttributeError                            Traceback (most recent call last)~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in unpack_apply_message(bufs, g, copy)
    203     args = []
    204     for i in range(info['nargs']):
--> 205         arg, arg_bufs = deserialize_object(arg_bufs, g)
    206         args.append(arg)
    207     args = tuple(args)
~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in deserialize_object(buffers, g)
    143     bufs = list(buffers)
    144     pobj = buffer_to_bytes_py2(bufs.pop(0))
--> 145     canned = pickle.loads(pobj)
    146     if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
    147         for c in canned:
AttributeError: Can't get attribute 'inputobject' on <IPython.core.interactiveshell.DummyMod object at 0x10b3ec748>

[4:apply]: 
---------------------------------------------------------------------------AttributeError                            Traceback (most recent call last)~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in unpack_apply_message(bufs, g, copy)
    203     args = []
    204     for i in range(info['nargs']):
--> 205         arg, arg_bufs = deserialize_object(arg_bufs, g)
    206         args.append(arg)
    207     args = tuple(args)
~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in deserialize_object(buffers, g)
    143     bufs = list(buffers)
    144     pobj = buffer_to_bytes_py2(bufs.pop(0))
--> 145     canned = pickle.loads(pobj)
    146     if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
    147         for c in canned:
AttributeError: Can't get attribute 'inputobject' on <IPython.core.interactiveshell.DummyMod object at 0x1107a3b70>

[5:apply]: 
---------------------------------------------------------------------------AttributeError                            Traceback (most recent call last)~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in unpack_apply_message(bufs, g, copy)
    203     args = []
    204     for i in range(info['nargs']):
--> 205         arg, arg_bufs = deserialize_object(arg_bufs, g)
    206         args.append(arg)
    207     args = tuple(args)
~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in deserialize_object(buffers, g)
    143     bufs = list(buffers)
    144     pobj = buffer_to_bytes_py2(bufs.pop(0))
--> 145     canned = pickle.loads(pobj)
    146     if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
    147         for c in canned:
AttributeError: Can't get attribute 'inputobject' on <IPython.core.interactiveshell.DummyMod object at 0x10769d6d8>

[6:apply]: 
---------------------------------------------------------------------------AttributeError                            Traceback (most recent call last)~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in unpack_apply_message(bufs, g, copy)
    203     args = []
    204     for i in range(info['nargs']):
--> 205         arg, arg_bufs = deserialize_object(arg_bufs, g)
    206         args.append(arg)
    207     args = tuple(args)
~/env/test-parallel/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py in deserialize_object(buffers, g)
    143     bufs = list(buffers)
    144     pobj = buffer_to_bytes_py2(bufs.pop(0))
--> 145     canned = pickle.loads(pobj)
    146     if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
    147         for c in canned:
AttributeError: Can't get attribute 'inputobject' on <IPython.core.interactiveshell.DummyMod object at 0x1080626d8>

... 28 more exceptions ...

In [6]:

rc[:].use_cloudpickle()

Out[6]:

<AsyncResult: use_cloudpickle>

In [7]:

view.map_async(lambda obj: obj.filepath, inputlist[x:x+100], ordered=False).get()

Out[7]:

['/Users/benjaminrk/dev/mine/notebooks/empty/test%20dir',
 '/Users/benjaminrk/dev/mine/notebooks/empty/Untitled1.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/traitlets-and-argparse.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/test_pypy.pyc',
 '/Users/benjaminrk/dev/mine/notebooks/empty/broken',
 '/Users/benjaminrk/dev/mine/notebooks/empty/docker-sockets.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/traitlets.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/ok',
 '/Users/benjaminrk/dev/mine/notebooks/empty/Untitled.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/ci',
 '/Users/benjaminrk/dev/mine/notebooks/empty/inorder.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/eventloops.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/client-socket.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/detect-abi.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/server-socket.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/sql.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/test.py',
 '/Users/benjaminrk/dev/mine/notebooks/empty/nbtoken.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/test-parallel.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/kernel-restart-problem.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/evan.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/server.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/test.txt',
 '/Users/benjaminrk/dev/mine/notebooks/empty/math.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/test-hub.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/polite-errors.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/datapub.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/stitches.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/test.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/redirect-output.ipynb',
 '/Users/benjaminrk/dev/mine/notebooks/empty/foo.py~',
 '/Users/benjaminrk/dev/mine/notebooks/empty/line.html']