! pip3 install cython
Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (0.28.5)
%load_ext Cython
The Cython extension is already loaded. To reload it, use: %reload_ext Cython
class Task(object):
__slots__ = ("function", "args", "kwargs", "annotations")
def __init__(self, function, *args, **kwargs):
self.function = function
self.args = args
self.annotations = kwargs.pop("annotations", None)
self.kwargs = kwargs
def __getstate__(self):
return (self.function, self.args, self.kwargs, self.annotations)
def __setstate__(self, state):
self.function = state[0]
self.args = state[1]
self.kwargs = state[2]
self.annotations = state[3]
def __repr__(self):
details = ", ".join("%s=%s" % (n, repr(self[i]))
for i, n in enumerate(self.__slots__)
if self[i])
return 'Task({})'.format(details)
%%cython
cdef class CythonTask:
cdef readonly object function
cdef readonly tuple args
cdef readonly dict kwargs
cdef readonly dict annotations
def __init__(self, function, *args, **kwargs):
self.function = function
self.args = args
self.annotations = kwargs.pop("annotations", None)
self.kwargs = kwargs
def __getstate__(self):
return (self.function, self.args, self.kwargs, self.annotations)
def __setstate__(self, state):
self.function = state[0]
self.args = state[1]
self.kwargs = state[2]
self.annotations = state[3]
def __repr__(self):
details = ", ".join("%s=%s" % (n, repr(self[i]))
for i, n in enumerate(self.__slots__)
if self[i])
return 'Task({})'.format(details)
cdef class CythonTask2:
cdef readonly object function
cdef readonly tuple args
cdef readonly dict kwargs
cdef readonly dict annotations
def __init__(self, function, args=(), kwargs={}, annotations={}):
self.function = function
self.args = args
self.kwargs = kwargs
self.annotations = annotations
def __reduce__(self):
state = (self.function, self.args, self.kwargs, self.annotations)
return (CythonTask2, state)
def __repr__(self):
details = ", ".join("%s=%s" % (n, repr(self[i]))
for i, n in enumerate(self.__slots__)
if self[i])
return 'Task({})'.format(details)
def f(x):
return x + 1
%timeit -r 30 [(f, i, {"bob": "foo"}, {"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 [Task(f, i, bob="foo", annotations={"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 [CythonTask(f, i, bob="foo", annotations={"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 [CythonTask2(f, (i,), {"bob": "foo"}, annotations={"resource": "GPU", "size": i}) for i in range(1000)]
The slowest run took 4.55 times longer than the fastest. This could mean that an intermediate result is being cached. 1000 loops, best of 30: 343 µs per loop 1000 loops, best of 30: 1.14 ms per loop 1000 loops, best of 30: 635 µs per loop 1000 loops, best of 30: 548 µs per loop
import pickle
tasks = [(f, i, {"bob": "foo"}, {"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 pickle.dumps(tasks, protocol=pickle.HIGHEST_PROTOCOL)
tasks = [Task(f, i, bob="foo", annotations={"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 pickle.dumps(tasks, protocol=pickle.HIGHEST_PROTOCOL)
tasks = [CythonTask(f, i, bob="foo", annotations={"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 pickle.dumps(tasks, protocol=pickle.HIGHEST_PROTOCOL)
tasks = [CythonTask2(f, (i,), {"bob": "foo"}, annotations={"resource": "GPU", "size": i}) for i in range(1000)]
%timeit -r 30 pickle.dumps(tasks, protocol=pickle.HIGHEST_PROTOCOL)
1000 loops, best of 30: 500 µs per loop 1000 loops, best of 30: 1.55 ms per loop 1000 loops, best of 30: 1.31 ms per loop 1000 loops, best of 30: 1.17 ms per loop