# !pip install coiled
Requirement already satisfied: coiled in c:\users\argenisleon\anaconda3\lib\site-packages (0.0.23) Requirement already satisfied: pandas>=1.1.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (1.1.2) Requirement already satisfied: aiohttp in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (3.6.2) Requirement already satisfied: click in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (7.1.1) Requirement already satisfied: distributed>=2.23.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (2.25.0) Requirement already satisfied: s3fs in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (0.5.1) Requirement already satisfied: dask[complete]>=2.23.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (2.25.0) Requirement already satisfied: ipython in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from coiled) (7.5.0) Requirement already satisfied: aiobotocore>=1.0.7 in c:\users\argenisleon\anaconda3\lib\site-packages (from coiled) (1.1.1) Requirement already satisfied: pytz>=2017.2 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from pandas>=1.1.0->coiled) (2019.3) Requirement already satisfied: numpy>=1.15.4 in c:\users\argenisleon\anaconda3\lib\site-packages (from pandas>=1.1.0->coiled) (1.19.1) Requirement already satisfied: python-dateutil>=2.7.3 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from pandas>=1.1.0->coiled) (2.8.1) Requirement already satisfied: yarl<2.0,>=1.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiohttp->coiled) (1.5.1) Requirement already satisfied: multidict<5.0,>=4.5 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiohttp->coiled) (4.7.6) Requirement already satisfied: async-timeout<4.0,>=3.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiohttp->coiled) (3.0.1) Requirement already satisfied: attrs>=17.3.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiohttp->coiled) (19.3.0) Requirement already satisfied: chardet<4.0,>=2.0 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from aiohttp->coiled) (3.0.4) Requirement already satisfied: psutil>=5.0 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from distributed>=2.23.0->coiled) (5.7.0) Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (2.1.0) Requirement already satisfied: cloudpickle>=1.5.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (1.5.0) Requirement already satisfied: setuptools in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (41.6.0) Requirement already satisfied: zict>=0.1.3 in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (2.0.0) Requirement already satisfied: toolz>=0.8.2 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from distributed>=2.23.0->coiled) (0.10.0) Requirement already satisfied: pyyaml in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from distributed>=2.23.0->coiled) (5.3.1) Requirement already satisfied: tornado>=5; python_version < "3.8" in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (6.0.4) Requirement already satisfied: msgpack>=0.6.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (1.0.0) Requirement already satisfied: tblib>=1.6.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from distributed>=2.23.0->coiled) (1.6.0) Requirement already satisfied: fsspec>=0.8.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from s3fs->coiled) (0.8.1) Requirement already satisfied: partd>=0.3.10; extra == "complete" in c:\users\argenisleon\anaconda3\lib\site-packages (from dask[complete]>=2.23.0->coiled) (1.1.0) Requirement already satisfied: bokeh>=1.0.0; extra == "complete" in c:\users\argenisleon\anaconda3\lib\site-packages (from dask[complete]>=2.23.0->coiled) (2.0.1) Requirement already satisfied: pygments in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (2.6.1) Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (2.0.10) Requirement already satisfied: decorator in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (4.4.2) Requirement already satisfied: traitlets>=4.2 in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (4.3.3) Requirement already satisfied: pickleshare in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (0.7.5) Requirement already satisfied: backcall in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (0.1.0) Requirement already satisfied: jedi>=0.10 in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (0.14.1) Requirement already satisfied: colorama; sys_platform == "win32" in c:\users\argenisleon\anaconda3\lib\site-packages (from ipython->coiled) (0.4.3) Requirement already satisfied: wrapt>=1.10.10 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiobotocore>=1.0.7->coiled) (1.12.1) Requirement already satisfied: botocore<1.17.45,>=1.17.44 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiobotocore>=1.0.7->coiled) (1.17.44) Requirement already satisfied: aioitertools>=0.5.1 in c:\users\argenisleon\anaconda3\lib\site-packages (from aiobotocore>=1.0.7->coiled) (0.7.0) Requirement already satisfied: six>=1.5 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from python-dateutil>=2.7.3->pandas>=1.1.0->coiled) (1.14.0) Requirement already satisfied: typing-extensions>=3.7.4; python_version < "3.8" in c:\users\argenisleon\anaconda3\lib\site-packages (from yarl<2.0,>=1.0->aiohttp->coiled) (3.7.4.1) Requirement already satisfied: idna>=2.0 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from yarl<2.0,>=1.0->aiohttp->coiled) (2.9) Requirement already satisfied: heapdict in c:\users\argenisleon\anaconda3\lib\site-packages (from zict>=0.1.3->distributed>=2.23.0->coiled) (1.0.1) Requirement already satisfied: locket in c:\users\argenisleon\anaconda3\lib\site-packages (from partd>=0.3.10; extra == "complete"->dask[complete]>=2.23.0->coiled) (0.2.0) Requirement already satisfied: Jinja2>=2.7 in c:\users\argenisleon\anaconda3\lib\site-packages (from bokeh>=1.0.0; extra == "complete"->dask[complete]>=2.23.0->coiled) (2.11.1) Requirement already satisfied: pillow>=4.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from bokeh>=1.0.0; extra == "complete"->dask[complete]>=2.23.0->coiled) (7.1.1) Requirement already satisfied: packaging>=16.8 in c:\users\argenisleon\anaconda3\lib\site-packages (from bokeh>=1.0.0; extra == "complete"->dask[complete]>=2.23.0->coiled) (20.3) Requirement already satisfied: wcwidth in c:\users\argenisleon\anaconda3\lib\site-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython->coiled) (0.1.9) Requirement already satisfied: ipython-genutils in c:\users\argenisleon\anaconda3\lib\site-packages (from traitlets>=4.2->ipython->coiled) (0.2.0) Requirement already satisfied: parso>=0.5.0 in c:\users\argenisleon\anaconda3\lib\site-packages (from jedi>=0.10->ipython->coiled) (0.5.2) Requirement already satisfied: urllib3<1.26,>=1.20; python_version != "3.4" in c:\users\argenisleon\anaconda3\lib\site-packages (from botocore<1.17.45,>=1.17.44->aiobotocore>=1.0.7->coiled) (1.24.3) Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in c:\users\argenisleon\anaconda3\lib\site-packages (from botocore<1.17.45,>=1.17.44->aiobotocore>=1.0.7->coiled) (0.9.4) Requirement already satisfied: docutils<0.16,>=0.10 in c:\users\argenisleon\appdata\roaming\python\python37\site-packages (from botocore<1.17.45,>=1.17.44->aiobotocore>=1.0.7->coiled) (0.15.2) Requirement already satisfied: MarkupSafe>=0.23 in c:\users\argenisleon\anaconda3\lib\site-packages (from Jinja2>=2.7->bokeh>=1.0.0; extra == "complete"->dask[complete]>=2.23.0->coiled) (1.1.1) Requirement already satisfied: pyparsing>=2.0.2 in c:\users\argenisleon\anaconda3\lib\site-packages (from packaging>=16.8->bokeh>=1.0.0; extra == "complete"->dask[complete]>=2.23.0->coiled) (2.4.6)
# !coiled login --token $coiled_token
Traceback (most recent call last):
File "c:\users\argenisleon\anaconda3\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "c:\users\argenisleon\anaconda3\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\argenisleon\Anaconda3\Scripts\coiled.exe\__main__.py", line 7, in <module>
File "c:\users\argenisleon\anaconda3\lib\site-packages\click\core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "c:\users\argenisleon\anaconda3\lib\site-packages\click\core.py", line 782, in main
rv = self.invoke(ctx)
File "c:\users\argenisleon\anaconda3\lib\site-packages\click\core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "c:\users\argenisleon\anaconda3\lib\site-packages\click\core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "c:\users\argenisleon\anaconda3\lib\site-packages\click\core.py", line 610, in invoke
return callback(*args, **kwargs)
File "c:\users\argenisleon\anaconda3\lib\site-packages\coiled\cli\login.py", line 17, in login
handle_credentials(server=server, token=token, save=True)
File "c:\users\argenisleon\anaconda3\lib\asyncio\base_events.py", line 583, in run_until_complete
return future.result()
File "c:\users\argenisleon\anaconda3\lib\site-packages\coiled\utils.py", line 137, in handle_credentials
await handle_api_exception(response)
File "c:\users\argenisleon\anaconda3\lib\site-packages\coiled\utils.py", line 64, in handle_api_exception
raise exception_cls(error_body)
Exception: {'detail': 'Authentication credentials were not provided.'}
import coiled
coiled.create_software_environment(name='zarr', conda=['xarray', 'zarr','dask=2.2.23','distributed'])
coiled.create_cluster_configuration(
name="bumblebee",
worker_memory="16 GiB",
worker_cpu=4,
scheduler_memory="4 GiB",
scheduler_cpu=1,
software="coiled/default"
)
cluster.name #salvar en bumblebee
coiled.list_clusters()
cluster = coiled.Cluster(configuration='bumblebee')
cluster.name # argenisleon-xxxxxxxx-x
coiled.list_clusters() # {"argenisleon-xxxxxxxx-x":{...}}
from dask.distributed import Client
client = Client(name=cluster.name)
print('Dashboard:', client.dashboard_link)
dir(coiled)
import dask.dataframe as dd
df = dd.read_csv(
"s3://nyc-tlc/trip data/yellow_tripdata_2019-*.csv",
dtype={
"payment_type": "UInt8",
"VendorID": "UInt8",
"passenger_count": "UInt8",
"RatecodeID": "UInt8",
},
storage_options={"anon": True},
blocksize="16 MiB",
).persist()
df.groupby("passenger_count").tip_amount.mean().compute()
op
--------------------------------------------------------------------------- NameError Traceback (most recent call last) <ipython-input-1-99393fff965a> in <module> ----> 1 op NameError: name 'op' is not defined