#!/usr/bin/env python
# coding: utf-8

# <img src="https://www.comet.com/images/logo_comet_light.png" width="200px"/>
# 
# # Comet.ml REST API
# 
# *This page is available as an executable or viewable **Jupyter Notebook**:* <br/>
# <a href="https://mybinder.org/v2/gh/comet-ml/comet-examples/master?filepath=notebooks%2FComet-REST-API.ipynb" target="_parent"><img align="left" src="https://mybinder.org/badge_logo.svg"></a>
# <a href="https://nbviewer.jupyter.org/github/comet-ml/comet-examples/blob/master/notebooks/Comet-REST-API.ipynb" target="_parent"><img align="right" src="https://raw.githubusercontent.com/jupyter/design/master/logos/Badges/nbviewer_badge.png" width="109" height="20"></a>
# <br/>
# <hr/>
# Comet.ml has an extensive interface to all of your data using a [REST API](https://en.wikipedia.org/wiki/Representational_state_transfer) through [Comet.ml endpoints](https://www.comet.com/docs/rest-api/endpoints/). Now, you can access this information easily through the Comet.ml Python SDK. Requires version comet_ml version 1.0.40 or greater.
# 
# ## Setup
# 
# To run the following experiments, you'll need to set your COMET_API_KEY and COMET_REST_API_KEY. The easiest way to to this is to set the values in a cell like this:
# 
# ```python
# import comet_ml
# 
# comet_ml.save(api_key="...", rest_api_key="...")
# ```
# where you replace the ...'s with your keys.
# 
# You can get your COMET_API_KEY under your quickstart link (replace YOUR_USERNAME with your Comet.ml username):
# 
# https://www.comet.com/YOUR_USERNAME/quickstart
# 
# You can get your COMET_REST_API_KEY under your settings (replace YOUR_USERNAME with your Comet.ml username):
# 
# https://www.comet.com/YOUR_USERNAME/settings/account
# 
# ## Quick Overview
# 
# To access the REST API through the Comet.ml SDK, you will need to make an API() instance. First, we import the API class and other libraries we will need:

# In[2]:


from comet_ml import API
import comet_ml

import matplotlib.pyplot as plt

get_ipython().run_line_magic('matplotlib', 'inline')


# and create the API instance:

# In[3]:


comet_api = API()


# Using the `comet_api` instance, you can get the name of your workspaces:

# In[4]:


comet_api.get()


# If you reference your workspace by name using comet_api.get(WORKSPACE_NAME), you'll see your projects:

# In[5]:


comet_api.get("dsblank")


# Or, get the projects from another user's workspace:

# In[6]:


comet_api.get("cometpublic")


# Using the same method, you can refer to a project by name:

# In[7]:


comet_api.get("dsblank", "keras")


# Or, using the slash delimiter:

# In[8]:


comet_api.get("dsblank/keras")


# And one more level, get the details of an experiment:

# In[9]:


comet_api.get("dsblank", "keras", "5dc346a883964bd2b8864c40940fd864")


# Or, again using the slash shorthand:

# In[10]:


comet_api.get("dsblank/keras/5dc346a883964bd2b8864c40940fd864")


# Let's get the experiment and save it to a variable named `exp`:

# In[11]:


exp = comet_api.get("dsblank/keras/51cf6e588a3346cdb560fd0c09d49610")


# In[12]:


exp


# There are a number of items you get from the APIExperiment instance:

# In[13]:


help(exp)


# For example, we can explore the `other` property, which shows items saved with Experiment.log_other(NAME, VALUE):

# In[15]:


exp.get_others_summary()


# In this example, we see that the experiment has the `Name` "example 001". We can use `Name` to also look up experiments:

# In[16]:


exp = comet_api.get("dsblank/keras/example 001")
exp.id, exp.name


# Perhaps one of the most useful abilities for the REST API is to access your experiment's data in order to create a variation of a plot. To access the raw metric data, use the `get_metrics` property of the experiment:

# In[36]:


exp.get_metrics()


# Thus, there were over 2000 metrics logged during the training of this experiment. We can get the first using indexing with an integer:

# In[37]:


exp.get_metrics()[0]


# That shows that the "acc" (accuracy) metric had a value of about 0.09 at step 1 of the experiment.
# 
# We can also use a string as an index to query all of the dictionaries in `get_metrics_summary` to only give those values at each step, like so:

# In[38]:


acc_metrics = exp.get_metrics("acc")
acc_metrics


# In[39]:


len(acc_metrics)


# In[40]:


acc_metrics[0]


# That's it for a quick overview. Now let's look in detail at each component, and introduce the low-level REST API as well.

# ## Workspaces
# 
# By default, comet_api.get() reports only your workspace names:

# In[45]:


comet_api.get()


# You can also interate over those names:

# In[46]:


for workspace in comet_api.get():
    print(workspace)


# As we saw above, you can also access other public workspaces as well:

# In[47]:


comet_api.get("cometpublic")


# ## Projects
# 
# Under get(WORKSPACE_NAME), you'll find the projects:

# In[48]:


comet_api.get("cometpublic")


# In[49]:


project = comet_api.get("cometpublic", "comet-notebooks")
## OR:
# project = comet_api.get("cometpublic/comet-notebooks")


# If you just print out, or iterate over a project, you get access to the experiment ids:

# In[50]:


project


# In[51]:


project[0].id, project[0].name


# In[52]:


project[1].id, project[1].name


# ## Experiments
# 
# Continuing with the dictionary-like access, you can see and iterate over the experiment ids:

# In[53]:


comet_api.get("cometpublic", "comet-notebooks")


# In[54]:


exp = comet_api.get(
    "cometpublic", "comet-notebooks", "d21f94a1c71841d2961da1e6ddb5ab20"
)
## OR
# exp = comet_api.get("cometpublic/comet-notebooks/d21f94a1c71841d2961da1e6ddb5ab20")
exp


# In[55]:


exp = comet_api.get("cometpublic", "comet-notebooks", "example 001")
## OR
## exp = comet_api.get("cometpublic/comet-notebooks/example 001")
exp


# ### Regular Expression Experiment Name Matching
# 
# You can also use regular expressions as the name for the experiment:

# In[56]:


comet_api.get("cometpublic", "comet-notebooks", "example.*")


# ### Experiment Properties

# In this brief dictionary representation, you will see that `other`, `metrics` and `parameters` give a list of names. However, as we saw above, you can get more information through properties of those same names:
# 
# names through exp.data["properties"] and more detail at exp.properties:

# In[58]:


exp.get_parameters_summary()


# names through exp.data["other"] and more detail at exp.other:

# In[60]:


exp.get_others_summary()[0]["name"], exp.get_others_summary()[0]["valueCurrent"]


# names through exp.data["metrics"] and more detail at exp.metrics:

# In[61]:


exp.get_metrics()


# You can see all of the methods and propeties on an experiment instance:

# In[62]:


help(exp)


# For example, just like when creating and logging data, you can also use the `.display()` method to show the Comet.ml page for that experiment right in the notebook:

# In[63]:


exp.display()


# You can get an existing experiment:

# In[ ]:


e = comet_api.get("dsblank", "chainer", "596d91ae1dbc420c9b13a3ced858de3c")
API_KEY = comet_ml.get_config()["comet.api_key"]
ee = comet_ml.start(API_KEY, experiment_key=e.id)


# You can make changes to the saved data using the existing experiment:
# 
# https://www.comet.com/docs/v2/api-and-sdk/python-sdk/reference/ExistingExperiment/

# In[65]:


ee.end()


# ### Examples
# 
# Comet.ml is working on a query API which will allow highly efficient queries of your data. However, you can also write your own query of sorts.
# 
# Here is some code that prints out the names of experiments that have associated HTML (this can take a long time if you have many experiments):

# In[ ]:


get_ipython().run_cell_magic('time', '', 'for workspace in comet_api.get():\n    print("processing workspace", workspace, "...")\n    for project in comet_api.get(workspace):\n        print("    processing project", project, "...")\n        for exp_id in comet_api.get(workspace, project):\n            exp = comet_api.get(workspace, project, exp_id)\n            if exp.html != None:\n                print("found html in %s!" % exp.key)\n')


# Here is a function that will find the first experiment that has associated images:

# In[67]:


def find_image():
    for workspace in comet_api.get():
        for project in comet_api.get(workspace):
            for exp_id in comet_api.get(workspace, project):
                exp = comet_api.get(workspace, project, exp_id)
                if exp.images != []:
                    return exp


# In[ ]:


find_image()


# Now, we get the experiment API and explore the `.images` property:

# In[69]:


comet_api.get("dsblank/pytorch/3b56611892b7447aa8c4486a6eeb27d0").get_images()


# We can get a URL for the image, and display it in the notebook:

# In[72]:


url = comet_api.get("dsblank/pytorch/3b56611892b7447aa8c4486a6eeb27d0").get_asset_list(
    asset_type="image"
)[0]["link"]
url


# In[73]:


from IPython.display import Image


# In[74]:


Image(url=url)


# Now, let's write a short program that will find the run with the best accuracy given a workspace/project string:

# In[75]:


def find_best_run(project):
    runs = []
    for exp_id in comet_api.get(project):
        exp = comet_api.get(project, experiment=exp_id)
        accs = [x["valueMax"] for x in exp.get_metrics() if x["name"] == "acc"]
        if len(accs) > 0:
            runs.append([float(accs[0]), exp])
    if runs:
        return sorted(runs, key=lambda v: v[0], reverse=True)[0]


# In[ ]:


find_best_run("cometpublic/fasttext")


# Can we get all of the `hidden_size` parameter values for the experiments in dsblank/pytorch?

# In[ ]:


[
    [p["valueCurrent"] for p in exp.get_parameters()]
    for exp in comet_api.get("dsblank/pytorch")
]


# In[79]:


experiments = [
    [
        (exp, "hidden_size", int(param["valueCurrent"]))
        for param in exp.get_parameters_summary()
        if param["name"] == "hidden_size"
    ]
    for exp in comet_api.get("dsblank/pytorch")
]
experiments = [e[0] for e in experiments if len(e) > 0]


# In[80]:


experiments[0]


# ### Assets
# 
# To get an asset, you need to get the asset_id. You can see all of the assets related to a project using the `APIExperiment.asset_list`:

# In[81]:


def find_asset():
    for ws in comet_api.get():
        for pj in comet_api.get(ws):
            for exp in comet_api.get(ws, pj):
                if exp.get_asset_list() != []:
                    return (exp, exp.get_asset_list())


exp, elist = find_asset()


# From there, you can use the `APIExperiment.get_asset(asset_id)` method to get the asset.

# In[ ]:


h5 = exp.get_asset("a6c75ebcfd344c06a4934b97641ea87e")


# We hope that this gives you some ideas of how you can use the Comet REST API!