#!/usr/bin/env python
# coding: utf-8

# # Distinguish Failed Experiments
# 
# Modeling runs can fail for a number of reasons. When logging with rubicon_ml, a failed run may result in an empty or incomplete experiment. In this example, we'll walk through how to handle such experiments.

# First lets simulate the problem. To do this we'll create an estimator that will fail on it's `fit()` 30% of the time.  We will consider any pipeline that has a learned attribute `self.state_` to have "succeeded," and any that does not to have "failed."

# In[1]:


from sklearn.base import BaseEstimator
import random

class BadEstimator(BaseEstimator):
    def __init__(self):
        super().__init__()
        self.knn = KNeighborsClassifier(n_neighbors=2)
    def fit(self, X, y):
        self.knn.fit(X, y)
        output=random.random()
        if output>.3:
            self.state_=output
    def score(self, X):
        knn_score = self.knn.score(X)
        return knn_score


# Next, lets create a `rubicon_ml` project to log our experimenation to.

# In[2]:


from rubicon_ml.sklearn import make_pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.impute import SimpleImputer
from rubicon_ml import Rubicon

random.seed(17)

rubicon = Rubicon(
    persistence="memory",
)
project = rubicon.get_or_create_project(name="Failed Experiments")


# Now let's create a `rubicon_ml.sklearn` pipeline with this sporadically failing estimator and attempt to `fit` the pipeline twenty times.  Tag any experiment that doesn't have a valid `state_` attribute for failures with `exp.add_tags(["failed"])` and passed experiments with `exp.add_tags(["passed"])`.

# In[3]:


X = [[1], [1], [1], [1]]
y = [1, 1, 1, 1]
for _ in range(20):
    pipe=make_pipeline(project, SimpleImputer(strategy="mean"),BadEstimator())
    pipe.fit(X,y)
    if not hasattr(pipe["badestimator"],"state_"):
         pipe.experiment.add_tags(["failed"])
    else:
         pipe.experiment.add_tags(["passed"])


# Finally, we can now retrieve all our failed experiments by passing the `tags=["failed"]` to `project.experiments()`. 

# In[4]:


for exp in project.experiments(tags=["failed"]):
    print(exp)


# We can also see that the pipeline passed ~70% of the time and ~30% of the time.

# In[5]:


len(project.experiments(tags=["failed"]))


# In[6]:


len(project.experiments(tags=["passed"]))