#!/usr/bin/env python
# coding: utf-8

# How can we sample *some* of the variables in a pyll configuration space, while assigning values to the others?
# 
# Let's look at a simple example involving 2 variables 'a' and 'b'.
# The 'a' variable controls whether our space returns -1 or some random number, 'b'.
# 
# If we just run optimization normally, then we'll find that 'a' should be 0 (the index of the choice that gives the lowest
# return value.

# In[1]:


from hyperopt import hp, fmin, rand
space = hp.choice('a', [-1, hp.uniform('b', 0, 1)])
best = fmin(fn=lambda x: x, space=space, algo=rand.suggest, max_evals=100)
print(best)


# But what if someone else already set up the space, and we just run the search over the other part of the space, which corresponds to the uniform draw?
# 
# The easiest way to do this is probably to *clone* the search space, while making some substitutions while we're at it.
# We can just make a new search space in which 'a' is no longer a hyperparameter.

# In[2]:


# put the configuration space in a local var
# so that we can work on it.
print(space)


# The transformation we want to make on the search space is to replace the `randint` with a constant value of 1, 
# corresponding to always choosing hyperparameter a to be the second element of the list of choices.
# 
# Now, if you don't have access to the code that generated a search space, then you'll have to go digging around for the
# node you need to replace. There are two approaches you can use to do this: navigation and search.

# In[3]:


from hyperopt import pyll

# The "navigation" approach to finding an internal
# search space node:
randint_node_nav = space.pos_args[0].pos_args[1]
print("by navigation:")
print(randint_node_nav)

# The "search" approach to finding an internal
# search space node:
randint_nodes = [node for node in pyll.dfs(space) if node.name == 'randint']
randint_node_srch, = randint_nodes
print("by search:")
print(randint_node_srch)

assert randint_node_nav == randint_node_srch


# In[4]:


space_with_fixed_a = pyll.clone(space, memo={randint_node_nav: pyll.as_apply(1)})
print(space_with_fixed_a)


# Now, having cloned the space with a new term for the randint, we can search the new space.  I wasn't sure if this would work because I haven't really tested the use of hyperopt_params that wrap around non-random nodes (here we replaced the randint with a constant) but it works for random search:

# In[5]:


best = fmin(fn=lambda x: x, space=space_with_fixed_a, algo=rand.suggest, max_evals=100)
print(best)


# Yep, sure enough: The TPE implementation is broken by a hyperparameter that turns out to be a constant. At implementation time, that was not part of the plan.

# In[6]:


from hyperopt import tpe
best = fmin(fn=lambda x: x, space=space_with_fixed_a, algo=tpe.suggest, max_evals=100)
print(best)


# The TPE algorithm works if we make a different replacement in the graph. If we replace the entire "hyperopt_param" node corresponding to hyperparameter "a", then it works fine.

# In[7]:


space_with_no_a = pyll.clone(space, memo={space.pos_args[0]: pyll.as_apply(1)})
best = fmin(fn=lambda x: x, space=space_with_no_a, algo=tpe.suggest, max_evals=100)
print(best)