#!/usr/bin/env python
# coding: utf-8

# <h1> Text Classification using TensorFlow on Cloud ML Engine </h1>
# 
# This notebook illustrates:
# <ol>
# <li> Creating datasets for Machine Learning using BigQuery
# <li> Creating a text classification model using the high-level Estimator API 
# <li> Training on Cloud ML Engine
# <li> Deploying model
# <li> Predicting with model
# </ol>

# In[1]:


# change these to try this notebook out
BUCKET = 'cloud-training-demos-ml'
PROJECT = 'cloud-training-demos'
REGION = 'us-central1'


# In[2]:


import os
os.environ['BUCKET'] = BUCKET
os.environ['PROJECT'] = PROJECT
os.environ['REGION'] = REGION


# In[3]:


get_ipython().run_line_magic('datalab', 'project set -p $PROJECT')


# In[ ]:


get_ipython().system('pip install --upgrade tensorflow')


# In[5]:


import tensorflow as tf
print tf.__version__


# The idea is to look at the title of a newspaper article and figure out whether the article came from the New York Times or from TechCrunch. There are very sophisticated approaches that we can try, but for now, let's go with something very simple.
# 
# <h2> Data exploration and preprocessing in BigQuery </h2>
# <p>
# What does the Hacker News dataset look like?

# In[10]:


get_ipython().run_line_magic('bq', 'query')
SELECT
  url, title, score
FROM
  `bigquery-public-data.hacker_news.stories`
WHERE
  LENGTH(title) > 10
  AND score > 10
LIMIT 10


# Let's do some regular expression parsing in BigQuery to get the source of the newspaper article from the URL. For example, if the url is http://mobile.nytimes.com/...., I want to be left with <i>nytimes</i>. To ensure that the parsing works for all URLs of interest, I'll group by the source to make sure there are no weird names left. This was an iterative process.

# In[26]:


query="""
SELECT
  ARRAY_REVERSE(SPLIT(REGEXP_EXTRACT(url, '.*://(.[^/]+)/'), '.'))[OFFSET(1)] AS source,
  COUNT(title) AS num_articles
FROM
  `bigquery-public-data.hacker_news.stories`
WHERE
  REGEXP_CONTAINS(REGEXP_EXTRACT(url, '.*://(.[^/]+)/'), '.com$')
  AND LENGTH(title) > 10
GROUP BY
  source
ORDER BY num_articles DESC
LIMIT 10
"""


# In[27]:


import google.datalab.bigquery as bq
df = bq.Query(query).execute().result().to_dataframe()
df


# Now that we have good parsing of the URL to get the source, let's put together a dataset of source and titles. This will be our labeled dataset for machine learning.

# In[92]:


query="""
SELECT source, REGEXP_REPLACE(title, '[^a-zA-Z0-9 $.-]', ' ') AS title FROM
(SELECT
  ARRAY_REVERSE(SPLIT(REGEXP_EXTRACT(url, '.*://(.[^/]+)/'), '.'))[OFFSET(1)] AS source,
  title
FROM
  `bigquery-public-data.hacker_news.stories`
WHERE
  REGEXP_CONTAINS(REGEXP_EXTRACT(url, '.*://(.[^/]+)/'), '.com$')
  AND LENGTH(title) > 10
)
WHERE (source = 'github' OR source = 'nytimes' OR source = 'techcrunch')
"""
df = bq.Query(query + " LIMIT 10").execute().result().to_dataframe()
df.head()


# For ML training, we will need to split our dataset into training and evaluation datasets (and perhaps an independent test dataset if we are going to do model or feature selection based on the evaluation dataset).  A simple way to do this is to use the hash of a well-distributed column in our data (See https://www.oreilly.com/learning/repeatable-sampling-of-data-sets-in-bigquery-for-machine-learning).
# <p>
# So, let's do that and save the results as CSV files.

# In[93]:


traindf = bq.Query(query + " AND ABS(MOD(FARM_FINGERPRINT(title), 4)) > 0").execute().result().to_dataframe()
evaldf  = bq.Query(query + " AND ABS(MOD(FARM_FINGERPRINT(title), 4)) = 0").execute().result().to_dataframe()
traindf.head()


# In[86]:


traindf['source'].value_counts()


# In[87]:


evaldf['source'].value_counts()


# In[94]:


traindf.to_csv('train.csv', header=False, index=False, encoding='utf-8', sep='\t')
evaldf.to_csv('eval.csv', header=False, index=False, encoding='utf-8', sep='\t')


# In[95]:


get_ipython().system('head -3 train.csv')


# In[96]:


get_ipython().system('wc -l *.csv')


# In[97]:


get_ipython().run_line_magic('bash', '')
gsutil cp *.csv gs://${BUCKET}/txtcls1/


# <h2> TensorFlow code </h2>
# 
# Please explore the code in this <a href="txtcls1/trainer">directory</a> -- <a href="txtcls1/trainer/model.py">model.py</a> contains the key TensorFlow model and <a href="txtcls1/trainer/task.py">task.py</a> has a main() that launches off the training job.
# 
# However, the following cells should give you an idea of what the model code does:

# In[6]:


import tensorflow as tf
from tensorflow.contrib import lookup
from tensorflow.python.platform import gfile

print tf.__version__
MAX_DOCUMENT_LENGTH = 5  
PADWORD = 'ZYXW'

# vocabulary
lines = ['Some title', 'A longer title', 'An even longer title', 'This is longer than doc length']

# create vocabulary
vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor(MAX_DOCUMENT_LENGTH)
vocab_processor.fit(lines)
with gfile.Open('vocab.tsv', 'wb') as f:
    f.write("{}\n".format(PADWORD))
    for word, index in vocab_processor.vocabulary_._mapping.iteritems():
      f.write("{}\n".format(word))
N_WORDS = len(vocab_processor.vocabulary_)
print '{} words into vocab.tsv'.format(N_WORDS)

# can use the vocabulary to convert words to numbers
table = lookup.index_table_from_file(
  vocabulary_file='vocab.tsv', num_oov_buckets=1, vocab_size=None, default_value=-1)
numbers = table.lookup(tf.constant(lines[0].split()))
with tf.Session() as sess:
  tf.tables_initializer().run()
  print "{} --> {}".format(lines[0], numbers.eval())   


# In[13]:


get_ipython().system('cat vocab.tsv')


# In[7]:


# string operations
titles = tf.constant(lines)
words = tf.string_split(titles)
densewords = tf.sparse_tensor_to_dense(words, default_value=PADWORD)
numbers = table.lookup(densewords)

# now pad out with zeros and then slice to constant length
padding = tf.constant([[0,0],[0,MAX_DOCUMENT_LENGTH]])
padded = tf.pad(numbers, padding)
sliced = tf.slice(padded, [0,0], [-1, MAX_DOCUMENT_LENGTH])

with tf.Session() as sess:
  tf.tables_initializer().run()
  print "titles=", titles.eval(), titles.shape
  print "words=", words.eval()
  print "dense=", densewords.eval(), densewords.shape
  print "numbers=", numbers.eval(), numbers.shape
  print "padding=", padding.eval(), padding.shape
  print "padded=", padded.eval(), padded.shape
  print "sliced=", sliced.eval(), sliced.shape 


# In[31]:


get_ipython().run_line_magic('bash', '')
grep "^def" txtcls1/trainer/model.py


# Let's make sure the code works locally on a small dataset for a few steps.

# In[ ]:


get_ipython().run_line_magic('bash', '')
echo "bucket=${BUCKET}"
rm -rf outputdir
export PYTHONPATH=${PYTHONPATH}:${PWD}/txtcls1
python -m trainer.task \
   --bucket=${BUCKET} \
   --output_dir=outputdir \
   --job-dir=./tmp --train_steps=200


# When I ran it, I got a 41% accuracy after a few steps. Because batchsize=32, 200 steps is essentially 6400 examples -- the full dataset is 72,000 examples, so this is not even the full dataset. And already, we are doing better than random chance.
# <p>
# Once the code works in standalone mode, you can run it on Cloud ML Engine. You can monitor the job from the GCP console in the Cloud Machine Learning Engine section.  Since we have 72,000 examples and batchsize=32, train_steps=36,000 essentially means 16 epochs.

# In[ ]:


get_ipython().run_line_magic('bash', '')
OUTDIR=gs://${BUCKET}/txtcls1/trained_model
JOBNAME=txtcls_$(date -u +%y%m%d_%H%M%S)
echo $OUTDIR $REGION $JOBNAME
gsutil -m rm -rf $OUTDIR
gsutil cp txtcls1/trainer/*.py $OUTDIR
gcloud ml-engine jobs submit training $JOBNAME \
   --region=$REGION \
   --module-name=trainer.task \
   --package-path=$(pwd)/txtcls1/trainer \
   --job-dir=$OUTDIR \
   --staging-bucket=gs://$BUCKET \
   --scale-tier=BASIC --runtime-version=1.2 \
   -- \
   --bucket=${BUCKET} \
   --output_dir=${OUTDIR} \
   --train_steps=36000


# Training finished with an accuracy of 73%.  Obviously, this was trained on a really small dataset and with more data will hopefully come even greater accuracy.

# <h2> Deploy trained model </h2>
# <p>
# Deploying the trained model to act as a REST web service is a simple gcloud call.

# In[20]:


get_ipython().run_line_magic('bash', '')
gsutil ls gs://${BUCKET}/txtcls1/trained_model/export/Servo/


# In[ ]:


get_ipython().run_line_magic('bash', '')
MODEL_NAME="txtcls"
MODEL_VERSION="v1"
MODEL_LOCATION=$(gsutil ls gs://${BUCKET}/txtcls1/trained_model/export/Servo/ | tail -1)
echo "Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes"
#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}
#gcloud ml-engine models delete ${MODEL_NAME}
gcloud ml-engine models create ${MODEL_NAME} --regions $REGION
gcloud ml-engine versions create ${MODEL_VERSION} --model ${MODEL_NAME} --origin ${MODEL_LOCATION}


# <h2> Use model to predict </h2>
# <p>
# Send a JSON request to the endpoint of the service to make it predict which publication the article is more likely to run in. These are actual titles of articles in the New York Times, github, and TechCrunch on June 19.   These titles were not part of the training or evaluation datasets.

# In[28]:


from googleapiclient import discovery
from oauth2client.client import GoogleCredentials
import json

credentials = GoogleCredentials.get_application_default()
api = discovery.build('ml', 'v1beta1', credentials=credentials,
            discoveryServiceUrl='https://storage.googleapis.com/cloud-ml/discovery/ml_v1beta1_discovery.json')

request_data = {'instances':
  [
      {
        'title': 'Supreme Court to Hear Major Case on Partisan Districts'
      },
      {
        'title': 'Furan -- build and push Docker images from GitHub to target'
      },
      {
        'title': 'Time Warner will spend $100M on Snapchat original shows and ads'
      },
  ]
}

parent = 'projects/%s/models/%s/versions/%s' % (PROJECT, 'txtcls', 'v1')
response = api.projects().predict(body=request_data, name=parent).execute()
print "response={0}".format(response)


# As you can see, the trained model predicts that the Supreme Court article is 78% likely to come from New York Times and 22% from TechCrunch. The Docker article is 89% likely to be from GitHub according to the service and the Time Warner one is 100% likely to be from TechCrunch.

# Copyright 2017 Google Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License