lst = [1,2,4,5]
list(map(lambda x: 'lower' if x < 3 else 'higher', lst))

# %load_ext watermark
# %watermark -a "Sebastian Raschka" -u -d -v -p numpy,pandas,pyprind,matplotlib,nltk,sklearn

# # 압축파일을 풀어서 저장하는 함수
# import gzip
# with gzip.open('movie_data.csv.gz') as f_in, open('movie_data.csv', 'wb') as f_out:
#     f_out.writelines(f_in)

# import nltk
# nltk.download('stopwords')

def get_minibatch(doc_stream, size):
    docs, y = [], []
    try:
        for _ in range(size):
            text, label = next(doc_stream)
            docs.append(text)
            y.append(label)
    except StopIteration:
        return None, None
    return docs, y

from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.linear_model import SGDClassifier

vect = HashingVectorizer(decode_error='ignore', 
                         n_features=2**21,
                         preprocessor=None, 
                         tokenizer=tokenizer)

# max_iter 를 설정시 tol=0.01 값을 0.21 ~ 0.01 사이의 값으로 추가를 해주면 warning이 사라짐
clf = SGDClassifier(loss='log', random_state=1, max_iter=1, tol=0.01)
doc_stream = stream_docs(path='data/movie_data.csv')

clf = clf.partial_fit(X_test, y_test)

import pickle, os
# dest = os.path.join('movieclassifier', 'pkl_objects')
# if not os.path.exists(dest):
#     os.makedirs(dest)

pickle.dump(stop, open('data/stopwords.pkl', 'wb'), protocol=4)   
pickle.dump(clf, open('data/classifier.pkl', 'wb'), protocol=4)

%%writefile movieclassifier/vectorizer.py

# 나중에 임포트할 수 있도록 별도의 파일에 `HashingVectorizer`를 저장합니다.
from sklearn.feature_extraction.text import HashingVectorizer
import re, os, pickle
cur_dir = os.path.dirname(__file__)
stop = pickle.load(open('data/stopwords.pkl'), 'rb')

def tokenizer(text):
    text = re.sub('<[^>]*>', '', text)
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)',
                           text.lower())
    text = re.sub('[\W]+', ' ', text.lower()) \
                   + ' '.join(emoticons).replace('-', '')
    tokenized = [w for w in text.split() if w not in stop]
    return tokenized

vect = HashingVectorizer(decode_error='ignore',
                         n_features=2**21,
                         preprocessor=None,
                         tokenizer=tokenizer)

# import os
# os.chdir('movieclassifier')

%%reset
import numpy as np
label = {0:'양성', 1:'음성'}

example = ['I love this movie']
X = vect.transform(example)
print('예측: %s\n확률: %.2f%%' %\
      (label[clf.predict(X)[0]], 
       np.max(clf.predict_proba(X))*100))

import sqlite3

conn = sqlite3.connect('reviews.sqlite')
c = conn.cursor()
c.execute('DROP TABLE IF EXISTS review_db')
c.execute('CREATE TABLE review_db (review TEXT, sentiment INTEGER, date TEXT)')

example1 = 'I love this movie'
c.execute("INSERT INTO review_db (review, sentiment, date) VALUES (?, ?, DATETIME('now'))", (example1, 1))

example2 = 'I disliked this movie'
c.execute("INSERT INTO review_db (review, sentiment, date) VALUES (?, ?, DATETIME('now'))", (example2, 0))

conn.commit()
conn.close()

conn = sqlite3.connect('reviews.sqlite')
c = conn.cursor()
c.execute("SELECT * FROM review_db WHERE date BETWEEN '2017-01-01 10:10:10' AND DATETIME('now')")
results = c.fetchall()
conn.close()

print(results)

import shutil

os.chdir('../movieclassifier_with_update')
shutil.copyfile('../movieclassifier/pkl_objects/classifier.pkl',
                './pkl_objects/classifier.pkl')

# SQLite 데이터베이스에 저장된 데이터로 분류기를 업데이트하는 함수를 정의합니다:
import pickle, sqlite3
import numpy as np

# 로컬 디렉토리에서 HashingVectorizer를 임포트합니다
from vectorizer import vect

def update_model(db_path, model, batch_size=10000):

    conn = sqlite3.connect(db_path)
    c    = conn.cursor()
    c.execute('SELECT * from review_db')    
    results  = c.fetchmany(batch_size)
    while results:
        data = np.array(results)
        X = data[:, 0]
        y = data[:, 1].astype(int)
        classes = np.array([0, 1])
        X_train = vect.transform(X)
        clf.partial_fit(X_train, y, classes=classes)
        results = c.fetchmany(batch_size) 
    conn.close()
    return None

# 모델을 업데이트합니다:
cur_dir = '.'

# app.py 파일에 이 코드를 삽입했다면 다음 경로를 사용하세요.
# import os
# cur_dir = os.path.dirname(__file__)

clf = pickle.load(open(os.path.join(cur_dir,
                 'pkl_objects',
                 'classifier.pkl'), 'rb'))
db = os.path.join(cur_dir, 'reviews.sqlite')

update_model(db_path=db, model=clf, batch_size=10000)

# classifier.pkl 파일을 업데이트하려면 다음 주석을 해제하세요.

# pickle.dump(clf, open(os.path.join(cur_dir, 
#             'pkl_objects', 'classifier.pkl'), 'wb')
#             , protocol=4)