import nltk
from numpy.random import randint, seed
from sklearn.feature_extraction.text import CountVectorizer #n-Gram 만들어줌
#train을 위한 텍스트 데이터
my_text = """Machine learning is the scientific study of algorithms and statistical models that computer systems use to effectively perform a specific task without using explicit instructions, relying on patterns and inference instead. It is seen as a subset of artificial intelligence. Machine learning algorithms build a mathematical model of sample data, known as "training data", in order to make predictions or decisions without being explicitly programmed to perform the task.[1][2]:2 Machine learning algorithms are used in the applications of email filtering, detection of network intruders, and computer vision, where it is infeasible to develop an algorithm of specific instructions for performing the task. Machine learning is closely related to computational statistics, which focuses on making predictions using computers. The study of mathematical optimization delivers methods, theory and application domains to the field of machine learning. Data mining is a field of study within machine learning, and focuses on exploratory data analysis through unsupervised learning In its application across business problems, machine learning is also referred to as predictive analytics."""
my_text = [my_text.lower()] # 소문자로 변환 => Required by the CountVectorizer().
my_text
['machine learning is the scientific study of algorithms and statistical models that computer systems use to effectively perform a specific task without using explicit instructions, relying on patterns and inference instead. it is seen as a subset of artificial intelligence. machine learning algorithms build a mathematical model of sample data, known as "training data", in order to make predictions or decisions without being explicitly programmed to perform the task.[1][2]:2 machine learning algorithms are used in the applications of email filtering, detection of network intruders, and computer vision, where it is infeasible to develop an algorithm of specific instructions for performing the task. machine learning is closely related to computational statistics, which focuses on making predictions using computers. the study of mathematical optimization delivers methods, theory and application domains to the field of machine learning. data mining is a field of study within machine learning, and focuses on exploratory data analysis through unsupervised learning in its application across business problems, machine learning is also referred to as predictive analytics.']
n = 3
n_min = n #n이 3이면 3까지만 가져오도록 함
n_max = n
n_gram_type = 'word'
vectorizer = CountVectorizer(ngram_range=(n_min,n_max), analyzer = n_gram_type) #CountVectorizer: ngram의 range를 보냄
n_grams = vectorizer.fit(my_text).get_feature_names() #get_feature_names: 리스트로 ngram 가져옴
n_gram_cts = vectorizer.transform(my_text).toarray() #출력물은 배열의 배열
#ngram에 해당하는 횟수
#transform: 도수분포표
#toarray를 안하면 도수분포표가 듬성듬성한 행렬로 만들어짐
n_gram_cts = list(n_gram_cts[0]) #첫 번째 원소 가져와야 제대로 된 도수분포표 나옴
list(zip(n_grams,n_gram_cts)) #도수분포표와 ngram 맞물리기위해 zip
[('across business problems', 1),
('algorithm of specific', 1),
('algorithms and statistical', 1),
('algorithms are used', 1),
('algorithms build mathematical', 1),
('also referred to', 1),
('an algorithm of', 1),
('analysis through unsupervised', 1),
('and application domains', 1),
('and computer vision', 1),
('and focuses on', 1),
('and inference instead', 1),
('and statistical models', 1),
('application across business', 1),
('application domains to', 1),
('applications of email', 1),
('are used in', 1),
('artificial intelligence machine', 1),
('as predictive analytics', 1),
('as subset of', 1),
('as training data', 1),
('being explicitly programmed', 1),
('build mathematical model', 1),
('business problems machine', 1),
('closely related to', 1),
('computational statistics which', 1),
('computer systems use', 1),
('computer vision where', 1),
('computers the study', 1),
('data analysis through', 1),
('data in order', 1),
('data known as', 1),
('data mining is', 1),
('decisions without being', 1),
('delivers methods theory', 1),
('detection of network', 1),
('develop an algorithm', 1),
('domains to the', 1),
('effectively perform specific', 1),
('email filtering detection', 1),
('explicit instructions relying', 1),
('explicitly programmed to', 1),
('exploratory data analysis', 1),
('field of machine', 1),
('field of study', 1),
('filtering detection of', 1),
('focuses on exploratory', 1),
('focuses on making', 1),
('for performing the', 1),
('in its application', 1),
('in order to', 1),
('in the applications', 1),
('infeasible to develop', 1),
('inference instead it', 1),
('instead it is', 1),
('instructions for performing', 1),
('instructions relying on', 1),
('intelligence machine learning', 1),
('intruders and computer', 1),
('is also referred', 1),
('is closely related', 1),
('is field of', 1),
('is infeasible to', 1),
('is seen as', 1),
('is the scientific', 1),
('it is infeasible', 1),
('it is seen', 1),
('its application across', 1),
('known as training', 1),
('learning algorithms are', 1),
('learning algorithms build', 1),
('learning and focuses', 1),
('learning data mining', 1),
('learning in its', 1),
('learning is also', 1),
('learning is closely', 1),
('learning is the', 1),
('machine learning algorithms', 2),
('machine learning and', 1),
('machine learning data', 1),
('machine learning is', 3),
('make predictions or', 1),
('making predictions using', 1),
('mathematical model of', 1),
('mathematical optimization delivers', 1),
('methods theory and', 1),
('mining is field', 1),
('model of sample', 1),
('models that computer', 1),
('network intruders and', 1),
('of algorithms and', 1),
('of artificial intelligence', 1),
('of email filtering', 1),
('of machine learning', 1),
('of mathematical optimization', 1),
('of network intruders', 1),
('of sample data', 1),
('of specific instructions', 1),
('of study within', 1),
('on exploratory data', 1),
('on making predictions', 1),
('on patterns and', 1),
('optimization delivers methods', 1),
('or decisions without', 1),
('order to make', 1),
('patterns and inference', 1),
('perform specific task', 1),
('perform the task', 1),
('performing the task', 1),
('predictions or decisions', 1),
('predictions using computers', 1),
('problems machine learning', 1),
('programmed to perform', 1),
('referred to as', 1),
('related to computational', 1),
('relying on patterns', 1),
('sample data known', 1),
('scientific study of', 1),
('seen as subset', 1),
('specific instructions for', 1),
('specific task without', 1),
('statistical models that', 1),
('statistics which focuses', 1),
('study of algorithms', 1),
('study of mathematical', 1),
('study within machine', 1),
('subset of artificial', 1),
('systems use to', 1),
('task machine learning', 2),
('task without using', 1),
('that computer systems', 1),
('the applications of', 1),
('the field of', 1),
('the scientific study', 1),
('the study of', 1),
('the task machine', 2),
('theory and application', 1),
('through unsupervised learning', 1),
('to as predictive', 1),
('to computational statistics', 1),
('to develop an', 1),
('to effectively perform', 1),
('to make predictions', 1),
('to perform the', 1),
('to the field', 1),
('training data in', 1),
('unsupervised learning in', 1),
('use to effectively', 1),
('used in the', 1),
('using computers the', 1),
('using explicit instructions', 1),
('vision where it', 1),
('where it is', 1),
('which focuses on', 1),
('within machine learning', 1),
('without being explicitly', 1),
('without using explicit', 1)]
Trigram근사란?
바로 직전의 두 단어만 가지고 그 다음 단어가 나올 확률에 구해 예측하는 것
n = 3
n_min = n
n_max = n
n_gram_type = 'word'
vectorizer = CountVectorizer(ngram_range=(n_min,n_max), analyzer = n_gram_type)
n_grams = vectorizer.fit(my_text).get_feature_names() #Trigram 만듬
my_dict = {}
for a_gram in n_grams:
#ngram가져와서 trigram이면 trigram에서 첫번째 두단어만 쪼갬, 그다음단어는 value로 넣음
#트라이 w1,w2,w3을 w1w2,w1로 쪼갬
#어폴시는 w3,w1w2 w1w2 두개가 발생했을 때 세번째 단어가 발생할 확률
#두단어 w1w2를 키로 두고 3을 value로 둠
words = nltk.word_tokenize(a_gram)
a_nm1_gram = ' '.join(words[0:n-1]) # (n-1)-Gram.
next_word = words[-1] # Word after the a_nm1_gram.
if a_nm1_gram not in my_dict.keys(): #딕셔너리
my_dict[a_nm1_gram] = [next_word] #딕셔너리 키에 없다면 리스트로 새로 만들고, # a_nm1_gram is a new key. So, initialize the dictionary entry.
#키 #value는 그 다음에 나오는 단어를 리스트로 추가
else: #키에 이미 있다면 추가
my_dict[a_nm1_gram] += [next_word] # an_nm1_gram is already in the dictionary.
#2개를 ㅜn-1로
#1개를 넥스트
#딕셔너리 보기
my_dict
{'across business': ['problems'],
'algorithm of': ['specific'],
'algorithms and': ['statistical'],
'algorithms are': ['used'],
'algorithms build': ['mathematical'],
'also referred': ['to'],
'an algorithm': ['of'],
'analysis through': ['unsupervised'],
'and application': ['domains'],
'and computer': ['vision'],
'and focuses': ['on'],
'and inference': ['instead'],
'and statistical': ['models'],
'application across': ['business'],
'application domains': ['to'],
'applications of': ['email'],
'are used': ['in'],
'artificial intelligence': ['machine'],
'as predictive': ['analytics'],
'as subset': ['of'],
'as training': ['data'],
'being explicitly': ['programmed'],
'build mathematical': ['model'],
'business problems': ['machine'],
'closely related': ['to'],
'computational statistics': ['which'],
'computer systems': ['use'],
'computer vision': ['where'],
'computers the': ['study'],
'data analysis': ['through'],
'data in': ['order'],
'data known': ['as'],
'data mining': ['is'],
'decisions without': ['being'],
'delivers methods': ['theory'],
'detection of': ['network'],
'develop an': ['algorithm'],
'domains to': ['the'],
'effectively perform': ['specific'],
'email filtering': ['detection'],
'explicit instructions': ['relying'],
'explicitly programmed': ['to'],
'exploratory data': ['analysis'],
'field of': ['machine', 'study'],
'filtering detection': ['of'],
'focuses on': ['exploratory', 'making'],
'for performing': ['the'],
'in its': ['application'],
'in order': ['to'],
'in the': ['applications'],
'infeasible to': ['develop'],
'inference instead': ['it'],
'instead it': ['is'],
'instructions for': ['performing'],
'instructions relying': ['on'],
'intelligence machine': ['learning'],
'intruders and': ['computer'],
'is also': ['referred'],
'is closely': ['related'],
'is field': ['of'],
'is infeasible': ['to'],
'is seen': ['as'],
'is the': ['scientific'],
'it is': ['infeasible', 'seen'],
'its application': ['across'],
'known as': ['training'],
'learning algorithms': ['are', 'build'],
'learning and': ['focuses'],
'learning data': ['mining'],
'learning in': ['its'],
'learning is': ['also', 'closely', 'the'],
'machine learning': ['algorithms', 'and', 'data', 'is'],
'make predictions': ['or'],
'making predictions': ['using'],
'mathematical model': ['of'],
'mathematical optimization': ['delivers'],
'methods theory': ['and'],
'mining is': ['field'],
'model of': ['sample'],
'models that': ['computer'],
'network intruders': ['and'],
'of algorithms': ['and'],
'of artificial': ['intelligence'],
'of email': ['filtering'],
'of machine': ['learning'],
'of mathematical': ['optimization'],
'of network': ['intruders'],
'of sample': ['data'],
'of specific': ['instructions'],
'of study': ['within'],
'on exploratory': ['data'],
'on making': ['predictions'],
'on patterns': ['and'],
'optimization delivers': ['methods'],
'or decisions': ['without'],
'order to': ['make'],
'patterns and': ['inference'],
'perform specific': ['task'],
'perform the': ['task'],
'performing the': ['task'],
'predictions or': ['decisions'],
'predictions using': ['computers'],
'problems machine': ['learning'],
'programmed to': ['perform'],
'referred to': ['as'],
'related to': ['computational'],
'relying on': ['patterns'],
'sample data': ['known'],
'scientific study': ['of'],
'seen as': ['subset'],
'specific instructions': ['for'],
'specific task': ['without'],
'statistical models': ['that'],
'statistics which': ['focuses'],
'study of': ['algorithms', 'mathematical'],
'study within': ['machine'],
'subset of': ['artificial'],
'systems use': ['to'],
'task machine': ['learning'],
'task without': ['using'],
'that computer': ['systems'],
'the applications': ['of'],
'the field': ['of'],
'the scientific': ['study'],
'the study': ['of'],
'the task': ['machine'],
'theory and': ['application'],
'through unsupervised': ['learning'],
'to as': ['predictive'],
'to computational': ['statistics'],
'to develop': ['an'],
'to effectively': ['perform'],
'to make': ['predictions'],
'to perform': ['the'],
'to the': ['field'],
'training data': ['in'],
'unsupervised learning': ['in'],
'use to': ['effectively'],
'used in': ['the'],
'using computers': ['the'],
'using explicit': ['instructions'],
'vision where': ['it'],
'where it': ['is'],
'which focuses': ['on'],
'within machine': ['learning'],
'without being': ['explicitly'],
'without using': ['explicit']}
위에서 만든 딕셔너리 가지고 그 다음 단어를 확률에 의해 예측
# Helper function that picks the following word.
def predict_next(a_nm1_gram):
value_list_size = len(my_dict[a_nm1_gram]) # 키에 해당하는 값의 길이 = a_nm1_gram.
i_pick = randint(0, value_list_size) # 0 ~ value_list_size의 랜덤숫자
return(my_dict[a_nm1_gram][i_pick]) # 랜덤으로 선택한 다음 다음 단어를 반환
# Test.
input_str = 'order to' # Has to be a VALID (n-1)-Gram!
predict_next(input_str) #그 다음 단어를 확률적으로 뽑아서 줌
#위 결과에서 'order to': ['make'] 2개뿐임
'make'
# Another test.
# 10 번 반복하고 다음 단어가 확률에 따라 무작위로 선택되는지 확인
input_str = 'machine learning' # Has to be a VALID (n-1)-Gram!
for i in range(10):
print(predict_next(input_str))
#machine learning을 넣으면 확률에 따라 결과값이 다르게 나옴
#range에 해당하는 원소를 랜덤으로 뽑음
#반복되는게 있으면 더 높은 확률로 뽑힐 것 (그래서 결과값이 다 다름)
is and algorithms algorithms and and data data data algorithms
seed 사용하는 방법도 있음
#random seed 초기화
seed(123)
# A seed string has to be input by the user.
my_seed_str = 'machine learning' # Has to be a VALID (n-1)-Gram!
# my_seed_str = 'in order' # Has to be a VALID (n-1)-Gram!
a_nm1_gram = my_seed_str #seed가지고 n-1gram 만듬
output_string = my_seed_str # Initialize the output string.
while a_nm1_gram in my_dict:
output_string += " " + predict_next(a_nm1_gram) #n-1바탕으로 예측해서 추가시킴
words = nltk.word_tokenize(output_string)
a_nm1_gram = ' '.join(words[-n+1:]) #만들어논 단어를 토큰화해서 끝에서 두개만 뽑아옴 해서
#n-1gram bigram 만들어 게속 감 # Update a_nm1_gram.
# Output the predicted sequence.
output_string
#trigram근사에 따라 bigram seed를 주면 그 다음 단어를 예측해 trigram 완성시킴
#완성시킨 끝에 두개를 bifram으로 뽑아 그걸 바탕으로 3번째 단어 예측해 완성시킴
#이 과정이 계속 돌고돌아 문장만들어 감
'machine learning data mining is field of study within machine learning data mining is field of machine learning algorithms are used in the applications of email filtering detection of network intruders and computer vision where it is infeasible to develop an algorithm of specific instructions for performing the task machine learning and focuses on making predictions using computers the study of algorithms and statistical models that computer systems use to effectively perform specific task without using explicit instructions relying on patterns and inference instead it is seen as subset of artificial intelligence machine learning and focuses on exploratory data analysis through unsupervised learning in its application across business problems machine learning and focuses on exploratory data analysis through unsupervised learning in its application across business problems machine learning and focuses on exploratory data analysis through unsupervised learning in its application across business problems machine learning is closely related to computational statistics which focuses on exploratory data analysis through unsupervised learning in its application across business problems machine learning data mining is field of machine learning is closely related to computational statistics which focuses on making predictions using computers the study of algorithms and statistical models that computer systems use to effectively perform specific task without using explicit instructions relying on patterns and inference instead it is seen as subset of artificial intelligence machine learning algorithms are used in the applications of email filtering detection of network intruders and computer vision where it is infeasible to develop an algorithm of specific instructions for performing the task machine learning algorithms build mathematical model of sample data known as training data in order to make predictions or decisions without being explicitly programmed to perform the task machine learning is the scientific study of algorithms and statistical models that computer systems use to effectively perform specific task without using explicit instructions relying on patterns and inference instead it is seen as subset of artificial intelligence machine learning data mining is field of machine learning is also referred to as predictive analytics'