In [0]:
! apt-get update
! apt-get install g++ openjdk-8-jdk
! pip3 install nltk konlpy wordcloud matplotlib gensim
! apt-get install fonts-nanum*
! apt-get install fontconfig
! fc-cache -fv
! cp /usr/share/fonts/truetype/nanum/Nanum* /usr/local/lib/python3.6/dist-packages/matplotlib/mpl-data/fonts/ttf/
! rm -rf /content/.cache/matplotlib/*
speech_text = "https://raw.githubusercontent.com/YongBeomKim/nltk_rnd/master/data/pyongyang_fin.txt"
font_file = "/usr/local/lib/python3.6/dist-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicCoding.ttf"
# speech_text = "../data/pyongyang_fin.txt"
# font_file = "../data/D2Coding.ttf"
Ign:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 InRelease
Ign:2 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 InRelease
Get:3 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]
Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release
Get:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release [564 B]
Get:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release.gpg [801 B]
Hit:7 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease
Get:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Packages [6,819 B]
Get:10 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease [15.4 kB]
Get:11 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease [3,609 B]
Hit:12 http://archive.ubuntu.com/ubuntu bionic InRelease
Get:13 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [161 kB]
Get:14 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]
Get:15 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [363 kB]
Get:16 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]
Get:17 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main Sources [1,629 kB]
Get:18 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [956 kB]
Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [727 kB]
Get:20 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main amd64 Packages [782 kB]
Fetched 4,896 kB in 5s (910 kB/s)
Reading package lists... Done
Reading package lists... Done
Building dependency tree
Reading state information... Done
g++ is already the newest version (4:7.3.0-3ubuntu2.1).
g++ set to manually installed.
The following package was automatically installed and is no longer required:
libnvidia-common-410
Use 'apt autoremove' to remove it.
The following additional packages will be installed:
fonts-dejavu-core fonts-dejavu-extra libatk-wrapper-java
libatk-wrapper-java-jni libxxf86dga1 openjdk-8-jre x11-utils
Suggested packages:
openjdk-8-demo openjdk-8-source visualvm icedtea-8-plugin mesa-utils
The following NEW packages will be installed:
fonts-dejavu-core fonts-dejavu-extra libatk-wrapper-java
libatk-wrapper-java-jni libxxf86dga1 openjdk-8-jdk openjdk-8-jre x11-utils
0 upgraded, 8 newly installed, 0 to remove and 8 not upgraded.
Need to get 4,771 kB of archives.
After this operation, 13.1 MB of additional disk space will be used.
Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libxxf86dga1 amd64 2:1.1.4-1 [13.7 kB]
Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 fonts-dejavu-core all 2.37-1 [1,041 kB]
Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 fonts-dejavu-extra all 2.37-1 [1,953 kB]
Get:4 http://archive.ubuntu.com/ubuntu bionic/main amd64 x11-utils amd64 7.7+3build1 [196 kB]
Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 libatk-wrapper-java all 0.33.3-20ubuntu0.1 [34.7 kB]
Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 libatk-wrapper-java-jni amd64 0.33.3-20ubuntu0.1 [28.3 kB]
Get:7 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 openjdk-8-jre amd64 8u191-b12-2ubuntu0.18.04.1 [69.7 kB]
Get:8 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 openjdk-8-jdk amd64 8u191-b12-2ubuntu0.18.04.1 [1,435 kB]
Fetched 4,771 kB in 2s (2,336 kB/s)
Selecting previously unselected package libxxf86dga1:amd64.
(Reading database ... 131294 files and directories currently installed.)
Preparing to unpack .../0-libxxf86dga1_2%3a1.1.4-1_amd64.deb ...
Unpacking libxxf86dga1:amd64 (2:1.1.4-1) ...
Selecting previously unselected package fonts-dejavu-core.
Preparing to unpack .../1-fonts-dejavu-core_2.37-1_all.deb ...
Unpacking fonts-dejavu-core (2.37-1) ...
Selecting previously unselected package fonts-dejavu-extra.
Preparing to unpack .../2-fonts-dejavu-extra_2.37-1_all.deb ...
Unpacking fonts-dejavu-extra (2.37-1) ...
Selecting previously unselected package x11-utils.
Preparing to unpack .../3-x11-utils_7.7+3build1_amd64.deb ...
Unpacking x11-utils (7.7+3build1) ...
Selecting previously unselected package libatk-wrapper-java.
Preparing to unpack .../4-libatk-wrapper-java_0.33.3-20ubuntu0.1_all.deb ...
Unpacking libatk-wrapper-java (0.33.3-20ubuntu0.1) ...
Selecting previously unselected package libatk-wrapper-java-jni:amd64.
Preparing to unpack .../5-libatk-wrapper-java-jni_0.33.3-20ubuntu0.1_amd64.deb ...
Unpacking libatk-wrapper-java-jni:amd64 (0.33.3-20ubuntu0.1) ...
Selecting previously unselected package openjdk-8-jre:amd64.
Preparing to unpack .../6-openjdk-8-jre_8u191-b12-2ubuntu0.18.04.1_amd64.deb ...
Unpacking openjdk-8-jre:amd64 (8u191-b12-2ubuntu0.18.04.1) ...
Selecting previously unselected package openjdk-8-jdk:amd64.
Preparing to unpack .../7-openjdk-8-jdk_8u191-b12-2ubuntu0.18.04.1_amd64.deb ...
Unpacking openjdk-8-jdk:amd64 (8u191-b12-2ubuntu0.18.04.1) ...
Processing triggers for mime-support (3.60ubuntu1) ...
Setting up fonts-dejavu-core (2.37-1) ...
Setting up libxxf86dga1:amd64 (2:1.1.4-1) ...
Processing triggers for libc-bin (2.27-3ubuntu1) ...
Processing triggers for man-db (2.8.3-2ubuntu0.1) ...
Setting up fonts-dejavu-extra (2.37-1) ...
Processing triggers for hicolor-icon-theme (0.17-2) ...
Processing triggers for fontconfig (2.12.6-0ubuntu2) ...
Setting up x11-utils (7.7+3build1) ...
Setting up libatk-wrapper-java (0.33.3-20ubuntu0.1) ...
Setting up libatk-wrapper-java-jni:amd64 (0.33.3-20ubuntu0.1) ...
Setting up openjdk-8-jre:amd64 (8u191-b12-2ubuntu0.18.04.1) ...
update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/policytool to provide /usr/bin/policytool (policytool) in auto mode
Setting up openjdk-8-jdk:amd64 (8u191-b12-2ubuntu0.18.04.1) ...
update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/bin/appletviewer to provide /usr/bin/appletviewer (appletviewer) in auto mode
update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/bin/jconsole to provide /usr/bin/jconsole (jconsole) in auto mode
Processing triggers for libc-bin (2.27-3ubuntu1) ...
Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)
Collecting konlpy
Downloading https://files.pythonhosted.org/packages/e5/3d/4e983cd98d87b50b2ab0387d73fa946f745aa8164e8888a714d5129f9765/konlpy-0.5.1-py2.py3-none-any.whl (19.4MB)
100% |████████████████████████████████| 19.4MB 1.7MB/s
Requirement already satisfied: wordcloud in /usr/local/lib/python3.6/dist-packages (1.5.0)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (3.0.3)
Requirement already satisfied: gensim in /usr/local/lib/python3.6/dist-packages (3.6.0)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.11.0)
Collecting JPype1>=0.5.7 (from konlpy)
Downloading https://files.pythonhosted.org/packages/c4/4b/60a3e63d51714d4d7ef1b1efdf84315d118a0a80a5b085bb52a7e2428cdc/JPype1-0.6.3.tar.gz (168kB)
100% |████████████████████████████████| 174kB 32.1MB/s
Requirement already satisfied: numpy>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from wordcloud) (1.14.6)
Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from wordcloud) (4.1.1)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (0.10.0)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.3.1)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.0.1)
Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.5.3)
Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.1.0)
Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.8.0)
Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow->wordcloud) (0.46)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib) (40.8.0)
Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.49.0)
Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (0.98)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.18.4)
Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (1.9.123)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2019.3.9)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)
Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.6)
Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.22)
Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.2.0)
Requirement already satisfied: botocore<1.13.0,>=1.12.123 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (1.12.123)
Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.9.4)
Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.123->boto3->smart-open>=1.2.1->gensim) (0.14)
Building wheels for collected packages: JPype1
Building wheel for JPype1 (setup.py) ... done
Stored in directory: /root/.cache/pip/wheels/0e/2b/e8/c0b818ac4b3d35104d35e48cdc7afe27fc06ea277feed2831a
Successfully built JPype1
Installing collected packages: JPype1, konlpy
Successfully installed JPype1-0.6.3 konlpy-0.5.1
Reading package lists... Done
Building dependency tree
Reading state information... Done
Note, selecting 'fonts-nanum-eco' for glob 'fonts-nanum*'
Note, selecting 'fonts-nanum' for glob 'fonts-nanum*'
Note, selecting 'fonts-nanum-gothic-light' for glob 'fonts-nanum*'
Note, selecting 'fonts-nanum-coding' for glob 'fonts-nanum*'
Note, selecting 'fonts-nanum-extra' for glob 'fonts-nanum*'
The following package was automatically installed and is no longer required:
libnvidia-common-410
Use 'apt autoremove' to remove it.
The following NEW packages will be installed:
fonts-nanum fonts-nanum-coding fonts-nanum-eco fonts-nanum-extra
0 upgraded, 4 newly installed, 0 to remove and 8 not upgraded.
Need to get 37.0 MB of archives.
After this operation, 145 MB of additional disk space will be used.
Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 fonts-nanum all 20170925-1 [9,604 kB]
Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 fonts-nanum-eco all 1.000-6 [14.0 MB]
Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 fonts-nanum-extra all 20170925-1 [12.2 MB]
Get:4 http://archive.ubuntu.com/ubuntu bionic/universe amd64 fonts-nanum-coding all 2.5-1 [1,083 kB]
Fetched 37.0 MB in 4s (9,238 kB/s)
Selecting previously unselected package fonts-nanum.
(Reading database ... 131460 files and directories currently installed.)
Preparing to unpack .../fonts-nanum_20170925-1_all.deb ...
Unpacking fonts-nanum (20170925-1) ...
Selecting previously unselected package fonts-nanum-eco.
Preparing to unpack .../fonts-nanum-eco_1.000-6_all.deb ...
Unpacking fonts-nanum-eco (1.000-6) ...
Selecting previously unselected package fonts-nanum-extra.
Preparing to unpack .../fonts-nanum-extra_20170925-1_all.deb ...
Unpacking fonts-nanum-extra (20170925-1) ...
Selecting previously unselected package fonts-nanum-coding.
Preparing to unpack .../fonts-nanum-coding_2.5-1_all.deb ...
Unpacking fonts-nanum-coding (2.5-1) ...
Setting up fonts-nanum-extra (20170925-1) ...
Setting up fonts-nanum (20170925-1) ...
Setting up fonts-nanum-coding (2.5-1) ...
Setting up fonts-nanum-eco (1.000-6) ...
Processing triggers for fontconfig (2.12.6-0ubuntu2) ...
Reading package lists... Done
Building dependency tree
Reading state information... Done
fontconfig is already the newest version (2.12.6-0ubuntu2).
fontconfig set to manually installed.
The following package was automatically installed and is no longer required:
libnvidia-common-410
Use 'apt autoremove' to remove it.
0 upgraded, 0 newly installed, 0 to remove and 8 not upgraded.
/usr/share/fonts: caching, new cache contents: 0 fonts, 1 dirs
/usr/share/fonts/truetype: caching, new cache contents: 0 fonts, 3 dirs
/usr/share/fonts/truetype/dejavu: caching, new cache contents: 22 fonts, 0 dirs
/usr/share/fonts/truetype/liberation: caching, new cache contents: 16 fonts, 0 dirs
/usr/share/fonts/truetype/nanum: caching, new cache contents: 31 fonts, 0 dirs
/usr/local/share/fonts: caching, new cache contents: 0 fonts, 0 dirs
/root/.local/share/fonts: skipping, no such directory
/root/.fonts: skipping, no such directory
/var/cache/fontconfig: cleaning cache directory
/root/.cache/fontconfig: not cleaning non-existent cache directory
/root/.fontconfig: not cleaning non-existent cache directory
fc-cache: succeeded
In [0]:
import nltk
nltk.download('punkt')
import pandas as pd
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from nltk import FreqDist
from nltk.tokenize import word_tokenize
# for Colab
import requests
texts = requests.get(speech_text).text
texts[:100]
# for LocalHost
# with open(speech_text, 'r') as f:
# texts = f.read()
# texts[:100]
[nltk_data] Downloading package punkt to /root/nltk_data... [nltk_data] Unzipping tokenizers/punkt.zip.
Out[0]:
'존경하는 국민여러분. 성원해 주신 덕분에 평양에 잘 다녀왔습니다. \n\n국민 여러분께서 보셨듯이 정상회담에서 좋은 합의를 이뤘고, 최상의 환대를 받았습니다. 무엇보다 3일동안 김정은'
In [0]:
%matplotlib inline
wcloud = WordCloud(font_file).generate(texts)
plt.figure(figsize=(12,12))
plt.imshow(wcloud)
plt.axis("off")
Out[0]:
(-0.5, 399.5, 199.5, -0.5)
In [0]:
from collections import Counter
dictionary = Counter(texts.split())
dictionary.most_common()[:10]
Out[0]:
[('김정은', 7),
('저는', 6),
('것을', 6),
('수', 5),
('위해', 5),
('우리', 4),
('것입니다.', 4),
('존경하는', 3),
('국민', 3),
('대화를', 3)]
## **명사만 추출하여 Wordcloud 만들기**
In [0]:
texts[:200]
Out[0]:
'존경하는 국민여러분. 성원해 주신 덕분에 평양에 잘 다녀왔습니다. \n\n국민 여러분께서 보셨듯이 정상회담에서 좋은 합의를 이뤘고, 최상의 환대를 받았습니다. 무엇보다 3일동안 김정은 위원장과 여러차례 만나 긴 시간 많은 대화를 허심탄회하게 나눌 수 있었던 것에 큰 의미를 두고 싶습니다.\n\n남북관계를 크게 진전시키고 두 정상 간의 신뢰 구축에도 큰 도움이 된 '
In [0]:
# Twitter 모듈을 활용하여 명사만 추출
from konlpy.tag import Okt
twitter = Okt()
tokens = twitter.pos(texts, stem=True)
tokens_noun = [token[0] for token in tokens
if token[1] == "Noun"]
texts_noun = " ".join(tokens_noun)
texts_noun[:300]
Out[0]:
'존경 국민 여러분 성원 주신 덕분 평양 국민 여러분 정상회담 합의 최상 환대 무엇 김정은 위원장 차례 시간 대화 허심 탄회 수 것 의미 남북관계 크게 진전 두 정상 간의 신뢰 구축 도움 방문 평가 북 준비 기간 불구 우리 대표 단 정성 다해 동안 공항 길가 환영 환송 평양 시민 각별 인사 수 백두산 동안 삼지연공항 배웅 준 지역 주민 감사 저 경기장 대규모 집단체조 공연 평양 시민 대한민국 대통령 사상 최초 연설 기회 그 한반도 영구 핵무기 핵 위협 평화 터전 저 연설 대해 박수 보 존경 국민 여러분 지난 저 김정은 위원장 비핵화 '
In [0]:
%matplotlib inline
plt.figure(figsize=(12,12))
wcloud = WordCloud(font_file).generate(texts_noun)
plt.imshow(wcloud)
plt.axis("off")
Out[0]:
(-0.5, 399.5, 199.5, -0.5)
In [0]:
# Token 빈도결과값
from collections import Counter
dictionary = Counter(texts_noun.split())
dictionary.most_common()[:10]
Out[0]:
[('것', 14),
('국민', 10),
('비핵화', 10),
('위원장', 9),
('합의', 8),
('대화', 8),
('우리', 8),
('저', 8),
('평양', 7),
('김정은', 7)]