# 한글 인코딩/ 디코딩 문법
from urllib import parse
temp = "%B8%D3%BD%C5%B7%AF%B4%D7"
parse.unquote_plus(temp, encoding='euc-kr')

parse.quote_plus("머신러닝", encoding='euc-kr')

# Base url 추출 및 추가경로 붙이기
from urllib import parse
url_root = "https://www.kamis.or.kr/test/join"
url_base = parse.urljoin(url_root, '/customer/price/retail/item.do?action=priceinfo')
url_base

query = {
    'examParam1' : 'value1',
    'examParam2' : ['aaa', 'bbb'],
    'Korquery' : "파이"
} 
parse.urlencode(query, encoding='UTF-8', doseq=True)

# Param 를 사용하여 Query Url 만들기
query = {
    "regday":"2019-10-18",
    "itemcode":"111",
    "convert_kg_yn":"N",
    "itemcategorycode":"100",
}
url_query = parse.urlencode(query, encoding='UTF-8', doseq=True)
url_query

# Base url 과 Params 연결하기
url = url_base + "&" + url_query
url

# 긴 Url 경로를 Parsing 하기
url_parse = parse.urlparse(url)
url_parse

# Root Url 추출하기 (url Net Location)
url_parse.netloc

# url Add Paths
url_parse.path

# url Query
url_parse.query

# Python 기본 모듈인 urllib 을 사용하여 Robots.txt 내용 확인하기
from urllib import request
url  = "https://news.naver.com/robots.txt"
resp = request.urlopen(url)
resp.read() # 통으로 불러오기

# 상세 페이지 크롤링
url    = "https://www.kamis.or.kr/customer/price/retail/item.do?action=priceinfo&regday=2019-10-18&itemcode=111&convert_kg_yn=N&itemcategorycode=100"
resp   = request.urlopen(url)
r_test = resp.readlines()[103:107]
r_test

from urllib import parse
txt_byte = parse.quote_plus(r_test[3])
type(txt_byte), txt_byte

# 변환된 str 중 인코딩 문제를 해결
parse.unquote_plus(txt_byte)

def str_korchar(str_text):
    r_lst = []
    for _ in list(str_text.strip()):
        if '가' <= _ <= '힣':                        # 영어는 구분작성 
            ch1 = (ord(_)-ord('가'))//588            # 588개 마다 초성변경            
            ch2 = ((ord(_)-ord('가'))-(588*ch1))//28 # 중성은 총 28개
            ch3 = (ord(_) - ord('가'))-(588*ch1)-28*ch2
            r_lst.append([str_start[ch1], str_mid[ch2], str_last[ch3]])
        else: r_lst.append([_])
    return r_lst

str_korchar("된장찌개 볶음abc")

list("깍두기")

str_korchar("깍두기")

(ord("깍")-ord("가"))//588

# Base url 경로
url = "https://www.kamis.or.kr/customer/price/retail/item.do?action=priceinfo"

from urllib import parse
parse.urlsplit(url)

params = {
    "regday" : "2019-10-18",
    "itemcategorycode" : "100",
    "itemcode" : "111",
    "convert_kg_yn" : "N",
}
query = '&' + parse.urlencode(params, encoding='UTF-8')
query

url + query

import pandas as pd
pd.read_html(url + query)[3].head(3)

url = "https://www.kamis.or.kr/customer/price/retail/item.do?action=priceinfo"
params = {
    "regday" : "2019-10-18",
    "itemcategorycode" : "100",
    "itemcode" : "111",
    "convert_kg_yn" : "N",
}

import requests
web = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2'
headers = {'User-Agent' : web}
resp = requests.get(url, params=params, headers=headers)
resp.text[2800:3000]

# OTP key 값 추출하기
# KRX 주가 데이터 수집예제
gen_otp_url = 'http://marketdata.krx.co.kr/contents/COM/GenerateOTP.jspx'
gen_otp_data = {
    'name':'fileDown', 
    'filetype':'xls',
    'market_gubun':'ALL', #시장구분: ALL=전체
    'url':'MKD/04/0404/04040200/mkd04040200_01',
    'schdate':'20191018',
    'pagePath':'/contents/MKD/04/0404/04040200/MKD04040200.jsp'
}
r = requests.post(gen_otp_url, gen_otp_data)
OTP_code = r.content
OTP_code

data = {
    "Accept" :"text/html,application/xhtml+xm…plication/xml;q=0.9,*/*;q=0.8",
    "Accept-Encoding" : "gzip, deflate, br",
    "Accept-Language" : "ko-KR,ko;q=0.8,en-US;q=0.5,en;q=0.3",
    "Connection" : "keep-alive",
    "Content-Length" : "76940",
    "Content-Type" : "application/x-www-form-urlencoded",
    "Host" : "www.kamis.or.kr",
    "Referer" : url + query,
    "Upgrade-Insecure-Requests" : "1",
    "User-Agent" : "Mozilla/5.0 (X11; Ubuntu; Linu…) Gecko/20100101 Firefox/69.0"
}
resp = requests.post("https://www.kamis.or.kr/jsp/common/excel_util.jsp", data=data)
resp.content

url = "https://www.kamis.or.kr/customer/price/retail/item.do?action=priceinfo"
params = {
    "regday" : "2015-10-14",
    "itemcategorycode" : "100",
    "itemcode" : "111",
    "convert_kg_yn" : "N",
}

import requests
web = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2'
headers = {'User-Agent' : web}
resp = requests.get(url, params=params, headers=headers)
resp

from lxml.html import fromstring, tostring
resp_lxml  = fromstring(resp.text)
resp_table = resp_lxml.xpath('//table[@id="itemTable_1"]')[0]

import pandas as pd
table = pd.read_html(tostring(resp_table))[0]
table_idx = [_ for _, txt in enumerate(list(table.columns))   if txt.find('당일') != -1]

if len(table_idx) == 1:
    table_idx = table_idx[0]
    table = table.iloc[:,[0, table_idx]].set_index(
        table.columns[0]).loc[["평균","최고값","최저값"]]
    table = list(table.to_dict(orient='list').values())[0]
else:
    table = None
table

# 조리 레시피 API 안내 페이지 https://www.foodsafetykorea.go.kr/api/openApiInfo.do
query = {
    "menu_no":"849",
    "menu_grp":"MENU_GRP31",
    "start_idx":"120",
    "svc_no":"COOKRCP01",
    "svc_type_cd":"API_TYPE06",
    "show_cnt":"10",
    "Referer":"https://www.foodsafetykorea.go.kr/api/sheetInfo.do",
}
import requests
resp = requests.post("https://www.foodsafetykorea.go.kr/api/openApiInfo.do", data=query)
resp
# with open("food.html", "w") as f:
#     f.write(resp.text)

%%time
from urllib import request, parse
resp = request.urlopen(url)
resp = parse.quote_plus(resp.read()) # Byte to String
resp = parse.unquote_plus(resp)      # Encoding Text

import json
resp = json.loads(resp) # Json 데이터를 Dict 으로 변환
resp
# len(resp['I0750']['row'])

resp['I0750']['RESULT']['CODE'].lower()#.find("error")

resp['I0750']['RESULT']['CODE'].lower().find("error")

import pandas as pd
df = pd.DataFrame(resp['I0750']['row'])
df = df.loc[:,list(col_to_kor.keys())]  # header 정렬
df.columns = [col_to_kor[_] for _ in list(df.columns)]    
# df.to_excel('data/food_nutrient.xls', index=False)
df