import matplotlib
# cloab 字體設定

!wget -O taipei_sans_tc_beta.ttf https://drive.google.com/uc?id=1eGAsTN1HBpJAkeVM57_C7ccp7hbgSz3_&export=download

# 新增字體
matplotlib.font_manager.fontManager.addfont('taipei_sans_tc_beta.ttf')

# 將 font-family 設為 Taipei Sans TC Beta
# 設定完後，之後的圖表都可以顯示中文了
matplotlib.rc('font', family='Taipei Sans TC Beta')

# 下載 GitHub 中的 dataset
!git clone https://github.com/leo85741/dataset.git
%cd dataset
!unzip softjob_23_25.csv.zip

#匯入資料
df = pd.read_csv('softjob_23_25.csv', encoding = 'UTF-8')
df.head(3)

MetaData = df.copy()

# 去除一些不需要的欄位
MetaData = MetaData.drop(['artPoster', 'artCatagory', 'artComment', 'e_ip', 'insertedDate', 'dataSource'], axis=1)
# 只留下中文字 (artContent -> sentence)
MetaData['sentence'] = MetaData['artContent'].apply(lambda x: re.sub('[^\u4e00-\u9fff]+', '', str(x)) if isinstance(x, str) else '')

MetaData.head(3)

data = MetaData.copy()

# 斷詞、去除停用字並將word欄位展開
# data['word'] = data.sentence.apply(getToken).explode('word')

data['word'] = data['sentence'].apply(getToken)
data = data.explode('word')
data.head(3)

data

softjob_df = MetaData.copy()
softjob_df.head(3)

# 留下要用到的欄位
softjob_df = softjob_df.loc[:,["system_id", "sentence"]]

# 使用空格連接斷完的詞
softjob_df['word'] = softjob_df.sentence.apply(getToken).map(' '.join)

softjob_df.head()

# Bag of Word
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(softjob_df["word"])
vocabulary = vectorizer.get_feature_names_out()

# 轉成 dataframe
DTM_df = pd.DataFrame(columns = vocabulary, data = X.toarray())
DTM_df

transformer = TfidfTransformer()
# 將詞頻矩陣X統計成TF-IDF值
tfidf = transformer.fit_transform(X)

# 轉成dataframe
TFIDF_df = pd.DataFrame(columns = vocabulary, data = tfidf.toarray())

TFIDF_df

softjob_tfidf = TFIDF_df.mean().to_frame().reset_index() ## !!
softjob_tfidf.columns = ["word", "avg"]

softjob_tfidf.sort_values('avg', ascending = False).head(10)

toptens = TFIDF_df.copy()
toptens.insert(0, 'doc_id', toptens.index+1)

toptens

toptens = toptens.melt(id_vars = "doc_id", var_name = "word", value_name = 'tfidf')
toptens

(
    # 從每篇文章挑選出tf-idf最大的前十個詞
    toptens.groupby("doc_id").apply(lambda x : x.nlargest(10, "tfidf")).reset_index(drop=True)

    # 計算每個詞被選中的次數
    .groupby(['word'],as_index=False).size()

).sort_values('size', ascending=False).head(10) # 排序看前十名

softjob_bigram = MetaData.copy()

softjob_bigram["word"] = softjob_bigram['sentence'].apply(lambda row: ngram_getToken(row, 2))
softjob_bigram = softjob_bigram.explode('word')

softjob_bigram.head(3)

# 計算每個組合出現的次數
softjob_bigram_count = softjob_bigram['word'].value_counts().reset_index()

softjob_bigram_count.sort_values('count', ascending=False).head(10)

softjob_bigram_count.sort_values('count', ascending=False).head(60) # 方便後續做lexicon

softjob_trigram = MetaData.copy()

softjob_trigram["word"] = softjob_trigram.sentence.apply(lambda row: ngram_getToken(row, 3))
softjob_trigram = softjob_trigram.explode('word')

softjob_trigram.head(3)

# 計算每個組合出現的次數
softjob_trigram_count = softjob_trigram['word'].value_counts().reset_index()

softjob_trigram_count.sort_values('count', ascending=False).head(10)

softjob_trigram_count.sort_values('count', ascending=False).head(40) # 方便後續做lexicon

# 剛才的斷詞結果沒有使用新增的辭典，因此我們重新進行斷詞
data2 = MetaData.copy()
data2['word'] = data2.sentence.apply(getToken)
data2 = data2.explode('word')

data2.head(3)

bigramfdist = MetaData.copy()

bigramfdist["word"] = bigramfdist['sentence'].apply(lambda row: ngram_getToken(row, 2))
bigramfdist = bigramfdist.explode('word')

bigramfdist.head(3)

bigramfdist['word'] = bigramfdist['word'].astype(str)

# 使用FreqDist 取得 bigram 斷詞 與 bigram 出現頻率
bigramfdist = FreqDist(bigramfdist['word'].apply(lambda x: tuple(x.split(' '))))
bigramfdist.most_common(5)

fig, ax = plt.subplots(figsize=(12, 10))

pos = nx.spring_layout(G, k=1.5)

# networks
nx.draw_networkx(G, pos,
                 font_size=16,
                 width=weights,
                 edge_color='grey',
                 node_color='purple',
                 with_labels = False,
                 ax=ax)

# 增加 labels
for key, value in pos.items():
    x, y = value[0]+.07, value[1]+.045
    ax.text(x, y,
            s=key,
            bbox=dict(facecolor='red', alpha=0.25),
            horizontalalignment='center', fontsize=13)

plt.show()

data_cor = MetaData.copy()

# 需要改成使用空格連接斷好的詞
data_cor['word'] = data_cor.sentence.apply(getToken).map(' '.join)
data_cor.head(3)

# Bag of Word
# 篩選至少出現在5篇文章以上且詞頻前300的詞彙
vectorizer = CountVectorizer(min_df = 5, max_features = 300)
X = vectorizer.fit_transform(data_cor["word"])
vocabulary = vectorizer.get_feature_names_out()

# 轉成dataframe
DTM_df = pd.DataFrame(columns = vocabulary, data = X.toarray())
DTM_df

# 計算詞之間的相關係數
corr_matrix = np.corrcoef(DTM_df.T)

# 轉成dataframe
Cor_df = pd.DataFrame(corr_matrix, index = DTM_df.columns, columns = DTM_df.columns)

Cor_df.insert(0, 'word1', Cor_df.columns)
Cor_df.reset_index(inplace = True, drop = True)

Cor_df

word_cor_df = Cor_df.melt(id_vars = 'word1', var_name = 'word2', value_name = 'cor')

# 去除兩個詞相同的情況
word_cor_df = word_cor_df[word_cor_df["word1"] != word_cor_df["word2"]]

word_cor_df.sort_values('cor', ascending=False).head(10)

plt.figure(figsize=(12,8))   # 顯示圖框架大小 (寬,高)
plt.style.use("ggplot")     # 使用ggplot主題樣式

plt.subplot(121)
plt.title('語言')
plt.xlabel('cor')
plt.barh(language_sum['word2'],language_sum['cor'])
plt.gca().invert_yaxis()

plt.subplot(122)
plt.title('工時')
plt.xlabel('cor')
plt.barh(workinghours_sum['word2'],workinghours_sum['cor'],color="darkblue")
plt.gca().invert_yaxis()

plt.show()

# 保留存在詞頻前60高之詞彙的組合
filtered_df = word_cor_df[(word_cor_df['word1'].isin(most_freq_word)) & (word_cor_df['word2'].isin(most_freq_word))]

# 篩選出相關係數大於0.3的組合
filtered_df = filtered_df[filtered_df['cor'] > 0.3]
filtered_df.reset_index(inplace=True, drop=True)
filtered_df

fig, ax = plt.subplots(figsize=(12, 8))

pos = nx.spring_layout(g, k=0.3)

# networks
nx.draw_networkx(g, pos,
                 font_size=16,
                 width=weights,
                 edge_color='grey',
                 node_color='lightblue',
                 with_labels = False,
                 ax=ax)

# 增加 labels
for key, value in pos.items():
    x, y = value[0]+.07, value[1]+.045
    ax.text(x, y,
            s=key,
            bbox=dict(facecolor='red', alpha=0.25),
            horizontalalignment='center', fontsize=12)

plt.show()

data_cos = data_cor.copy()
data_cos.head(3)

transformer = TfidfTransformer()
print(transformer)
# 將詞頻矩陣X統計成TF-IDF值
tfidf = transformer.fit_transform(X)

# 轉成dataframe
TFIDF_df = pd.DataFrame(columns = vocabulary, data = tfidf.toarray())
TFIDF_df

cos_df = pd.DataFrame(cosine_matrix[0], columns = ['cos_similarity'])
cos_df

cos_df = cos_df.merge(data_cos, how = 'left', left_index=True, right_index=True)
cos_df.loc[:,["cos_similarity", "artTitle", "artUrl"]].sort_values(by=['cos_similarity'], ascending=False).head(10)

cos_df_14 = pd.DataFrame(cosine_matrix[14], columns=['cos_similarity'])

cos_df_14 = cos_df_14.merge(data_cos, how = 'left', left_index=True, right_index=True)
cos_df_14.loc[:,["cos_similarity", "artTitle", "artUrl"]].sort_values(by=['cos_similarity'], ascending=False).head(10)

text = '喜歡'
next_words = list(bi_prediction[text])
next_words.sort(key = lambda s: s[1], reverse = True)
for next_word in next_words[:5]:
    print('next word: {}, probability: {}'.format(next_word.word, next_word.prob))

text = '提供'
next_words = list(bi_prediction[text])
next_words.sort(key = lambda s: s[1], reverse = True)
for next_word in next_words[:5]:
    print('next word: {}, probability: {}'.format(next_word.word, next_word.prob))

from google.colab import drive
drive.mount('/content/drive')

# 設定圖的中文字體
!wget -O TaipeiSansTCBeta-Regular.ttf https://drive.google.com/uc?id=1eGAsTN1HBpJAkeVM57_C7ccp7hbgSz3_&export=download

import matplotlib

matplotlib.font_manager.fontManager.addfont('TaipeiSansTCBeta-Regular.ttf')
matplotlib.rc('font', family='Taipei Sans TC Beta')


# 設定圖的中文字體 (無法顯示的話可以試試‘Microsoft JhengHei’字體)
# 也可參考：https://pyecontech.com/2020/03/27/python_matplotlib_chinese/
#plt.rcParams['font.sans-serif'] = ['Noto Sans CJK TC'] #使圖中中文能正常顯示
plt.rcParams['axes.unicode_minus'] = False #使負號能夠顯示

folder_path = "/content/drive/MyDrive/SMA_2025S-main/"
csv_files = glob.glob(os.path.join(folder_path, '*.csv'))

# 用來存放每一個 DataFrame
df_list = []

# 讀取每個 CSV 並加到 list 裡
for file in csv_files:
    df = pd.read_csv(file)
    df_list.append(df)

# 合併所有 DataFrame
df = pd.concat(df_list, ignore_index=True)

df.head()

# 幾篇文章
print(f"number of posts: {df.shape[0]}")
print(f"date range: {(df['artDate'].min(), df['artDate'].max())}")
print("="*60)

for file in csv_files:
    csv = pd.read_csv(file)
    post_count = csv.shape[0]
    date_range = (csv['artDate'].min(), csv['artDate'].max())

    filename = os.path.basename(file)

    print(f"({filename})")
    print(f"number of posts: {post_count}")
    print(f"date range: {date_range[0]} ~ {date_range[1]}\n")

# 過濾 nan 的資料
df = df.dropna(subset=['artTitle'])
df = df.dropna(subset=['artContent'])
# 移除網址格式
df["artContent"] = df.artContent.apply(
    lambda x: re.sub("(http|https)://.*", "", x)
)
df["artTitle"] = df["artTitle"].apply(
    lambda x: re.sub("(http|https)://.*", "", x)
)
# 只留下中文字
df["artContent"] = df.artContent.apply(
    lambda x: re.sub("[^\u4e00-\u9fa5]+", "", x)
)
df["artTitle"] = df["artTitle"].apply(
    lambda x: re.sub("[^\u4e00-\u9fa5]+", "", x)
)
df.head(3)

# 留下 content
df["content"] = df["artTitle"] + df["artContent"]
df = df[["content", "artUrl", "artCatagory"]]  # 文章內容 文章連結
df.head()

print(f"total docs: {df.shape[0]}")

# 設定斷詞 function
def getToken(row):
    seg_list = jieba.cut(row, cut_all=False)
    seg_list = [
        w for w in seg_list if w not in stopWords and len(w) > 1
    ]  # 篩選掉停用字與字元數大於1的詞彙
    return seg_list

df["words"] = df["content"].apply(getToken).map(" ".join)
df.head()