In [3]:
import tweepy
import pandas as pd
import json
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
Gute Referenz, der ich den meisten Code für den Datendownload verdanke: https://www.packtpub.com/big-data-and-business-intelligence/mastering-social-media-mining-python.
Falls der Code selbst ausgeführt werden soll, müssen hier die eigenen Twitter-Keys verwendet werden. Details siehe auch in der Referenz oben.
In [5]:
consumer_key = 'XXX'
consumer_secret = 'XXX'
access_token = 'XXX'
access_secret = 'XXX'
In [6]:
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
In [7]:
client = tweepy.API(auth, wait_on_rate_limit=True)
In [8]:
users = ('derStandardat', 'KURIERat', 'ArminWolf', 'SPIEGEL_Top')
# last tweet from @diepresse_com in 2012
# last tweet from @oe24News in April 2016
Herunterladen der Tweets¶
Da derStandard.at mit dem 2016-10-26 das 3.200 Tweet Limit der Twitter API erreicht, lade ich nur Nov und Dez herunter
In [9]:
for user in users:
print('getting data for {}'.format(user))
fname = "user_timeline_{}.jsonl".format(user)
with open(fname, 'w') as f:
for page in tweepy.Cursor(client.user_timeline, screen_name=user, count=200).pages(16):
for status in page:
if (status.created_at.year == 2016) & (status.created_at.month > 10):
f.write(json.dumps(status._json)+"\n")
else:
break
Tweets in Datenframe zusammenfassen¶
In [10]:
tweets = [pd.read_json(fname, lines=True) for fname in ["user_timeline_{}.jsonl".format(user) for user in users]]
df = pd.concat(tweets).reset_index(drop=True)
In [11]:
df[:3]
Out[11]:
In [12]:
df.info()
In [13]:
df.columns
Out[13]:
Username aus Spalte 'user' extrahieren und ergänzen
In [14]:
df_user = pd.DataFrame(df.user.tolist())
df_user.columns
Out[14]:
In [15]:
df_user.name.value_counts()
Out[15]:
In [17]:
df_user[:2]
Out[17]:
In [18]:
df = df.join([df_user.name])
Wer hat wieviele Followers?
In [32]:
df_user.drop_duplicates(subset=('name'))[['name', 'followers_count']].sort_values('followers_count', ascending=False)
Out[32]:
Tweets analysieren¶
In [19]:
df = df.sort_values('created_at', ascending=True)
In [20]:
df[:3]
Out[20]:
In [21]:
for name in df.name.unique():
print(name, df.loc[df.name == name, 'created_at'].min())
In [ ]:
In [ ]:
In [33]:
df_temp = df.groupby([df.name])['id'].count()
df_temp
Out[33]:
In [34]:
df_temp = df.groupby([df.retweeted_status.isnull() == False, df.name])['id'].count().unstack()
df_temp
Out[34]:
In [36]:
df = df[df.retweeted_status.isnull()]
In [91]:
df_temp = df.groupby([df.retweeted_status.isnull() == False, df.name])['id'].count().unstack()
df_temp
Out[91]:
In [92]:
df_temp = df.groupby([df.name])['retweet_count'].sum()
df_temp
Out[92]:
In [ ]:
In [112]:
df_temp = df.groupby([df.name, df.created_at.dt.dayofweek])['id'].count().unstack()
df_temp.columns = ['Mo', 'Di', 'Mi', 'Do', 'Fr', 'Sa', 'So']
df_temp
Out[112]:
In [113]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp, annot=True, linewidths=.5, fmt='.0f')
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets')
sns.set(font_scale=1)
ax.set(xlabel='Wochentag', ylabel='Twitter Account')
Out[113]:
In [115]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp.div(df_temp.sum(axis=1), axis=0) * 100, annot=True, linewidths=.5, fmt='.0f')
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets in Prozent je Account')
sns.set(font_scale=1)
ax.set(xlabel='Wochentag', ylabel='Twitter Account')
sns.plt.savefig('tweets_pro_wochentag.png')
In [96]:
df_temp = df.groupby([ df.name, df.created_at.dt.hour])['id'].count().unstack()
df_temp[:3]
Out[96]:
In [97]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Twitter Account')
Out[97]:
In [98]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp.div(df_temp.sum(axis=1), axis=0) * 100)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets in Prozent je Account')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Twitter Account')
sns.plt.savefig('tweets_pro_stunde.png')
In [99]:
df_temp = df[df.name == 'Armin Wolf'].groupby([ df.created_at.dt.weekday,
df.created_at.dt.hour])['id'].count().unstack()
df_temp.index = ['Mo', 'Di', 'Mi', 'Do', 'Fr', 'Sa', 'So']
df_temp
Out[99]:
In [100]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets von Armin Wolf')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Wochentag')
Out[100]:
In [101]:
df_temp = df.groupby([df.created_at.dt.month, df.created_at.dt.day,
df.created_at.dt.hour])['id'].count().unstack()
df_temp[:3]
Out[101]:
In [102]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets pro Tag')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Tag')
sns.plt.savefig('tweets_pro_tag.png')
In [103]:
df_temp = df[df.name == 'Armin Wolf'].groupby([df.created_at.dt.month, df.created_at.dt.day,
df.created_at.dt.hour])['id'].count().unstack()
df_temp[:3]
Out[103]:
In [104]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Tweets von Armin Wolf')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Tag')
#sns.plt.savefig('tweets_armin_wolf.svg')
Out[104]:
In [105]:
df.loc[(df.name == 'Armin Wolf') & (df.created_at.dt.month == 11) &
(df.created_at.dt.day == 9) & (df.created_at.dt.hour == 3),
['created_at', 'text']]
Out[105]:
In [106]:
df_temp = df.groupby([df.created_at.dt.month, df.created_at.dt.day,
df.created_at.dt.hour])['retweet_count'].sum().unstack()
df_temp[:3]
Out[106]:
In [107]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Retweets pro Tag')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Tag')
Out[107]:
In [108]:
sns.set(style='whitegrid')
ax = sns.distplot(df.loc[df.retweet_count <= 50,'retweet_count'], kde=False)
sns.set(font_scale=1.5)
sns.plt.title('Verteilung Tweets je Retweets <= 50')
sns.set(font_scale=1)
ax.set(xlabel='Retweets', ylabel='Anzahl Tweets')
sns.plt.savefig('retweets_hist_bis_50.png')
In [109]:
sns.set(style='whitegrid')
ax = sns.distplot(df.loc[df.retweet_count > 50,'retweet_count'], kde=False)
sns.set(font_scale=1.5)
sns.plt.title('Verteilung Tweets je Retweets > 50')
sns.set(font_scale=1)
ax.set(xlabel='Retweets', ylabel='Anzahl Tweets')
sns.plt.savefig('retweets_hist_ab_50.png')
In [110]:
df.sort_values('retweet_count', ascending=False).iloc[0,:]
Out[110]:
In [111]:
for name in df.name.unique():
print('*' * 80)
print(name)
print(df[df.name == name].sort_values('retweet_count', ascending=False).iloc[0,:])
print()
In [ ]:
In [59]:
df_temp = df[df.name == 'Armin Wolf'].groupby([df.created_at.dt.month, df.created_at.dt.day,
df.created_at.dt.hour])['retweet_count'].sum().unstack()
df_temp[:3]
Out[59]:
In [60]:
sns.set(style='whitegrid')
ax = sns.heatmap(df_temp)
sns.set(font_scale=1.5)
sns.plt.title('Anzahl Retweets von Tweets von Armin Wolf')
sns.set(font_scale=1)
ax.set(xlabel='Uhrzeit - Stunde', ylabel='Tag')
#sns.plt.savefig('reteets_armin_wolf.svg')
Analyse Hashtags¶
In [157]:
def get_hashtags(entities):
hashtags = entities.get('hashtags', [])
return [tag['text'].lower() for tag in hashtags]
In [170]:
df_temp_hashtags = df.entities.apply(get_hashtags).apply(pd.Series)
In [171]:
df_temp_hashtags[7:10]
Out[171]:
In [173]:
df_temp_hashtags2 = pd.concat([df, df_temp_hashtags], axis=1)
In [174]:
df_temp_hashtags2[7:10]
Out[174]:
In [175]:
df_hashtags = pd.melt(df_extended, id_vars=['id_str', 'name'] ,value_vars=[0,1,2,3],
value_name='hashtag', var_name='hashtag_nr').dropna()
df_hashtags[:3]
Out[175]:
In [178]:
df_hashtags.hashtag.value_counts()[:5]
Out[178]:
In [179]:
from wordcloud import WordCloud
# https://github.com/amueller/word_cloud
# Achtung, ich habe an der Erweiterung wordcloud - die für englische Sprache entwickelt wurde - etwas
# herumgebastelt, um zu verhindern, dass ein 's' am Ende eines Worts als Pluar interpretiert wird.
# Ansonst würden zum Beispiel die Wörter hau und Haus zusammen erfasst.
In [180]:
#stopword Liste 1 von http://members.unine.ch/jacques.savoy/clef/index.html
STOPWORDS_GENERAL = set([x.strip() for x in open('germanST.txt').read().split('\n')])
STOPWORDS_GENERAL = STOPWORDS_GENERAL.union(set(('nämlich',)))
In [186]:
text_hashtags = ' '.join(df_hashtags.hashtag.tolist())
In [203]:
wordcloud = WordCloud(max_font_size=40, relative_scaling=.5, background_color='white',stopwords=STOPWORDS_GENERAL,
scale=3, random_state=42).generate(text_hashtags)
plt.imshow(wordcloud)
plt.axis("off")
#plt.title('Hashtags');
plt.savefig('hashtags.png', transparent=True, bbox_inches='tight', pad_inches=0)
plt.show()
In [204]:
df_hashtags.name.value_counts()
Out[204]:
In [209]:
for name in df_hashtags.name.unique():
text_hashtags_name = ' '.join(df_hashtags[df_hashtags.name == name].hashtag.tolist())
wordcloud = WordCloud(max_font_size=40, relative_scaling=.5, background_color='white',stopwords=STOPWORDS_GENERAL,
scale=3, random_state=42).generate(text_hashtags_name)
plt.imshow(wordcloud)
plt.axis("off");
plt.title('Hashtags: ' + name);
plt.savefig('hashtags_' + name.replace(' ','_') + '.png',
transparent=True, bbox_inches='tight', pad_inches=0)
plt.show()