import pandas as pd, numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import lzma,json,gzip
import urllib.request
from utils import load_file, party_normalizer, save_local, load_local
members=load_file('ep_meps_current.json.xz')
loading ep_meps_current.json.xz...
countries=['Hungary','Romania']
eu={}
parties={}
groups={}
names={}
for j in members:
z='Constituencies'
w='Groups'
if z in j:
if j[z][0]['country'] in countries:
if j[z][0]['country'] not in eu:eu[j[z][0]['country']]={}
eu[j[z][0]['country']][j['UserID']]=j
names[j['Name']['full']]=j
for i in j[z]:
if type(i['party'])==str:
party=i['party']
else:
party=i['party'][0]
party=str(party)
start=int(str(i['start'])[:4])
end=int(str(i['end'])[:4])
if end==9999:
end=2019
if party not in parties:
parties[party]={'min':9999,'max':0}
parties[party]['min']=min(start,parties[party]['min'])
parties[party]['max']=max(end,parties[party]['max'])
if w in j:
for i in j[w]:
party=i['Organization']
party=str(party)
if type(i['groupid'])==str:
code=i['groupid']
else:
code=i['groupid'][0]
start=int(str(i['start'])[:4])
end=int(str(i['end'])[:4])
if end==9999:
end=2019
if party not in groups:
groups[party]={'min':9999,'max':0}
groups[party]['min']=min(start,groups[party]['min'])
groups[party]['max']=max(end,groups[party]['max'])
groups[party]['code']=code
save_local(names,'names')
save_local(groups,'groups')
save_local(parties,'parties')
del members
votes=load_file('ep_votes.json.xz')
loading ep_votes.json.xz...
def get_allegiance(allegiance,voteid,outcome,name):
if voteid not in allegiance:
allegiance[voteid]={'title':j['title'],'url':j['url'],'ts':j['ts']}
if outcome not in allegiance[voteid]:
allegiance[voteid][outcome]=[]
allegiance[voteid][outcome].append(name)
return allegiance
eu_allegiance={}
eu_vt={}
eu_joint_allegiance={}
eu_joint_vt={}
for country in countries:
hu=eu[country]
hu_allegiance={}
hu_vt={}
for j in votes:
ts=j['ts']
year=str(ts)[:4]
if year not in hu_vt:hu_vt[year]=[]
if year not in hu_allegiance:hu_allegiance[year]={'name':{},'group':{},'party':{}}
if year not in eu_joint_vt:eu_joint_vt[year]=[]
if year not in eu_joint_allegiance:eu_joint_allegiance[year]={'name':{},'group':{},'party':{}}
if j['title'] not in ["Modification de l'ordre du jour"]:
for outcome in ['For','Against']:
if outcome in j:
for group in j[outcome]['groups']:
for i in group['votes']:
if i['ep_id'] in hu:
dummy={}
dummy['vote']=j['voteid']
dummy['party']='-'
for k in hu[i['ep_id']]['Constituencies']:
if k['start']<ts<k['end']:
dummy['party']=k['party']
dummy['name']=hu[i['ep_id']]['Name']['full']
dummy['outcome']=outcome
dummy['group']=group['group']
dummy['party']=party_normalizer(dummy['party'])
dummy['group']=party_normalizer(dummy['group'])
dummy['title']=j['title']
dummy['url']=j['url']
dummy['ts']=ts
dummy['year']=year
hu_vt[year].append(dummy)
eu_joint_vt[year].append(dummy)
for allegiance_type in ['name','group','party']:
hu_allegiance[year][allegiance_type]=\
get_allegiance(hu_allegiance[year][allegiance_type],j['voteid'],
outcome,dummy[allegiance_type])
eu_joint_allegiance[year][allegiance_type]=\
get_allegiance(eu_joint_allegiance[year][allegiance_type],j['voteid'],
outcome,dummy[allegiance_type])
eu_allegiance[country]=hu_allegiance
eu_vt[country]=hu_vt
print(country)
Hungary Romania
eu_allegiance['Joint']=eu_joint_allegiance
eu_vt['Joint']=eu_joint_vt
countries=countries+['Joint']
save_local(eu_vt,'eu_vt')
save_local(eu_allegiance,'eu_allegiance')
del eu_vt
del eu_allegiance
dossiers=load_file('ep_dossiers.json.xz')
loading ep_dossiers.json.xz...
#amendments=load_file('ep_amendments.json.xz')
loading ep_amendments.json.xz...
dossiers_by_id={}
dossiers_by_report={}
for d in dossiers:
dossiers_by_id[d['_id']]=d['procedure']
voteid_2_dossierid={}
for v in votes:
if v['voteid'] not in voteid_2_dossierid:voteid_2_dossierid[v['voteid']]=[]
if 'dossierid' in v:
voteid_2_dossierid[v['voteid']].append(dossiers_by_id[v['dossierid']])
save_local(voteid_2_dossierid,'voteid_2_dossierid')