StatsBomb 360 Data Parsing¶

# Python ≥3.5 (ideally)
import platform
import sys, getopt
assert sys.version_info >= (3, 5)
import csv

# Import Dependencies
%matplotlib inline

# Math Operations
import numpy as np
from math import pi

# Datetime
import datetime
from datetime import date
import time

# Data Preprocessing
import pandas as pd
import pandas_profiling as pp
import os
import re
import chardet
import random
from io import BytesIO
from pathlib import Path

# Reading Directories
import glob
import os

# Working with JSON
import json
from pandas.io.json import json_normalize

# Data Visualisation
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
import missingno as msno

# Progress Bar
from tqdm import tqdm

# Display in Jupyter
from IPython.display import Image, YouTubeVideo
from IPython.core.display import HTML

# Ignore Warnings
import warnings
warnings.filterwarnings(action="ignore", message="^internal gelsd")

print("Setup Complete")

# Python / module versions used here for reference
print('Python: {}'.format(platform.python_version()))
print('NumPy: {}'.format(np.__version__))
print('pandas: {}'.format(pd.__version__))
print('matplotlib: {}'.format(mpl.__version__))

# Show files in directory
print(glob.glob(os.path.join(data_dir_sb, 'raw', 'competitions/*')))

# Read in exported CSV file if exists, if not, read in JSON file

## 
if not os.path.exists(os.path.join(data_dir_sb, 'raw', 'competitions', 'competitions.csv')):
    json_competitions = read_json_file(os.path.join(data_dir_sb, 'open-data', 'data', 'competitions.json'))
    df_competitions_flat = pd.read_json(json_competitions)

##     
else:
    df_competitions_flat = pd.read_csv(os.path.join(data_dir_sb, 'raw', 'competitions', 'competitions.csv'))    


# Display DataFrame
df_competitions_flat

df_competitions_flat.shape

df_competitions_flat

# Define a list to select only the competitions of interest. 

# Flatmap all Competition IDs to use all available competitions
lst_competitions = df_competitions_flat['competition_id'].unique().tolist()

"""
# Define list of competitions
lst_competitions = [2,     # Premier League
                    11,    # La Liga
                    16,    # Champions League
                   #37,    # FA Women's Super League
                    43,    # FIFA World Cup
                   #49,    # NWSL
                   #55,    # UEFA Euro
                   #72,    # Women's World Cup
                   ]

"""

# Display list of competitions
lst_competitions

# Display the number of competitions
len(lst_competitions)

# Show files in directory
print(glob.glob(os.path.join(data_dir_sb, 'raw', 'matches/*')))

# Read in selected matches

## Read in exported CSV file if exists, if not, read in JSON file
if not os.path.exists(os.path.join(data_dir_sb, 'raw', 'matches', 'matches_sb_360.csv')):
    
    ### Create empty list for DataFrames
    dfs_matches_all = []
    
    ### Loop through the selected competitions
    for competition in lst_competitions:

        ### Create empty list for DataFrames
        dfs_matches_competition = []
        
        #### Show files in directory
        lst_filepaths = list(glob.glob(data_dir_sb + '/open-data/data/matches/' + str(competition) + '/*'))
            
        for filepath in lst_filepaths:
        
            ##### Open the JSON filepath with defined Competition and Season IDs
            try:

                ###### Import all StatsBomb JSON Match data for the mens matches
                with open(filepath) as f:
                    json_sb_match_data = json.load(f)

                ###### Flatten the JSON Match data
                df_matches_flat = json_normalize(json_sb_match_data)

                ###### Append each Match data to 
                dfs_matches_competition.append(df_matches_flat)

                ## Concatenate DataFrames to one DataFrame
                df_matches_competition = pd.concat(dfs_matches_competition)

            #####
            except:
                pass

        ## Concatenate DataFrames to one DataFrame
        dfs_matches_all.append(df_matches_competition)
            
    ## Concatenate DataFrames to one DataFrame
    df_matches_flat = pd.concat(dfs_matches_all)
    
##
else:    
    df_matches_flat = pd.read_csv(os.path.join(data_dir_sb, 'raw', 'matches', 'matches_sb_360.csv'))
    
    
## Display DataFrame
df_matches_flat.head()

df_matches_flat.shape

# Shot outcomes types and their frequency
df_matches_flat.groupby(['competition.competition_name', 'season.season_name']).match_id.count()

# Flatmap all Match IDs to use all available matches
lst_matches = df_matches_flat['match_id'].tolist()

# Display the number of matches
len(lst_matches)

# Show files in directory
print(glob.glob(os.path.join(data_dir_sb, 'raw', 'events/*')))

# Read in exported CSV file if exists, if not, read in JSON file

##
if not os.path.exists(os.path.join(data_dir_sb, 'raw', 'events', 'events_sb_360.csv')):

    ### Create empty list for DataFrames
    dfs_events = []

    ### Loop through event files for the selected matches and append DataFrame to dfs_events list
    for match_id in lst_matches:
        
        #### 
        with open(data_dir_sb + '/open-data/data/events/' + str(match_id) + '.json') as f:
            event = json.load(f)
           #match_id = str(match_id)
            df_event_flat = json_normalize(event)
            df_event_flat['match_id'] = match_id
            dfs_events.append(df_event_flat)    

    ### Concatenate DataFrames to one DataFrame
    df_events = pd.concat(dfs_events)
    
    ### Flatten the nested columns
    df_events_flat = flatten_nested_json_df(df_events)
    
    
##
else:    
    df_events_flat = pd.read_csv(os.path.join(data_dir_sb, 'raw', 'events', 'events_sb_360.csv'))
    
    
## Display DataFrame
df_events_flat.head()

df_events_flat.shape

# Read in exported CSV file if exists, if not, merge the individual DataFrames
if not os.path.exists(os.path.join(data_dir_sb, 'raw', 'combined', 'combined_sb_360.csv')):
    
    # Join the Matches DataFrame to the Events DataFrame
    df_events_matches = pd.merge(df_events_flat, df_matches_flat, left_on=['match_id'], right_on=['match_id'])

    # Join the Competitions DataFrame to the Events-Matches DataFrame
    df_events_matches_competitions = pd.merge(df_events_matches, df_competitions_flat, left_on=['competition.competition_id', 'season.season_id'], right_on=['competition_id', 'season_id'])
    
else:    
    df_events_matches_competitions = pd.read_csv(os.path.join(data_dir_sb, 'raw', 'combined', 'combined_sb_360.csv'))
    
    
# Display DataFrame
df_events_matches_competitions.head()

print('No. rows in Events DataFrame BEFORE join to Matches and Competitions DataFrames: {}'.format(len(df_events_flat)))
print('No. rows in DataFrame AFTER join: {}\n'.format(len(df_events_matches_competitions)))
print('-'*10+'\n')
print('Variance in rows before and after join: {}\n'.format(len(df_events_matches_competitions) - len(df_events_flat)))

# Display the first five rows of the DataFrame, df_events_matches_competitions
df_events_matches_competitions.head()

# Display the last five rows of the DataFrame, df_events_matches_competitions
df_events_matches_competitions.tail()