Python pandas Q&A video series by Data School¶

# examine the first 5 rows
orders.head()

# examine the first 5 rows
users.head()

# examine the first 5 rows
ufo.head()

# select the 'City' Series using bracket notation
ufo['City']

# or equivalently, use dot notation
ufo.City

# create a new 'Location' Series (must use bracket notation to define the Series name)
ufo['Location'] = ufo.City + ', ' + ufo.State
ufo.head()

# example method: show the first 5 rows
movies.head()

# example method: calculate summary statistics
movies.describe()

# example attribute: number of rows and columns
movies.shape

# example attribute: data type of each column
movies.dtypes

# use an optional parameter to the describe method to summarize only 'object' columns
movies.describe(include=['object'])

# examine the column names
ufo.columns

# rename two of the columns by using the 'rename' method
ufo.rename(columns={'Colors Reported':'Colors_Reported', 'Shape Reported':'Shape_Reported'}, inplace=True)
ufo.columns

# replace all of the column names by overwriting the 'columns' attribute
ufo_cols = ['city', 'colors reported', 'shape reported', 'state', 'time']
ufo.columns = ufo_cols
ufo.columns

# replace the column names during the file reading process by using the 'names' parameter
ufo = pd.read_csv('http://bit.ly/uforeports', header=0, names=ufo_cols)
ufo.columns

# replace all spaces with underscores in the column names by using the 'str.replace' method
ufo.columns = ufo.columns.str.replace(' ', '_')
ufo.columns

# read a dataset of UFO reports into a DataFrame
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.head()

# remove a single column (axis=1 refers to columns)
ufo.drop('Colors Reported', axis=1, inplace=True)
ufo.head()

# remove multiple columns at once
ufo.drop(['City', 'State'], axis=1, inplace=True)
ufo.head()

# remove multiple rows at once (axis=0 refers to rows)
ufo.drop([0, 1], axis=0, inplace=True)
ufo.head()

# read a dataset of top-rated IMDb movies into a DataFrame
movies = pd.read_csv('http://bit.ly/imdbratings')
movies.head()

# sort the 'title' Series in ascending order (returns a Series)
movies.title.sort_values().head()

# sort in descending order instead
movies.title.sort_values(ascending=False).head()

# sort the entire DataFrame by the 'title' Series (returns a DataFrame)
movies.sort_values('title').head()

# sort in descending order instead
movies.sort_values('title', ascending=False).head()

# sort the DataFrame first by 'content_rating', then by 'duration'
movies.sort_values(['content_rating', 'duration']).head()

# read a dataset of top-rated IMDb movies into a DataFrame
movies = pd.read_csv('http://bit.ly/imdbratings')
movies.head()

# examine the number of rows and columns
movies.shape

# confirm that the list has the same length as the DataFrame
len(booleans)

# examine the first five list elements
booleans[0:5]

# convert the list to a Series
is_long = pd.Series(booleans)
is_long.head()

# use bracket notation with the boolean Series to tell the DataFrame which rows to display
movies[is_long]

# simplify the steps above: no need to write a for loop to create 'is_long' since pandas will broadcast the comparison
is_long = movies.duration >= 200
movies[is_long]

# or equivalently, write it in one line (no need to create the 'is_long' object)
movies[movies.duration >= 200]

# select the 'genre' Series from the filtered DataFrame
movies[movies.duration >= 200].genre

# or equivalently, use the 'loc' method
movies.loc[movies.duration >= 200, 'genre']

# read a dataset of top-rated IMDb movies into a DataFrame
movies = pd.read_csv('http://bit.ly/imdbratings')
movies.head()

# filter the DataFrame to only show movies with a 'duration' of at least 200 minutes
movies[movies.duration >= 200]

# demonstration of the 'and' operator
print(True and True)
print(True and False)
print(False and False)

# demonstration of the 'or' operator
print(True or True)
print(True or False)
print(False or False)

# CORRECT: use the '&' operator to specify that both conditions are required
movies[(movies.duration >=200) & (movies.genre == 'Drama')]

# INCORRECT: using the '|' operator would have shown movies that are either long or dramas (or both)
movies[(movies.duration >=200) | (movies.genre == 'Drama')].head()

# use the '|' operator to specify that a row can match any of the three criteria
movies[(movies.genre == 'Crime') | (movies.genre == 'Drama') | (movies.genre == 'Action')].head(10)

# or equivalently, use the 'isin' method
movies[movies.genre.isin(['Crime', 'Drama', 'Action'])].head(10)

# read a dataset of UFO reports into a DataFrame, and check the columns
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.columns

# specify which columns to include by name
ufo = pd.read_csv('http://bit.ly/uforeports', usecols=['City', 'State'])

# or equivalently, specify columns by position
ufo = pd.read_csv('http://bit.ly/uforeports', usecols=[0, 4])
ufo.columns

# specify how many rows to read
ufo = pd.read_csv('http://bit.ly/uforeports', nrows=3)
ufo

# Series are directly iterable (like a list)
for c in ufo.City:
    print(c)

# various methods are available to iterate through a DataFrame
for index, row in ufo.iterrows():
    print(index, row.City, row.State)

# read a dataset of alcohol consumption into a DataFrame, and check the data types
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.dtypes

# only include numeric columns in the DataFrame
import numpy as np
drinks.select_dtypes(include=[np.number]).dtypes

# describe all of the numeric columns
drinks.describe()

# pass the string 'all' to describe all columns
drinks.describe(include='all')

# pass a list of data types to only describe certain types
drinks.describe(include=['object', 'float64'])

# pass a list even if you only want to describe a single data type
drinks.describe(include=['object'])

# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()

# drop a column (temporarily)
drinks.drop('continent', axis=1).head()

# drop a row (temporarily)
drinks.drop(2, axis=0).head()

# calculate the mean of each numeric column
drinks.mean()

# or equivalently, specify the axis explicitly
drinks.mean(axis=0)

# calculate the mean of each row
drinks.mean(axis=1).head()

# 'index' is an alias for axis 0
drinks.mean(axis='index')

# 'columns' is an alias for axis 1
drinks.mean(axis='columns').head()

# read a dataset of Chipotle orders into a DataFrame
orders = pd.read_table('http://bit.ly/chiporders')
orders.head()

# normal way to access string methods in Python
'hello'.upper()

# string methods for pandas Series are accessed via 'str'
orders.item_name.str.upper().head()

# string method 'contains' checks for a substring and returns a boolean Series
orders.item_name.str.contains('Chicken').head()

# use the boolean Series to filter the DataFrame
orders[orders.item_name.str.contains('Chicken')].head()

# string methods can be chained together
orders.choice_description.str.replace('[', '').str.replace(']', '').head()

# many pandas string methods support regular expressions (regex)
orders.choice_description.str.replace('[\[\]]', '').head()

# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()

# examine the data type of each Series
drinks.dtypes

# change the data type of an existing Series
drinks['beer_servings'] = drinks.beer_servings.astype(float)
drinks.dtypes

# alternatively, change the data type of a Series while reading in a file
drinks = pd.read_csv('http://bit.ly/drinksbycountry', dtype={'beer_servings':float})
drinks.dtypes

# read a dataset of Chipotle orders into a DataFrame
orders = pd.read_table('http://bit.ly/chiporders')
orders.head()

# examine the data type of each Series
orders.dtypes

# convert a string to a number in order to do math
orders.item_price.str.replace('$', '').astype(float).mean()

# string method 'contains' checks for a substring and returns a boolean Series
orders.item_name.str.contains('Chicken').head()

# convert a boolean Series to an integer (False = 0, True = 1)
orders.item_name.str.contains('Chicken').astype(int).head()

# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()

# calculate the mean beer servings across the entire dataset
drinks.beer_servings.mean()

# calculate the mean beer servings just for countries in Africa
drinks[drinks.continent=='Africa'].beer_servings.mean()

# calculate the mean beer servings for each continent
drinks.groupby('continent').beer_servings.mean()

# other aggregation functions (such as 'max') can also be used with groupby
drinks.groupby('continent').beer_servings.max()

# multiple aggregation functions can be applied simultaneously
drinks.groupby('continent').beer_servings.agg(['count', 'mean', 'min', 'max'])

# specifying a column to which the aggregation function should be applied is not required
drinks.groupby('continent').mean()

# side-by-side bar plot of the DataFrame directly above
drinks.groupby('continent').mean().plot(kind='bar')

# read a dataset of top-rated IMDb movies into a DataFrame
movies = pd.read_csv('http://bit.ly/imdbratings')
movies.head()

# examine the data type of each Series
movies.dtypes

# count the non-null values, unique values, and frequency of the most common value
movies.genre.describe()

# count how many times each value in the Series occurs
movies.genre.value_counts()

# display percentages instead of raw counts
movies.genre.value_counts(normalize=True)

# 'value_counts' (like many pandas methods) outputs a Series
type(movies.genre.value_counts())

# thus, you can add another Series method on the end
movies.genre.value_counts().head()

# display the unique values in the Series
movies.genre.unique()

# count the number of unique values in the Series
movies.genre.nunique()

# compute a cross-tabulation of two Series
pd.crosstab(movies.genre, movies.content_rating)

# calculate various summary statistics
movies.duration.describe()

# many statistics are implemented as Series methods
movies.duration.mean()

# 'value_counts' is primarily useful for categorical data, not numerical data
movies.duration.value_counts().head()

# histogram of the 'duration' Series (shows the distribution of a numerical variable)
movies.duration.plot(kind='hist')

# bar plot of the 'value_counts' for the 'genre' Series
movies.genre.value_counts().plot(kind='bar')

# read a dataset of UFO reports into a DataFrame
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.tail()

# 'isnull' returns a DataFrame of booleans (True if missing, False if not missing)
ufo.isnull().tail()

# 'nonnull' returns the opposite of 'isnull' (True if not missing, False if missing)
ufo.notnull().tail()

# count the number of missing values in each Series
ufo.isnull().sum()

# use the 'isnull' Series method to filter the DataFrame rows
ufo[ufo.City.isnull()].head()

# examine the number of rows and columns
ufo.shape

# if 'any' values are missing in a row, then drop that row
ufo.dropna(how='any').shape

# 'inplace' parameter for 'dropna' is False by default, thus rows were only dropped temporarily
ufo.shape

# if 'all' values are missing in a row, then drop that row (none are dropped in this case)
ufo.dropna(how='all').shape

# if 'any' values are missing in a row (considering only 'City' and 'Shape Reported'), then drop that row
ufo.dropna(subset=['City', 'Shape Reported'], how='any').shape

# if 'all' values are missing in a row (considering only 'City' and 'Shape Reported'), then drop that row
ufo.dropna(subset=['City', 'Shape Reported'], how='all').shape

# 'value_counts' does not include missing values by default
ufo['Shape Reported'].value_counts().head()

# explicitly include missing values
ufo['Shape Reported'].value_counts(dropna=False).head()

# confirm that the missing values were filled in
ufo['Shape Reported'].value_counts().head()

# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()

# every DataFrame has an index (sometimes called the "row labels")
drinks.index

# column names are also stored in a special "index" object
drinks.columns

# neither the index nor the columns are included in the shape
drinks.shape

# index and columns both default to integers if you don't define them
pd.read_table('http://bit.ly/movieusers', header=None, sep='|').head()

# identification: index remains with each row when filtering the DataFrame
drinks[drinks.continent=='South America']

# selection: select a portion of the DataFrame using the index
drinks.loc[23, 'beer_servings']

# set an existing column as the index
drinks.set_index('country', inplace=True)
drinks.head()

# 'country' is now the index
drinks.index

# 'country' is no longer a column
drinks.columns

# 'country' data is no longer part of the DataFrame contents
drinks.shape

# country name can now be used for selection
drinks.loc['Brazil', 'beer_servings']

# index name is optional
drinks.index.name = None
drinks.head()

# restore the index name, and move the index back to a column
drinks.index.name = 'country'
drinks.reset_index(inplace=True)
drinks.head()

# many DataFrame methods output a DataFrame
drinks.describe()

# you can interact with any DataFrame using its index and columns
drinks.describe().loc['25%', 'beer_servings']

# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()

# every DataFrame has an index
drinks.index

# every Series also has an index (which carries over from the DataFrame)
drinks.continent.head()

# Series index is on the left, values are on the right
drinks.continent.head()