# To install this library in Jupyter notebook
#import sys
#!{sys.executable} -m pip install pandas
import pandas as pd
pd.__version__ , pd.__path__
df.sort_values()
df.sort_index()
import pandas as pd
df = pd.read_csv('datasets/groupdata.csv')
df.head()
# `shape` attribute of a dataframe object return a two value tuple containing rows and columns
# Note the rows count does not include the column labels and column count does not include the row index
df.shape
# `index` attribute of a dataframe object return the list of row indices and its datatype
df.index
# `columns` attribute of a dataframe object return the list of column labels and its datatype
df.columns
# `dtypes` attribute of a dataframe object return the data type of each column in the dataframe
df.dtypes
df.columns
df.subj1 + df.subj2
df.subj1.add(df.subj2)
ser1 = df.subj1.add(df.subj2, fill_value=0)
ser1
# On the left side of assignment you must use `[]` operator, while on the right you can use dot operator as well
df['total'] = ser1
Note that once, nothing appears to happen after you execute a Jupyter notebook cell, that means some processing has been done in the background. Over here, a new column has been added to the dataframe named df. Let us confirm this
df.head(3)
del df['colname']
, which will remove the column, but will not return itdf.pop('colname')
method which will remove that column as well as return the deleted column as a seriesdf.drop()
is a better method than the above two. It can delete more than one columns and is not inplace. Moreover, it can be used to delete rows as well
del df['colname']
¶del df['colname']
will remove the column without returning it. It is inplacedf.columns
del df['total']
df.head(3)
df.pop('colname')
¶df.pop('colname')
method will remove the column and will return the Series that has been removed from the dataframe. It is inplacedf.pop('address')
df.head(3)
df.drop()
¶df.drop()
method is used to remove one or more columns and will return a Series or Dataframe object accordingly.df.drop(columns=[---], axis=1, inplace=False)
columns
parameter and to the axis
argument pass 1. (axis
argument specifies the direction of operation, which is left to right while deleting columns)import pandas as pd
df = pd.read_csv('datasets/groupdata.csv')
df.head(3)
df.drop(columns='name')
# Remember axis is the direction of operation, and axis=1 is the column axis that goes from left to right
df.drop(columns=['name', 'age', 'address'], axis=1)
It has just returned the resulting dataframe after removing the columns. No change has made to the original dataframe
df.head(3)
Let us repeat the operation, with inplace=True
. Note this time it will return None. However, changes has been made to the original dataframe
df.drop(columns=['age', 'address', 'name'], axis=1, inplace=True)
df.head(3)
df.append()
method, which will return a new dataframe with the row added.df.append(other, ignore_index=False)
More on append in next session
import numpy as np
import pandas as pd
df = pd.read_csv('datasets/groupdata.csv')
df.tail()
# Let us create a new dataframe having a single row
newdf = pd.DataFrame(data=[['MS222', 100, 'Kamokey', 'AFT', 'group D', 'Male', 55.0, 55.0, 9999]],
columns=['roll no', 'age', 'address', 'session', 'group', 'gender','subj1', 'subj2', 'scholarship'])
newdf
# Let us create a new dataframe having a single row (Can always create one having multiple rows).
# Do note that we have not mentioned the scholarship data value as well as the scholarship column name
newdf = pd.DataFrame(data=[['New Student', 'MS222', 100, 'Kamokey', 'AFT', 'group D', 'Male', 55.0, 55.0]],
columns=['name', 'roll no', 'age', 'address', 'session', 'group', 'gender','subj1', 'subj2'])
newdf
Note: The index associated with the only row in above dataframe is 0. Moreover, the sequence of columns is not same (name is coming before rollno)
df = df.append(newdf , ignore_index=True)
df.tail()
ignore_index=True
argument it has been assigned the next available index. Otherwise, the new row will also have row index 0.df.append()
method do not have inplace
argument, so you always have to assign the resulting dataframe to the original df
.df.drop()
has inplace
argument, while df.append()
does not have inplace
argument.
df.drop()
method is used to remove one or more rows (other than columns) and will return a Series or Dataframe object accordingly.df.drop(index=[---], axis=0, inplace=False)
index
parameter and to the axis
argument pass 0. (axis
argument specifies the direction of operation, which is top to bottom while deleting rows)import pandas as pd
df = pd.read_csv('datasets/groupdata.csv')
df.head()
df.drop(index=[2,4], axis=0, inplace = True)
df.head()
Create a Simple Dataframe
import pandas as pd
df = pd.read_csv('datasets/groupdata.csv')
df.head()
Example: Add a new column total
that contains sum of marks in subj1
and subj2
. Then add a new column that should contain string "Good"
if total>150
other wise string "Bad"
df['total'] = df.subj1 + df.subj2
df.head()
list1 = ['Good' if i >=150 else 'Bad' for i in df.total]
list1
df['grade'] = ['Good' if i >=150 else 'Bad' for i in df.total]
df.head()
df = pd.read_csv('datasets/groupdata.csv')
df.head()
df[df['address'] == 'Lahore']
df[df['address'] == 'Lahore'].index
df.drop(index = df[df['address'] == 'Lahore'].index, axis = 0, inplace = True)
df
# Let us drop an entire row from the data frame, in which session is 'AFT'
# Get the indices where session == 'AFT' using the .index function
indices = df[df['session'] == 'AFT'].index
indices
# Pass those indices to the drop method to delete those rows
df.drop(index = indices, inplace = True)
df
df = pd.read_csv('datasets/groupdata.csv')
df
df.info()
Example: Let us drop the column(s) from above data frame, if it has >= than 2 NaN values
mylist_mask=df.apply(lambda col: col.isnull().sum() >= 1)
mylist_mask
mylist_names=df.columns[mylist_mask]
mylist_names
df.drop(columns=mylist_names, axis=1, inplace=True)
df.info()
# It will delete the Scholarship column
df.drop(columns=df.columns[df.apply(lambda col: col.isnull().sum() >= 2)], axis=1, inplace=True)
# Verify
df.info()
df.head(3)
astype(dtype)
method to cast a pandas object to a specified dtype dtype
.int64
to float64
¶df = pd.read_csv('datasets/groupdata.csv')
df.dtypes
#Suppose we want to change the datatype of `age` column to float64 dtype
df['age'] = df.age.astype(float)
df.dtypes
df.head()
df = pd.read_csv('datasets/groupdata.csv')
df.head()
df.gender.str.contains('Male')
df.gender.str.contains('Male').astype(int)
df['gender'] = df.gender.str.contains('Male').astype(int)
df
df.sort_values()
¶Pandas data frame has two useful functions.
df.sort_values()
to sort by values of one or more columns anddf.sort_index()
to sort by the index. Each of these functions come with numerous options, like sorting in specific order (ascending or descending), sorting in place, sorting with missing values, sorting by specific algorithm etc.
df.sort_values()
function sort by the values along either axis.df.sort_values(by,axis=0,ascending=True,inplace=False,kind='quicksort',na_position='last',ignore_index=False)
Where,
by
: str or list of str to sortaxis
: If axis
is 0 or 'index' then 'by' may contain index levels and/or column labels. If axis
is 1 or 'columns' then 'by' may contain column levels and/or index labels.ascending
: if True then ascending and if False then descendinginplace
: If True, perform operation in-place.kind
: {'quicksort', 'mergesort', 'heapsort', 'stable'}, default 'quicksort'. This option is only applied when sorting on a single column or label.na_position
: If first then puts NaNs at the beginning. Default is lastignore_index
: If True, the resulting axis will be labeled 0, 1, …, n - 1. Default Falseimport pandas as pd
df = pd.DataFrame({
'roll_no': [ 102, 101, 104, 103, 105],
'name' : ['Kamal', 'Saima', 'Jamal', 'Shaikh', 'Farzana'],
'gender' : ['M', 'F', 'M', 'M', 'F'],
'grade' : ['A', 'A', 'B', 'B', 'A'],
'marks' : [ 21, 23, 12, 14, 20],
'city' : ['Lahore', 'Peshawer', 'Lahore', 'Karachi', 'Peshawer']
})
df
# Let us sort the data by grade column
# By default the sorting is done in ascending order and is not inplace
df1 = df.sort_values(by=['grade'])
df1
grade
column. You can observe that some of the students with higher marks are ranked lower.# sort the dataframe
df2 = df.sort_values(by=['grade','marks'])
df2
df3 = df.sort_values(by=['grade','marks'], ascending=[True,False])
df3
reset_index()
function.df3.reset_index()
drop = True
and also inplace = True
to save the state.df3.reset_index(inplace=True, drop=True)
df3
df = pd.read_csv('datasets/groupdata.csv')
df
# If there is a missing value NaN, by default it is listed at the end when using sort_values function
# Regardless of the sorting order (Ascending or Descending)
df.sort_values(by='scholarship')
# If the argument na_position='first', it will be listed at the top.
df.sort_values(by=['scholarship'], na_position='first')
- Checkout the
df.nlargest()
method which return the firstn
rows ordered bycolumns
in descending order.- Checkout the
df.nsmallest()
method which return the firstn
rows ordered bycolumns
in ascending order.
df.sort_index()
¶We have observed while using
df.sort_values()
, by default the sorting is performed in the vertical direction. If you want to sort in the row direction, we can set theaxis
argument ofdf.sort_values()
method to 1, which is by default set to zero. However, it may cause problems when a number and a string are mixed
df.sort_index()
method.df.sort_index(axis=0,ascending=True,inplace=False,kind='quicksort',na_position='last',ignore_index=False)
Where,
axis
: The axis along which to sort. The value 0 identifies the rows, and 1 identifies the columns. (default is 0)ascending
: If True then ascending and If False then descendinginplace
: If True, perform operation in-place.kind
: {'quicksort', 'mergesort', 'heapsort', 'stable'}, default 'quicksort'. This option is only applied when sorting on a single column or label.na_position
: If first then puts NaNs at the beginning. Default is lastignore_index
: If True, the resulting axis will be labeled 0, 1, …, n - 1. Default False# Let us create a simple data frame
import pandas as pd
df = pd.DataFrame({
'roll_no': [ 102, 101, 104, 105, 103],
'name' : ['Kamal', 'Saima', 'Jamal','Farzana', 'Shaikh'],
'gender' : ['M', 'F', 'M', 'M', 'F'],
'grade' : ['A', 'A', 'B', 'B', 'A'],
'marks' : [ 21, 23, 12, 14, 20],
'city' : ['Lahore', 'Peshawer', 'Lahore', 'Karachi', 'Peshawer']
})
df
df1 = df.sort_index(axis=1)
df1
reset_index()
method to make the row indices again in increasing numeric order.roll_no
column as indexsort_index()
with axis=0reset_index()
df
Let us sort by roll_no
# Lets us set the roll_no column as index
df1 = df.set_index(["roll_no"])
df1
Note that the implicit index collumn is dropped and the roll_no column has become the index of this dataframe.
# sort the datframe by index
df2 = df1.sort_index(axis=0)
df2
# After sort you can reset the index if you want
df3 = df2.reset_index(drop=False)
df3