# To install this library in Jupyter notebook
#import sys
#!{sys.executable} -m pip install pandas
import pandas as pd
pd.__version__ , pd.__path__
('1.3.4', ['/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas'])
A Pandas Dataframe is a two-dimensional labeled data structure (like SQL table) with heterogeneously typed columns, having both a row and a column index.
pd.DataFrame(data=None, index=None, columns=None, dtype=None)
data
: It can be a 2-D NumPy Array, a Dictionary of Python Lists, or a Dictionary of Panda Series (You can also create a dataframe from a file in CSV, Excel, JSON, HTML format or may be from a database table as well).index
: These are the row indices. Will default to RangeIndex (0, 1, 2, ..., n), if index argument is not passed and no indexing information is part of input data.columns
: These are the column indices or labels. Will default to RangeIndex (0, 1, 2, ..., n), if index argument is not passed and no indexing information is part of input data.dtype
: Data type to force. Only a single dtype is allowed. If None, infer.
import pandas as pd
import numpy as np
df = pd.DataFrame()
print(df)
Empty DataFrame Columns: [] Index: []
arr = np.random.randint(10,100, size= (6,5))
print("Numpy Array:\n",arr)
df = pd.DataFrame(data=arr)
print("Pandas Dataframe:\n",df)
Numpy Array: [[51 10 67 20 67] [55 24 35 57 95] [47 74 25 37 10] [96 64 19 21 86] [10 49 10 50 45] [94 33 76 66 37]] Pandas Dataframe: 0 1 2 3 4 0 51 10 67 20 67 1 55 24 35 57 95 2 47 74 25 37 10 3 96 64 19 21 86 4 10 49 10 50 45 5 94 33 76 66 37
# Let us name the column labels of our choice, while creating it
col_labels=['Col1', 'Col2', 'Col3', 'Col4', 'Col5']
df = pd.DataFrame(data=arr, columns=col_labels)
df
# Let us name the row labels of our choice, while creating it
df = pd.DataFrame(data=arr, index=['Row0', 'Row1', 'Row2', 'Row3', 'Row4', 'Row5'])
df
# Let us name the both row labels and column labels to strings of our choice, while creating it
row_labels = ['Row0', 'Row1', 'Row2', 'Row3', 'Row4', 'Row5']
col_labels = ['Col0', 'Col1', 'Col2', 'Col3', 'Col4']
df = pd.DataFrame(data=arr, index=row_labels, columns=col_labels)
df
Col0 | Col1 | Col2 | Col3 | Col4 | |
---|---|---|---|---|---|
Row0 | 51 | 10 | 67 | 20 | 67 |
Row1 | 55 | 24 | 35 | 57 | 95 |
Row2 | 47 | 74 | 25 | 37 | 10 |
Row3 | 96 | 64 | 19 | 21 | 86 |
Row4 | 10 | 49 | 10 | 50 | 45 |
Row5 | 94 | 33 | 76 | 66 | 37 |
df['Row0']
--------------------------------------------------------------------------- KeyError Traceback (most recent call last) /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance) 3360 try: -> 3361 return self._engine.get_loc(casted_key) 3362 except KeyError as err: /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc() /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc() pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item() pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item() KeyError: 'Row0' The above exception was the direct cause of the following exception: KeyError Traceback (most recent call last) /var/folders/1t/g3ylw8h50cjdqmk5d6jh1qmm0000gn/T/ipykernel_28361/1613391253.py in <module> ----> 1 df['Row0'] /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/frame.py in __getitem__(self, key) 3456 if self.columns.nlevels > 1: 3457 return self._getitem_multilevel(key) -> 3458 indexer = self.columns.get_loc(key) 3459 if is_integer(indexer): 3460 indexer = [indexer] /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance) 3361 return self._engine.get_loc(casted_key) 3362 except KeyError as err: -> 3363 raise KeyError(key) from err 3364 3365 if is_scalar(key) and isna(key) and not self.hasnans: KeyError: 'Row0'
index
and columns
attributes of a dataframe object.
arr = np.random.randint(10,100, size= (6,5))
df = pd.DataFrame(data=arr)
df
row_labels = ['Row0', 'Row1', 'Row2', 'Row3', 'Row4', 'Row5']
col_labels = ['Col0', 'Col1', 'Col2', 'Col3', 'Col4']
df.columns = col_labels
df.index = row_labels
df
Keys
become the column names, andValues
are lists/arrays containing data for the respective columns.people = {
"name" : ["Rauf", "Arif", "Maaz", "Hadeed", "Mujahid", "Mohid"],
"age" : [52, 51, 26, 22, 18, 17],
"address": ["Lahore", "Karachi", "Lahore", "Islamabad", "Kakul", "Karachi"],
"cell" : ["321-123", "320-431", "321-478", "324-446", "321-967", "320-678"],
"bg": ["B+", "A-", "B+", "O-", "A-", "B+"]
}
people
# Pass this Dictionary of Python Lists to pd.Dataframe()
df_people = pd.DataFrame(data=people)
df_people
pd.DataFrame()
method, or can do that later by assigning the new values to the index
and columns
attributes of a dataframe object.# Let us change the row labels of above dataframe
row_labels = ['MS01', 'MS02', 'MS03', 'MS04', 'MS05', 'MS06']
df_people.index = row_labels
df_people
One can think of a dataframe as a dictionary of Panda Series:
Keys
are column names, andValues
are Series object for the respective columns.dict = {
"name": pd.Series(['Arif', 'Hadeed', 'Mujahid']),
"age": pd.Series([50, 22, 18]),
"addr": pd.Series(['Lahore', 'Islamabad','Karachi']),
}
df = pd.DataFrame(data=dict)
df
Note from the above output, that every series object becomes the data of the appropriate column. Moreover, the keys of the dictionary become the column labels.
dict = {
"name": pd.Series(data=['Arif', 'Hadeed', 'Mujahid', 'Maaz'], index=['a','b','c', 'd']),
"age": pd.Series(data=[50, 22,np.nan, 18], index=['a','b','c','d']),
"addr": pd.Series(data=['Lahore', '', 'Peshawer','Karachi'], index=['a','b','c', 'd']),
}
df = pd.DataFrame(dict)
df
- In the above code and its output, note that every series object has four data values and four corresponding indices.
- Also note that in the
age
series, we have a NaN value, and in theaddr
series we have an empty string.- Another point to note that the row indices of the three series exactly match, in number as well as in sequence/value.
- A question arise, what if the indices of series are different. See the following code to understand this concept.
import pandas as pd
import numpy as np
dict = {
"name": pd.Series(['Arif', 'Hadeed', 'Mujahid', 'Maaz'], index=['a','b','c', 'd']),
"age": pd.Series([50, 22,np.nan, 18], index=['a','x','y','d']),
"addr": pd.Series(['Lahore', '','Karachi'], index=['a', 'd', 'x']),
}
df = pd.DataFrame(dict)
df
name | age | addr | |
---|---|---|---|
a | Arif | 50.0 | Lahore |
b | Hadeed | NaN | NaN |
c | Mujahid | NaN | NaN |
d | Maaz | 18.0 | |
x | NaN | 22.0 | Karachi |
y | NaN | NaN | NaN |
df.head(-4)
name | age | addr | |
---|---|---|---|
a | Arif | 50.0 | Lahore |
b | Hadeed | NaN | NaN |
df.tail(-4)
name | age | addr | |
---|---|---|---|
x | NaN | 22.0 | Karachi |
y | NaN | NaN | NaN |
- In the above code and its output, note that first series object has four data values and four corresponding indices. Similarly, second series object has four data values (with one
np.nan
value) and four corresponding indices, which are a bit different from the first series object. Third series has three data values (with one empty string) and three indices.- Note the resulting Dataframe has six rows and three columns.
- For index 'a' we have value in all the three series objects or columns.
- For index 'b' we have a value in first series object, and NaN for the second and third column, since the second and third series object has no value corresponding to row index 'b.
.
notationpeople = {
"name" : ["Rauf", "Arif", "Maaz", "Hadeed", "Mujahid", "Mohid"],
"age" : [52, 51, 26, 22, 18, 17],
"address": ["Lahore", "Karachi", "Lahore", "Islamabad", "Kakul", "Karachi"],
"cell" : ["321-123", "320-431", "321-478", "324-446", "321-967", "320-678"],
"bg": ["B+", "A-", "B+", "O-", "A-", "B+"]
}
df_people = pd.DataFrame(data=people, index=['MS01', 'MS02', 'MS03', 'MS04', 'MS05', 'MS06'])
df_people
name | age | address | cell | bg | |
---|---|---|---|---|---|
MS01 | Rauf | 52 | Lahore | 321-123 | B+ |
MS02 | Arif | 51 | Karachi | 320-431 | A- |
MS03 | Maaz | 26 | Lahore | 321-478 | B+ |
MS04 | Hadeed | 22 | Islamabad | 324-446 | O- |
MS05 | Mujahid | 18 | Kakul | 321-967 | A- |
MS06 | Mohid | 17 | Karachi | 320-678 | B+ |
# `shape` attribute of a dataframe object return a two value tuple containing rows and columns
# Note the rows count does not include the column labels and column count does not include the row index
df_people.shape
(6, 5)
# `ndim` attribute of a dataframe object returns number of dimensions (which is always 2)
df_people.ndim
2
# `size` attribute of a dataframe object returns the number of elements in the underlying data
df_people.size
30
# `index` attribute of a dataframe object return the list of row indices and its datatype
df_people.index
Index(['MS01', 'MS02', 'MS03', 'MS04', 'MS05', 'MS06'], dtype='object')
# `columns` attribute of a dataframe object return the list of column labels and its datatype
df_people.columns
Index(['name', 'age', 'address', 'cell', 'bg'], dtype='object')
#This attribute is used to fetch both index and column names.
df_people.axes
[Index(['MS01', 'MS02', 'MS03', 'MS04', 'MS05', 'MS06'], dtype='object'), Index(['name', 'age', 'address', 'cell', 'bg'], dtype='object')]
# `values` attribute of a dataframe object returns a NumPy 2-D having all the values in the DataFrame,
# without the row indices and column labels
df_people.values
array([['Rauf', 52, 'Lahore', '321-123', 'B+'], ['Arif', 51, 'Karachi', '320-431', 'A-'], ['Maaz', 26, 'Lahore', '321-478', 'B+'], ['Hadeed', 22, 'Islamabad', '324-446', 'O-'], ['Mujahid', 18, 'Kakul', '321-967', 'A-'], ['Mohid', 17, 'Karachi', '320-678', 'B+']], dtype=object)
df.empty
False
# `dtypes` attribute of a dataframe object return the data type of each column in the dataframe
df_people.dtypes
name object age int64 address object cell object bg object dtype: object
# To check number on non-NA values
df_people.count()
df.info()
Method¶#This method prints information about a DataFrame including the row indices, column labels,
# non-null values count in each column, datatype and memory usage
df_people.info()
--------------------------------------------------------------------------- NameError Traceback (most recent call last) /var/folders/1t/g3ylw8h50cjdqmk5d6jh1qmm0000gn/T/ipykernel_41559/1037101920.py in <module> 1 #This method prints information about a DataFrame including the row indices, column labels, 2 # non-null values count in each column, datatype and memory usage ----> 3 df_people.info() NameError: name 'df_people' is not defined