Notebook

1a. A demo of using pandas to bulk download files...¶

Building off the method we just looked at of using Pandas to grab a single file, we see here how Python can be quite effective at downloading industrial water data for all US states.

Compare running this script with doing the downloads by hand...

In [ ]:

#import os & pandas
import os
import pandas as pd

#import the us package (install if necessary)
try:
    import us
except:
    !pip install us
    print('installing the US package')
    import us

In [ ]:

#Create a folder to hold all the downloads
outFolder = "WaterData"
if not os.path.exists(outFolder): os.mkdir(outFolder)

In [ ]:

#Loop through each state, download it's data, and save to a local file
for state in us.STATES:
    stateAbbr = state.abbr.lower()
    print("{}".format(stateAbbr),end='>')
    
    #Update the url with the state code
    theURL = 'https://waterdata.usgs.gov/{}/nwis/water_use?format=rdb&rdb_compression=value&wu_area=County&wu_year=ALL&wu_county=ALL&wu_category=IN&wu_county_nms=--ALL%2BCounties--&wu_category_nms=Industrial'
    theURL = theURL.format(stateAbbr)
    
    #Get the data as a dataframe
    dfState = pd.read_csv(theURL,delimiter='\t',skiprows=list(range(49))+[50])
    
    #write df to csv in the WaterData folder
    dfState.to_csv("WaterData/{}.csv".format(stateAbbr),index=False)

In [ ]: