In [1]:

# This file is part of the Minnesota Population Center's NHGISXWALK.
# For copyright and licensing information, see the NOTICE and LICENSE files
# in this project's top-level directory, and also on-line at:
#   https://github.com/ipums/nhgisxwalk

Test Subsets for testing: 1990¶

blocks, block groups parts, and blocks¶

From a national crosswalk:
Create target state-level subsets for NHGIS base crosswalks
Create target state-level subsets for NHGIS base tabular data
Record unit tests values for posterity

This is currently only intended for use with block-level data as base units.

James Gaboardi (jgaboardi@gmail.com), 2020-05

In [2]:

%load_ext watermark
%watermark

Last updated: 2021-07-07T19:49:42.757472-04:00

Python implementation: CPython
Python version       : 3.9.6
IPython version      : 7.25.0

Compiler    : Clang 11.1.0 
OS          : Darwin
Release     : 20.5.0
Machine     : x86_64
Processor   : i386
CPU cores   : 8
Architecture: 64bit

In [3]:

import inspect
import nhgisxwalk
import numpy
import pandas

%load_ext autoreload
%autoreload 2
%watermark -w
%watermark -iv

Watermark: 2.2.0

json      : 2.0.9
numpy     : 1.21.0
pandas    : 1.3.0
nhgisxwalk: 0.1.1

Set the state (for subsetting), source & target, and year & geography¶

In [4]:

source_year, target_year = "1990", "2010"
gj_src, gj_trg = "GJOIN%s"%source_year, "GJOIN%s"%target_year

In [5]:

data_path = "../testing_data_subsets/"
block_file = "%s_block" % source_year
supp_file = "%s_blck_grp_598" % source_year

Set the base-level crosswalk file name¶

In [6]:

base_xwalk_name = "nhgis_blk%s_blk%s_gj" % (source_year, target_year)
base_xwalk_name

Out[6]:

'nhgis_blk1990_blk2010_gj'

Set the base (source) summary file name¶

In [7]:

base_source_name = "%s.csv.zip" % block_file
base_source_file = "%s%s" % (data_path, base_source_name)
base_source_file

Out[7]:

'../testing_data_subsets/1990_block.csv.zip'

Set the supplementary summary file name¶

In [8]:

supp_source_name = "%s.csv.zip" % supp_file
supp_source_file = "%s%s" % (data_path, supp_source_name)
supp_source_file

Out[8]:

'../testing_data_subsets/1990_blck_grp_598.csv.zip'

Read in the national the base-level crosswalk¶

In [9]:

data_types = nhgisxwalk.str_types([gj_src, gj_trg])
from_csv_kws = {"path": data_path, "archived": True, "remove_unpacked": True}
read_csv_kws = {"dtype": data_types}
base_xwalk = nhgisxwalk.xwalk_df_from_csv(
    base_xwalk_name, **from_csv_kws, **read_csv_kws
)
base_xwalk.head()

Out[9]:

	GJOIN1990	GJOIN2010	WEIGHT	PAREA_VIA_BLK00
0	G10000100401101	G10000100401001000	1.000000	1.000000
1	G10000100401102	G10000100401001001	0.921750	0.976774
2	G10000100401102	G10000100401001002	0.078219	0.023215
3	G10000100401102	G10000100401001003	0.000031	0.000012
4	G10000100401103	G10000100401001003	1.000000	1.000000

Declare input variable¶

not needed for creating a subset perse, but should do regardless

In [10]:

input_vars = [
    nhgisxwalk.desc_code_1990["Persons"]["Total"],
    nhgisxwalk.desc_code_1990["Families"]["Total"],
    nhgisxwalk.desc_code_1990["Households"]["Total"],
    nhgisxwalk.desc_code_1990["Housing Units"]["Total"]
]
input_var_tags = ["pop", "fam", "hh", "hu"]

Generate the desired crosswalk and subset down to the target state¶

In [11]:

state_bgp1990tr2010 = nhgisxwalk.GeoCrossWalk(
    base_xwalk,
    source_year=source_year,
    target_year=target_year,
    source_geo="bgp",
    target_geo="tr",
    base_source_table=base_source_file,
    supp_source_table=supp_source_file,
    input_var=input_vars,
    weight_var=input_var_tags,
    keep_base=True,
    add_geoid=True
)
state_bgp1990tr2010.xwalk

Out[11]:

	bgp1990gj	tr2010gj	tr2010ge	wt_pop	wt_fam	wt_hh	wt_hu
0	G100001090444072500423009999999999921	G1000010043202	10001043202	1.0	1.0	1.0	1.0
1	G100001090444444300422009999999999926	G1000010042202	10001042202	1.0	1.0	1.0	1.0
2	G100001090444612650422009999999219011	G1000010041200	10001041200	0.0	0.0	0.0	0.0
3	G100001090444612650422009999999219011	G1000010042201	10001042201	1.0	1.0	1.0	1.0
4	G100001090444612650422009999999219012	G1000010042201	10001042201	1.0	1.0	1.0	1.0
...	...	...	...	...	...	...	...
1058	G100005093552999990515009999999999923	G1000050051500	10005051500	1.0	1.0	1.0	1.0
1059	G100005093552999990515009999999999924	G1000050051500	10005051500	1.0	1.0	1.0	1.0
1060	G100005093552999990516009999999999921	G1000050051702	10005051702	1.0	1.0	1.0	1.0
1061	G340033010610106000204029999999916014	G1000030990100	10003990100	0.0	0.0	0.0	0.0
1062	NaN	G1000050990000	10005990000	0.0	0.0	0.0	0.0

1063 rows × 7 columns

unittests¶

In [12]:

state_bgp1990tr2010.xwalk.drop_duplicates(subset=["bgp1990gj", "tr2010gj"])

Out[12]:

	bgp1990gj	tr2010gj	tr2010ge	wt_pop	wt_fam	wt_hh	wt_hu
0	G100001090444072500423009999999999921	G1000010043202	10001043202	1.0	1.0	1.0	1.0
1	G100001090444444300422009999999999926	G1000010042202	10001042202	1.0	1.0	1.0	1.0
2	G100001090444612650422009999999219011	G1000010041200	10001041200	0.0	0.0	0.0	0.0
3	G100001090444612650422009999999219011	G1000010042201	10001042201	1.0	1.0	1.0	1.0
4	G100001090444612650422009999999219012	G1000010042201	10001042201	1.0	1.0	1.0	1.0
...	...	...	...	...	...	...	...
1058	G100005093552999990515009999999999923	G1000050051500	10005051500	1.0	1.0	1.0	1.0
1059	G100005093552999990515009999999999924	G1000050051500	10005051500	1.0	1.0	1.0	1.0
1060	G100005093552999990516009999999999921	G1000050051702	10005051702	1.0	1.0	1.0	1.0
1061	G340033010610106000204029999999916014	G1000030990100	10003990100	0.0	0.0	0.0	0.0
1062	NaN	G1000050990000	10005990000	0.0	0.0	0.0	0.0

1063 rows × 7 columns

In [13]:

ix1, ix2 = 13, 17
state_bgp1990tr2010.xwalk.loc[ix1:ix2-1]

Out[13]:

	bgp1990gj	tr2010gj	tr2010ge	wt_pop	wt_fam	wt_hh	wt_hu
13	G100001090444999990421009999999219012	G1000010042100	10001042100	1.000000	1.000000	1.000000	1.000000
14	G100001090444999990421009999999999921	G1000010042100	10001042100	0.997664	0.997166	0.997148	0.997278
15	G100001090444999990421009999999999921	G1000010042201	10001042201	0.002336	0.002834	0.002852	0.002722
16	G100001090444999990421009999999999922	G1000010042100	10001042100	1.000000	1.000000	1.000000	1.000000

In [14]:

id_cols = ["bgp1990gj", "tr2010gj", "tr2010ge"]
obs_str_vals = state_bgp1990tr2010.xwalk[id_cols][ix1:ix2].values
obs_str_vals

Out[14]:

array([['G100001090444999990421009999999219012', 'G1000010042100',
        '10001042100'],
       ['G100001090444999990421009999999999921', 'G1000010042100',
        '10001042100'],
       ['G100001090444999990421009999999999921', 'G1000010042201',
        '10001042201'],
       ['G100001090444999990421009999999999922', 'G1000010042100',
        '10001042100']], dtype=object)

In [15]:

wgt_cols = ["wt_pop", "wt_fam", "wt_hh", "wt_hu"]
obs_num_vals = state_bgp1990tr2010.xwalk[wgt_cols][ix1:ix2].values
obs_num_vals

Out[15]:

array([[1.        , 1.        , 1.        , 1.        ],
       [0.99766436, 0.99716625, 0.99714829, 0.99727768],
       [0.00233564, 0.00283375, 0.00285171, 0.00272232],
       [1.        , 1.        , 1.        , 1.        ]])

In [16]:

state_bgp1990tr2010.xwalk[wgt_cols][ix1:ix2]

Out[16]:

	wt_pop	wt_fam	wt_hh	wt_hu
13	1.000000	1.000000	1.000000	1.000000
14	0.997664	0.997166	0.997148	0.997278
15	0.002336	0.002834	0.002852	0.002722
16	1.000000	1.000000	1.000000	1.000000