In [1]:

%load_ext autoreload
%autoreload 2

In [2]:

import sys
sys.path.append("..")

In [3]:

from optimus import Optimus

In [4]:

op = Optimus("vaex")

In [50]:

import polars as pl

q = (
    pl.scan_parquet("../../Data input/Base Pre2Pos para Modelo - Abr 22.parquet")    
)

df = q.collect()

In [72]:

df

Out[72]:

NO	CO_ID	ARPU	TENURE	FECHA_INST_TELEFONO	FECHA_INSTALACION	STATUS_TELEFONO	CANAL_VENTA	SUB_CANAL_VENTA	ID_VENDEDOR	ID_PLAN	PLAN_PREPAGO	ID_PLAN_ANTERIOR	FECHA_CAMBIO_PLAN	TRA_IN_LOC_Mean_6_MESES	TRA_IN_INT_Mean_6_MESES	TRA_IN_NAT_Mean_6_MESES	TRA_IN_MOBILE_Mean_6_MESES	TRA_IN_MOBILE_OTHER_Mean_6_MESES	TRA_OUT_LOC_Mean_6_MESES	TRA_OUT_INT_Mean_6_MESES	TRA_OUT_NAT_Mean_6_MESES	TRA_OUT_MOBILE_Mean_6_MESES	TRA_OUT_MOBILE_OTHER_Mean_6_MESES	SMS_OUT_AMT_Mean_6_MESES	RECHARGE_AMT_Mean_6_MESES	AMT_RENT_Mean_6_MESES	AMT_MIN_OUT_INT_Mean_6_MESES	AMT_MIN_OUT_MOBILE_OTHER_Mean_6_MESES	AMT_MIN_OUT_OFF_NET_FIX_Mean_6_MESES	AMT_MIN_OUT_MOBILE_Mean_6_MESES	AMT_MIN_OUT_MOBILE_SPE_TARIFF_Mean_6_MESES	AMT_ROAMING_SMS_Mean_6_MESES	AMT_ROAMING_DATA_Mean_6_MESES	AMT_ROAMING_VOICE_Mean_6_MESES	AMT_PAQUETICO_VOZ_Mean_6_MESES	AMT_PAQUETICO_DATA_Mean_6_MESES	...	TRA_IN_NAT_Mean_3_MESES	TRA_IN_MOBILE_Mean_3_MESES	TRA_IN_MOBILE_OTHER_Mean_3_MESES	TRA_OUT_LOC_Mean_3_MESES	TRA_OUT_INT_Mean_3_MESES	TRA_OUT_NAT_Mean_3_MESES	TRA_OUT_MOBILE_Mean_3_MESES	TRA_OUT_MOBILE_OTHER_Mean_3_MESES	SMS_OUT_AMT_Mean_3_MESES	RECHARGE_AMT_Mean_3_MESES	AMT_RENT_Mean_3_MESES	AMT_MIN_OUT_INT_Mean_3_MESES	AMT_MIN_OUT_MOBILE_OTHER_Mean_3_MESES	AMT_MIN_OUT_OFF_NET_FIX_Mean_3_MESES	AMT_MIN_OUT_MOBILE_Mean_3_MESES	AMT_MIN_OUT_MOBILE_SPE_TARIFF_Mean_3_MESES	AMT_ROAMING_SMS_Mean_3_MESES	AMT_ROAMING_DATA_Mean_3_MESES	AMT_ROAMING_VOICE_Mean_3_MESES	AMT_PAQUETICO_VOZ_Mean_3_MESES	AMT_PAQUETICO_DATA_Mean_3_MESES	AMT_MIN_OUT_ON_NET_FIX_Mean_3_MESES	MB_Mean_3_MESES	FECHA_NACIMIENTO	SEXO	SEGMENTO_BI	DESCRIPCION_PROVINCIA_CUENTA	DESCRIPCION_PROVINCIA_CVOZ	DESCRIPCION_PROVINCIA_CDATA	PROVINCIA	MONTO_RECARGA_Sum_MES_ANTERIOR	CANT_RECARGAS_MES_ANTERIOR	CABLE	TELEFONO_FIJO	INTERNET	MBB	MOVIL_POSPAGO
i64	i64	i64	i64	str	str	str	i64	str	str	str	str	str	str	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	...	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	f64	str	str	str	str	str	str	str	str	str	str	str	str	str	str
1	12881892	384	109	"2013-01-30 00:00:00"	"2013-01-30 00:00:00"	"A"	6	"4601"	"18425"	"CEPREVO005"	"S"	"$null$"	"$null$"	1.55	0.986	2.384	17.903	35.277	4.271	0.0	2.16	6.134	7.396	3.143	340.651	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	3.703	17.527	39.467	2.58	0.0	0.76	8.0	5.837	5.333	282.047	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.011	"1982-04-12 00:00:00"	"M"	"E"	"SAN PEDRO DE MACORIS"	"SAN PEDRO DE MACORIS"	"$null$"	"SAN PEDRO DE MACORIS"	"76.920"	"1"	"1"	"$null$"	"$null$"	"$null$"	"$null$"
2	136449998	410	32	"2019-07-19 00:00:00"	"2022-02-06 00:00:00"	"A"	3	"2903"	"24425"	"CEPREVO004"	"S"	"$null$"	"$null$"	3.323	0.0	0.0	126.01	5.379	0.007	0.0	0.0	18.44	1.474	4.0	393.626	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	128.703	4.597	0.017	0.0	0.0	0.753	0.1	5.333	197.433	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	2940.907	"1995-05-10 00:00:00"	"F"	"$null$"	"DUARTE"	"DUARTE"	"$null$"	"DUARTE"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"1"	"$null$"
3	15661796	654	93	"2014-06-20 00:00:00"	"2014-06-20 00:00:00"	"A"	6	"358"	"19394"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.124	0.0	0.0	110.039	87.179	0.549	0.0	0.0	9.119	5.939	16.143	515.26	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	112.61	19.15	0.0	0.0	0.0	12.01	9.71	14.333	575.98	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	23524.509	"1993-02-15 00:00:00"	"F"	"E"	"DISTRITO NACIONAL"	"SAN JUAN"	"$null$"	"SAN JUAN"	"804.880"	"8"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
4	127068927	350	37	"2019-01-29 00:00:00"	"2019-01-31 00:00:00"	"A"	3	"1919"	"23502"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.566	0.0	0.0	3.311	3.483	0.0	0.0	0.0	0.069	0.471	0.0	212.297	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	0.493	5.147	0.0	0.0	0.0	0.16	1.1	0.0	38.46	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.238	"1973-09-30 00:00:00"	"F"	"E"	"AZUA"	"SANTIAGO"	"$null$"	"SANTIAGO"	"115.380"	"2"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
5	156732494	660	9	"2021-06-14 00:00:00"	"2021-06-14 00:00:00"	"A"	6	"11361"	"E6314"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.049	0.0	0.0	1.137	10.38	0.263	0.0	0.0	1.071	4.966	2.0	348.34	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	0.653	0.433	0.613	0.0	0.0	1.12	2.133	2.667	307.68	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	11349.316	"1997-07-30 00:00:00"	"F"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
6	158282264	741	7	"2021-07-30 00:00:00"	"2021-07-30 00:00:00"	"A"	3	"5598"	"25430"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.0	0.0	0.0	0.3	3.246	0.0	0.0	0.0	1.094	5.943	1.857	63.954	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	0.0	0.713	0.0	0.0	0.0	0.0	0.0	2.333	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	"1990-12-24 00:00:00"	"M"	"E"	"BAHORUCO"	"$null$"	"$null$"	"BAHORUCO"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
7	73882646	596	59	"2017-04-20 00:00:00"	"2020-07-07 00:00:00"	"A"	3	"1938"	"S4801"	"CEPREVO004"	"S"	"$null$"	"$null$"	0.976	0.0	0.0	15.146	36.881	0.0	0.0	0.0	0.946	1.706	0.0	495.603	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	27.12	36.703	0.0	0.0	0.0	0.72	0.0	0.0	215.38	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	2453.693	"1976-09-09 00:00:00"	"M"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
8	3385917	288	181	"2007-02-19 00:00:00"	"2015-12-20 00:00:00"	"A"	6	"66"	"999"	"CEPREVO006"	"S"	"$null$"	"$null$"	0.481	0.0	0.0	1.901	3.589	0.583	1.4	0.0	0.0	1.174	0.0	159.337	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	4.437	8.373	1.36	3.267	0.0	0.0	2.74	0.0	371.787	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	900.414	"1972-04-19 00:00:00"	"F"	"E"	"DISTRITO NACIONAL"	"DISTRITO NACIONAL"	"$null$"	"DISTRITO NACIONAL"	"346.140"	"4"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
9	116622990	713	44	"2018-06-29 00:00:00"	"2018-06-29 00:00:00"	"A"	6	"10700"	"D8917"	"CEPREVO005"	"S"	"CEFLEVO006"	"2020-10-26 15:41:53"	4.917	4.233	0.094	8.399	12.047	0.637	0.003	0.243	5.5	1.93	2.143	691.541	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.22	9.877	8.323	1.267	0.007	0.0	4.613	2.01	3.0	736.663	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	20158.096	"1952-04-30 00:00:00"	"F"	"D"	"PUERTO PLATA"	"SANTIAGO"	"$null$"	"SANTIAGO"	"645.390"	"5"	"$null$"	"$null$"	"$null$"	"1"	"$null$"
10	81969494	702	57	"2017-06-20 00:00:00"	"2019-08-14 00:00:00"	"A"	6	"215"	"D3359"	"CEPREVO005"	"S"	"$null$"	"$null$"	5.589	35.849	0.0	8.55	22.851	1.337	0.0	0.0	28.023	12.806	5.714	679.137	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	3.423	8.573	0.0	0.0	0.0	11.827	2.413	6.667	320.5	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.003	"1988-11-24 00:00:00"	"F"	"E"	"MARIA TRINIDAD SANCHEZ"	"SAMANA"	"SAMANA"	"SAMANA"	"192.300"	"4"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
11	132727116	341	34	"2019-05-14 00:00:00"	"2019-05-14 00:00:00"	"A"	7	"2799"	"F3816"	"CEPREVO005"	"S"	"$null$"	"$null$"	1.84	0.0	0.0	1.764	17.446	4.134	0.0	0.0	20.21	27.907	0.0	362.646	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	2.67	14.993	0.0	0.0	0.0	31.097	22.003	0.0	325.647	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	"1956-11-04 00:00:00"	"F"	"E"	"SAN CRISTOBAL"	"SANTIAGO"	"$null$"	"SANTIAGO"	"392.300"	"10"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
12	4216752	790	171	"2007-11-27 00:00:00"	"2022-02-09 00:00:00"	"A"	6	"419"	"999"	"CEPREVO006"	"S"	"$null$"	"$null$"	1.561	1.463	0.0	14.507	9.597	2.13	0.014	0.0	7.163	4.449	3.714	834.593	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	20.37	11.4	4.97	0.033	0.0	13.26	6.3	5.333	998.697	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	35598.158	"1992-12-24 00:00:00"	"M"	"C"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"996.150"	"8"	"$null$"	"$null$"	"1"	"$null$"	"$null$"
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
163809	161877472	278	4	"2021-11-05 00:00:00"	"2021-11-09 00:00:00"	"A"	6	"855"	"E4572"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.0	0.157	0.0	5.321	6.807	0.0	0.0	0.0	3.103	2.599	8.571	274.714	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	8.863	11.23	0.0	0.0	0.0	2.127	3.423	17.333	307.68	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	19950.533	"1969-04-28 00:00:00"	"F"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"307.680"	"4"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163810	83408377	770	56	"2017-06-28 00:00:00"	"2020-06-30 00:00:00"	"A"	3	"3277"	"23494"	"CEPREVO004"	"S"	"$null$"	"$null$"	1.114	0.0	0.0	63.78	23.873	0.0	0.0	0.0	7.384	2.387	0.571	905.476	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	89.087	18.937	0.0	0.0	0.0	10.403	4.003	1.333	999.997	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	35385.762	"1991-03-10 00:00:00"	"F"	"D"	"LA ALTAGRACIA"	"LA ALTAGRACIA"	"$null$"	"LA ALTAGRACIA"	"999.990"	"10"	"$null$"	"$null$"	"$null$"	"$null$"	"1"
163811	78090567	544	57	"2017-05-27 00:00:00"	"2017-05-27 00:00:00"	"A"	7	"2739"	"F4606"	"CEPREVO005"	"S"	"$null$"	"$null$"	1.611	0.0	0.0	2.627	8.549	0.033	0.0	0.0	0.191	2.237	3.857	598.877	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	5.397	2.627	0.077	0.0	0.0	0.06	2.633	4.333	602.54	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	21934.294	"1998-10-01 00:00:00"	"F"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"653.820"	"8"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163812	14473167	675	99	"2013-11-30 00:00:00"	"2013-11-30 00:00:00"	"A"	6	"6750"	"21818"	"CEPREVO005"	"S"	"$null$"	"$null$"	1.821	0.0	0.0	7.881	15.559	0.737	0.0	0.0	4.089	4.499	0.714	681.183	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	8.49	11.037	1.22	0.0	0.0	3.153	1.597	1.0	599.72	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	28011.121	"1988-02-14 00:00:00"	"M"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"568.440"	"5"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163813	2901452	713	186	"2006-09-20 00:00:00"	"2006-09-20 00:00:00"	"A"	6	"179"	"999"	"CEPREVO006"	"S"	"$null$"	"$null$"	1.123	0.0	0.0	7.473	32.44	4.226	0.0	0.0	104.69	0.891	1.714	701.74	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	7.543	28.86	7.747	0.0	0.0	101.043	1.867	2.0	742.04	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	8808.32	"1974-07-14 00:00:00"	"F"	"C"	"DISTRITO NACIONAL"	"$null$"	"$null$"	"DISTRITO NACIONAL"	"644.600"	"6"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163814	16347965	398	89	"2014-10-16 00:00:00"	"2019-05-03 00:00:00"	"A"	3	"6079"	"20892"	"CEPREVO005"	"S"	"$null$"	"$null$"	4.714	7.034	0.0	5.173	9.966	3.277	0.211	0.0	10.224	5.794	4.143	456.626	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	3.243	7.473	0.187	0.247	0.0	5.337	3.687	3.667	271.787	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	2815.963	"1997-06-16 00:00:00"	"F"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"230.760"	"4"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163815	128849253	750	36	"2019-03-05 00:00:00"	"2019-03-05 00:00:00"	"A"	21	"10707"	"D3829"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.486	0.0	0.0	6.431	18.706	0.241	0.0	0.0	17.741	10.827	0.0	760.447	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	6.607	18.233	0.543	0.0	0.0	20.797	7.797	0.0	800.01	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	26178.42	"2000-09-06 00:00:00"	"M"	"C"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"784.630"	"5"	"$null$"	"$null$"	"$null$"	"$null$"	"2"
163816	152479773	506	12	"2021-03-03 00:00:00"	"2021-03-03 00:00:00"	"A"	6	"11129"	"D3507"	"CEPREVO004"	"S"	"$null$"	"$null$"	0.666	23.066	0.0	58.133	4.089	1.123	0.029	0.0	19.007	3.486	5.143	488.601	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	71.823	7.293	0.0	0.0	0.0	25.117	5.613	8.667	348.717	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	7550.089	"1976-12-25 00:00:00"	"M"	"E"	"PEDERNALES"	"$null$"	"$null$"	"PEDERNALES"	"430.770"	"5"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163817	2581628	732	190	"2006-05-08 00:00:00"	"2006-05-08 00:00:00"	"A"	6	"33019"	"999"	"CEPREVO006"	"S"	"$null$"	"$null$"	13.349	0.029	1.149	21.274	22.909	6.024	0.0	0.0	14.344	8.821	2.857	705.501	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	1.713	25.02	15.087	5.863	0.0	0.0	12.623	5.117	2.667	564.117	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	9895.634	"1986-05-11 00:00:00"	"M"	"C"	"DISTRITO NACIONAL"	"$null$"	"$null$"	"DISTRITO NACIONAL"	"461.550"	"3"	"1"	"1"	"$null$"	"$null$"	"$null$"
163818	96685857	641	53	"2017-10-20 00:00:00"	"2017-10-20 00:00:00"	"A"	6	"871"	"F7110"	"CEPREVO005"	"S"	"CEFLEVO006"	"2020-12-02 17:04:53"	0.0	0.0	0.0	3.44	0.933	1.554	0.0	0.0	1.587	0.423	5.286	725.27	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	1.587	1.177	2.133	0.0	0.0	0.737	0.773	7.0	846.157	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	18884.18	"1998-05-05 00:00:00"	"M"	"E"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"692.310"	"6"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163819	157842933	624	8	"2021-07-17 00:00:00"	"2021-07-17 00:00:00"	"A"	23	"11548"	"C0050"	"CEPREVO005"	"S"	"$null$"	"$null$"	0.0	0.0	0.0	1.656	1.896	1.343	0.477	0.0	1.731	0.596	1.429	571.406	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	2.677	1.577	0.067	1.113	0.0	3.62	0.31	0.667	410.24	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	17264.106	"1932-04-08 00:00:00"	"M"	"E"	"DISTRITO NACIONAL"	"SAN CRISTOBAL"	"$null$"	"SAN CRISTOBAL"	"153.840"	"2"	"$null$"	"$null$"	"$null$"	"$null$"	"$null$"
163820	17487527	623	82	"2015-04-30 00:00:00"	"2015-04-30 00:00:00"	"A"	6	"435"	"20218"	"CEPREVO005"	"S"	"$null$"	"$null$"	9.781	1.329	0.891	307.201	33.189	5.796	0.0	0.249	109.38	2.609	2.0	651.641	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	...	0.0	244.99	23.027	0.31	0.0	0.0	83.587	3.967	2.667	656.403	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	3191.814	"1975-12-27 00:00:00"	"M"	"D"	"DISTRITO NACIONAL"	"SANTO DOMINGO"	"$null$"	"SANTO DOMINGO"	"538.450"	"5"	"$null$"	"$null$"	"$null$"	"$null$"	"1"

In [71]:

%%time
(df["NO"].cast(str).str.contains(r"^\d+$").sum(), df["CO_ID"].cast(str).str.contains(r"^\d+$").sum())

Wall time: 19.2 ms

Out[71]:

(163820, 163820)

In [5]:

%%time
# df =op.load.file("data/crime.csv")
df =op.load.file("../../Data input/Base Pre2Pos para Modelo - Abr 22.parquet")

Wall time: 2.22 s

In [28]:

df["NO"].mask.int()

Out[28]:

Viewing 10 of 163.8 thousand rows / 1 columns

None partition(s)

NO 1 (bool)
True
True
True
True
True
True
True
True
True
True
True

Viewing 10 of 163.8 thousand rows / 1 columns

None partition(s) <class 'optimus.engines.vaex.dataframe.VaexDataFrame'>

In [36]:

for col_name in df.cols.names():    
    print(col_name)
    %prun df.cols.profile(col_name)

NO
Wall time: 1.43 s
CO_ID
Wall time: 1.63 s
ARPU
Wall time: 1.51 s
TENURE
Wall time: 1.61 s
FECHA_INST_TELEFONO
Wall time: 1.68 s
FECHA_INSTALACION
Wall time: 1.58 s
STATUS_TELEFONO
Wall time: 1.66 s
CANAL_VENTA
Wall time: 1.71 s
SUB_CANAL_VENTA
Wall time: 2.37 s
ID_VENDEDOR
Wall time: 1.87 s
ID_PLAN
Wall time: 2.15 s
PLAN_PREPAGO
Wall time: 1.84 s
ID_PLAN_ANTERIOR
Wall time: 1.67 s
FECHA_CAMBIO_PLAN
Wall time: 1.71 s
TRA_IN_LOC_Mean_6_MESES
Wall time: 1.74 s
TRA_IN_INT_Mean_6_MESES
Wall time: 1.67 s
TRA_IN_NAT_Mean_6_MESES
Wall time: 1.69 s
TRA_IN_MOBILE_Mean_6_MESES
Wall time: 2.19 s
TRA_IN_MOBILE_OTHER_Mean_6_MESES
Wall time: 1.66 s
TRA_OUT_LOC_Mean_6_MESES
Wall time: 1.56 s
TRA_OUT_INT_Mean_6_MESES
Wall time: 1.54 s
TRA_OUT_NAT_Mean_6_MESES
Wall time: 1.56 s
TRA_OUT_MOBILE_Mean_6_MESES
Wall time: 1.51 s
TRA_OUT_MOBILE_OTHER_Mean_6_MESES
Wall time: 1.57 s
SMS_OUT_AMT_Mean_6_MESES
Wall time: 1.74 s
RECHARGE_AMT_Mean_6_MESES
Wall time: 1.98 s
AMT_RENT_Mean_6_MESES

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<timed eval> in <module>

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in profile(self, cols, bins, flush)
    827         """
    828         # Uses profile on self instead of calculate_profile to get the data only when it's necessary
--> 829         self.root.profile(cols=cols, bins=bins, flush=flush)
    830         df = self.root
    831 

~\Documents\Optimus\examples\..\optimus\engines\base\profile.py in __call__(self, cols, bins, force_hist, output, flush, size)
     84 
     85             if calculate:
---> 86                 df = df.profile._calculate(cols, bins, force_hist, flush, size)
     87                 profile = Meta.get(df.meta, "profile")
     88                 self.root.meta = df.meta

~\Documents\Optimus\examples\..\optimus\engines\base\profile.py in _calculate(self, cols, bins, force_hist, flush, size)
    146 
    147             if cols_to_infer:
--> 148                 cols_data_types = {**cols_data_types, **df.cols.infer_type(cols_to_infer, tidy=False)["infer_type"]}
    149                 cols_data_types = {col: cols_data_types[col] for col in cols_to_profile if col in cols_data_types}
    150 

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in infer_type(self, cols, sample_count, tidy)
   3657 
   3658         # Infer the data type from every element in a Series.
-> 3659         sample_df = df.cols.select(cols).sample(sample_count).to_optimus_pandas()
   3660         rows_count = sample_count
   3661         sample_dtypes = sample_df.cols.infer_data_types().cols.frequency()

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in select(self, cols, regex, data_type, invert, accepts_missing_cols)
    208                              accepts_missing_cols=accepts_missing_cols)
    209         # meta = df.meta
--> 210         meta = Meta.select_columns(meta, cols)
    211         dfd = df.data
    212         if cols is not None:

~\Documents\Optimus\examples\..\optimus\engines\base\meta.py in select_columns(meta, cols)
    168 
    169         to_drop = list(set(all_cols) - set(cols))
--> 170         return Meta.drop_columns(meta, to_drop)

~\Documents\Optimus\examples\..\optimus\engines\base\meta.py in drop_columns(meta, cols)
    155             meta = Meta.reset(meta, f'columns_data_types.{col}')
    156             meta = Meta.reset(meta, f'max_cell_length.{col}')
--> 157             meta = Meta.reset(meta, f'profile.columns.{col}')
    158         return meta
    159 

~\Documents\Optimus\examples\..\optimus\engines\base\meta.py in reset(meta, spec)
     53         if spec is not None:
     54             data = deepcopy(meta)
---> 55             delete(data, spec, ignore_missing=True)
     56         else:
     57             data = meta

~\Anaconda3\envs\python38\lib\site-packages\glom\mutation.py in delete(obj, path, ignore_missing)
    341     .. versionadded:: 20.5.0
    342     """
--> 343     return glom(obj, Delete(path, ignore_missing=ignore_missing))
    344 
    345 

~\Anaconda3\envs\python38\lib\site-packages\glom\mutation.py in __init__(self, path, ignore_missing)
    261     def __init__(self, path, ignore_missing=False):
    262         if isinstance(path, basestring):
--> 263             path = Path.from_text(path)
    264         elif type(path) is TType:
    265             path = Path(path)

~\Anaconda3\envs\python38\lib\site-packages\glom\core.py in from_text(cls, text)
    637 
    638         """
--> 639         if text not in cls._CACHE:
    640             if len(cls._CACHE) > cls._MAX_CACHE:
    641                 return cls(*text.split('.'))

KeyboardInterrupt:

AMT_MIN_OUT_INT_Mean_6_MESES
Wall time: 1.88 s
AMT_MIN_OUT_MOBILE_OTHER_Mean_6_MESES
Wall time: 1.82 s
AMT_MIN_OUT_OFF_NET_FIX_Mean_6_MESES
Wall time: 1.7 s
AMT_MIN_OUT_MOBILE_Mean_6_MESES
Wall time: 1.63 s
AMT_MIN_OUT_MOBILE_SPE_TARIFF_Mean_6_MESES
Wall time: 1.68 s
AMT_ROAMING_SMS_Mean_6_MESES
Wall time: 1.65 s
AMT_ROAMING_DATA_Mean_6_MESES
Wall time: 1.79 s
AMT_ROAMING_VOICE_Mean_6_MESES
Wall time: 1.63 s
AMT_PAQUETICO_VOZ_Mean_6_MESES
Wall time: 1.63 s
AMT_PAQUETICO_DATA_Mean_6_MESES
Wall time: 1.65 s
AMT_MIN_OUT_ON_NET_FIX_Mean_6_MESES
Wall time: 1.8 s
MB_Mean_6_MESES
Wall time: 1.85 s
TRA_IN_LOC_Mean_3_MESES
Wall time: 1.71 s
TRA_IN_INT_Mean_3_MESES
Wall time: 1.65 s
TRA_IN_NAT_Mean_3_MESES
Wall time: 1.54 s
TRA_IN_MOBILE_Mean_3_MESES
Wall time: 1.54 s
TRA_IN_MOBILE_OTHER_Mean_3_MESES
Wall time: 1.64 s
TRA_OUT_LOC_Mean_3_MESES
Wall time: 1.66 s
TRA_OUT_INT_Mean_3_MESES
Wall time: 1.57 s
TRA_OUT_NAT_Mean_3_MESES
Wall time: 1.71 s
TRA_OUT_MOBILE_Mean_3_MESES
Wall time: 2.42 s
TRA_OUT_MOBILE_OTHER_Mean_3_MESES
Wall time: 2.33 s
SMS_OUT_AMT_Mean_3_MESES
Wall time: 1.93 s
RECHARGE_AMT_Mean_3_MESES
Wall time: 1.6 s
AMT_RENT_Mean_3_MESES
Wall time: 1.59 s
AMT_MIN_OUT_INT_Mean_3_MESES
Wall time: 1.56 s
AMT_MIN_OUT_MOBILE_OTHER_Mean_3_MESES
Wall time: 1.63 s
AMT_MIN_OUT_OFF_NET_FIX_Mean_3_MESES
Wall time: 1.65 s
AMT_MIN_OUT_MOBILE_Mean_3_MESES
Wall time: 1.82 s
AMT_MIN_OUT_MOBILE_SPE_TARIFF_Mean_3_MESES
Wall time: 1.68 s
AMT_ROAMING_SMS_Mean_3_MESES
Wall time: 1.67 s
AMT_ROAMING_DATA_Mean_3_MESES
Wall time: 1.56 s
AMT_ROAMING_VOICE_Mean_3_MESES
Wall time: 1.55 s
AMT_PAQUETICO_VOZ_Mean_3_MESES
Wall time: 1.55 s
AMT_PAQUETICO_DATA_Mean_3_MESES
Wall time: 1.69 s
AMT_MIN_OUT_ON_NET_FIX_Mean_3_MESES
Wall time: 1.73 s
MB_Mean_3_MESES
Wall time: 1.56 s
FECHA_NACIMIENTO
Wall time: 1.69 s
SEXO
Wall time: 1.93 s
SEGMENTO_BI
Wall time: 1.84 s
DESCRIPCION_PROVINCIA_CUENTA
Wall time: 1.67 s
DESCRIPCION_PROVINCIA_CVOZ
Wall time: 2.02 s
DESCRIPCION_PROVINCIA_CDATA
Wall time: 1.61 s
PROVINCIA
Wall time: 1.58 s
MONTO_RECARGA_Sum_MES_ANTERIOR
Wall time: 1.65 s
CANT_RECARGAS_MES_ANTERIOR
Wall time: 1.6 s
CABLE
Wall time: 1.6 s
TELEFONO_FIJO
Wall time: 1.77 s
INTERNET
Wall time: 1.75 s
MBB
Wall time: 1.69 s
MOVIL_POSPAGO
Wall time: 1.71 s

In [49]:

col_name = "NO"
%prun df[col_name].cols.profile(col_name)

In [48]:

%prun df.data["NO"].astype(str).str.match(r"^\d+$").sum()

In [14]:

for col_name in df.cols.names():
    print(col_name)
    %prun df[col_name].cols.quality(col_name)

NO
 CO_ID
 ARPU
 TENURE
 FECHA_INST_TELEFONO
 FECHA_INSTALACION
 STATUS_TELEFONO
 CANAL_VENTA
 SUB_CANAL_VENTA
 ID_VENDEDOR
 ID_PLAN
 PLAN_PREPAGO
 ID_PLAN_ANTERIOR
 FECHA_CAMBIO_PLAN
 TRA_IN_LOC_Mean_6_MESES
 TRA_IN_INT_Mean_6_MESES
 TRA_IN_NAT_Mean_6_MESES
 TRA_IN_MOBILE_Mean_6_MESES
 TRA_IN_MOBILE_OTHER_Mean_6_MESES
 TRA_OUT_LOC_Mean_6_MESES
 TRA_OUT_INT_Mean_6_MESES

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
C:\Users\ARGENI~1\AppData\Local\Temp/ipykernel_19916/1976845657.py in <module>
      1 for col_name in df.cols.names():
      2     print(col_name)
----> 3     get_ipython().run_line_magic('prun', 'df[col_name].cols.quality(col_name)')

~\Anaconda3\envs\python38\lib\site-packages\IPython\core\interactiveshell.py in run_line_magic(self, magic_name, line, _stack_depth)
   2349                 kwargs['local_ns'] = self.get_local_scope(stack_depth)
   2350             with self.builtin_trap:
-> 2351                 result = fn(*args, **kwargs)
   2352             return result
   2353 

~\Anaconda3\envs\python38\lib\site-packages\decorator.py in fun(*args, **kw)
    230             if not kwsyntax:
    231                 args, kw = fix(args, kw, sig)
--> 232             return caller(func, *(extras + args), **kw)
    233     fun.__name__ = func.__name__
    234     fun.__doc__ = func.__doc__

~\Anaconda3\envs\python38\lib\site-packages\IPython\core\magic.py in <lambda>(f, *a, **k)
    185     # but it's overkill for just that one bit of state.
    186     def magic_deco(arg):
--> 187         call = lambda f, *a, **k: f(*a, **k)
    188 
    189         if callable(arg):

~\Anaconda3\envs\python38\lib\site-packages\IPython\core\magics\execution.py in prun(self, parameter_s, cell)
    315             arg_str += '\n' + cell
    316         arg_str = self.shell.transform_cell(arg_str)
--> 317         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
    318 
    319     def _run_with_profiler(self, code, opts, namespace):

~\Anaconda3\envs\python38\lib\site-packages\IPython\core\magics\execution.py in _run_with_profiler(self, code, opts, namespace)
    337         prof = profile.Profile()
    338         try:
--> 339             prof = prof.runctx(code, namespace, namespace)
    340             sys_exit = ''
    341         except SystemExit:

~\Anaconda3\envs\python38\lib\cProfile.py in runctx(self, cmd, globals, locals)
     98         self.enable()
     99         try:
--> 100             exec(cmd, globals, locals)
    101         finally:
    102             self.disable()

<string> in <module>

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in quality(self, cols, flush, compute)
   3611 
   3612             dtype = df.constants.INTERNAL_TO_OPTIMUS.get(dtype, dtype)
-> 3613             matches_mismatches = getattr(df[col_name].mask, dtype)(col_name).cols.frequency()
   3614 
   3615             missing = df.mask.null(col_name).cols.sum()

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in frequency(self, cols, n, percentage, total_rows, count_uniques, compute, tidy)
   3835             return _value_counts
   3836 
-> 3837         n_largest = [calculate_n_largest(df.data[col], count_uniques) for col in cols]
   3838 
   3839         b = [series_to_dict(_n_largest, _cols) for _n_largest, _cols in zip(n_largest, cols)]

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in <listcomp>(.0)
   3835             return _value_counts
   3836 
-> 3837         n_largest = [calculate_n_largest(df.data[col], count_uniques) for col in cols]
   3838 
   3839         b = [series_to_dict(_n_largest, _cols) for _n_largest, _cols in zip(n_largest, cols)]

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in calculate_n_largest(_series, include_uniques)
   3788         @self.F.delayed
   3789         def calculate_n_largest(_series, include_uniques):
-> 3790             _value_counts = _series.value_counts()
   3791             if n is not None:
   3792                 _n_largest = _value_counts.nlargest(n)

~\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py in value_counts(self, dropna, dropnan, dropmissing, ascending, progress, axis)
   1041             return a+b
   1042         progressbar = vaex.utils.progressbars(progress, title="value counts")
-> 1043         self.ds.map_reduce(map, reduce, [self.expression], delay=False, progress=progressbar, name='value_counts', info=True, to_numpy=False)
   1044         counters = [k for k in counters if k is not None]
   1045         counter = counters[0]

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in map_reduce(self, map, reduce, arguments, progress, delay, info, to_numpy, ignore_filter, pre_filter, name, selection)
    432         progressbar.add_task(task, f'map reduce: {name}')
    433         task = self.executor.schedule(task)
--> 434         return self._delay(delay, task)
    435 
    436     def apply(self, f, arguments=None, vectorize=False, multiprocessing=True):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _delay(self, delay, task, progressbar)
   1773             return task
   1774         else:
-> 1775             self.execute()
   1776             return task.get()
   1777 

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in execute(self)
    415                 print(repr(task))
    416         if self.executor.tasks:
--> 417             self.executor.execute()
    418 
    419     async def execute_async(self):

~\Anaconda3\envs\python38\lib\site-packages\vaex\execution.py in execute(self)
    306 
    307     def execute(self):
--> 308         for _ in self.execute_generator():
    309             pass  # just eat all elements
    310 

~\Anaconda3\envs\python38\lib\site-packages\vaex\execution.py in execute_generator(self, use_async)
    430                         logger.debug("Pass cancelled because of the global progress event: %r", self.signal_progress.callbacks)
    431                     return ok_tasks and ok_executor and not all_stopped
--> 432                 yield from self.thread_pool.map(self.process_part, dataset.chunk_iterator(run.dataset_deps, chunk_size),
    433                                                     dataset.row_count,
    434                                                     progress=progress,

~\Anaconda3\envs\python38\lib\site-packages\vaex\multithreading.py in map(self, callable, iterator, count, on_error, progress, cancel, unpack, use_async, **kwargs_extra)
    108         iterator = iter(buffer(iterator, self._max_workers + 3))
    109         try:
--> 110             for value in iterator:
    111                 if use_async:
    112                     value = yield value

~\Anaconda3\envs\python38\lib\site-packages\vaex\itertools.py in buffer(i, n)
      3     try:
      4         for _ in range(n-1):
----> 5             values.append(next(i))
      6         while True:
      7             values.append(next(i))

~\Anaconda3\envs\python38\lib\concurrent\futures\_base.py in result_iterator()
    617                     # Careful not to keep a reference to the popped future
    618                     if timeout is None:
--> 619                         yield fs.pop().result()
    620                     else:
    621                         yield fs.pop().result(end_time - time.monotonic())

~\Anaconda3\envs\python38\lib\concurrent\futures\_base.py in result(self, timeout)
    437                     return self.__get_result()
    438 
--> 439                 self._condition.wait(timeout)
    440 
    441                 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

~\Anaconda3\envs\python38\lib\threading.py in wait(self, timeout)
    300         try:    # restore state no matter what (e.g., KeyboardInterrupt)
    301             if timeout is None:
--> 302                 waiter.acquire()
    303                 gotit = True
    304             else:

KeyboardInterrupt:

In [18]:

df.cols.count_uniques()

---------------------------------------------------------------------------
RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
"""
Traceback (most recent call last):
  File "C:\Users\argenisleon\Anaconda3\envs\python38\lib\multiprocessing\pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
  File "C:\Users\argenisleon\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py", line 1696, in _apply
    scalar_result = self.f(*[fix_type(k[i]) for k in args], **{key: value[i] for key, value in kwargs.items()})
  File "C:\Users\argenisleon\Documents\Optimus\examples\..\optimus\engines\vaex\functions.py", line 94, in to_string_vaex
    return series.astype(str)
AttributeError: 'int' object has no attribute 'astype'
"""

The above exception was the direct cause of the following exception:

AttributeError                            Traceback (most recent call last)
C:\Users\ARGENI~1\AppData\Local\Temp/ipykernel_19528/3714096341.py in <module>
----> 1 df.cols.count_uniques()

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in count_uniques(self, cols, estimate, compute, tidy)
   3170         if df.op.engine != Engine.SPARK.value and estimate is not False:
   3171             logger.warn(f"'estimate' argument is only supported on {EnginePretty.SPARK.value}")
-> 3172         return df.cols.agg_exprs(cols, self.F.count_uniques, estimate, tidy=tidy, compute=compute)
   3173 
   3174     def _math(self, cols="*", value=None, operator=None, output_cols=None, output_col=None, name="",

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in agg_exprs(self, cols, funcs, compute, tidy, parallel, *args)
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}
   1363         else:
-> 1364             agg_result = {func.__name__: {col_name: self.exec_agg(func(df.data[col_name], *args), compute=False) for
   1365                                           col_name in cols} for func in funcs}
   1366 

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in <dictcomp>(.0)
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}
   1363         else:
-> 1364             agg_result = {func.__name__: {col_name: self.exec_agg(func(df.data[col_name], *args), compute=False) for
   1365                                           col_name in cols} for func in funcs}
   1366 

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in <dictcomp>(.0)
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}
   1363         else:
-> 1364             agg_result = {func.__name__: {col_name: self.exec_agg(func(df.data[col_name], *args), compute=False) for
   1365                                           col_name in cols} for func in funcs}
   1366 

~\Documents\Optimus\examples\..\optimus\engines\base\functions.py in count_uniques(self, series, estimate)
    450         Get the count of unique values in a series
    451         """
--> 452         return self.to_string(series).nunique()
    453 
    454     def unique_values(self, series, estimate=False):

~\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py in nunique(self, dropna, dropnan, dropmissing, selection, axis, limit, limit_raise, progress, delay)
   1149             else:
   1150                 return len(value)
-> 1151         return f()
   1152 
   1153     def countna(self):

~\Anaconda3\envs\python38\lib\site-packages\vaex\cache.py in wrapper(*args, **kwargs)
    485                         return value
    486             else:
--> 487                 return f(*args, **kwargs)
    488         return wrapper
    489     if f is None:

~\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py in f()
   1144         @vaex.cache._memoize(key_function=key_function, delay=delay)
   1145         def f():
-> 1146             value = self.unique(dropna=dropna, dropnan=dropnan, dropmissing=dropmissing, selection=selection, axis=axis, limit=limit, limit_raise=limit_raise, array_type=None, progress=progress, delay=delay)
   1147             if delay:
   1148                 return value.then(len)

~\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py in unique(self, dropna, dropnan, dropmissing, selection, axis, limit, limit_raise, array_type, progress, delay)
   1123         :param bool delay: {delay}
   1124         """
-> 1125         return self.ds.unique(self, dropna=dropna, dropnan=dropnan, dropmissing=dropmissing, selection=selection, array_type=array_type, axis=axis, limit=limit, limit_raise=limit_raise, progress=progress, delay=delay)
   1126 
   1127     @docsubst

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in unique(self, expression, return_inverse, dropna, dropnan, dropmissing, progress, selection, axis, delay, limit, limit_raise, array_type)
    671                     return keys
    672             progressbar = vaex.utils.progressbars(progress, title="unique")
--> 673             hash_map_result = self._hash_map_unique(expression, progress=progressbar, selection=selection, flatten=axis is None, delay=True, limit=limit, limit_raise=limit_raise)
    674             if return_inverse:
    675                 progress_inverse = progressbar.add("find inverse")

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _hash_map_unique(self, expression, progress, selection, flatten, delay, limit, limit_raise, return_inverse)
    498             selection = str(selection)
    499         expression = _ensure_string_from_expression(expression)
--> 500         task = vaex.tasks.TaskHashmapUniqueCreate(self, expression, flatten, limit=limit, selection=selection, return_inverse=return_inverse, limit_raise=limit_raise)
    501         task = self.executor.schedule(task)
    502         progressbar = vaex.utils.progressbars(progress)

~\Anaconda3\envs\python38\lib\site-packages\vaex\tasks.py in __init__(self, df, expression, flatten, limit, limit_raise, selection, return_inverse)
    181         super().__init__(df=df, expressions=[expression], pre_filter=df.filtered, name=self.snake_name)
    182         self.flatten = flatten
--> 183         self.dtype = self.df.data_type(expression)
    184         self.dtype_item = self.df.data_type(expression, axis=-1 if flatten else 0)
    185         self.limit = limit

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in data_type(self, expression, array_type, internal, axis)
   2268                 data = self.evaluate(expression, 0, 1, filtered=False, array_type=array_type, parallel=False)
   2269             except:
-> 2270                 data = self.evaluate(expression, 0, 1, filtered=True, array_type=array_type, parallel=False)
   2271         if data_type is None:
   2272             # means we have to determine it from the data

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in evaluate(self, expression, i1, i2, out, selection, filtered, array_type, parallel, chunk_size, progress)
   3088             return self.evaluate_iterator(expression, s1=i1, s2=i2, out=out, selection=selection, filtered=filtered, array_type=array_type, parallel=parallel, chunk_size=chunk_size, progress=progress)
   3089         else:
-> 3090             return self._evaluate_implementation(expression, i1=i1, i2=i2, out=out, selection=selection, filtered=filtered, array_type=array_type, parallel=parallel, chunk_size=chunk_size, progress=progress)
   3091 
   3092     @docsubst

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _evaluate_implementation(self, expression, i1, i2, out, selection, filtered, array_type, parallel, chunk_size, raw, progress)
   6520 
   6521             for expression in expressions:
-> 6522                 value = block_scope.evaluate(expression)
   6523                 value = array_types.convert(value, array_type)
   6524                 values.append(value)

~\Anaconda3\envs\python38\lib\site-packages\vaex\scopes.py in evaluate(self, expression, out)
    117             # logger.debug("in eval")
    118             # eval("def f(")
--> 119             result = eval(expression, expression_namespace, self)
    120             result = auto_encode(self.df, expression, result)
    121             self.values[expression] = wrap(result)

<string> in <module>

~\Anaconda3\envs\python38\lib\site-packages\vaex\arrow\numpy_dispatch.py in wrapper(*args, **kwargs)
    134         args = list(map(unwrap, args))
    135         kwargs = {k: unwrap(v) for k, v, in kwargs.items()}
--> 136         result = f(*args, **kwargs)
    137         return wrap(result)
    138     return wrapper

~\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py in __call__(self, *args, **kwargs)
   1679     def __call__(self, *args, **kwargs):
   1680         import vaex.multiprocessing
-> 1681         return vaex.multiprocessing.apply(self._apply, args, kwargs, self.multiprocessing)
   1682 
   1683     def _apply(self, *args, **kwargs):

~\Anaconda3\envs\python38\lib\site-packages\vaex\multiprocessing.py in apply(f, args, kwargs, multiprocessing)
     35         args = [_trim(k) for k in args]
     36         kwargs = {k:_trim(v) for k, v in kwargs.items()}
---> 37         result = _get_pool().apply(f, args, kwargs)
     38         return result
     39     else:

~\Anaconda3\envs\python38\lib\multiprocessing\pool.py in apply(self, func, args, kwds)
    355         Pool must be running.
    356         '''
--> 357         return self.apply_async(func, args, kwds).get()
    358 
    359     def map(self, func, iterable, chunksize=None):

~\Anaconda3\envs\python38\lib\multiprocessing\pool.py in get(self, timeout)
    769             return self._value
    770         else:
--> 771             raise self._value
    772 
    773     def _set(self, i, obj):

~\Anaconda3\envs\python38\lib\multiprocessing\pool.py in worker()
    123         job, i, func, args, kwds = task
    124         try:
--> 125             result = (True, func(*args, **kwds))
    126         except Exception as e:
    127             if wrap_exception and func is not _helper_reraises_exception:

~\Anaconda3\envs\python38\lib\site-packages\vaex\expression.py in _apply()
   1694         args = [vaex.array_types.tolist(k) for k in args]
   1695         for i in range(length):
-> 1696             scalar_result = self.f(*[fix_type(k[i]) for k in args], **{key: value[i] for key, value in kwargs.items()})
   1697             result.append(scalar_result)
   1698         result = np.array(result)

~\Documents\Optimus\examples\..\optimus\engines\vaex\functions.py in to_string_vaex()
     92     def to_string(self, series):
     93         def to_string_vaex(series):
---> 94             return series.astype(str)
     95 
     96         return series.apply(to_string_vaex)

AttributeError: 'int' object has no attribute 'astype'

In [ ]:

%prun df.cols.profile()

In [78]:

df.save.parquet("../../Data input/Base Pre2Pos para Modelo - Abr 22.parquet")

In [49]:

dfv = df.data

In [52]:

df.mask.null(col_name)

Out[52]:

Viewing 10 of 163.8 thousand rows / 1 columns

1 partition(s)

MOVIL_POSPAGO 1 (bool)
False
False
False
False
False
False
False
False
False
False
False

Viewing 10 of 163.8 thousand rows / 1 columns

1 partition(s) <class 'optimus.engines.pandas.dataframe.PandasDataFrame'>

In [10]:

df.cols.sum()

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
C:\Users\ARGENI~1\AppData\Local\Temp/ipykernel_35616/3714096341.py in <module>
----> 1 df.cols.count_uniques()

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in count_uniques(self, cols, estimate, compute, tidy)
   3170         if df.op.engine != Engine.SPARK.value and estimate is not False:
   3171             logger.warn(f"'estimate' argument is only supported on {EnginePretty.SPARK.value}")
-> 3172         return df.cols.agg_exprs(cols, self.F.count_uniques, estimate, tidy=tidy, compute=compute)
   3173 
   3174     def _math(self, cols="*", value=None, operator=None, output_cols=None, output_col=None, name="",

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in agg_exprs(self, cols, funcs, compute, tidy, parallel, *args)
   1358 
   1359         if parallel:
-> 1360             all_funcs = [getattr(df[cols].data, func.__name__)()
   1361                          for func in funcs]
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in <listcomp>(.0)
   1358 
   1359         if parallel:
-> 1360             all_funcs = [getattr(df[cols].data, func.__name__)()
   1361                          for func in funcs]
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in __getattr__(self, name)
    282             return self.__hidden__[name].__get__(self)
    283         else:
--> 284             return object.__getattribute__(self, name)
    285 
    286     def _ipython_key_completions_(self):

AttributeError: 'DataFrameLocal' object has no attribute 'count_uniques'

In [9]:

%%time
df.cols.profile(flush=True)

<class 'optimus.engines.pandas.dataframe.PandasDataFrame'>

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<timed eval> in <module>

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in profile(self, cols, bins, flush)
    827         """
    828         # Uses profile on self instead of calculate_profile to get the data only when it's necessary
--> 829         self.root.profile(cols=cols, bins=bins, flush=flush)
    830         df = self.root
    831 

~\Documents\Optimus\examples\..\optimus\engines\base\profile.py in __call__(self, cols, bins, force_hist, output, flush, size)
     84 
     85             if calculate:
---> 86                 df = df.profile._calculate(cols, bins, force_hist, flush, size)
     87                 profile = Meta.get(df.meta, "profile")
     88                 self.root.meta = df.meta

~\Documents\Optimus\examples\..\optimus\engines\base\profile.py in _calculate(self, cols, bins, force_hist, flush, size)
    146 
    147             if cols_to_infer:
--> 148                 cols_data_types = {**cols_data_types, **df.cols.infer_type(cols_to_infer, tidy=False)["infer_type"]}
    149                 cols_data_types = {col: cols_data_types[col] for col in cols_to_profile if col in cols_data_types}
    150 

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in infer_type(self, cols, sample_count, tidy)
   3662         sample_dtypes = sample_df.cols.infer_data_types().cols.frequency()
   3663         print(type(sample_df))
-> 3664         unique_counts = sample_df.cols.count_uniques(tidy=False)['count_uniques']
   3665 
   3666         cols_and_inferred_dtype = {}

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in count_uniques(self, cols, estimate, compute, tidy)
   3170         if df.op.engine != Engine.SPARK.value and estimate is not False:
   3171             logger.warn(f"'estimate' argument is only supported on {EnginePretty.SPARK.value}")
-> 3172         return df.cols.agg_exprs(cols, self.F.count_uniques, estimate, tidy=tidy, compute=compute)
   3173 
   3174     def _math(self, cols="*", value=None, operator=None, output_cols=None, output_col=None, name="",

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in agg_exprs(self, cols, funcs, compute, tidy, parallel, *args)
   1358 
   1359         if parallel:
-> 1360             all_funcs = [getattr(df[cols].data, func.__name__)()
   1361                          for func in funcs]
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}

~\Documents\Optimus\examples\..\optimus\engines\base\columns.py in <listcomp>(.0)
   1358 
   1359         if parallel:
-> 1360             all_funcs = [getattr(df[cols].data, func.__name__)()
   1361                          for func in funcs]
   1362             agg_result = {func.__name__: self.exec_agg(all_funcs, compute=False) for func in funcs}

~\Anaconda3\envs\python38\lib\site-packages\pandas\core\generic.py in __getattr__(self, name)
   5485         ):
   5486             return self[name]
-> 5487         return object.__getattribute__(self, name)
   5488 
   5489     def __setattr__(self, name: str, value) -> None:

AttributeError: 'DataFrame' object has no attribute 'count_uniques'

In [23]:

for col_name in df.cols.names():
    print(col_name)
    %time df[col_name].cols.quality(col_name)

NO
missing 0.0
Wall time: 86.9 ms
CO_ID
missing 0.0
Wall time: 104 ms
ARPU
missing 0.0
Wall time: 94.1 ms
TENURE
missing 0.0
Wall time: 91.6 ms
FECHA_INST_TELEFONO
missing 0.0
Wall time: 276 ms
FECHA_INSTALACION
missing 0.0
Wall time: 262 ms
STATUS_TELEFONO
missing 0.0
Wall time: 251 ms
CANAL_VENTA
missing 0.0
Wall time: 113 ms
SUB_CANAL_VENTA
missing 0.0
Wall time: 298 ms
ID_VENDEDOR
missing 0.0
Wall time: 317 ms
ID_PLAN
missing 0.0
Wall time: 265 ms
PLAN_PREPAGO
missing 0.0
Wall time: 234 ms
ID_PLAN_ANTERIOR
missing 0.0
Wall time: 260 ms
FECHA_CAMBIO_PLAN
missing 0.0
Wall time: 251 ms
TRA_IN_LOC_Mean_6_MESES
missing 0.0
Wall time: 99.8 ms
TRA_IN_INT_Mean_6_MESES
missing 0.0
Wall time: 95.6 ms
TRA_IN_NAT_Mean_6_MESES
missing 0.0
Wall time: 111 ms
TRA_IN_MOBILE_Mean_6_MESES
missing 0.0
Wall time: 112 ms
TRA_IN_MOBILE_OTHER_Mean_6_MESES
missing 0.0
Wall time: 104 ms
TRA_OUT_LOC_Mean_6_MESES
missing 0.0
Wall time: 107 ms
TRA_OUT_INT_Mean_6_MESES
missing 0.0
Wall time: 126 ms
TRA_OUT_NAT_Mean_6_MESES
missing 0.0
Wall time: 113 ms
TRA_OUT_MOBILE_Mean_6_MESES
missing 0.0
Wall time: 109 ms
TRA_OUT_MOBILE_OTHER_Mean_6_MESES
missing 0.0
Wall time: 101 ms
SMS_OUT_AMT_Mean_6_MESES
missing 0.0
Wall time: 103 ms
RECHARGE_AMT_Mean_6_MESES
missing 0.0
Wall time: 115 ms
AMT_RENT_Mean_6_MESES
missing 0.0
Wall time: 105 ms
AMT_MIN_OUT_INT_Mean_6_MESES
missing 0.0
Wall time: 107 ms
AMT_MIN_OUT_MOBILE_OTHER_Mean_6_MESES
missing 0.0
Wall time: 117 ms
AMT_MIN_OUT_OFF_NET_FIX_Mean_6_MESES
missing 0.0
Wall time: 125 ms
AMT_MIN_OUT_MOBILE_Mean_6_MESES
missing 0.0
Wall time: 114 ms
AMT_MIN_OUT_MOBILE_SPE_TARIFF_Mean_6_MESES
missing 0.0
Wall time: 110 ms
AMT_ROAMING_SMS_Mean_6_MESES
missing 0.0
Wall time: 104 ms
AMT_ROAMING_DATA_Mean_6_MESES
missing 0.0
Wall time: 107 ms
AMT_ROAMING_VOICE_Mean_6_MESES
missing 0.0
Wall time: 123 ms
AMT_PAQUETICO_VOZ_Mean_6_MESES
missing 0.0
Wall time: 116 ms
AMT_PAQUETICO_DATA_Mean_6_MESES
missing 0.0
Wall time: 131 ms
AMT_MIN_OUT_ON_NET_FIX_Mean_6_MESES
missing 0.0
Wall time: 131 ms
MB_Mean_6_MESES
missing 0.0
Wall time: 127 ms
TRA_IN_LOC_Mean_3_MESES
missing 0.0
Wall time: 133 ms
TRA_IN_INT_Mean_3_MESES
missing 0.0
Wall time: 119 ms
TRA_IN_NAT_Mean_3_MESES
missing 0.0
Wall time: 171 ms
TRA_IN_MOBILE_Mean_3_MESES
missing 0.0
Wall time: 143 ms
TRA_IN_MOBILE_OTHER_Mean_3_MESES
missing 0.0
Wall time: 138 ms
TRA_OUT_LOC_Mean_3_MESES
missing 0.0
Wall time: 126 ms
TRA_OUT_INT_Mean_3_MESES
missing 0.0
Wall time: 129 ms
TRA_OUT_NAT_Mean_3_MESES
missing 0.0
Wall time: 132 ms
TRA_OUT_MOBILE_Mean_3_MESES
missing 0.0
Wall time: 136 ms
TRA_OUT_MOBILE_OTHER_Mean_3_MESES
missing 0.0
Wall time: 132 ms
SMS_OUT_AMT_Mean_3_MESES
missing 0.0
Wall time: 142 ms
RECHARGE_AMT_Mean_3_MESES
missing 0.0
Wall time: 111 ms
AMT_RENT_Mean_3_MESES
missing 0.0
Wall time: 114 ms
AMT_MIN_OUT_INT_Mean_3_MESES
missing 0.0
Wall time: 111 ms
AMT_MIN_OUT_MOBILE_OTHER_Mean_3_MESES
missing 0.0
Wall time: 107 ms
AMT_MIN_OUT_OFF_NET_FIX_Mean_3_MESES
missing 0.0
Wall time: 102 ms
AMT_MIN_OUT_MOBILE_Mean_3_MESES
missing 0.0
Wall time: 101 ms
AMT_MIN_OUT_MOBILE_SPE_TARIFF_Mean_3_MESES
missing 0.0
Wall time: 107 ms
AMT_ROAMING_SMS_Mean_3_MESES
missing 0.0
Wall time: 99.9 ms
AMT_ROAMING_DATA_Mean_3_MESES
missing 0.0
Wall time: 116 ms
AMT_ROAMING_VOICE_Mean_3_MESES
missing 0.0
Wall time: 108 ms
AMT_PAQUETICO_VOZ_Mean_3_MESES
missing 0.0
Wall time: 100 ms
AMT_PAQUETICO_DATA_Mean_3_MESES
missing 0.0
Wall time: 107 ms
AMT_MIN_OUT_ON_NET_FIX_Mean_3_MESES
missing 0.0
Wall time: 111 ms
MB_Mean_3_MESES
missing 0.0
Wall time: 119 ms
FECHA_NACIMIENTO
missing 0.0
Wall time: 331 ms
SEXO
missing 0.0
Wall time: 302 ms
SEGMENTO_BI
missing 0.0
Wall time: 290 ms
DESCRIPCION_PROVINCIA_CUENTA
missing 0.0
Wall time: 351 ms
DESCRIPCION_PROVINCIA_CVOZ
missing 0.0
Wall time: 320 ms
DESCRIPCION_PROVINCIA_CDATA
missing 0.0
Wall time: 268 ms
PROVINCIA
missing 0.0
Wall time: 276 ms
MONTO_RECARGA_Sum_MES_ANTERIOR
missing 0.0
Wall time: 257 ms
CANT_RECARGAS_MES_ANTERIOR
missing 0.0
Wall time: 268 ms
CABLE
missing 0.0
Wall time: 278 ms
TELEFONO_FIJO
missing 0.0
Wall time: 282 ms
INTERNET
missing 0.0
Wall time: 322 ms
MBB
missing 0.0
Wall time: 344 ms
MOVIL_POSPAGO
missing 0.0
Wall time: 308 ms

In [39]:

df.mask.null("NO")

Out[39]:

Viewing 10 of 163.8 thousand rows / 78 columns

None partition(s)

NO 1 (bool)	CO_ID 2 (int64)	ARPU 3 (int64)	TENURE 4 (int64)	FECHA_INST_TELEFONO 5 (string)	FECHA_INSTALACION 6 (string)	STATUS_TELEFONO 7 (string)	CANAL_VENTA 8 (int64)	SUB_CANAL_VENTA 9 (string)	ID_VENDEDOR 10 (string)	ID_PLAN 11 (string)	PLAN_PREPAGO 12 (string)	ID_PLAN_ANTERIOR 13 (string)	FECHA_CAMBIO_PLAN 14 (string)	TRA_IN_LOC_Mean_6_MESES 15 (float64)	TRA_IN_INT_Mean_6_MESES 16 (float64)	TRA_IN_NAT_Mean_6_MESES 17 (float64)	TRA_IN_MOBILE_Mean_6_MESES 18 (float64)	TRA_IN_MOBILE_OTHER_Mean_6_MESES 19 (float64)	TRA_OUT_LOC_Mean_6_MESES 20 (float64)	TRA_OUT_INT_Mean_6_MESES 21 (float64)	TRA_OUT_NAT_Mean_6_MESES 22 (float64)	TRA_OUT_MOBILE_Mean_6_MESES 23 (float64)	TRA_OUT_MOBILE_OTHER_Mean_6_MESES 24 (float64)	SMS_OUT_AMT_Mean_6_MESES 25 (float64)	RECHARGE_AMT_Mean_6_MESES 26 (float64)	MB_Mean_6_MESES 39 (float64)	TRA_IN_LOC_Mean_3_MESES 40 (float64)	TRA_IN_INT_Mean_3_MESES 41 (float64)	TRA_IN_NAT_Mean_3_MESES 42 (float64)	TRA_IN_MOBILE_Mean_3_MESES 43 (float64)	TRA_IN_MOBILE_OTHER_Mean_3_MESES 44 (float64)	TRA_OUT_LOC_Mean_3_MESES 45 (float64)	TRA_OUT_INT_Mean_3_MESES 46 (float64)	TRA_OUT_NAT_Mean_3_MESES 47 (float64)	TRA_OUT_MOBILE_Mean_3_MESES 48 (float64)	TRA_OUT_MOBILE_OTHER_Mean_3_MESES 49 (float64)	SMS_OUT_AMT_Mean_3_MESES 50 (float64)	RECHARGE_AMT_Mean_3_MESES 51 (float64)	MB_Mean_3_MESES 64 (float64)	FECHA_NACIMIENTO 65 (string)	SEXO 66 (string)	SEGMENTO_BI 67 (string)	DESCRIPCION_PROVINCIA_CUENTA 68 (string)	DESCRIPCION_PROVINCIA_CVOZ 69 (string)	DESCRIPCION_PROVINCIA_CDATA 70 (string)	PROVINCIA 71 (string)	MONTO_RECARGA_Sum_MES_ANTERIOR 72 (string)	CANT_RECARGAS_MES_ANTERIOR 73 (string)	CABLE 74 (string)	TELEFONO_FIJO 75 (string)	INTERNET 76 (string)	MBB 77 (string)	MOVIL_POSPAGO 78 (string)
False	12881892	384	109	2013-01-30⋅00:00:00	2013-01-30⋅00:00:00	A	6	4601	18425	CEPREVO005	S	$null$	$null$	1.55	0.986	2.384	17.903	35.277	4.271	0.0	2.16	6.134	7.396	3.143	340.651	0.014	1.763	1.32	3.703	17.527	39.467	2.58	0.0	0.76	8.0	5.837	5.333	282.047	0.011	1982-04-12⋅00:00:00	M	E	SAN⋅PEDRO⋅DE⋅MACORIS	SAN⋅PEDRO⋅DE⋅MACORIS	$null$	SAN⋅PEDRO⋅DE⋅MACORIS	76.920	1	1	$null$	$null$	$null$	$null$
False	136449998	410	32	2019-07-19⋅00:00:00	2022-02-06⋅00:00:00	A	3	2903	24425	CEPREVO004	S	$null$	$null$	3.323	0.0	0.0	126.01	5.379	0.007	0.0	0.0	18.44	1.474	4.0	393.626	3505.539	2.173	0.0	0.0	128.703	4.597	0.017	0.0	0.0	0.753	0.1	5.333	197.433	2940.907	1995-05-10⋅00:00:00	F	$null$	DUARTE	DUARTE	$null$	DUARTE	$null$	$null$	$null$	$null$	$null$	1	$null$
False	15661796	654	93	2014-06-20⋅00:00:00	2014-06-20⋅00:00:00	A	6	358	19394	CEPREVO005	S	$null$	$null$	0.124	0.0	0.0	110.039	87.179	0.549	0.0	0.0	9.119	5.939	16.143	515.26	20121.611	0.29	0.0	0.0	112.61	19.15	0.0	0.0	0.0	12.01	9.71	14.333	575.98	23524.509	1993-02-15⋅00:00:00	F	E	DISTRITO⋅NACIONAL	SAN⋅JUAN	$null$	SAN⋅JUAN	804.880	8	$null$	$null$	$null$	$null$	$null$
False	127068927	350	37	2019-01-29⋅00:00:00	2019-01-31⋅00:00:00	A	3	1919	23502	CEPREVO005	S	$null$	$null$	0.566	0.0	0.0	3.311	3.483	0.0	0.0	0.0	0.069	0.471	0.0	212.297	1296.564	0.0	0.0	0.0	0.493	5.147	0.0	0.0	0.0	0.16	1.1	0.0	38.46	1.238	1973-09-30⋅00:00:00	F	E	AZUA	SANTIAGO	$null$	SANTIAGO	115.380	2	$null$	$null$	$null$	$null$	$null$
False	156732494	660	9	2021-06-14⋅00:00:00	2021-06-14⋅00:00:00	A	6	11361	E6314	CEPREVO005	S	$null$	$null$	0.049	0.0	0.0	1.137	10.38	0.263	0.0	0.0	1.071	4.966	2.0	348.34	11663.497	0.0	0.0	0.0	0.653	0.433	0.613	0.0	0.0	1.12	2.133	2.667	307.68	11349.316	1997-07-30⋅00:00:00	F	E	DISTRITO⋅NACIONAL	SANTO⋅DOMINGO	$null$	SANTO⋅DOMINGO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
False	158282264	741	7	2021-07-30⋅00:00:00	2021-07-30⋅00:00:00	A	3	5598	25430	CEPREVO005	S	$null$	$null$	0.0	0.0	0.0	0.3	3.246	0.0	0.0	0.0	1.094	5.943	1.857	63.954	164.832	0.0	0.0	0.0	0.0	0.713	0.0	0.0	0.0	0.0	0.0	2.333	0.0	0.0	1990-12-24⋅00:00:00	M	E	BAHORUCO	$null$	$null$	BAHORUCO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
False	73882646	596	59	2017-04-20⋅00:00:00	2020-07-07⋅00:00:00	A	3	1938	S4801	CEPREVO004	S	$null$	$null$	0.976	0.0	0.0	15.146	36.881	0.0	0.0	0.0	0.946	1.706	0.0	495.603	10044.85	2.277	0.0	0.0	27.12	36.703	0.0	0.0	0.0	0.72	0.0	0.0	215.38	2453.693	1976-09-09⋅00:00:00	M	E	DISTRITO⋅NACIONAL	SANTO⋅DOMINGO	$null$	SANTO⋅DOMINGO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
False	3385917	288	181	2007-02-19⋅00:00:00	2015-12-20⋅00:00:00	A	6	66	999	CEPREVO006	S	$null$	$null$	0.481	0.0	0.0	1.901	3.589	0.583	1.4	0.0	0.0	1.174	0.0	159.337	385.892	1.123	0.0	0.0	4.437	8.373	1.36	3.267	0.0	0.0	2.74	0.0	371.787	900.414	1972-04-19⋅00:00:00	F	E	DISTRITO⋅NACIONAL	DISTRITO⋅NACIONAL	$null$	DISTRITO⋅NACIONAL	346.140	4	$null$	$null$	$null$	$null$	$null$
False	116622990	713	44	2018-06-29⋅00:00:00	2018-06-29⋅00:00:00	A	6	10700	D8917	CEPREVO005	S	CEFLEVO006	2020-10-26⋅15:41:53	4.917	4.233	0.094	8.399	12.047	0.637	0.003	0.243	5.5	1.93	2.143	691.541	16461.451	5.42	9.877	0.22	9.877	8.323	1.267	0.007	0.0	4.613	2.01	3.0	736.663	20158.096	1952-04-30⋅00:00:00	F	D	PUERTO⋅PLATA	SANTIAGO	$null$	SANTIAGO	645.390	5	$null$	$null$	$null$	1	$null$
False	81969494	702	57	2017-06-20⋅00:00:00	2019-08-14⋅00:00:00	A	6	215	D3359	CEPREVO005	S	$null$	$null$	5.589	35.849	0.0	8.55	22.851	1.337	0.0	0.0	28.023	12.806	5.714	679.137	0.03	3.253	51.24	0.0	3.423	8.573	0.0	0.0	0.0	11.827	2.413	6.667	320.5	0.003	1988-11-24⋅00:00:00	F	E	MARIA⋅TRINIDAD⋅SANCHEZ	SAMANA	SAMANA	SAMANA	192.300	4	$null$	$null$	$null$	$null$	$null$
False	132727116	341	34	2019-05-14⋅00:00:00	2019-05-14⋅00:00:00	A	7	2799	F3816	CEPREVO005	S	$null$	$null$	1.84	0.0	0.0	1.764	17.446	4.134	0.0	0.0	20.21	27.907	0.0	362.646	0.0	2.113	0.0	0.0	2.67	14.993	0.0	0.0	0.0	31.097	22.003	0.0	325.647	0.0	1956-11-04⋅00:00:00	F	E	SAN⋅CRISTOBAL	SANTIAGO	$null$	SANTIAGO	392.300	10	$null$	$null$	$null$	$null$	$null$

Viewing 10 of 163.8 thousand rows / 78 columns

None partition(s) <class 'optimus.engines.vaex.dataframe.VaexDataFrame'>

In [26]:

df.data["CO_ID"] ="1"

In [27]:

dfv = df.data["CO_ID"]
dfv
def func(value):
    return value

dfv = df.data.apply(func, dfv)

In [28]:

df.data

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\Anaconda3\envs\python38\lib\site-packages\IPython\core\formatters.py in __call__(self, obj, include, exclude)
    968 
    969             if method is not None:
--> 970                 return method(include=include, exclude=exclude)
    971             return None
    972         else:

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _repr_mimebundle_(self, include, exclude, **kwargs)
   4207         # TODO: optimize, since we use the same data in both versions
   4208         # TODO: include latex version
-> 4209         return {'text/html':self._head_and_tail_table(format='html'), 'text/plain': self._head_and_tail_table(format='plain')}
   4210 
   4211     def _repr_html_(self):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _head_and_tail_table(self, n, format)
   3962             return self._as_table(0, N, format=format)
   3963         else:
-> 3964             return self._as_table(0, math.ceil(n / 2), N - math.floor(n / 2), N, format=format)
   3965 
   3966     def head_and_tail_print(self, n=5):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _as_table(self, i1, i2, j1, j2, format, ellipsis)
   4116             # return values_list
   4117         if i2 - i1 > 0:
-> 4118             parts = table_part(i1, i2, parts)
   4119             if j1 is not None and j2 is not None:
   4120                 values_list[0][1].append(ellipsis)

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in table_part(k1, k2, parts)
   4110                     if columns_sliced is not None and j >= columns_sliced:
   4111                         column_index += 1  # skip over the slice/ellipsis
-> 4112                     value = values[name][i]
   4113                     value = _format_value(value)
   4114                     values_list[column_index+1][1].append(value)

TypeError: 'int' object is not subscriptable

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\Anaconda3\envs\python38\lib\site-packages\IPython\core\formatters.py in __call__(self, obj)
    700                 type_pprinters=self.type_printers,
    701                 deferred_pprinters=self.deferred_printers)
--> 702             printer.pretty(obj)
    703             printer.flush()
    704             return stream.getvalue()

~\Anaconda3\envs\python38\lib\site-packages\IPython\lib\pretty.py in pretty(self, obj)
    392                         if cls is not object \
    393                                 and callable(cls.__dict__.get('__repr__')):
--> 394                             return _repr_pprint(obj, self, cycle)
    395 
    396             return _default_pprint(obj, self, cycle)

~\Anaconda3\envs\python38\lib\site-packages\IPython\lib\pretty.py in _repr_pprint(obj, p, cycle)
    698     """A pprint that just redirects to the normal repr function."""
    699     # Find newlines and replace them with p.break_()
--> 700     output = repr(obj)
    701     lines = output.splitlines()
    702     with p.group():

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in __repr__(self)
   4219     if not _DEBUG:
   4220         def __repr__(self):
-> 4221             return self._head_and_tail_table(format='plain')
   4222 
   4223     def __current_sequence_index(self):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _head_and_tail_table(self, n, format)
   3962             return self._as_table(0, N, format=format)
   3963         else:
-> 3964             return self._as_table(0, math.ceil(n / 2), N - math.floor(n / 2), N, format=format)
   3965 
   3966     def head_and_tail_print(self, n=5):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _as_table(self, i1, i2, j1, j2, format, ellipsis)
   4116             # return values_list
   4117         if i2 - i1 > 0:
-> 4118             parts = table_part(i1, i2, parts)
   4119             if j1 is not None and j2 is not None:
   4120                 values_list[0][1].append(ellipsis)

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in table_part(k1, k2, parts)
   4110                     if columns_sliced is not None and j >= columns_sliced:
   4111                         column_index += 1  # skip over the slice/ellipsis
-> 4112                     value = values[name][i]
   4113                     value = _format_value(value)
   4114                     values_list[column_index+1][1].append(value)

TypeError: 'int' object is not subscriptable

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\Anaconda3\envs\python38\lib\site-packages\IPython\core\formatters.py in __call__(self, obj)
    343             method = get_real_method(obj, self.print_method)
    344             if method is not None:
--> 345                 return method()
    346             return None
    347         else:

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _repr_html_(self)
   4212         """Representation for Jupyter."""
   4213         self._output_css()
-> 4214         return self._head_and_tail_table()
   4215 
   4216     def __str__(self):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _head_and_tail_table(self, n, format)
   3962             return self._as_table(0, N, format=format)
   3963         else:
-> 3964             return self._as_table(0, math.ceil(n / 2), N - math.floor(n / 2), N, format=format)
   3965 
   3966     def head_and_tail_print(self, n=5):

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in _as_table(self, i1, i2, j1, j2, format, ellipsis)
   4116             # return values_list
   4117         if i2 - i1 > 0:
-> 4118             parts = table_part(i1, i2, parts)
   4119             if j1 is not None and j2 is not None:
   4120                 values_list[0][1].append(ellipsis)

~\Anaconda3\envs\python38\lib\site-packages\vaex\dataframe.py in table_part(k1, k2, parts)
   4110                     if columns_sliced is not None and j >= columns_sliced:
   4111                         column_index += 1  # skip over the slice/ellipsis
-> 4112                     value = values[name][i]
   4113                     value = _format_value(value)
   4114                     values_list[column_index+1][1].append(value)

TypeError: 'int' object is not subscriptable

In [ ]:

import vaex
with vaex.progress.tree('rich', title="Two passes"):
    df.cols.sum("*")

Output()

In [6]:

df

limit 10

Out[6]:

Viewing 10 of 163.8 thousand rows / 78 columns

None partition(s)

NO 1 (int64)	CO_ID 2 (int64)	ARPU 3 (int64)	TENURE 4 (int64)	FECHA_INST_TELEFONO 5 (string)	FECHA_INSTALACION 6 (string)	STATUS_TELEFONO 7 (string)	CANAL_VENTA 8 (int64)	SUB_CANAL_VENTA 9 (string)	ID_VENDEDOR 10 (string)	ID_PLAN 11 (string)	PLAN_PREPAGO 12 (string)	ID_PLAN_ANTERIOR 13 (string)	FECHA_CAMBIO_PLAN 14 (string)	TRA_IN_LOC_Mean_6_MESES 15 (float64)	TRA_IN_INT_Mean_6_MESES 16 (float64)	TRA_IN_NAT_Mean_6_MESES 17 (float64)	TRA_IN_MOBILE_Mean_6_MESES 18 (float64)	TRA_IN_MOBILE_OTHER_Mean_6_MESES 19 (float64)	TRA_OUT_LOC_Mean_6_MESES 20 (float64)	TRA_OUT_INT_Mean_6_MESES 21 (float64)	TRA_OUT_NAT_Mean_6_MESES 22 (float64)	TRA_OUT_MOBILE_Mean_6_MESES 23 (float64)	TRA_OUT_MOBILE_OTHER_Mean_6_MESES 24 (float64)	SMS_OUT_AMT_Mean_6_MESES 25 (float64)	RECHARGE_AMT_Mean_6_MESES 26 (float64)	MB_Mean_6_MESES 39 (float64)	TRA_IN_LOC_Mean_3_MESES 40 (float64)	TRA_IN_INT_Mean_3_MESES 41 (float64)	TRA_IN_NAT_Mean_3_MESES 42 (float64)	TRA_IN_MOBILE_Mean_3_MESES 43 (float64)	TRA_IN_MOBILE_OTHER_Mean_3_MESES 44 (float64)	TRA_OUT_LOC_Mean_3_MESES 45 (float64)	TRA_OUT_INT_Mean_3_MESES 46 (float64)	TRA_OUT_NAT_Mean_3_MESES 47 (float64)	TRA_OUT_MOBILE_Mean_3_MESES 48 (float64)	TRA_OUT_MOBILE_OTHER_Mean_3_MESES 49 (float64)	SMS_OUT_AMT_Mean_3_MESES 50 (float64)	RECHARGE_AMT_Mean_3_MESES 51 (float64)	MB_Mean_3_MESES 64 (float64)	FECHA_NACIMIENTO 65 (string)	SEXO 66 (string)	SEGMENTO_BI 67 (string)	DESCRIPCION_PROVINCIA_CUENTA 68 (string)	DESCRIPCION_PROVINCIA_CVOZ 69 (string)	DESCRIPCION_PROVINCIA_CDATA 70 (string)	PROVINCIA 71 (string)	MONTO_RECARGA_Sum_MES_ANTERIOR 72 (string)	CANT_RECARGAS_MES_ANTERIOR 73 (string)	CABLE 74 (string)	TELEFONO_FIJO 75 (string)	INTERNET 76 (string)	MBB 77 (string)	MOVIL_POSPAGO 78 (string)
1	12881892	384	109	2013-01-30⋅00:00:00	2013-01-30⋅00:00:00	A	6	4601	18425	CEPREVO005	S	$null$	$null$	1.55	0.986	2.384	17.903	35.277	4.271	0.0	2.16	6.134	7.396	3.143	340.651	0.014	1.763	1.32	3.703	17.527	39.467	2.58	0.0	0.76	8.0	5.837	5.333	282.047	0.011	1982-04-12⋅00:00:00	M	E	SAN⋅PEDRO⋅DE⋅MACORIS	SAN⋅PEDRO⋅DE⋅MACORIS	$null$	SAN⋅PEDRO⋅DE⋅MACORIS	76.920	1	1	$null$	$null$	$null$	$null$
2	136449998	410	32	2019-07-19⋅00:00:00	2022-02-06⋅00:00:00	A	3	2903	24425	CEPREVO004	S	$null$	$null$	3.323	0.0	0.0	126.01	5.379	0.007	0.0	0.0	18.44	1.474	4.0	393.626	3505.539	2.173	0.0	0.0	128.703	4.597	0.017	0.0	0.0	0.753	0.1	5.333	197.433	2940.907	1995-05-10⋅00:00:00	F	$null$	DUARTE	DUARTE	$null$	DUARTE	$null$	$null$	$null$	$null$	$null$	1	$null$
3	15661796	654	93	2014-06-20⋅00:00:00	2014-06-20⋅00:00:00	A	6	358	19394	CEPREVO005	S	$null$	$null$	0.124	0.0	0.0	110.039	87.179	0.549	0.0	0.0	9.119	5.939	16.143	515.26	20121.611	0.29	0.0	0.0	112.61	19.15	0.0	0.0	0.0	12.01	9.71	14.333	575.98	23524.509	1993-02-15⋅00:00:00	F	E	DISTRITO⋅NACIONAL	SAN⋅JUAN	$null$	SAN⋅JUAN	804.880	8	$null$	$null$	$null$	$null$	$null$
4	127068927	350	37	2019-01-29⋅00:00:00	2019-01-31⋅00:00:00	A	3	1919	23502	CEPREVO005	S	$null$	$null$	0.566	0.0	0.0	3.311	3.483	0.0	0.0	0.0	0.069	0.471	0.0	212.297	1296.564	0.0	0.0	0.0	0.493	5.147	0.0	0.0	0.0	0.16	1.1	0.0	38.46	1.238	1973-09-30⋅00:00:00	F	E	AZUA	SANTIAGO	$null$	SANTIAGO	115.380	2	$null$	$null$	$null$	$null$	$null$
5	156732494	660	9	2021-06-14⋅00:00:00	2021-06-14⋅00:00:00	A	6	11361	E6314	CEPREVO005	S	$null$	$null$	0.049	0.0	0.0	1.137	10.38	0.263	0.0	0.0	1.071	4.966	2.0	348.34	11663.497	0.0	0.0	0.0	0.653	0.433	0.613	0.0	0.0	1.12	2.133	2.667	307.68	11349.316	1997-07-30⋅00:00:00	F	E	DISTRITO⋅NACIONAL	SANTO⋅DOMINGO	$null$	SANTO⋅DOMINGO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
6	158282264	741	7	2021-07-30⋅00:00:00	2021-07-30⋅00:00:00	A	3	5598	25430	CEPREVO005	S	$null$	$null$	0.0	0.0	0.0	0.3	3.246	0.0	0.0	0.0	1.094	5.943	1.857	63.954	164.832	0.0	0.0	0.0	0.0	0.713	0.0	0.0	0.0	0.0	0.0	2.333	0.0	0.0	1990-12-24⋅00:00:00	M	E	BAHORUCO	$null$	$null$	BAHORUCO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
7	73882646	596	59	2017-04-20⋅00:00:00	2020-07-07⋅00:00:00	A	3	1938	S4801	CEPREVO004	S	$null$	$null$	0.976	0.0	0.0	15.146	36.881	0.0	0.0	0.0	0.946	1.706	0.0	495.603	10044.85	2.277	0.0	0.0	27.12	36.703	0.0	0.0	0.0	0.72	0.0	0.0	215.38	2453.693	1976-09-09⋅00:00:00	M	E	DISTRITO⋅NACIONAL	SANTO⋅DOMINGO	$null$	SANTO⋅DOMINGO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
8	3385917	288	181	2007-02-19⋅00:00:00	2015-12-20⋅00:00:00	A	6	66	999	CEPREVO006	S	$null$	$null$	0.481	0.0	0.0	1.901	3.589	0.583	1.4	0.0	0.0	1.174	0.0	159.337	385.892	1.123	0.0	0.0	4.437	8.373	1.36	3.267	0.0	0.0	2.74	0.0	371.787	900.414	1972-04-19⋅00:00:00	F	E	DISTRITO⋅NACIONAL	DISTRITO⋅NACIONAL	$null$	DISTRITO⋅NACIONAL	346.140	4	$null$	$null$	$null$	$null$	$null$
9	116622990	713	44	2018-06-29⋅00:00:00	2018-06-29⋅00:00:00	A	6	10700	D8917	CEPREVO005	S	CEFLEVO006	2020-10-26⋅15:41:53	4.917	4.233	0.094	8.399	12.047	0.637	0.003	0.243	5.5	1.93	2.143	691.541	16461.451	5.42	9.877	0.22	9.877	8.323	1.267	0.007	0.0	4.613	2.01	3.0	736.663	20158.096	1952-04-30⋅00:00:00	F	D	PUERTO⋅PLATA	SANTIAGO	$null$	SANTIAGO	645.390	5	$null$	$null$	$null$	1	$null$
10	81969494	702	57	2017-06-20⋅00:00:00	2019-08-14⋅00:00:00	A	6	215	D3359	CEPREVO005	S	$null$	$null$	5.589	35.849	0.0	8.55	22.851	1.337	0.0	0.0	28.023	12.806	5.714	679.137	0.03	3.253	51.24	0.0	3.423	8.573	0.0	0.0	0.0	11.827	2.413	6.667	320.5	0.003	1988-11-24⋅00:00:00	F	E	MARIA⋅TRINIDAD⋅SANCHEZ	SAMANA	SAMANA	SAMANA	192.300	4	$null$	$null$	$null$	$null$	$null$
11	132727116	341	34	2019-05-14⋅00:00:00	2019-05-14⋅00:00:00	A	7	2799	F3816	CEPREVO005	S	$null$	$null$	1.84	0.0	0.0	1.764	17.446	4.134	0.0	0.0	20.21	27.907	0.0	362.646	0.0	2.113	0.0	0.0	2.67	14.993	0.0	0.0	0.0	31.097	22.003	0.0	325.647	0.0	1956-11-04⋅00:00:00	F	E	SAN⋅CRISTOBAL	SANTIAGO	$null$	SANTIAGO	392.300	10	$null$	$null$	$null$	$null$	$null$

Viewing 10 of 163.8 thousand rows / 78 columns

None partition(s) <class 'optimus.engines.vaex.dataframe.VaexDataFrame'>

In [12]:

%%time
df

Wall time: 0 ns
limit 10

Out[12]:

Viewing 10 of 163.8 thousand rows / 78 columns

None partition(s)

NO 1 (int64)	CO_ID 2 (int64)	ARPU 3 (int64)	TENURE 4 (int64)	FECHA_INST_TELEFONO 5 (string)	FECHA_INSTALACION 6 (string)	STATUS_TELEFONO 7 (string)	CANAL_VENTA 8 (int64)	SUB_CANAL_VENTA 9 (string)	ID_VENDEDOR 10 (string)	ID_PLAN 11 (string)	PLAN_PREPAGO 12 (string)	ID_PLAN_ANTERIOR 13 (string)	FECHA_CAMBIO_PLAN 14 (string)	TRA_IN_LOC_Mean_6_MESES 15 (float64)	TRA_IN_INT_Mean_6_MESES 16 (float64)	TRA_IN_NAT_Mean_6_MESES 17 (float64)	TRA_IN_MOBILE_Mean_6_MESES 18 (float64)	TRA_IN_MOBILE_OTHER_Mean_6_MESES 19 (float64)	TRA_OUT_LOC_Mean_6_MESES 20 (float64)	TRA_OUT_INT_Mean_6_MESES 21 (float64)	TRA_OUT_NAT_Mean_6_MESES 22 (float64)	TRA_OUT_MOBILE_Mean_6_MESES 23 (float64)	TRA_OUT_MOBILE_OTHER_Mean_6_MESES 24 (float64)	SMS_OUT_AMT_Mean_6_MESES 25 (float64)	RECHARGE_AMT_Mean_6_MESES 26 (float64)	MB_Mean_6_MESES 39 (float64)	TRA_IN_LOC_Mean_3_MESES 40 (float64)	TRA_IN_INT_Mean_3_MESES 41 (float64)	TRA_IN_NAT_Mean_3_MESES 42 (float64)	TRA_IN_MOBILE_Mean_3_MESES 43 (float64)	TRA_IN_MOBILE_OTHER_Mean_3_MESES 44 (float64)	TRA_OUT_LOC_Mean_3_MESES 45 (float64)	TRA_OUT_INT_Mean_3_MESES 46 (float64)	TRA_OUT_NAT_Mean_3_MESES 47 (float64)	TRA_OUT_MOBILE_Mean_3_MESES 48 (float64)	TRA_OUT_MOBILE_OTHER_Mean_3_MESES 49 (float64)	SMS_OUT_AMT_Mean_3_MESES 50 (float64)	RECHARGE_AMT_Mean_3_MESES 51 (float64)	MB_Mean_3_MESES 64 (float64)	FECHA_NACIMIENTO 65 (string)	SEXO 66 (string)	SEGMENTO_BI 67 (string)	DESCRIPCION_PROVINCIA_CUENTA 68 (string)	DESCRIPCION_PROVINCIA_CVOZ 69 (string)	DESCRIPCION_PROVINCIA_CDATA 70 (string)	PROVINCIA 71 (string)	MONTO_RECARGA_Sum_MES_ANTERIOR 72 (string)	CANT_RECARGAS_MES_ANTERIOR 73 (string)	CABLE 74 (string)	TELEFONO_FIJO 75 (string)	INTERNET 76 (string)	MBB 77 (string)	MOVIL_POSPAGO 78 (string)
1	12881892	384	109	2013-01-30⋅00:00:00	2013-01-30⋅00:00:00	A	6	4601	18425	CEPREVO005	S	$null$	$null$	1.55	0.986	2.384	17.903	35.277	4.271	0.0	2.16	6.134	7.396	3.143	340.651	0.014	1.763	1.32	3.703	17.527	39.467	2.58	0.0	0.76	8.0	5.837	5.333	282.047	0.011	1982-04-12⋅00:00:00	M	E	SAN⋅PEDRO⋅DE⋅MACORIS	SAN⋅PEDRO⋅DE⋅MACORIS	$null$	SAN⋅PEDRO⋅DE⋅MACORIS	76.920	1	1	$null$	$null$	$null$	$null$
2	136449998	410	32	2019-07-19⋅00:00:00	2022-02-06⋅00:00:00	A	3	2903	24425	CEPREVO004	S	$null$	$null$	3.323	0.0	0.0	126.01	5.379	0.007	0.0	0.0	18.44	1.474	4.0	393.626	3505.539	2.173	0.0	0.0	128.703	4.597	0.017	0.0	0.0	0.753	0.1	5.333	197.433	2940.907	1995-05-10⋅00:00:00	F	$null$	DUARTE	DUARTE	$null$	DUARTE	$null$	$null$	$null$	$null$	$null$	1	$null$
3	15661796	654	93	2014-06-20⋅00:00:00	2014-06-20⋅00:00:00	A	6	358	19394	CEPREVO005	S	$null$	$null$	0.124	0.0	0.0	110.039	87.179	0.549	0.0	0.0	9.119	5.939	16.143	515.26	20121.611	0.29	0.0	0.0	112.61	19.15	0.0	0.0	0.0	12.01	9.71	14.333	575.98	23524.509	1993-02-15⋅00:00:00	F	E	DISTRITO⋅NACIONAL	SAN⋅JUAN	$null$	SAN⋅JUAN	804.880	8	$null$	$null$	$null$	$null$	$null$
4	127068927	350	37	2019-01-29⋅00:00:00	2019-01-31⋅00:00:00	A	3	1919	23502	CEPREVO005	S	$null$	$null$	0.566	0.0	0.0	3.311	3.483	0.0	0.0	0.0	0.069	0.471	0.0	212.297	1296.564	0.0	0.0	0.0	0.493	5.147	0.0	0.0	0.0	0.16	1.1	0.0	38.46	1.238	1973-09-30⋅00:00:00	F	E	AZUA	SANTIAGO	$null$	SANTIAGO	115.380	2	$null$	$null$	$null$	$null$	$null$
5	156732494	660	9	2021-06-14⋅00:00:00	2021-06-14⋅00:00:00	A	6	11361	E6314	CEPREVO005	S	$null$	$null$	0.049	0.0	0.0	1.137	10.38	0.263	0.0	0.0	1.071	4.966	2.0	348.34	11663.497	0.0	0.0	0.0	0.653	0.433	0.613	0.0	0.0	1.12	2.133	2.667	307.68	11349.316	1997-07-30⋅00:00:00	F	E	DISTRITO⋅NACIONAL	SANTO⋅DOMINGO	$null$	SANTO⋅DOMINGO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
6	158282264	741	7	2021-07-30⋅00:00:00	2021-07-30⋅00:00:00	A	3	5598	25430	CEPREVO005	S	$null$	$null$	0.0	0.0	0.0	0.3	3.246	0.0	0.0	0.0	1.094	5.943	1.857	63.954	164.832	0.0	0.0	0.0	0.0	0.713	0.0	0.0	0.0	0.0	0.0	2.333	0.0	0.0	1990-12-24⋅00:00:00	M	E	BAHORUCO	$null$	$null$	BAHORUCO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
7	73882646	596	59	2017-04-20⋅00:00:00	2020-07-07⋅00:00:00	A	3	1938	S4801	CEPREVO004	S	$null$	$null$	0.976	0.0	0.0	15.146	36.881	0.0	0.0	0.0	0.946	1.706	0.0	495.603	10044.85	2.277	0.0	0.0	27.12	36.703	0.0	0.0	0.0	0.72	0.0	0.0	215.38	2453.693	1976-09-09⋅00:00:00	M	E	DISTRITO⋅NACIONAL	SANTO⋅DOMINGO	$null$	SANTO⋅DOMINGO	$null$	$null$	$null$	$null$	$null$	$null$	$null$
8	3385917	288	181	2007-02-19⋅00:00:00	2015-12-20⋅00:00:00	A	6	66	999	CEPREVO006	S	$null$	$null$	0.481	0.0	0.0	1.901	3.589	0.583	1.4	0.0	0.0	1.174	0.0	159.337	385.892	1.123	0.0	0.0	4.437	8.373	1.36	3.267	0.0	0.0	2.74	0.0	371.787	900.414	1972-04-19⋅00:00:00	F	E	DISTRITO⋅NACIONAL	DISTRITO⋅NACIONAL	$null$	DISTRITO⋅NACIONAL	346.140	4	$null$	$null$	$null$	$null$	$null$
9	116622990	713	44	2018-06-29⋅00:00:00	2018-06-29⋅00:00:00	A	6	10700	D8917	CEPREVO005	S	CEFLEVO006	2020-10-26⋅15:41:53	4.917	4.233	0.094	8.399	12.047	0.637	0.003	0.243	5.5	1.93	2.143	691.541	16461.451	5.42	9.877	0.22	9.877	8.323	1.267	0.007	0.0	4.613	2.01	3.0	736.663	20158.096	1952-04-30⋅00:00:00	F	D	PUERTO⋅PLATA	SANTIAGO	$null$	SANTIAGO	645.390	5	$null$	$null$	$null$	1	$null$
10	81969494	702	57	2017-06-20⋅00:00:00	2019-08-14⋅00:00:00	A	6	215	D3359	CEPREVO005	S	$null$	$null$	5.589	35.849	0.0	8.55	22.851	1.337	0.0	0.0	28.023	12.806	5.714	679.137	0.03	3.253	51.24	0.0	3.423	8.573	0.0	0.0	0.0	11.827	2.413	6.667	320.5	0.003	1988-11-24⋅00:00:00	F	E	MARIA⋅TRINIDAD⋅SANCHEZ	SAMANA	SAMANA	SAMANA	192.300	4	$null$	$null$	$null$	$null$	$null$
11	132727116	341	34	2019-05-14⋅00:00:00	2019-05-14⋅00:00:00	A	7	2799	F3816	CEPREVO005	S	$null$	$null$	1.84	0.0	0.0	1.764	17.446	4.134	0.0	0.0	20.21	27.907	0.0	362.646	0.0	2.113	0.0	0.0	2.67	14.993	0.0	0.0	0.0	31.097	22.003	0.0	325.647	0.0	1956-11-04⋅00:00:00	F	E	SAN⋅CRISTOBAL	SANTIAGO	$null$	SANTIAGO	392.300	10	$null$	$null$	$null$	$null$	$null$

Viewing 10 of 163.8 thousand rows / 78 columns

None partition(s) <class 'optimus.engines.vaex.dataframe.VaexDataFrame'>

In [62]:

%timeit df.data.sum(df.cols.names([0,1,2,3]))

21 ms ± 2.06 ms per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [7]:

%%time
import vaex
with vaex.progress.tree('rich', title="Two passes"):
    dff = df.cols.to_float()

Output()

Wall time: 603 ms

In [8]:

dfv = dff.data