load config data
import pandas as pd
import os
import re
import gc
from datetime import date
import pickle
# key: filename, [0]=book_long, [1]=book_num, [3]=book_short
bo2book = {'01-matthew': ['Matthew', '1', 'Matt'],
'02-mark': ['Mark', '2', 'Mark'],
'03-luke': ['Luke', '3', 'Luke'],
'04-john': ['John', '4', 'John'],
'05-acts': ['Acts', '5', 'Acts'],
'06-romans': ['Romans', '6', 'Rom'],
'07-1corinthians': ['I_Corinthians', '7', '1Cor'],
'08-2corinthians': ['II_Corinthians', '8', '2Cor'],
'09-galatians': ['Galatians', '9', 'Gal'],
'10-ephesians': ['Ephesians', '10', 'Eph'],
'11-philippians': ['Philippians', '11', 'Phil'],
'12-colossians': ['Colossians', '12', 'Col'],
'13-1thessalonians':['I_Thessalonians', '13', '1Thess'],
'14-2thessalonians':['II_Thessalonians','14', '2Thess'],
'15-1timothy': ['I_Timothy', '15', '1Tim'],
'16-2timothy': ['II_Timothy', '16', '2Tim'],
'17-titus': ['Titus', '17', 'Titus'],
'18-philemon': ['Philemon', '18', 'Phlm'],
'19-hebrews': ['Hebrews', '19', 'Heb'],
'20-james': ['James', '20', 'Jas'],
'21-1peter': ['I_Peter', '21', '1Pet'],
'22-2peter': ['II_Peter', '22', '2Pet'],
'23-1john': ['I_John', '23', '1John'],
'24-2john': ['II_John', '24', '2John'],
'25-3john': ['III_John', '25', '3John'],
'26-jude': ['Jude', '26', 'Jude'],
'27-revelation': ['Revelation', '27', 'Rev']}
load and sort the data in pickle file and store in new fileset.
BaseDir = 'C:\\Users\\tonyj\\my_new_Jupyter_folder\\test_of_xml_etree\\'
source_dir = BaseDir+'outputfiles\\' #the input files (with 'wordjumps')
output_dir = BaseDir+'outputfiles_sorted\\' #the output files (words in order of running text)
for bo in bo2book:
'''
load all data into a dataframe
process books in order (bookinfo is a list!)
'''
InputFile = os.path.join(source_dir, f'{bo}.pkl')
OutputFile = os.path.join(output_dir, f'{bo}.pkl')
print(f'\tloading {InputFile}...')
pkl_file = open(InputFile, 'rb')
df = pickle.load(pkl_file)
pkl_file.close()
# not sure if this is needed
# fill dictionary of column names for this book
IndexDict = {} # init an empty dictionary
ItemsInRow=1
for itemname in df.columns.to_list():
IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
ItemsInRow+=1
# sort by id
df.sort_values(by=['nodeId'])
#store the resulting DataFrame per book into a pickle file for further processing
#df = df.convert_dtypes(convert_string=True)
output = open(r"{}".format(OutputFile), 'wb')
pickle.dump(df, output)
output.close()
loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\01-matthew.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\02-mark.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\03-luke.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\04-john.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\05-acts.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\06-romans.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\07-1corinthians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\08-2corinthians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\09-galatians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\10-ephesians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\11-philippians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\12-colossians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\13-1thessalonians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\14-2thessalonians.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\15-1timothy.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\16-2timothy.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\17-titus.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\18-philemon.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\19-hebrews.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\20-james.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\21-1peter.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\22-2peter.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\23-1john.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\24-2john.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\25-3john.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\26-jude.pkl... loading C:\Users\tonyj\my_new_Jupyter_folder\test_of_xml_etree\outputfiles\27-revelation.pkl...
from pprint import pprint
# dump test data
BaseDir = 'C:\\Users\\tonyj\\my_new_Jupyter_folder\\test_of_xml_etree\\'
source_dir = BaseDir+'outputfiles\\' #the input files (with 'wordjumps')
output_dir = BaseDir+'outputfiles_sorted\\' #the output files (words in order of running text)
InputFile = os.path.join(source_dir, f'25-3john.pkl')
OutputFile = os.path.join(output_dir, f'25-3john.pkl')
pkl_file = open(InputFile, 'rb')
df = pickle.load(pkl_file)
pkl_file.close()
pprint(df)
pkl_file = open(OutputFile, 'rb')
df = pickle.load(pkl_file)
pkl_file.close()
pprint(df)
{http://www.w3.org/XML/1998/namespace}id ref Cat Start End \
127273 n64001001001 3JN 1:1!1 det 0 0
127274 n64001001002 3JN 1:1!2 adj 1 1
127275 n64001001003 3JN 1:1!3 noun 2 2
127276 n64001001004 3JN 1:1!4 det 3 3
127277 n64001001005 3JN 1:1!5 adj 4 4
... ... ... ... ... ..
127487 n64001015007 3JN 1:15!7 verb 0 0
127488 n64001015008 3JN 1:15!8 det 1 1
127489 n64001015009 3JN 1:15!9 adj 2 2
127490 n64001015010 3JN 1:15!10 prep 3 3
127491 n64001015011 3JN 1:15!11 noun 4 4
StrongNumber UnicodeLemma Gender Number FunctionalTag ... \
127273 3588 ὁ Masculine Singular T-NSM ...
127274 4245 πρεσβύτερος Masculine Singular A-NSM-C ...
127275 1050 Γάϊος Masculine Singular N-DSM ...
127276 3588 ὁ Masculine Singular T-DSM ...
127277 27 ἀγαπητός Masculine Singular A-DSM ...
... ... ... ... ... ... ...
127487 782 ἀσπάζομαι NaN Singular V-PNM-2S ...
127488 3588 ὁ Masculine Plural T-APM ...
127489 5384 φίλος Masculine Plural A-APM ...
127490 2596 κατά NaN NaN PREP ...
127491 3686 ὄνομα Neuter Singular N-ASN ...
Parent16Name Parent16Type Parent16Cat Parent16Start Parent16End \
127273 NaN NaN NaN NaN NaN
127274 NaN NaN NaN NaN NaN
127275 NaN NaN NaN NaN NaN
127276 NaN NaN NaN NaN NaN
127277 NaN NaN NaN NaN NaN
... ... ... ... ... ...
127487 NaN NaN NaN NaN NaN
127488 NaN NaN NaN NaN NaN
127489 NaN NaN NaN NaN NaN
127490 NaN NaN NaN NaN NaN
127491 NaN NaN NaN NaN NaN
Parent16Rule Parent16Head Parent16NodeId Parent16ClType Parent16HasDet
127273 NaN NaN NaN NaN NaN
127274 NaN NaN NaN NaN NaN
127275 NaN NaN NaN NaN NaN
127276 NaN NaN NaN NaN NaN
127277 NaN NaN NaN NaN NaN
... ... ... ... ... ...
127487 NaN NaN NaN NaN NaN
127488 NaN NaN NaN NaN NaN
127489 NaN NaN NaN NaN NaN
127490 NaN NaN NaN NaN NaN
127491 NaN NaN NaN NaN NaN
[219 rows x 198 columns]
{http://www.w3.org/XML/1998/namespace}id ref Cat Start End \
127273 n64001001001 3JN 1:1!1 det 0 0
127274 n64001001002 3JN 1:1!2 adj 1 1
127275 n64001001003 3JN 1:1!3 noun 2 2
127276 n64001001004 3JN 1:1!4 det 3 3
127277 n64001001005 3JN 1:1!5 adj 4 4
... ... ... ... ... ..
127487 n64001015007 3JN 1:15!7 verb 0 0
127488 n64001015008 3JN 1:15!8 det 1 1
127489 n64001015009 3JN 1:15!9 adj 2 2
127490 n64001015010 3JN 1:15!10 prep 3 3
127491 n64001015011 3JN 1:15!11 noun 4 4
StrongNumber UnicodeLemma Gender Number FunctionalTag ... \
127273 3588 ὁ Masculine Singular T-NSM ...
127274 4245 πρεσβύτερος Masculine Singular A-NSM-C ...
127275 1050 Γάϊος Masculine Singular N-DSM ...
127276 3588 ὁ Masculine Singular T-DSM ...
127277 27 ἀγαπητός Masculine Singular A-DSM ...
... ... ... ... ... ... ...
127487 782 ἀσπάζομαι NaN Singular V-PNM-2S ...
127488 3588 ὁ Masculine Plural T-APM ...
127489 5384 φίλος Masculine Plural A-APM ...
127490 2596 κατά NaN NaN PREP ...
127491 3686 ὄνομα Neuter Singular N-ASN ...
Parent16Name Parent16Type Parent16Cat Parent16Start Parent16End \
127273 NaN NaN NaN NaN NaN
127274 NaN NaN NaN NaN NaN
127275 NaN NaN NaN NaN NaN
127276 NaN NaN NaN NaN NaN
127277 NaN NaN NaN NaN NaN
... ... ... ... ... ...
127487 NaN NaN NaN NaN NaN
127488 NaN NaN NaN NaN NaN
127489 NaN NaN NaN NaN NaN
127490 NaN NaN NaN NaN NaN
127491 NaN NaN NaN NaN NaN
Parent16Rule Parent16Head Parent16NodeId Parent16ClType Parent16HasDet
127273 NaN NaN NaN NaN NaN
127274 NaN NaN NaN NaN NaN
127275 NaN NaN NaN NaN NaN
127276 NaN NaN NaN NaN NaN
127277 NaN NaN NaN NaN NaN
... ... ... ... ... ...
127487 NaN NaN NaN NaN NaN
127488 NaN NaN NaN NaN NaN
127489 NaN NaN NaN NaN NaN
127490 NaN NaN NaN NaN NaN
127491 NaN NaN NaN NaN NaN
[219 rows x 198 columns]