Creating Text-Fabric from LowFat XML trees¶

# set some globals
monad=1
CollectedItems= 0

# process books in order
for bo, bookinfo in bo2book.items():
  CollectedItems=0
  SentenceNumber=0
  WordGroupNumber=0
  full_df=pd.DataFrame({})
  book_long=bookinfo[0]
  booknum=bookinfo[1]
  book_short=bookinfo[2]
  InputFile = os.path.join(XmlDir, f'{bo}.xml')
  OutputFile = os.path.join(PklDir, f'{bo}.pkl')
  print(f'Processing {book_long} at {InputFile}')

  # send xml document to parsing process
  tree = ET.parse(InputFile)
  # Now add all the parent info to the nodes in the xtree [important!]
  addParentInfo(tree.getroot())
  start_time = time.time()
    
  # walk over all the XML data
  for elem in tree.iter():
    if elem.tag == 'sentence':
        # add running number to 'sentence' tags
        SentenceNumber+=1
        elem.set('SN', SentenceNumber)
    if elem.tag == 'wg':
        # add running number to 'wg' tags
        WordGroupNumber+=1
        elem.set('WGN', WordGroupNumber)
    if elem.tag == 'w':
        # all nodes containing words are tagged with 'w'
     
        # show  progress on screen
        CollectedItems+=1
        if (CollectedItems%100==0): print (".",end='')
        
        #Leafref will contain list with book, chapter verse and wordnumber
        Leafref = re.sub(r'[!: ]'," ", elem.attrib.get('ref')).split()
        
        #push value for monad to element tree 
        elem.set('monad', monad)
        monad+=1
        
        # add some important computed data to the leaf
        elem.set('LeafName', elem.tag)
        elem.set('word', elem.text)
        elem.set('book_long', book_long)
        elem.set('booknum', int(booknum))
        elem.set('book_short', book_short)
        elem.set('chapter', int(Leafref[1]))
        elem.set('verse', int(Leafref[2]))
       
        # folling code will trace down parents upto the tree and store found attributes
        parentnode=getParent(elem)
        index=0
        while (parentnode):
           index+=1
           elem.set('Parent{}Name'.format(index),      parentnode.tag)
           elem.set('Parent{}Type'.format(index),      parentnode.attrib.get('type'))
           elem.set('Parent{}Appos'.format(index),     parentnode.attrib.get('appositioncontainer'))
           elem.set('Parent{}Class'.format(index),     parentnode.attrib.get('class'))
           elem.set('Parent{}Rule'.format(index),      parentnode.attrib.get('rule'))
           elem.set('Parent{}Role'.format(index),      parentnode.attrib.get('role'))
           elem.set('Parent{}Cltype'.format(index),    parentnode.attrib.get('cltype'))
           elem.set('Parent{}Unit'.format(index),      parentnode.attrib.get('unit'))
           elem.set('Parent{}Junction'.format(index),  parentnode.attrib.get('junction'))
           elem.set('Parent{}SN'.format(index),        parentnode.attrib.get('SN'))
           elem.set('Parent{}WGN'.format(index),       parentnode.attrib.get('WGN'))
           currentnode=parentnode
           parentnode=getParent(currentnode)      
        elem.set('parents', int(index))
        
        #this will push all elements found in the tree into a DataFrame
        df=pd.DataFrame(elem.attrib, index={monad})
        full_df=pd.concat([full_df,df])
        
  #store the resulting DataFrame per book into a pickle file for further processing
  df = df.convert_dtypes(convert_string=True)
    
  # sort by s=id
  sortkey='{http://www.w3.org/XML/1998/namespace}id'
  full_df.rename(columns={sortkey: 'id'}, inplace=True)
  full_df.sort_values(by=['id'])

  output = open(r"{}".format(OutputFile), 'wb')
  pickle.dump(full_df, output)
  output.close()
  print("\nFound ",CollectedItems, " items in  %s seconds\n" % (time.time() - start_time))  
        

TF = Fabric(locations=BaseDir, silent=False)
cv = CV(TF)
version = "0.1.7 (added role info to each word)"

###############################################
#   Common helper functions                   #
###############################################

#Function to prevent errors during conversion due to missing data
def sanitize(input):
    if isinstance(input, float): return ''
    if isinstance(input, type(None)): return ''
    else: return (input)


# Function to expand the syntactic categories of words or wordgroup
# See also "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" 
# page 5&6 (section 2.4 Syntactic Categories at Clause Level)
def ExpandRole(input):
    if input=="adv": return 'Adverbial'
    if input=="io":  return 'Indirect Object'
    if input=="o":   return 'Object'
    if input=="o2":  return 'Second Object'
    if input=="s":   return 'Subject'
    if input=="p":   return 'Predicate'
    if input=="v":   return 'Verbal'
    if input=="vc":  return 'Verbal Copula'
    if input=='aux': return 'Auxiliar'
    return ''

# Function to expantion of Part of Speech labels. See also the description in 
# "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 6&7
# (2.2. Syntactic Categories at Word Level: Part of Speech Labels)
def ExpandSP(input):
    if input=='adj':  return 'adjective'
    if input=='conj': return 'conjunction'
    if input=='det':  return 'determiner' 
    if input=='intj': return 'interjection' 
    if input=='noun': return 'noun' 
    if input=='num':  return 'numeral' 
    if input=='prep': return 'preposition' 
    if input=='ptcl': return 'particle' 
    if input=='pron': return 'pronoun' 
    if input=='verb': return 'verb' 
    return ''

###############################################
#          The director routine               #
###############################################

def director(cv):
    
    ###############################################
    #   Innitial setup of data etc.               #
    ###############################################
    NoneType = type(None)      # needed as tool to validate certain data
    IndexDict = {}             # init an empty dictionary
    WordGroupDict={}    # init a dummy dictionary
    PrevWordGroupSet = WordGroupSet = []
    PrevWordGroupList = WordGroupList = []
    RootWordGroup = 0
    WordNumber=FoundWords=WordGroupTrack=0
    # The following is required to recover succesfully from an abnormal condition
    # in the LowFat tree data where a <wg> element is labeled as <error>
    # this number is arbitrary but should be high enough not to clash with 'real' WG numbers
    DummyWGN=200000  
    
    for bo,bookinfo in bo2book.items(): 
        
        ###############################################
        #   start of section executed for each book   #
        ###############################################
        
        # note: bookinfo is a list! Split the data
        Book        = bookinfo[0]  
        BookNumber  = int(bookinfo[1])
        BookShort   = bookinfo[2]
        BookLoc     = os.path.join(PklDir, f'{bo}.pkl') 
        
        
        # load  data for this book into a dataframe. 
        # make sure wordorder is correct
        print(f'\tWe are loading {BookLoc}...')
        pkl_file = open(BookLoc, 'rb')
        df_unsorted = pickle.load(pkl_file)
        pkl_file.close()
        df=df_unsorted.sort_values(by=['id'])
 
        
        # set up nodes for new book
        ThisBookPointer = cv.node('book')
        cv.feature(ThisBookPointer, book=Book, booknumber=BookNumber, bookshort=BookShort)
        
        ThisChapterPointer = cv.node('chapter')
        cv.feature(ThisChapterPointer, chapter=1)
        PreviousChapter=1
        
        ThisVersePointer = cv.node('verse')
        cv.feature(ThisVersePointer, verse=1)
        PreviousVerse=1
        
        ThisSentencePointer = cv.node('sentence')
        cv.feature(ThisSentencePointer, verse=1)
        PreviousSentence=1                


        '''
        fill dictionary of column names for this book 
        sort to ensure proper wordorder
        '''
        ItemsInRow=1
        for itemname in df.columns.to_list():
            IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
            ItemsInRow+=1
        df.sort_values(by=['id'])
        

        ###############################################
        # Iterate through words and construct objects #
        ###############################################
        
        for row in df.itertuples():
            WordNumber += 1
            FoundWords +=1
            
            # Detect and act upon changes in sentences, verse and chapter 
            # the order of terminating and creating the nodes is critical: 
            # close verse - close chapter - open chapter - open verse 
            NumberOfParents = row[IndexDict.get("i_parents")]
            ThisSentence=int(row[IndexDict.get("i_Parent{}SN".format(NumberOfParents-1))])
            ThisVerse = sanitize(row[IndexDict.get("i_verse")])
            ThisChapter = sanitize(row[IndexDict.get("i_chapter")])
            if (ThisSentence!=PreviousSentence):
                #cv.feature(ThisSentencePointer, statdata?)
                cv.terminate(ThisSentencePointer)
                
            if (ThisVerse!=PreviousVerse):
                #cv.feature(ThisVersePointer, statdata?)
                cv.terminate(ThisVersePointer)

            if (ThisChapter!=PreviousChapter):
                #cv.feature(ThisChapterPointer, statdata?)
                cv.terminate(ThisChapterPointer)
                PreviousChapter = ThisChapter
                ThisChapterPointer = cv.node('chapter')
                cv.feature(ThisChapterPointer, chapter=ThisChapter)
                
            if (ThisVerse!=PreviousVerse):
                PreviousVerse = ThisVerse  
                ThisVersePointer = cv.node('verse')
                cv.feature(ThisVersePointer, verse=ThisVerse, chapter=ThisChapter)
                
            if (ThisSentence!=PreviousSentence):
                PreviousSentence=ThisSentence
                ThisSentencePointer = cv.node('sentence')
                cv.feature(ThisSentencePointer, verse=ThisVerse, chapter=ThisChapter)       

        
            ###############################################
            #         analyze and process <WG> tags       #
            ###############################################
                    
            # get number of parent nodes (this differs per word)
            PrevWordGroupList=WordGroupList
            WordGroupList=[]  # stores current active WordGroup numbers

            for i in range(NumberOfParents-2,0,-1): # important: reversed itteration!
                _WGN=row[IndexDict.get("i_Parent{}WGN".format(i))]
                if isinstance(_WGN, type(None)): 
                    # handling conditions where XML data has <error role="err_clause-complex-met-no-conditionsClCl2"> e.g. Acts 26:12
                    # to recover, we need to create a dummy WG with a sufficient high WGN so it can never match any real WGN. 
                    WGN=DummyWGN
                else:
                    WGN=int(_WGN)
                if WGN!='':
                     WordGroupList.append(WGN)
                     WordGroupDict[(WGN,0)]=WGN
                     WordGroupDict[(WGN,1)]=sanitize(row[IndexDict.get("i_Parent{}Rule".format(i))])
                     WordGroupDict[(WGN,2)]=sanitize(row[IndexDict.get("i_Parent{}Cltype".format(i))])
                     WordGroupDict[(WGN,3)]=sanitize(row[IndexDict.get("i_Parent{}Junction".format(i))])
                     WordGroupDict[(WGN,6)]=sanitize(row[IndexDict.get("i_Parent{}Class".format(i))])
                     WordGroupDict[(WGN,7)]=sanitize(row[IndexDict.get("i_Parent{}Role".format(i))])
                     WordGroupDict[(WGN,8)]=sanitize(row[IndexDict.get("i_Parent{}Type".format(i))])
                     WordGroupDict[(WGN,9)]=sanitize(row[IndexDict.get("i_Parent{}Appos".format(i))])  
                     WordGroupDict[(WGN,10)]=NumberOfParents-1-i  # = number of parent wordgroups     
            if not PrevWordGroupList==WordGroupList:
                if RootWordGroup != WordGroupList[0]:
                    RootWordGroup = WordGroupList[0]
                    SuspendableWordGoupList = []
                    # we have a new sentence. rebuild suspendable wordgroup list
                    # some cleaning of data may be added here to save on memmory... 
                    #for k in range(6): del WordGroupDict[item,k]
                for item in reversed(PrevWordGroupList):
                    if (item not in WordGroupList):
                         # CLOSE/SUSPEND CASE
                         SuspendableWordGoupList.append(item)
                         cv.terminate(WordGroupDict[item,4])
                for item in WordGroupList:
                    if (item not in PrevWordGroupList):
                        if (item in SuspendableWordGoupList):
                              # RESUME CASE
                              #print ('\n resume: '+str(item),end=' ')
                              cv.resume(WordGroupDict[(item,4)])
                        else:
                              # CREATE CASE
                              #print ('\n create: '+str(item),end=' ')
                              WordGroupDict[(item,4)]=cv.node('wg')
                              WordGroupDict[(item,5)]=WordGroupTrack
                              WordGroupTrack += 1
                              cv.feature(WordGroupDict[(item,4)], wordgroup=WordGroupDict[(item,0)], junction=WordGroupDict[(item,3)], 
                                         clausetype=WordGroupDict[(item,2)], rule=WordGroupDict[(item,1)], wgclass=WordGroupDict[(item,6)], 
                                         wgrole=WordGroupDict[(item,7)],wgrolelong=ExpandRole(WordGroupDict[(item,7)]),
                                         wgtype=WordGroupDict[(item,8)],appos=WordGroupDict[(item,8)],wglevel=WordGroupDict[(item,10)])

               
     
            # These roles are performed either by a WG or just a single word.
            Role=row[IndexDict.get("i_role")]
            ValidRoles=["adv","io","o","o2","s","p","v","vc","aux"]
            DistanceToRoleClause=0
            if isinstance (Role,str) and Role in ValidRoles: 
                # role is assign to this word (uniqely)
                WordRole=Role
                WordRoleLong=ExpandRole(WordRole)
            else:
                # role details needs to be taken from some uptree wordgroup 
                WordRole=WordRoleLong=''
                for i in range(1,NumberOfParents-1):
                    Role = row[IndexDict.get("i_Parent{}Role".format(i))]
                    if isinstance (Role,str) and Role in ValidRoles: 
                        WordRole=Role
                        WordRoleLong=ExpandRole(WordRole)
                        DistanceToRoleClause=i
                        break
      

            ###############################################
            #         analyze and process <W> tags        #
            ###############################################
                
            # determine syntactic categories at word level. 
            PartOfSpeech=sanitize(row[IndexDict.get("i_class")])
            PartOfSpeechFull=ExpandSP(PartOfSpeech)
                    
            # some attributes are not present inside some (small) books. The following is to prevent exceptions.
            degree='' 
            if 'i_degree'  in IndexDict: degree=sanitize(row[IndexDict.get("i_degree")]) 
            subjref=''
            if 'i_subjref' in IndexDict: subjref=sanitize(row[IndexDict.get("i_subjref")]) 

                    
            # create the word slots
            this_word = cv.slot()
            cv.feature(this_word, 
                    after=         sanitize(row[IndexDict.get("i_after")]),
                    id=            sanitize(row[IndexDict.get("i_id")]),
                    unicode=       sanitize(row[IndexDict.get("i_unicode")]),
                    word=          sanitize(row[IndexDict.get("i_word")]),
                    monad=         sanitize(row[IndexDict.get("i_monad")]),
                    orig_order=    FoundWords,
                    book_long=     sanitize(row[IndexDict.get("i_book_long")]),
                    booknumber=    BookNumber,
                    bookshort=     sanitize(row[IndexDict.get("i_book_short")]),
                    chapter=       ThisChapter,
                    ref=           sanitize(row[IndexDict.get("i_ref")]),
                    sp=            PartOfSpeech,
                    sp_full=       PartOfSpeechFull,
                    verse=         ThisVerse,
                    sentence=      ThisSentence,
                    normalized=    sanitize(row[IndexDict.get("i_normalized")]),
                    morph=         sanitize(row[IndexDict.get("i_morph")]),
                    strongs=       sanitize(row[IndexDict.get("i_strong")]),
                    lex_dom=       sanitize(row[IndexDict.get("i_domain")]),
                    ln=            sanitize(row[IndexDict.get("i_ln")]),
                    gloss=         sanitize(row[IndexDict.get("i_gloss")]),
                    gn=            sanitize(row[IndexDict.get("i_gender")]),
                    nu=            sanitize(row[IndexDict.get("i_number")]),
                    case=          sanitize(row[IndexDict.get("i_case")]),
                    lemma=         sanitize(row[IndexDict.get("i_lemma")]),
                    person=        sanitize(row[IndexDict.get("i_person")]),
                    mood=          sanitize(row[IndexDict.get("i_mood")]),
                    tense=         sanitize(row[IndexDict.get("i_tense")]),
                    number=        sanitize(row[IndexDict.get("i_number")]),
                    voice=         sanitize(row[IndexDict.get("i_voice")]),
                    degree=        degree,
                    type=          sanitize(row[IndexDict.get("i_type")]),
                    reference=     sanitize(row[IndexDict.get("i_ref")]),     
                    subj_ref=      subjref,
                    nodeID=        sanitize(row[1]),                        #this is a fixed position in dataframe
                    wordrole=      WordRole,
                    wordrolelong=  WordRoleLong,
                    wordlevel=     NumberOfParents-1,
                    roleclausedistance = DistanceToRoleClause
                    )
            cv.terminate(this_word)

        
        '''
        wrap up the book. At the end of the book we need to close all nodes in proper order.
        '''   
        for item in WordGroupList:
            #cv.feature(WordGroupDict[(item,4)], add some stats?)
            cv.terminate(WordGroupDict[item,4])
        #cv.feature(ThisSentencePointer, statdata?)
        cv.terminate(ThisSentencePointer)
        #cv.feature(ThisVersePointer, statdata?)
        cv.terminate(ThisVersePointer)
        #cv.feature(ThisChapterPonter, statdata?)
        cv.terminate(ThisChapterPointer)      
        #cv.feature(ThisBookPointer, statdata?)
        cv.terminate(ThisBookPointer)

        # clear dataframe for this book, clear the index dictionary
        del df
        IndexDict.clear()
        gc.collect()
        
        ###############################################
        #    end of section executed for each book    #
        ###############################################

    ###############################################
    #      end of director function               #
    ###############################################
        
###############################################
#            Output definitions               #
###############################################
        
slotType = 'word'  
otext = {  # dictionary of config data for sections and text formats
        'fmt:text-orig-full':'{word}{after}',
        'sectionTypes':'book,chapter,verse',
        'sectionFeatures':'book,chapter,verse',
        'structureFeatures': 'book,chapter,verse',
        'structureTypes': 'book,chapter,verse',
        }

# configure metadata
generic = {  # dictionary of metadata meant for all features
         'Name': 'Greek New Testament (NA1904)',
         'Version': '1904',
         'Editors': 'Nestle',
         'Data source': 'MACULA Greek Linguistic Datasets, available at https://github.com/Clear-Bible/macula-greek/tree/main/Nestle1904/lowfat',
         'Availability': 'Creative Commons Attribution 4.0 International (CC BY 4.0)', 
         'Converter_author': 'Tony Jurg, Vrije Universiteit Amsterdam, Netherlands', 
         'Converter_execution': 'Tony Jurg, Vrije Universiteit Amsterdam, Netherlands', 
         'Convertor_source': 'https://github.com/tonyjurg/n1904_lft',
         'Converter_version': '{}'.format(version),
         'TextFabric version': '{}'.format(VERSION)  #imported from tf.parameters
         }

# set of integer valued feature names
intFeatures = {  
         'booknumber',
         'chapter',
         'verse',
         'sentence',
         'wordgroup',
         'orig_order',
         'monad',
         'wglevel'
         }

# per feature dicts with metadata
featureMeta = {  
                 'after':       {'description': 'Characters (eg. punctuations) following the word'},
                 'id':          {'description': 'id of the word'},
                 'book':        {'description': 'Book'},
                 'book_long':   {'description': 'Book name (fully spelled out)'},
                 'booknumber':  {'description': 'NT book number (Matthew=1, Mark=2, ..., Revelation=27)'},
                 'bookshort':   {'description': 'Book name (abbreviated)'},
                 'chapter':     {'description': 'Chapter number inside book'},
                 'verse':       {'description': 'Verse number inside chapter'},
                 'sentence':    {'description': 'Sentence number (counted per chapter)'},
                 'type':        {'description': 'Wordgroup type information (verb, verbless, elided, minor, etc.)'},
                 'rule':        {'description': 'Wordgroup rule information '},
                 'orig_order':  {'description': 'Word order within corpus (per book)'},
                 'monad':       {'description': 'Monad (currently: order of words in XML tree file!)'},
                 'word':        {'description': 'Word as it appears in the text (excl. punctuations)'},
                 'unicode':     {'description': 'Word as it arears in the text in Unicode (incl. punctuations)'},
                 'ref':         {'description': 'ref Id'},
                 'sp':          {'description': 'Part of Speech (abbreviated)'},
                 'sp_full':     {'description': 'Part of Speech (long description)'}, 
                 'normalized':  {'description': 'Surface word stripped of punctations'},
                 'lemma':       {'description': 'Lexeme (lemma)'},
                 'morph':       {'description': 'Morphological tag (Sandborg-Petersen morphology)'},
                                 # see also discussion on relation between lex_dom and ln 
                                 # @ https://github.com/Clear-Bible/macula-greek/issues/29
                 'lex_dom':     {'description': 'Lexical domain according to Semantic Dictionary of Biblical Greek, SDBG (not present everywhere?)'},
                 'ln':          {'description': 'Lauw-Nida lexical classification (not present everywhere?)'},
                 'strongs':     {'description': 'Strongs number'},
                 'gloss':       {'description': 'English gloss'},
                 'gn':          {'description': 'Gramatical gender (Masculine, Feminine, Neuter)'},
                 'nu':          {'description': 'Gramatical number (Singular, Plural)'},
                 'case':        {'description': 'Gramatical case (Nominative, Genitive, Dative, Accusative, Vocative)'},
                 'person':      {'description': 'Gramatical person of the verb (first, second, third)'},
                 'mood':        {'description': 'Gramatical mood of the verb (passive, etc)'},
                 'tense':       {'description': 'Gramatical tense of the verb (e.g. Present, Aorist)'},
                 'number':      {'description': 'Gramatical number of the verb'},
                 'voice':       {'description': 'Gramatical voice of the verb'},
                 'degree':      {'description': 'Degree (e.g. Comparitative, Superlative)'},
                 'type':        {'description': 'Gramatical type  of noun or pronoun (e.g. Common, Personal)'},
                 'reference':   {'description': 'Reference (to nodeID in XML source data, not yet post-processes)'},
                 'subj_ref':    {'description': 'Subject reference (to nodeID in XML source data, not yet post-processes)'},
                 'nodeID':      {'description': 'Node ID (as in the XML source data, not yet post-processes)'},
                 'junction':    {'description': 'Junction data related to a wordgroup'},
                 'wordgroup':   {'description': 'Wordgroup number (counted per book)'},
                 'wgclass':     {'description': 'Class of the wordgroup ()'},
                 'wgrole':      {'description': 'Role of the wordgroup (abbreviated)'},
                 'wgrolelong':  {'description': 'Role of the wordgroup (abbreviated)'},
                 'wordrole':    {'description': 'Role of the word (abbreviated)'},
                 'wordrolelong':{'description': 'Role of the word (full)'},
                 'wgtype':      {'description': 'Wordgroup type details'},
                 'clausetype':  {'description': 'Clause type details'},
                 'appos':       {'description': 'Apposition details'},
                 'wglevel':     {'description': 'number of parent wordgroups for a wordgroup'},
                 'wordlevel':   {'description': 'number of parent wordgroups for a word'},
                 'roleclausedistance': {'description': 'distance to wordgroup defining the role of this word'}
                 }


###############################################
#            the main function                #
###############################################

good = cv.walk(
    director,
    slotType,
    otext=otext,
    generic=generic,
    intFeatures=intFeatures,
    featureMeta=featureMeta,
    warn=True,
    force=True
)

if good:
  print ("done")

# Loading-the-New-Testament-Text-Fabric (add a specific version, eg. 0.1.2)
NA = use ("tonyjurg/n1904_lft", version="0.1.7", hoist=globals())

N.otypeRank

T.formats

Search0 = '''
book book=Matthew
  chapter chapter=1
     
     verse 
'''
Search0 = NA.search(Search0)
NA.show(Search0, start=1, end=21, condensed=True, extraFeatures={'sp','gloss','wordrole', 'wglevel', 'wordlevel','roleclausedistance'},  suppress={'chapter'}, withNodes=False)