Creating Text-Fabric from LowFat XML trees¶

# set some globals
monad=1
CollectedItems= 0

# process books in order
for bo, bookinfo in bo2book.items():
  CollectedItems=0
  SentenceNumber=0
  WordGroupNumber=0
  full_df=pd.DataFrame({})
  book_long=bookinfo[0]
  booknum=bookinfo[1]
  book_short=bookinfo[2]
  InputFile = os.path.join(XmlDir, f'{bo}.xml')
  OutputFile = os.path.join(PklDir, f'{bo}.pkl')
  print(f'Processing {book_long} at {InputFile}')

  # send xml document to parsing process
  tree = ET.parse(InputFile)
  # Now add all the parent info to the nodes in the xtree [important!]
  addParentInfo(tree.getroot())
  start_time = time.time()
    
  # walk over all the XML data
  for elem in tree.iter():
    if elem.tag == 'sentence':
        # add running number to 'sentence' tags
        SentenceNumber+=1
        elem.set('SN', SentenceNumber)
    if elem.tag == 'wg':
        # add running number to 'wg' tags
        WordGroupNumber+=1
        elem.set('WGN', WordGroupNumber)
    if elem.tag == 'w':
        # all nodes containing words are tagged with 'w'
     
        # show  progress on screen
        CollectedItems+=1
        if (CollectedItems%100==0): print (".",end='')
        
        #Leafref will contain list with book, chapter verse and wordnumber
        Leafref = re.sub(r'[!: ]'," ", elem.attrib.get('ref')).split()
        
        #push value for monad to element tree 
        elem.set('monad', monad)
        monad+=1
        
        # add some important computed data to the leaf
        elem.set('LeafName', elem.tag)
        elem.set('word', elem.text)
        elem.set('book_long', book_long)
        elem.set('booknum', int(booknum))
        elem.set('book_short', book_short)
        elem.set('chapter', int(Leafref[1]))
        elem.set('verse', int(Leafref[2]))
       
        # folling code will trace down parents upto the tree and store found attributes
        parentnode=getParent(elem)
        index=0
        while (parentnode):
           index+=1
           elem.set('Parent{}Name'.format(index),      parentnode.tag)
           elem.set('Parent{}Type'.format(index),      parentnode.attrib.get('type'))
           elem.set('Parent{}Appos'.format(index),     parentnode.attrib.get('appositioncontainer'))
           elem.set('Parent{}Class'.format(index),     parentnode.attrib.get('class'))
           elem.set('Parent{}Rule'.format(index),      parentnode.attrib.get('rule'))
           elem.set('Parent{}Role'.format(index),      parentnode.attrib.get('role'))
           elem.set('Parent{}Cltype'.format(index),    parentnode.attrib.get('cltype'))
           elem.set('Parent{}Unit'.format(index),      parentnode.attrib.get('unit'))
           elem.set('Parent{}Junction'.format(index),  parentnode.attrib.get('junction'))
           elem.set('Parent{}SN'.format(index),        parentnode.attrib.get('SN'))
           elem.set('Parent{}WGN'.format(index),       parentnode.attrib.get('WGN'))
           currentnode=parentnode
           parentnode=getParent(currentnode)      
        elem.set('parents', int(index))
        
        #this will push all elements found in the tree into a DataFrame
        df=pd.DataFrame(elem.attrib, index={monad})
        full_df=pd.concat([full_df,df])
        
  #store the resulting DataFrame per book into a pickle file for further processing
  df = df.convert_dtypes(convert_string=True)
    
  # sort by s=id
  sortkey='{http://www.w3.org/XML/1998/namespace}id'
  full_df.rename(columns={sortkey: 'id'}, inplace=True)
  full_df.sort_values(by=['id'])

  output = open(r"{}".format(OutputFile), 'wb')
  pickle.dump(full_df, output)
  output.close()
  print("\nFound ",CollectedItems, " items in  %s seconds\n" % (time.time() - start_time))  
        

TF = Fabric(locations=BaseDir, silent=False)
cv = CV(TF)
version = "0.1.3 (recursive clause generation)"


def sanitize(input):
    if isinstance(input, float): return ''
    if isinstance(input, type(None)): return ''
    else: return (input)
    



def director(cv):
    
 
    NoneType = type(None)      # needed as tool to validate certain data
    prev_book = "Matthew"      # start at first book
    IndexDict = {}             # init an empty dictionary
    
    Arrays2Dump=200
    DumpedArrays=0
    ClauseDict={}    # init a dummy dictionary
    PrevClauseSet = []
    PrevClauseList = []

    for bo,bookinfo in bo2book.items():
        
        '''
        load all data into a dataframe
        process books in order (bookinfo is a list!)
        '''   
        book=bookinfo[0]  
        booknum=int(bookinfo[1])
        book_short=bookinfo[2]
        book_loc = os.path.join(PklDir, f'{bo}.pkl') 
        
        print(f'\tWe are loading {book_loc}...')
        pkl_file = open(book_loc, 'rb')
        df_unsorted = pickle.load(pkl_file)
        pkl_file.close()
        df=df_unsorted.sort_values(by=['id'])
 
    
        FoundWords=0
        phrasefunction='TBD'
        phrasefunction_long='TBD'
        this_clausetype="unknown"  #just signal a not found case
        this_clauserule="unknown"
        this_clauseconst="unknown"
        prev_clausetype = "unknown"
        this_phrasetype="unknown"       #just signal a not found case
        this_phrasefunction="unknown"
        this_phrasefunction_long="unknown"
        this_role="unknown"
        ClausesSuspended = False
        PreSuspendClauseList = {}
        phrase_resume = False
        ClosedPhrase=''
        PhraseToRestoreTo=''
        phrase_close=False
        
        
        prev_chapter = int(1)      # start at 1
        prev_verse = int(1)        # start at 1
        prev_sentence = int(1)     # start at 1
        prev_clause = int(1)       # start at 1
        prev_phrase = int(1)       # start at 1
        prev_role = int(1)         # start at 1
        
        # ClauseArrayNumbers contains the active clause numbers (wg in the LowFatTree data)
        

     
        # reset/load the following initial variables (we are at the start of a new book)
        sentence_track = clause_track = clauseconst_track = phrase_track = 1
        sentence_done = clause_done = clauseconst_done = phrase_done = verse_done = chapter_done = book_done = False
        PhraseInterupted = False
        wrdnum = 0 # start at 0

        # fill dictionary of column names for this book 
        ItemsInRow=1
        for itemname in df.columns.to_list():
            IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
            ItemsInRow+=1
            
        df.sort_values(by=['id'])

        '''
        Walks through the texts and triggers
        slot and node creation events.
        '''
        
        # iterate through words and construct objects
        for row in df.itertuples():
            wrdnum += 1
            FoundWords +=1
            
          
            '''
            First get all the relevant information from the dataframe
            '''   
            
            # get number of parent nodes
            parents = row[IndexDict.get("i_parents")]
            
      
            # get chapter and verse from the data
            chapter = sanitize(row[IndexDict.get("i_chapter")])
            verse = sanitize(row[IndexDict.get("i_verse")])
         

                    
            # experimental (decoding the clause structure)
            ClauseList=[]  # stores current active clauses (wg numbers)
            if True:
                for i in range(1,parents-1):
                    item = IndexDict.get("i_Parent{}Class".format(i))
                    if row[item]=="cl":
                        WGN=row[IndexDict.get("i_Parent{}WGN".format(i))]
                        ClauseList.append(WGN)
                        ClauseDict[(WGN,0)]=WGN
                        ClauseDict[(WGN,1)]=sanitize(row[IndexDict.get("i_Parent{}Rule".format(i))])
                        ClauseDict[(WGN,2)]=sanitize(row[IndexDict.get("i_Parent{}Cltype".format(i))])
                        ClauseDict[(WGN,3)]=sanitize(row[IndexDict.get("i_Parent{}Junction".format(i))])
                if not PrevClauseList==ClauseList:
                    #print ('\nPrevClauseDict ={',end='')
                    #for item in PrevClauseDict:
                    #    print (' ',item,'=',PrevClauseDict[item],end='')
                    #print ('}')
                    #print('\nClauseList=',ClauseList, 'PrevClauseList=',PrevClauseList, 'ClausesSuspended=',ClausesSuspended,end='')
                    #print('\nAction(s) = ',end=' ')
                    ResumePhrase=False
                    if ClausesSuspended==True:
                        for item in ClauseList:
                            try:
                                ItemToResume=ClauseDict[(item,4)]
                                #print (' resume: '+str(item),end=' ')
                                cv.resume(ClauseDict[(item,4)])
                                #ResumePhrase=True
                                PhraseToRestoreTo=prev_phrasefunction
                            except:
                                #print("This is the odd condition for wg=",item)
                                # CREATE CASE
                                #print ('\n create: '+str(item),end=' ')
                                ClauseDict[(item,4)]=cv.node('clause')
                                ClauseDict[(item,5)]=clause_track
                                clause_track += 1
                                #print ('link=',ClauseDict[(item,4)])
                                
                            

                    if not bool(ClauseList): # this means it is a part outside a clause
                        for item in PrevClauseList:
                            # SUSPEND CASE
                            #print (' suspend: '+str(item),end=' ')
                            cv.feature(ClauseDict[(item,4)], clause=ClauseDict[(item,5)], wg=ClauseDict[(item,0)], junction=ClauseDict[(item,3)], clausetype=ClauseDict[(item,2)], clauserule=ClauseDict[(item,1)])
                            cv.terminate(ClauseDict[(item,4)])
                        ClausesSuspended= True
                        PreSuspendClauseList = PrevClauseList
                    else:
                        for item in PrevClauseList:
                           if (item not in ClauseList):
                                # CLOSE CASE
                                #print ('\n close: '+str(item),end=' ')
                                cv.feature(ClauseDict[(item,4)], clause=ClauseDict[(item,5)], wg=ClauseDict[(item,0)], junction=ClauseDict[(item,3)], clausetype=ClauseDict[(item,2)], clauserule=ClauseDict[(item,1)])
                                cv.terminate(ClauseDict[item,4])
                                for k in range(6): del ClauseDict[item,k]
                        for item in ClauseList:
                           if (item not in PrevClauseList):
                                if not ClausesSuspended:
                                    # CREATE CASE
                                    #print ('\n create: '+str(item),end=' ')
                                    ClauseDict[(item,4)]=cv.node('clause')
                                    ClauseDict[(item,5)]=clause_track
                                    clause_track += 1
                                    #print ('link=',ClauseDict[(item,4)])
                        ClausesSuspended=False

                    PrevClauseList = ClauseList    
                    clause_done=True
                    #print ('\nClauseDict=',ClauseDict,end='')
                    #print ('\n\n',book,chapter,":",verse,'\t',row[IndexDict.get("i_word")]+row[IndexDict.get("i_after")],end='')

                #else:
                    #print (row[IndexDict.get("i_word")]+row[IndexDict.get("i_after")],end='')


     

            
            # determine syntactic categories of clause parts. See also the description in 
            # "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 5&6
            # (section 2.4 Syntactic Categories at Clause Level)
            prev_phrasefunction=this_phrasefunction
            prev_phrasefunction_long=this_phrasefunction_long
            role=row[IndexDict.get("i_role")]
            ValidRoles=["adv","io","o","o2","s","p","v","vc"]
            this_phrasefunction=''
            PhraseWGN=''
            if isinstance (role,str) and role in ValidRoles: 
                this_phrasefunction=role
            else:
                for i in range(1,parents-1):
                    role = row[IndexDict.get("i_Parent{}Role".format(i))]
                    if isinstance (role,str) and role in ValidRoles: 
                        this_phrasefunction=role
                        PhraseWGN=row[IndexDict.get("i_Parent{}WGN".format(i))]
                        break
            if this_phrasefunction!=prev_phrasefunction:
                #print ('\n',chapter,':',verse,'  this_phrasefunction=',this_phrasefunction,' prev_phrasefunction=',prev_phrasefunction)
                if phrase_close:
                     cv.resume(ClosedPhrase)
                     #print ('resume phrase',ClosedPhrase)
                if this_phrasefunction!='':
                     phrase_done = True
                else:
                     phrase_close = True
                     ClosedPhrase=this_phrase
                     PhraseToRestoreTo=prev_phrasefunction



            if   this_phrasefunction=="adv":  this_phrasefunction_long='Adverbial'
            elif this_phrasefunction=="io":   this_phrasefunction_long='Indirect Object'
            elif this_phrasefunction=="o":    this_phrasefunction_long='Object'
            elif this_phrasefunction=="o2":   this_phrasefunction_long='Second Object'
            elif this_phrasefunction=="s":    this_phrasefunction_long='Subject'
            elif this_phrasefunction=="p":    this_phrasefunction_long='Predicate'
            elif this_phrasefunction=="v":    this_phrasefunction_long='Verbal'
            elif this_phrasefunction=="vc":   this_phrasefunction_long='Verbal Copula'
            #print ('this_phrasefunction=',this_phrasefunction, end=" | ")

       
                
            '''
            determine if conditions are met to trigger some action 
            action will be executed after next word
            '''   
            
            # detect book boundary
            if prev_book != book:
                prev_book=book
                book_done = True
                chapter_done = True
                verse_done=True
                sentence_done = True
                clause_done = True
                phrase_done = True

            # detect chapter boundary
            if prev_chapter != chapter:
                chapter_done = True
                verse_done=True
                sentence_done = True
                clause_done = True
                phrase_done = True
                
            # detect verse boundary
            if prev_verse != verse:
                verse_done=True
                

            # determine syntactic categories at word level. See also the description in 
            # "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 6&7
            # (2.2. Syntactic Categories at Word Level: Part of Speech Labels)
            sp=sanitize(row[IndexDict.get("i_class")])
            if sp=='adj':
                sp_full='adjective'
            elif sp=='adj':
                sp_full='adjective'
            elif sp=='conj':
                sp_full='conjunction'
            elif sp=='det':
                sp_full='determiner' 
            elif sp=='intj':
                sp_full='interjection' 
            elif sp=='noun':
                sp_full='noun' 
            elif sp=='num':
                sp_full='numeral' 
            elif sp=='prep':
                sp_full='preposition' 
            elif sp=='ptcl':
                sp_full='particle' 
            elif sp=='pron':
                sp_full='pronoun' 
            elif sp=='verb':
                sp_full='verb' 
                    
                    
            # Manage first word per book
            if wrdnum==1: 
                prev_phrasetype=this_phrasetype
                prev_phrasefunction=this_phrasefunction
                prev_phrasefunction_long=this_phrasefunction_long
                prev_clauserule=this_clauserule
                book_done = chapter_done = verse_done = phrase_done = clause_done = sentence_done = False
                # create the first set of nodes
                this_book = cv.node('book')
                cv.feature(this_book, book=prev_book)
                this_chapter = cv.node('chapter')
                this_verse = cv.node('verse')
                this_sentence = cv.node('sentence')
                #this_clause = cv.node('clause')
                this_phrase = cv.node('phrase')
                #print ('new phrase',this_phrase)
                sentence_track += 1
                #clause_track += 1
                phrase_track += 1

                       
            
            '''
             -- handle TF events --
            Determine what actions need to be done if proper condition is met.
            '''   

            # act upon end of phrase (close)
            if phrase_done or phrase_close:
                cv.feature(this_phrase, phrase=prev_phrase, phrasetype=prev_phrasetype, phrasefunction=prev_phrasefunction, phrasefunction_long=prev_phrasefunction_long,wg=PhraseWGN)
                cv.terminate(this_phrase)
                #print ('terminate phrase',this_phrase,':',prev_phrasefunction,'\n')
                ClosedPhrase=this_phrase
                PhraseToRestoreTo=prev_phrasefunction
                phrase_close=False


                
            #    cv.feature(this_clause, clause=prev_clause, clausetype=prev_clausetype, clauserule=prev_clauserule)
            #    cv.terminate(this_clause)
 
            # act upon end of sentence (close)
            if sentence_done:
                cv.feature(this_sentence, sentence=prev_sentence)
                cv.terminate(this_sentence)
                
            # act upon end of verse (close)
            if verse_done:
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)
                prev_verse = verse  

            # act upon end of chapter (close)
            if chapter_done:
                cv.feature(this_chapter, chapter=prev_chapter)
                cv.terminate(this_chapter)
                prev_chapter = chapter

            # act upon end of book (close and open new)
            if book_done:
                cv.terminate(this_book)
                this_book = cv.node('book')
                cv.feature(this_book, book=book) 
                prev_book = book
                wrdnum = 1
                phrase_track = 1
                clause_track = 1
                sentence_track = 1
                book_done = False
                
            # start of chapter (create new)
            if chapter_done:
                this_chapter = cv.node('chapter')
                chapter_done = False
                
            # start of verse (create new)
            if verse_done:
                this_verse = cv.node('verse')
                verse_done = False  
                
            # start of sentence (create new)
            if sentence_done:
                this_sentence= cv.node('sentence')
                prev_sentence = sentence_track
                sentence_track += 1
                sentence_done = False

                
            # start of phrase (create new)
            if phrase_done and not ResumePhrase:
                this_phrase = cv.node('phrase')
                #print ('new phrase',this_phrase)
                prev_phrase = phrase_track
                prev_phrasefunction=this_phrasefunction
                prev_phrasefunction_long=this_phrasefunction_long
                phrase_track += 1
                phrase_done = False
                
      
            # Detect boundaries of sentences
            text=sanitize(row[IndexDict.get("i_after")])[-1:]
            if text == "." :        
                sentence_done = True
                #phrase_done = True
            #if text == ";" or text == ",":
            #    phrase_done = True       
                    
            
            '''
            -- create word nodes --
            '''   
                    
            # some attributes are not present inside some (small) books. The following is to prevent exceptions.
            degree='' 
            if 'i_degree' in IndexDict: 
                   degree=sanitize(row[IndexDict.get("i_degree")]) 
            subjref=''
            if 'i_subjref' in IndexDict:
                   subjref=sanitize(row[IndexDict.get("i_subjref")]) 
                    
            #print (chapter,':',verse,"  ",row[IndexDict.get("i_word")],' - ',row[IndexDict.get("i_gloss")],' - ', sp,end='\n')

            # make word object
            this_word = cv.slot()
            cv.feature(this_word, 
                    after=sanitize(row[IndexDict.get("i_after")]),
                    id=sanitize(row[IndexDict.get("i_id")]),
                    unicode=sanitize(row[IndexDict.get("i_unicode")]),
                    word=sanitize(row[IndexDict.get("i_word")]),
                    monad=sanitize(row[IndexDict.get("i_monad")]),
                    orig_order=FoundWords,
                    book_long=sanitize(row[IndexDict.get("i_book_long")]),
                    booknum=booknum,
                    book_short=sanitize(row[IndexDict.get("i_book_short")]),
                    chapter=chapter,
                    ref=sanitize(row[IndexDict.get("i_ref")]),
                    sp=sanitize(sp),
                    sp_full=sanitize(sp_full),
                    verse=verse,
                    sentence=sanitize(prev_sentence),
                    clause=sanitize(prev_clause),
                    phrase=sanitize(prev_phrase),
                    normalized=sanitize(row[IndexDict.get("i_normalized")]),
                    morph=sanitize(row[IndexDict.get("i_morph")]),
                    strongs=sanitize(row[IndexDict.get("i_strong")]),
                    lex_dom=sanitize(row[IndexDict.get("i_domain")]),
                    ln=sanitize(row[IndexDict.get("i_ln")]),
                    gloss=sanitize(row[IndexDict.get("i_gloss")]),
                    gn=sanitize(row[IndexDict.get("i_gender")]),
                    nu=sanitize(row[IndexDict.get("i_number")]),
                    case=sanitize(row[IndexDict.get("i_case")]),
                    lemma=sanitize(row[IndexDict.get("i_lemma")]),
                    person=sanitize(row[IndexDict.get("i_person")]),
                    mood=sanitize(row[IndexDict.get("i_mood")]),
                    tense=sanitize(row[IndexDict.get("i_tense")]),
                    number=sanitize(row[IndexDict.get("i_number")]),
                    voice=sanitize(row[IndexDict.get("i_voice")]),
                    degree=degree,
                    type=sanitize(row[IndexDict.get("i_type")]),
                    reference=sanitize(row[IndexDict.get("i_ref")]),     
                    subj_ref=subjref,
                    nodeID=sanitize(row[1])                              #this is a fixed position.
                    )
            cv.terminate(this_word)

        
        '''
        -- wrap up the book --
        '''   
        
        # close all nodes (phrase, clause, sentence, verse, chapter and book)
        cv.feature(this_phrase, phrase=phrase_track, phrasetype=prev_phrasetype,phrasefunction=prev_phrasefunction,phrasefunction_long=prev_phrasefunction_long, wg=PhraseWGN)
        cv.terminate(this_phrase)
        #print ('terminate phrase',this_phrase,':',prev_phrasetype,'\n')

        
        #print (ClauseDict)
        for item in ClauseList:
            cv.feature(ClauseDict[(item,4)], clause=ClauseDict[(item,5)], clausetype=prev_clausetype, clauserule=prev_clauserule)
            cv.terminate(ClauseDict[item,4])
                                
                                
        cv.feature(this_sentence, sentence=prev_sentence)
        cv.terminate(this_sentence)
        cv.feature(this_verse, verse=prev_verse)
        cv.terminate(this_verse)
        cv.feature(this_chapter, chapter=prev_chapter)
        cv.terminate(this_chapter)
        cv.feature(this_book, book=prev_book)
        cv.terminate(this_book)
        
        # clear dataframe for this book 
        del df
        # clear the index dictionary
        IndexDict.clear()
        gc.collect()
        
        
'''
-- output definitions --
'''  
        
slotType = 'word'  # or whatever you choose
otext = {  # dictionary of config data for sections and text formats
        'fmt:text-orig-full':'{word}{after}',
        'sectionTypes':'book,chapter,verse',
        'sectionFeatures':'book,chapter,verse',
        'structureFeatures': 'book,chapter,verse',
        'structureTypes': 'book,chapter,verse',
        }

# configure metadata
generic = {  # dictionary of metadata meant for all features
         'Name': 'Greek New Testament (NA1904)',
         'Version': '1904',
         'Editors': 'Nestle',
         'Data source': 'MACULA Greek Linguistic Datasets, available at https://github.com/Clear-Bible/macula-greek/tree/main/Nestle1904/lowfat',
         'Availability': 'Creative Commons Attribution 4.0 International (CC BY 4.0)', 
         'Converter_author': 'Tony Jurg, Vrije Universiteit Amsterdam, Netherlands', 
         'Converter_execution': 'Tony Jurg, Vrije Universiteit Amsterdam, Netherlands', 
         'Convertor_source': 'https://github.com/tonyjurg/n1904_lft',
         'Converter_version': '{}'.format(version),
         'TextFabric version': '{}'.format(VERSION)  #imported from tf.parameters
         }

intFeatures = {  # set of integer valued feature names
         'booknum',
         'chapter',
         'verse',
         'sentence',
         'clause',
         'phrase',
         'orig_order',
         'monad'
         }

featureMeta = {  # per feature dicts with metadata
                 'after': {'description': 'Characters (eg. punctuations) following the word'},
                 'id': {'description': 'id of the word'},
                 'book': {'description': 'Book'},
                 'book_long': {'description': 'Book name (fully spelled out)'},
                 'booknum': {'description': 'NT book number (Matthew=1, Mark=2, ..., Revelation=27)'},
                 'book_short': {'description': 'Book name (abbreviated)'},
                 'chapter': {'description': 'Chapter number inside book'},
                 'verse': {'description': 'Verse number inside chapter'},
                 'sentence': {'description': 'Sentence number (counted per chapter)'},
                 'clause': {'description': 'Clause number (counted per chapter)'},
                 'clausetype' : {'description': 'Clause type information (verb, verbless, elided, minor, etc.)'},
                 'clauserule' : {'description': 'Clause rule information '},
                 'phrase' : {'description': 'Phrase number (counted per book)'},
                 'phrasetype' : {'description': 'Phrase type information'},
                 'phrasefunction' : {'description': 'Phrase function (abbreviated)'},
                 'phrasefunction_long' : {'description': 'Phrase function (long description)'},
                 'orig_order': {'description': 'Word order within corpus (per book)'},
                 'monad':{'description': 'Monad (currently: order of words in XML tree file!)'},
                 'word': {'description': 'Word as it appears in the text (excl. punctuations)'},
                 'unicode': {'description': 'Word as it arears in the text in Unicode (incl. punctuations)'},
                 'ref': {'description': 'ref Id'},
                 'sp': {'description': 'Part of Speech (abbreviated)'},
                 'sp_full': {'description': 'Part of Speech (long description)'}, 
                 'normalized': {'description': 'Surface word stripped of punctations'},
                 'lemma': {'description': 'Lexeme (lemma)'},
                 'morph': {'description': 'Morphological tag (Sandborg-Petersen morphology)'},
                 # see also discussion on relation between lex_dom and ln @ https://github.com/Clear-Bible/macula-greek/issues/29
                 'lex_dom': {'description': 'Lexical domain according to Semantic Dictionary of Biblical Greek, SDBG (not present everywhere?)'},
                 'ln': {'description': 'Lauw-Nida lexical classification (not present everywhere?)'},
                 'strongs': {'description': 'Strongs number'},
                 'gloss': {'description': 'English gloss'},
                 'gn': {'description': 'Gramatical gender (Masculine, Feminine, Neuter)'},
                 'nu': {'description': 'Gramatical number (Singular, Plural)'},
                 'case': {'description': 'Gramatical case (Nominative, Genitive, Dative, Accusative, Vocative)'},
                 'person': {'description': 'Gramatical person of the verb (first, second, third)'},
                 'mood': {'description': 'Gramatical mood of the verb (passive, etc)'},
                 'tense': {'description': 'Gramatical tense of the verb (e.g. Present, Aorist)'},
                 'number': {'description': 'Gramatical number of the verb'},
                 'voice': {'description': 'Gramatical voice of the verb'},
                 'degree': {'description': 'Degree (e.g. Comparitative, Superlative)'},
                 'type': {'description': 'Gramatical type  of noun or pronoun (e.g. Common, Personal)'},
                 'reference': {'description': 'Reference (to nodeID in XML source data, not yet post-processes)'},
                 'subj_ref': {'description': 'Subject reference (to nodeID in XML source data, not yet post-processes)'},
                 'nodeID': {'description': 'Node ID (as in the XML source data, not yet post-processes)'},
                 'junction': {'description': 'Junction data related to a clause'},
                 'wg'      : {'description': 'wg number in orig xml'}
                 }

'''
 -- the main function  --
'''   

good = cv.walk(
    director,
    slotType,
    otext=otext,
    generic=generic,
    intFeatures=intFeatures,
    featureMeta=featureMeta,
    warn=True,
    force=True
)

if good:
  print ("done")

# Loading-the-New-Testament-Text-Fabric (add a specific version, eg. 0.1.2)
NA = use ("tonyjurg/n1904_lft", version="0.1.3", hoist=globals())

N.otypeRank

Search0 = '''
book book=Jude
  chapter chapter=1
  
     verse 
'''
Search0 = NA.search(Search0)
NA.show(Search0, start=1, end=20, condensed=True,  extraFeatures={'sp','wg','clausetype','gloss','clauserule', 'junction','phrasefunction_long', }, withNodes=True)