# Set some globals
monad=1 # Smallest meaningful unit of text (in this corpus: a single word)

# Process all the books (files) in order
for bo, bookinfo in bo2book.items():
  CollectedItems=0
  full_df=pd.DataFrame({})
  book_long=bookinfo[0]
  booknum=bookinfo[1]
  book_short=bookinfo[2]
  InputFile = os.path.join(InputDir, f'{bo}.xml')
  OutputFile = os.path.join(OutputDir, f'{bo}.pkl')
  print(f'Processing {book_long} at {InputFile}')
  DataFrameList = []

  # Send the loaded XML document to the parsing routine
  tree = ET.parse(InputFile)
    
  # Now add all the parent info to the nodes in the XML tree [this step is important!]
  addParentInfo(tree.getroot())
  start_time = time.time()
    
  # Walk over all the leaves and harvest the data
  for elem in tree.iter():
    if not list(elem):
        # If no child elements exist, this must be a leaf/terminal node
     
        # Show  progress on screen by printing a dot for each 100 words processed
        CollectedItems+=1
        if (CollectedItems%100==0): print (".",end='')
        
        # Leafref will contain list with book, chapter verse and wordnumber
        Leafref = re.sub(r'[!: ]'," ", elem.attrib.get('ref')).split()
        
        # Push value for monad to element tree 
        elem.set('monad', monad)
        monad+=1
        
        # Add some important computed data to the leaf
        elem.set('LeafName', elem.tag)
        elem.set('word', elem.text)
        elem.set('book_long', book_long)
        elem.set('booknum', int(booknum))
        elem.set('book_short', book_short)
        elem.set('chapter', int(Leafref[1]))
        elem.set('verse', int(Leafref[2]))
       
        # The following code traces the parents up the tree and stores the discovered attributes.
        parentnode=getParent(elem)
        index=0
        while (parentnode):
           index+=1
           elem.set('Parent{}Name'.format(index),  parentnode.tag)
           elem.set('Parent{}Type'.format(index),  parentnode.attrib.get('Type'))
           elem.set('Parent{}Cat'.format(index),   parentnode.attrib.get('Cat'))
           elem.set('Parent{}Start'.format(index), parentnode.attrib.get('Start'))
           elem.set('Parent{}End'.format(index),   parentnode.attrib.get('End'))
           elem.set('Parent{}Rule'.format(index),  parentnode.attrib.get('Rule'))
           elem.set('Parent{}Head'.format(index),  parentnode.attrib.get('Head'))
           elem.set('Parent{}NodeId'.format(index),parentnode.attrib.get('nodeId'))
           elem.set('Parent{}ClType'.format(index),parentnode.attrib.get('ClType'))
           elem.set('Parent{}HasDet'.format(index),parentnode.attrib.get('HasDet'))
           currentnode=parentnode
           parentnode=getParent(currentnode)      
        elem.set('parents', int(index))
        
        # This will push all elements found in the tree into a DataFrame
        DataFrameChunk=pd.DataFrame(elem.attrib, index={monad})
        DataFrameList.append(DataFrameChunk)
        
  # Store the resulting DataFrame per book into a pickle file for further processing
  full_df = pd.concat([df for df in DataFrameList])
    
  output = open(r"{}".format(OutputFile), 'wb')
  pickle.dump(full_df, output)
  output.close()
  print("\nFound ",CollectedItems, " items in  %s seconds\n" % (time.time() - start_time))  
        

TF = Fabric(locations=output_dir, silent=False)
cv = CV(TF)

###############################################
#   Common helper functions                   #
###############################################

# The following sanitizer function is required to prevent passing float data to the walker function
def sanitize(input):
    if isinstance(input, float): return ''
    else: return (input)
   
###############################################
#          The director routine               #
###############################################

def director(cv):
    
    ###############################################
    #   Innitial setup of data etc.               #
    ###############################################
    
    NoneType = type(None)      # needed as tool to validate certain data
    IndexDict = {}             # init an empty dictionary

    for bo,bookinfo in bo2book.items():
        
        ###############################################
        #   start of section executed for each book   #
        ###############################################
        
        # load all data into a dataframe and process books in order (note that bookinfo is a list)
        Book=bookinfo[0]  
        BookNum=int(bookinfo[1])
        BookShort=bookinfo[2]
        BookLoc = os.path.join(source_dir, f'{bo}.pkl') 
        
        # Read data from PKL file and report progress
        print(f'\tloading {BookLoc}...')
        PklFile = open(BookLoc, 'rb')
        df_unsorted = pickle.load(PklFile)
        PklFile.close()
        
        # Fill dictionary of column names for this book
        ItemsInRow=1
        for itemname in df_unsorted.columns.to_list():
            IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
            # This is to identify the collumn containing the key to sort upon
            if itemname=="{http://www.w3.org/XML/1998/namespace}id": SortKey=ItemsInRow-1
            ItemsInRow+=1
            
        # Sort the nodes
        df=df_unsorted.sort_values(by=df_unsorted.columns[SortKey])
        del df_unsorted
        
        # Reset/load the following initial  variables (we are at the start of a new book)     
        phrasefunction = prev_phrasefunction = phrasefunctionlong = prev_phrasefunctionlong='TBD'
        this_clausetype = this_clauserule = phrasetype="unknown"
        prev_chapter = prev_verse = prev_sentence = prev_clause = prev_phrase = int(1)      
        sentence_track = clause_track = phrase_track = 1
        sentence_done = clause_done = phrase_done = verse_done = chapter_done = book_done = False  
        wrdnum = 0 # start at 0   

        # Create a set of nodes at the start a new book
       
        ThisBookPointer = cv.node('book')
        cv.feature(ThisBookPointer, book=Book, booknum=BookNum, bookshort=BookShort)
        
        ThisChapterPointer = cv.node('chapter')
        cv.feature(ThisChapterPointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=1)
        
        ThisVersePointer = cv.node('verse')
        cv.feature(ThisVersePointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=1, verse=1)
        
        ThisSentencePointer = cv.node('sentence')
        cv.feature(ThisSentencePointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=1, verse=1, sentence=1)
        
        ThisClausePointer = cv.node('clause')
        cv.feature(ThisClausePointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=1, verse=1, sentence=1, clause=1)
        
        ThisPhrasePointer = cv.node('phrase')
        cv.feature(ThisPhrasePointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=1, verse=1, sentence=1, clause=1, phrase=1)
          
        ###############################################
        # Iterate through words and construct objects #
        ###############################################
        
        for row in df.itertuples():
            wrdnum += 1
           
            # Get the number of parent nodes for this word
            parents = row[IndexDict.get("i_parents")]
      
            # Get chapter and verse for this word from the data
            chapter = row[IndexDict.get("i_chapter")]
            verse = row[IndexDict.get("i_verse")]
            
            # Get clause rule and type info of parent clause
            for i in range(1,parents-1):
                 item = IndexDict.get("i_Parent{}Cat".format(i))
                 if row[item]=="CL":
                    clauseparent=i
                    this_clauserule=row[IndexDict.get("i_Parent{}Rule".format(i))]  
                    this_clausetype=row[IndexDict.get("i_Parent{}ClType".format(i))]  
                    break
            cv.feature(ThisClausePointer, clause=clause_track, clauserule=this_clauserule, clausetype=this_clausetype, book=Book, booknum=BookNum, bookshort=BookShort, chapter=chapter, verse=verse)
            

            # Get phrase type info
            prev_phrasetype=phrasetype
            for i in range(1,parents-1):
                 item = IndexDict.get("i_Parent{}Cat".format(i))
                 if row[item]=="np":
                    _item ="i_Parent{}Rule".format(i)
                    phrasetype=row[IndexDict.get(_item)]
                    break
            functionaltag=row[IndexDict.get('i_FunctionalTag')]

            
            # Determine syntactic categories of clause parts. See also the description in 
            # "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 5&6
            # (section 2.4 Syntactic Categories at Clause Level)
            prev_phrasefunction=phrasefunction
            for i in range(1,clauseparent):              
                phrasefunction = row[IndexDict.get("i_Parent{}Cat".format(i))] 
                if phrasefunction=="ADV": 
                    phrasefunctionlong='Adverbial function'
                    break
                elif phrasefunction=="IO":
                    phrasefunctionlong='Indirect Object function'
                    break
                elif phrasefunction=="O":
                    phrasefunctionlong='Object function'
                    break
                elif phrasefunction=="O2":
                    phrasefunctionlong='Second Object function'
                    break
                elif phrasefunction=="S":
                    phrasefunctionlong='Subject function'
                    break
                elif phrasefunction=='P':
                    phrasefunctionlong='Predicate function'
                    break
                elif phrasefunction=="V":
                    phrasefunctionlong='Verbal function'
                    break
                elif phrasefunction=="VC":
                    phrasefunctionlong='Verbal Copula function'
            if prev_phrasefunction!=phrasefunction and wrdnum!=1:
                phrase_done = True


            # Determine syntactic categories at word level. See also the description in 
            # "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 6&7
            # (2.2. Syntactic Categories at Word Level: Part of Speech Labels)
            sp=sanitize(row[IndexDict.get("i_Cat")])
            if   sp=='adj':  splong='adjective'
            elif sp=='adj':  splong='adjective'
            elif sp=='conj': splong='conjunction'
            elif sp=='det':  splong='determiner' 
            elif sp=='intj': splong='interjection' 
            elif sp=='noun': splong='noun' 
            elif sp=='num':  splong='numeral' 
            elif sp=='prep': splong='preposition' 
            elif sp=='ptcl': splong='particle' 
            elif sp=='pron': splong='pronoun' 
            elif sp=='verb': splong='verb' 
                    
                    
            '''
            Determine if conditions are met to trigger some action 
            action will be executed after next word
            '''   
 
            # Detect chapter boundary
            if prev_chapter != chapter:
                chapter_done = True
                verse_done=True
                sentence_done = True
                clause_done = True
                phrase_done = True
                
            # Detect verse boundary
            if prev_verse != verse:
                verse_done=True
  
            
            '''
            Handle TF events and determine what actions need to be done if proper condition is met.
            '''   

            # Act upon end of phrase (close)
            if phrase_done or clause_done or sentence_done:
                cv.feature(ThisPhrasePointer, phrase=phrase_track, phrasetype=prev_phrasetype, phrasefunction=prev_phrasefunction, phrasefunctionlong=prev_phrasefunctionlong)
                cv.terminate(ThisPhrasePointer)
                prev_phrasefunction=phrasefunction
                prev_phrasefunctionlong=phrasefunctionlong
                
            # act upon end of clause (close)  
            if clause_done:
                cv.terminate(ThisClausePointer)
 
            # act upon end of sentence (close)
            if sentence_done:
                cv.terminate(ThisSentencePointer)
                
            # act upon end of verse (close)
            if verse_done:
                cv.terminate(ThisVersePointer)
                prev_verse = verse  

            # act upon end of chapter (close)
            if chapter_done:
                cv.terminate(ThisChapterPointer)
                prev_chapter = chapter

                
            # Start of chapter (create new)
            if chapter_done:
                ThisChapterPointer = cv.node('chapter')
                cv.feature(ThisChapterPointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=chapter)
                chapter_done = False
                
            # Start of verse (create new)
            if verse_done:
                ThisVersePointer = cv.node('verse')
                cv.feature(ThisVersePointer, book=Book, booknum=BookNum, bookshort=BookShort, chapter=chapter, verse=verse)
                verse_done = False  
                
            # Start of sentence (create new)
            if sentence_done:
                ThisSentencePointer= cv.node('sentence')
                cv.feature(ThisSentencePointer, sentence=sentence_track)
                sentence_track += 1
                sentence_done = False
                
            # Start of clause (create new)  
            if clause_done:
                ThisClausePointer = cv.node('clause')
                cv.feature(ThisClausePointer, clause=clause_track, clauserule=this_clauserule,clausetype=this_clausetype)
                clause_track += 1
                clause_done = False
                phrase_done = True 
                
            # Start of phrase (create new)
            if phrase_done:
                ThisPhrasePointer = cv.node('phrase')
                cv.feature(ThisPhrasePointer, phrase=phrase_track, phrasefunction=phrasefunction, phrasefunctionlong=phrasefunctionlong)
                prev_phrase = phrase_track
                prev_phrasefunction=phrasefunction
                prev_phrasefunctionlong=phrasefunctionlong
                phrase_track += 1
                phrase_done = False
                
      
            # Detect boundaries of sentences, clauses and phrases 
            text=row[IndexDict.get("i_Unicode")]
            if text[-1:] == "." :        
                sentence_done = True
                clause_done = True
                phrase_done = True
            if text[-1:] == ";" or text[-1:] == ",":
                clause_done = True
                phrase_done = True       
                    
            
            '''
            -- create word nodes --
            '''   
            
            # Get the word details and detect presence of punctuations
            word=row[IndexDict.get("i_Unicode")]
            match = re.search(r"([\.·—,;])$", word)
            if match: 
                # The group(0) method is used to retrieve the matched punctuation sign
                after=match.group(0)+' '
                # Remove the punctuation from the end of the word
                word=word[:-1]
                
            else: 
                after=' '
                    
            # Some attributes are not present inside some (small) books. The following is to prevent exceptions.
            degree='' 
            if 'i_Degree' in IndexDict: 
                   degree=sanitize(row[IndexDict.get("i_Degree")]) 
            subjref=''
            if 'i_SubjRef' in IndexDict:
                   subjref=sanitize(row[IndexDict.get("i_SubjRef")]) 
        

            # Create the word node 
            ThisWordPointer = cv.slot()
            cv.feature(ThisWordPointer, 
                    after=after,
                    word=word,
                    monad=row[IndexDict.get("i_monad")],
                    book=Book,
                    booknum=BookNum,
                    bookshort=BookShort,
                    chapter=chapter,
                    sp=sp,
                    splong=splong,
                    verse=verse,
                    sentence=sentence_track,
                    clause=clause_track,
                    phrase=phrase_track,
                    normalized=sanitize(row[IndexDict.get("i_NormalizedForm")]),
                    formaltag=sanitize(row[IndexDict.get("i_FormalTag")]),
                    functionaltag=functionaltag,
                    strongs=sanitize(row[IndexDict.get("i_StrongNumber")]),
                    lex_dom=sanitize(row[IndexDict.get("i_LexDomain")]),
                    ln=sanitize(row[IndexDict.get("i_LN")]),
                    gloss=sanitize(row[IndexDict.get("i_Gloss")]),
                    gn=sanitize(row[IndexDict.get("i_Gender")]),
                    nu=sanitize(row[IndexDict.get("i_Number")]),
                    case=sanitize(row[IndexDict.get("i_Case")]),
                    lemma=sanitize(row[IndexDict.get("i_UnicodeLemma")]),
                    person=sanitize(row[IndexDict.get("i_Person")]),
                    mood=sanitize(row[IndexDict.get("i_Mood")]),
                    tense=sanitize(row[IndexDict.get("i_Tense")]),
                    number=sanitize(row[IndexDict.get("i_Number")]),
                    voice=sanitize(row[IndexDict.get("i_Voice")]),
                    degree=degree,
                    type=sanitize(row[IndexDict.get("i_Type")]),
                    reference=sanitize(row[IndexDict.get("i_Ref")]),     # the capital R is critical here!
                    subj_ref=subjref,
                    nodeID=row[1]                              #this is a fixed position.
                    )
            cv.terminate(ThisWordPointer)

        
        '''
        Wrap up the book. At the end of the book we need to close all nodes in proper order.
        '''   
        
        # Close all nodes (phrase, clause, sentence, verse, chapter and book)
        cv.feature(ThisPhrasePointer, phrase=phrase_track, phrasetype=prev_phrasetype,phrasefunction=prev_phrasefunction,phrasefunctionlong=prev_phrasefunctionlong)
        cv.terminate(ThisPhrasePointer)
        cv.feature(ThisClausePointer, clause=clause_track, clauserule=this_clauserule, clausetype=this_clausetype)
        cv.terminate(ThisClausePointer)
        cv.terminate(ThisSentencePointer)
        cv.terminate(ThisVersePointer)
        cv.terminate(ThisChapterPointer)
        cv.terminate(ThisBookPointer)
        
        # Clear dataframe for this book 
        del df
        # Clear the index dictionary
        IndexDict.clear()
        gc.collect()
        
        ###############################################
        #    End of section executed for each book    #
        ###############################################
        
    ###############################################
    #      End of director function               #
    ###############################################
        
###############################################
#            Output definitions               #
###############################################
        
slotType = 'word'  # or whatever you choose
otext = {  # dictionary of config data for sections and text formats
        'fmt:text-orig-full':'{word}{after}',
        'sectionTypes':'book,chapter,verse',
        'sectionFeatures':'book,chapter,verse',
        'structureFeatures': 'book,chapter,verse',
        'structureTypes': 'book,chapter,verse',
        }

# Configure metadata
generic = {  # dictionary of metadata which will be included in all feature files
        'textFabricVersion': '{}'.format(VERSION),  #imported from tf.parameter
        'xmlSourceLocation': 'https://github.com/tonyjurg/Nestle1904GBI/tree/main/resources/sourcedata/apr_6_2023',
        'xmlSourceDate': 'April 6, 2023',
        'author': 'Evangelists and apostles',
        'availability': 'Creative Commons Attribution 4.0 International (CC BY 4.0)',
        'converters': 'Tony Jurg',
        'converterSource': 'https://github.com/tonyjurg/Nestle1904GBI/tree/main/resources/converter',
        'converterVersion': '0.4',
        'dataSource': 'MACULA Greek Linguistic Datasets, available at https://github.com/Clear-Bible/macula-greek/tree/main/Nestle1904/nodes',
        'editors': 'Eberhart Nestle (1904)',
        'sourceDescription': 'Greek New Testment (British Foreign Bible Society, 1904)',
        'sourceFormat': 'XML (GBI tree node data)',
        'title': 'Greek New Testament (Nestle 1904 GBI)'
         }

intFeatures = {  # set of integer valued feature names
         'booknum',
         'chapter',
         'verse',
         'sentence',
         'clause',
         'phrase',
         'monad'
         }

# note: 'description' should start with a lower case. Using an uppercase leaves this info out of node overview 
featureMeta = {  # per feature dicts with metadata
                 'after':                {'description': 'Character after the word (space or punctuation)'},
                 'book':                 {'description': 'Book name (fully spelled out)'},
                 'booknum':              {'description': 'NT book number (Matthew=1, Mark=2, ..., Revelation=27)'},
                 'bookshort':            {'description': 'Book name (abbreviated)'},
                 'chapter':              {'description': 'Chapter number inside book'},
                 'verse':                {'description': 'Verse number inside chapter'},
                 'sentence':             {'description': 'Sentence number (counted per chapter)'},
                 'clause':               {'description': 'Clause number (counted per chapter)'},
                 'clauserule':           {'description': 'Clause rule'},
                 'clausetype':           {'description': 'Clause type'},
                 'phrase' :              {'description': 'Phrase number (counted per chapter)'},
                 'phrasetype' :          {'description': 'Phrase type information'},
                 'phrasefunction' :      {'description': 'Phrase function (abbreviated)'},
                 'phrasefunctionlong'  : {'description': 'Phrase function (long description)'},
                 'monad':                {'description': 'Sequence number of the smallest meaningful unit of text (single word)'},
                 'word':                 {'description': 'Word as it appears in the text'},
                 'sp':                   {'description': 'Speech Part (abbreviated)'},
                 'splong':               {'description': 'Speech Part (long description)'}, 
                 'normalized':           {'description': 'Surface word stripped of punctations'},
                 'lemma':                {'description': 'Lexeme (lemma)'},
                 'formaltag':            {'description': 'Formal tag (Sandborg-Petersen morphology)'},
                 'functionaltag':        {'escription': 'Functional tag (Sandborg-Petersen morphology)'},
                 # see also discussion on relation between lex_dom and ln @ https://github.com/Clear-Bible/macula-greek/issues/29
                 'lex_dom':              {'description': 'Lexical domain according to Semantic Dictionary of Biblical Greek, SDBG'},
                 'ln':                   {'description': 'Lauw-Nida lexical classification'},
                 'strongs':              {'description': 'Strongs number'},
                 'gloss':                {'description': 'English gloss'},
                 'gn':                   {'description': 'Gramatical gender (Masculine, Feminine, Neuter)'},
                 'nu':                   {'description': 'Gramatical number (Singular, Plural)'},
                 'case':                 {'description': 'Gramatical case (Nominative, Genitive, Dative, Accusative, Vocative)'},
                 'person':               {'description': 'Gramatical person of the verb (first, second, third)'},
                 'mood':                 {'description': 'Gramatical mood of the verb (passive, etc)'},
                 'tense':                {'description': 'Gramatical tense of the verb (e.g. Present, Aorist)'},
                 'number':               {'description': 'Gramatical number of the verb'},
                 'voice':                {'description': 'Gramatical voice of the verb'},
                 'degree':               {'description': 'Degree (e.g. Comparitative, Superlative)'},
                 'type':                 {'description': 'Gramatical type  of noun or pronoun (e.g. Common, Personal)'},
                 'reference':            {'description': 'Reference (to nodeID in XML source data, not yet post-processes)'},
                 'subj_ref':             {'description': 'Subject reference (to nodeID in XML source data)'},
                 'nodeID':               {'description': 'Node ID (as in the XML source data)'}
                 }

'''
 -- The main function  --
'''   


good = cv.walk(
    director,
    slotType,
    otext=otext,
    generic=generic,
    intFeatures=intFeatures,
    featureMeta=featureMeta,
    warn=True,
    force=True
)

if good:
  print ("done")