#!/usr/bin/env python
# coding: utf-8

# # Creating Text-Fabric from LowFat XML trees
# The source data for the conversion are the LowFat XML trees files representing the macula-greek version of the Nestle 1904 Greek New Testment.  The most recent source data can be found on github https://github.com/Clear-Bible/macula-greek/tree/main/Nestle1904/lowfat. Attribution: "MACULA Greek Linguistic Datasets, available at https://github.com/Clear-Bible/macula-greek/". 
# 
# The production of the Text-Fabric files consist of two steps. First the creation of piclke files (part 1). Secondly the actual Text-Fabric creation process (part 2). Both steps are independent allowing to start from Part 2 by using the pickle files as input. 
# 
# Be advised that this Text-Fabric version is a test version (proof of concept) and requires further finetuning, especialy with regards of nomenclature and presentation of (sub)phrases and clauses.

# ## Table of content <a class="anchor" id="TOC">
# * [Part 1: Read LowFat XML data and store in pickle](#first-bullet)
# * [part 2: Sort the nodes](#second-bullet)
# * [Part 3: Nestle1904 production from pickle input](#third-bullet)
# * [Part 4: Testing the created textfabric data](#fourth-bullet)

# ## Part 1: Read LowFat XML data and store in pickle <a class="anchor" id="first-bullet"></a>
# ##### [back to TOC](#TOC)
# 
# This script harvests all information from the LowFat tree data (XML nodes), puts it into a Panda DataFrame and stores the result per book in a pickle file. Note: pickling (in Python) is serialising an object into a disk file (or buffer). 
# 
# In the context of this script, 'Leaf' refers to those node containing the Greek word as data, which happen to be the nodes without any child (hence the analogy with the leaves on the tree). These 'leafs' can also be refered to as 'terminal nodes'. Futher, Parent1 is the leaf's parent, Parent2 is Parent1's parent, etc.
# 
# For a full description of the source data see document [MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf](https://github.com/Clear-Bible/macula-greek/blob/main/doc/MACULA%20Greek%20Treebank%20for%20the%20Nestle%201904%20Greek%20New%20Testament.pdf)

# ### Step 1: import various libraries

# In[8]:


import pandas as pd
import sys
import os
import time
import pickle

import re  #regular expressions
from os import listdir
from os.path import isfile, join
import xml.etree.ElementTree as ET


# ### Step 2: initialize global data
# 
# Change BaseDir, XmlDir and PklDir to match location of the datalocation and the OS used.

# In[5]:


BaseDir = 'C:\\Users\\tonyj\\my_new_Jupyter_folder\\Read_from_lowfat\\data\\'
XmlDir = BaseDir+'xml\\'
PklDir = BaseDir+'pkl\\'
XlsxDir = BaseDir+'xlsx\\'
# note: create output directory prior running this part

#         key: filename,       [0]=book_long,   [1]=book_num,  [3]=book_short
bo2book = {'01-matthew':       ['Matthew',         '1',        'Matt'],
           '02-mark':          ['Mark',            '2',        'Mark'],
           '03-luke':          ['Luke',            '3',        'Luke'],
           '04-john':          ['John',            '4',        'John'],
           '05-acts':          ['Acts',            '5',        'Acts'],
           '06-romans':        ['Romans',          '6',        'Rom'],
           '07-1corinthians':  ['I_Corinthians',   '7',        '1Cor'],
           '08-2corinthians':  ['II_Corinthians',  '8',        '2Cor'],
           '09-galatians':     ['Galatians',       '9',        'Gal'],
           '10-ephesians':     ['Ephesians',       '10',       'Eph'],
           '11-philippians':   ['Philippians',     '11',       'Phil'],
           '12-colossians':    ['Colossians',      '12',       'Col'],
           '13-1thessalonians':['I_Thessalonians', '13',       '1Thess'],
           '14-2thessalonians':['II_Thessalonians','14',       '2Thess'],
           '15-1timothy':      ['I_Timothy',       '15',       '1Tim'],
           '16-2timothy':      ['II_Timothy',      '16',       '2Tim'],
           '17-titus':         ['Titus',           '17',       'Titus'],
           '18-philemon':      ['Philemon',        '18',       'Phlm'],
           '19-hebrews':       ['Hebrews',         '19',       'Heb'],
           '20-james':         ['James',           '20',       'Jas'],
           '21-1peter':        ['I_Peter',         '21',       '1Pet'],
           '22-2peter':        ['II_Peter',        '22',       '2Pet'],
           '23-1john':         ['I_John',          '23',       '1John'],
           '24-2john':         ['II_John',         '24',       '2John'],
           '25-3john':         ['III_John',        '25',       '3John'],     
           '26-jude':          ['Jude',            '26',       'Jude'],
           '27-revelation':    ['Revelation',      '27',       'Rev']}

bo2book_ = {'25-3john':         ['III_John',        '25',       '3John']}


# ### step 3: define Function to add parent info to each node of the XML tree
# 
# In order to traverse from the 'leafs' (terminating nodes) upto the root of the tree, it is required to add information to each node pointing to the parent of each node.
# 
# (concept taken from https://stackoverflow.com/questions/2170610/access-elementtree-node-parent-node)

# In[6]:


def addParentInfo(et):
    for child in et:
        child.attrib['parent'] = et
        addParentInfo(child)

def getParent(et):
    if 'parent' in et.attrib:
        return et.attrib['parent']
    else:
        return None


# ### Step 4: read and process the XML data and store panda dataframe in pickle

# In[48]:


# set some globals
monad=1
CollectedItems= 0

# process books in order
for bo, bookinfo in bo2book.items():
  CollectedItems=0
  SentenceNumber=0
  WordGroupNumber=0
  full_df=pd.DataFrame({})
  book_long=bookinfo[0]
  booknum=bookinfo[1]
  book_short=bookinfo[2]
  InputFile = os.path.join(XmlDir, f'{bo}.xml')
  OutputFile = os.path.join(PklDir, f'{bo}.pkl')
  print(f'Processing {book_long} at {InputFile}')

  # send xml document to parsing process
  tree = ET.parse(InputFile)
  # Now add all the parent info to the nodes in the xtree [important!]
  addParentInfo(tree.getroot())
  start_time = time.time()
    
  # walk over all the XML data
  for elem in tree.iter():
    if elem.tag == 'sentence':
        # add running number to 'sentence' tags
        SentenceNumber+=1
        elem.set('SN', SentenceNumber)
    if elem.tag == 'wg':
        # add running number to 'wg' tags
        WordGroupNumber+=1
        elem.set('WGN', WordGroupNumber)
    if elem.tag == 'w':
        # all nodes containing words are tagged with 'w'
     
        # show  progress on screen
        CollectedItems+=1
        if (CollectedItems%100==0): print (".",end='')
        
        #Leafref will contain list with book, chapter verse and wordnumber
        Leafref = re.sub(r'[!: ]'," ", elem.attrib.get('ref')).split()
        
        #push value for monad to element tree 
        elem.set('monad', monad)
        monad+=1
        
        # add some important computed data to the leaf
        elem.set('LeafName', elem.tag)
        elem.set('word', elem.text)
        elem.set('book_long', book_long)
        elem.set('booknum', int(booknum))
        elem.set('book_short', book_short)
        elem.set('chapter', int(Leafref[1]))
        elem.set('verse', int(Leafref[2]))
       
        # folling code will trace down parents upto the tree and store found attributes
        parentnode=getParent(elem)
        index=0
        while (parentnode):
           index+=1
           elem.set('Parent{}Name'.format(index),      parentnode.tag)
           elem.set('Parent{}Type'.format(index),      parentnode.attrib.get('type'))
           elem.set('Parent{}Appos'.format(index),     parentnode.attrib.get('appositioncontainer'))
           elem.set('Parent{}Class'.format(index),     parentnode.attrib.get('class'))
           elem.set('Parent{}Rule'.format(index),      parentnode.attrib.get('rule'))
           elem.set('Parent{}Role'.format(index),      parentnode.attrib.get('role'))
           elem.set('Parent{}Cltype'.format(index),    parentnode.attrib.get('cltype'))
           elem.set('Parent{}Unit'.format(index),      parentnode.attrib.get('unit'))
           elem.set('Parent{}Junction'.format(index),  parentnode.attrib.get('junction'))
           elem.set('Parent{}SN'.format(index),        parentnode.attrib.get('SN'))
           elem.set('Parent{}WGN'.format(index),       parentnode.attrib.get('WGN'))
           currentnode=parentnode
           parentnode=getParent(currentnode)      
        elem.set('parents', int(index))
        
        #this will push all elements found in the tree into a DataFrame
        df=pd.DataFrame(elem.attrib, index={monad})
        full_df=pd.concat([full_df,df])
        
  #store the resulting DataFrame per book into a pickle file for further processing
  df = df.convert_dtypes(convert_string=True)
    
  # sort by s=id
  sortkey='{http://www.w3.org/XML/1998/namespace}id'
  full_df.rename(columns={sortkey: 'id'}, inplace=True)
  full_df.sort_values(by=['id'])

  output = open(r"{}".format(OutputFile), 'wb')
  pickle.dump(full_df, output)
  output.close()
  print("\nFound ",CollectedItems, " items in  %s seconds\n" % (time.time() - start_time))  
        


# In[ ]:


# just dump some things to test the result


for bo in bo2book:
        '''
        load all data into a dataframe
        process books in order (bookinfo is a list!)
        '''   
        InputFile = os.path.join(PklDir, f'{bo}.pkl')
       
        print(f'\tloading {InputFile}...')
        pkl_file = open(InputFile, 'rb')
        df = pickle.load(pkl_file)
        pkl_file.close()
   
        # not sure if this is needed
        # fill dictionary of column names for this book 
        IndexDict = {}             # init an empty dictionary
        ItemsInRow=1
        for itemname in df.columns.to_list():
            IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
            print (itemname)
            ItemsInRow+=1
        


# ## Part 3: Nestle1904 Text-Fabric production from pickle input <a class="anchor" id="second-bullet"></a>
# ##### [back to TOC](#TOC)
# 
# This script creates the Text-Fabric files by recursive calling the TF walker function.
# API info: https://annotation.github.io/text-fabric/tf/convert/walker.html
# 
# The pickle files created by step 1 are stored on Github location T.B.D.

# ### Step 1: Load libraries and initialize some data
# 
# 

# In[11]:


import pandas as pd
import os
import re
import gc
from tf.fabric import Fabric
from tf.convert.walker import CV
from tf.parameters import VERSION
from datetime import date
import pickle

BaseDir = 'C:\\Users\\tonyj\\my_new_Jupyter_folder\\Read_from_lowfat\\data\\'
XmlDir = BaseDir+'xml\\'
PklDir = BaseDir+'pkl\\'
XlsxDir = BaseDir+'xlsx\\'

#         key: filename,       [0]=book_long,   [1]=book_num,  [3]=book_short
bo2book = {'01-matthew':       ['Matthew',         '1',        'Matt'],
           '02-mark':          ['Mark',            '2',        'Mark'],
           '03-luke':          ['Luke',            '3',        'Luke'],
           '04-john':          ['John',            '4',        'John'],
           '05-acts':          ['Acts',            '5',        'Acts'],
           '06-romans':        ['Romans',          '6',        'Rom'],
           '07-1corinthians':  ['I_Corinthians',   '7',        '1Cor'],
           '08-2corinthians':  ['II_Corinthians',  '8',        '2Cor'],
           '09-galatians':     ['Galatians',       '9',        'Gal'],
           '10-ephesians':     ['Ephesians',       '10',       'Eph'],
           '11-philippians':   ['Philippians',     '11',       'Phil'],
           '12-colossians':    ['Colossians',      '12',       'Col'],
           '13-1thessalonians':['I_Thessalonians', '13',       '1Thess'],
           '14-2thessalonians':['II_Thessalonians','14',       '2Thess'],
           '15-1timothy':      ['I_Timothy',       '15',       '1Tim'],
           '16-2timothy':      ['II_Timothy',      '16',       '2Tim'],
           '17-titus':         ['Titus',           '17',       'Titus'],
           '18-philemon':      ['Philemon',        '18',       'Phlm'],
           '19-hebrews':       ['Hebrews',         '19',       'Heb'],
           '20-james':         ['James',           '20',       'Jas'],
           '21-1peter':        ['I_Peter',         '21',       '1Pet'],
           '22-2peter':        ['II_Peter',        '22',       '2Pet'],
           '23-1john':         ['I_John',          '23',       '1John'],
           '24-2john':         ['II_John',         '24',       '2John'],
           '25-3john':         ['III_John',        '25',       '3John'],     
           '26-jude':          ['Jude',            '26',       'Jude'],
           '27-revelation':    ['Revelation',      '27',       'Rev']}

bo2book_ = {'26-jude':          ['Jude',            '26',       'Jude']}


# ## Optional: export to Excel for investigation

# In[ ]:


# test: sorting the data
import openpyxl
import pickle

#if True:
for bo in bo2book:
        '''
        load all data into a dataframe
        process books in order (bookinfo is a list!)
        '''   
        InputFile = os.path.join(PklDir, f'{bo}.pkl')
        #InputFile = os.path.join(PklDir, '01-matthew.pkl')
       
        print(f'\tloading {InputFile}...')
        pkl_file = open(InputFile, 'rb')
        df = pickle.load(pkl_file)
        pkl_file.close()
   
        # not sure if this is needed
        # fill dictionary of column names for this book 
        IndexDict = {}             # init an empty dictionary
        ItemsInRow=1
        for itemname in df.columns.to_list():
            IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
            ItemsInRow+=1
            #print(itemname)
        
        # sort by id
        #print(df)
        df_sorted=df.sort_values(by=['id'])
        df_sorted.to_excel(os.path.join(XlsxDir, f'{bo}.xlsx'), index=False)


# ### Step 2 Running the TF walker function
# 
# API info: https://annotation.github.io/text-fabric/tf/convert/walker.html
# 
# The logic of interpreting the data is included in the director function.

# In[15]:


TF = Fabric(locations=BaseDir, silent=False)
cv = CV(TF)
version = "0.1.3 (recursive clause generation)"


def sanitize(input):
    if isinstance(input, float): return ''
    if isinstance(input, type(None)): return ''
    else: return (input)
    



def director(cv):
    
 
    NoneType = type(None)      # needed as tool to validate certain data
    prev_book = "Matthew"      # start at first book
    IndexDict = {}             # init an empty dictionary
    
    Arrays2Dump=200
    DumpedArrays=0
    ClauseDict={}    # init a dummy dictionary
    PrevClauseSet = []
    PrevClauseList = []

    for bo,bookinfo in bo2book.items():
        
        '''
        load all data into a dataframe
        process books in order (bookinfo is a list!)
        '''   
        book=bookinfo[0]  
        booknum=int(bookinfo[1])
        book_short=bookinfo[2]
        book_loc = os.path.join(PklDir, f'{bo}.pkl') 
        
        print(f'\tWe are loading {book_loc}...')
        pkl_file = open(book_loc, 'rb')
        df_unsorted = pickle.load(pkl_file)
        pkl_file.close()
        df=df_unsorted.sort_values(by=['id'])
 
    
        FoundWords=0
        phrasefunction='TBD'
        phrasefunction_long='TBD'
        this_clausetype="unknown"  #just signal a not found case
        this_clauserule="unknown"
        this_clauseconst="unknown"
        prev_clausetype = "unknown"
        this_phrasetype="unknown"       #just signal a not found case
        this_phrasefunction="unknown"
        this_phrasefunction_long="unknown"
        this_role="unknown"
        ClausesSuspended = False
        PreSuspendClauseList = {}
        phrase_resume = False
        ClosedPhrase=''
        PhraseToRestoreTo=''
        phrase_close=False
        
        
        prev_chapter = int(1)      # start at 1
        prev_verse = int(1)        # start at 1
        prev_sentence = int(1)     # start at 1
        prev_clause = int(1)       # start at 1
        prev_phrase = int(1)       # start at 1
        prev_role = int(1)         # start at 1
        
        # ClauseArrayNumbers contains the active clause numbers (wg in the LowFatTree data)
        

     
        # reset/load the following initial variables (we are at the start of a new book)
        sentence_track = clause_track = clauseconst_track = phrase_track = 1
        sentence_done = clause_done = clauseconst_done = phrase_done = verse_done = chapter_done = book_done = False
        PhraseInterupted = False
        wrdnum = 0 # start at 0

        # fill dictionary of column names for this book 
        ItemsInRow=1
        for itemname in df.columns.to_list():
            IndexDict.update({'i_{}'.format(itemname): ItemsInRow})
            ItemsInRow+=1
            
        df.sort_values(by=['id'])

        '''
        Walks through the texts and triggers
        slot and node creation events.
        '''
        
        # iterate through words and construct objects
        for row in df.itertuples():
            wrdnum += 1
            FoundWords +=1
            
          
            '''
            First get all the relevant information from the dataframe
            '''   
            
            # get number of parent nodes
            parents = row[IndexDict.get("i_parents")]
            
      
            # get chapter and verse from the data
            chapter = sanitize(row[IndexDict.get("i_chapter")])
            verse = sanitize(row[IndexDict.get("i_verse")])
         

                    
            # experimental (decoding the clause structure)
            ClauseList=[]  # stores current active clauses (wg numbers)
            if True:
                for i in range(1,parents-1):
                    item = IndexDict.get("i_Parent{}Class".format(i))
                    if row[item]=="cl":
                        WGN=row[IndexDict.get("i_Parent{}WGN".format(i))]
                        ClauseList.append(WGN)
                        ClauseDict[(WGN,0)]=WGN
                        ClauseDict[(WGN,1)]=sanitize(row[IndexDict.get("i_Parent{}Rule".format(i))])
                        ClauseDict[(WGN,2)]=sanitize(row[IndexDict.get("i_Parent{}Cltype".format(i))])
                        ClauseDict[(WGN,3)]=sanitize(row[IndexDict.get("i_Parent{}Junction".format(i))])
                if not PrevClauseList==ClauseList:
                    #print ('\nPrevClauseDict ={',end='')
                    #for item in PrevClauseDict:
                    #    print (' ',item,'=',PrevClauseDict[item],end='')
                    #print ('}')
                    #print('\nClauseList=',ClauseList, 'PrevClauseList=',PrevClauseList, 'ClausesSuspended=',ClausesSuspended,end='')
                    #print('\nAction(s) = ',end=' ')
                    ResumePhrase=False
                    if ClausesSuspended==True:
                        for item in ClauseList:
                            try:
                                ItemToResume=ClauseDict[(item,4)]
                                #print (' resume: '+str(item),end=' ')
                                cv.resume(ClauseDict[(item,4)])
                                #ResumePhrase=True
                                PhraseToRestoreTo=prev_phrasefunction
                            except:
                                #print("This is the odd condition for wg=",item)
                                # CREATE CASE
                                #print ('\n create: '+str(item),end=' ')
                                ClauseDict[(item,4)]=cv.node('clause')
                                ClauseDict[(item,5)]=clause_track
                                clause_track += 1
                                #print ('link=',ClauseDict[(item,4)])
                                
                            

                    if not bool(ClauseList): # this means it is a part outside a clause
                        for item in PrevClauseList:
                            # SUSPEND CASE
                            #print (' suspend: '+str(item),end=' ')
                            cv.feature(ClauseDict[(item,4)], clause=ClauseDict[(item,5)], wg=ClauseDict[(item,0)], junction=ClauseDict[(item,3)], clausetype=ClauseDict[(item,2)], clauserule=ClauseDict[(item,1)])
                            cv.terminate(ClauseDict[(item,4)])
                        ClausesSuspended= True
                        PreSuspendClauseList = PrevClauseList
                    else:
                        for item in PrevClauseList:
                           if (item not in ClauseList):
                                # CLOSE CASE
                                #print ('\n close: '+str(item),end=' ')
                                cv.feature(ClauseDict[(item,4)], clause=ClauseDict[(item,5)], wg=ClauseDict[(item,0)], junction=ClauseDict[(item,3)], clausetype=ClauseDict[(item,2)], clauserule=ClauseDict[(item,1)])
                                cv.terminate(ClauseDict[item,4])
                                for k in range(6): del ClauseDict[item,k]
                        for item in ClauseList:
                           if (item not in PrevClauseList):
                                if not ClausesSuspended:
                                    # CREATE CASE
                                    #print ('\n create: '+str(item),end=' ')
                                    ClauseDict[(item,4)]=cv.node('clause')
                                    ClauseDict[(item,5)]=clause_track
                                    clause_track += 1
                                    #print ('link=',ClauseDict[(item,4)])
                        ClausesSuspended=False

                    PrevClauseList = ClauseList    
                    clause_done=True
                    #print ('\nClauseDict=',ClauseDict,end='')
                    #print ('\n\n',book,chapter,":",verse,'\t',row[IndexDict.get("i_word")]+row[IndexDict.get("i_after")],end='')

                #else:
                    #print (row[IndexDict.get("i_word")]+row[IndexDict.get("i_after")],end='')


     

            
            # determine syntactic categories of clause parts. See also the description in 
            # "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 5&6
            # (section 2.4 Syntactic Categories at Clause Level)
            prev_phrasefunction=this_phrasefunction
            prev_phrasefunction_long=this_phrasefunction_long
            role=row[IndexDict.get("i_role")]
            ValidRoles=["adv","io","o","o2","s","p","v","vc"]
            this_phrasefunction=''
            PhraseWGN=''
            if isinstance (role,str) and role in ValidRoles: 
                this_phrasefunction=role
            else:
                for i in range(1,parents-1):
                    role = row[IndexDict.get("i_Parent{}Role".format(i))]
                    if isinstance (role,str) and role in ValidRoles: 
                        this_phrasefunction=role
                        PhraseWGN=row[IndexDict.get("i_Parent{}WGN".format(i))]
                        break
            if this_phrasefunction!=prev_phrasefunction:
                #print ('\n',chapter,':',verse,'  this_phrasefunction=',this_phrasefunction,' prev_phrasefunction=',prev_phrasefunction)
                if phrase_close:
                     cv.resume(ClosedPhrase)
                     #print ('resume phrase',ClosedPhrase)
                if this_phrasefunction!='':
                     phrase_done = True
                else:
                     phrase_close = True
                     ClosedPhrase=this_phrase
                     PhraseToRestoreTo=prev_phrasefunction



            if   this_phrasefunction=="adv":  this_phrasefunction_long='Adverbial'
            elif this_phrasefunction=="io":   this_phrasefunction_long='Indirect Object'
            elif this_phrasefunction=="o":    this_phrasefunction_long='Object'
            elif this_phrasefunction=="o2":   this_phrasefunction_long='Second Object'
            elif this_phrasefunction=="s":    this_phrasefunction_long='Subject'
            elif this_phrasefunction=="p":    this_phrasefunction_long='Predicate'
            elif this_phrasefunction=="v":    this_phrasefunction_long='Verbal'
            elif this_phrasefunction=="vc":   this_phrasefunction_long='Verbal Copula'
            #print ('this_phrasefunction=',this_phrasefunction, end=" | ")

       
                
            '''
            determine if conditions are met to trigger some action 
            action will be executed after next word
            '''   
            
            # detect book boundary
            if prev_book != book:
                prev_book=book
                book_done = True
                chapter_done = True
                verse_done=True
                sentence_done = True
                clause_done = True
                phrase_done = True

            # detect chapter boundary
            if prev_chapter != chapter:
                chapter_done = True
                verse_done=True
                sentence_done = True
                clause_done = True
                phrase_done = True
                
            # detect verse boundary
            if prev_verse != verse:
                verse_done=True
                

            # determine syntactic categories at word level. See also the description in 
            # "MACULA Greek Treebank for the Nestle 1904 Greek New Testament.pdf" page 6&7
            # (2.2. Syntactic Categories at Word Level: Part of Speech Labels)
            sp=sanitize(row[IndexDict.get("i_class")])
            if sp=='adj':
                sp_full='adjective'
            elif sp=='adj':
                sp_full='adjective'
            elif sp=='conj':
                sp_full='conjunction'
            elif sp=='det':
                sp_full='determiner' 
            elif sp=='intj':
                sp_full='interjection' 
            elif sp=='noun':
                sp_full='noun' 
            elif sp=='num':
                sp_full='numeral' 
            elif sp=='prep':
                sp_full='preposition' 
            elif sp=='ptcl':
                sp_full='particle' 
            elif sp=='pron':
                sp_full='pronoun' 
            elif sp=='verb':
                sp_full='verb' 
                    
                    
            # Manage first word per book
            if wrdnum==1: 
                prev_phrasetype=this_phrasetype
                prev_phrasefunction=this_phrasefunction
                prev_phrasefunction_long=this_phrasefunction_long
                prev_clauserule=this_clauserule
                book_done = chapter_done = verse_done = phrase_done = clause_done = sentence_done = False
                # create the first set of nodes
                this_book = cv.node('book')
                cv.feature(this_book, book=prev_book)
                this_chapter = cv.node('chapter')
                this_verse = cv.node('verse')
                this_sentence = cv.node('sentence')
                #this_clause = cv.node('clause')
                this_phrase = cv.node('phrase')
                #print ('new phrase',this_phrase)
                sentence_track += 1
                #clause_track += 1
                phrase_track += 1

                       
            
            '''
             -- handle TF events --
            Determine what actions need to be done if proper condition is met.
            '''   

            # act upon end of phrase (close)
            if phrase_done or phrase_close:
                cv.feature(this_phrase, phrase=prev_phrase, phrasetype=prev_phrasetype, phrasefunction=prev_phrasefunction, phrasefunction_long=prev_phrasefunction_long,wg=PhraseWGN)
                cv.terminate(this_phrase)
                #print ('terminate phrase',this_phrase,':',prev_phrasefunction,'\n')
                ClosedPhrase=this_phrase
                PhraseToRestoreTo=prev_phrasefunction
                phrase_close=False


                
            #    cv.feature(this_clause, clause=prev_clause, clausetype=prev_clausetype, clauserule=prev_clauserule)
            #    cv.terminate(this_clause)
 
            # act upon end of sentence (close)
            if sentence_done:
                cv.feature(this_sentence, sentence=prev_sentence)
                cv.terminate(this_sentence)
                
            # act upon end of verse (close)
            if verse_done:
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)
                prev_verse = verse  

            # act upon end of chapter (close)
            if chapter_done:
                cv.feature(this_chapter, chapter=prev_chapter)
                cv.terminate(this_chapter)
                prev_chapter = chapter

            # act upon end of book (close and open new)
            if book_done:
                cv.terminate(this_book)
                this_book = cv.node('book')
                cv.feature(this_book, book=book) 
                prev_book = book
                wrdnum = 1
                phrase_track = 1
                clause_track = 1
                sentence_track = 1
                book_done = False
                
            # start of chapter (create new)
            if chapter_done:
                this_chapter = cv.node('chapter')
                chapter_done = False
                
            # start of verse (create new)
            if verse_done:
                this_verse = cv.node('verse')
                verse_done = False  
                
            # start of sentence (create new)
            if sentence_done:
                this_sentence= cv.node('sentence')
                prev_sentence = sentence_track
                sentence_track += 1
                sentence_done = False

                
            # start of phrase (create new)
            if phrase_done and not ResumePhrase:
                this_phrase = cv.node('phrase')
                #print ('new phrase',this_phrase)
                prev_phrase = phrase_track
                prev_phrasefunction=this_phrasefunction
                prev_phrasefunction_long=this_phrasefunction_long
                phrase_track += 1
                phrase_done = False
                
      
            # Detect boundaries of sentences
            text=sanitize(row[IndexDict.get("i_after")])[-1:]
            if text == "." :        
                sentence_done = True
                #phrase_done = True
            #if text == ";" or text == ",":
            #    phrase_done = True       
                    
            
            '''
            -- create word nodes --
            '''   
                    
            # some attributes are not present inside some (small) books. The following is to prevent exceptions.
            degree='' 
            if 'i_degree' in IndexDict: 
                   degree=sanitize(row[IndexDict.get("i_degree")]) 
            subjref=''
            if 'i_subjref' in IndexDict:
                   subjref=sanitize(row[IndexDict.get("i_subjref")]) 
                    
            #print (chapter,':',verse,"  ",row[IndexDict.get("i_word")],' - ',row[IndexDict.get("i_gloss")],' - ', sp,end='\n')

            # make word object
            this_word = cv.slot()
            cv.feature(this_word, 
                    after=sanitize(row[IndexDict.get("i_after")]),
                    id=sanitize(row[IndexDict.get("i_id")]),
                    unicode=sanitize(row[IndexDict.get("i_unicode")]),
                    word=sanitize(row[IndexDict.get("i_word")]),
                    monad=sanitize(row[IndexDict.get("i_monad")]),
                    orig_order=FoundWords,
                    book_long=sanitize(row[IndexDict.get("i_book_long")]),
                    booknum=booknum,
                    book_short=sanitize(row[IndexDict.get("i_book_short")]),
                    chapter=chapter,
                    ref=sanitize(row[IndexDict.get("i_ref")]),
                    sp=sanitize(sp),
                    sp_full=sanitize(sp_full),
                    verse=verse,
                    sentence=sanitize(prev_sentence),
                    clause=sanitize(prev_clause),
                    phrase=sanitize(prev_phrase),
                    normalized=sanitize(row[IndexDict.get("i_normalized")]),
                    morph=sanitize(row[IndexDict.get("i_morph")]),
                    strongs=sanitize(row[IndexDict.get("i_strong")]),
                    lex_dom=sanitize(row[IndexDict.get("i_domain")]),
                    ln=sanitize(row[IndexDict.get("i_ln")]),
                    gloss=sanitize(row[IndexDict.get("i_gloss")]),
                    gn=sanitize(row[IndexDict.get("i_gender")]),
                    nu=sanitize(row[IndexDict.get("i_number")]),
                    case=sanitize(row[IndexDict.get("i_case")]),
                    lemma=sanitize(row[IndexDict.get("i_lemma")]),
                    person=sanitize(row[IndexDict.get("i_person")]),
                    mood=sanitize(row[IndexDict.get("i_mood")]),
                    tense=sanitize(row[IndexDict.get("i_tense")]),
                    number=sanitize(row[IndexDict.get("i_number")]),
                    voice=sanitize(row[IndexDict.get("i_voice")]),
                    degree=degree,
                    type=sanitize(row[IndexDict.get("i_type")]),
                    reference=sanitize(row[IndexDict.get("i_ref")]),     
                    subj_ref=subjref,
                    nodeID=sanitize(row[1])                              #this is a fixed position.
                    )
            cv.terminate(this_word)

        
        '''
        -- wrap up the book --
        '''   
        
        # close all nodes (phrase, clause, sentence, verse, chapter and book)
        cv.feature(this_phrase, phrase=phrase_track, phrasetype=prev_phrasetype,phrasefunction=prev_phrasefunction,phrasefunction_long=prev_phrasefunction_long, wg=PhraseWGN)
        cv.terminate(this_phrase)
        #print ('terminate phrase',this_phrase,':',prev_phrasetype,'\n')

        
        #print (ClauseDict)
        for item in ClauseList:
            cv.feature(ClauseDict[(item,4)], clause=ClauseDict[(item,5)], clausetype=prev_clausetype, clauserule=prev_clauserule)
            cv.terminate(ClauseDict[item,4])
                                
                                
        cv.feature(this_sentence, sentence=prev_sentence)
        cv.terminate(this_sentence)
        cv.feature(this_verse, verse=prev_verse)
        cv.terminate(this_verse)
        cv.feature(this_chapter, chapter=prev_chapter)
        cv.terminate(this_chapter)
        cv.feature(this_book, book=prev_book)
        cv.terminate(this_book)
        
        # clear dataframe for this book 
        del df
        # clear the index dictionary
        IndexDict.clear()
        gc.collect()
        
        
'''
-- output definitions --
'''  
        
slotType = 'word'  # or whatever you choose
otext = {  # dictionary of config data for sections and text formats
        'fmt:text-orig-full':'{word}{after}',
        'sectionTypes':'book,chapter,verse',
        'sectionFeatures':'book,chapter,verse',
        'structureFeatures': 'book,chapter,verse',
        'structureTypes': 'book,chapter,verse',
        }

# configure metadata
generic = {  # dictionary of metadata meant for all features
         'Name': 'Greek New Testament (NA1904)',
         'Version': '1904',
         'Editors': 'Nestle',
         'Data source': 'MACULA Greek Linguistic Datasets, available at https://github.com/Clear-Bible/macula-greek/tree/main/Nestle1904/lowfat',
         'Availability': 'Creative Commons Attribution 4.0 International (CC BY 4.0)', 
         'Converter_author': 'Tony Jurg, Vrije Universiteit Amsterdam, Netherlands', 
         'Converter_execution': 'Tony Jurg, Vrije Universiteit Amsterdam, Netherlands', 
         'Convertor_source': 'https://github.com/tonyjurg/n1904_lft',
         'Converter_version': '{}'.format(version),
         'TextFabric version': '{}'.format(VERSION)  #imported from tf.parameters
         }

intFeatures = {  # set of integer valued feature names
         'booknum',
         'chapter',
         'verse',
         'sentence',
         'clause',
         'phrase',
         'orig_order',
         'monad'
         }

featureMeta = {  # per feature dicts with metadata
                 'after': {'description': 'Characters (eg. punctuations) following the word'},
                 'id': {'description': 'id of the word'},
                 'book': {'description': 'Book'},
                 'book_long': {'description': 'Book name (fully spelled out)'},
                 'booknum': {'description': 'NT book number (Matthew=1, Mark=2, ..., Revelation=27)'},
                 'book_short': {'description': 'Book name (abbreviated)'},
                 'chapter': {'description': 'Chapter number inside book'},
                 'verse': {'description': 'Verse number inside chapter'},
                 'sentence': {'description': 'Sentence number (counted per chapter)'},
                 'clause': {'description': 'Clause number (counted per chapter)'},
                 'clausetype' : {'description': 'Clause type information (verb, verbless, elided, minor, etc.)'},
                 'clauserule' : {'description': 'Clause rule information '},
                 'phrase' : {'description': 'Phrase number (counted per book)'},
                 'phrasetype' : {'description': 'Phrase type information'},
                 'phrasefunction' : {'description': 'Phrase function (abbreviated)'},
                 'phrasefunction_long' : {'description': 'Phrase function (long description)'},
                 'orig_order': {'description': 'Word order within corpus (per book)'},
                 'monad':{'description': 'Monad (currently: order of words in XML tree file!)'},
                 'word': {'description': 'Word as it appears in the text (excl. punctuations)'},
                 'unicode': {'description': 'Word as it arears in the text in Unicode (incl. punctuations)'},
                 'ref': {'description': 'ref Id'},
                 'sp': {'description': 'Part of Speech (abbreviated)'},
                 'sp_full': {'description': 'Part of Speech (long description)'}, 
                 'normalized': {'description': 'Surface word stripped of punctations'},
                 'lemma': {'description': 'Lexeme (lemma)'},
                 'morph': {'description': 'Morphological tag (Sandborg-Petersen morphology)'},
                 # see also discussion on relation between lex_dom and ln @ https://github.com/Clear-Bible/macula-greek/issues/29
                 'lex_dom': {'description': 'Lexical domain according to Semantic Dictionary of Biblical Greek, SDBG (not present everywhere?)'},
                 'ln': {'description': 'Lauw-Nida lexical classification (not present everywhere?)'},
                 'strongs': {'description': 'Strongs number'},
                 'gloss': {'description': 'English gloss'},
                 'gn': {'description': 'Gramatical gender (Masculine, Feminine, Neuter)'},
                 'nu': {'description': 'Gramatical number (Singular, Plural)'},
                 'case': {'description': 'Gramatical case (Nominative, Genitive, Dative, Accusative, Vocative)'},
                 'person': {'description': 'Gramatical person of the verb (first, second, third)'},
                 'mood': {'description': 'Gramatical mood of the verb (passive, etc)'},
                 'tense': {'description': 'Gramatical tense of the verb (e.g. Present, Aorist)'},
                 'number': {'description': 'Gramatical number of the verb'},
                 'voice': {'description': 'Gramatical voice of the verb'},
                 'degree': {'description': 'Degree (e.g. Comparitative, Superlative)'},
                 'type': {'description': 'Gramatical type  of noun or pronoun (e.g. Common, Personal)'},
                 'reference': {'description': 'Reference (to nodeID in XML source data, not yet post-processes)'},
                 'subj_ref': {'description': 'Subject reference (to nodeID in XML source data, not yet post-processes)'},
                 'nodeID': {'description': 'Node ID (as in the XML source data, not yet post-processes)'},
                 'junction': {'description': 'Junction data related to a clause'},
                 'wg'      : {'description': 'wg number in orig xml'}
                 }

'''
 -- the main function  --
'''   

good = cv.walk(
    director,
    slotType,
    otext=otext,
    generic=generic,
    intFeatures=intFeatures,
    featureMeta=featureMeta,
    warn=True,
    force=True
)

if good:
  print ("done")


# ## Part 4: Testing the created textfabric data <a class="anchor" id="fourth-bullet"></a>
# ##### [back to TOC](#TOC)

# ### Step 1 load the TF data
# 
# The TF will be loaded from github repository https://github.com/tonyjurg/n1904_lft

# In[1]:


get_ipython().run_line_magic('load_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')


# In[18]:


# First, I have to laod different modules that I use for analyzing the data and for plotting:
import sys, os, collections
import pandas as pd
import numpy as np
import re


from tf.fabric import Fabric
from tf.app import use


# The following cell loads the TextFabric files from github repository. 

# In[17]:


# Loading-the-New-Testament-Text-Fabric (add a specific version, eg. 0.1.2)
NA = use ("tonyjurg/n1904_lft", version="0.1.3", hoist=globals())


# In[19]:


N.otypeRank


# In[ ]:





# ### Step 2 Perform some basic display 
# 
# note: the implementation with regards how phrases need to be displayed (esp. with regards to conjunctions) is still to be done.

# In[20]:


Search0 = '''
book book=Jude
  chapter chapter=1
  
     verse 
'''
Search0 = NA.search(Search0)
NA.show(Search0, start=1, end=20, condensed=True,  extraFeatures={'sp','wg','clausetype','gloss','clauserule', 'junction','phrasefunction_long', }, withNodes=True)


# In[ ]:





# ### Step 3 dump some structure information

# In[233]:


T.structureInfo()


# In[234]:


T.structure(20619)


# In[232]:


TF.features['otext'].metaData


# ## Running text fabric browser <a class="anchor" id="fourth-bullet"></a>
# ##### [back to TOC](#TOC)

# In[33]:


get_ipython().system('text-fabric app')


# In[34]:


get_ipython().system('text-fabric app -k')


# In[44]:


tf.core.nodes.Nodes.otypeRank


# In[ ]: