pd.set_option('display.max_columns', 100)
NA1904.head()

print(pd.read_csv("/Users/oliverglanz/Library/CloudStorage/OneDrive-AndrewsUniversity/1200_AUS-research/Fabric-TEXT/0_data_NA1904/input/Matthew.txt", delimiter='\t', low_memory=False))

import os
import re
import collections
import json
import csv
# from glob import glob
from tf.fabric import Fabric
from tf.convert.walker import CV
# from tf.compose import modify

source_dirs = 'input' # "input" is the name of the input folder that contains the source file
output_dirs = 'output' # "output" is the name of the output folder to which the finished TF files will be dumped into

bo2book = {line.split()[0]:line.split()[1] for line in '''
Matthew Matthew
Mark Mark
Luke Luke
John John
Acts Acts
Romans Romans
I_Corinthians I_Corinthians
II_Corinthians II_Corinthians
Galatians Galatians
Ephesians Ephesians
Philippians Philippians
Colossians Colossians
I_Thessalonians I_Thessalonians
II_Thessalonians II_Thessalonians
I_Timothy I_Timothy
II_Timothy II_Timothy
Titus Titus
Philemon Philemon
Hebrews Hebrews
James James
I_Peter I_Peter
II_Peter II_Peter
I_John I_John
II_John II_John
III_John III_John
Jude Jude
Revelation Revelation

'''.split('\n') if line} # "OT" is the name of the file in the input folder AND "split()" splits at space

# patts = {'section': re.compile('(\d*):(\d*)\.(\d*)')}

def director(cv):
        
    '''
    Walks through NA1904 texts and triggers
    slot and node creation events.
    '''
        
    # process books in order
    for bo, book in bo2book.items():
        
        book_loc = os.path.join(source_dirs, f'{bo}.txt') #for some reason the files with fileextension "xlsx" and "csv" do not work
        
        print(f'\thandling {book_loc}...')
        
        with open(book_loc, 'r', encoding="utf8") as infile:
            text = [w for w in infile.read().split('\n') if w]
            
        this_book = cv.node('book')
        cv.feature(this_book, book=book) ##added 20220928
            
        # keep track of when to trigger paragraph, chapter, and verse objects
        # para_track = 1 # keep counts of paragraphs
        prev_book = "Matthew" # start at Genesis ##removed 20220928
        prev_chap = 1 # start at 1
        prev_verse = 1 # start at 1
        
        sentence_track = 1
        sentence_done = False
        clause_track = 1
        clause_done = False
        wrdnum = 0 # start at 0

        this_chap = cv.node('chapter')
        # this_para = cv.node('paragraph')
        this_verse = cv.node('verse')
        # this_subverse = cv.node('subverse')
        this_clause = cv.node('clause')
        this_sentence = cv.node('sentence')
        
        
        # iterate through words and construct objects
        for word in text:

            wrdnum += 1

            data = word.split('\t')
            # word_data, lemmas = data[:7], data[7:]

            word_data = data[:58] #the number here is the column positions
            morphology = ' '.join(data[58:]) #the number here is the column positions
            
            # segment out word data
            # bo_code, ref, brake, ketiv, qere, morph, strongs = word_data
            book_long, booknum, book_short, chapter, verse, bol_ref, orig_order, bol_monad_num, word, bol_surface, normalized, normalized_noaccent, lemma, bol_lemma, lemma_noaccent, lemma_translit, form_tag, functional_tag, strongs, strongs_unreliable, lemma_dict, bol_lemma_dict, gloss_EN, bol_gloss_EN, abc_order, bol_dict_abc, freq_lemma, bol_lemma_occ, bol_frequency_rank, sp, bol_psp, gn, bol_gender, nu, bol_number, nu_poss, bol_possessor_number, ps, bol_person, case, bol_case, tense, bol_tense, voice, bol_voice, mood, bol_mood, degree, extra, bol_suffix, bol_verb_type, bol_noun_stem, word_stem, declension, bol_noun_declension, vocab_ReadGreekIn30Days, alt_morphology, verb_class = word_data



            #try:
            #    verse = int(verse)
            #except ValueError:
            #    subverse = verse[-1:]
            #    verse = verse[:-1]

            if verse == "":
                print(f'{orig_order}: {verse} {subverse}')


            # strongs_lemma, anlex_lemma = ' '.join(lemmas).split('!') # reconstitute lemmas and split on !

            # chapt, verse, wrdnum = [int(v) for v in patts['section'].match(ref).groups()]

            # -- handle TF events --

            # detect book boundary
            if prev_book != book:

                # end subverse
                # cv.feature(this_subverse, subverse=prev_subverse)
                # cv.terminate(this_subverse)

                # end verse
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)
                
                # end chapter
                cv.feature(this_chap, chapter=prev_chap)
                cv.terminate(this_chap)

                # end book
                cv.feature(this_book, book=prev_book)
                cv.terminate(this_book)
                
                # new book, chapter, verse, and subverse begin
                this_book = cv.node('book')
                prev_book = book
                this_chap = cv.node('chapter')
                prev_chap = chapter
                this_verse = cv.node('verse')
                prev_verse = verse
                #this_subverse = cv.node('subverse')
                #prev_subverse = subverse
                wrdnum = 1
            
            # detect chapter boundary
            elif prev_chap != chapter:

                # end subverse
                #cv.feature(this_subverse, subverse=prev_subverse)
                #cv.terminate(this_subverse)
                
                # end verse
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)
                
                # end chapter
                cv.feature(this_chap, chapter=prev_chap)
                cv.terminate(this_chap)
                
                # new chapter, verse, and subverse begin
                this_chap = cv.node('chapter')
                prev_chap = chapter
                this_verse = cv.node('verse')
                prev_verse = verse
                #this_subverse = cv.node('subverse')
                #prev_subverse = subverse
                wrdnum = 1
            
            # detect verse boundary
            elif prev_verse != verse:

                # end subverse
                #cv.feature(this_subverse, subverse=prev_subverse)
                #cv.terminate(this_subverse)

                # end verse
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)

                # new verse and subverse begin
                this_verse = cv.node('verse')
                prev_verse = verse
                #this_subverse = cv.node('subverse')
                #prev_subverse = subverse
                wrdnum = 1

            # detect subverse boundary
            #elif prev_subverse != subverse:
            #    cv.feature(this_subverse, subverse=prev_subverse)
            #    cv.terminate(this_subverse)
            #    this_subverse = cv.node('subverse')
            #    prev_subverse = subverse

                
            # detect paragraph boundary
            # if brake == 'P':
            #     cv.feature(this_para, para=para_track)
            #     cv.terminate(this_para)
            #     this_para = cv.node('paragraph') # start a new paragraph
            #     para_track += 1 # count paragraphs in the book
            
            
            if sentence_done:
                cv.feature(this_clause, clause=clause_track)
                cv.terminate(this_clause)
                cv.feature(this_sentence, sentence=sentence_track)
                cv.terminate(this_sentence)
                this_sentence = cv.node('sentence')
                sentence_track += 1
                this_clause = cv.node('clause')
                clause_track += 1

                sentence_done = False
                clause_done = False

            elif clause_done:
                cv.feature(this_clause, clause=clause_track)
                cv.terminate(this_clause)
                this_clause = cv.node('clause')
                clause_track += 1
                clause_done = False

            if text[-1:] == "." or text[-1:] == ";":
                sentence_done = True

            elif text[-1:] == "," or text[-1:] == "·":
                clause_done = True
            
            
                
            # make word object
            this_word = cv.slot()
            cv.feature(this_word, 
                    orig_order=orig_order,
                    bol_monad_num=bol_monad_num,
                    book_long=book_long,
                    booknum=booknum,
                    book_short=book_short,
                    chapter=chapter,
                    verse=verse,
                    bol_ref=bol_ref,
                    word=word,
                    bol_surface=bol_surface,
                    normalized=normalized,
                    lemma=lemma,
                    bol_lemma=bol_lemma,
                    lemma_translit=lemma_translit,
                    form_tag=form_tag,
                    functional_tag=functional_tag,
                    strongs=strongs,
                    strongs_unreliable=strongs_unreliable,
                    lemma_dict=lemma_dict,
                    bol_lemma_dict=bol_lemma_dict,
                    gloss_EN=gloss_EN,
                    bol_gloss_EN=bol_gloss_EN,
                    abc_order=abc_order,
                    bol_dict_abc=bol_dict_abc,
                    freq_lemma=freq_lemma,
                    bol_lemma_occ=bol_lemma_occ,
                    bol_frequency_rank=bol_frequency_rank,
                    sp=sp,
                    bol_psp=bol_psp,
                    gn=gn,
                    bol_gender=bol_gender,
                    nu=nu,
                    bol_number=bol_number,
                    nu_poss=nu_poss,
                    bol_possessor_number=bol_possessor_number,
                    ps=ps,
                    bol_person=bol_person,
                    case=case,
                    bol_case=bol_case,
                    tense=tense,
                    bol_tense=bol_tense,
                    voice=voice,
                    bol_voice=bol_voice,
                    mood=mood,
                    bol_mood=bol_mood,
                    degree=degree,
                    extra=extra,
                    bol_suffix=bol_suffix,
                    bol_verb_type=bol_verb_type,
                    bol_noun_stem=bol_noun_stem,
                    word_stem=word_stem,
                    declension=declension,
                    bol_noun_declension=bol_noun_declension,
                    vocab_ReadGreekIn30Days=vocab_ReadGreekIn30Days,
                    normalized_noaccent=normalized_noaccent,
                    lemma_noaccent=lemma_noaccent,
                    alt_morphology=alt_morphology,
                    verb_class=verb_class,
                       # ketiv=ketiv, 
                       # qere=qere, 
                       # strongs=strongs, 
                       # str_lem=strongs_lemma.strip(),
                       # anlex_lem=anlex_lemma.strip()
                      )
            cv.terminate(this_word)
        
        # end book and its objects
        # - end subverse
        #cv.feature(this_subverse, subverse=prev_subverse)
        #cv.terminate(this_subverse)
        
       # - end clause
        cv.feature(this_clause, clause=clause_track)
        cv.terminate(this_clause)

        # - end sentence
        cv.feature(this_sentence, sentence=sentence_track)
        cv.terminate(this_sentence)
        

        # - end verse
        cv.feature(this_verse, verse=prev_verse)
        cv.terminate(this_verse)
        
        # - end paragraph
        # cv.feature(this_para, para=para_track)
        # cv.terminate(this_para)
        
        # - end chapter
        cv.feature(this_chap, chapter=prev_chap)
        cv.terminate(this_chap)
        
        # - end book
        cv.feature(this_book, book=prev_book, book_code=bo)
        cv.terminate(this_book)


slotType = 'word'
otext = {'fmt:text-orig-full':'{word} ',
         'sectionTypes':'book,chapter,verse',
         'sectionFeatures':'book,chapter,verse'}

generic = {'Name': 'NA1904',
           'Version': '1904',
           'Author': 'Nestle & Aland',
           'Editors': 'Ulrik Sandborg Petersen, Eliran Wong, Oliver Glanz, BOL',
           'Converter': 'Oliver Glanz', 
           'Source:':'https://github.com/biblicalhumanities/Nestle1904/blob/master/morph/Nestle1904.csv',
           'Note':'?'}

intFeatures = {'chapter', 'verse'}

featureMeta = {  'book': {'description': 'book'},
                 'book_code':{'description': 'Short book abbreviation'},
                 'sentence': {'description': 'sentence unit'},
                 'clause': {'description': 'clause unit'},
                 'orig_order': {'description': 'word order within corpus'},
                 'bol_monad_num': {'description': 'BOL based word order within corpus'},
                 'book_long': {'description': 'fully spelled out book name'},
                 'booknum': {'description': 'NT book number (Matthew=1, Mark=2, Revelation=27)'},
                 'book_short': {'description': 'book name abbreviated'},
                 'chapter': {'description': 'book chapter'},
                 'verse': {'description': 'verse'},
                 'bol_ref': {'description': 'BOL based Bible reference'},
                 'word': {'description': 'word as it appears in the text'},
                 'bol_surface': {'description': 'BOL based word as it appears in the text'},
                 'normalized': {'description': 'surface word stripped of punctation'},
                 'lemma': {'description': 'lexeme'},
                 'bol_lemma': {'description': 'BOL based lexeme'},
                 'lemma_translit': {'description': 'lexeme translated'},
                 'form_tag': {'description': 'form tag'},
                 'functional_tag': {'description': 'functional tag'},
                 'strongs': {'description': 'strongs number'},
                 'strongs_unreliable': {'description': 'unrealiable strongs number'},
                 'lemma_dict': {'description': 'lexeme as it appears in the dictionary'},
                 'bol_lemma_dict': {'description': 'BOL based lexeme as it appears in the dictionary'},
                 'gloss_EN': {'description': 'English gloss'},
                 'bol_gloss_EN': {'description': 'BOL based English gloss'},
                 'abc_order': {'description': 'the dictionary position of a given word'},
                 'bol_dict_abc': {'description': 'BOL based dictionary position of a given word'},
                 'freq_lemma': {'description': 'number of occurence of a given lexeme'},
                 'bol_lemma_occ': {'description': 'BOL based number of occurence of a given lexeme'},
                 'bol_frequency_rank': {'description': 'BOL based frequency rank of a given lexeme'},
                 'sp': {'description': 'part of speech'},
                 'bol_psp': {'description': 'BOL based part of speech'},
                 'gn': {'description': 'gender'},
                 'bol_gender': {'description': 'BOL based gender'},
                 'nu': {'description': 'number'},
                 'bol_number': {'description': 'BOL based number'},
                 'nu_poss': {'description': 'number of posessor'},
                 'bol_possessor_number': {'description': 'BOL based number of posessor'},
                 'ps': {'description': 'person'},
                 'bol_person': {'description': 'BOL based person'},
                 'case': {'description': 'case'},
                 'bol_case': {'description': 'BOL based case'},
                 'tense': {'description': 'tense'},
                 'bol_tense': {'description': 'BOL based tense'},
                 'voice': {'description': 'voice'},
                 'bol_voice': {'description': 'BOL based voice'},
                 'mood': {'description': 'mood'},
                 'bol_mood': {'description': 'BOL based mood'},
                 'degree': {'description': 'degree'},
                 'extra': {'description': 'extra'},
                 'bol_suffix': {'description': 'BOL based suffix'},
                 'bol_verb_type': {'description': 'BOL based verb_type'},
                 'bol_noun_stem': {'description': 'BOL based noun stem/noun_type'},
                 'word_stem': {'description': 'stem of a word'},
                 'declension': {'description': 'noun declension'},
                 'bol_noun_declension': {'description': 'BOL based noun declension'},
                 'vocab_ReadGreekIn30Days': {'description': 'chapter categoried vocab as it appears in Larry Richards Textbook "Learning Greek in 30 days"'},
                 'normalized_noaccent': {'description': 'surface word stripped of punctations and accents'},
                 'lemma_noaccent': {'description': 'lemma stripped of accents"'},
                 'alt_morphology': {'description': 'alternative morphological interpretation of ambiguous nominal or verbal form'},            
                 'verb_class': {'description': 'verb classes according to Teresa Reeve and Oliver Glanz'},




               #'subverse': {'description': 'Subverse information'},
               # 'para': {'description': 'A paragraph number'},
               # 'ketiv': {'descrption': 'The text as it is written in the printed Tischendorf'},
               # 'qere': {'description': 'The text as the editor thinks it should have been'},
               # 'strongs': {'description': 'A word\'s number in Strongs'},
               # 'str_lem': {'description': 'Word lemma that corresponds to The NEW Strong\'sComplete Dictionary of Bible Words'},
               # 'anlex_lem': {'description': 'Word lemma that corresponds to Friberg, Friberg and Miller\'s ANLEX'}
              }


# configure metadata/output
version = '1904'
generic['Version'] = version

output = os.path.join(output_dirs, version)

print(f'Processing Version {version}')
output_dir = output_dirs.format(version=version)

TF = Fabric(locations=output_dir, silent=True)
cv = CV(TF)

cv.walk(director,
                slotType,
                otext=otext,
                generic=generic,
                intFeatures=intFeatures,
                featureMeta=featureMeta,
                warn=True,
                force=False,)

import os
import re
import collections
import json
import csv
# from glob import glob
from tf.fabric import Fabric
from tf.convert.walker import CV
# from tf.compose import modify

source_dirs = 'input' # "input" is the name of the input folder that contains the source file
output_dirs = 'output' # "output" is the name of the output folder to which the finished TF files will be dumped into

bo2book = {line.split()[0]:line.split()[1] for line in '''
NTt5 New_Testament
'''.split('\n') if line} # "OT" is the name of the file in the input folder AND "split()" splits at space

# patts = {'section': re.compile('(\d*):(\d*)\.(\d*)')}

def director(cv):
        
    '''
    Walks through NA1904 and triggers
    slot and node creation events.
    '''
        
    # process books in order
    for bo, book in bo2book.items():
        
        book_loc = os.path.join(source_dirs, f'{bo}.txt') #for some reason the files with fileextension "xlsx" and "csv" do not work
        
        print(f'\thandling {book_loc}...')
        
        with open(book_loc, 'r', encoding="utf8") as infile:
            text = [w for w in infile.read().split('\n') if w]
            
        this_book = cv.node('book')
            
        # keep track of when to trigger paragraph, chapter, and verse objects
        # para_track = 1 # keep counts of paragraphs
        prev_book = "Matthew" # start at Genesis
        prev_chap = 1 # start at 1
        prev_verse = 1 # start at 1
        
        sentence_track = 1
        sentence_done = False
        clause_track = 1
        clause_done = False
        
        wrdnum = 0 # start at 0
        this_chap = cv.node('chapter')
        # this_para = cv.node('paragraph')
        this_verse = cv.node('verse')
        this_sentence = cv.node('sentence')
        this_clause = cv.node('clause')
        
        
        
        
        # iterate through words and construct objects
        for word in text:

            wrdnum += 1

            data = word.split('\t')
            # word_data, lemmas = data[:7], data[7:]

            word_data = data[:29] #the number here is the amount of columns
            morphology = ' '.join(data[29:]) #the number here is the amount of columns
            
            # segment out word data
            # bo_code, ref, brake, ketiv, qere, morph, strongs = word_data
            orig_order, book, book_short, chapter, verse, word, normalized, lemma, morph_form, morph_functional, strongs, lexeme_dict, gloss, bol_lemma_dict, bol_gloss, lemma_translit, bol_dict_abc, abc_order, freq_lemma, sp, gn, nu, ps, case, tense, voice, mood, degree, normalized_noaccent, extra = word_data



            #try:
            #    verse = int(verse)
            #except ValueError:
            #    subverse = verse[-1:]
            #    verse = verse[:-1]

            if verse == "":
                print(f'{orig_order}: {verse} {subverse}')


            # strongs_lemma, anlex_lemma = ' '.join(lemmas).split('!') # reconstitute lemmas and split on !

            # chapt, verse, wrdnum = [int(v) for v in patts['section'].match(ref).groups()]

            # -- handle TF events --

            # detect book boundary
            if prev_book != book:

                # end subverse
                # cv.feature(this_subverse, subverse=prev_subverse)
                # cv.terminate(this_subverse)

                # end verse
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)
                
                # end chapter
                cv.feature(this_chap, chapter=prev_chap)
                cv.terminate(this_chap)

                # end book
                cv.feature(this_book, book=prev_book)
                cv.terminate(this_book)
                
                # new book, chapter, verse, and subverse begin
                this_book = cv.node('book')
                prev_book = book
                this_chap = cv.node('chapter')
                prev_chap = chapter
                this_verse = cv.node('verse')
                prev_verse = verse
                #this_subverse = cv.node('subverse')
                #prev_subverse = subverse
                wrdnum = 1
            
            # detect chapter boundary
            elif prev_chap != chapter:

                # end subverse
                #cv.feature(this_subverse, subverse=prev_subverse)
                #cv.terminate(this_subverse)
                
                # end verse
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)
                
                # end chapter
                cv.feature(this_chap, chapter=prev_chap)
                cv.terminate(this_chap)
                
                # new chapter, verse, and subverse begin
                this_chap = cv.node('chapter')
                prev_chap = chapter
                this_verse = cv.node('verse')
                prev_verse = verse
                #this_subverse = cv.node('subverse')
                #prev_subverse = subverse
                wrdnum = 1
            
            # detect verse boundary
            elif prev_verse != verse:

                # end subverse
                #cv.feature(this_subverse, subverse=prev_subverse)
                #cv.terminate(this_subverse)

                # end verse
                cv.feature(this_verse, verse=prev_verse)
                cv.terminate(this_verse)

                # new verse and subverse begin
                this_verse = cv.node('verse')
                prev_verse = verse
                #this_subverse = cv.node('subverse')
                #prev_subverse = subverse
                wrdnum = 1

            # detect subverse boundary
            #elif prev_subverse != subverse:
            #    cv.feature(this_subverse, subverse=prev_subverse)
            #    cv.terminate(this_subverse)
            #    this_subverse = cv.node('subverse')
            #    prev_subverse = subverse

                
            # detect paragraph boundary
            # if brake == 'P':
            #     cv.feature(this_para, para=para_track)
            #     cv.terminate(this_para)
            #     this_para = cv.node('paragraph') # start a new paragraph
            #     para_track += 1 # count paragraphs in the book
            
            
            if sentence_done:
                cv.feature(this_clause, clause=clause_track)
                cv.terminate(this_clause)
                cv.feature(this_sentence, sentence=sentence_track)
                cv.terminate(this_sentence)
                this_sentence = cv.node('sentence')
                sentence_track += 1
                this_clause = cv.node('clause')
                clause_track += 1

                sentence_done = False
                clause_done = False

            elif clause_done:
                cv.feature(this_clause, clause=clause_track)
                cv.terminate(this_clause)
                this_clause = cv.node('clause')
                clause_track += 1
                clause_done = False

            if text[-1:] == "." or text[-1:] == ";":
                sentence_done = True

            elif text[-1:] == "," or text[-1:] == "·":
                clause_done = True
            
            
                
            # make word object
            this_word = cv.slot()
            cv.feature(this_word, 
                        orig_order=orig_order,
                        book=book,
                        book_short=book_short,
                        chapter=chapter,
                        verse=verse,
                        word=word,
                        normalized=normalized,
                        lemma=lemma,
                        morph_form=morph_form,
                        morph_functional=morph_functional,
                        strongs=strongs,
                        lexeme_dict=lexeme_dict,
                        gloss=gloss,
                        bol_lemma_dict=bol_lemma_dict,
                        bol_gloss=bol_gloss,
                        lemma_translit=lemma_translit,
                        bol_dict_abc=bol_dict_abc,
                        abc_order=abc_order,
                        freq_lemma=freq_lemma,
                        sp=sp,
                        gn=gn,
                        nu=nu,
                        ps=ps,
                        case=case,
                        tense=tense,
                        voice=voice,
                        mood=mood,
                        degree=degree,
                        extra=extra          
                       
                       # ketiv=ketiv, 
                       # qere=qere, 
                       # strongs=strongs, 
                       # str_lem=strongs_lemma.strip(),
                       # anlex_lem=anlex_lemma.strip()
                      )
            cv.terminate(this_word)
        
        # end book and its objects
        # - end subverse
        #cv.feature(this_subverse, subverse=prev_subverse)
        #cv.terminate(this_subverse)
        
       # - end clause
        cv.feature(this_clause, clause=clause_track)
        cv.terminate(this_clause)

        # - end sentence
        cv.feature(this_sentence, sentence=sentence_track)
        cv.terminate(this_sentence)
        

        # - end verse
        cv.feature(this_verse, verse=prev_verse)
        cv.terminate(this_verse)
        
        # - end paragraph
        # cv.feature(this_para, para=para_track)
        # cv.terminate(this_para)
        
        # - end chapter
        cv.feature(this_chap, chapter=prev_chap)
        cv.terminate(this_chap)
        
        # - end book
        cv.feature(this_book, book=prev_book, book_code=bo)
        cv.terminate(this_book)


slotType = 'word'
otext = {'fmt:text-orig-full':'{word} ',
         'sectionTypes':'book,chapter,verse',
         'sectionFeatures':'book,chapter,verse'}

generic = {'Name': 'NA1904',
           'Version': '1904',
           'Author': 'Nestle & Aland',
           'Editors': 'Ulrik Sandborg Petersen, Eliran Wong, Oliver Glanz, BOL',
           'Converter': 'Oliver Glanz', 
           'Source:':'https://github.com/biblicalhumanities/Nestle1904/blob/master/morph/Nestle1904.csv',
           'Note':'?'}

intFeatures = {'chapter', 'verse'}

featureMeta = {
                'orig_order': {'description': 'word order within corpus'},
                'book': {'description': 'book'},
                'book_short': {'description': 'Short book abbreviation'},
                'book_code':{'description': 'Short book abbreviation'},				
                'chapter': {'description': 'chapter'},
                'verse': {'description': 'verse'},
                'sentence': {'description': 'sentence'},
                'clause': {'description': 'clause'},
                'word': {'description': 'word as it appears in the text'},
                'normalized': {'description': 'normalized word'},
                'lemma': {'description': 'lemma'},
                'morph_form': {'description': 'formal morphology coding'},
                'morph_functional': {'description': 'functional morphology coding'},
                'strongs': {'description': 'Strong\'s numbers'},
                'lexeme_dict': {'description': 'dictionary form of lemma'},
                'gloss': {'description': 'English translation'},
                'bol_lemma_dict': {'description': 'BOL based dictionary form'},
                'bol_gloss': {'description': 'BOL based English gloss'},    
                'lemma_translit': {'description': 'lemma transliteration'},    
                'bol_dict_abc': {'description': 'BOL based alphabetic order of words'},   
                'abc_order': {'description': 'NA1904 alphabetic order of words'},    
                'freq_lemma': {'description': 'word frequency in corpus'},    
                'sp': {'description': 'part of speech'},
                'gn': {'description': 'gender'},
                'nu': {'description': 'number'},
                'ps': {'description': 'person'},
                'case': {'description': 'case'},
                'tense': {'description': 'tense'},
                'voice': {'description': 'voice'},
                'mood': {'description': 'mood'},
                'degree': {'description': 'degree'},
                'extra': {'description': 'extra notes'},





               #'subverse': {'description': 'Subverse information'},
               # 'para': {'description': 'A paragraph number'},
               # 'ketiv': {'descrption': 'The text as it is written in the printed Tischendorf'},
               # 'qere': {'description': 'The text as the editor thinks it should have been'},
               # 'strongs': {'description': 'A word\'s number in Strongs'},
               # 'str_lem': {'description': 'Word lemma that corresponds to The NEW Strong\'sComplete Dictionary of Bible Words'},
               # 'anlex_lem': {'description': 'Word lemma that corresponds to Friberg, Friberg and Miller\'s ANLEX'}
              }


# configure metadata/output
version = '1904'
generic['Version'] = version

output = os.path.join(output_dirs, version)

print(f'Processing Version {version}')
output_dir = output_dirs.format(version=version)

TF = Fabric(locations=output_dir, silent=True)
cv = CV(TF)

cv.walk(director,
                slotType,
                otext=otext,
                generic=generic,
                intFeatures=intFeatures,
                featureMeta=featureMeta,
                warn=True,
                force=False,)

NA = use ("CenterBLC/NA", version="1904")

Search0 = '''
book book=Matthew
  chapter chapter=1
     verse verse=1
       word
     
'''
Search0 = NA.search(Search0)
NA.show(Search0, start=1, end=1, condensed=True, colorMap={1:'pink'}, extraFeatures={'clause','sentence','lemma','morph_form','morph_functional','normalized','strongs','lexeme_dict','gloss', 'sp', 'gn', 'nu', 'ps', 'case', 'tense', 'voice', 'mood', 'degree', 'extra', 'freq_lemma', 'bol_dict_abc', 'lemma_translit', 'abc_order', 'bol_lemma_dict','bol_gloss'})

Search1 = '''
verse book=Matt chapter=1 verse=1
      word
'''
Search1 = NA.search(Search1)
NA.show(Search1, start=1, end=1, condensed=True, colorMap={1:'pink'}, extraFeatures={'lemma'})

Search2 = '''
book book=Gen
  chapter chapter=1
    verse verse=1
        word word* lex_utf8 g_cons_utf8 morphology* translit_SBL
'''
Search2 = LXX.search(Search2)
LXX.show(Search2, start=1, end=1, condensed=True, colorMap={1:'pink'}, extraFeatures={'subverse'})

featureprep=pd.read_excel('d:/OneDrive/1200_AUS-research/Fabric-TEXT/0_data_NA1904/Nestle1904_source_v1.41.xlsx')
pd.set_option('display.max_columns', 100)
featureprep.head(5)

featureprep['bookshort'].unique()

featureprep['fullbookname']=featureprep['bookshort'].apply(bookname)
featureprep.head(30000)

morph['sp']=morph['morph_form'].apply(spconditions)
morph.head(20)

morph['gn']=morph['morph_form'].apply(gender)
morph.head(20)