# load the BHSL app and data
BHS = use ("etcbc/BHSA",hoist=globals())

# The following will push the Text-Fabric stylesheet to this notebook (to facilitate proper display with notebook viewer)
BHS.dh(BHS.getCss())

F.trailer.freqList()

# find the parashots petuchot
petuchaQuery = '''
book book=Genesis|Exodus|Leviticus|Numeri|Deuteronomium
  word trailer~_P
'''

petuchaResults = BHS.search(petuchaQuery)

# find the parashot setumot
setumaQuery = '''
book book=Genesis|Exodus|Leviticus|Numeri|Deuteronomium
  word trailer~_S
'''

setumaResults = BHS.search(setumaQuery)

# Import necessary libraries
from collections import defaultdict

# Initialize a dictionary to store counts per book
petuchaCounts = defaultdict(int)

# Iterate over the results and count petuchot per book
for book, petucha in petuchaResults:
    petuchaCounts[book] += 1

# Sort the books alphabetically
sortedBooks = sorted(petuchaCounts.keys())

# Display the results in a formatted table
print(f"{'Book':<20}{'Number of Petuchot'}")
print('-' * 35)
for book in sortedBooks:
    print(f"{F.book.v(book):<20}{petuchaCounts[book]}")

# Import necessary libraries
from collections import defaultdict

# Initialize a dictionary to store counts per book
setumaCounts = defaultdict(int)

# Iterate over the results and count setuma per book
for book, setuma in setumaResults:
    setumaCounts[book] += 1

# Sort the books alphabetically
sortedBooks = sorted(setumaCounts.keys())

# Display the results in a formatted table
print(f"{'Book':<20}{'Number of Setumot'}")
print('-' * 35)
for book in sortedBooks:
    print(f"{F.book.v(book):<20}{setumaCounts[book]}")

# Import necessary libraries
import pandas as pd
from tf.app import use

# Function to get reference string from verse nodes
def getVerseReference(node):
    section = T.sectionFromNode(node)
    return f"{section[0]} {section[1]}:{section[2]}" if section else 'Unknown Reference'

# Function to process each petucha and append to list
def addPetuchaInfo(petuchaList, index, startWordNode, endWordNode, length, bookName):
    startVerseNodes = L.u(startWordNode, otype='verse')
    endVerseNodes = L.u(endWordNode, otype='verse')
    
    startRefStr = getVerseReference(startVerseNodes[0]) if startVerseNodes else 'Unknown Reference'
    endRefStr = getVerseReference(endVerseNodes[0]) if endVerseNodes else 'Unknown Reference'
    
    petuchaList.append({
        'Index': index,
        'StartRef': startRefStr,
        'EndRef': endRefStr,
        'Length': length,
        'Book': bookName
    })

# Initialize variables
petuchaInfo = []
currentPetuchaLength = 0
currentPetuchaStartWord = None
index = 1

# Find all words in the Torah
wordsInTorahQuery = '''
book book=Genesis|Exodus|Leviticus|Numeri|Deuteronomium
  word
'''
wordsInTorah = BHS.search(wordsInTorahQuery)

# Iterate over all words in the dataset
for bookNode, wordNode in wordsInTorah:
    # Get the trailer feature of the word
    trailer = F.trailer.v(wordNode) or ''  # prevent 'NoneType' errors
    
    # If starting a new petucha, record the start word
    if currentPetuchaStartWord is None:
        currentPetuchaStartWord = wordNode
        currentBookName = F.book.v(bookNode)
    
    # Increment the length counter
    currentPetuchaLength += 1

    # Check if the word ends with a petucha (represented by 'P' in the trailer)
    if 'P' in trailer:
        addPetuchaInfo(petuchaInfo, index, currentPetuchaStartWord, wordNode, currentPetuchaLength, currentBookName)
        
        # Reset the variables for the next petucha
        currentPetuchaLength = 0
        currentPetuchaStartWord = None
        index += 1

# Handle any remaining words after the last petucha
if currentPetuchaLength > 0 and currentPetuchaStartWord is not None:
    addPetuchaInfo(petuchaInfo, index, currentPetuchaStartWord, wordNode, currentPetuchaLength, currentBookName)

# Convert the petuchaInfo list to a pandas DataFrame for analysis
df = pd.DataFrame(petuchaInfo)

# Define the desired book order
orderedBooks = ['Genesis', 'Exodus', 'Leviticus', 'Numeri', 'Deuteronomium']

# Display per-book statistics using a specified formatting
print("\nStatistical overview of petucha lengths per book:")
bookStats = df.groupby('Book')['Length'].describe().round(2)
bookStats['count'] = bookStats['count'].astype(int)
bookStats['min'] = bookStats['min'].astype(int)
bookStats['max'] = bookStats['max'].astype(int)

# Calculate total row across all books
totalStats = pd.DataFrame({
    'count': [int(bookStats['count'].sum())],
    'mean': [round(bookStats['mean'].mean(), 2)],
    'std': [round(bookStats['std'].mean(), 2)],
    'min': [int(bookStats['min'].min())],
    '25%': [round(bookStats['25%'].mean(), 2)],
    '50%': [round(bookStats['50%'].mean(), 2)],
    '75%': [round(bookStats['75%'].mean(), 2)],
    'max': [int(bookStats['max'].max())]
}, index=['Total'])

# Concatenate the total row with the book_stats DataFrame
bookStats = pd.concat([bookStats, totalStats])

# Reorder bookStats based on the original order of books in petuchaInfo
bookStats = bookStats.reindex(orderedBooks + ['Total'])

# Configure display options to show all data on a single line for each book
pd.set_option('display.max_columns', None)  # Show all columns
pd.set_option('display.width', 1000)  # Wide display to avoid line wrapping
print(bookStats)

from bokeh.plotting import figure, show, output_notebook
from bokeh.models import ColumnDataSource, HoverTool
from bokeh.transform import factor_cmap
from bokeh.palettes import Category10

# Add a petucha index for plotting
df['petuchaIndex'] = df.index + 1

# Prepare data for Bokeh
source = ColumnDataSource(df)

# Define the color palette
books = df['Book'].unique()
palette = Category10[len(books)]
color_map = factor_cmap('Book', palette=palette, factors=books)

# Create the figure
output_notebook()  # To display the plot in a Jupyter notebook

p = figure(
    width=1000,
    height=700,
    title='Petucha lengths in the Torah (in words)',
    x_axis_label='Petucha index',
    y_axis_label='Length (in words)',
    tools="pan,wheel_zoom,box_zoom,reset,save"
)

# Add the scatter plot using scatter()
p.scatter(
    x='petuchaIndex',
    y='Length',
    source=source,
    size=8,
    color=color_map,
    legend_field='Book',
    marker='circle',
    line_color='black',
    fill_alpha=0.8
)

# Add hover tool
hover = HoverTool()
hover.tooltips = [
    ('Petucha index', '@Index'),
    ('Length', '@Length'),
    ('Start verse', '@StartRef'),
    ('End verse', '@EndRef'),
    (' ', ' ') # to get a blank line when multiple datapoint are grouped when hovering  
]
p.add_tools(hover)

# Customize legend
p.legend.location = 'top_right'
p.legend.click_policy = 'hide'

# Show the plot
show(p)

# Import necessary libraries
import pandas as pd
from tf.app import use

# Function to get reference string from verse nodes
def getVerseReference(node):
    section = T.sectionFromNode(node)
    return f"{section[0]} {section[1]}:{section[2]}" if section else 'Unknown Reference'

# Function to process each setuma and append to list
def addSetumaInfo(setumaList, index, startWordNode, endWordNode, length, bookName):
    startVerseNodes = L.u(startWordNode, otype='verse')
    endVerseNodes = L.u(endWordNode, otype='verse')
    
    startRefStr = getVerseReference(startVerseNodes[0]) if startVerseNodes else 'Unknown Reference'
    endRefStr = getVerseReference(endVerseNodes[0]) if endVerseNodes else 'Unknown Reference'
    
    setumaList.append({
        'Index': index,
        'StartRef': startRefStr,
        'EndRef': endRefStr,
        'Length': length,
        'Book': bookName
    })

# Initialize variables
setumaInfo = []
currentSetumaLength = 0
currentSetumaStartWord = None
index = 1

# Find all words in the Torah
wordsInTorahQuery = '''
book book=Genesis|Exodus|Leviticus|Numeri|Deuteronomium
  word
'''
wordsInTorah = BHS.search(wordsInTorahQuery)

# Iterate over all words in the dataset
for bookNode, wordNode in wordsInTorah:
    # Get the trailer feature of the word
    trailer = F.trailer.v(wordNode) or ''  # prevent 'NoneType' errors
    
    # If starting a new setuma, record the start word
    if currentSetumaStartWord is None:
        currentSetumaStartWord = wordNode
        currentBookName = F.book.v(bookNode)
    
    # Increment the length counter
    currentSetumaLength += 1

    # Check if the word ends with a Setuma (represented by 'S' in the trailer)
    if 'S' in trailer:
        addSetumaInfo(setumaInfo, index, currentSetumaStartWord, wordNode, currentSetumaLength, currentBookName)
        
        # Reset the variables for the next setuma
        currentSetumaLength = 0
        currentSetumaStartWord = None
        index += 1

# Handle any remaining words after the last setuma
if currentSetumaLength > 0 and currentSetumaStartWord is not None:
    addSetumaInfo(setumaInfo, index, currentSetumaStartWord, wordNode, currentSetumaLength, currentBookName)

# Convert the setumaInfo list to a pandas DataFrame for analysis
df = pd.DataFrame(setumaInfo)

# Define the desired book order
orderedBooks = ['Genesis', 'Exodus', 'Leviticus', 'Numeri', 'Deuteronomium']

# Display per-book statistics using a specified formatting
print("\nStatistical overview of setuma lengths per book:")
bookStats = df.groupby('Book')['Length'].describe().round(2)
bookStats['count'] = bookStats['count'].astype(int)
bookStats['min'] = bookStats['min'].astype(int)
bookStats['max'] = bookStats['max'].astype(int)

# Calculate total row across all books
totalStats = pd.DataFrame({
    'count': [int(bookStats['count'].sum())],
    'mean': [round(bookStats['mean'].mean(), 2)],
    'std': [round(bookStats['std'].mean(), 2)],
    'min': [int(bookStats['min'].min())],
    '25%': [round(bookStats['25%'].mean(), 2)],
    '50%': [round(bookStats['50%'].mean(), 2)],
    '75%': [round(bookStats['75%'].mean(), 2)],
    'max': [int(bookStats['max'].max())]
}, index=['Total'])

# Concatenate the total row with the book_stats DataFrame
bookStats = pd.concat([bookStats, totalStats])

# Reorder bookStats based on the original order of books in setumaInfo
bookStats = bookStats.reindex(orderedBooks + ['Total'])

# Configure display options to show all data on a single line for each book
pd.set_option('display.max_columns', None)  # Show all columns
pd.set_option('display.width', 1000)  # Wide display to avoid line wrapping
print(bookStats)

from bokeh.plotting import figure, show, output_notebook
from bokeh.models import ColumnDataSource, HoverTool
from bokeh.transform import factor_cmap
from bokeh.palettes import Category10

# Add a setuma index for plotting
df['setumaIndex'] = df.index + 1

# Prepare data for Bokeh
source = ColumnDataSource(df)

# Define the color palette
books = df['Book'].unique()
palette = Category10[len(books)]
color_map = factor_cmap('Book', palette=palette, factors=books)

# Create the figure
output_notebook()  # To display the plot in a Jupyter notebook

p = figure(
    width=1000,
    height=700,
    title='Setuma lengths in the Torah (in words)',
    x_axis_label='Setuma index',
    y_axis_label='Length (in words)',
    tools="pan,wheel_zoom,box_zoom,reset,save"
)

# Add the scatter plot using scatter()
p.scatter(
    x='setumaIndex',
    y='Length',
    source=source,
    size=8,
    color=color_map,
    legend_field='Book',
    marker='circle',
    line_color='black',
    fill_alpha=0.8
)

# Add hover tool
hover = HoverTool()
hover.tooltips = [
    ('Setuma index', '@Index'),
    ('Length', '@Length'),
    ('Start verse', '@StartRef'),
    ('End verse', '@EndRef'),
    (' ', ' ') # to get a blank line when multiple datapoint are grouped when hovering  
]
p.add_tools(hover)

# Customize legend
p.legend.location = 'top_right'
p.legend.click_policy = 'hide'

# Show the plot
show(p)