# load the N1904 app and data
N1904 = use ("centerblc/N1904", version="1.0.0", hoist=globals())

# The following will push the Text-Fabric stylesheet to this notebook (to facilitate proper display with notebook viewer)
N1904.dh(N1904.getCss())

# define query template
findLnValueEqual ='''
phrase
    word ln=49.5
'''
equalSearch=N1904.search(findLnValueEqual)

# print the results
for phrase, node in equalSearch:
    print (f'word node:{node}, ln feature:{F.ln.v(node)}')

# define query template
findLnValueRegexp ='''
phrase
    word ln~49.5
'''
regexpSearch=N1904.search(findLnValueRegexp)

# print the results
for phrase, node in regexpSearch:
    print (f'word node:{node}, ln feature:{F.ln.v(node)}')

# define query template
# The preceding 'r' before the template allows for a raw strings, preventing Python from altering the regex.
findUpdatedLnValueRegex =r'''
phrase
    word ln~\b49[.]5\b
'''

updatedRegexSearch=N1904.search(findUpdatedLnValueRegex)

# print the results
for phrase, node in updatedRegexSearch:
    print (f'word node:{node}, ln feature:{F.ln.v(node)}')

from collections import defaultdict
import pandas as pd
from IPython.display import display

# Initialize a dictionary to count the number of Catagories per word node
lnCategoryCounts = defaultdict(int)

# Iterate over each word in the Greek New Testament
for word in F.otype.s("word"):
    ln = F.ln.v(word)  # Retrieve the Louw-Nida classification for the word
    
    if ln:
        # Count the number of Louw-Nida categories for this word
        numLNs = len(ln.split())
    else:
        numLNs = 0  # For words with no Louw-Nida classification
    
    # Increment the count for this LN category count
    lnCategoryCounts[numLNs] += 1

# Convert to a DataFrame for easier display as a table
lnCategoryCountsDf = pd.DataFrame.from_dict(lnCategoryCounts, orient='index', columns=['Count'])
lnCategoryCountsDf.index.name = 'Number of LN categories'
lnCategoryCountsDf = lnCategoryCountsDf.sort_index()  # Sort by the number of LNs

# Display the table
display(lnCategoryCountsDf)

from collections import defaultdict
import pandas as pd
from IPython.display import display

# Initialize a dictionary to hold counts and delta for each Louw-Nida top-level classification
lnCounts = defaultdict(lambda: {'countAllItems': 0, 'countFirstItem': 0, 'delta': 0})

# Iterate over each word in the Greek New Testament
for word in F.otype.s("word"):
    ln = F.ln.v(word)  # Retrieve the Louw-Nida classification for the word
    
    if ln:  # Check if there is a valid Louw-Nida classification
        # Split the ln value into it's individual codes
        codes = ln.split()
        
        # Count all items: increment for each top-level value in codes
        for code in codes:
            topLevelValue = int(code.split('.')[0])
            lnCounts[topLevelValue]['countAllItems'] += 1

        # Count only the first item: increment for the first top-level value
        firstTopLevelValue = int(codes[0].split('.')[0])
        lnCounts[firstTopLevelValue]['countFirstItem'] += 1

# Calculate delta for each top-level classification
for topLevelValue, counts in lnCounts.items():
    counts['delta'] = counts['countAllItems'] - counts['countFirstItem']

# Convert to a DataFrame
lnCountsDf = pd.DataFrame.from_dict(lnCounts, orient='index')
lnCountsDf.index.name = 'Louw-Nida top-level'
lnCountsDf.columns = ['count all items', 'count only first item', 'delta']

# Sort by 'delta' in descending order
lnCountsDf = lnCountsDf.sort_values(by='delta', ascending=False)

# Display the sorted table
display(lnCountsDf)

print (f'word node:39180, ln feature:{F.ln.v(39180)}, domain feature:{F.domain.v(39180)}')

# define query template
# The preceding 'r' before the template allows for a raw strings, preventing Python from altering the regex.
findDomainRegex =r'''
phrase
    word domain~\b088015\b
'''

domainResults=N1904.search(findDomainRegex)

# print the first 10 results
printCounter=0
for phrase, node in domainResults:
    print (f'word node:{node}, domain feature:{F.domain.v(node)}')
    printCounter+=1
    if printCounter==10:break

# define query template
# The preceding 'r' before the template allows for a raw strings, preventing Python from altering the regex.
lightQuery =r'''
phrase
   word ln~\b14[.](3[6-9]|4[0-9]|5[0-2])\b
'''

lightSearch=N1904.search(lightQuery)

# print the first 5 results
printCounter=0
for phrase, node in lightSearch:
    print (f'word node:{node}, ln feature:{F.ln.v(node)}')
    printCounter+=1
    if printCounter==5:break

import pandas as pd
from collections import defaultdict
from IPython.display import display

# Initialize a dictionary to store counts for each lemma and ln classification
lemmaLnCounts = defaultdict(int)

# Populate the dictionary with counts from the search results
for phrase, node in lightSearch:
    ln = F.ln.v(node)  # Retrieve LN classification for the word node
    lemma = F.lemma.v(node)  # Retrieve lemma for the word node
    gloss = F.gloss.v(node)  # Retrieve english gloss for the word node
    if ln and lemma:
        lemmaLnCounts[(lemma, ln, gloss)] += 1

# Create the DataFrame without a default index
df = pd.DataFrame(
    [{'lemma': lemma, 'ln': ln, 'gloss': gloss, 'count': count} for (lemma, ln, gloss), count in lemmaLnCounts.items()]
).sort_values(by='count', ascending=False)

# Display the DataFrame without the index
display(df.style.hide(axis="index"))

import pandas as pd
from collections import defaultdict

# Initialize a dictionary to store counts for each book
bookCounts = defaultdict(int)

# Populate the dictionary with counts from the search results
for phrase, node in lightSearch:
    book = F.book.v(node)  # Retrieve book name
    if book:
        bookCounts[book] += 1

# Convert dictionary to a DataFrame
bookTotals = pd.DataFrame(list(bookCounts.items()), columns=['Book', 'Total Count'])

# Sort by total count in descending order
bookTotals = bookTotals.sort_values(by='Total Count', ascending=False).reset_index(drop=True)

# Display the totals per book as a nice-looking table
display(bookTotals)

import pandas as pd
from collections import defaultdict
from math import pi, cos, sin
from bokeh.io import show, output_notebook, save
from bokeh.plotting import figure
from bokeh.models import HoverTool, Label
from bokeh.palettes import Category20
from bokeh.resources import INLINE

# Enable Bokeh output for Jupyter Notebook and JupyterLab
output_notebook(resources=INLINE)

# Initialize a dictionary to store counts for each lemma, LN classification, gloss, and book
lemmaLnCounts = defaultdict(int)

# Populate the dictionary with counts from the search results
for phrase, node in lightSearch:
    ln = F.ln.v(node)  # Retrieve LN classification for the word node
    lemma = F.lemma.v(node)  # Retrieve lemma for the word node
    gloss = F.gloss.v(node)  # Retrieve English gloss for the word node
    book = F.book.v(node)  # Retrieve book name
    if ln and lemma:
        lemmaLnCounts[(lemma, ln, gloss, book)] += 1

# Convert dictionary to a DataFrame
df = pd.DataFrame(
    [(lemma, ln, gloss, book, count) for (lemma, ln, gloss, book), count in lemmaLnCounts.items()],
    columns=['lemma', 'ln', 'gloss', 'book', 'count']
)

# Group by book and aggregate lemma information per book
bookLemmaCounts = df.groupby(['book', 'lemma']).agg({'count': 'sum'}).reset_index()

# Sort lemmas by count in descending order within each book
bookLemmaCounts = bookLemmaCounts.sort_values(by=['book', 'count'], ascending=[True, False])

# Aggregate total counts per book for the pie chart and sort by total count in descending order
bookCounts = bookLemmaCounts.groupby('book')['count'].sum().reset_index()
bookCounts = bookCounts.sort_values(by='count', ascending=False).reset_index(drop=True)

# Create a custom column to hold sorted lemma breakdown in descending order for each book
tooltipData = bookLemmaCounts.groupby('book')[['lemma', 'count']].apply(
    lambda group: '\n'.join(f"{row['lemma']}: {row['count']}" for _, row in group.iterrows())
).reset_index(name='lemmaInfo')

# Merge sorted lemma information with book counts
bookCounts = bookCounts.merge(tooltipData, on='book', how='left')

# Total count and cumulative angle calculation
totalCount = bookCounts['count'].sum()
bookCounts['angle'] = bookCounts['count'] / totalCount * 2 * pi

# Sequential start and end angle calculation
angles = [0] + list(bookCounts['angle'].cumsum())
bookCounts['startAngle'] = angles[:-1]
bookCounts['endAngle'] = angles[1:]

# Assign colors from Category20 palette
colors = Category20[len(bookCounts)] if len(bookCounts) <= 20 else Category20[20]
bookCounts['color'] = colors[:len(bookCounts)]

# Initialize Bokeh figure with increased size and adjusted y-axis range for offset
pieChart = figure(height=800, width=1000, title="Distribution of 'light-related' lemmas per book",
           toolbar_location=None, tools="")

# Draw each wedge separately with a smaller radius and set individual hover data
for i, row in bookCounts.iterrows():
    wedge = pieChart.wedge(x=0, y=0.5, radius=0.5,  # Reduced radius to make the pie chart smaller
                           start_angle=row['startAngle'], end_angle=row['endAngle'],
                           line_color="white", fill_color=row['color'])

    # Add a custom hover tool for each wedge with book-specific info in a multi-line format
    hover = HoverTool(renderers=[wedge],
                      tooltips=[
                          ("Book", row['book']),
                          ("Total Count", f"{row['count']}"),
                          ("Lemmas", row['lemmaInfo'])
                      ])
    pieChart.add_tools(hover)
    
    # Calculate label position closer to the pie chart at a fixed distance
    angle = (row['startAngle'] + row['endAngle']) / 2
    labelX = 0.7 * cos(angle)  # Position labels at 0.7 distance from the center
    labelY = 0.5 + (0.7 * sin(angle))  # Adjust y-position by 0.5 to account for shifted chart center
    
    # Determine alignment for the left half
    alignment = "left" if -pi/2 < angle < pi/2 else "right"
    
    # Fine-tune label alignment on the left side
    if pi/2 < angle < 3 * pi / 2:
        labelX += 0.05  # Move labels on the left side slightly to the right

    # Draw connector line from pie segment edge to the center of the label
    lineEndX = labelX - 0.02 if alignment == "right" else labelX + 0.02  # Adjust line end to center of label
    lineX = [0.5 * cos(angle), lineEndX]
    lineY = [0.5 + (0.5 * sin(angle)), labelY]
    pieChart.line(x=lineX, y=lineY, line_width=1, color=row['color'])
    
    # Set label alignment for book name
    label = Label(x=labelX, y=labelY, text=row['book'], text_align=alignment, text_baseline="middle", text_font_size="10pt")
    pieChart.add_layout(label)

    # Position total count within the wedge
    countX = 0.35 * cos(angle)  # Position count text within the wedge, closer to the center
    countY = 0.5 + (0.35 * sin(angle))  # Adjust y-position by 0.5 to account for shifted chart center
    countLabel = Label(x=countX, y=countY, text=str(row['count']), text_align="center", text_baseline="middle", text_font_size="9pt", text_color="black")
    pieChart.add_layout(countLabel)

# Adjust grid/axis settings
pieChart.axis.axis_label = None
pieChart.axis.visible = False
pieChart.grid.grid_line_color = None

# Show plot in notebook
show(pieChart)

from IPython.display import HTML
import base64  # used to encode the data to be downloaded
from bokeh.embed import file_html
from bokeh.resources import CDN

def createDownloadLink(htmlContent, fileName, documentTitle, buttonText):

    # Convert plot to HTML string
    htmlContent = file_html(htmlContent, CDN, documentTitle)
    
    # Encode the HTML content to base64
    b64Html = base64.b64encode(htmlContent.encode()).decode()
    
    # Create the HTML download link
    downloadLink = f'''
    <a download="{fileName}" href="data:text/html;base64,{b64Html}">
        <button>{buttonText}</button>
    </a>
    '''
    return HTML(downloadLink)

# Display the download link in the notebook 
createDownloadLink(pieChart, 'light_related_lemmas_per_book.html', 'Distribution of \'light-related\' lemmas per book', 'Download pie-chart')

import pandas as pd
from bokeh.io import output_notebook, show
from bokeh.plotting import figure
from bokeh.models import ColumnDataSource, ColorBar
from bokeh.transform import linear_cmap
from bokeh.resources import INLINE
from bokeh.palettes import Viridis256

# Enable Bokeh output for Jupyter Notebook and JupyterLab
output_notebook(resources=INLINE)

# Prepare data dictionary for DataFrame creation
data = {
    'lemma': [],
    'topLevelLnCode': [],
    'count': []
}

# Populate data dictionary with lemma, top-level LN code, and count
for lemma, (totalCount, lnDict) in result.items():
    for lnCode, count in lnDict.items():
        topLevelCode = int(lnCode.split('.')[0])  # Extract top-level LN code as integer
        data['lemma'].append(lemma.lower())  # Convert lemma to lowercase
        data['topLevelLnCode'].append(topLevelCode)
        data['count'].append(count)

# Create DataFrame and normalize counts per lemma
df = pd.DataFrame(data)
dfGrouped = df.groupby(['lemma', 'topLevelLnCode'], as_index=False)['count'].sum()
dfGrouped['normalizedCount'] = dfGrouped.groupby('lemma')['count'].transform(lambda x: x / x.sum())

# Map top-level LN codes to descriptions
dfGrouped['louwNidaDescription'] = dfGrouped['topLevelLnCode'].map(louwNidaMapping)

# Sort lemmas alphabetically and LN codes numerically for plot arrangement
dfGrouped['lemma'] = pd.Categorical(dfGrouped['lemma'], categories=sorted(dfGrouped['lemma'].unique()))
dfGrouped['topLevelLnCode'] = pd.Categorical(dfGrouped['topLevelLnCode'], ordered=True, categories=sorted(set(dfGrouped['topLevelLnCode']), key=int))

# Set plot height dynamically based on the number of unique lemmas
numLemmas = len(dfGrouped['lemma'].unique())
plotHeight = 20 * numLemmas  # Adjust multiplier for spacing

# Prepare data source for plotting
sortedLnCodes = [str(code) for code in sorted(dfGrouped['topLevelLnCode'].cat.categories)]
source = ColumnDataSource(dfGrouped)

# Initialize Bokeh figure with dynamic height and increased width
lnHeatmap = figure(
    width=1250, height=plotHeight,
    title="Per-Lemma Normalized Heatmap by Top-Level LN Code",
    x_range=sortedLnCodes,  # Sorted x-axis labels
    y_range=sorted(dfGrouped['lemma'].unique(), reverse=True),
    tools="hover", tooltips=[("Lemma", "@lemma"), ("LN Code", "@topLevelLnCode"), ("LN Description", "@louwNidaDescription"), ("Fraction", "@normalizedCount{0.00}")],
)

# Rotate x-axis labels for readability
lnHeatmap.xaxis.major_label_orientation = 1.57  # 90 degrees in radians

# Define color mapper for heatmap
mapper = linear_cmap(field_name='normalizedCount', palette=Viridis256, low=0, high=1)

# Add heatmap rectangles to the plot
lnHeatmap.rect(x="topLevelLnCode", y="lemma", width=1, height=1, source=source, fill_color=mapper, line_color=None)

# Add color bar for normalized count reference
colorBar = ColorBar(color_mapper=mapper['transform'], width=8, location=(0, 0))
lnHeatmap.add_layout(colorBar, 'right')

# Display the plot
show(lnHeatmap)

from IPython.display import HTML
import base64  # used to encode the data to be downloaded
from bokeh.embed import file_html
from bokeh.resources import CDN

def createDownloadLink(htmlContent, fileName, documentTitle, buttonText):

    # Convert plot to HTML string
    htmlContent = file_html(htmlContent, CDN, documentTitle)
    
    # Encode the HTML content to base64
    b64Html = base64.b64encode(htmlContent.encode()).decode()
    
    # Create the HTML download link
    downloadLink = f'''
    <a download="{fileName}" href="data:text/html;base64,{b64Html}">
        <button>{buttonText}</button>
    </a>
    '''
    return HTML(downloadLink)

# Display the download link in the notebook 
createDownloadLink(lnHeatmap, 'louw_nida_heatmap.html', 'Interactive Louw-Nida Heatmap', 'Download Louw-Nida heatmap')

import subprocess
from IPython.display import display, HTML

# Display the active conda environment
!conda env list | findstr "*"

# Run conda list and capture the output
condaListOutput = subprocess.check_output("conda list", shell=True).decode("utf-8")

# Wrap the output with <details> and <summary> HTML tags
htmlOutput = "<details><summary>Click to view installed packages</summary><pre>"
htmlOutput += condaListOutput
htmlOutput += "</pre></details>"

# Display the HTML in the notebook
display(HTML(htmlOutput))