Edge Cases: The Making of Network Navigator and Critical Approaches to DH Tools¶

import pandas as pd
import os
import json
import statsmodels.api as sm
import warnings
warnings.filterwarnings('ignore')

import altair as alt
alt.data_transformers.disable_max_rows()
alt.data_transformers.enable('json')
# Check if VS Code is running the script
if os.environ.get('VSCODE_PID') is not None:
    alt.renderers.enable('mimetype')
else:
    alt.renderers.enable('default')


import nltk
from nltk.tokenize import word_tokenize
## If you haven't downloaded the NLTK data sets yet, do so:
def download_nltk_data_if_needed(packages):
    for package in packages:
        try:
            nltk.data.find(package)
        except LookupError:
            nltk.download(package)

download_nltk_data_if_needed(['tokenizers/punkt', 'corpora/stopwords'])

article_data_directory_path = "./script/data/csvs/" # Where all the article datasets are stored
article_chart_directory_path = "./script/data/charts/" # Where all the article charts are stored

# Define the path to the output CSV file
output_csv_path = f"{article_data_directory_path}heppler_data/whatisdh_df.csv"

# Check if the output CSV file exists
if os.path.exists(output_csv_path):
    # If it exists, load the data from the output CSV file
    whatisdh_quotes_df = pd.read_csv(output_csv_path)
else:
    # If it doesn't exist, load the data from the original CSV file
    whatisdh_quotes_df = pd.read_csv(f"{article_data_directory_path}heppler_data/dayofquotes_full.csv")

    # Add a 'quote_id' column based on the DataFrame index
    whatisdh_quotes_df['quote_id'] = whatisdh_quotes_df.index

    # Add a 'lower_quote' column with the lowercase version of the 'quote' column
    whatisdh_quotes_df['lower_quote'] = whatisdh_quotes_df['quote'].str.lower()

    # Add a 'lower_tokenized_quote' column with the tokenized version of the 'lower_quote' column
    whatisdh_quotes_df['lower_tokenized_quote'] = whatisdh_quotes_df['lower_quote'].apply(word_tokenize)

    # Convert the 'date' column to datetime format
    whatisdh_quotes_df['date'] = pd.to_datetime(whatisdh_quotes_df['date'])

    # Add a 'year' column based on the 'date' column
    whatisdh_quotes_df['year'] = whatisdh_quotes_df['date'].dt.year

    # Add a 'quote_length' column with the length of each tokenized quote
    whatisdh_quotes_df['quote_length'] = whatisdh_quotes_df['lower_tokenized_quote'].apply(len)

    # Add a 'tool_frequency' column with the count of 'tool' in each tokenized quote
    whatisdh_quotes_df['tool_frequency'] = whatisdh_quotes_df['lower_tokenized_quote'].apply(lambda x: sum([1 for token in x if 'tool' in token]))

    # Add a 'scaled_tool_frequency' column with the 'tool_frequency' scaled by the 'quote_length'
    whatisdh_quotes_df['scaled_tool_frequency'] = whatisdh_quotes_df['tool_frequency'] / whatisdh_quotes_df['quote_length']

    # Group the DataFrame by 'year' and calculate the sum of 'quote_length', 'tool_frequency', and 'scaled_tool_frequency' for each year
    yearly_tool_counts_df = whatisdh_quotes_df.groupby(['year']).agg({'quote_length': 'sum', 'tool_frequency': 'sum', 'scaled_tool_frequency': 'sum'}).reset_index()

    # Group the DataFrame by 'year' and calculate the total count for each year
    yearly_counts_df = whatisdh_quotes_df.groupby('year').size().reset_index(name='total_counts')

    # Group the DataFrame by 'year' and calculate the count of quotes with 'tool' for each year
    yearly_tool_whatisdh_quotes_df = whatisdh_quotes_df[whatisdh_quotes_df.tool_frequency > 0].groupby(['year']).size().reset_index(name='tool_counts')

    # Merge the total counts and tool counts DataFrames on the 'year' column
    merged_counts_df = pd.merge(yearly_counts_df, yearly_tool_whatisdh_quotes_df, on='year', how='outer')

    # Merge the merged counts DataFrame and the tool counts DataFrame on the 'year' column
    final_counts_whatisdh_df = pd.merge(merged_counts_df, yearly_tool_counts_df, on='year', how='outer')

    # Add a 'ratio' column with the ratio of 'tool_counts' to 'total_counts'
    final_counts_whatisdh_df['ratio'] = final_counts_whatisdh_df['tool_counts'] / final_counts_whatisdh_df['total_counts']

    # Reshape the DataFrame from wide format to long format, with 'year' and 'ratio' as the identifier variables, and 'total_counts' and 'tool_counts' as the variable names
    reshaped_whatisdh_df = pd.melt(final_counts_whatisdh_df, id_vars=['year', 'ratio'], value_vars=['total_counts', 'tool_counts'])

    # Replace 'total_counts' with 'Total Number of Quotes By Year' in the 'variable' column
    reshaped_whatisdh_df.loc[reshaped_whatisdh_df['variable'] == 'total_counts', 'variable'] = 'Total Number of Quotes By Year'

    # Replace 'tool_counts' with 'Total Number of Quotes Containing "Tool" By Year' in the 'variable' column
    reshaped_whatisdh_df.loc[reshaped_whatisdh_df['variable'] == 'tool_counts', 'variable'] = 'Total Number of Quotes Containing "Tool" By Year'

    # Write the DataFrame to the output CSV file
    whatisdh_quotes_df.to_csv(output_csv_path, index=False)

whatisdh_chart_output_path = f"{article_chart_directory_path}whatisdh_chart.json"

# Check if the chart file exists
if os.path.exists(whatisdh_chart_output_path):
    # Load the chart from the JSON file
    with open(whatisdh_chart_output_path) as json_file:
        whatisdh_chart_dict = json.load(json_file)
    whatisdh_chart = alt.Chart.from_dict(whatisdh_chart_dict)
else:
    # Create a bar chart
    time_chart = alt.Chart(reshaped_whatisdh_df).mark_bar().encode(
        x=alt.X('year:O', title='Year', axis=alt.Axis(labelAngle=0)),
        y=alt.Y('value:Q', title='Total number of quotes defining DH'),
        color=alt.Color('variable:N', legend=alt.Legend(title='', orient='bottom', symbolLimit=0, labelLimit=400)),
        tooltip=[
            alt.Tooltip('year', title='Year'),
            alt.Tooltip('value', title='Total Number of Quotes'),
            alt.Tooltip('variable', title='Quote Category'),
            alt.Tooltip('ratio', title='Ratio of Quotes', format='.2f')
        ]
    ).properties(
        title={'text': 'Change in Number of Quotes Containing "Tool" Over Time', 'fontSize': 12},
        width=400,
        height=300
    )

    # Define the dependent and independent variables
    X = final_counts_whatisdh_df['tool_frequency']
    y = final_counts_whatisdh_df['quote_length']

    # Add a constant to the independent variable
    X = sm.add_constant(X)

    # Perform the OLS regression
    model = sm.OLS(y, X)
    results = model.fit()

    # Get the R-squared value
    r_squared = results.rsquared

    # Create the chart
    chart = alt.Chart(final_counts_whatisdh_df).mark_circle().encode(
        x=alt.X('tool_frequency', axis=alt.Axis(title='Frequency of "tool" in any quote defining DH')),
        y=alt.Y('quote_length', axis=alt.Axis(title='Length of any quote defining DH')),
        color=alt.Color('year:N', legend=alt.Legend(title='Year of Quote'), scale=alt.Scale(scheme='plasma')),
        size='ratio',
        tooltip=[
            alt.Tooltip('year', title='Year'),
            alt.Tooltip('tool_frequency', title='Frequency of Tool in Quote'),
            alt.Tooltip('quote_length', title='Quote Length'),
            alt.Tooltip('scaled_tool_frequency', title='Scaled Frequency of Tool in Quote', format='.2f'),
        ]
    ).properties(
        title={'text': 'Correlation Between Frequency of "Tool" in Quotes', 'fontSize': 12},
        width=300,
    )

    # Add the regression line
    combined_chart = chart + chart.transform_regression('tool_frequency', 'quote_length', method="linear").mark_line(color='red').encode(color=alt.Color(legend=None))

    # Add the R-squared value as a text mark
    text = alt.Chart({'values':[{'x': 0, 'y': 0}]}).mark_text(
        text=f'R-squared: {r_squared:.2f}',
        align='left',
        baseline='top',
        color='black',
        dx=5  # Nudges text to right so it doesn't appear on top of the bar
    ).encode(
        x=alt.value(0),  # pixels from left
        y=alt.value(0),  # pixels from top
    )

    # Combine the charts
    final_chart = combined_chart + text

    whatisdh_chart = (alt.hconcat(time_chart, final_chart).resolve_scale(color='independent', y="independent").properties(title="Frequency of 'Tool' in Quotes From whatisdigitalhumanities.com and Day of DH").configure(
            title=alt.TitleConfig(fontSize=20, font='Garamond', anchor='middle'),
            axis=alt.AxisConfig(labelFont='Garamond', titleFont='Garamond'),
            legend=alt.LegendConfig(labelFont='Garamond', titleFont='Garamond')
        ))
    whatisdh_chart.save(whatisdh_chart_output_path)

# Create a list of all primariy network analysis tools
network_tools = ['Gephi', 'Palladio', 'nodegoat', 'igraph', 'Textexture', 'Netlytic', 'sigma.js', 'Neo4j', 'NetworkX', 'NodeXL', 'Graphviz', 'Cytoscape']

import numpy as np
from typing import List, Optional, Tuple
from sklearn.feature_extraction.text import CountVectorizer

def count_term_frequencies(df: pd.DataFrame, text_column: str, term_list: List[str]) -> pd.DataFrame:
    """
    Vectorize the text and count the frequencies of specific terms.

    Parameters:
    df (pd.DataFrame): The input DataFrame.
    text_column (str): The name of the column in df that contains the text to process.
    term_list (List[str]): A list of terms for which to calculate frequencies.

    Returns:
    term_freq_df (pd.DataFrame): A DataFrame where each column corresponds to a term in term_list and the values are the frequencies of the term in each document.
    """
    # Initialize CountVectorizer with the provided terms as the vocabulary
    count_vec = CountVectorizer(ngram_range=(1,2), vocabulary=term_list)

    # Fit and transform the text data
    X = count_vec.fit_transform(df[text_column]).toarray()

    # Create a DataFrame with the term frequencies
    term_freq_df = pd.DataFrame(X, columns=count_vec.get_feature_names_out())

    # For terms that contain a period, use straight string matching
    for term in term_list:
        if '.' in term:
            term_freq_df[term] = df[text_column].str.count(term)

    return term_freq_df


def generate_word_counts(subset_df: pd.DataFrame, text_column: str, terms_list: List[str]) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    """
    Process a DataFrame to calculate term frequencies and tokenize text.

    Parameters:
    subset_df (pd.DataFrame): The input DataFrame.
    text_column (str): The name of the column in subset_df that contains the text to process.
    terms_list (List[str]): A list of terms for which to calculate frequencies.

    Returns:
    combined_df (pd.DataFrame): The original DataFrame, with additional columns for the lowercased text, tokenized text, token frequencies, and total tokens.
    """
    if 'lower_text' not in subset_df.columns:
        subset_df['lower_text'] = subset_df[text_column].str.lower()

    if 'total_tokens' not in subset_df.columns:
        subset_df['total_tokens'] = subset_df['lower_text'].str.split(' ').str.len()
    
    count_df = count_term_frequencies(subset_df, 'lower_text', terms_list)
    combined_df = pd.concat([subset_df, count_df], axis=1)
    
    return combined_df

def process_data(file_path: str, text_column: str, date_column: str, terms_list: List, data_origin: str, title: str, term_type: str, term_mapping: bool, joined_term: Optional[str] = None):
    """
    Process the data and calculate term frequencies.

    Args:
        file_path (str): The path to the CSV file.
        text_column (str): The name of the column containing the text data.
        date_column (str): The name of the column containing the date data.
        terms_list (List): A list of terms to calculate frequencies for.
        data_origin (str): The origin of the data.
        title (str): The title of the data.
        term_type (str): The type of term data.
        term_mapping (bool): Whether to map term names for formatting variables.
        joined_term (str, optional): A term that we've searched for in multiple formats that we want to normalize into one term. Defaults to None.

    Returns:
        pd.DataFrame: The processed and calculated DataFrame.
    """
    # Load the CSV file into a DataFrame
    df = pd.read_csv(file_path)

    # Create a subset of the DataFrame that only includes rows where 'full_text' is not null
    subset_df = df[df[text_column].notna()]
    subset_df = subset_df.reset_index(drop=True)
    # Create a 'cleaned_conference_year' column by converting the 'conference_year' column to string and appending "-01-01"
    subset_df[f"cleaned_{date_column}"] = subset_df[date_column].astype(str) + "-01-01"

    # Convert the 'cleaned_conference_year' column to datetime format
    subset_df[f"cleaned_{date_column}"] = pd.to_datetime(subset_df[f"cleaned_{date_column}"])

    original_terms_list = terms_list.copy()
    lower_terms_list = [term.lower() for term in terms_list]

    # Process the DataFrame to calculate term frequencies and tokenize text
    combined_df = generate_word_counts(subset_df, text_column, lower_terms_list)
    # If a tool column is provided, create the 'finalized_tool' column
    if joined_term:
        plural_joined_term = joined_term + "s"
        combined_df[f'finalized_{joined_term}'] = np.where(combined_df[joined_term].notnull(), combined_df[joined_term], combined_df[plural_joined_term])
        combined_df = combined_df.drop(columns=[joined_term, plural_joined_term])
        combined_df = combined_df.rename(columns={f"finalized_{joined_term}": joined_term})

    id_vars_columns = subset_df.columns.tolist()
    melted_combined_df = pd.melt(combined_df, id_vars=id_vars_columns, var_name=term_type, value_name='counts')
    melted_combined_df['scaled_counts'] = melted_combined_df['counts'] / melted_combined_df['total_tokens']
    
    # If term mapping is enabled, map the terms
    if term_mapping:
        # Create a mapping from lowercase tool name to the correct name
        term_mapping = {term.lower(): term for term in original_terms_list}
        melted_combined_df[term_type] = melted_combined_df[term_type].replace(term_mapping)

    # Group the DataFrame by 'cleaned_conference_year' and 'tool' and calculate the sum of 'counts' for each group, then reset the index
    summed_df = melted_combined_df.groupby([f"cleaned_{date_column}", term_type]).counts.sum().reset_index()

    # Group the DataFrame by 'cleaned_conference_year' and 'tool' and calculate the sum of 'scaled_counts' for each group, then reset the index
    scaled_df = melted_combined_df.groupby([f"cleaned_{date_column}", term_type]).scaled_counts.sum().reset_index()

    # Group the DataFrame by 'cleaned_conference_year' and calculate the sum of 'total_tokens' for each group, then reset the index
    total_tokens_df = melted_combined_df.groupby([f"cleaned_{date_column}", term_type]).total_tokens.sum().reset_index()

    # Merge the grouped DataFrames
    grouped_df = pd.merge(summed_df, scaled_df, on=[f"cleaned_{date_column}", term_type])

    # Merge the grouped DataFrame with the total tokens DataFrame
    grouped_df = pd.merge(grouped_df, total_tokens_df, on=[f"cleaned_{date_column}", term_type])

    # Multiply the 'scaled_counts' column by 100
    grouped_df.scaled_counts = grouped_df.scaled_counts * 100

    # Rename the 'cleaned_conference_year' column to 'date'
    grouped_df = grouped_df.rename(columns={f"cleaned_{date_column}": "date"})

    # Add a 'data_origin' column 
    grouped_df["data_origin"] = data_origin

    # Add a 'title' column
    grouped_df["title"] = title

    return grouped_df

# Define paths to charts json
network_tools_over_time_chart_output_path = f"{article_chart_directory_path}network_tools_over_time_chart.json"
relative_popularity_chart_output_path = f"{article_chart_directory_path}relative_popularity_chart.json"
rerun_chart_code = False # Set to True to rerun the code to generate the charts

# Check if the files exist and if the code should be rerun
if (os.path.exists(network_tools_over_time_chart_output_path) == False) or (os.path.exists(relative_popularity_chart_output_path) == False) or rerun_chart_code:
    # Load the CSV file into a DataFrame
    tapor_adho_df = pd.read_csv(f"{article_data_directory_path}barbot_et_al_data/tools-dh-proceedings.csv")

    # Reshape the DataFrame from wide format to long format, with 'Tool' as the identifier variable, 'year' as the variable name, and 'counts' as the value name
    melted_tapor_adho_df = tapor_adho_df.melt(id_vars=['Tool'], var_name='year', value_name='counts')

    # Create a 'date' column by concatenating the 'year' column with '-01-01' to represent the first day of the year
    melted_tapor_adho_df['date'] = melted_tapor_adho_df['year'].astype(str) + '-01-01'

    # Convert the 'date' column to datetime format
    melted_tapor_adho_df['date'] = pd.to_datetime(melted_tapor_adho_df['date'])

    # Group the DataFrame by 'Tool' and calculate the sum of 'counts' for each tool, then reset the index
    grouped_dh_tools = melted_tapor_adho_df.groupby(['year'])['counts'].sum().reset_index().rename(columns={"counts": "total_tokens"})

    # Merge the melted DataFrame with the grouped DataFrame on the 'year' column
    merged_tapor_adho_df = pd.merge(melted_tapor_adho_df, grouped_dh_tools, on='year', how='outer')

    # Create a subset of the DataFrame that only includes rows where 'Tool' is in 'network_tools'
    subset_tapor_adho_df = merged_tapor_adho_df[merged_tapor_adho_df.Tool.isin(network_tools)]

    # Drop the 'year' column from the subset DataFrame
    subset_tapor_adho_df = subset_tapor_adho_df.drop(columns=["year"])

    # Rename the 'Tool' column to 'tool'
    subset_tapor_adho_df = subset_tapor_adho_df.rename(columns={"Tool": "tools"})

    # Add a 'data_origin' column with the value 'TaPoR'
    subset_tapor_adho_df["data_origin"] = "TaPoR"

    # Add a 'title' column with the value 'TaPoR and DH Conferences Data by Barbot et al. (2019)'
    subset_tapor_adho_df["title"] = "TaPoR and DH Conferences Data by Barbot et al. (2019)"

    # Set scaled counts to None
    subset_tapor_adho_df['scaled_counts'] = subset_tapor_adho_df['counts'] / subset_tapor_adho_df['total_tokens'] * 100

# Check if the files exist and if the code should be rerun
if (os.path.exists(network_tools_over_time_chart_output_path) == False) or (os.path.exists(relative_popularity_chart_output_path) == False) or rerun_chart_code:
    grouped_index_conferences_df = process_data(
        file_path=f"{article_data_directory_path}/weingart_et_al_data/dh_conferences_works.csv",
        text_column="full_text",
        date_column="conference_year",
        terms_list=network_tools,
        data_origin="Index of DH Conferences",
        title="Index of DH Conferences by Weingart et al (2023)",
        term_type="tools",
        term_mapping=True,
        joined_term=None
    )

# Check if the files exist and if the code should be rerun
if (os.path.exists(network_tools_over_time_chart_output_path) == False) or (os.path.exists(relative_popularity_chart_output_path) == False) or rerun_chart_code:
    grouped_dhq_df = process_data(
        file_path=f"{article_data_directory_path}scraped_dhq_data/dhq_articles.csv",
        text_column="body_text",
        date_column="date_processed",
        terms_list=network_tools,
        data_origin="DHQ",
        title="DHQ Articles (2023)",
        term_type="tools",
        term_mapping=True,
        joined_term=None
    )
   

# Define colors for each tool to use the same colors across charts
color_network_map = {
    'Gephi': '#4c78a8', 
    'Palladio': '#aec7e8', 
    'nodegoat': '#ff7f0e', 
    'igraph': '#ffbb78', 
    'Textexture': '#2ca02c', 
    'Netlytic': '#98df8a', 
    'sigma.js': '#d62728', 
    'Neo4j': '#ff9896', 
    'NetworkX': '#9467bd', 
    'NodeXL': '#c5b0d5', 
    'Graphviz': '#8c564b', 
    'Cytoscape': '#c49c94'
    }

# Check if the chart file exists
if os.path.exists(network_tools_over_time_chart_output_path) and rerun_chart_code == False:
    # Load the chart from the JSON file
    with open(network_tools_over_time_chart_output_path) as json_file:
        network_tools_over_time_chart_dict = json.load(json_file)
    network_tools_over_time_chart = alt.Chart.from_dict(network_tools_over_time_chart_dict)
else:
    # Concatenate the three DataFrames into one
    combined_df = pd.concat([subset_tapor_adho_df, grouped_index_conferences_df, grouped_dhq_df])
    combined_df = combined_df.reset_index(drop=True)

    combined_df.date = combined_df.date.astype(str)
    combined_df.date = combined_df.date.str.split(" ").str[0]
    combined_df['date'] = pd.to_datetime(combined_df.date, format='%Y-%m-%d')
    combined_df = combined_df.rename(columns={"counts": "Raw Frequency of Mentions", "scaled_counts": "Frequency of Mentions Scaled by Total Tokens"})
    # Get a list of unique data origins
    data_origins = combined_df.data_origin.unique().tolist()

    # Initialize an empty list to store the charts
    charts = []

    # Get a list of unique tools
    list_tools = combined_df.tools.unique().tolist()

    # Create a list of colors for each tool
    color_list = [color_network_map[tool] for tool in list_tools]

    # Create a dropdown selection
    dropdown = alt.binding_select(
        options=['Raw Frequency of Mentions', 'Frequency of Mentions Scaled by Total Tokens'], 
        name='Select Counting Method on Y Axis'
    )
    ycol_param = alt.param(
        value="Raw Frequency of Mentions", 
        bind=dropdown
    )
    # Create a selection that allows the user to select points in the legend
    selection = alt.selection_multi(fields=['tools'], bind='legend')
    # Loop over each data origin
    # Create a color scale for the chart
    color_scale = alt.Scale(
            domain=list_tools,
            range=color_list
        )
    for data_origin in data_origins:
        # Create a subset of the DataFrame that only includes rows where 'data_origin' is the current data origin
        subset_df = combined_df[combined_df.data_origin == data_origin]
        # Set the width of the chart depending on the data origin
        width = 500 if data_origin != "TaPoR" else 300

        # Get the title of the chart from the 'title' column of the subset DataFrame
        title = subset_df.title.unique()[0]

        new_chart = alt.Chart(subset_df).mark_bar().encode(
            # Encode the 'year' column as the x-axis
            x=alt.X('year(date):T',  axis=alt.Axis(title="Year")),
            # Encode the 'counts' column as the y-axis
            y=alt.Y('y:Q', axis=alt.Axis(title="Frequency of Mentions")),
            # Encode the 'tool' column as the color
            color=alt.Color('tools:N', legend=alt.Legend(title="Tool"), scale=color_scale),
            # Set the opacity of the bars depending on the selection
            opacity=alt.condition(selection, alt.value(1), alt.value(0.2)),
            # Set the tooltip to show the 'tool', 'counts', 'date', and 'Frequency of Mentions Scaled by Total Tokens' columns
            tooltip=[
                alt.Tooltip('tools', title='Tool'),
                alt.Tooltip('Raw Frequency of Mentions'),
                alt.Tooltip('date:T', title='Date', format='%Y'),
                alt.Tooltip('Frequency of Mentions Scaled by Total Tokens', title="Frequency of Mentions Scaled by Total Tokens (Percentage)", format='.2f'),
                alt.Tooltip('total_tokens', title='Total Tokens'),
            ]
        ).transform_calculate(
                y=f'datum[{ycol_param.name}]'
            ).add_params(
                ycol_param, selection
            ).properties(
            # Set the title and width of the chart
            title=f"{title}",
            width=width
        )
        

        # Add the chart to the list of charts
        charts.append(new_chart)

    # Concatenate the charts horizontally
    network_tools_over_time_chart = alt.hconcat(*charts).properties(
        # Set the title of the concatenated chart
        title={
            "text": "Popularity of Network Analysis Tools Over Time and Across Multiple Datasets",
            "subtitle": ["The Y-Axis can be changed to show either the raw frequency of a tool name in each respective dataset or the frequency of mentions scaled by all tokens for that year.", "You can hover over individual tools for more information and the legend is also interactive, allowing you to select specific tools to highlight."],
        }
    ).configure(
        # Configure the title, axis, and legend of the chart
        title=alt.TitleConfig(fontSize=20, font='Garamond', anchor='middle'),
        axis=alt.AxisConfig(labelFont='Garamond', titleFont='Garamond'),
        legend=alt.LegendConfig(labelFont='Garamond', titleFont='Garamond')
    )
    network_tools_over_time_chart.save(network_tools_over_time_chart_output_path)

# Check if the chart file exists
if os.path.exists(relative_popularity_chart_output_path) and rerun_chart_code == False:
    # Load the chart from the JSON file
    with open(relative_popularity_chart_output_path) as json_file:
        relative_popularity_chart_dict = json.load(json_file)
    relative_popularity_chart = alt.Chart.from_dict(relative_popularity_chart_dict)
else:
    # Get a list of unique tools
    grouped_merged_dh_tools = merged_tapor_adho_df.groupby(['Tool'])['counts'].sum().reset_index()

    total_tokens = grouped_merged_dh_tools['counts'].sum()
    grouped_merged_dh_tools['scaled_counts'] = grouped_merged_dh_tools['counts'] / total_tokens * 100
    all_dh_tools = grouped_merged_dh_tools.Tool.unique().tolist()

    # Get the count of the second most frequent network tool
    second_frequent_network_tool = grouped_merged_dh_tools[grouped_merged_dh_tools.Tool.isin(network_tools)].sort_values(by='counts', ascending=False).head(2)[1:2].counts.values[0]

    # Get a list of tools that have a count greater than or equal to the count of the second most frequent network tool
    updated_network_tools = grouped_merged_dh_tools[grouped_merged_dh_tools.counts >= second_frequent_network_tool].sort_values(by='counts', ascending=False).Tool.unique().tolist()

    # Create a list of colors for each tool
    updated_network_colors = [color_network_map[tool] if tool in color_network_map else '#bab0ac' for tool in updated_network_tools]
    grouped_merged_dh_tools = grouped_merged_dh_tools.rename(columns={"counts": "Raw Frequency of Mentions", "scaled_counts": "Frequency of Mentions Scaled by All Tool Mentions"})
    # Create a dropdown selection
    dropdown = alt.binding_select(
        options=["Raw Frequency of Mentions",  "Frequency of Mentions Scaled by All Tool Mentions"], 
        name='Select Counting Method on Y Axis'
    )
    xcol_param = alt.param(
        value="Raw Frequency of Mentions",
        bind=dropdown
    )

    # Create a bar chart of the total tools
    total_tools = alt.Chart(grouped_merged_dh_tools[grouped_merged_dh_tools["Raw Frequency of Mentions"] >= second_frequent_network_tool]).mark_bar().encode(
        x=alt.X('x:Q', title='Number of mentions'),
        y=alt.Y('Tool:N', title='Tool', sort='-x'),
        color=alt.Color('Tool:N', legend=None, scale=alt.Scale(domain=updated_network_tools, range=updated_network_colors)),
        tooltip=[
            alt.Tooltip('Tool', title='Tool'),
            alt.Tooltip('Raw Frequency of Mentions', title='Raw Frequency of Mentions'),
            alt.Tooltip('Frequency of Mentions Scaled by All Tool Mentions', title='Frequency of Mentions Scaled by All Tool Mentions (Percentage)', format='.2f'),
        ]
    ).transform_calculate(
        x=f'datum[{xcol_param.name}]'
    ).add_params(xcol_param).properties(
        title="TaPoR and DH Conferences Data by Barbot et al. (2019)",
    )

    # Process the DataFrame to calculate term frequencies and tokenize text
    grouped_index_conferences_df = process_data(
        file_path=f"{article_data_directory_path}weingart_et_al_data/dh_conferences_works.csv",
        text_column="full_text",
        date_column="conference_year",
        terms_list=all_dh_tools,
        data_origin="Index of DH Conferences",
        title="Index of DH Conferences by Weingart et al (2023)",
        term_type="all_tools",
        term_mapping=True,
        joined_term=None
    )
    all_tools_index_conferences_df = grouped_index_conferences_df.groupby('all_tools').agg({'counts': 'sum'}).reset_index()

    total_index_tools = all_tools_index_conferences_df['counts'].sum()
    all_tools_index_conferences_df['scaled_counts'] = all_tools_index_conferences_df['counts'] / total_index_tools * 100

    # Get the frequency of the second most frequent network tool
    conference_second_frequent_network_tool = all_tools_index_conferences_df[all_tools_index_conferences_df.all_tools.isin(network_tools)].sort_values(by='counts', ascending=False).head(2)[1:2].counts.values[0]

    # Get a list of tools that have a frequency greater than or equal to the frequency of the second most frequent network tool
    conference_updated_network_tools = all_tools_index_conferences_df[all_tools_index_conferences_df.counts >= conference_second_frequent_network_tool].sort_values(by='counts', ascending=False).all_tools.unique().tolist()

    # Create a list of colors for each tool
    conference_updated_network_colors = [color_network_map[tool] if tool in color_network_map else '#bab0ac' for tool in conference_updated_network_tools]


    all_tools_index_conferences_df = all_tools_index_conferences_df.rename(columns={"all_tools": "Tool", 'counts': 'Raw Frequency of Mentions', 'scaled_counts': 'Frequency of Mentions Scaled by All Tool Mentions'})

    # Create a bar chart of the total tools
    conference_total_tools = alt.Chart(all_tools_index_conferences_df[all_tools_index_conferences_df["Raw Frequency of Mentions"] >= conference_second_frequent_network_tool]).mark_bar().encode(
        x=alt.X('x:Q', title='Number of mentions'),
        y=alt.Y('Tool:N', sort='-x'),
        color=alt.Color('Tool:N', legend=None, scale=alt.Scale(domain=conference_updated_network_tools, range=conference_updated_network_colors)),
        tooltip=[
            alt.Tooltip('Tool', title='Tool'),
            alt.Tooltip('Raw Frequency of Mentions', title='Raw Frequency of Mentions'),
            alt.Tooltip('Frequency of Mentions Scaled by All Tool Mentions', title='Frequency of Mentions Scaled by All Tool Mentions (Percentage)', format='.2f'),
        ]
    ).transform_calculate(
        x=f'datum[{xcol_param.name}]'
    ).add_params(xcol_param).properties(
        title="Index of DH Conferences by Weingart et al (2023)",
    )

    # Process the DataFrame to calculate term frequencies and tokenize text
    grouped_dhq_df = process_data(
        file_path=f"{article_data_directory_path}scraped_dhq_data/dhq_articles.csv",
        text_column="body_text",
        date_column="date_processed",
        terms_list=all_dh_tools,
        data_origin="DHQ",
        title="DHQ Articles (2023)",
        term_type="all_tools",
        term_mapping=True,
        joined_term=None
    )

    all_tools_dhq_df = grouped_dhq_df.groupby('all_tools').agg({'counts': 'sum'}).reset_index()
    total_dhq_tools = all_tools_dhq_df['counts'].sum()
    all_tools_dhq_df['scaled_counts'] = all_tools_dhq_df['counts'] / total_dhq_tools * 100

    # Get the frequency of the second most frequent network tool
    dhq_second_frequent_network_tool = all_tools_dhq_df[all_tools_dhq_df.all_tools.isin(network_tools)].sort_values(by='counts', ascending=False).head(2)[1:2].counts.values[0]

    # Get a list of tools that have a frequency greater than or equal to the frequency of the second most frequent network tool
    dhq_updated_network_tools = all_tools_dhq_df[all_tools_dhq_df.counts >= dhq_second_frequent_network_tool].sort_values(by='counts', ascending=False).all_tools.unique().tolist()

    # Create a list of colors for each tool
    dhq_updated_network_colors = [color_network_map[tool] if tool in color_network_map else '#bab0ac' for tool in dhq_updated_network_tools]

    all_tools_dhq_df = all_tools_dhq_df.rename(columns={"all_tools": "Tool", 'counts': 'Raw Frequency of Mentions', 'scaled_counts': 'Frequency of Mentions Scaled by All Tool Mentions'})
    # Create a bar chart of the total tools
    dhq_total_tools = alt.Chart(all_tools_dhq_df[all_tools_dhq_df["Raw Frequency of Mentions"] >= dhq_second_frequent_network_tool]).mark_bar().encode(
        x=alt.X('x:Q', title='Number of mentions'),
        y=alt.Y('Tool:N', sort='-x'),
        color=alt.Color('Tool:N', legend=None, scale=alt.Scale(domain=dhq_updated_network_tools, range=dhq_updated_network_colors)),
        tooltip=[
            alt.Tooltip('Tool', title='Tool'),
            alt.Tooltip('Raw Frequency of Mentions', title='Raw Frequency of Mentions'),
            alt.Tooltip('Frequency of Mentions Scaled by All Tool Mentions', title='Frequency of Mentions Scaled by All Tool Mentions (Percentage)', format='.2f'),
        ]
    ).transform_calculate(
        x=f'datum[{xcol_param.name}]'
    ).add_params(xcol_param).properties(
        title="DHQ Articles (2023)",
    )

    # Concatenate the charts horizontally
    relative_popularity_chart = alt.hconcat(total_tools, conference_total_tools, dhq_total_tools).configure(
        title=alt.TitleConfig(fontSize=20, font='Garamond', anchor='middle'),
        axis=alt.AxisConfig(labelFont='Garamond', titleFont='Garamond'),
        legend=alt.LegendConfig(labelFont='Garamond', titleFont='Garamond')
    ).resolve_scale(color='independent').properties(
        title={
            "text":"Ranking of Most Popular Network Analysis Tools Relative to Other DH Tools Across Multiple Datasets",
            "subtitle": ["The Y-Axis can be changed to show either the raw frequency of a tool name in each respective dataset", "or the frequency of mentions scaled by total references to any tool for that dataset.", "You can also hover over individual tools for more information."],
        }
    )
    relative_popularity_chart.save(relative_popularity_chart_output_path)

nn_chart_output_path = f"{article_chart_directory_path}/nn_chart.json"

# Check if the CSV files exist
if os.path.exists(nn_chart_output_path) and rerun_chart_code == False:
    with open(nn_chart_output_path) as json_file:
        nn_chart_dict = json.load(json_file)
    nn_chart = alt.Chart.from_dict(nn_chart_dict)
else:
    # Load the CSV files into DataFrames
    nn_commits_df = pd.read_csv(f"{article_data_directory_path}github_data/network_navigator_commits.csv")

    # Convert the 'commit.author.date' column to datetime format
    nn_commits_df['commit.author.date'] = pd.to_datetime(nn_commits_df['commit.author.date'], errors='coerce')

    # Replace 'Zoe LeBlanc' with 'ZoeLeBlanc' in the 'commit.author.name' column
    nn_commits_df.loc[nn_commits_df['commit.author.name'] == 'Zoe LeBlanc', 'commit.author.name'] = 'ZoeLeBlanc'

    # Create a subset of the DataFrame that only includes certain columns
    subset_commits_df = nn_commits_df[['commit.author.name', 'commit.author.date', 'stats.total', 'stats.additions', 'stats.deletions']]

    # Replace '.' with '_' in the column names
    cols = subset_commits_df.columns
    cols = [col.replace('.', '_') for col in cols]
    subset_commits_df.columns = cols

    # Prepare data for symmetric bar chart
    df_additions = subset_commits_df[["commit_author_name", "commit_author_date", "stats_additions"]].copy()
    df_additions["type"] = "additions"
    df_additions.rename(columns={"stats_additions": "value"}, inplace=True)

    df_deletions = subset_commits_df[["commit_author_name", "commit_author_date", "stats_deletions"]].copy()
    df_deletions["type"] = "deletions"
    df_deletions["stats_deletions"] = -df_deletions["stats_deletions"]
    df_deletions.rename(columns={"stats_deletions": "value"}, inplace=True)

    # Concatenate the additions and deletions DataFrames
    df_graph = pd.concat([df_additions, df_deletions])

    # Create the symmetric bar chart
    project_history = alt.Chart(df_graph).mark_bar().encode(
        x=alt.X('commit_author_date:T', title="Date"),
        y=alt.Y('value:Q', title="Additions/Deletions"),
        color=alt.Color('type:N', title="Type of Change"),
        tooltip=['commit_author_name:N', 'commit_author_date:T', 'value:Q', 'type:N'],
    ).properties(
        title="Additions and Deletions to Network Navigator",
    )

    # Create a selection that allows the user to select points in the legend
    selection = alt.selection_point(fields=['commit_author_name'], bind='legend')

    # Create a bar chart of the total contributions by each author
    author_chart = alt.Chart(subset_commits_df).mark_bar(opacity=0.7).encode(
        x=alt.X('commit_author_date:T', title='Date'),
        y=alt.Y('stats_total:Q', title='Total Contributions').stack(None),
        color=alt.Color('commit_author_name:N',
            scale=alt.Scale(scheme='tableau20'), 
            legend=alt.Legend(title="Author"),
            sort=alt.EncodingSortField(
                field="stats_total",  # The field to use for the sort
                op="sum",            # The operation to run on the field prior to sorting
                order="descending"   # The sort order
            )
        ),
        tooltip=['commit_author_name:N', 'commit_author_date:T', 'stats_total:Q'],
        opacity=alt.condition(selection, alt.value(1), alt.value(0.2)),
    ).add_params(selection).interactive(bind_y=False).properties(
        title="Contributors to Network Navigator",
    )

    # Concatenate the charts horizontally
    nn_chart = alt.hconcat(project_history, author_chart).resolve_scale(y='independent', color='independent').properties(
        title="History of Network Navigator via GitHub"
    ).configure(
        title=alt.TitleConfig(fontSize=20, font='Garamond', anchor='middle'),
        axis=alt.AxisConfig(labelFont='Garamond', titleFont='Garamond'),
        legend=alt.LegendConfig(labelFont='Garamond', titleFont='Garamond')
    )
    nn_chart.save(nn_chart_output_path)

terms = ['tool', 'tools', 'infrastructure']
tool_infrastructure_chart_output_path = f"{article_chart_directory_path}tool_infrastructure_chart.json"
rerun_chart_code = False # Set to True to rerun the code to generate the charts

def process_jstor_data(df: pd.DataFrame, terms: List[str], date_column: str, id_column: str, origin_column: str, word_count_column: str, joined_term: Optional[str] = None) -> pd.DataFrame:
    """
    Process and group a DataFrame to calculate term frequencies.

    Parameters:
    df (pd.DataFrame): The input DataFrame.
    terms (List[str]): A list of terms for which to calculate frequencies.
    date_column (str): The name of the column in df that contains the date.
    id_column (str): The name of the column in df that contains the ID.
    origin_column (str): The name of the column in df that contains the origin of the data.
    word_count_column (str): The name of the column in df that contains the word count.
    joined_term (str, optional): A term that we've searched for in multiple formats that we want to normalize into one term. Defaults to None.

    Returns:
    final_grouped_df (pd.DataFrame): A DataFrame containing the terms and their frequencies, grouped by date and origin.
    """
    def process_counts(row):
        for term in terms:
            split_term = term.split()
            column_length = 'unigramCount' if len(split_term) == 1 else 'bigramCount' if len(split_term) == 2 else 'trigramCount'
            counts = row[column_length][term] if term in row[column_length] else 0
            row[term] = counts
        return row

    df = df.apply(process_counts, axis=1)

    subset_df = df[[date_column, id_column, origin_column, word_count_column] + terms]
    # If a tool column is provided, create the 'finalized_tool' column
    if joined_term:
        plural_joined_term = joined_term + "s"
        subset_df[f'finalized_{joined_term}'] = np.where(subset_df[joined_term].notnull(), subset_df[joined_term], subset_df[plural_joined_term])
        subset_df = subset_df.drop(columns=[joined_term, plural_joined_term])
        subset_df = subset_df.rename(columns={f"finalized_{joined_term}": joined_term})
    melted_df = pd.melt(subset_df, id_vars=[date_column, id_column, origin_column, word_count_column], var_name='terms', value_name='counts')
    melted_df['scaled_counts'] = melted_df['counts'] / melted_df[word_count_column]
    total_df = melted_df.groupby([origin_column, date_column])[word_count_column].sum().reset_index().rename(columns={word_count_column: 'total_tokens'})
    grouped_df = melted_df.groupby([origin_column, date_column, 'terms']).counts.sum().reset_index()
    scaled_grouped_df = melted_df.groupby([origin_column, date_column, 'terms']).scaled_counts.sum().reset_index()
    final_grouped_df = pd.merge(grouped_df, scaled_grouped_df, on=[origin_column, date_column, 'terms'])
    final_grouped_df = pd.merge(final_grouped_df, total_df, on=[origin_column, date_column])
    final_grouped_df.scaled_counts = final_grouped_df.scaled_counts * 100
    final_grouped_df = final_grouped_df.rename(columns={origin_column: 'data_origin', date_column: 'date'})
    final_grouped_df["title"] = final_grouped_df.data_origin

    return final_grouped_df

terms = ['network', 'networks', 'network analysis']
network_analysis_chart_output_path = f"{article_chart_directory_path}network_analysis_chart.json"
rerun_chart_code = False # Set to True to rerun the code to generate the charts

# Check if the chart file exists
if os.path.exists(network_analysis_chart_output_path) and rerun_chart_code == False:
    # Load the chart from the JSON file
    with open(network_analysis_chart_output_path) as json_file:
        network_analysis_chart_dict = json.load(json_file)
    network_analysis_chart = alt.Chart.from_dict(network_analysis_chart_dict)
else:

    # Process the DataFrame to calculate term frequencies and tokenize text
    grouped_term_index_conferences_df = process_data(
        file_path=f"{article_data_directory_path}/weingart_et_al_data/dh_conferences_works.csv",
        text_column="full_text",
        date_column="conference_year",
        terms_list=terms,
        data_origin="Index of DH Conferences",
        title="Index of DH Conferences by Weingart et al (2023)",
        term_type="terms",
        term_mapping=False,
        joined_term="network"
    )

    # Process the DataFrame to calculate term frequencies and tokenize text
    grouped_term_dhq_df = process_data(
        file_path=f"{article_data_directory_path}scraped_dhq_data/dhq_articles.csv",
        text_column="body_text",
        date_column="date_processed",
        terms_list=terms,
        data_origin="DHQ",
        title="DHQ Articles (2023)",
        term_type="terms",
        term_mapping=False,
        joined_term="network"
    )

    if os.path.exists(f"{article_data_directory_path}constellate_data/jstor_data_network_analysis.csv") == False:
        jstor_data = pd.read_json(f"{article_data_directory_path}constellate_data/jstor_data-jsonl.jsonl", lines=True)
        jstor_data['datePublished'] = pd.to_datetime(jstor_data['datePublished'])
        grouped_jstor_data = process_jstor_data(jstor_data, terms, 'datePublished', 'id', 'isPartOf', 'wordCount', joined_term='network')
    else:
        grouped_jstor_data = pd.read_csv(f"{article_data_directory_path}constellate_data/jstor_data_network_analysis.csv")

    term_combined_df = pd.concat([grouped_term_index_conferences_df, grouped_term_dhq_df, grouped_jstor_data])
    term_combined_df = term_combined_df.reset_index(drop=True)
    term_combined_df = term_combined_df.rename(columns={"counts": "Raw Frequency of Mentions", "scaled_counts": "Frequency of Mentions Scaled by Total Tokens"})

    term_combined_df.date = term_combined_df.date.astype(str)
    term_combined_df.date = term_combined_df.date.str.split(" ").str[0]
    term_combined_df['date'] = pd.to_datetime(term_combined_df.date, format='%Y-%m-%d')


    dropdown = alt.binding_select(
        options=['Raw Frequency of Mentions', 'Frequency of Mentions Scaled by Total Tokens'], 
        name='Select Counting Method on Y Axis'
    )
    ycol_param = alt.param(
        value="Raw Frequency of Mentions",
        bind=dropdown
    )

    network_term = term_combined_df[term_combined_df.terms == 'network']
    selection = alt.selection_point(fields=['data_origin'], bind='legend')
    network_chart = alt.Chart(network_term).mark_bar().encode(
        x=alt.X('year(date):T', axis=alt.Axis(title="Year")),
        y=alt.Y('y:Q', axis=alt.Axis(title="Frequency of Mentions")),
        color=alt.Color('data_origin:N', legend=alt.Legend(title="Data Origin", symbolLimit=0, labelLimit=0), scale=alt.Scale( scheme='tableau20'), sort=alt.EncodingSortField(field="counts", op="sum", order='descending')),
        tooltip=[
            alt.Tooltip('data_origin', title='Data Origin'),
            alt.Tooltip('Raw Frequency of Mentions'),
            alt.Tooltip('date:T', title='Date', format='%Y'),
            alt.Tooltip('Frequency of Mentions Scaled by Total Tokens', title="Frequency of Mentions Scaled by Total Tokens (Percentage)", format='.2f'),
            alt.Tooltip('total_tokens', title='Total Tokens'),
        ],
        opacity=alt.condition(selection, alt.value(1), alt.value(0.2)),
    ).transform_calculate(
        y=f'datum[{ycol_param.name}]'
    ).add_params(
        ycol_param, selection
    ).properties(
        title="Popularity of the Term 'Network' Over Time and Across Multiple Datasets",
        width=500
    )

    network_analysis_term = term_combined_df[term_combined_df.terms == 'network analysis']

    selection = alt.selection_point(fields=['data_origin'], bind='legend')

    network_analysis_chart = alt.Chart(network_analysis_term).mark_bar().encode(
        x=alt.X('year(date):T', axis=alt.Axis(title="Year")),
        y=alt.Y('y:Q', axis=alt.Axis(title="Frequency of Mentions")),
        color=alt.Color('data_origin:N', legend=alt.Legend(title="Data Origin", symbolLimit=0, labelLimit=0), scale=alt.Scale( scheme='tableau20'), sort=alt.EncodingSortField(field="counts", op="sum", order='descending')),
        tooltip=[
            alt.Tooltip('data_origin', title='Data Origin'),
            alt.Tooltip('Raw Frequency of Mentions'),
            alt.Tooltip('date:T', title='Date', format='%Y'),
            alt.Tooltip('Frequency of Mentions Scaled by Total Tokens', title="Frequency of Mentions Scaled by Total Tokens (Percentage)", format='.2f'),
            alt.Tooltip('total_tokens', title='Total Tokens'),
        ],
        opacity=alt.condition(selection, alt.value(1), alt.value(0.2)),
    ).transform_calculate(
        y=f'datum[{ycol_param.name}]'
    ).add_params(
        ycol_param, selection
    ).properties(
        title=f"Popularity of the Term 'Network Analysis' Over Time and Across Multiple Datasets",
        width=500
    )

    network_analysis_chart = alt.vconcat(network_chart, network_analysis_chart).configure(
        title=alt.TitleConfig(fontSize=20, font='Garamond', anchor='middle'),
        axis=alt.AxisConfig(labelFont='Garamond', titleFont='Garamond'),
        legend=alt.LegendConfig(labelFont='Garamond', titleFont='Garamond')
    )
    network_analysis_chart.save(network_analysis_chart_output_path)

terms = ['tool criticism', 'algorithmic criticism']
tool_algo_criticism_chart_output_path = f"{article_chart_directory_path}tool_algo_criticism_chart.json"
rerun_chart_code = False # Set to True to rerun the code to generate the charts

# Check if the chart file exists
if os.path.exists(tool_algo_criticism_chart_output_path) and rerun_chart_code == False:
    # Load the chart from the JSON file
    with open(tool_algo_criticism_chart_output_path) as json_file:
        tool_algo_criticism_chart_dict = json.load(json_file)
    tool_algo_criticism_chart = alt.Chart.from_dict(tool_algo_criticism_chart_dict)
else:

    # Process the DataFrame to calculate term frequencies and tokenize text
    grouped_term_index_conferences_df = process_data(
        file_path=f"{article_data_directory_path}weingart_et_al_data/dh_conferences_works.csv",
        text_column="full_text",
        date_column="conference_year",
        terms_list=terms,
        data_origin="Index of DH Conferences",
        title="Index of DH Conferences by Weingart et al (2023)",
        term_type="terms",
        term_mapping=False,
        joined_term=None
    )

    # Process the DataFrame to calculate term frequencies and tokenize text
    grouped_term_dhq_df = process_data(
        file_path=f"{article_data_directory_path}scraped_dhq_data/dhq_articles.csv",
        text_column="body_text",
        date_column="date_processed",
        terms_list=terms,
        data_origin="DHQ",
        title="DHQ Articles (2023)",
        term_type="terms",
        term_mapping=False,
        joined_term=None
    )

    if os.path.exists(f"{article_data_directory_path}constellate_data/jstor_data_tool_algorithmic_criticism.csv") == False:
        jstor_data = pd.read_json(f"{article_data_directory_path}constellate_data/jstor_data-jsonl.jsonl", lines=True)
        jstor_data['datePublished'] = pd.to_datetime(jstor_data['datePublished'])
        grouped_jstor_data = process_jstor_data(jstor_data, terms, 'datePublished', 'id', 'isPartOf', 'wordCount', joined_term=None)
    else:
        grouped_jstor_data = pd.read_csv(f"{article_data_directory_path}constellate_data/jstor_data_tool_algorithmic_criticism.csv")

    term_combined_df = pd.concat([grouped_term_index_conferences_df, grouped_term_dhq_df, grouped_jstor_data])
    term_combined_df = term_combined_df.reset_index(drop=True)
    term_combined_df = term_combined_df.rename(columns={"counts": "Raw Frequency of Mentions", "scaled_counts": "Frequency of Mentions Scaled by Total Tokens"})

    term_combined_df.date = term_combined_df.date.astype(str)
    term_combined_df.date = term_combined_df.date.str.split(" ").str[0]
    term_combined_df['date'] = pd.to_datetime(term_combined_df.date, format='%Y-%m-%d')


    dropdown = alt.binding_select(
        options=['Raw Frequency of Mentions', 'Frequency of Mentions Scaled by Total Tokens'], 
        name='Select Counting Method on Y Axis'
    )
    ycol_param = alt.param(
        value="Raw Frequency of Mentions",
        bind=dropdown
    )

    tool_criticism_term = term_combined_df[term_combined_df.terms == 'tool criticism']
    selection = alt.selection_point(fields=['data_origin'], bind='legend')
    tool_criticism_chart = alt.Chart(tool_criticism_term).mark_bar().encode(
        x=alt.X('year(date):T', axis=alt.Axis(title="Year")),
        y=alt.Y('y:Q', axis=alt.Axis(title="Frequency of Mentions")),
        color=alt.Color('data_origin:N', legend=alt.Legend(title="Data Origin", symbolLimit=0, labelLimit=0), scale=alt.Scale( scheme='tableau20'), sort=alt.EncodingSortField(field="counts", op="sum", order='descending')),
        tooltip=[
            alt.Tooltip('data_origin', title='Data Origin'),
            alt.Tooltip('Raw Frequency of Mentions'),
            alt.Tooltip('date:T', title='Date', format='%Y'),
            alt.Tooltip('Frequency of Mentions Scaled by Total Tokens', title="Frequency of Mentions Scaled by Total Tokens (Percentage)", format='.2f'),
            alt.Tooltip('total_tokens', title='Total Tokens'),
        ],
        opacity=alt.condition(selection, alt.value(1), alt.value(0.2)),
    ).transform_calculate(
        y=f'datum[{ycol_param.name}]'
    ).add_params(
        ycol_param, selection
    ).properties(
        title="Popularity of the Term 'Tool Criticism' Over Time and Across Multiple Datasets",
        width=500
    )

    algorithmic_criticism_term = term_combined_df[term_combined_df.terms == 'algorithmic criticism']

    selection = alt.selection_point(fields=['data_origin'], bind='legend')

    algorithmic_criticism_chart = alt.Chart(algorithmic_criticism_term).mark_bar().encode(
        x=alt.X('year(date):T', axis=alt.Axis(title="Year")),
        y=alt.Y('y:Q', axis=alt.Axis(title="Frequency of Mentions")),
        color=alt.Color('data_origin:N', legend=alt.Legend(title="Data Origin", symbolLimit=0, labelLimit=0), scale=alt.Scale( scheme='tableau20'), sort=alt.EncodingSortField(field="counts", op="sum", order='descending')),
        tooltip=[
            alt.Tooltip('data_origin', title='Data Origin'),
            alt.Tooltip('Raw Frequency of Mentions'),
            alt.Tooltip('date:T', title='Date', format='%Y'),
            alt.Tooltip('Frequency of Mentions Scaled by Total Tokens', title="Frequency of Mentions Scaled by Total Tokens (Percentage)", format='.2f'),
            alt.Tooltip('total_tokens', title='Total Tokens'),
        ],
        opacity=alt.condition(selection, alt.value(1), alt.value(0.2)),
    ).transform_calculate(
        y=f'datum[{ycol_param.name}]'
    ).add_params(
        ycol_param, selection
    ).properties(
        title=f"Popularity of the Term 'Algorithmic Criticism' Over Time and Across Multiple Datasets",
        width=500
    )

    tool_algo_criticism_chart = alt.vconcat(tool_criticism_chart, algorithmic_criticism_chart).configure(
        title=alt.TitleConfig(fontSize=20, font='Garamond', anchor='middle'),
        axis=alt.AxisConfig(labelFont='Garamond', titleFont='Garamond'),
        legend=alt.LegendConfig(labelFont='Garamond', titleFont='Garamond')
    )
    tool_algo_criticism_chart.save(tool_algo_criticism_chart_output_path)