Academic Research: Exploring Fairness in Wikipedia Article Search¶

# example of a document in the corpus
file = '/usa/fmchen/trec_2022/trec_corpus_20220301_plain.json.gz'

with gzip.open(file, 'rb') as f:
    line = f.readline()
one_line = json.loads(line)
print(json.dumps(one_line, indent=2))

# setup spark session to process page meta data
spark = SparkSession.builder.appName("ReadingParquet").config("spark.executor.memory", "8g") \
    .config("spark.driver.memory", "8g") \
    .config("spark.sql.autoBroadcastJoinThreshold", "-1") \
    .config("spark.executor.extraJavaOptions", "-XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:InitiatingHeapOccupancyPercent=35") \
    .getOrCreate()

meta_df = spark.read.parquet("/usa/fmchen/trec_2022/trec_2022_articles_discrete.parquet")
eval_df = spark.read.json('/usa/fmchen/trec_2022/trec_2022_eval_reldocs.jsonl')

# Replace empty lists with "Unknown" in meta_df
meta_df = meta_df.withColumn(
    'page_subcont_regions', 
    F.when(F.size(F.col('page_subcont_regions')) == 0, F.array(F.lit("Unknown")))
     .otherwise(F.col('page_subcont_regions'))
)
# Explode the 'page_subcont_regions' array into individual records
meta_df = meta_df.withColumn("page_subcont_region", F.explode("page_subcont_regions"))

meta_df.select(['page_id','page_subcont_region']).take(20)

eval_df.show()

# Count occurrences of each unique value
frequency_df = meta_df.groupBy('page_subcont_region').count()
frequency_df = frequency_df.toPandas().sort_values(by='count')

# Plotting
plt.figure(figsize=(9, 6))
plt.bar(frequency_df['page_subcont_region'], frequency_df['count'], color='skyblue')
plt.xlabel('Page Topic Region')
plt.ylabel('Frequency')
plt.title('Frequency of Each Unique Value in page_subcont_region')
plt.xticks(rotation=90, fontsize=16)
plt.show()

# Explode rel_docs in eval_df
eval_df_exploded = eval_df.withColumn("page_id", F.explode("rel_docs"))

# Join the exploded eval_df with meta_df
joined_df = eval_df_exploded.join(meta_df, "page_id")

# Group by title and page_subcont_region, and count each group
region_counts = joined_df.groupBy("title", "page_subcont_region").count()

# Calculate the total counts per title for percentage calculation
total_counts = region_counts.groupBy("title").sum("count").withColumnRenamed("sum(count)", "total")

# Calculate percentages
final_df = region_counts.join(total_counts, "title")
final_df = final_df.withColumn("percentage", (F.col("count") / F.col("total")) * 100)

# Pivot the DataFrame to get one row per title with a column for each subcontinent region
pivot_df = final_df.groupBy("title").pivot("page_subcont_region").sum("percentage")

# Collect the data for plotting
plot_data_geo = pivot_df.toPandas().set_index("title")

# Ensure 'Unknown' is the last column
plot_data_geo = plot_data_geo[[col for col in plot_data_geo.columns if col != 'Unknown'] + ['Unknown']]
plot_data_geo = plot_data_geo.sort_values(by='Unknown')
# Define the number of light colors needed (excluding 'Unknown')
n_colors_geo = len(plot_data_geo.columns) - 1

base_color = sns.color_palette(["coral"])[0]
light_palette_geo = sns.light_palette(base_color, n_colors_geo + 1)[1:]
light_palette_geo.append('black')  # Ensure 'Unknown' is black

fig, ax = plt.subplots(figsize=(16, 9))

# Plot the geographic location distribution
plot_data_geo[::2].plot(kind='bar', stacked=True, ax=ax, color=light_palette_geo)
ax.set_title('Topic Geographic Location Group Membership Distribution', fontsize=20)
ax.set_ylabel('Percentage', fontsize=20)
ax.set_xlabel('')
ax.set_xticklabels([label[:30] + '...' if len(label) > 30 else label for label in plot_data_geo[::2].index], rotation=90, fontsize=20)
plt.tight_layout()

# Create custom patches for the legend
known_patch = mpatches.Patch(color=light_palette_geo[-2], label='Known Groups')
unknown_patch = mpatches.Patch(color='black', label='Unknown Group')

# Add the legend to the plot
ax.legend(bbox_to_anchor=(1.34, 1.34), loc='upper right', fontsize=20) # handles=[known_patch, unknown_patch]

# Show the plot
plt.show()

# group 1
plot_data_geo[plot_data_geo['Unknown']>=50].index

# group 2
plot_data_geo[plot_data_geo['Unknown']<=30].index

# Load spaCy model
nlp = spacy.load('en_core_web_lg')

# Define topics in both groups
group1 = ['Jazz', 'Linguistics', 'Songs', 'Chemistry', 'Mathematics', 'Fishes']
group2 = ['Trains', 'Rivers', 'Schools', 'Canada/WikiProject Political parties and politicians in Canada',
          'Film', 'Higher education', 'LGBT studies/LGBT Person task force', 'Ice Hockey', 'Architecture',
          'Football', 'Biography/sports and games work group', 'Olympics', 'Baseball', 'Basketball',
          'Women\'s sport', 'Women writers', 'Athletics', 'Biography/arts and entertainment work group',
          'Elections and Referendums', 'Journalism', 'LGBT studies']

# Geographic keywords
geo_keywords = ['geographic location']
geo_tokens = [nlp(keyword) for keyword in geo_keywords]

# Function to calculate geographic relevance scores
def calculate_scores(topics):
    scores = {}
    for topic in topics:
        topic_token = nlp(topic)
        similarities = [topic_token.similarity(geo_token) for geo_token in geo_tokens]
        scores[topic] = sum(similarities) / len(similarities)
    return scores

# Calculate scores for each group
group1_scores = calculate_scores(group1)
group2_scores = calculate_scores(group2)

# Merging the scores into a single dictionary for plotting
all_scores = {**{topic: score for topic, score in group1_scores.items()},
              **{topic: score for topic, score in group2_scores.items()}}

# Sorting topics for plotting (merge both groups)
all_topics_sorted = {k: v for k, v in sorted(all_scores.items(), key=lambda item: item[1], reverse=True)}

# Creating color mapping based on group membership
colors = ['blue' if topic in group1 else 'green' for topic in all_topics_sorted.keys()]

# Plotting
plt.figure(figsize=(10, 8))
bars = plt.barh(list(all_topics_sorted.keys()), list(all_topics_sorted.values()), color=colors)
plt.xlabel('Geographic Relevance Score')
plt.title('Geographic Relevance of Topics Across Two Groups')
plt.gca().invert_yaxis()  # Higher scores at the top

# Adding a legend
blue_patch = plt.Rectangle((0,0),1,1,fc="blue", edgecolor = 'none', label='Group (Unknown > 50%)')
green_patch = plt.Rectangle((0,0),1,1,fc='green', edgecolor = 'none', label='Group (Unknown < 30%)')
plt.legend(handles=[blue_patch, green_patch], loc='best')

plt.show()

# Load GeoJSON data
with open('world_geo_low.json', 'r') as f:
    geo_json_data = json.load(f)

# Function to create a Choropleth map for a given topic
def create_map(topic):
    # Prepare DataFrame for mapping
    df = result_df.loc[topic].reset_index()
    df.columns = ['Continent', 'Percentage']  # Make sure these match your DataFrame structure
    
    # Merge the DataFrame with the GeoJSON properties
    for feature in geo_json_data['features']:
        continent = feature['properties']['continent']
        # Match the continent and attach the percentage value for the tooltip
        if continent in df['Continent'].values:
            feature['properties']['Percentage'] = float(df[df['Continent'] == continent]['Percentage'])
        else:
            feature['properties']['Percentage'] = "No data"  # Handling missing data

    # Create map
    m = folium.Map(location=[20, 0], zoom_start=2,
                   zoom_control=False,
               scrollWheelZoom=False,
               dragging=False)

    # Choropleth layer
    folium.Choropleth(
        geo_data=geo_json_data,
        name='choropleth',
        data=df,
        columns=['Continent', 'Percentage'],
        key_on='feature.properties.continent',
        fill_color='YlOrRd',
        fill_opacity=0.7,
        line_opacity=0.001,  # Reduced opacity for country borders
        legend_name=f"Percentage of relevant documents come from (%)"
    ).add_to(m)

    # Add tooltips
    folium.GeoJson(
        data=geo_json_data,
        style_function=lambda x: {'fillColor': '#ffffff00', 'color': '#000000', 'weight': 2},  # Thicker continent borders
        tooltip=folium.GeoJsonTooltip(
            fields=['continent', 'Percentage'],
            aliases=['Continent: ', 'Percentage: '],
            localize=True
        )
    ).add_to(m)

    folium.LayerControl().add_to(m)
    return m

# nbviewer.org does not render dynamic content. So, I pick one topic here only. 
# For dropdown menu and select different topic, uncomment the following code

# Widget setup for topic selection
topics = result_df.index.tolist()
dropdown = widgets.Dropdown(
    options=topics,
    value=topics[0],
    description='Topic:',
    disabled=False,
)

map_widget = widgets.Output()

def on_change(change):
    if change['type'] == 'change' and change['name'] == 'value':
        with map_widget:
            map_widget.clear_output(wait=True)
            display(create_map(change['new']))

dropdown.observe(on_change)

# Display the dropdown and the initial map
display(dropdown, map_widget)
on_change({'type': 'change', 'name': 'value', 'new': topics[0]})  # Display initial topic map


# m = create_map('Rivers')

m

from IPython.display import Image 
Image(url="https://dl.dropbox.com/scl/fi/k10r6zxqyug9wgyjq7ssk/outsampleloss.png?rlkey=u9vfwsdn1qurhow5htslf3lt7", width=500)