In [73]:

import celltypist
from celltypist import models
import scanpy as sc
import pandas as pd 
import numpy as np
import anndata
import re
import h5py
import scipy.sparse as scs
import concurrent.futures
import scanpy.external as sce
import gc
from concurrent.futures import ProcessPoolExecutor
import copy

In [74]:

adata=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_2023-11-23.h5ad')

In [75]:

np.shape(adata.obs)

Out[75]:

(1193859, 57)

Get T cells if annotation is final on first Round of clustering¶

In [76]:

T_Cells_Annotation=pd.read_csv('T_Cells_Res1.5_Annotation.csv')
T_Cells_Annotation['leiden_resolution_1.5']=[str (x) for x in T_Cells_Annotation['leiden_resolution_1.5']]

In [77]:

T_Cells_Annotation_filtered=T_Cells_Annotation.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [78]:

adata.obs=adata.obs.merge(T_Cells_Annotation_filtered, on=["leiden_resolution_1.5"], how="left")

In [79]:

df_Part_1=adata.obs[adata.obs['leiden_resolution_1.5'].isin(T_Cells_Annotation_filtered['leiden_resolution_1.5'])]

In [80]:

df_Part_1.to_parquet("Annotation_Part_1.parquet")

In [81]:

np.shape(df_Part_1)

Out[81]:

(886753, 63)

C5+C12¶

In [82]:

T_Cells_Annotation_C5_C12=pd.read_csv('T_Cells_Res3_C5_C12_Annotation.csv')
T_Cells_Annotation_C5_C12['leiden_resolution_3_C5_C12']=[str (x) for x in T_Cells_Annotation_C5_C12['leiden_resolution_3_C5_C12']]

In [83]:

T_Cells_Annotation_C5_C12=T_Cells_Annotation_C5_C12.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [84]:

adata_C5_C12=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C5_C12_2023-11-28_redo.h5ad')

In [85]:

adata_C5_C12.obs=adata_C5_C12.obs.merge(T_Cells_Annotation_C5_C12, on=["leiden_resolution_3_C5_C12"], how="left")

In [86]:

df_Part_2=adata_C5_C12.obs[adata_C5_C12.obs['leiden_resolution_3_C5_C12'].isin(T_Cells_Annotation_C5_C12['leiden_resolution_3_C5_C12'])]

In [87]:

df_Part_2.to_parquet("Annotation_Part_2.parquet")

In [88]:

np.shape(df_Part_2)

Out[88]:

(114083, 66)

In [89]:

np.shape(adata_C5_C12.obs)

Out[89]:

(123022, 66)

In [90]:

np.shape(adata_C5_C12.obs.query("leiden_resolution_3_C5_C12.isin(['8','14'])"))

Out[90]:

(8939, 66)

In [91]:

(8939+114083)==np.shape(adata_C5_C12.obs)[0]

Out[91]:

True

C11¶

In [92]:

T_Cells_Annotation_C11=pd.read_csv('T_Cells_Res1.5_C11_Annotation.csv')
T_Cells_Annotation_C11['leiden_resolution_1.5_C11']=[str (x) for x in T_Cells_Annotation_C11['leiden_resolution_1.5_C11']]

In [93]:

T_Cells_Annotation_C11=T_Cells_Annotation_C11.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [94]:

adata_C11=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C11_2023-11-27.h5ad')

In [95]:

adata_C11.obs=adata_C11.obs.merge(T_Cells_Annotation_C11, on=["leiden_resolution_1.5_C11"], how="left")

In [96]:

df_Part_3=adata_C11.obs[adata_C11.obs['leiden_resolution_1.5_C11'].isin(T_Cells_Annotation_C11['leiden_resolution_1.5_C11'])]

In [97]:

df_Part_3.to_parquet("Annotation_Part_3.parquet")

In [98]:

np.shape(df_Part_3)

Out[98]:

(48095, 64)

In [99]:

np.shape(adata_C11.obs)

Out[99]:

(50060, 64)

In [100]:

np.shape(adata_C11.obs.query("`leiden_resolution_1.5_C11`.isin(['8'])"))

Out[100]:

(1965, 64)

In [101]:

(1965+48095)==np.shape(adata_C11.obs)[0]

Out[101]:

True

In [ ]:

C13¶

In [102]:

T_Cells_Annotation_C13=pd.read_csv('T_Cells_Res1.5_C13_Annotation.csv')
T_Cells_Annotation_C13['leiden_resolution_1.5_C13']=[str (x) for x in T_Cells_Annotation_C13['leiden_resolution_1.5_C13']]

In [103]:

T_Cells_Annotation_C13=T_Cells_Annotation_C13.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [104]:

adata_C13=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C13_2023-11-27_retry.h5ad')

In [105]:

adata_C13.obs=adata_C13.obs.merge(T_Cells_Annotation_C13, on=["leiden_resolution_1.5_C13"], how="left")

In [106]:

df_Part_4=adata_C13.obs[adata_C13.obs['leiden_resolution_1.5_C13'].isin(T_Cells_Annotation_C13['leiden_resolution_1.5_C13'])]

In [107]:

df_Part_4.to_parquet("Annotation_Part_4.parquet")

In [108]:

np.shape(df_Part_4)

Out[108]:

(36364, 64)

In [109]:

np.shape(adata_C13.obs)

Out[109]:

(42191, 64)

In [110]:

np.shape(adata_C13.obs.query("`leiden_resolution_1.5_C13`.isin(['1','11'])"))

Out[110]:

(5827, 64)

In [111]:

(5827+36364)==np.shape(adata_C13.obs)[0]

Out[111]:

True

In [ ]:

C14¶

In [112]:

T_Cells_Annotation_C14=pd.read_csv('T_Cells_Res1.5_C14_Annotation.csv')
T_Cells_Annotation_C14['leiden_resolution_1.5_C14']=[str (x) for x in T_Cells_Annotation_C14['leiden_resolution_1.5_C14']]

In [113]:

T_Cells_Annotation_C14=T_Cells_Annotation_C14.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [114]:

adata_C14=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C14_2023-11-27.h5ad')

In [115]:

adata_C14.obs=adata_C14.obs.merge(T_Cells_Annotation_C14, on=["leiden_resolution_1.5_C14"], how="left")

In [116]:

df_Part_5=adata_C14.obs[adata_C14.obs['leiden_resolution_1.5_C14'].isin(T_Cells_Annotation_C14['leiden_resolution_1.5_C14'])]

In [117]:

df_Part_5.to_parquet("Annotation_Part_5.parquet")

In [118]:

np.shape(df_Part_5)

Out[118]:

(40051, 64)

In [119]:

np.shape(adata_C14.obs)

Out[119]:

(40051, 64)

C15 gdT¶

In [120]:

T_Cells_Annotation_C15=pd.read_csv('T_Cells_Res1.5_C15_Annotation.csv')
T_Cells_Annotation_C15['leiden_resolution_1.5_C15_with_gdT_from_C5_C12_C11_C13']=[str (x) for x in T_Cells_Annotation_C15['leiden_resolution_1.5_C15_with_gdT_from_C5_C12_C11_C13']]

In [121]:

T_Cells_Annotation_C15=T_Cells_Annotation_C15.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [122]:

adata_C15=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C15_with_gdT_from_C5_C12_C11_C13_2023-11-29.h5ad')

In [123]:

adata_C15.obs=adata_C15.obs.merge(T_Cells_Annotation_C15, on=["leiden_resolution_1.5_C15_with_gdT_from_C5_C12_C11_C13"], how="left")

In [124]:

df_Part_6=adata_C15.obs[adata_C15.obs['leiden_resolution_1.5_C15_with_gdT_from_C5_C12_C11_C13'].isin(T_Cells_Annotation_C15['leiden_resolution_1.5_C15_with_gdT_from_C5_C12_C11_C13'])]

In [125]:

df_Part_6.to_parquet("Annotation_Part_6.parquet")

In [126]:

np.shape(df_Part_6)

Out[126]:

(51009, 66)

In [127]:

np.shape(adata_C15.obs)

Out[127]:

(51009, 66)

In [ ]:

C19¶

In [128]:

T_Cells_Annotation_C19=pd.read_csv('T_Cells_Res2_C19_Annotation.csv')
T_Cells_Annotation_C19['leiden_resolution_2_C19']=[str (x) for x in T_Cells_Annotation_C19['leiden_resolution_2_C19']]

In [129]:

T_Cells_Annotation_C19=T_Cells_Annotation_C19.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [130]:

adata_C19=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C19_2023-12-05.h5ad')

In [131]:

adata_C19.obs=adata_C19.obs.merge(T_Cells_Annotation_C19, on=["leiden_resolution_2_C19"], how="left")

In [132]:

df_Part_7=adata_C19.obs[adata_C19.obs['leiden_resolution_2_C19'].isin(T_Cells_Annotation_C19['leiden_resolution_2_C19'])]

In [133]:

df_Part_7.to_parquet("Annotation_Part_7.parquet")

In [134]:

np.shape(df_Part_7)

Out[134]:

(13029, 65)

In [135]:

np.shape(adata_C19.obs)

Out[135]:

(13029, 65)

In [ ]:

C21¶

In [136]:

T_Cells_Annotation_C21=pd.read_csv('T_Cells_Res1.5_C21_Annotation.csv')
T_Cells_Annotation_C21['leiden_resolution_1.5_C21']=[str (x) for x in T_Cells_Annotation_C21['leiden_resolution_1.5_C21']]

In [137]:

T_Cells_Annotation_C21=T_Cells_Annotation_C21.query("AIFI_L2_Final=='Yes' & AIFI_L3_Final=='Yes'")

In [138]:

adata_C21=sc.read_h5ad('/home/jupyter/reference_generating_new/T/Tcells_processed_LV2_C21_2023-11-27.h5ad')

In [139]:

adata_C21.obs=adata_C21.obs.merge(T_Cells_Annotation_C21, on=["leiden_resolution_1.5_C21"], how="left")

In [140]:

df_Part_8=adata_C21.obs[adata_C21.obs['leiden_resolution_1.5_C21'].isin(T_Cells_Annotation_C21['leiden_resolution_1.5_C21'])]

In [141]:

df_Part_8.to_parquet("Annotation_Part_8.parquet")

Check if cell number is same with origional adata¶

In [145]:

np.shape(adata.obs)[0]==np.shape(df_Part_1)[0]+np.shape(df_Part_2)[0]+np.shape(df_Part_3)[0]+np.shape(df_Part_4)[0]+np.shape(df_Part_5)[0]+np.shape(df_Part_6)[0]+np.shape(df_Part_7)[0]+np.shape(df_Part_8)[0]

Out[145]:

True

combine table¶

In [143]:

import os
import pandas as pd

In [144]:

csv_directory = '.' 

output_excel = 'combined.xlsx'  
writer = pd.ExcelWriter(output_excel, engine='openpyxl')
with pd.ExcelWriter(output_excel, engine='openpyxl') as writer:
    for csv_file in os.listdir(csv_directory):
        if csv_file.endswith('.csv'):
            file_path = os.path.join(csv_directory, csv_file)
            
            df = pd.read_csv(file_path)
            
            sheet_name = os.path.splitext(csv_file)[0]
            df.to_excel(writer, sheet_name=sheet_name, index=False)

In [ ]: