
import os
import pandas as pd
import numpy as np

import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
from tqdm import tqdm
from collections import Counter, defaultdict
from copy import deepcopy

from lifelines import CoxPHFitter
from lifelines import KaplanMeierFitter
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from scipy.stats import zscore

from IPython.core.display import display, HTML
display(HTML("<style>.container { width:80% !important; }</style>"))

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)


datadir = "/data/spatial/"
outdir = f"{datadir}/output/"

in_f1 = "TCGA-HNSC.htseq_counts.tsv.tpm.gene.txt"
in_f2 = 'DEGs_Metastatic_Cluster.txt'
in_f3 = "TCGA-HNSC.gdc_phenotype.LNM.txt"
in_f4 = "OSCC.RNA.gdc_sample_sheet.TCGA-HNSC.txt"
in_f5 = "tcga-xena-hub.s3.us-east-1.amazonaws.com_download_survival%2FHNSC_survival.txt"


df = pd.read_table(os.path.join(datadir, "input", in_f1))
df


gene_df = pd.read_table(os.path.join(datadir, "input", in_f2))
gene_df


meta_genes = gene_df["Metastatic_pri_GeneNames"]

# Exclude genes not in common
remove_items = ['TARS', 'HIST1H4A', 'HIST1H1E', 'HIST1H2BN', 'PALM2-AKAP2', 'HIST1H1D', 'SOD2', 'H2AFX', 'HIST1H4F', 'C16orf58', 'FAM207A']
filtered_list = [item for item in meta_genes if item not in remove_items]

df2 = df.T[filtered_list].apply(lambda x: np.log(1 + x)).apply(zscore).mean(axis=1).reset_index()
df2.columns = ['submitter_id.samples', 'Metastasis Sig. Mean Expr.']
df2['submitter_id.samples'] = df2['submitter_id.samples'].str.replace('.', '-')
df2


filtered_df = df.T[filtered_list].apply(lambda x: np.log(1 + x)).apply(zscore).reset_index()
df3 = pd.concat([df2, filtered_df], axis='columns')
df3


pheno = pd.read_table(os.path.join(datadir, "input", in_f3))
pheno


df4 = pd.merge(df3, pheno, on='submitter_id.samples', how='inner')
df4


duplicate_counts = df4['submitter_id'].value_counts()
duplicates = duplicate_counts[duplicate_counts > 1]
print(len(duplicates))

45


# Delete duplicate lines (exclude normal tissue)
df4 = df4.drop_duplicates(subset='submitter_id', keep='first')
duplicate_counts = df4['submitter_id'].value_counts()
duplicates = duplicate_counts[duplicate_counts > 1]
print(len(duplicates))

0


oscc_rna = pd.read_table(os.path.join(datadir, "input", in_f4), index_col="Unnamed: 0")
oscc_rna = oscc_rna.rename(columns={'colname':'index'})
oscc_rna


df5 = pd.merge(oscc_rna, df4, left_on='case_submitter_id', right_on='submitter_id', how='left')
df5


df6 = df5[['index_x', 'submitter_id', 'Metastasis Sig. Mean Expr.', 'LNM', 'met'] + filtered_list]
df6


surv_df = pd.read_table(datadir + "/input/" + in_f5)
surv_df


# Delete duplicate lines (exclude normal tissue)
cols_to_check = surv_df.columns.difference(['sample'])
surv_dup_df = surv_df.drop_duplicates(subset=cols_to_check)
surv_dup_df


merged_df = pd.merge(df6, surv_dup_df, left_on='submitter_id', right_on='_PATIENT', how='left')
merged_df


def evaluate_on_test(test, cph, selected_genes, avg_coefficients, iteration):
    """
    Evaluates the fitted Cox Proportional Hazard model on the test dataset by computing risk scores and plotting
    the Kaplan-Meier Survival Curve based on these scores.

    Parameters:
    - test: A DataFrame containing the test data.
    - cph: A fitted CoxPHFitter object.
    - selected_genes: A list of genes that were selected during regularization.
    - avg_coefficients: A dictionary mapping genes to their average coefficients.
    - iteration: The current iteration number (for display).

    Returns:
    A DataFrame containing the 'OS.time', 'OS', and computed 'risk_score' for the test dataset.
    """

    test_X = test.drop(['OS.time', 'OS'], axis=1)
    test_copy = test.copy()

    # Calculate the risk score
    test_copy['risk_score'] = test_X[selected_genes].dot(np.array([avg_coefficients[gene] for gene in selected_genes]))

    # Kaplan-Meier estimation
    kmf = KaplanMeierFitter()

    # Categorize risk scores into two groups: high and low risk for visualization purposes
    high_risk = test_copy[test_copy['risk_score'] > test_copy['risk_score'].median()]
    low_risk = test_copy[test_copy['risk_score'] <= test_copy['risk_score'].median()]

    plt.figure(figsize=(10,7))
    kmf.fit(high_risk['OS.time'], event_observed=high_risk['OS'], label='High Risk')
    ax = kmf.plot(ci_show=False)

    kmf.fit(low_risk['OS.time'], event_observed=low_risk['OS'], label='Low Risk')
    kmf.plot(ax=ax, ci_show=False)
    plt.title(f'Kaplan-Meier Survival Curve based on Risk Score for Iteration {iteration}')
    plt.show()

    return test_copy[['OS.time', 'OS', 'risk_score']]


def plot_coefficient_distribution(cph, iteration):
    """
    Plots the distribution of coefficients after fitting a model.

    Parameters:
    - cph: A fitted CoxPHFitter object.
    - iteration: The current iteration number (for display).
    """
    coef_values = cph.summary["coef"].values

    plt.figure(figsize=(10, 6))
    sns.histplot(coef_values, bins=50, kde=True)
    plt.title(f'Coefficient Distribution for Iteration {iteration}')
    plt.xlabel('Coefficient Value')
    plt.ylabel('Frequency')
    plt.grid(True)
    plt.tight_layout()
    plt.show()


# Prepare data
X = merged_df[filtered_list]
y = merged_df[['OS.time', 'OS']]  # Ensure 'OS.time' is the duration and 'OS' is the event

# 1,000 iterations of leave-out-10% cross validation
num_iterations = 1000
penalizer = 0.02
epsilon = 1e-3

# Combine scaled features and target variables
data = pd.concat([X, y], axis=1)
print(data.shape)
data = data.dropna()
print(data.shape)

gene_counts = {}
coefficients_sum = {}
results = []

for i in tqdm(range(num_iterations)):
    train, test = train_test_split(data, test_size=0.1, random_state=i) #　iterations of leave-out-10% cross validation

    # LASSO penalization (l1_ratio=1)
    cph = CoxPHFitter(penalizer=penalizer, l1_ratio=1)
    cph.fit(train, duration_col='OS.time', event_col='OS')

    # Select only genes that are risk for survival
    risk_coeffs = cph.summary[cph.summary["coef"].abs() > epsilon]["coef"]

    for gene, coef in risk_coeffs.items():
        gene_counts[gene] = gene_counts.get(gene, 0) + 1
        coefficients_sum[gene] = coefficients_sum.get(gene, 0) + coef

    # Calculate selected genes and their average coefficients
    selected_genes_iter = [gene for gene, count in gene_counts.items() if count >= ((i+1) * 0.9)]
    avg_coefficients_iter = {gene: round(coefficients_sum[gene] / gene_counts[gene], 3) for gene in selected_genes_iter}

    print(f"No. of selected genes: {len(selected_genes_iter)}")
    print(f"Selected genes: {selected_genes_iter}")
    rounded_avg_coefficients = {gene: round(value, 3) for gene, value in avg_coefficients_iter.items()}
    print(f"Average coefficients: {rounded_avg_coefficients}")

    # Plot the coefficients
    plot_coefficient_distribution(cph, i+1)

    # Evaluated with test data from iterations
    result = evaluate_on_test(test, cph, selected_genes_iter, avg_coefficients_iter, i+1)
    results.append(result)

    if (i + 1) % 100 == 0 or i == num_iterations - 1:
      sorted_genes = sorted(avg_coefficients_iter.items(), key=lambda x: abs(x[1]), reverse=True)
      res = pd.DataFrame(sorted_genes, columns=["Gene", "Coefficient"])
      filename = f'{DAY}_sorted_coefficients_iteration_{i+1}.penalizer{penalizer}.{Version}.txt'
      res.to_csv(outdir + filename, sep='\t', index=False)

print(f"No. of selected genes: {len(selected_genes_iter)}")
print(f"Selected genes: {selected_genes_iter}")
print(f"Average coefficients: {avg_coefficients_iter}")
print(f"Results on the last test split: {results[-1]}")

(221, 289)
(220, 289)

  0%|          | 0/1000 [00:00<?, ?it/s]

No. of selected genes: 64
Selected genes: ['IGKC', 'ARGLU1', 'DES', 'IFI6', 'ARF3', 'S100A4', 'C4orf3', 'SFRP2', 'CCN2', 'GTF3C6', 'S100A6', 'THBS4', 'MRFAP1L1', 'LUC7L3', 'UBXN6', 'RBM42', 'ARAP1', 'IGHMBP2', 'PRRC2C', 'NES', 'ADAM9', 'TXNDC12', 'SPARC', 'RTF2', 'TNNT1', 'PEA15', 'DIDO1', 'IFNGR2', 'GABARAPL2', 'TXNDC17', 'FLOT1', 'MRC2', 'LSP1', 'PI4K2A', 'ITPRIPL2', 'TNC', 'CDC20', 'FBXO32', 'LPCAT1', 'TMEM70', 'CIAO2B', 'NTMT1', 'YIPF4', 'MIB2', 'MFGE8', 'CIITA', 'NOM1', 'IL4R', 'ADAMTS4', 'COL15A1', 'EML3', 'TSPAN4', 'NUPR1', 'RASSF1', 'VASH1', 'TNNT2', 'OLR1', 'PIM3', 'C3', 'ST3GAL2', 'CWC25', 'CFD', 'PTX3', 'PLEKHG5']
Average coefficients: {'IGKC': -0.157, 'ARGLU1': 0.341, 'DES': 0.608, 'IFI6': 0.055, 'ARF3': 0.282, 'S100A4': -0.688, 'C4orf3': 0.158, 'SFRP2': 0.285, 'CCN2': 0.616, 'GTF3C6': -0.068, 'S100A6': -0.104, 'THBS4': 0.271, 'MRFAP1L1': -0.294, 'LUC7L3': -0.41, 'UBXN6': -0.254, 'RBM42': 0.004, 'ARAP1': -0.065, 'IGHMBP2': 0.021, 'PRRC2C': -0.097, 'NES': -0.278, 'ADAM9': -0.228, 'TXNDC12': 0.088, 'SPARC': -0.301, 'RTF2': -0.042, 'TNNT1': -0.113, 'PEA15': -0.194, 'DIDO1': -0.174, 'IFNGR2': 0.155, 'GABARAPL2': -0.02, 'TXNDC17': 0.183, 'FLOT1': -0.115, 'MRC2': 0.234, 'LSP1': -0.004, 'PI4K2A': 0.166, 'ITPRIPL2': 0.136, 'TNC': -0.336, 'CDC20': 0.157, 'FBXO32': 0.021, 'LPCAT1': 0.247, 'TMEM70': -0.062, 'CIAO2B': -0.08, 'NTMT1': 0.253, 'YIPF4': 0.128, 'MIB2': 0.003, 'MFGE8': -0.243, 'CIITA': 0.036, 'NOM1': 0.057, 'IL4R': 0.02, 'ADAMTS4': 0.012, 'COL15A1': -0.029, 'EML3': -0.081, 'TSPAN4': 0.064, 'NUPR1': -0.009, 'RASSF1': -0.044, 'VASH1': -0.382, 'TNNT2': 0.013, 'OLR1': 0.25, 'PIM3': 0.296, 'C3': 0.098, 'ST3GAL2': -0.277, 'CWC25': 0.316, 'CFD': 0.102, 'PTX3': 0.239, 'PLEKHG5': -0.091}

  0%|          | 1/1000 [00:14<4:01:37, 14.51s/it]

No. of selected genes: 38
Selected genes: ['ARGLU1', 'DES', 'IFI6', 'ARF3', 'S100A4', 'C4orf3', 'SFRP2', 'CCN2', 'GTF3C6', 'THBS4', 'MRFAP1L1', 'LUC7L3', 'UBXN6', 'NES', 'TNNT1', 'IFNGR2', 'TXNDC17', 'FLOT1', 'MRC2', 'LSP1', 'PI4K2A', 'ITPRIPL2', 'TNC', 'FBXO32', 'LPCAT1', 'CIAO2B', 'NTMT1', 'YIPF4', 'MFGE8', 'CIITA', 'NOM1', 'COL15A1', 'OLR1', 'PIM3', 'C3', 'CFD', 'PTX3', 'PLEKHG5']
Average coefficients: {'ARGLU1': 0.175, 'DES': 0.458, 'IFI6': 0.079, 'ARF3': 0.232, 'S100A4': -0.517, 'C4orf3': 0.111, 'SFRP2': 0.28, 'CCN2': 0.439, 'GTF3C6': -0.072, 'THBS4': 0.185, 'MRFAP1L1': -0.293, 'LUC7L3': -0.34, 'UBXN6': -0.191, 'NES': -0.251, 'TNNT1': -0.085, 'IFNGR2': 0.213, 'TXNDC17': 0.141, 'FLOT1': -0.137, 'MRC2': 0.364, 'LSP1': -0.043, 'PI4K2A': 0.161, 'ITPRIPL2': 0.074, 'TNC': -0.171, 'FBXO32': 0.012, 'LPCAT1': 0.134, 'CIAO2B': -0.066, 'NTMT1': 0.235, 'YIPF4': 0.076, 'MFGE8': -0.404, 'CIITA': 0.085, 'NOM1': 0.058, 'COL15A1': -0.238, 'OLR1': 0.213, 'PIM3': 0.279, 'C3': 0.126, 'CFD': 0.183, 'PTX3': 0.213, 'PLEKHG5': -0.143}

  0%|          | 2/1000 [00:28<4:01:07, 14.50s/it]

No. of selected genes: 32
Selected genes: ['ARGLU1', 'DES', 'ARF3', 'S100A4', 'C4orf3', 'SFRP2', 'CCN2', 'GTF3C6', 'THBS4', 'MRFAP1L1', 'LUC7L3', 'UBXN6', 'NES', 'TNNT1', 'IFNGR2', 'TXNDC17', 'FLOT1', 'MRC2', 'PI4K2A', 'ITPRIPL2', 'LPCAT1', 'CIAO2B', 'NTMT1', 'MFGE8', 'NOM1', 'COL15A1', 'OLR1', 'PIM3', 'C3', 'CFD', 'PTX3', 'PLEKHG5']
Average coefficients: {'ARGLU1': 0.149, 'DES': 0.434, 'ARF3': 0.197, 'S100A4': -0.45, 'C4orf3': 0.077, 'SFRP2': 0.239, 'CCN2': 0.418, 'GTF3C6': -0.105, 'THBS4': 0.151, 'MRFAP1L1': -0.295, 'LUC7L3': -0.281, 'UBXN6': -0.145, 'NES': -0.176, 'TNNT1': -0.082, 'IFNGR2': 0.225, 'TXNDC17': 0.168, 'FLOT1': -0.169, 'MRC2': 0.367, 'PI4K2A': 0.153, 'ITPRIPL2': 0.141, 'LPCAT1': 0.112, 'CIAO2B': -0.066, 'NTMT1': 0.28, 'MFGE8': -0.369, 'NOM1': 0.092, 'COL15A1': -0.305, 'OLR1': 0.198, 'PIM3': 0.271, 'C3': 0.12, 'CFD': 0.215, 'PTX3': 0.156, 'PLEKHG5': -0.205}

  0%|          | 3/1000 [00:43<4:00:23, 14.47s/it]

No. of selected genes: 25
Selected genes: ['DES', 'ARF3', 'S100A4', 'C4orf3', 'SFRP2', 'CCN2', 'MRFAP1L1', 'LUC7L3', 'UBXN6', 'NES', 'IFNGR2', 'TXNDC17', 'FLOT1', 'MRC2', 'PI4K2A', 'ITPRIPL2', 'NTMT1', 'MFGE8', 'COL15A1', 'OLR1', 'PIM3', 'C3', 'CFD', 'PTX3', 'PLEKHG5']
Average coefficients: {'DES': 0.394, 'ARF3': 0.188, 'S100A4': -0.43, 'C4orf3': 0.097, 'SFRP2': 0.221, 'CCN2': 0.402, 'MRFAP1L1': -0.316, 'LUC7L3': -0.312, 'UBXN6': -0.142, 'NES': -0.169, 'IFNGR2': 0.248, 'TXNDC17': 0.186, 'FLOT1': -0.2, 'MRC2': 0.388, 'PI4K2A': 0.186, 'ITPRIPL2': 0.143, 'NTMT1': 0.291, 'MFGE8': -0.41, 'COL15A1': -0.31, 'OLR1': 0.17, 'PIM3': 0.29, 'C3': 0.179, 'CFD': 0.232, 'PTX3': 0.147, 'PLEKHG5': -0.194}

  0%|          | 4/1000 [00:58<4:02:50, 14.63s/it]


# 23-gene signature
res

	TCGA.BB.4224.01A	TCGA.H7.7774.01A	TCGA.CV.6943.01A	TCGA.CN.5374.01A	TCGA.CQ.6227.01A	TCGA.CV.6959.01A	TCGA.F7.A61V.01A	TCGA.CV.7413.01A	TCGA.CV.7247.01A	TCGA.CR.5249.01A	...	TCGA.CV.6960.11A	TCGA.CV.A464.01A	TCGA.C9.A47Z.01A	TCGA.CN.6010.01A	TCGA.WA.A7GZ.11A	TCGA.CV.7235.01A	TCGA.CX.7086.01A	TCGA.CV.6935.11A	TCGA.P3.A6SW.01A	TCGA.HD.A6HZ.01A
A1BG	0.405295	0.000000	0.274815	0.293833	0.162432	0.086511	0.027869	0.400248	0.030948	0.072121	...	0.014832	0.123858	0.071607	0.111782	0.087451	0.378804	0.066455	0.009181	0.301440	0.177668
A1BG-AS1	1.409574	0.212527	1.938291	2.436886	3.463670	0.938631	0.559192	3.508232	0.331183	1.077281	...	0.218245	2.319501	1.436772	0.814085	0.506909	1.852104	0.903753	0.171940	2.182144	0.784272
A1CF	0.018589	0.000000	0.002865	0.020216	0.009861	0.006613	0.004793	0.010870	0.000000	0.000000	...	0.005102	0.009468	0.000000	0.004273	0.020055	0.000000	0.003810	0.000000	0.006100	0.091676
A2M	82.569969	75.826356	488.800604	191.724219	1310.090781	579.945717	136.370044	186.228680	470.819984	300.763953	...	165.588515	499.634744	78.655454	374.319751	361.653242	499.158742	155.176401	174.166560	163.238289	423.929599
A2M-AS1	0.263864	0.058412	0.214699	0.252513	0.335913	0.120154	0.054432	0.213949	0.177308	0.253551	...	0.127465	0.419311	0.111886	0.339617	0.614891	0.285736	0.138448	0.129113	0.207794	0.638495
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
TBC1D3C	0.005990	0.000000	0.000000	0.000000	0.000000	0.000000	0.024713	0.000000	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.019643	0.000000	0.015724	0.000000
TMSB15B	0.057724	0.051913	0.120980	0.029294	0.107507	0.065714	0.063508	0.088508	0.095503	0.130110	...	0.010562	0.190125	0.061192	0.150362	0.020759	0.063253	0.067831	0.032691	0.035356	0.101218
TP53TG3C	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
USP17L8	0.015620	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.012178	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.010271
ZFP62	17.259246	8.904212	11.143712	26.036895	6.873685	8.911097	7.891183	12.697112	10.238681	21.011883	...	14.155443	13.472712	7.136761	16.940617	16.087040	15.831563	9.533326	12.465659	20.494577	14.304321

	Metastatic_pri_GeneNames	Metastatic_pri_Scores	Metastatic_pri_LogFoldchanges	Metastatic_pri_Pvals_adj
0	COL1A2	16.591194	1.301025	6.930000e-58
1	COL3A1	16.544867	1.387061	9.970000e-58
2	DCN	16.373690	1.360238	1.260000e-56
3	TPM2	16.190931	1.203350	1.670000e-55
4	IGKC	15.516564	1.832264	5.750000e-51
...	...	...	...	...
293	PTX3	2.378346	0.158925	3.464982e-02
294	ABHD16A	2.320436	0.385030	3.998464e-02
295	CUEDC2	2.313150	0.179848	4.069606e-02
296	ANTKMT	2.293972	0.180900	4.262316e-02
297	PLEKHG5	2.286873	0.506338	4.337287e-02

	submitter_id.samples	Metastasis Sig. Mean Expr.
0	TCGA-BB-4224-01A	-0.120999
1	TCGA-H7-7774-01A	-1.010822
2	TCGA-CV-6943-01A	0.231542
3	TCGA-CN-5374-01A	0.016997
4	TCGA-CQ-6227-01A	0.450236
...	...	...
541	TCGA-CV-7235-01A	0.229792
542	TCGA-CX-7086-01A	-0.330176
543	TCGA-CV-6935-11A	-0.762011
544	TCGA-P3-A6SW-01A	-0.027633
545	TCGA-HD-A6HZ-01A	0.320100

	submitter_id.samples	Metastasis Sig. Mean Expr.	index	COL1A2	COL3A1	DCN	TPM2	IGKC	PCOLCE	PRRX1	...	COL10A1	CWC25	CFD	ZNF707	CCNDBP1	PTX3	ABHD16A	CUEDC2	ANTKMT	PLEKHG5
0	TCGA-BB-4224-01A	-0.120999	TCGA.BB.4224.01A	-1.033177	-1.172174	0.012038	0.053268	-2.381722	-1.505695	-1.132512	...	-0.187951	1.078821	-0.195891	1.360465	0.082921	-0.924137	1.379503	-0.600103	-0.234087	-0.790115
1	TCGA-H7-7774-01A	-1.010822	TCGA.H7.7774.01A	-1.790722	-1.411683	-1.153653	-1.619565	-0.167134	-1.851165	-0.672788	...	-1.490375	-2.584782	-0.019745	-2.527397	-0.595203	-0.975431	-0.468209	-1.740594	-2.027355	-0.575345
2	TCGA-CV-6943-01A	0.231542	TCGA.CV.6943.01A	1.679412	1.625742	1.396474	-0.015438	1.171078	1.350195	1.068263	...	0.857605	-0.738831	0.140341	0.025660	0.661148	-0.349996	0.179159	-0.292268	-0.084748	0.051891
3	TCGA-CN-5374-01A	0.016997	TCGA.CN.5374.01A	-0.877316	-0.887038	-0.425125	0.241661	0.951927	-0.234991	-0.412873	...	-0.147564	0.376399	-0.532619	0.758023	-0.726857	-0.918837	0.915561	0.159292	-0.018111	0.346687
4	TCGA-CQ-6227-01A	0.450236	TCGA.CQ.6227.01A	0.592867	0.670386	0.879832	1.542557	-0.270512	1.178650	0.656216	...	-0.059872	-0.412874	-0.632155	-0.377059	-0.208318	2.128243	-0.785440	0.570198	-0.340434	0.398842
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
541	TCGA-CV-7235-01A	0.229792	TCGA.CV.7235.01A	1.406201	1.396792	1.450056	0.351174	-1.255399	1.505254	0.891355	...	1.316211	0.759759	-0.698104	-0.685183	0.215873	0.787945	0.960420	0.153506	-0.345568	0.602089
542	TCGA-CX-7086-01A	-0.330176	TCGA.CX.7086.01A	-0.975291	-0.869763	-0.207855	-1.783048	0.223749	-1.570646	-1.145911	...	-1.289943	1.098170	0.597675	0.082691	-0.282989	-0.973404	-0.536360	-1.510854	0.779925	0.168974
543	TCGA-CV-6935-11A	-0.762011	TCGA.CV.6935.11A	-1.369474	-1.587022	0.306273	-1.906342	-0.252972	-0.458843	-0.953090	...	-1.490569	-1.522217	2.152302	-2.236460	0.814242	0.284371	-0.332312	-1.385302	-1.177959	-0.148917
544	TCGA-P3-A6SW-01A	-0.027633	TCGA.P3.A6SW.01A	-0.031399	0.086048	0.120332	0.003798	1.690902	0.671448	-0.675508	...	-1.005509	1.438178	0.166622	-0.198283	0.296772	-0.706736	0.822380	0.847812	2.455449	-0.016310
545	TCGA-HD-A6HZ-01A	0.320100	TCGA.HD.A6HZ.01A	0.196346	0.461012	0.663521	0.942504	0.980765	0.106807	0.321138	...	-0.473867	0.583801	-0.322395	-0.472772	0.872122	-0.478880	-0.178745	-0.649946	-0.018093	0.414971

	submitter_id.samples	submitter_id	number_of_lymphnodes_positive_by_he	number_of_lymphnodes_positive_by_ihc	pathologic_N	LNM	LNM_stage	_stage
0	TCGA-DQ-5631-01A	TCGA-DQ-5631	1.0	NaN	N1	1.0	N1	1.0
1	TCGA-BA-7269-01A	TCGA-BA-7269	1.0	NaN	N1	1.0	N1	1.0
2	TCGA-BA-A4IH-01A	TCGA-BA-A4IH	2.0	NaN	N1	1.0	N1	1.0
3	TCGA-CV-6954-01A	TCGA-CV-6954	0.0	0.0	N0	0.0	N0	0.0
4	TCGA-CV-6954-11A	TCGA-CV-6954	0.0	0.0	N0	0.0	N0	0.0
...	...	...	...	...	...	...	...	...
607	TCGA-HD-A6I0-01A	TCGA-HD-A6I0	1.0	NaN	N1	1.0	N1	1.0
608	TCGA-HD-A6I0-11A	TCGA-HD-A6I0	1.0	NaN	N1	1.0	N1	1.0
609	TCGA-BA-4077-01B	TCGA-BA-4077	NaN	NaN	N0	0.0	N0	0.0
610	TCGA-IQ-7630-01A	TCGA-IQ-7630	0.0	NaN	N0	0.0	N0	0.0
611	TCGA-MT-A67A-01A	TCGA-MT-A67A	0.0	NaN	N0	0.0	N0	0.0

	case_id	case_submitter_id	project_id	ajcc_pathologic_n	LNM	index
0	ae90972d-5bc2-4b53-b5ff-1b8c31f39342	TCGA-CQ-A4CH	TCGA-HNSC	N0	0	Bulk_N0_1
1	c0edde5e-d229-4061-8820-14afc712c5b6	TCGA-HD-A633	TCGA-HNSC	N0	0	Bulk_N0_2
2	42bc2336-2408-441a-83b9-aadc9700f750	TCGA-D6-6825	TCGA-HNSC	N0	0	Bulk_N0_3
3	9413432e-24fe-428a-b464-1a5bcdbf5f74	TCGA-CV-A45V	TCGA-HNSC	N0	0	Bulk_N0_4
4	e1527704-8a97-445f-a5ee-357920cb28f7	TCGA-CQ-A4CE	TCGA-HNSC	N0	0	Bulk_N0_5
...	...	...	...	...	...	...
216	32128246-1258-43d1-b1e5-30ae63822c32	TCGA-CV-6950	TCGA-HNSC	N2c	3	Bulk_N2c_217
217	0de19185-3517-4e30-925b-7eb1f5079ec2	TCGA-BA-A6DJ	TCGA-HNSC	N2c	3	Bulk_N2c_218
218	b815fae9-c595-4c76-acc6-6d38b9af9578	TCGA-RS-A6TO	TCGA-HNSC	N2c	3	Bulk_N2c_219
219	a1ded1e8-eb28-49dd-8f3d-1ce8f40eed8f	TCGA-CN-A642	TCGA-HNSC	N3	3	Bulk_N3_220
220	fcf0dc48-b889-4593-a15b-aa715aae7bf5	TCGA-CV-6948	TCGA-HNSC	N3	3	Bulk_N3_221

	index_x	submitter_id	Metastasis Sig. Mean Expr.	LNM	met	COL1A2	COL3A1	DCN	TPM2	IGKC	...	COL10A1	CWC25	CFD	ZNF707	CCNDBP1	PTX3	ABHD16A	CUEDC2	ANTKMT	PLEKHG5
0	Bulk_N0_1	TCGA-CQ-A4CH	0.281043	N0	LNM(-)	0.097462	0.402146	0.078208	0.065333	0.242735	...	1.124225	1.333174	-1.995392	1.265721	1.297547	-0.695401	1.246309	-0.144208	0.498140	0.157581
1	Bulk_N0_2	TCGA-HD-A633	-0.063757	N0	LNM(-)	0.110391	0.255975	-0.068605	-0.204737	-0.867158	...	0.636623	0.285078	0.441313	-0.611770	2.305755	-0.169926	0.068640	-0.235548	0.644635	0.672512
2	Bulk_N0_3	TCGA-D6-6825	0.243959	N0	LNM(-)	0.407491	0.597626	0.230511	0.191198	-0.238637	...	-0.003704	0.736972	-1.388775	-0.169590	0.613043	-0.512571	0.549435	-1.435477	-1.586910	-0.254789
3	Bulk_N0_4	TCGA-CV-A45V	-0.219422	N0	LNM(-)	0.413852	0.446143	0.268884	-0.698043	-0.007244	...	0.194782	-0.563319	0.352268	-1.190062	0.352757	-0.291938	0.277646	-0.725440	-0.493274	1.127072
4	Bulk_N0_5	TCGA-CQ-A4CE	-0.525274	N0	LNM(-)	-0.804055	-0.803690	-0.703758	-1.418650	0.451997	...	-0.999622	0.758186	0.483518	0.541351	-0.768932	-0.618403	-0.000168	-2.034215	-0.308857	0.745962
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
216	Bulk_N2c_217	TCGA-CV-6950	0.292028	N3	LNM(+)	0.272828	0.218251	-0.609819	0.825255	-0.564574	...	1.258988	1.221221	-0.842402	0.507332	0.495160	-0.287747	-0.470765	-1.235215	-1.216764	0.279407
217	Bulk_N2c_218	TCGA-BA-A6DJ	-0.182780	N3	LNM(+)	-0.550448	-0.371672	-0.126053	0.036691	-0.228131	...	-0.405429	0.248659	0.598812	0.555555	-0.151848	-0.588209	0.672620	0.125368	-0.268224	2.001442
218	Bulk_N2c_219	TCGA-RS-A6TO	0.243472	N3	LNM(+)	0.720005	0.579678	0.499783	0.203628	-0.676634	...	1.481050	0.851368	0.897618	2.421766	-0.305575	-0.423292	1.433802	0.386176	0.317127	-0.101006
219	Bulk_N3_220	TCGA-CN-A642	0.257824	N3	LNM(+)	-0.027097	0.008198	-0.293648	0.246242	-0.759478	...	0.054022	0.850130	-0.311755	0.819312	1.142223	-0.758257	1.543740	1.007891	0.607908	0.373052
220	Bulk_N3_221	TCGA-CV-6948	-0.077607	N3	LNM(+)	-0.362869	-0.349182	-0.573629	-1.220422	0.243774	...	0.008537	0.128888	0.231496	1.391224	-1.078031	-0.858884	0.440530	0.576399	0.590245	-0.539611

	sample	_PATIENT	OS	OS.time	DSS	DSS.time	DFI	DFI.time	PFI	PFI.time	Redaction
0	TCGA-4P-AA8J-01	TCGA-4P-AA8J	0	102.0	0.0	102.0	NaN	NaN	0	102.0	NaN
1	TCGA-BA-4074-01	TCGA-BA-4074	1	462.0	1.0	462.0	NaN	NaN	1	396.0	NaN
2	TCGA-BA-4075-01	TCGA-BA-4075	1	283.0	1.0	283.0	NaN	NaN	1	236.0	NaN
3	TCGA-BA-4076-01	TCGA-BA-4076	1	415.0	1.0	415.0	NaN	NaN	1	286.0	NaN
4	TCGA-BA-4077-01	TCGA-BA-4077	1	1134.0	1.0	1134.0	NaN	NaN	1	1134.0	NaN
...	...	...	...	...	...	...	...	...	...	...	...
599	TCGA-UF-A7JV-01	TCGA-UF-A7JV	1	90.0	1.0	90.0	NaN	NaN	1	55.0	NaN
600	TCGA-UP-A6WW-01	TCGA-UP-A6WW	0	518.0	0.0	518.0	NaN	NaN	0	518.0	NaN
601	TCGA-WA-A7GZ-01	TCGA-WA-A7GZ	1	625.0	0.0	625.0	0.0	625.0	0	625.0	NaN
602	TCGA-WA-A7GZ-11	TCGA-WA-A7GZ	1	625.0	0.0	625.0	0.0	625.0	0	625.0	NaN
603	TCGA-WA-A7H4-01	TCGA-WA-A7H4	0	443.0	0.0	443.0	0.0	443.0	0	443.0	NaN

	Gene	Coefficient
0	S100A4	-0.511
1	MFGE8	-0.408
2	COL15A1	-0.407
3	DES	0.374
4	MRC2	0.364
5	MRFAP1L1	-0.338
6	SFRP2	0.318
7	CCN2	0.315
8	PIM3	0.289
9	LUC7L3	-0.280
10	IFNGR2	0.245
11	CORO1A	-0.243
12	CFD	0.242
13	ABHD16A	0.235
14	OLR1	0.227
15	NTMT1	0.214
16	NOM1	0.213
17	C3	0.191
18	UBXN6	-0.180
19	TXNDC17	0.166
20	NES	-0.162
21	PTX3	0.131
22	C4orf3	0.122