create_eval_graphs.py

#!/usr/bin/env python
# coding: utf-8

# In[1]:


import os
import matplotlib.pyplot as plt
import seaborn as sns
import pickle
import pandas as pd
import glob

# In[2]:


# Constants
GRAPH_PERCEPTION = False
GRAPH_SINGLE_NODE = False
PERCEPTION_TITLE = "with Perception" if GRAPH_PERCEPTION else "without Perception"
NODE_TITLE = "single node" if GRAPH_SINGLE_NODE else "multi node"
EVALUATION_FOLDER_PATH = './evaluation_outputs'
CUMULATIVE_STATS_FOLDER_PATH = './evaluation_outputs/cumulative_stats_dist_with_perception' if GRAPH_PERCEPTION else './evaluation_outputs/cumulative_stats_dist_no_perception'
CUMULATIVE_STATS_FOLDERS = glob.glob(os.path.join(EVALUATION_FOLDER_PATH + '/cumulative_stats_*'))

print(CUMULATIVE_STATS_FOLDERS)

SHOULD_SHOW = False

# In[3]:


def get_stats_df(file_path=""):
    """
    Load a DataFrame from a pickle file.

    Args:
    file_path (str): The path of the pickle file.

    Returns:
    pd.DataFrame: The DataFrame loaded from the pickle file.
    """
    if not os.path.exists(file_path):
        print(f"Cannot find file {file_path}")
        return None

    try:
        with open(file_path, 'rb') as picklefile:
            stats_df = pickle.load(picklefile)
        return stats_df
    except Exception as e:
        print(f"Error loading file {file_path}: {e}")


# In[4]:


def create_box_plot(data, x, y, labels):
    """
    Create a box plot using seaborn.

    Args:
    data (pd.DataFrame): The DataFrame containing the data to be plotted.
    x (str): The column name for the x-axis variable.
    y (str): The column name for the y-axis variable.
    labels (dict): A dictionary containing the labels for the plot (xlabel, ylabel, title).

    Returns:
    Axes: The axis object containing the box plot.
    """
    ax = sns.boxplot(data=data, x=x, y=y)
    plt.yscale('log')
    ax.set(xlabel=labels['xlabel'],
           ylabel=labels['ylabel'],
           title=labels['title'])
    return ax


# In[5]:
def create_bar_plot(data, x, y, labels):
    """
    Create a bar plot using seaborn.

    Args:
    data (pd.DataFrame): The DataFrame containing the data to be plotted.
    x (str): The column name for the x-axis variable.
    y (str): The column name for the y-axis variable.
    labels (dict): A dictionary containing the labels for the plot (xlabel, ylabel, title).

    Returns:
    Axes: The axis object containing the box plot.
    """
    ax = sns.barplot(data=data, x=x, y=y)
    ax.set(xlabel=labels['xlabel'],
           ylabel=labels['ylabel'],
           title=labels['title'])
    return ax


def create_stacked_bar_chart(data,  y, labels):
    """
    Create a scatter plot using seaborn.

    Args:
    data (pd.DataFrame): The DataFrame containing the data to be plotted.
    x (str): The column name for the x-axis variable.
    y (str): The column name for the y-axis variable.
    labels (dict): A dictionary containing the labels for the plot (xlabel, ylabel, title).

    Returns:
    Axes: The axis object containing the scatter plot.
    """
    colors = [
        'black', 
        'silver', 
        'chartreuse', 
        'red', 
        'chocolate', 
        'darkorange',
        'tan',
        'gold',
        'beige',
        'olivedrab',
        'lime',
        'powderblue',
        'royalblue',
        'indigo',
        'violet',
        'pink',
     ]
    
    labels_legend = [
        #'agent_step_0',
        #'agent_step_1',
        #'agent_step_2', 
        #'agent_step_3', 
        #'agent_step_4', 
        #'agent_step_5', 
        #'agent_step_6', 
        #'agent_step_10', 
        #'agent_step_11', 
        #'client_control', 
        #'client_perception', 
        #'client_localization', 
        #'client_agent_update_info', 
        #'client_controller_update_info', 
        #'client_controller_step',  
        #'client_control',
        "client_process",
        "idle",
        "network_latency",]

    ax = data.plot(y=y, kind='bar', stacked=True, color=colors, figsize=(11, 8))
    plt.xlabel(labels['xlabel'])
    plt.ylabel(labels['ylabel'])
    plt.yscale('log')
    plt.title(labels['title'])
    ax.legend(labels=labels_legend)
    return ax


# In[6]:


def save_ax(ax=None, file_path=""):
    """
    Save the plot to a file.

    Args:
    ax (Axes): The axis object containing the plot.
    file_path (str): The path where the plot should be saved.
    """
    if not file_path or not ax:
        print("File path or axis object not provided.")
        return

    ax.figure.savefig(file_path)
    print(f"Saved file: {file_path}")


# In[273]:


def plot_simulation_time(path):
    sim_time_df_path = f'{path}/df_total_sim_time'
    sim_stats_df = get_stats_df(sim_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Total Runtime (s)',
              "title": f'eCloudSim: Total Runtime \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # sns.set(font_scale=1.2)
    sns.set_style('whitegrid')  

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='time_s', labels=labels)
    save_file_path = f'{path}/total_sim_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

    # Scatter plot
    # ax = create_scatter_plot(data=sim_stats_df, x='num_cars', y='time_s', labels=labels)
    # save_file_path = f'{path}/total_sim_time_scatterplot.png'
    # save_ax(ax, save_file_path)
    # if SHOULD_SHOW:
    #    plt.show()
    # plt.clf()


# In[274]:


# In[275]:


def plot_world_step_time(path):
    step_time_df_path = f'{path}/df_world_step_time'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Simulation Step Time (ms)',
              "title": f'eCloudSim: Simulation Step Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='world_step_time_ms', labels=labels)
    save_file_path = f'{path}/world_step_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

    # Scatter plot
    # ax = create_scatter_plot(data=sim_stats_df, x='num_cars', y='step_time_ms', labels=labels)
    # save_file_path = f'{path}/step_time_scatterplot.png'
    # save_ax(ax, save_file_path)
    # if SHOULD_SHOW:
    #    plt.show()
    # plt.clf()

def plot_client_step_time(path):
    step_time_df_path = f'{path}/df_client_step_time'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Simulation Step Time (ms)',
              "title": f'eCloudSim: Total Client Step Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='client_step_time_ms', labels=labels)
    ax.set(ylim=(0,3000))
    save_file_path = f'{path}/client_step_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

    new_df = sim_stats_df.groupby(by='num_cars').mean()
    pd.options.display.max_columns = 99
    print(new_df)
    # Scatter plot
    # ax = create_scatter_plot(data=sim_stats_df, x='num_cars', y='step_time_ms', labels=labels)
    # save_file_path = f'{path}/step_time_scatterplot.png'
    # save_ax(ax, save_file_path)
    # if SHOULD_SHOW:
    #    plt.show()
    # plt.clf()

def plot_client_perception_time(path):
    step_time_df_path = f'{path}/df_client_perception_time'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Client Perception Time (ms)',
              "title": f'eCloudSim: Client Perception Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='client_perception_time_ms', labels=labels)
    save_file_path = f'{path}/client_perception_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

def plot_client_localization_time(path):
    step_time_df_path = f'{path}/df_client_localization_time'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Client Localization Time (ms)',
              "title": f'eCloudSim: Client Localization Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='client_localization_time_ms', labels=labels)
    save_file_path = f'{path}/client_localization_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

def plot_client_control_time(path):
    step_time_df_path = f'{path}/df_client_control_time'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Client Control Time (ms)',
              "title": f'eCloudSim: Client Control Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='client_control_time_ms', labels=labels)
    save_file_path = f'{path}/client_control_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

def plot_agent_step_times(path):
    TOTAL_AGENT_STEPS = 12
    AGENT_STEPS = {
            0: "sim end",
            1: "lights",
            2: "temp route",
            3: "path generation",
            4: "lane change",
            5: "collision",
            6: "no lane change composite",
            7: "push",
            8: "blocking",
            9: "overtake",
            10: "following",
            11: "normal",
    }

    for i in range(TOTAL_AGENT_STEPS):
        step_time_df_path = f'{path}/df_agent_step_list_{i}'
        sim_stats_df = get_stats_df(step_time_df_path)

        labels = {"xlabel": 'Number of Cars',
              "ylabel": f'Agent Step {i} Time (ms)',
              "title": f'eCloudSim: Agent Step {i} Time - {AGENT_STEPS[i].title()} \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

        # Box plot
        plt.figure(figsize=(10, 6))
        ax = create_box_plot(data=sim_stats_df, x='num_cars', y=f'agent_step_list_{i}_ms', labels=labels)
        save_file_path = f'{path}/agent_step_{i}_time_boxplot.png'
        save_ax(ax, save_file_path)
        if SHOULD_SHOW:
            plt.show()
        plt.clf()


def plot_client_stacked_barchart(path):
    TOTAL_AGENT_STEPS = 7
    AGENT_STEPS = {
            0: "sim end",
            1: "lights",
            2: "temp route",
            3: "path generation",
            4: "lane change",
            5: "collision",
            6: "no lane change composite",
            10: "following",
            11: "normal",
    }

    step_time_df_path = f'{path}/df_agent_step_list_0'
    agent_df = get_stats_df(step_time_df_path)
    y_columns = ['agent_step_list_0_ms']

    labels = {"xlabel": 'Number of Cars',
              "ylabel": f'Agent Step Time',
              "title": f'eCloudSim: Agent Step Time - Composite \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}


    for i, step in AGENT_STEPS.items():
      if i == 0:
        continue
      step_time_df_path = f'{path}/df_agent_step_list_{i}'
      sim_stats_df = get_stats_df(step_time_df_path)
      #print(sim_stats_df[f'agent_step_list_{i}_ms'])
      y_columns.append(f'agent_step_list_{i}_ms')
      agent_df = agent_df.join(sim_stats_df[f'agent_step_list_{i}_ms'])
      #print(agent_df)

    client_debug_data = [ # see ClientDebugHelper.debug_data
            "client_control_time",
            "client_perception_time",
            "client_localization_time",
            # "client_update_info_time", # NOT NEEDED --> v2x
            "client_agent_update_info_time",
            "client_controller_update_info_time_list",
            # "client_agent_step_time_list", # handled by AGENT_STEPS
            "client_controller_step_time_list",
            # "client_vehicle_step_time_list", # COMPOSITE: client_agent_step_time_list + client_controller_step_time_list
            "client_control_time_list",
            "network_latency",
    ]

    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='client_perception_time_ms', labels=labels)
    save_file_path = f'{path}/client_perception_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

    for list_name in client_debug_data:
        step_time_df_path = f'{path}/df_{list_name}'
        sim_stats_df = get_stats_df(step_time_df_path)
        agent_df = agent_df.join(sim_stats_df[f'{list_name}_ms'])
        y_columns.append(f'{list_name}_ms')

    print(agent_df)

    new_df = agent_df.groupby(by='num_cars').mean()

    pd.options.display.max_columns = 99
    print(new_df)
    print(y_columns)

    ax = create_stacked_bar_chart(y=y_columns, data=new_df , labels=labels)

    save_file_path = f'{path}/agent_step_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
      plt.show()
    plt.clf()


def plot_client_process_time(path):

    agent_df = pd.DataFrame()
    y_columns = []

    labels = {"xlabel": 'Number of Cars',
              "ylabel": f' Step Time',
              "title": f'eCloudSim: Client and Network Times \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}


    client_debug_data = [ # see ClientDebugHelper.debug_data
            "client_process",
    ]
    for list_name in client_debug_data:
        step_time_df_path = f'{path}/df_{list_name}'
        sim_stats_df = get_stats_df(step_time_df_path)
        agent_df[f'num_cars'] = sim_stats_df[f'num_cars'] 
        agent_df[f'{list_name}_ms'] = sim_stats_df[f'{list_name}_ms']
        y_columns.append(f'{list_name}_ms')

    print(agent_df)

    new_df = agent_df.groupby(by='num_cars').mean()

    pd.options.display.max_columns = 99
    print(new_df)
    print(y_columns)

    ax = create_box_plot(y=y_columns, data=new_df , labels=labels)

    save_file_path = f'{path}/client_network_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
      plt.show()
    plt.clf()

def plot_client_process_time(path):
    step_time_df_path = f'{path}/df_client_process'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Client Processing Time (ms)',
              "title": f'eCloudSim: Client Processing Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='client_process_ms', labels=labels)
    save_file_path = f'{path}/client_process_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

def plot_barrier_time(path):

    step_time_df_path = f'{path}/df_idle'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Barrier Time (ms)',
              "title": f'eCloudSim: Client Processing Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='idle_ms', labels=labels)
    save_file_path = f'{path}/barrier_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()

def plot_network_overhead(path):

    step_time_df_path = f'{path}/df_network_latency'
    sim_stats_df = get_stats_df(step_time_df_path)

    labels = {"xlabel": 'Number of Cars',
              "ylabel": 'Network Overhead (ms)',
              "title": f'eCloudSim: Network Overhead \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    # Box plot
    plt.figure(figsize=(10, 6))
    ax = create_box_plot(data=sim_stats_df, x='num_cars', y='network_latency_ms', labels=labels)
    save_file_path = f'{path}/network_overhead_time_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()


def plot_individual_client_boxplot(path):

    agent_df = pd.DataFrame()
    y_columns = []

    labels = {"xlabel": 'Car_Index',
              "ylabel": f' Step Time',
              "title": f'eCloudSim: Individual Client Step Time \n per Number of Vehicles ({PERCEPTION_TITLE}) - {NODE_TITLE}'}

    
    step_time_df_path = f'{path}/df_client_individual_step_times_dict'
    sim_stats_df = get_stats_df(step_time_df_path)
    num_cars = sim_stats_df['num_cars'][0]

    ax = sns.boxplot(data=sim_stats_df)
    plt.yscale('log')
    ax.set(xlabel=labels['xlabel'],
           ylabel=labels['ylabel'],
           title=labels['title'])
 

    save_file_path = f'{path}/individual_client_time_dict_boxplot.png'
    save_ax(ax, save_file_path)
    if SHOULD_SHOW:
        plt.show()
    plt.clf()
# In[276]:


#step_time_df_path = f'{path}/df_world_step_time'
#client_step_time_df_path = f'{path}/df_client_step_time'
#client_perception_time_df_path =  f'{path}/df_client_perception_time' 

# In[277]:


def plot_comparison_chart_time(data):
    # Plotting
    sns.set_style("whitegrid")
    sns.set_palette("deep")
    sns.set_context("talk")

    plt.figure(figsize=(10, 6))

    # Convert the DataFrame to a long format for easy plotting
    data_long = data.melt(id_vars=['num_vehicles'], value_vars=['eCloudSim', 'openCDA'], var_name='Simulation', value_name='Value')

    sns.lineplot(x='num_vehicles', y='Value', hue='Simulation', data=data_long, marker='o')

    plt.xlabel('Number of Vehicles')
    plt.xticks(data['num_vehicles'].unique())
    plt.ylabel('Total Simulation Time (s)')
    plt.title('Simulation Time with Perception on a Single Node\n(Multi2lane Scenario)')

    plt.legend(['eCloudSim', 'openCDA'])

    if SHOULD_SHOW:
        plt.show()


# In[278]:


def plot_comparison_chart_cpu(data):
    # Plotting
    sns.set_style("whitegrid")
    sns.set_palette("deep")
    sns.set_context("talk")

    plt.figure(figsize=(10, 6))

    sns.lineplot(x='num_vehicles', y='cpu_util', hue='Simulation', data=data, marker='o')

    plt.xlabel('Number of Vehicles')
    plt.xticks(data['num_vehicles'].unique())
    plt.ylabel('CPU Utilization (%)')
    plt.title('CPU Utilization without Perception on a Single Node\n(Multi2lane Scenario)')

    plt.legend(['eCloudSim', 'openCDA'])

    plt.ylim([40, 80])

    if SHOULD_SHOW:
        plt.show()


# In[279]:


if __name__ == '__main__':

    for path in CUMULATIVE_STATS_FOLDERS:
        # Plotting simulation total run time stats
        plot_simulation_time(path)

        # Plotting simulation step time stats
        plot_world_step_time(path)

        # Plotting simulation step time stats
        plot_client_step_time(path)

        # Plotting simulation step time stats
        #plot_client_perception_time()

        # Plotting simulation step time stats
        #plot_client_localization_time()

        # Plotting simulation step time stats
        #plot_client_control_time()

        #plot_agent_step_times()

        plot_network_overhead(path)

        plot_barrier_time(path)

        plot_client_process_time(path)
        
        #plot_individual_client_boxplot(path)


   # Example DataFrame for comparison chart
    comparison_data = pd.DataFrame({
        'num_vehicles': [4, 8, 16],
        'eCloudSim': [72.3, 94.159503, 156.235791],
        'openCDA': [54.68, 90, 170]
    })

    # Plotting comparison chart
    plot_comparison_chart_time(comparison_data)

    data = {
        "Simulation": ["eCloudSim", "eCloudSim", "eCloudSim", "openCDA", "openCDA", "openCDA"],
        "num_vehicles": [4, 8, 16, 4, 8, 16],
        "cpu_util": [54.68, 73.70, 72.21, 55.00, 73.00, 74.00],
    }

    df = pd.DataFrame(data)
    plot_comparison_chart_cpu(df)


# In[ ]: