sim_integrated_mass.py

"""
This script runs 15 simulations (each corresponding to a different starting
ratio) in Cantera.
Reactor conditions are replicated from: "Methane catalytic partial oxidation on
autothermal Rh and Pt foam catalysts: Oxidation and reforming zones, transport
effects,and approach to thermodynamic equilibrium"
Horn 2007, doi:10.1016/j.jcat.2007.05.011
Ref 17: "Syngas by catalytic partial oxidation of methane on rhodium:
Mechanistic conclusions from spatially resolved measurements and numerical
simulations"
Horn 2006, doi:10.1016/j.jcat.2006.05.008
Ref 18: "Spatial and temporal profiles in millisecond partial oxidation
processes"
Horn 2006, doi:10.1007/s10562-006-0117-8
"""
import cantera as ct
import numpy as np
import scipy
import pylab
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from matplotlib.pyplot import cm
from matplotlib.ticker import NullFormatter, MaxNLocator, LogLocator
plt.switch_backend('agg')  # needed for saving figures
import csv
import os
# import rmgpy
# import rmg
import re
import operator
import pandas as pd
import pylab
from cycler import cycler
import seaborn as sns
import os
import multiprocessing
from functools import partial
import threading
import itertools
from rmgpy.molecule import Molecule
from rmgpy.data.base import Database
import subprocess
# unit conversion factors to SI
mm = 0.001
cm = 0.01
ms = mm
minute = 60.0

#######################################################################
# Input Parameters
#######################################################################
t_in = 800  # K - in the paper, it was ~698.15K at the start of the cat surface and ~373.15 for the gas inlet temp
t_cat = t_in
length = 70 * mm  # Reactor length - m
diam = 16.5 * mm  # Reactor diameter - in m, from Ref 17 & Ref 18
area = (diam/2.0)**2*np.pi  # Reactor cross section area (area of tube) in m^2
porosity = 0.81  # Monolith channel porosity, from Ref 17, sec 2.2.2
cat_area_per_vol = 1.6e4  # m2/m3, which is 160 cm2/cm3, as used in Horn 2006
cat_area_per_vol *= 0.05 # to make the concentrations change slower
flow_rate = 4.7  # slpm, as seen in as seen in Horn 2007
tot_flow = 0.208  # constant inlet flow rate in mol/min, equivalent to 4.7 slpm
flow_rate = flow_rate * .001 / 60  # m^3/s, as seen in as seen in Horn 2007
velocity = flow_rate / area  # m/s
MOLECULAR_WEIGHTS = {"H": 1.008, "O": 15.999, "C": 12.011, "N": 14.0067}

# The PFR will be simulated by a chain of 'N_reactors' stirred reactors.
N_reactors = 7001

on_catalyst = 1000  # catalyst length 10mm, from Ref 17
off_catalyst = 2000
dt = 1.0

# new sensitivities
# length = 510 * mm  # Reactor length - m
# N_reactors = 51001
# on_catalyst = 1000  # catalyst length 10mm, from Ref 17
# off_catalyst = 51000

reactor_len = length/(N_reactors-1)
rvol = area * reactor_len * porosity

# catalyst area in one reactor
cat_area = cat_area_per_vol * rvol

# root directory for output files
out_root = '/home/xu.chao/sketches/cpox_sim/rmg_models/base_cathub'

residence_time = reactor_len / velocity # unit in s

def setup_ct_solution(path_to_cti):

    # this chemkin file is from the cti generated by rmg
    gas = ct.Solution(path_to_cti, 'gas')
    surf = ct.Interface(path_to_cti, 'surface1', [gas])

    print("This mechanism contains {} gas reactions and {} surface reactions".format(gas.n_reactions, surf.n_reactions))
    print(f"Thread ID from threading{threading.get_ident()}")
    i_ar = gas.species_index('Ar')


    return {'gas':gas, 'surf':surf,"i_ar":i_ar,"n_surf_reactions":surf.n_reactions}

def transient_flux_diagram(reaction_index, surf, out_dir, ratio):
    """
    Plot the transient flux diagram at a certain location
    """
    location = str(int(reaction_index / 100))
    diagram = ct.ReactionPathDiagram(surf, 'X')
    diagram.title = 'rxn path'
    diagram.label_threshold = 1e-9
    dot_file = f"{out_dir}/rxnpath-{ratio:.1f}-x-{location}mm.dot"
    img_file = f"{out_dir}/rxnpath-{ratio:.1f}-x-{location}mm.pdf"
    diagram.write_dot(dot_file)
    os.system('dot {0} -Tpng -o{1} -Gdpi=200'.format(dot_file, img_file))

    for element in elements:
        diagram = ct.ReactionPathDiagram(surf, element)
        diagram.title = element + 'rxn path'
        diagram.label_threshold = 1e-9
        dot_file = f"{out_dir}/rxnpath-{ratio:.1f}-x-{location}mm-{element}.dot"
        img_file = f"{out_dir}/rxnpath-{ratio:.1f}-x-{location}mm-{element}.pdf"
        diagram.write_dot(dot_file)
        os.system('dot {0} -Tpng -o{1} -Gdpi=200'.format(dot_file, img_file))

def prettydot(dotfilepath, strip_line_labels=False):
    """
    Make a prettier version of the dot file (flux diagram)

    Assumes the species pictures are stored in a directory
    called 'species_pictures' alongside the dot file.
    """

    # pictures_directory = os.path.join(os.path.split(dotfilepath)[0], "species_pictures")
    pictures_directory = "/work/westgroup/chao/sketches/cpox_sim/rmg_models/base_cathub/species"

    if strip_line_labels:
        print("stripping edge (line) labels")

    # replace this:
    #  s10 [ fontname="Helvetica", label="C11H23J"];
    # with this:
    #  s10 [ shapefile="mols/C11H23J.png" label="" width="1" height="1" imagescale=true fixedsize=true color="white" ];

    reSize = re.compile('size="5,6"\;page="5,6"')
    reNode = re.compile(
        '(?P<node>s\d+)\ \[\ fontname="Helvetica",\ label="(?P<label>[^"]*)"\]\;'
    )

    rePicture = re.compile("(?P<smiles>.+?)\((?P<id>\d+)\)\.png")
    reLabel = re.compile("(?P<name>.+?)\((?P<id>\d+)\)$")

    species_pictures = dict()
    for picturefile in os.listdir(pictures_directory):
        match = rePicture.match(picturefile)
        if match:
            species_pictures[match.group("id")] = picturefile
        else:
            pass
            # print(picturefile, "didn't look like a picture")

    filepath = dotfilepath

    if not open(filepath).readline().startswith("digraph"):
        raise ValueError("{0} - not a digraph".format(filepath))

    infile = open(filepath)
    prettypath = filepath + "-pretty"
    outfile = open(prettypath, "w")

    for line in infile:
        (line, changed_size) = reSize.subn('size="12,12";page="12,12"', line)
        match = reNode.search(line)
        if match:
            label = match.group("label")
            idmatch = reLabel.match(label)
            if idmatch:
                idnumber = idmatch.group("id")
                if idnumber in species_pictures:
                    line = (
                        '%s [ image="%s/%s" label="" width="0.5" height="0.5" imagescale=false fixedsize=false color="none" ];\n'
                        % (match.group("node"),pictures_directory, species_pictures[idnumber])
                    )

        # rankdir="LR" to make graph go left>right instead of top>bottom

        if strip_line_labels:
            line = re.sub('label\s*=\s*"\s*[\d.]+"', 'label=""', line)

        # change colours
        line = re.sub('color="0.7,\ (.*?),\ 0.9"', r'color="1.0, \1, 0.7*\1"', line)

        outfile.write(line)

    outfile.close()
    infile.close()
    print(f"Graph saved to: {prettypath}")
    return prettypath

def get_current_fluxes(gas, surf):
    """
    Get all the current fluxes.
    Returns a dict like:
        `fluxes['gas']['C'] = ct.ReactionPathDiagram(self.gas, 'C')` etc.
    """
    fluxes = {"gas": dict(), "surf": dict()}
    for element in "HOC":
        fluxes["gas"][element] = ct.ReactionPathDiagram(gas, element)
        fluxes["surf"][element] = ct.ReactionPathDiagram(surf, element)
    element = "X"  # just the surface
    fluxes["surf"][element] = ct.ReactionPathDiagram(surf, element)
    return fluxes

def combine_fluxes(fluxes_dict):
    """
    Combined a dict of dicts of flux diagrams into one.

    Fluxes should be a dict with entries like
        fluxes['gas']['C'] = ct.ReactionPathDiagram(self.gas, 'C')

    Returns the flux diagram a string in the format you'd get from
    ct.ReactionPathdiagram.get_data()
    """
    # getting the entire net rates of the system
    temp_flux_data = dict()
    species = set()
    for element in "HOC":
        for phase in ("gas", "surf"):
            data = fluxes_dict[phase][element].get_data().strip().splitlines()
            if not data:
                # eg. if there's no gas-phase reactions involving C
                continue
            species.update(data[0].split())  # First line is a list of species
            for line in data[1:]:  # skip the first line

                s1, s2, fwd, rev = line.split()
                these_fluxes = np.array([float(fwd), float(rev)])

                if all(these_fluxes == 0):
                    continue

                # multiply by atomic mass of the element
                these_fluxes *= MOLECULAR_WEIGHTS[element]

                # for surface reactions, multiply by the catalyst area per volume in a reactor
                if phase == "surf":
                    these_fluxes *= cat_area_per_vol

                try:
                    # Try adding in this direction
                    temp_flux_data[(s1, s2)] += these_fluxes
                except KeyError:
                    try:
                        # Try adding in reverse direction
                        temp_flux_data[(s2, s1)] -= these_fluxes
                    except KeyError:
                        # Neither direction there yet, so create in this direction
                        temp_flux_data[(s1, s2)] = these_fluxes

    output = " ".join(species) + "\n"
    output += "\n".join(
        f"{s1} {s2} {fwd} {rev}" for (s1, s2), (fwd, rev) in temp_flux_data.items()
    )
    return output

def write_flux_dot(flux_data_string, out_file_path, threshold=0.01, title=""):
    """
    Takes a flux data string fromatted as from ct.ReactionPathdiagram.get_data()
    (or from combine_fluxes) and makes a graphviz .dot file.

    Fluxes below 'threshold' are not plotted.
    """

    output = ["digraph reaction_paths {", "center=1;"]

    flux_data = {}
    species_dict = {}
    flux_data_lines = flux_data_string.splitlines()
    species = flux_data_lines[0].split()

    for line in flux_data_lines[1:]:
        s1, s2, fwd, rev = line.split()
        net = float(fwd) + float(rev)
        if net < 0.0:  # if net is negative, switch s1 and s2 so it is positive
            flux_data[(s2, s1)] = -1 * net
        else:
            flux_data[(s1, s2)] = net

        # renaming species to dot compatible names
        if s1 not in species_dict:
            species_dict[s1] = "s" + str(len(species_dict) + 1)
        if s2 not in species_dict:
            species_dict[s2] = "s" + str(len(species_dict) + 1)

    # getting the arrow widths
    largest_rate = max(flux_data.values())

    added_species = {}  # dictionary of species that show up on the diagram
    for (s1, s2), net in flux_data.items():  # writing the node connections

        flux_ratio = net / largest_rate
        if abs(flux_ratio) < threshold:
            continue  # don't include the paths that are below the threshold

        pen_width = (
            1.0 - 4.0 * np.log10(flux_ratio / threshold) / np.log10(threshold) + 1.0
        )
        # pen_width = ((net - smallest_rate) / (largest_rate - smallest_rate)) * 4 + 2
        arrow_size = min(6.0, 0.5 * pen_width)
        output.append(
            f'{species_dict[s1]} -> {species_dict[s2]} [fontname="Helvetica", penwidth={pen_width:.2f}, arrowsize={arrow_size:.2f}, color="0.7, {flux_ratio+0.5:.3f}, 0.9", label="{flux_ratio:0.3g}"];'
        )

        added_species[s1] = species_dict[s1]
        added_species[s2] = species_dict[s2]

    for (
        species,
        s_index,
    ) in added_species.items():  # writing the species translations
        output.append(f'{s_index} [ fontname="Helvetica", label="{species}"];')

    title_string = (r"\l " + title) if title else ""
    output.append(f' label = "Scale = {largest_rate/1000}kg/m^3/s{title_string}";')
    output.append(' fontname = "Helvetica";')
    output.append("}\n")

    directory = os.path.split(out_file_path)[0]
    if directory:
        os.makedirs(directory, exist_ok=True)
    with open(out_file_path, "w") as out_file:
        out_file.write("\n".join(output))
    return "\n".join(output)


def save_flux_diagrams(gas, surf, total_fluxes, path="", suffix="", fmt="png"):
    """
    Saves the flux diagrams, in the provided path.
    The filenames have a suffix if provided,
    so you can keep them separate and not over-write.
    fmt can be 'pdf' or 'png'
    """
    strings = []
    # Now do the combined flux
    for name, fluxes_dict in [
        ("mass", get_current_fluxes(gas, surf)),
        ("integrated mass", total_fluxes),
    ]:

        flux_data_string = combine_fluxes(fluxes_dict)
        strings.append(flux_data_string)
        dot_file = os.path.join(
            path,
            f"reaction_path_{name.replace(' ','_')}{'_' if suffix else ''}{suffix}_total.dot",
        )
        img_file = os.path.join(
            path,
            f"reaction_path_{name.replace(' ','_')}{'_' if suffix else ''}{suffix}_total.{fmt}",
        )
        write_flux_dot(
            flux_data_string,
            dot_file,
            threshold=0.01,
            title=f"Reaction path diagram showing combined {name}",
        )
        # Unufortunately this code is duplicated above,
        # so be sure to duplicate any changes you make!
        pretty_dot_file = prettydot(dot_file)

        subprocess.run(
            [
                "dot",
                os.path.abspath(pretty_dot_file),
                f"-T{fmt}",
                "-o",
                os.path.abspath(img_file),
                "-Gdpi=72",
            ],
            cwd=path,
            check=True,
        )
        print(
            f"Wrote graphviz output file to '{os.path.abspath(os.path.join(os.getcwd(), img_file))}'."
        )

def plot_gas(data, path_to_cti, x_lim=None):
    """
    Plots gas-phase species profiles through the PFR.
    xlim is either None or a tuple (x_min, x_max)
    """
    gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions = data

    # Plot in mol/min
    fig, axs = plt.subplots()

    axs.set_prop_cycle(cycler('color', ['m', 'g', 'b', 'y', 'c', 'k', 'g']))

    for i in range(len(gas_out[0, :])):
        if i != i_ar:
            if gas_out[:, i].max() > 5.e-3:
                axs.plot(dist_array, gas_out[:, i], label=gas_names[i])
                species_name = gas_names[i]
                if species_name.endswith(')'):
                    if species_name[-3] == '(':
                        species_name = species_name[0:-3]
                    else:
                        species_name = species_name[0:-4]
            else:
                axs.plot(0, 0)

    axs.set_prop_cycle(cycler('color', ['m', 'g', 'b', 'y', 'c', 'r', 'k', 'g']))

    ax2 = axs.twinx()
    ax2.plot(dist_array, T_array, label='temperature', color='r')

    axs.plot([dist_array[on_catalyst], dist_array[on_catalyst]], [0, 0.15], linestyle='--', color='xkcd:grey')
    axs.plot([dist_array[off_catalyst], dist_array[off_catalyst]], [0, 0.15], linestyle='--', color='xkcd:grey')
    axs.annotate("catalyst", fontsize=13, xy=(dist_array[1500], 0.14), va='center', ha='center')

    for item in (
            axs.get_xticklabels() + axs.get_yticklabels() + ax2.get_xticklabels() + ax2.get_yticklabels()):
        item.set_fontsize(13)

    axs.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), fancybox=False, shadow=False, ncol=4)

    if x_lim is not None:
        x_min, x_max = x_lim
        axs.set_xlim(float(x_min), float(x_max))
        ax2.set_xlim(float(x_min), float(x_max))
    else:
        axs.set_xlim(0.0, length / mm)
        ax2.set_xlim(0.0, length / mm)
    axs.set_ylim(0., 0.15)
    ax2.set_ylim(600, 2400)
    axs.set_xlabel('Position (mm)', fontsize=16)
    axs.set_ylabel('Flow (mol/min)', fontsize=16)
    ax2.set_ylabel('Temperature (K)', fontsize=16)

    for n in range(len(gas_names)):
        if gas_names[n] == 'CH4(2)':
            c_in = gas_out[0][n]
        if gas_names[n] == 'O2(3)':
            o_in = gas_out[0][n]
    ratio = round(c_in / (o_in * 2), 1)

    out_dir = os.path.join(out_root, 'figures')
    os.path.exists(out_dir) or os.makedirs(out_dir)

    if x_lim is not None:
        fig.savefig(out_dir + '/' + str(ratio) + 'ratioZoom.pdf', bbox_inches='tight',)
    else:
        fig.savefig(out_dir + '/' + str(ratio) + 'ratioFull.pdf', bbox_inches='tight',)


def plot_surf(data, path_to_cti):
    """Plots surface site fractions through the PFR."""
    gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions = data

    fig, axs = plt.subplots()
    axs.set_prop_cycle(cycler('color', ['m', 'g', 'b', 'y', 'c', 'r', 'k', 'g']))

    # Plot two temperatures (of gas-phase and surface vs only surface.)
    for i in range(len(surf_out[0, :])):
        if surf_out[:, i].max() > 5.e-3:
            axs.plot(dist_array, surf_out[:, i], label=surf_names[i])
    axs.plot([dist_array[on_catalyst], dist_array[on_catalyst]], [0, 1.2], linestyle='--', color='xkcd:grey')
#     axs.plot([dist_array[off_catalyst], dist_array[off_catalyst]], [0, 1.2], linestyle='--', color='xkcd:grey')
    axs.annotate("catalyst", fontsize=13, xy=(dist_array[1500], 1.1), va='center', ha='center')

    for item in (
            axs.get_xticklabels() + axs.get_yticklabels()):
        item.set_fontsize(13)

    axs.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), fancybox=False, shadow=False, ncol=4)
    axs.set_ylim(0, 1.2)
    axs.set_xlim(8, 22)
    axs.set_xlabel('Position (mm)', fontsize=16)
    axs.set_ylabel('Site fraction', fontsize=16)

    #     temperature = np.round(T_array[0],0)
    for n in range(len(gas_names)):
        if gas_names[n] == 'CH4(2)':
            c_in = gas_out[0][n]
        if gas_names[n] == 'O2(3)':
            o_in = gas_out[0][n]
    ratio = c_in / (o_in * 2)
    ratio = round(ratio, 1)

    surface_name = os.path.split(path_to_cti)[-1][0:-5]
    out_dir = os.path.join(out_root, 'figures')
    os.path.exists(out_dir) or os.makedirs(out_dir)
    fig.savefig(out_dir + '/' + str(ratio) + 'surf.pdf', bbox_inches='tight')


def monolith_simulation(path_to_cti, temp, mol_in, rtol, atol, verbose=False, sens=False):
    """
    Set up and solve the monolith reactor simulation.
    Verbose prints out values as you go along
    Sens is for sensitivity, in the form [perturbation, reaction #]
    Args:
        path_to_cti: full path to the cti file
        temp (float): The temperature in Kelvin
        mol_in (3-tuple or iterable): the inlet molar ratios of (CH4, O2, He)
        verbose (Boolean): whether to print intermediate results
        sens (False or 2-tuple/list): if not False, then should be a 2-tuple or list [dk, rxn]
                in which dk = relative change (eg. 0.01) and rxn = the index of the surface reaction rate to change
    Returns:
        gas_out, # gas molar flow rate in moles/minute
        surf_out, # surface mole fractions
        gas_names, # gas species names
        surf_names, # surface species names
        dist_array, # distances (in mm)
        T_array # temperatures (in K)
    """
    sols_dict = setup_ct_solution(path_to_cti)
    gas, surf, i_ar, n_surf_reactions= sols_dict['gas'], sols_dict['surf'], sols_dict['i_ar'],sols_dict['n_surf_reactions']
    print(f"Running monolith simulation with CH4 and O2 concs {mol_in[0], mol_in[1]} on thread {threading.get_ident()}")
    ch4, o2, ar = mol_in
    ratio = ch4 / (2 * o2)

    X = f"CH4(2):{ch4}, O2(3):{o2}, Ar:{ar}"
    gas.TPX = 273.15, ct.one_atm, X  # need to initialize mass flow rate at STP
    # mass_flow_rate = velocity * gas.density_mass * area  # kg/s
    mass_flow_rate = flow_rate * gas.density_mass
    gas.TPX = temp, ct.one_atm, X
    temp_cat = temp
    surf.TP = temp_cat, ct.one_atm
    surf.coverages = 'X(1):1.0'
    gas.set_multiplier(1.0)

    TDY = gas.TDY
    cov = surf.coverages

    if verbose is True:
        print('  distance(mm)   X_CH4        X_O2        X_H2       X_CO       X_H2O       X_CO2')

    # create a new reactor
    gas.TDY = TDY
    r = ct.IdealGasReactor(gas)
    r.volume = rvol

    # create a reservoir to represent the reactor immediately upstream. Note
    # that the gas object is set already to the state of the upstream reactor
    upstream = ct.Reservoir(gas, name='upstream')

    # create a reservoir for the reactor to exhaust into. The composition of
    # this reservoir is irrelevant.
    downstream = ct.Reservoir(gas, name='downstream')

    # Add the reacting surface to the reactor. The area is set to the desired
    # catalyst area in the reactor.
    rsurf = ct.ReactorSurface(surf, r, A=cat_area)

    # The mass flow rate into the reactor will be fixed by using a
    # MassFlowController object.
    # mass_flow_rate = velocity * gas.density_mass * area  # kg/s
    # mass_flow_rate = flow_rate * gas.density_mass
    m = ct.MassFlowController(upstream, r, mdot=mass_flow_rate)

    # We need an outlet to the downstream reservoir. This will determine the
    # pressure in the reactor. The value of K will only affect the transient
    # pressure difference.
    v = ct.PressureController(r, downstream, master=m, K=1e-5)

    sim = ct.ReactorNet([r])
    sim.max_err_test_fails = 12

    # set relative and absolute tolerances on the simulation
    sim.rtol = rtol
    sim.atol = atol

    gas_names = gas.species_names
    surf_names = surf.species_names
    gas_out = []
    surf_out = []
    dist_array = []
    T_array = []
    net_rates_of_progress = []
    
    total_flux = {"gas": dict(), "surf": dict()}
    rsurf.kinetics.set_multiplier(0.0)  # no surface reactions until the gauze
    for n in range(N_reactors):
        # Set the state of the reservoir to match that of the previous reactor
        gas.TDY = r.thermo.TDY
        upstream.syncState()
        if n == on_catalyst:
            rsurf.kinetics.set_multiplier(1.0)
            if sens is not False:
                rsurf.kinetics.set_multiplier(1.0 + sens[0], sens[1])
        if n == off_catalyst:
            rsurf.kinetics.set_multiplier(0.0)
        sim.reinitialize()
        sim.advance(sim.time + 1e4 * residence_time)
        dist = n * reactor_len * 1.0e3  # distance in mm
        dist_array.append(dist)
        T_array.append(rsurf.kinetics.T)
        kmole_flow_rate = mass_flow_rate / gas.mean_molecular_weight  # kmol/s
        gas_out.append(1000 * 60 * kmole_flow_rate * r.kinetics.X.copy())  # molar flow rate in moles/minute
        surf_out.append(rsurf.kinetics.X.copy())
        net_rates_of_progress.append(rsurf.kinetics.net_rates_of_progress)
        # stop simulation when things are done changing, to avoid getting so many COVDES errors
        if n >= 1001:
            if np.max(abs(np.subtract(gas_out[-2], gas_out[-1]))) < 1e-15:
                break
        if n>=1000 and n<2000:
            # total_flux['gas'], total_flux['surf'] = add_fluxes(gas, surf, total_flux)
            gas_fluxes = total_flux["gas"]
            surf_fluxes = total_flux["surf"]
            for element in "HOC":
                new_diagram_gas = ct.ReactionPathDiagram(gas, element)
                new_diagram_gas.get_data() # neccessary before adding this into the other diagrams, this calls build function to build the fluxes first
                new_diagram_surf = ct.ReactionPathDiagram(surf, element)
                new_diagram_surf.get_data()
                try:
                    gas_fluxes[element].add(new_diagram_gas)
                except KeyError:
                    gas_fluxes[element] = new_diagram_gas
                try:
                    surf_fluxes[element].add(new_diagram_surf)
                except KeyError:
                    surf_fluxes[element] = new_diagram_surf
            # Now do the 'X' for just the surface
            element = "X"
            try:
                new_diagram_surf = ct.ReactionPathDiagram(surf, element)
                new_diagram_surf.get_data() # neccessary before adding this into the other diagrams, this calls build to build the fluxes first
                surf_fluxes[element].add(new_diagram_surf)
            except KeyError:
                surf_fluxes[element] = new_diagram_surf

        # make reaction diagrams
        out_dir = 'rxnpath'
        os.path.exists(out_dir) or os.makedirs(out_dir)
        elements = ['H', 'O', 'C']
        locations_of_interest = [1045, 1800]
        if sens is False:
            if n in locations_of_interest:
                # # plot the transient flux_diagrams
                # transient_flux_diagram(n, surf, out_dir, ratio)
                save_flux_diagrams(gas, surf, total_flux, path='rxnpath', suffix=f'{ratio:.1f}_{n}')
                df = pd.DataFrame(surf.net_rates_of_progress)
                df.to_csv(f"{out_dir}/net_rates_{ratio:.1f}_{n}.csv")

        if verbose is True:
            if not n % 100:
                print('  {0:10f}  {1:10f}  {2:10f}  {3:10f} {4:10f} {5:10f} {6:10f}'.format(dist, *gas[
                    'CH4(2)', 'O2(3)', 'H2(6)', 'CO(7)', 'H2O(5)', 'CO2(4)'].X * 1000 * 60 * kmole_flow_rate))

    gas_out = np.array(gas_out)
    surf_out = np.array(surf_out)
    gas_names = np.array(gas_names)
    surf_names = np.array(surf_names)
    data_out = gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions
    
    net_rates_of_progress = np.array(net_rates_of_progress)
    df = pd.DataFrame(net_rates_of_progress)
    
    out_dir = os.path.join(out_root, 'rates_of_progress')
    os.path.exists(out_dir) or os.makedirs(out_dir)
    df.to_csv(f"{out_dir}/net_rates_{ratio:.1f}.csv")
    print(len(dist_array))
    print(f"Finished monolith simulation for CH4 and O2 concs {mol_in[0], mol_in[1]} on thread {threading.get_ident()}")
    return data_out

def run_one_simulation(path_to_cti, rtol, atol, ratio):
    """
    Start all of the simulations all at once using multiprocessing
    """
    plot_generated = False
    fo2 = 1 / (2. * ratio + 1 + 79. / 21.)
    fch4 = 2 * fo2 * ratio
    far = 79 * fo2 / 21
    ratio_in = [fch4, fo2, far]  # mol fractions
    
    tl_list = [[tl, tl**2] for tl in [1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3]]
    
    try:
        a = monolith_simulation(path_to_cti, t_in, ratio_in, rtol, atol)
        gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions = a 
    except ct.CanteraError:
        tl = 0
        while tl < len(tl_list):
            try:
                a = monolith_simulation(path_to_cti, t_in, ratio_in, tl_list[tl][0], tl_list[tl][1])
                gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions = a
            except ct.CanteraError as e:
                if tl == len(tl_list) - 1:
                    raise e
                else:
                    print("SOLVER ERROR, GOING TO TRY AGAIN........")
            tl += 1
            if len(dist_array) == 7001:
                break
    
    if len(dist_array) < 7001:
        for tl in tl_list:
            try:
                sim_result = monolith_simulation(path_to_cti, t_in, ratio_in, tl[0], tl[1])
                gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions = sim_result
            except ct.CanteraError as e:
                gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions = a
            if len(dist_array) == 7001:
                    plot_gas(sim_result, path_to_cti)
                    plot_gas(sim_result,path_to_cti, x_lim=(8,25))
                    plot_surf(sim_result, path_to_cti)
                    plot_generated = True
                    break
                    
    if plot_generated is False:
        try:
            plot_gas(a, path_to_cti)
            plot_gas(a,path_to_cti, x_lim=(8,25))
            plot_surf(a, path_to_cti)
        except Exception as e:
            print(f"somthing wrong at ratio={ratio}")
            print(str(e))
    return [ratio, [gas_out, gas_names, dist_array, T_array, n_surf_reactions]]


def deriv(gas_out):
    deriv = []
    for x in range(len(gas_out) - 1):
        deriv.append((gas_out[x+1] - gas_out[x])/.01)
    deriv.append(0.)
    return deriv

def calculate(data, type='sens'):
    """
    Calculate properties of interest from the raw data
    :param data: the data
    :param type: 'sens' for sensitivity analyses
                 'output' for saving the output csv
                 'ratio' for plotting
    :return:
    """
    gas_out_data, gas_names_data, dist_array_data, T_array_data, n_surf_reactions = data

    reference = []
    for a in range(len(gas_names_data)):
        reference.append([gas_names_data[a], [gas_out_data[:, a]]])
    # This is the index the choose 
    sens_id = 1045
    
    for x in reference:
        if x[0] == 'CH4(2)':
            ch4_in = x[1][0][0]
            ch4_out = x[1][0][sens_id]
            if ch4_out < 0:
                ch4_out = 0.
            ch4_depletion = ch4_in - ch4_out
            reference_ch4_conv = ch4_depletion / ch4_in  # Sensitivity definition 7: CH4 conversion

            d_ch4 = deriv(x[1][0])
            reference_max_ch4_conv = min(d_ch4)  # Sensitivity definition 15: maximum rate of CH4 conversion

            conv50_pos = 0
            for y in range(len(x[1][0])):
                if (ch4_in - x[1][0][y]) / ch4_in >= 0.5:
                    if conv50_pos == 0:
                        conv50_pos = y
                        reference_dist_to_50_ch4_conv = dist_array_data[conv50_pos] # Sensitivity definition 14: distance to 50% CH4 conversion
                else:
                    # never reached 50% conversion
                    reference_dist_to_50_ch4_conv = 510.
        if x[0] == 'Ar':
            ar = x[1][0][sens_id]
        if x[0] == 'O2(3)':
            o2_in = x[1][0][0]
            o2_out = x[1][0][sens_id]
            if o2_out < 0:
                o2_out = 0.  # O2 can't be negative
            elif o2_out > o2_in:
                o2_out = o2_in  # O2 can't be created, to make it equal to O2 in
            o2_depletion = o2_in - o2_out
            reference_o2_conv = o2_depletion / o2_in  # Sensitivity definition 13: O2 conversion
        if x[0] == 'CO(7)':
            co_out = x[1][0][sens_id]
        if x[0] == 'H2(6)':
            h2_out = x[1][0][sens_id]
        if x[0] == 'H2O(5)':
            h2o_out = x[1][0][sens_id]
        if x[0] == 'CO2(4)':
            co2_out = x[1][0][sens_id]

    ratio = ch4_in / (2 * o2_in)

    # negative sensitivity is higher selectivity
    reference_h2_sel = h2_out / (ch4_depletion * 2)  # Sensitivity definition 5: H2 selectivity
    if reference_h2_sel <= 0:
        reference_h2_sel = 1.0e-15  # selectivity can't be 0

    reference_co_sel = co_out / ch4_depletion  # Sensitivity definition 3: CO selectivity
    if reference_co_sel <= 0:
        reference_co_sel = 1.0e-15  # selectivity can't be 0

    reference_syngas_selectivity = reference_co_sel + reference_h2_sel  # Sensitivity definition 1: SYNGAS selectivity

    reference_syngas_yield = reference_syngas_selectivity * reference_ch4_conv  # Sensitivity definition 2: SYNGAS yield
    if reference_syngas_yield <= 0:
        reference_syngas_yield = 1.0e-15  # yield can't be 0

    reference_co_yield = co_out / ch4_in  # Sensitivity definition 4: CO % yield
    # reference_co_yield = reference_co_sel * reference_ch4_conv

    reference_h2_yield = h2_out / (2 * ch4_in)  # Sensitivity definition 6: H2 % yield
    # reference_h2_yield = reference_h2_sel * reference_ch4_conv

    # Sensitivity definition 8: H2O + CO2 selectivity
    reference_h2o_sel = h2o_out / (ch4_depletion * 2)
    reference_co2_sel = co2_out / ch4_depletion
    if reference_h2o_sel <= 0:
        reference_h2o_sel = 1.0e-15  # H2O selectivity can't be 0
    if reference_co2_sel <= 0:
        reference_co2_sel = 1.0e-15  # CO2 selectivity can't be 0
    reference_full_oxidation_selectivity = reference_h2o_sel + reference_co2_sel

    # Sensitivity definition 9: H2O + CO2 yield
    reference_full_oxidation_yield = reference_full_oxidation_selectivity * reference_ch4_conv

    # Sensitivity definition 10: exit temperature
    reference_exit_temp = T_array_data[-1]

    # Sensitivity definition 11: peak temperature
    reference_peak_temp = max(T_array_data)

    # Sensitivity definition 12: distance to peak temperautre
    reference_peak_temp_dist = dist_array_data[T_array_data.index(max(T_array_data))]

    if type is 'sens':
        return reference_syngas_selectivity, reference_syngas_yield, reference_co_sel, reference_co_yield, reference_h2_sel, reference_h2_yield, reference_ch4_conv, reference_full_oxidation_selectivity, reference_full_oxidation_yield, reference_exit_temp, reference_peak_temp, reference_peak_temp_dist, reference_o2_conv, reference_max_ch4_conv, reference_dist_to_50_ch4_conv
    elif type is 'ratio':
        return reference_co_sel, reference_h2_sel, reference_ch4_conv, reference_exit_temp, reference_o2_conv, reference_co2_sel, reference_h2o_sel
    elif type is 'gas_data':
        return ratio, reference
    else:
        return ratio, ch4_in, ch4_out, co_out, h2_out, h2o_out, co2_out, reference_exit_temp, reference_peak_temp, reference_peak_temp_dist, reference_o2_conv, reference_max_ch4_conv, reference_dist_to_50_ch4_conv

def calc_sensitivities(reference, new, surf, index=None):
    """Calculates sensitivities given old simulation results and perturbed simulation results"""
    reference_syngas_selectivity, reference_syngas_yield, reference_co_sel, reference_co_yield, reference_h2_sel, reference_h2_yield, reference_ch4_conv, reference_full_oxidation_selectivity, reference_full_oxidation_yield, reference_exit_temp, reference_peak_temp, reference_peak_temp_dist, reference_o2_conv, reference_max_ch4_conv, reference_dist_to_50_ch4_conv = reference
    new_syngas_selectivity, new_syngas_yield, new_co_sel, new_co_yield, new_h2_sel, new_h2_yield, new_ch4_conv, new_full_oxidation_selectivity, new_full_oxidation_yield, new_exit_temp, new_peak_temp, new_peak_temp_dist, new_o2_conv, new_max_ch4_conv, new_dist_to_50_ch4_conv = new

    Sens5 = (new_h2_sel - reference_h2_sel) / (reference_h2_sel * dk)
    Sens3 = (new_co_sel - reference_co_sel) / (reference_co_sel * dk)
    Sens1 = (new_syngas_selectivity - reference_syngas_selectivity) / (reference_syngas_selectivity * dk)
    Sens2 = (new_syngas_yield - reference_syngas_yield) / (reference_syngas_yield * dk)
    Sens4 = (new_co_yield - reference_co_yield) / (reference_co_yield * dk)
    Sens6 = (new_h2_yield - reference_h2_yield) / (reference_h2_yield * dk)
    Sens7 = (new_ch4_conv - reference_ch4_conv) / (
            reference_ch4_conv * dk)
    Sens13 = (new_o2_conv - reference_o2_conv) / (reference_o2_conv * dk)
    Sens8 = (new_full_oxidation_selectivity - reference_full_oxidation_selectivity) / (
            reference_full_oxidation_selectivity * dk)
    Sens9 = (new_full_oxidation_yield - reference_full_oxidation_yield) / (reference_full_oxidation_yield * dk)
    Sens10 = (new_exit_temp - reference_exit_temp) / (reference_exit_temp * dk)
    Sens11 = (new_peak_temp - reference_peak_temp) / (reference_peak_temp * dk)
    Sens12 = (new_peak_temp_dist - reference_peak_temp_dist) / (reference_peak_temp_dist * dk)
    Sens14 = (new_max_ch4_conv - reference_max_ch4_conv) / (reference_max_ch4_conv * dk)
    Sens15 = (new_dist_to_50_ch4_conv - reference_dist_to_50_ch4_conv) / (reference_dist_to_50_ch4_conv * dk)

    if index is not None:
        rxn = surf.reaction_equations()[index]
        return rxn, Sens1, Sens2, Sens3, Sens4, Sens5, Sens6, Sens7, Sens8, Sens9, Sens10, Sens11, Sens12, Sens13, Sens14, Sens15
    else:
        return Sens1, Sens2, Sens3, Sens4, Sens5, Sens6, Sens7, Sens8, Sens9, Sens10, Sens11, Sens12, Sens13, Sens14, Sens15


def plot_ratio_comparisions(path_to_cti, data):
    ratios = [d[0] for d in data]
    fig, axs = plt.subplots(1, 2)

    # plot exit conversion and temp
    ch4_conv = [d[1][2] for d in data]
    axs[0].plot(ratios, ch4_conv, 'bo-', label='CH4', color='limegreen')
    o2_conv = [d[1][4] for d in data]
    axs[0].plot(ratios, o2_conv, 'bo-', label='O2', color='blue')
    ax2 = axs[0].twinx()
    temp = [d[1][3] for d in data]
    ax2.plot(ratios, temp, 'bo-', label='temp', color='orange')
    ax2.set_ylim(600.0, 2000)

    # plot exit selectivities
    co_sel = [d[1][0] for d in data]
    axs[1].plot(ratios, co_sel, 'bo-', label='CO', color='green')
    h2_sel = [d[1][1] for d in data]
    axs[1].plot(ratios, h2_sel, 'bo-', label='H2', color='purple')
    co2_sel = [d[1][5] for d in data]
    axs[1].plot(ratios, co2_sel, 'bo-', label='CO2', color='navy')
    h2o_sel = [d[1][6] for d in data]
    axs[1].plot(ratios, h2o_sel, 'bo-', label='H2O', color='dodgerblue')

    axs[0].legend()
    axs[1].legend()
    axs[0].set_ylabel('Exit conversion (%)', fontsize=13)
    ax2.set_ylabel('Exit temperature (K)', fontsize=13)
    axs[0].set_xlabel('C/O Ratio', fontsize=13)
    axs[1].set_xlabel('C/O Ratio', fontsize=13)
    axs[1].set_ylabel('Exit selectivity (%)', fontsize=13)
    plt.tight_layout()
    fig.set_figheight(6)
    fig.set_figwidth(16)
    out_dir = os.path.join(out_root, 'figures')
    os.path.exists(out_dir) or os.makedirs(out_dir)
    fig.savefig(out_dir + '/' + 'conversion&selectivity.pdf', bbox_inches='tight')


def sensitivity(path_to_cti, old_data, temp, dk, rtol, atol):
    """
    Rerun simulations for each perturbed surface reaction and compare to the
    original simulation (data) to get a numerical value for sensitivity.
    """
    sensitivity_results = []
    gas_out_data, gas_names_data, dist_array_data, T_array_data, n_surf_reactions = old_data
    surf = setup_ct_solution(path_to_cti)['surf']
    reference = []
    for a in range(len(gas_names_data)):
        reference.append([gas_names_data[a], [gas_out_data[:, a]]])

    # getting the ratio
    for x in reference:
        if x[0] == 'CH4(2)':
            ch4_in = x[1][0][0]
        if x[0] == 'O2(3)':
            o2_in = x[1][0][0]
        if x[0] == 'Ar':
            ar_in = x[1][0][0]
    ratio = ch4_in / (2 * o2_in)
    moles_in = [ch4_in, o2_in, ar_in]
    
    reference_data = calculate(old_data)

    # run the simulations
    for rxn in range(n_surf_reactions):
        try:
            gas_out, surf_out, gas_names, surf_names, dist_array, T_array, i_ar, n_surf_reactions_from_sim = monolith_simulation(path_to_cti, temp, moles_in, rtol, atol, sens=[dk, rxn])
            c = [gas_out, gas_names, dist_array, T_array, n_surf_reactions_from_sim]
            new_data = calculate(c, type='sens')
            sensitivities = calc_sensitivities(reference_data, new_data, surf, index=rxn)
            sensitivity_results.append(sensitivities)
        except Exception as e:
            print(str(e))
            print(f"solver issue for reaction {rxn}, writing everything to zero")
            sensitivities = ('solver crash', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
            sensitivity_results.append(sensitivities)
    return sensitivity_results


def export(path_to_cti, rxns_translated, ratio, rtol, atol):
    k = (pd.DataFrame.from_dict(data=rxns_translated, orient='columns'))
    k.columns = ['Reaction', 'SYNGAS Selec', 'SYNGAS Yield', 'CO Selectivity', 'CO % Yield', 'H2 Selectivity', 'H2 % Yield',
                 'CH4 Conversion', 'H2O+CO2 Selectivity', 'H2O+CO2 yield', 'Exit Temp', 'Peak Temp',
                 'Dist to peak temp', 'O2 Conversion', 'Max CH4 Conv', 'Dist to 50 CH4 Conv']
    surface_name = os.path.split(path_to_cti)[-1][0:-5]
    out_dir = os.path.join(out_root, f'sensitivities/{ratio}')
    os.path.exists(out_dir) or os.makedirs(out_dir)

    k.to_csv(out_dir + '/{:.1f}RxnSensitivity_rtol{:.2e}_atol{:.2e}.csv'.format(ratio, rtol, atol), header=True)


def sensitivity_worker(path_to_cti, rtol, atol, data):
    print('Starting sensitivity simulation for a C/O ratio of {:.1f}'.format(data[0]))
    old_data = data[1][0]
    ratio = data[0]
    sensitivities = sensitivity(path_to_cti, old_data, t_in, dk, rtol, atol)
    print('Finished sensitivity simulation for a C/O ratio of {:.1f}'.format(ratio))
    reactions = [d[0] for d in sensitivities]  # getting the reactions
    sensitivities = [list(s) for s in sensitivities]
    for x in range(len(reactions)):
        sensitivities[x][0] = reactions[x]
    export(path_to_cti, sensitivities, ratio, tols[0], tols[1])
    
if __name__ == "__main__":
    
    # ratios = [.6, .7, .8, .9, 1., 1.1, 1.2, 1.3, 1.4, 1.6, 1.8, 2., 2.2, 2.4, 2.6]
    rtols = [1.0e-8]
    atols = [1.0e-16]
    tol_comb = []
    for rtol in rtols:
        for atol in atols:
            tol_comb.append([rtol, atol])
    for tols in tol_comb:
        # ratios = [.6, .7, .8, .9, 1., 1.1, 1.2, 1.3, 1.4, 1.6, 1.8, 2., 2.2, 2.4, 2.6]
        # ratios = [.6, 1., 1.1, 1.2, 1.6, 2., 2.6]
        ratios = [.6, 1.]
        data = []
        num_threads = min(multiprocessing.cpu_count(), len(ratios))
        pool = multiprocessing.Pool(processes=num_threads)
        data = pool.map(partial(run_one_simulation, 'cantera.yaml', tols[0], tols[1]), ratios, 1) #use functools partial
        pool.close()
        pool.join()
        for r in data:
            output.append(calculate(r[1], type='output'))
        k = (pd.DataFrame.from_dict(data=output, orient='columns'))
        k.columns = ['C/O ratio', 'CH4 in', 'CH4 out', 'CO out', 'H2 out', 'H2O out', 'CO2 out', 'Exit temp', 'Max temp', 'Dist to max temp', 'O2 conv', 'Max CH4 Conv', 'Dist to 50 CH4 Conv']
        data_dir = os.path.join(out_root, 'sim_data')
        os.path.exists(data_dir) or os.makedirs(data_dir)
        k.to_csv(os.path.join(out_root, f'sim_data/rtol_{tols[0]}_atol_{tols[1]}_data.csv'), header=True)  # raw data
        
        # save gas profiles
        out_dir = os.path.join(out_root, f'gas_profiles')
        os.path.exists(out_dir) or os.makedirs(out_dir)
        for r in data:
            ratio, gas_profiles = calculate(r[1], type='gas_data')
            names = [i[0] for i in gas_profiles]
            gas_profiles_output = [i[1] for i in gas_profiles]
            gas_profiles_out = []
            for x in gas_profiles_output:
                gas_profiles_out.append(x[0].tolist())
            k = (pd.DataFrame(gas_profiles_out))
            k.to_csv(f'{out_dir}/gas_out' + str(ratio) + '.csv', header=True)
        
        ratio_comparison = []
        for r in data:
            ratio_comparison.append([r[0], calculate(r[1], type='ratio')])
        
        plot_ratio_comparisions('cantera.yaml', ratio_comparison)