From 63f30b2bf0de3e811610210531b3de79e7601182 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Mon, 13 Jan 2025 10:09:02 -0500
Subject: [PATCH 01/72] analysis script

---
 scripts/cce/athk_to_specter.py | 684 +++++++++++++++++++++++++++++++++
 1 file changed, 684 insertions(+)
 create mode 100755 scripts/cce/athk_to_specter.py

diff --git a/scripts/cce/athk_to_specter.py b/scripts/cce/athk_to_specter.py
new file mode 100755
index 000000000..84f5462ea
--- /dev/null
+++ b/scripts/cce/athk_to_specter.py
@@ -0,0 +1,684 @@
+#!/usr/bin/env python3
+## Alireza Rashti - Oct 2024 (C)
+## usage:
+## $ ./me -h
+##
+
+import sys
+import os
+import numpy as np
+from scipy import special
+import math
+import argparse
+import h5py
+import struct
+import glob
+from scipy.interpolate import interp1d
+
+# from itertools import product
+# import matplotlib.pyplot as plt
+# import glob
+# import sympy
+# import re
+
+## ---------------------------------------------------------------------- ##
+
+## field names
+g_field_names = [
+    "gxx",
+    "gxy",
+    "gxz",
+    "gyy",
+    "gyz",
+    "gzz",
+    "betax",
+    "betay",
+    "betaz",
+    "alp",
+]
+
+g_name_map = {
+    "gxx": "gxx",
+    "gxy": "gxy",
+    "gxz": "gxz",
+    "gyy": "gyy",
+    "gyz": "gyz",
+    "gzz": "gzz",
+    "betax": "Shiftx",
+    "betay": "Shifty",
+    "betaz": "Shiftz",
+    "alp": "Lapse",
+}
+
+g_name_index = {
+    "gxx": 4,
+    "gxy": 5,
+    "gxz": 6,
+    "gyy": 7,
+    "gyz": 8,
+    "gzz": 9,
+    "betax": 1,
+    "betay": 2,
+    "betaz": 3,
+    "alp": 0,
+}
+
+## real/imag
+g_re = 0
+g_im = 1
+
+## args
+g_args = None
+## various attrs
+g_attrs = None
+
+## debug
+g_debug_max_l = 2
+
+
+def parse_cli():
+  """
+    arg parser
+    """
+  p = argparse.ArgumentParser(
+      description="convert Athenak CCE dumps to Spectre CCE")
+  p.add_argument("-fpath", type=str, required=True, help="/path/to/cce/h5/dumps")
+  p.add_argument("-ftype", type=str, required=True, help="input file type:{h5,bin}")
+  p.add_argument("-d_out", type=str, required=True, help="/path/to/output/dir")
+  p.add_argument("-debug", type=str, default="n", help="debug=[y,n]")
+  p.add_argument(
+      "-radius",
+      type=float,
+      required=True,
+      help="interpolate all fields and their derivatives at this radius.",
+  )
+  p.add_argument(
+      "-t_deriv",
+      type=str,
+      default="Fourier",
+      help="method to take the time derivative of fields:{Fourier,fin_diff}",
+  )
+  p.add_argument(
+      "-r_deriv",
+      type=str,
+      default="ChebU",
+      help=
+      "method to take the radial derivative of fields:{ChebU:Chebyshev of second kind}",
+  )
+  p.add_argument(
+      "-interpolation",
+      type=str,
+      default="ChebU",
+      help=
+      "method to interpolate fields at a given r:{ChebU:Chebyshev of second kind}",
+  )
+
+  args = p.parse_args()
+  return args
+
+
+def load(fpath: str, field_name: str, attrs: dict) -> list:
+  """
+    read the field accroding to attrs.
+    return convention:
+      ret[real/imag, time_level, n, lm], eg:
+      ret[g_re,3,2,:] = Re(C_2lm(t=3)) for all lm
+      ret[g_im,3,2,:] = Im(C_2lm(t=3)) for all lm
+    """
+
+  if attrs["file_type"] == "h5":
+    lev_t = attrs["lev_t"]
+    max_n = attrs["max_n"]
+    max_lm = attrs["max_lm"]
+    shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
+    ret = np.empty(shape=shape, dtype=float)
+    with h5py.File(fpath, "r") as h5f:
+      # read & save
+      for i in range(0, lev_t):
+        key = f"{i}"
+        h5_re = h5f[f"{key}/{field_name}/re"]
+        h5_im = h5f[f"{key}/{field_name}/im"]
+        ret[g_re, i, :] = h5_re
+        ret[g_im, i, :] = h5_im
+  elif attrs["file_type"] == "bin":
+    lev_t = attrs["lev_t"]
+    max_n = attrs["max_n"]
+    max_lm = attrs["max_lm"]
+    shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
+    ret = np.empty(shape=shape, dtype=float)
+    # Load the list of files
+    flist = sorted(glob.glob(fpath+'/cce_*.bin'))
+    data = []
+    dat_real = []
+    dat_imag = []
+    t = []
+    for f in flist:
+      nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm = read_cce_file(f)
+      t.append(time)
+      dat_real.append(data_real[:,g_name_index[field_name],:])
+      dat_imag.append(data_imag[:,g_name_index[field_name],:])
+    dat_real = np.array(dat_real)
+    dat_imag = np.array(dat_imag)
+    t = np.array(t)
+    f_real = interp1d(t, dat_real,axis=0)
+    f_imag = interp1d(t, dat_imag,axis=0)
+
+    ret[g_re, :] = f_real(attrs["time"])
+    ret[g_im, :] = f_imag(attrs["time"])
+  else:
+    raise ValueError("no such option")
+
+  # print(ret)
+  return ret
+
+
+def lm_mode(l, m):
+  """
+    l and m mode convention
+    """
+  return l * l + l + m
+
+
+def read_cce_file(filename):
+    """
+    Reads binary data from a CCE output file generated by the provided C++ function.
+
+    Parameters:
+        filename (str): The path to the binary file.
+
+    Returns:
+        nr (int): Number of radial points.
+        num_l_modes (int): Number of l modes.
+        time (float): Time value from the simulation.
+        rin (float): Inner radial boundary.
+        rout (float): Outer radial boundary.
+        data_real (numpy.ndarray): Real part of the data, shape (nr, 10, num_angular_modes).
+        data_imag (numpy.ndarray): Imaginary part of the data, shape (nr, 10, num_angular_modes).
+        index_to_lm (dict): Mapping from data index to (l, m) values.
+    """
+    with open(filename, 'rb') as f:
+        # Read number of radial points (nr)
+        nr_bytes = f.read(4)
+        nr = struct.unpack('<i', nr_bytes)[0]  # little-endian integer
+
+        # Read number of l modes (num_l_modes)
+        num_l_modes_bytes = f.read(4)
+        num_l_modes = struct.unpack('<i', num_l_modes_bytes)[0]
+
+        # Read time
+        time_bytes = f.read(8)
+        time = struct.unpack('<d', time_bytes)[0]  # little-endian double
+
+        # Read inner and outer radial boundaries (rin and rout)
+        rin_bytes = f.read(8)
+        rin = struct.unpack('<d', rin_bytes)[0]
+
+        rout_bytes = f.read(8)
+        rout = struct.unpack('<d', rout_bytes)[0]
+
+        # Calculate the number of angular modes
+        num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
+
+        # Total number of data points
+        count = 10 * nr * num_angular_modes
+
+        # Read data_real array
+        data_real = np.fromfile(f, dtype='<d', count=count)  # little-endian double
+        data_real = data_real.reshape((nr, 10, num_angular_modes))
+
+        # Read data_imag array
+        data_imag = np.fromfile(f, dtype='<d', count=count)
+        data_imag = data_imag.reshape((nr, 10, num_angular_modes))
+
+        # Create a mapping from index to (l, m)
+        index_to_lm = {}
+        for l in range(1, num_l_modes + 1):
+            for m in range(-l, l + 1):
+                index = l * l + l + m
+                index_to_lm[index] = (l, m)
+
+    return nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm
+
+def get_attribute(fpath: str,
+                  field_name: str = "gxx",
+                  type: str = "h5",
+                  args=None) -> dict:
+  """
+    find attributes such as num. of time level, and n, lm in C_nlm
+    also saves the time value at each slice.
+    """
+  attrs = {}
+  if type == "h5":
+    attrs["file_type"] = "h5"
+    with h5py.File(fpath, "r") as h5f:
+      # find attribute about num. of time level, and n,l,m in C_nlm
+      attrs["lev_t"] = len(h5f.keys()) - 1
+      attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
+      attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1
+      attrs["r_in"] = h5f["metadata"].attrs["Rin"]
+      attrs["r_out"] = h5f["metadata"].attrs["Rout"]
+      # read & save time
+      time = []
+      for i in range(0, attrs["lev_t"]):
+        key = f"{i}"
+        t = h5f[key].attrs["Time"][0]
+        time.append(t)
+      attrs["time"] = np.array(time)
+  elif type == "bin":
+    attrs["file_type"] = "bin"
+    # Load the list of files
+    flist = sorted(glob.glob(fpath+'/cce_*.bin'))
+    t = []
+    data = []
+    for f in flist:
+      nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm = read_cce_file(f)
+      t.append(time)
+      #data.append(data_real + 1j * data_imag)
+    t = np.array(t)
+    #data = np.array(data)
+    attrs["lev_t"] = len(flist)
+    attrs["max_n"]  = nr
+    attrs["max_l"] = num_l_modes
+    attrs["max_lm"] = (num_l_modes + 1) ** 2
+    attrs["r_in"] = np.array([rin])
+    attrs["r_out"] = np.array([rout])
+    attrs["time"] = np.linspace(t.min(),t.max(),t.shape[0])
+  else:
+    raise ValueError("no such option")
+
+  # print(attrs)
+  return attrs
+
+
+def time_derivative_findiff(field: np.array, field_name: str, attrs: dict,
+                            args) -> np.array:
+  """
+    return the time derivative of the given field using finite diff. 2nd order
+    field(t,rel/img,n,lm)
+    """
+
+  print(f"finite difference time derivative: {field_name}", flush=True)
+  _, len_t, len_n, len_lm = field.shape
+  time = attrs["time"]
+  dt = np.gradient(time, 2)
+  dfield = np.empty_like(field)
+
+  for n in range(len_n):
+    for lm in range(len_lm):
+      dfield[g_re, :, n, lm] = np.gradient(field[g_re, :, n, lm], 2) / dt
+      dfield[g_im, :, n, lm] = np.gradient(field[g_im, :, n, lm], 2) / dt
+  
+  return dfield
+
+
+def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
+                            args) -> np.array:
+  """
+    return the time derivative of the given field using Fourier method
+    field(t,rel/img,n,lm)
+    """
+
+  print(f"Fourier time derivative: {field_name}", flush=True)
+  _, len_t, len_n, len_lm = field.shape
+  dt = attrs["time"][2] - attrs["time"][1]
+  wm = math.pi * 2.0 / (len_t * dt)
+
+  dfield = np.empty_like(field)
+  for n in range(len_n):
+    for lm in range(len_lm):
+      coeff = field[g_re, :, n, lm] + 1j * field[g_im, :, n, lm]
+      # F. transform
+      fft_coeff = np.fft.fft(coeff)
+      # if args["debug"] == 'y':
+      #  print("debug: normalization?",round(coeff[1],6) == round(np.fft.ifft(fft_coeff)[1],6))
+
+      # time derivative
+      half = len_t // 2 + 1
+      omega = np.empty(shape=half)
+      for i in range(0, half):
+        omega[i] = i * wm
+
+      dfft_coeff = np.empty_like(fft_coeff)
+      dfft_coeff[0] = 0
+      dfft_coeff[1:half] = (-np.imag(fft_coeff[1:half]) +
+                            1j * np.real(fft_coeff[1:half])) * omega[1:]
+      dfft_coeff[half:] = (np.imag(fft_coeff[half:]) -
+                           1j * np.real(fft_coeff[half:])) * omega[::-1][1:]
+
+      # not optimized version
+      """
+      dfft_coeff[0] = 0
+      for i in range(1, half):
+        omega = i * wm
+        re = np.real(fft_coeff[i])
+        im = np.imag(fft_coeff[i])
+        re2 = np.real(fft_coeff[-i])
+        im2 = np.imag(fft_coeff[-i])
+
+        dfft_coeff[i] = omega*complex(-im, re)
+        dfft_coeff[-i] = omega*complex(im2, -re2)
+
+      """
+      # F. inverse
+      coeff = np.fft.ifft(dfft_coeff)
+      dfield[g_re, :, n, lm] = np.real(coeff)
+      dfield[g_im, :, n, lm] = np.imag(coeff)
+
+  if args["debug"] == "y":
+    for n in range(len_n):
+      for l in range(2, g_debug_max_l + 1):
+        for m in range(-l, l + 1):
+          hfile = (f"{args['d_out']}/debug_{field_name}_n{n}l{l}m{m}.txt")
+          write_data = np.column_stack((
+              attrs["time"],
+              dfield[g_re, :, n, lm_mode(l, m)],
+              dfield[g_im, :, n, lm_mode(l, m)],
+              field[g_re, :, n, lm_mode(l, m)],
+              field[g_im, :, n, lm_mode(l, m)],
+          ))
+          np.savetxt(hfile, write_data, header="t dre/dt dim/dt re im")
+
+  return dfield
+
+
+def dUk_dx(order: int, x: float) -> float:
+  """
+    d(Chebyshev of second kind)/dx
+    """
+  assert x != 1 and x != -1
+  t = special.chebyt(order + 1)(x)
+  u = special.chebyu(order)(x)
+  duk_dx = (order + 1) * t - x * u
+  duk_dx /= x**2 - 1
+
+  return duk_dx
+
+
+def radial_derivative_at_r_chebu(field: np.array,
+                                 field_name: str,
+                                 attrs: dict,
+                                 args) -> np.array:
+  """
+    return the radial derivative of the given field using Chebyshev of
+    2nd kind method at the radius of interest.
+
+    f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
+    collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
+    x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+
+    field(t,rel/img,n,lm)
+    """
+
+  print(f"ChebyU radial derivative: {field_name}", flush=True)
+  _, len_t, len_n, len_lm = field.shape
+
+  r_1 = attrs["r_in"][0]
+  r_2 = attrs["r_out"][0]
+  assert r_1 != r_2
+  dx_dr = 2 / (r_2 - r_1)
+
+  if args["debug"] == "y":
+    # populate collocation points, roots of U_i
+    x_i = np.empty(shape=len_n, dtype=float)
+    for i in range(len_n):
+      x_i[i] = math.cos(math.pi * (i + 1) / (len_n + 1))
+
+    # dU_k/dx|x=x_i
+    duk_dx = np.empty(shape=(len_n, len_n), dtype=float)
+    for k in range(len_n):
+      for i in range(len_n):
+        t = special.chebyt(k + 1)(x_i[i])
+        u = special.chebyu(k)(x_i[i])
+        duk_dx[k, i] = (k + 1) * t - x_i[i] * u
+
+    duk_dx /= np.square(x_i) - 1
+
+    uk = np.empty(shape=len_n, dtype=float)
+    tk = np.empty(shape=len_n, dtype=float)
+    for k in range(len_n):
+      hfile = f"{args['d_out']}/cheb_k{k}.txt"
+      for i in range(len_n):
+        tk[i] = special.chebyt(k)(x_i[i])
+        uk[i] = special.chebyu(k)(x_i[i])
+
+      write_data = np.column_stack((x_i, uk, tk, duk_dx[k, :]))
+      np.savetxt(
+          hfile,
+          write_data,
+          header=f"x_i uk{k}(x_i) tk{k}(x_i) duk{k}(x_i)/dx",
+      )
+
+  dfield = np.zeros(shape=(len([g_re, g_im]), len_t, len_lm))
+  r = args["radius"]
+  x = (2 * r - r_1 - r_2) / (r_2 - r_1)
+  for k in range(len_n):
+    dfield[:, :, :] += field[:, :, k, :] * dUk_dx(k, x)
+
+  return dfield * dx_dr
+
+
+def time_derivative(field: np.array, field_name: str, attrs: dict, args):
+  """
+    return the time derivative of the given field
+    """
+
+  if args["t_deriv"] == "Fourier":
+    return time_derivative_fourier(field, field_name, attrs, args)
+  elif args["t_deriv"] == "fin_diff":
+    return time_derivative_findiff(field, field_name, attrs, args)
+  else:
+    raise ValueError("no such option")
+
+
+def radial_derivative_at_r(field: np.array, field_name: str, attrs: dict, args):
+  """
+    return the radial derivative of the given field at R=r
+    """
+
+  if args["r_deriv"] == "ChebU":
+    return radial_derivative_at_r_chebu(field, field_name, attrs, args)
+  else:
+    raise ValueError("no such option")
+
+
+class Interpolate_at_r:
+
+  def __init__(self, attrs: dict, args: dict):
+    """
+        interpolate the given field at R=r
+        """
+    self.attrs = attrs
+    self.args = args
+    self.len_t = attrs["lev_t"]
+    self.len_n = attrs["max_n"]
+    self.len_lm = attrs["max_lm"]
+    r_1 = attrs["r_in"][0]
+    r_2 = attrs["r_out"][0]
+    self.r = r = args["radius"]
+    self.x = (2 * r - r_1 - r_2) / (r_2 - r_1)
+
+    if args["interpolation"] == "ChebU":
+      self.Uk = np.empty(shape=self.len_n)
+      for k in range(self.len_n):
+        self.Uk[k] = special.chebyu(k)(self.x)
+      self.interp = self.interpolate_at_r_chebu
+    else:
+      raise ValueError("no such option")
+
+  def interpolate_at_r_chebu(self, field: np.array, field_name: str):
+    """
+        interpolate at R=r using Cheb U.
+        """
+    print(f"Interpolating at R={self.r}: {field_name}", flush=True)
+
+    field_r = np.zeros(shape=(len([g_re, g_im]), self.len_t, self.len_lm))
+    for k in range(self.len_n):
+      field_r[:, :, :] += field[:, :, k, :] * self.Uk[k]
+
+    return field_r
+
+  def interpolate(self, field: np.array, field_name: str):
+    return self.interp(field, field_name)
+
+
+def process_field(field_name: str) -> dict:
+  """
+    - read data
+    - find time derives
+    - find radial derives
+    - interpolate at R=r
+    """
+
+  # return
+  attrs = g_attrs
+  args = g_args
+  db = {}
+
+  # load data
+  field = load(args["fpath"], field_name, attrs)
+  # db[f"{field_name}"] = field
+
+  # time derivative
+  dfield_dt = time_derivative(field, field_name, attrs, args)
+
+  # interpolate at a specific radii
+  interpolate = Interpolate_at_r(attrs, args)
+  field_at_r = interpolate.interpolate(field, field_name)
+  db[f"{field_name}|r"] = field_at_r
+
+  dfield_dt_at_r = interpolate.interpolate(dfield_dt, f"d{field_name}/dt")
+  db[f"d{field_name}/dt|r"] = dfield_dt_at_r
+
+  # radial derivative at R=r
+  dfield_dr_at_r = radial_derivative_at_r(field, field_name, attrs, args)
+  db[f"d{field_name}/dr|r"] = dfield_dr_at_r
+
+  return db
+
+
+def h5_create_group(h5file, group_name: str):
+  """
+    create a group for h5
+    """
+  h5group = None
+
+  # create group if not exists
+  if h5file.get(group_name, default=None) == None:
+    h5group = h5file.create_group(group_name)
+  else:
+    raise ValueError("this group {group_name} is already exists.")
+
+  return h5group
+
+
+def h5_write_data(h5file,
+                  data: np.array,
+                  data_name: str,
+                  attrs: dict,
+                  args: dict):
+  """
+    reminder:
+      data[real/imag, time_level, lm]
+
+    write syntax, eg:
+
+    h5["gxx.dat"] =
+      [time_level, ['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)', 'gxx_Re(1,1)', 'gxx_Im(1,1)', ...] ]
+
+    h5["gxx.dat"].attrs['Legend'] = the associated column =
+      array(['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)', 'gxx_Re(1,1)', 'gxx_Im(1,1)', ...])
+
+    # => h5["gxx.dat"][3,0] = value of time at the dump level 3
+    # => h5["gxx.dat"][4,1] = value of gxx_Re(0,0) at the dump level 4
+
+    """
+
+  dataset_conf = dict(
+      name=f"{data_name}",
+      shape=(attrs["lev_t"], len([g_re, g_im]) * (attrs["max_l"]**2) + 1),
+      dtype=float, # chunks=True,
+      # compression="gzip",
+      # shuffle=True,
+  )
+
+  data_attrs = ["time"]
+
+  if args["debug"] == "y":
+    print(dataset_conf, flush=True)
+
+  h5file.create_dataset(**dataset_conf)
+
+  flat = 0
+  h5file[f"{data_name}"][:, flat] = attrs["time"]
+  flat += 1
+  for l in range(0, attrs["max_l"]):
+    for m in range(l, -l - 1,-1):
+      data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
+      data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")
+      h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(l, m)]
+      h5file[f"{data_name}"][:, flat + 1] = data[g_im, :, lm_mode(l, m)]
+      flat += 2
+  Legend = [s.encode('ascii', 'ignore') for s in data_attrs]
+  h5file[f"{data_name}"].attrs["Legend"] = Legend
+
+
+def write(f: str, db: dict, attrs: dict, args: dict):
+  """
+    write data on disk
+    """
+  print(f"writing: {f}", flush=True)
+
+  field_name = g_name_map[f"{f}"]
+  field_name_key = f"{field_name}.dat"
+  dfield_name_dr_key = f"Dr{field_name}.dat"
+  dfield_name_dt_key = f"Dt{field_name}.dat"
+
+  r = args["radius"]
+  file_name = os.path.join(args["d_out"], f"CceR{r:07.2f}.h5")
+  with h5py.File(file_name, "a") as h5file:
+
+    name = field_name_key
+    data = db[f"{f}|r"]
+    h5_write_data(h5file, data, name, attrs, args)
+
+    name = dfield_name_dr_key
+    data = db[f"d{f}/dr|r"]
+    h5_write_data(h5file, data, name, attrs, args)
+
+    name = dfield_name_dt_key
+    data = db[f"d{f}/dt|r"]
+    h5_write_data(h5file, data, name, attrs, args)
+
+
+def main(args):
+  """
+    create output required by Specter code
+    ref: https://spectre-code.org/tutorial_cce.html
+    """
+
+  global g_attrs
+  global g_args
+
+  g_args = args
+
+  # check if output dir exist, if not, mkdir
+  if not os.path.exists(g_args["d_out"]):
+    os.makedirs(g_args["d_out"])
+  # find attribute for an arbitrary field
+  g_attrs = get_attribute(args["fpath"],type=args["ftype"])
+
+  # for each field
+  # I'm afraid this method takes too much memory
+  # from multiprocessing import Pool
+  # with Pool(processes=len(g_field_names)) as p:
+  #  db = p.map(process_field, g_field_names)
+
+  for f in g_field_names:
+    db = process_field(f)
+    # write on disk
+    write(f, db, g_attrs, g_args)
+
+
+if __name__ == "__main__":
+  args = parse_cli()
+  main(args.__dict__)

From 74b341705c8e89b1988f1a8c6be6fd4b1e8c3f97 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 10:35:46 -0500
Subject: [PATCH 02/72] feat: nan checker.

---
 scripts/cce/athk_to_specter.py |  4 ++++
 scripts/cce/cce_run.sh         | 13 +++++++++++++
 scripts/cce/pre_run.sh         |  7 +++++++
 3 files changed, 24 insertions(+)
 create mode 100755 scripts/cce/cce_run.sh
 create mode 100755 scripts/cce/pre_run.sh

diff --git a/scripts/cce/athk_to_specter.py b/scripts/cce/athk_to_specter.py
index 84f5462ea..57933554c 100755
--- a/scripts/cce/athk_to_specter.py
+++ b/scripts/cce/athk_to_specter.py
@@ -613,6 +613,10 @@ def h5_write_data(h5file,
   flat += 1
   for l in range(0, attrs["max_l"]):
     for m in range(l, -l - 1,-1):
+      if args["debug"] == "y":
+        assert all(data[g_re, :, lm_mode(l, m)] != np.nan)
+        assert all(data[g_im, :, lm_mode(l, m)] != np.nan)
+      
       data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
       data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")
       h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(l, m)]
diff --git a/scripts/cce/cce_run.sh b/scripts/cce/cce_run.sh
new file mode 100755
index 000000000..5037849f2
--- /dev/null
+++ b/scripts/cce/cce_run.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+cd runs
+
+echo "running preproc...\n"
+./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_test.yaml
+
+echo "-------------------------"
+
+
+echo "running CCE..."
+./CharacteristicExtract --input-file cce_par.yaml
+
diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
new file mode 100755
index 000000000..beae51279
--- /dev/null
+++ b/scripts/cce/pre_run.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+./athk_to_specter.py -ftype bin -d_out ./debug -debug y \
+-t_deriv fin_diff -r_deriv ChebU -interpolation ChebU \
+-radius 30 \
+-fpath ./dat/bin/cce_bbh/
+

From d97f67752b66b5d5bd273e5d41188fe89af0bd3c Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 10:39:08 -0500
Subject: [PATCH 03/72] fix: name

---
 scripts/cce/{athk_to_specter.py => athk_to_spectre.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scripts/cce/{athk_to_specter.py => athk_to_spectre.py} (100%)

diff --git a/scripts/cce/athk_to_specter.py b/scripts/cce/athk_to_spectre.py
similarity index 100%
rename from scripts/cce/athk_to_specter.py
rename to scripts/cce/athk_to_spectre.py

From 0c8e36297f82f3dee0c4c65ae3d29e79642e18f5 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 11:32:19 -0500
Subject: [PATCH 04/72] feat: add debug script.

---
 scripts/cce/debug_athk_to_spectre.py | 120 +++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100755 scripts/cce/debug_athk_to_spectre.py

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
new file mode 100755
index 000000000..229e815bc
--- /dev/null
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+## Alireza Rashti - Jan 2025 (C)
+## usage:
+## $ ./me -h
+##
+
+import sys
+import os
+import numpy as np
+import math as m
+import argparse
+import h5py
+
+# import matplotlib.pyplot as plt
+# import glob
+# import sympy
+## ---------------------------------------------------------------------- ##
+
+g_name_map = {
+    "gxx": "gxx",
+    "gxy": "gxy",
+    "gxz": "gxz",
+    "gyy": "gyy",
+    "gyz": "gyz",
+    "gzz": "gzz",
+    "betax": "Shiftx",
+    "betay": "Shifty",
+    "betaz": "Shiftz",
+    "alp": "Lapse",
+}
+
+
+def parse_cli():
+  """
+    arg parser
+    """
+  p = argparse.ArgumentParser(description="debugging athk_to_spectre.py")
+  p.add_argument(
+      "-debug",
+      type=str,
+      default="plot_simple",
+      help="debug type=[plot_simple]",
+  )
+  p.add_argument(
+      "-fpath",
+      type=str,
+      required=True,
+      help="path/to/output/athk_to_spectre.py/h5",
+  )
+  p.add_argument(
+      "-field_name",
+      type=str,
+      default="gxx",
+      help="plot for this field [gxx,gxy,...]",
+  )
+  p.add_argument(
+      "-field_mode",
+      type=str,
+      default="Re_(2,2)",
+      help="plot this mode[Re_(l,m),Im_(l,m)]",
+  )
+
+  args = p.parse_args()
+  return args
+
+
+def find_h5_mode(h5f, field_name, mode_name, args):
+  mode = 0
+  flag = False
+  for m in h5f[field_name].attrs["Legend"]:
+    if m.find(mode_name) != -1:
+      flag = True
+      break
+    mode += 1
+
+  assert flag == True
+  return mode
+
+
+def read_h5_mode_and_derivs(args):
+
+  field_name = g_name_map[f"{field_name}"]
+  field_name_key = f"{field_name}.dat"
+  dfield_name_dr_key = f"Dr{field_name}.dat"
+  dfield_name_dt_key = f"Dt{field_name}.dat"
+
+  with h5py.File(args["fpath"], "r") as h5f:
+    h5file[f"{data_name}"].attrs["Legend"] = Legend
+
+    mode = find_h5_mode(h5f, f"{field_name_key}", args["field_mode"], args)
+    f = h5f[f"{field_name_key}"][:, mode]
+
+    mode = find_h5_mode(h5f, f"{dfield_name_dr_key}", args["field_mode"], args)
+    drf = h5f[f"{dfield_name_dr_key}"][:, mode]
+
+    mode = find_h5_mode(h5f, f"{dfield_name_dt_key}", args["field_mode"], args)
+    dtf = h5f[f"{dfield_name_dt_key}"][:, mode]
+
+  return f, drf, dtf
+
+
+def debug_plot_simple(args):
+  dat = read_h5_mode_and_derivs(args)
+  plot_simple_live(dat, args)
+
+
+def main(args):
+  """
+    debug
+    """
+
+  if args["debug"] == "plot_simple":
+    debug_plot_simple(args)
+  else:
+    raise ValueError("no such option")
+
+
+if __name__ == "__main__":
+  args = parse_cli()
+  main(args.__dict__)

From be07710fee6eb78335f4306435c2546d5055058f Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 11:36:25 -0500
Subject: [PATCH 05/72] fix: bug.

---
 scripts/cce/debug_athk_to_spectre.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index 229e815bc..e9904f764 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -56,8 +56,8 @@ def parse_cli():
   p.add_argument(
       "-field_mode",
       type=str,
-      default="Re_(2,2)",
-      help="plot this mode[Re_(l,m),Im_(l,m)]",
+      default="Re(2,2)",
+      help="plot this mode[Re(l,m),Im(l,m)]",
   )
 
   args = p.parse_args()
@@ -69,6 +69,7 @@ def find_h5_mode(h5f, field_name, mode_name, args):
   flag = False
   for m in h5f[field_name].attrs["Legend"]:
     if m.find(mode_name) != -1:
+      print("found mode for", field_name, m,mode_name)
       flag = True
       break
     mode += 1
@@ -79,14 +80,12 @@ def find_h5_mode(h5f, field_name, mode_name, args):
 
 def read_h5_mode_and_derivs(args):
 
-  field_name = g_name_map[f"{field_name}"]
+  field_name = g_name_map[args["field_name"]]
   field_name_key = f"{field_name}.dat"
   dfield_name_dr_key = f"Dr{field_name}.dat"
   dfield_name_dt_key = f"Dt{field_name}.dat"
 
   with h5py.File(args["fpath"], "r") as h5f:
-    h5file[f"{data_name}"].attrs["Legend"] = Legend
-
     mode = find_h5_mode(h5f, f"{field_name_key}", args["field_mode"], args)
     f = h5f[f"{field_name_key}"][:, mode]
 

From 21680c224f1214cc8ff394e0d7866a32b1bda0ec Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 12:12:13 -0500
Subject: [PATCH 06/72] feat: plot

---
 scripts/cce/debug_athk_to_spectre.py | 56 ++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 4 deletions(-)

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index e9904f764..d828fa992 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -10,8 +10,8 @@
 import math as m
 import argparse
 import h5py
+import matplotlib.pyplot as plt
 
-# import matplotlib.pyplot as plt
 # import glob
 # import sympy
 ## ---------------------------------------------------------------------- ##
@@ -87,6 +87,7 @@ def read_h5_mode_and_derivs(args):
 
   with h5py.File(args["fpath"], "r") as h5f:
     mode = find_h5_mode(h5f, f"{field_name_key}", args["field_mode"], args)
+    t = h5f[f"{field_name_key}"][:, 0]
     f = h5f[f"{field_name_key}"][:, mode]
 
     mode = find_h5_mode(h5f, f"{dfield_name_dr_key}", args["field_mode"], args)
@@ -95,13 +96,60 @@ def read_h5_mode_and_derivs(args):
     mode = find_h5_mode(h5f, f"{dfield_name_dt_key}", args["field_mode"], args)
     dtf = h5f[f"{dfield_name_dt_key}"][:, mode]
 
-  return f, drf, dtf
+  return (f, drf, dtf, t)
 
 
+def plot_simple_v_t(dat, args):
+  """
+    plot value vs time
+  """
+  fig, axes = plt.subplots(3, 1, sharex=True)
+  
+  # f
+  ax = axes[0]
+  label=args["field_name"]
+  conf = dict(ls="-", label=label, color="k")
+  ax.plot(dat[-1], dat[0], **conf)
+  ax.set_ylabel(label)
+  ax.grid(True)
+
+  # drf
+  ax = axes[1]
+  label="d"+args["field_name"] + "/dr"
+  conf = dict(ls="-", label=label, color="k")
+  ax.plot(dat[-1], dat[1], **conf)
+  ax.set_ylabel(label)
+  ax.grid(True)
+
+  # dtf
+  ax = axes[2]
+  label="d"+args["field_name"] + "/dt"
+  conf = dict(ls="-", label=label, color="k")
+  ax.plot(dat[-1], dat[2], **conf)
+  ax.grid(True)
+  ax.set_ylabel(label)
+  ax.set_xlabel("t/M")
+  
+  plt.tight_layout()
+  plt.show()
+
+  """ 
+  file_out = args["o"]
+  if args["debug"] == "y":
+    plt.show()
+  else:
+    plt.savefig(file_out)
+    os.system("pdfcrop " + file_out + " " + file_out)
+  """
+
+
+  
+
+  
+
 def debug_plot_simple(args):
   dat = read_h5_mode_and_derivs(args)
-  plot_simple_live(dat, args)
-
+  plot_simple_v_t(dat, args)
 
 def main(args):
   """

From 4641d85bff17b54aba5e23a909b1277f0d18169d Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 12:27:22 -0500
Subject: [PATCH 07/72] fix.

---
 scripts/cce/debug_athk_to_spectre.py | 42 +++++++++++++++-------------
 scripts/cce/pre_run.sh               |  7 ++++-
 2 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index d828fa992..fc463bf86 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -47,6 +47,12 @@ def parse_cli():
       required=True,
       help="path/to/output/athk_to_spectre.py/h5",
   )
+  p.add_argument(
+      "-dout",
+      type=str,
+      default="./",
+      help="path/to/output/dir",
+  )
   p.add_argument(
       "-field_name",
       type=str,
@@ -69,7 +75,7 @@ def find_h5_mode(h5f, field_name, mode_name, args):
   flag = False
   for m in h5f[field_name].attrs["Legend"]:
     if m.find(mode_name) != -1:
-      print("found mode for", field_name, m,mode_name)
+      print("found mode for", field_name, m, mode_name)
       flag = True
       break
     mode += 1
@@ -102,12 +108,12 @@ def read_h5_mode_and_derivs(args):
 def plot_simple_v_t(dat, args):
   """
     plot value vs time
-  """
+    """
   fig, axes = plt.subplots(3, 1, sharex=True)
-  
+
   # f
   ax = axes[0]
-  label=args["field_name"]
+  label = args["field_name"]
   conf = dict(ls="-", label=label, color="k")
   ax.plot(dat[-1], dat[0], **conf)
   ax.set_ylabel(label)
@@ -115,7 +121,7 @@ def plot_simple_v_t(dat, args):
 
   # drf
   ax = axes[1]
-  label="d"+args["field_name"] + "/dr"
+  label = "d" + args["field_name"] + "/dr"
   conf = dict(ls="-", label=label, color="k")
   ax.plot(dat[-1], dat[1], **conf)
   ax.set_ylabel(label)
@@ -123,34 +129,30 @@ def plot_simple_v_t(dat, args):
 
   # dtf
   ax = axes[2]
-  label="d"+args["field_name"] + "/dt"
+  label = "d" + args["field_name"] + "/dt"
   conf = dict(ls="-", label=label, color="k")
   ax.plot(dat[-1], dat[2], **conf)
   ax.grid(True)
   ax.set_ylabel(label)
   ax.set_xlabel("t/M")
-  
-  plt.tight_layout()
-  plt.show()
-
-  """ 
-  file_out = args["o"]
-  if args["debug"] == "y":
-    plt.show()
-  else:
-    plt.savefig(file_out)
-    os.system("pdfcrop " + file_out + " " + file_out)
-  """
 
+  plt.tight_layout()
+  # plt.show()
 
-  
+  mode = args["field_mode"][3:-1]
+  lm = mode.split(",")
+  file_out = os.path.join(
+      args["dout"],
+      args["field_name"] + "_" + f"l{lm[0]}m{lm[1]}" + "_vs_time.png",
+  )
+  plt.savefig(file_out, dpi=200)
 
-  
 
 def debug_plot_simple(args):
   dat = read_h5_mode_and_derivs(args)
   plot_simple_v_t(dat, args)
 
+
 def main(args):
   """
     debug
diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index beae51279..fb0a23594 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -1,7 +1,12 @@
 #!/usr/bin/env bash
 
-./athk_to_specter.py -ftype bin -d_out ./debug -debug y \
+if false
+then
+./athk_to_spectre.py -ftype bin -d_out ./debug -debug y \
 -t_deriv fin_diff -r_deriv ChebU -interpolation ChebU \
 -radius 30 \
 -fpath ./dat/bin/cce_bbh/
+fi
 
+# plot fig
+./debug_athk_to_spectre.py -debug plot_simple -fpath ./debug/CceR0030.00.h5 -dout ./debug

From c618d78fdbf7045a516d6405dd31865e6944f558 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 16 Jan 2025 12:32:00 -0500
Subject: [PATCH 08/72] feat: add plot script.

---
 scripts/cce/pre_run.sh | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index fb0a23594..098e7d536 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -8,5 +8,34 @@ then
 -fpath ./dat/bin/cce_bbh/
 fi
 
+file="./debug/CceR0030.00.h5"
 # plot fig
-./debug_athk_to_spectre.py -debug plot_simple -fpath ./debug/CceR0030.00.h5 -dout ./debug
+fname="gxx"
+mode="Re(0,0)"
+./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+	-fpath $file  -field_name $fname -field_mode $mode
+
+mode="Re(2,2)"
+./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+	-fpath $file  -field_name $fname -field_mode $mode
+
+
+fname="alp"
+mode="Re(0,0)"
+./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+	-fpath $file  -field_name $fname -field_mode $mode
+
+mode="Re(2,2)"
+./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+	-fpath $file  -field_name $fname -field_mode $mode
+
+
+fname="betax"
+mode="Re(0,0)"
+./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+	-fpath $file  -field_name $fname -field_mode $mode
+
+mode="Re(2,2)"
+./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+	-fpath $file  -field_name $fname -field_mode $mode
+

From a7d66a10a8521b0177e3dbbb3e17509d28627a91 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 17 Jan 2025 08:26:04 -0500
Subject: [PATCH 09/72] feat: add mode convergence.

---
 scripts/cce/debug_athk_to_spectre.py | 104 +++++++++++++++++++++++++--
 1 file changed, 97 insertions(+), 7 deletions(-)

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index fc463bf86..4fd110154 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -66,11 +66,18 @@ def parse_cli():
       help="plot this mode[Re(l,m),Im(l,m)]",
   )
 
+  p.add_argument(
+      "-time_dump",
+      type=int,
+      default=100,
+      help="how often dump for mode convergence",
+  )
+
   args = p.parse_args()
   return args
 
 
-def find_h5_mode(h5f, field_name, mode_name, args):
+def find_h5_1mode(h5f, field_name, mode_name, args):
   mode = 0
   flag = False
   for m in h5f[field_name].attrs["Legend"]:
@@ -84,7 +91,32 @@ def find_h5_mode(h5f, field_name, mode_name, args):
   return mode
 
 
-def read_h5_mode_and_derivs(args):
+def find_h5_all_modes(h5f, field_name, mode_name, args):
+  modes = []
+  names = []
+  re_or_im = mode_name[0:2] ## Re(...)
+  print(re_or_im)
+  assert re_or_im == "Re" or re_or_im == "Im"
+
+  legends = h5f[field_name].attrs["Legend"]
+  for i in range(len(legends)):
+    m = legends[i]
+    if m.find(re_or_im) != -1:
+      names.append(m)
+      modes.append(i)
+
+  assert len(modes)
+
+  # print(modes)
+  # print(names)
+
+  return modes
+
+
+def read_h5_1mode(args):
+  """
+    return t, f(t)|mode, df(t)/dr|mod, df(t)/dt|mode
+    """
 
   field_name = g_name_map[args["field_name"]]
   field_name_key = f"{field_name}.dat"
@@ -92,19 +124,72 @@ def read_h5_mode_and_derivs(args):
   dfield_name_dt_key = f"Dt{field_name}.dat"
 
   with h5py.File(args["fpath"], "r") as h5f:
-    mode = find_h5_mode(h5f, f"{field_name_key}", args["field_mode"], args)
+    mode = find_h5_1mode(h5f, f"{field_name_key}", args["field_mode"], args)
     t = h5f[f"{field_name_key}"][:, 0]
     f = h5f[f"{field_name_key}"][:, mode]
 
-    mode = find_h5_mode(h5f, f"{dfield_name_dr_key}", args["field_mode"], args)
+    mode = find_h5_1mode(h5f, f"{dfield_name_dr_key}", args["field_mode"], args)
     drf = h5f[f"{dfield_name_dr_key}"][:, mode]
 
-    mode = find_h5_mode(h5f, f"{dfield_name_dt_key}", args["field_mode"], args)
+    mode = find_h5_1mode(h5f, f"{dfield_name_dt_key}", args["field_mode"], args)
     dtf = h5f[f"{dfield_name_dt_key}"][:, mode]
 
   return (f, drf, dtf, t)
 
 
+def read_h5_all_modes(args):
+  """
+    return t, f(mode)|t, df(mode)/dr|t, df(mode)/dt|t
+
+    f[t_i,modes] = at t_i what are the modes
+    """
+
+  field_name = g_name_map[args["field_name"]]
+  field_name_key = f"{field_name}.dat"
+  dfield_name_dr_key = f"Dr{field_name}.dat"
+  dfield_name_dt_key = f"Dt{field_name}.dat"
+
+  with h5py.File(args["fpath"], "r") as h5f:
+    t = h5f[f"{field_name_key}"][:, 0]
+    n_dumps = len(t) // args["time_dump"]
+    n_dumps = n_dumps + 1 if len(t) % args["time_dump"] else n_dumps
+
+    modes = find_h5_all_modes(h5f,
+                              f"{field_name_key}",
+                              args["field_mode"],
+                              args)
+    f = np.empty(shape=(n_dumps, len(modes)))
+    k = 0 # note: f[k,.] is 0,1,... but i jumps
+    for i in range(0, len(t), args["time_dump"]):
+      for j in range(len(modes)):
+        f[k, j] = h5f[f"{field_name_key}"][i, modes[j]]
+      k += 1
+
+    modes = find_h5_all_modes(h5f,
+                              f"{dfield_name_dr_key}",
+                              args["field_mode"],
+                              args)
+    drf = np.empty(shape=(n_dumps, len(modes)))
+    k = 0
+    for i in range(0, len(t), args["time_dump"]):
+      for j in range(len(modes)):
+        drf[k, j] = h5f[f"{dfield_name_dr_key}"][i, modes[j]]
+      k += 1
+
+    modes = find_h5_all_modes(h5f,
+                              f"{dfield_name_dt_key}",
+                              args["field_mode"],
+                              args)
+    dtf = np.empty(shape=(n_dumps, len(modes)))
+    k = 0
+    for i in range(0, len(t), args["time_dump"]):
+      for j in range(len(modes)):
+        dtf[k, j] = h5f[f"{dfield_name_dt_key}"][i, modes[j]]
+      k += 1
+
+  return (f, drf, dtf, t)
+
+
 def plot_simple_v_t(dat, args):
   """
     plot value vs time
@@ -149,8 +234,13 @@ def plot_simple_v_t(dat, args):
 
 
 def debug_plot_simple(args):
-  dat = read_h5_mode_and_derivs(args)
-  plot_simple_v_t(dat, args)
+  # value vs time
+  # dat = read_h5_1mode(args)
+  # plot_simple_v_t(dat, args)
+
+  # conv test
+  dat = read_h5_all_modes(args)
+  # plot_simple_mode(dat, args)
 
 
 def main(args):

From 54791157c9696cb05752f39ec89a92d508c85199 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 17 Jan 2025 08:52:48 -0500
Subject: [PATCH 10/72] fix: plots.

---
 scripts/cce/debug_athk_to_spectre.py | 85 +++++++++++++++++++++++++---
 1 file changed, 77 insertions(+), 8 deletions(-)

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index 4fd110154..5f3cc09f3 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -93,16 +93,16 @@ def find_h5_1mode(h5f, field_name, mode_name, args):
 
 def find_h5_all_modes(h5f, field_name, mode_name, args):
   modes = []
-  names = []
+  # names = []
   re_or_im = mode_name[0:2] ## Re(...)
-  print(re_or_im)
+  # print(re_or_im)
   assert re_or_im == "Re" or re_or_im == "Im"
 
   legends = h5f[field_name].attrs["Legend"]
   for i in range(len(legends)):
     m = legends[i]
     if m.find(re_or_im) != -1:
-      names.append(m)
+      # names.append(m)
       modes.append(i)
 
   assert len(modes)
@@ -137,13 +137,14 @@ def read_h5_1mode(args):
   return (f, drf, dtf, t)
 
 
-def read_h5_all_modes(args):
+def read_h5_all_modes(args) -> dict:
   """
     return t, f(mode)|t, df(mode)/dr|t, df(mode)/dt|t
 
     f[t_i,modes] = at t_i what are the modes
     """
 
+  dat = {}
   field_name = g_name_map[args["field_name"]]
   field_name_key = f"{field_name}.dat"
   dfield_name_dr_key = f"Dr{field_name}.dat"
@@ -187,13 +188,21 @@ def read_h5_all_modes(args):
         dtf[k, j] = h5f[f"{dfield_name_dt_key}"][i, modes[j]]
       k += 1
 
-  return (f, drf, dtf, t)
+  dat["t"] = t
+  dat["f"] = f
+  dat["drf"] = drf
+  dat["dtf"] = dtf
+  dat["n_dumps"] = n_dumps
+
+  return dat
 
 
 def plot_simple_v_t(dat, args):
   """
     plot value vs time
     """
+
+  print("plot value vs time ...")
   fig, axes = plt.subplots(3, 1, sharex=True)
 
   # f
@@ -233,14 +242,74 @@ def plot_simple_v_t(dat, args):
   plt.savefig(file_out, dpi=200)
 
 
+def plot_simple_modes(dat, args):
+  """
+    plot modes for different times
+    """
+
+  print("plot modes for different times ...")
+
+  t = dat["t"]
+  f = dat["f"]
+  drf = dat["drf"]
+  dtf = dat["dtf"]
+  n_dumps = dat["n_dumps"]
+  x = np.arange(0, len(f[0,:]), dtype=int)
+  p = 0
+  for i in range(0, len(t), args["time_dump"]):
+    title = f"t_{t[i]}"
+
+    fig, axes = plt.subplots(3, 1, sharex=True)
+
+    # f
+    ax = axes[0]
+    ax.set_title(title)
+    label = args["field_name"]
+    conf = dict(ls="-", label=label, color="k")
+    ax.plot(x, np.abs(f[p, :]), **conf)
+    ax.set_ylabel(label)
+    ax.set_yscale('log')
+    ax.grid(True)
+
+    # drf
+    ax = axes[1]
+    label = "d" + args["field_name"] + "/dr"
+    conf = dict(ls="-", label=label, color="k")
+    ax.plot(x, np.abs(drf[p,:]), **conf)
+    ax.set_ylabel(label)
+    ax.set_yscale('log')
+    ax.grid(True)
+
+    # dtf
+    ax = axes[2]
+    label = "d" + args["field_name"] + "/dt"
+    conf = dict(ls="-", label=label, color="k")
+    ax.plot(x, np.abs(dtf[p,:]), **conf)
+    ax.grid(True)
+    ax.set_ylabel(label)
+    ax.set_yscale('log')
+    ax.set_xlabel("modes")
+
+    plt.tight_layout()
+    # plt.show()
+
+    file_out = os.path.join(
+        args["dout"],
+        args["field_name"] + "_" + "modes_" + f"{title}.png",
+    )
+    plt.savefig(file_out, dpi=200)
+
+    p += 1
+
+
 def debug_plot_simple(args):
   # value vs time
-  # dat = read_h5_1mode(args)
-  # plot_simple_v_t(dat, args)
+  dat = read_h5_1mode(args)
+  plot_simple_v_t(dat, args)
 
   # conv test
   dat = read_h5_all_modes(args)
-  # plot_simple_mode(dat, args)
+  plot_simple_modes(dat, args)
 
 
 def main(args):

From 193da9b17b640887cb0778af909ff3274bcf1dc5 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 07:24:20 -0500
Subject: [PATCH 11/72] fix: minor.

---
 scripts/cce/debug_athk_to_spectre.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index 5f3cc09f3..b56cf1800 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -298,7 +298,7 @@ def plot_simple_modes(dat, args):
         args["field_name"] + "_" + "modes_" + f"{title}.png",
     )
     plt.savefig(file_out, dpi=200)
-
+    plt.close()
     p += 1
 
 

From 47636c01edcb8c3358fd36d372966d4f85e5f8d3 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 09:57:50 -0500
Subject: [PATCH 12/72] fix: minor bug.

---
 scripts/cce/athk_to_spectre.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 57933554c..d27c0ba80 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -95,7 +95,7 @@ def parse_cli():
   p.add_argument(
       "-t_deriv",
       type=str,
-      default="Fourier",
+      default="fin_diff",
       help="method to take the time derivative of fields:{Fourier,fin_diff}",
   )
   p.add_argument(

From 2d01df15868d8568133e1e6e3716d83fbd3f3006 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 09:58:13 -0500
Subject: [PATCH 13/72] update debug scripts.

---
 scripts/cce/cce_run.sh | 28 ++++++++++++++++++++++++----
 scripts/cce/pre_run.sh | 37 ++++++++++++++++++++++++-------------
 2 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/scripts/cce/cce_run.sh b/scripts/cce/cce_run.sh
index 5037849f2..919bf90e7 100755
--- a/scripts/cce/cce_run.sh
+++ b/scripts/cce/cce_run.sh
@@ -1,13 +1,33 @@
 #!/usr/bin/env bash
+set -x
 
+if false
+then
 cd runs
-
 echo "running preproc...\n"
-./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_test.yaml
-
+./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_bin.yaml
 echo "-------------------------"
+echo "running CCE..."
+./CharacteristicExtract --input-file cce_par_bin.yaml
 
+# h5 test
+elif false
+then
+cd runs
+echo "running preproc...\n"
+./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_h5.yaml
+echo "-------------------------"
+echo "running CCE..."
+./CharacteristicExtract --input-file cce_par_h5.yaml
 
+# q2
+elif true
+then
+cd runs
+echo "running preproc...\n"
+./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_q2_h5.yaml
+echo "-------------------------"
 echo "running CCE..."
-./CharacteristicExtract --input-file cce_par.yaml
+./CharacteristicExtract --input-file cce_par_q2_h5.yaml
+fi
 
diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index 098e7d536..33a6c0cd8 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -9,33 +9,44 @@ then
 fi
 
 file="./debug/CceR0030.00.h5"
+dout='./debug/bin'
+
+if false
+then
+./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 \
+-ftype h5 -d_out ./debug/q2/ -radius 100
+fi
+
+file="./debug/q2/CceR0100.00.h5"
+dout='./debug/q2'
+
 # plot fig
 fname="gxx"
-mode="Re(0,0)"
-./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
-	-fpath $file  -field_name $fname -field_mode $mode
-
 mode="Re(2,2)"
-./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 	-fpath $file  -field_name $fname -field_mode $mode
 
+#mode="Re(0,0)"
+#./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+#	-fpath $file  -field_name $fname -field_mode $mode
+
 
 fname="alp"
-mode="Re(0,0)"
-./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
-	-fpath $file  -field_name $fname -field_mode $mode
+#mode="Re(0,0)"
+#./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+#	-fpath $file  -field_name $fname -field_mode $mode
 
 mode="Re(2,2)"
-./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 	-fpath $file  -field_name $fname -field_mode $mode
 
 
 fname="betax"
-mode="Re(0,0)"
-./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
-	-fpath $file  -field_name $fname -field_mode $mode
+#mode="Re(0,0)"
+#./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+#	-fpath $file  -field_name $fname -field_mode $mode
 
 mode="Re(2,2)"
-./debug_athk_to_spectre.py -debug plot_simple -dout ./debug \
+./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 	-fpath $file  -field_name $fname -field_mode $mode
 

From 7482abdde26bca28c62a6982706848c6a52c9cf1 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 09:58:52 -0500
Subject: [PATCH 14/72] feat: add cce spectre plot.

---
 scripts/cce/cce_plot.py | 97 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100755 scripts/cce/cce_plot.py

diff --git a/scripts/cce/cce_plot.py b/scripts/cce/cce_plot.py
new file mode 100755
index 000000000..6006847a3
--- /dev/null
+++ b/scripts/cce/cce_plot.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+## Alireza Rashti - Jan 2025 (C)
+## usage:
+## $ ./me -h
+##
+
+import sys
+import os
+import numpy as np
+import math as m
+import argparse
+import re
+import h5py
+
+# import matplotlib.pyplot as plt
+# import glob
+# import sympy
+## ---------------------------------------------------------------------- ##
+
+
+def parse_cli():
+  """
+    arg parser
+    """
+  p = argparse.ArgumentParser(description="plotting cce spectre output")
+  p.add_argument("-f",
+                 type=str,
+                 required=True,
+                 help="/path/to/specter/cce/output.h5")
+  p.add_argument("-dout",
+                 type=str,
+                 required=True,
+                 help="/path/to/output/file.txt")
+  p.add_argument(
+      "-field",
+      type=str,
+      default="Strain",
+      help="field names:[News,Psi0,Psi1,Psi2,Psi3,Psi4,Strain]",
+  )
+  p.add_argument(
+      "-mode",
+      type=str,
+      default="Real Y_2,2",
+      help='field modes:["Real Y_2,-2","Imag Y_2,-2", ...]',
+  )
+  args = p.parse_args()
+  return args
+
+
+def find_h5_1mode(h5f, field_name, mode_name, args):
+  mode = 0
+  flag = False
+  for m in h5f[field_name].attrs["Legend"]:
+    if m.find(mode_name) != -1:
+      print("found mode for", field_name, m, mode_name)
+      flag = True
+      break
+    mode += 1
+
+  assert flag == True
+  return mode
+
+
+def read_save_to_txt(args):
+
+  with h5py.File(args["f"], "r") as h5f:
+    # find group
+    keys = h5f.keys()
+    group = [k for k in keys if k.find("Spectre") != -1][0]
+    group = "/" + group + "/"
+    dataset = group+args["field"]
+    mode = find_h5_1mode(h5f, dataset, args["mode"], args)
+    t = h5f[dataset][:, 0]
+    f = h5f[dataset][:, mode]
+
+  mode_txt = args["mode"].replace(",", "_")
+  mode_txt = mode_txt.replace(" ", "")
+  out = os.path.join(args["dout"], args["field"] + "_" + mode_txt + ".txt")
+  #print(out)
+  stack = np.column_stack((t, f))
+  np.savetxt(out,
+             stack,
+             header=f'# t {args["mode"]}',
+             fmt="%.16e %.16e")
+
+
+def main(args):
+  """
+    read cce spectre file
+    """
+
+  read_save_to_txt(args)
+
+
+if __name__ == "__main__":
+  args = parse_cli()
+  main(args.__dict__)

From af2703473910399c790e53bc6451590fdf792a51 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 10:42:56 -0500
Subject: [PATCH 15/72] fix: bugs.

---
 scripts/cce/cce_plot.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/scripts/cce/cce_plot.py b/scripts/cce/cce_plot.py
index 6006847a3..2afcd65cf 100755
--- a/scripts/cce/cce_plot.py
+++ b/scripts/cce/cce_plot.py
@@ -34,14 +34,14 @@ def parse_cli():
   p.add_argument(
       "-field",
       type=str,
-      default="Strain",
+      default="Psi4",
       help="field names:[News,Psi0,Psi1,Psi2,Psi3,Psi4,Strain]",
   )
   p.add_argument(
       "-mode",
       type=str,
-      default="Real Y_2,2",
-      help='field modes:["Real Y_2,-2","Imag Y_2,-2", ...]',
+      default="Y_2,2",
+      help='field modes:["Y_2,-2","Y_2,-2", ...]',
   )
   args = p.parse_args()
   return args
@@ -69,20 +69,21 @@ def read_save_to_txt(args):
     group = [k for k in keys if k.find("Spectre") != -1][0]
     group = "/" + group + "/"
     dataset = group+args["field"]
-    mode = find_h5_1mode(h5f, dataset, args["mode"], args)
+    mode_re = find_h5_1mode(h5f, dataset, "Real "+args["mode"], args)
+    mode_im = find_h5_1mode(h5f, dataset, "Imag "+args["mode"], args)
     t = h5f[dataset][:, 0]
-    f = h5f[dataset][:, mode]
+    re = h5f[dataset][:, mode_re]
+    im = h5f[dataset][:, mode_im]
 
   mode_txt = args["mode"].replace(",", "_")
   mode_txt = mode_txt.replace(" ", "")
   out = os.path.join(args["dout"], args["field"] + "_" + mode_txt + ".txt")
   #print(out)
-  stack = np.column_stack((t, f))
+  stack = np.column_stack((t, re, im))
   np.savetxt(out,
              stack,
-             header=f'# t {args["mode"]}',
-             fmt="%.16e %.16e")
-
+             header=f'# t re{args["mode"]} im{args["mode"]}',
+             fmt="%.16e %.16e %.16e")
 
 def main(args):
   """

From 11c04dca5286a9a2c7155e110d27da1d845efb53 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 10:43:19 -0500
Subject: [PATCH 16/72] renamed:    cce_plot.py -> cce_txt.py

---
 scripts/cce/{cce_plot.py => cce_txt.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scripts/cce/{cce_plot.py => cce_txt.py} (100%)

diff --git a/scripts/cce/cce_plot.py b/scripts/cce/cce_txt.py
similarity index 100%
rename from scripts/cce/cce_plot.py
rename to scripts/cce/cce_txt.py

From a97cc9a7d5bddbb52ac0b3d568c8a8e7881984ea Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 11:03:29 -0500
Subject: [PATCH 17/72] renamed:    cce_txt.py -> cce_extract_txt.py

---
 scripts/cce/{cce_txt.py => cce_extract_txt.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scripts/cce/{cce_txt.py => cce_extract_txt.py} (100%)

diff --git a/scripts/cce/cce_txt.py b/scripts/cce/cce_extract_txt.py
similarity index 100%
rename from scripts/cce/cce_txt.py
rename to scripts/cce/cce_extract_txt.py

From 436c93448bdc8ddce603fc2d59c689cc7c663802 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 11:05:14 -0500
Subject: [PATCH 18/72] fix: minors.

---
 scripts/cce/cce_extract_txt.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/scripts/cce/cce_extract_txt.py b/scripts/cce/cce_extract_txt.py
index 2afcd65cf..93c13fdc2 100755
--- a/scripts/cce/cce_extract_txt.py
+++ b/scripts/cce/cce_extract_txt.py
@@ -22,7 +22,8 @@ def parse_cli():
   """
     arg parser
     """
-  p = argparse.ArgumentParser(description="plotting cce spectre output")
+  p = argparse.ArgumentParser(
+      description="extract cce spectre output in a txt file.")
   p.add_argument("-f",
                  type=str,
                  required=True,
@@ -68,9 +69,9 @@ def read_save_to_txt(args):
     keys = h5f.keys()
     group = [k for k in keys if k.find("Spectre") != -1][0]
     group = "/" + group + "/"
-    dataset = group+args["field"]
-    mode_re = find_h5_1mode(h5f, dataset, "Real "+args["mode"], args)
-    mode_im = find_h5_1mode(h5f, dataset, "Imag "+args["mode"], args)
+    dataset = group + args["field"]
+    mode_re = find_h5_1mode(h5f, dataset, "Real " + args["mode"], args)
+    mode_im = find_h5_1mode(h5f, dataset, "Imag " + args["mode"], args)
     t = h5f[dataset][:, 0]
     re = h5f[dataset][:, mode_re]
     im = h5f[dataset][:, mode_im]
@@ -78,12 +79,15 @@ def read_save_to_txt(args):
   mode_txt = args["mode"].replace(",", "_")
   mode_txt = mode_txt.replace(" ", "")
   out = os.path.join(args["dout"], args["field"] + "_" + mode_txt + ".txt")
-  #print(out)
+  # print(out)
   stack = np.column_stack((t, re, im))
-  np.savetxt(out,
-             stack,
-             header=f'# t re{args["mode"]} im{args["mode"]}',
-             fmt="%.16e %.16e %.16e")
+  np.savetxt(
+      out,
+      stack,
+      header=f'# t re{args["mode"]} im{args["mode"]}',
+      fmt="%.16e %.16e %.16e",
+  )
+
 
 def main(args):
   """

From 2de7fa8b938443ad5091d7ad4be4feaca239f698 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 22 Jan 2025 11:09:35 -0500
Subject: [PATCH 19/72] add: memo.

---
 scripts/cce/readme.txt | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 scripts/cce/readme.txt

diff --git a/scripts/cce/readme.txt b/scripts/cce/readme.txt
new file mode 100644
index 000000000..b7f202df5
--- /dev/null
+++ b/scripts/cce/readme.txt
@@ -0,0 +1,7 @@
+# h5
+./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 -ftype h5 -d_out ./debug/q2/ -radius 100
+
+# extract txt
+./cce_extract_txt.py -f ./runs/CharacteristicExtractReduction.h5 -dout  ./debug/q2/ -field Strain
+./cce_extract_txt.py -f ./runs/CharacteristicExtractReduction.h5 -dout  ./debug/q2/ -field Psi4
+

From 49e9dcaa59ec608aa8fbf4882fbb9ad2a34eee86 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 27 Jan 2025 10:44:39 -0500
Subject: [PATCH 20/72] fix & feat: fix some bugs and improve bin read speed.

---
 scripts/cce/athk_to_spectre.py | 145 +++++++++++++++++----------------
 scripts/cce/pre_run.sh         |  43 ++++++----
 2 files changed, 103 insertions(+), 85 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index d27c0ba80..70efaa27b 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -3,6 +3,9 @@
 ## usage:
 ## $ ./me -h
 ##
+## BUG:
+## -[] see TODOs
+## -[] see NOTEs
 
 import sys
 import os
@@ -82,8 +85,14 @@ def parse_cli():
     """
   p = argparse.ArgumentParser(
       description="convert Athenak CCE dumps to Spectre CCE")
-  p.add_argument("-fpath", type=str, required=True, help="/path/to/cce/h5/dumps")
-  p.add_argument("-ftype", type=str, required=True, help="input file type:{h5,bin}")
+  p.add_argument("-fpath",
+                 type=str,
+                 required=True,
+                 help="/path/to/cce/dir/or/file/dumps")
+  p.add_argument("-ftype",
+                 type=str,
+                 required=True,
+                 help="input file type:{h5,bin}")
   p.add_argument("-d_out", type=str, required=True, help="/path/to/output/dir")
   p.add_argument("-debug", type=str, default="n", help="debug=[y,n]")
   p.add_argument(
@@ -141,28 +150,44 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         ret[g_re, i, :] = h5_re
         ret[g_im, i, :] = h5_im
   elif attrs["file_type"] == "bin":
-    lev_t = attrs["lev_t"]
-    max_n = attrs["max_n"]
-    max_lm = attrs["max_lm"]
-    shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
-    ret = np.empty(shape=shape, dtype=float)
     # Load the list of files
-    flist = sorted(glob.glob(fpath+'/cce_*.bin'))
+    ##TODO: this depends on file name
+    flist = sorted(glob.glob(fpath + "/cce_*.bin"))
     data = []
     dat_real = []
     dat_imag = []
     t = []
     for f in flist:
-      nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm = read_cce_file(f)
+      (
+          nr,
+          num_l_modes,
+          time,
+          rin,
+          rout,
+          data_real,
+          data_imag,
+          index_to_lm,
+      ) = read_cce_file(f)
       t.append(time)
-      dat_real.append(data_real[:,g_name_index[field_name],:])
-      dat_imag.append(data_imag[:,g_name_index[field_name],:])
+      dat_real.append(data_real[:, g_name_index[field_name], :])
+      dat_imag.append(data_imag[:, g_name_index[field_name], :])
     dat_real = np.array(dat_real)
     dat_imag = np.array(dat_imag)
     t = np.array(t)
-    f_real = interp1d(t, dat_real,axis=0)
-    f_imag = interp1d(t, dat_imag,axis=0)
+    f_real = interp1d(t, dat_real, axis=0)
+    f_imag = interp1d(t, dat_imag, axis=0)
 
+    attrs["lev_t"] = len(flist)
+    attrs["max_n"] = nr
+    attrs["max_l"] = num_l_modes
+    attrs["max_lm"] = (num_l_modes + 1)**2
+    attrs["r_in"] = np.array([rin])
+    attrs["r_out"] = np.array([rout])
+    attrs["time"] = np.linspace(t.min(), t.max(), t.shape[0])
+
+    shape = (len([g_re, g_im]), attrs["lev_t"], attrs["max_n"], attrs["max_lm"])
+    ret = np.empty(shape=shape, dtype=float)
+    ## NOTE: is it fine that we do interpolation?
     ret[g_re, :] = f_real(attrs["time"])
     ret[g_im, :] = f_imag(attrs["time"])
   else:
@@ -180,7 +205,7 @@ def lm_mode(l, m):
 
 
 def read_cce_file(filename):
-    """
+  """
     Reads binary data from a CCE output file generated by the provided C++ function.
 
     Parameters:
@@ -196,48 +221,49 @@ def read_cce_file(filename):
         data_imag (numpy.ndarray): Imaginary part of the data, shape (nr, 10, num_angular_modes).
         index_to_lm (dict): Mapping from data index to (l, m) values.
     """
-    with open(filename, 'rb') as f:
-        # Read number of radial points (nr)
-        nr_bytes = f.read(4)
-        nr = struct.unpack('<i', nr_bytes)[0]  # little-endian integer
+  with open(filename, "rb") as f:
+    # Read number of radial points (nr)
+    nr_bytes = f.read(4)
+    nr = struct.unpack("<i", nr_bytes)[0] # little-endian integer
 
-        # Read number of l modes (num_l_modes)
-        num_l_modes_bytes = f.read(4)
-        num_l_modes = struct.unpack('<i', num_l_modes_bytes)[0]
+    # Read number of l modes (num_l_modes)
+    num_l_modes_bytes = f.read(4)
+    num_l_modes = struct.unpack("<i", num_l_modes_bytes)[0]
 
-        # Read time
-        time_bytes = f.read(8)
-        time = struct.unpack('<d', time_bytes)[0]  # little-endian double
+    # Read time
+    time_bytes = f.read(8)
+    time = struct.unpack("<d", time_bytes)[0] # little-endian double
 
-        # Read inner and outer radial boundaries (rin and rout)
-        rin_bytes = f.read(8)
-        rin = struct.unpack('<d', rin_bytes)[0]
+    # Read inner and outer radial boundaries (rin and rout)
+    rin_bytes = f.read(8)
+    rin = struct.unpack("<d", rin_bytes)[0]
 
-        rout_bytes = f.read(8)
-        rout = struct.unpack('<d', rout_bytes)[0]
+    rout_bytes = f.read(8)
+    rout = struct.unpack("<d", rout_bytes)[0]
 
-        # Calculate the number of angular modes
-        num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
+    # Calculate the number of angular modes
+    num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
 
-        # Total number of data points
-        count = 10 * nr * num_angular_modes
+    # Total number of data points
+    count = 10 * nr * num_angular_modes
 
-        # Read data_real array
-        data_real = np.fromfile(f, dtype='<d', count=count)  # little-endian double
-        data_real = data_real.reshape((nr, 10, num_angular_modes))
+    # Read data_real array
+    data_real = np.fromfile(f, dtype="<d", count=count) # little-endian double
+    data_real = data_real.reshape((nr, 10, num_angular_modes))
 
-        # Read data_imag array
-        data_imag = np.fromfile(f, dtype='<d', count=count)
-        data_imag = data_imag.reshape((nr, 10, num_angular_modes))
+    # Read data_imag array
+    data_imag = np.fromfile(f, dtype="<d", count=count)
+    data_imag = data_imag.reshape((nr, 10, num_angular_modes))
 
-        # Create a mapping from index to (l, m)
-        index_to_lm = {}
-        for l in range(1, num_l_modes + 1):
-            for m in range(-l, l + 1):
-                index = l * l + l + m
-                index_to_lm[index] = (l, m)
+    # Create a mapping from index to (l, m)
+    index_to_lm = {}
+    for l in range(1, num_l_modes + 1):
+      for m in range(-l, l + 1):
+        index = l * l + l + m
+        index_to_lm[index] = (l, m)
+
+  return nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm
 
-    return nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm
 
 def get_attribute(fpath: str,
                   field_name: str = "gxx",
@@ -266,23 +292,6 @@ def get_attribute(fpath: str,
       attrs["time"] = np.array(time)
   elif type == "bin":
     attrs["file_type"] = "bin"
-    # Load the list of files
-    flist = sorted(glob.glob(fpath+'/cce_*.bin'))
-    t = []
-    data = []
-    for f in flist:
-      nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm = read_cce_file(f)
-      t.append(time)
-      #data.append(data_real + 1j * data_imag)
-    t = np.array(t)
-    #data = np.array(data)
-    attrs["lev_t"] = len(flist)
-    attrs["max_n"]  = nr
-    attrs["max_l"] = num_l_modes
-    attrs["max_lm"] = (num_l_modes + 1) ** 2
-    attrs["r_in"] = np.array([rin])
-    attrs["r_out"] = np.array([rout])
-    attrs["time"] = np.linspace(t.min(),t.max(),t.shape[0])
   else:
     raise ValueError("no such option")
 
@@ -307,7 +316,7 @@ def time_derivative_findiff(field: np.array, field_name: str, attrs: dict,
     for lm in range(len_lm):
       dfield[g_re, :, n, lm] = np.gradient(field[g_re, :, n, lm], 2) / dt
       dfield[g_im, :, n, lm] = np.gradient(field[g_im, :, n, lm], 2) / dt
-  
+
   return dfield
 
 
@@ -612,17 +621,17 @@ def h5_write_data(h5file,
   h5file[f"{data_name}"][:, flat] = attrs["time"]
   flat += 1
   for l in range(0, attrs["max_l"]):
-    for m in range(l, -l - 1,-1):
+    for m in range(l, -l - 1, -1):
       if args["debug"] == "y":
         assert all(data[g_re, :, lm_mode(l, m)] != np.nan)
         assert all(data[g_im, :, lm_mode(l, m)] != np.nan)
-      
+
       data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
       data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")
       h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(l, m)]
       h5file[f"{data_name}"][:, flat + 1] = data[g_im, :, lm_mode(l, m)]
       flat += 2
-  Legend = [s.encode('ascii', 'ignore') for s in data_attrs]
+  Legend = [s.encode("ascii", "ignore") for s in data_attrs]
   h5file[f"{data_name}"].attrs["Legend"] = Legend
 
 
@@ -669,7 +678,7 @@ def main(args):
   if not os.path.exists(g_args["d_out"]):
     os.makedirs(g_args["d_out"])
   # find attribute for an arbitrary field
-  g_attrs = get_attribute(args["fpath"],type=args["ftype"])
+  g_attrs = get_attribute(args["fpath"], type=args["ftype"])
 
   # for each field
   # I'm afraid this method takes too much memory
diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index 33a6c0cd8..72e4d8b4f 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -1,26 +1,34 @@
 #!/usr/bin/env bash
 
-if false
+# bin
+if true
 then
-./athk_to_spectre.py -ftype bin -d_out ./debug -debug y \
+# create
+dout='./debug/dev'
+file='./dat/bin/cce_bbh/'
+fdebug='./debug/dev/CceR0030.00.h5'
+
+./athk_to_spectre.py -ftype bin -d_out "$dout" -debug y \
 -t_deriv fin_diff -r_deriv ChebU -interpolation ChebU \
 -radius 30 \
--fpath ./dat/bin/cce_bbh/
-fi
+-fpath $file
 
-file="./debug/CceR0030.00.h5"
-dout='./debug/bin'
+# plot fig:
+fname="gxx"
+mode="Re(2,2)"
+./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+	-fpath $fdebug  -field_name $fname -field_mode $mode
 
-if false
+# h5 bbh, q=2
+elif false
 then
 ./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 \
 -ftype h5 -d_out ./debug/q2/ -radius 100
-fi
 
 file="./debug/q2/CceR0100.00.h5"
 dout='./debug/q2'
 
-# plot fig
+# plot fig:
 fname="gxx"
 mode="Re(2,2)"
 ./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
@@ -31,22 +39,23 @@ mode="Re(2,2)"
 #	-fpath $file  -field_name $fname -field_mode $mode
 
 
-fname="alp"
+#fname="alp"
 #mode="Re(0,0)"
 #./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 #	-fpath $file  -field_name $fname -field_mode $mode
 
-mode="Re(2,2)"
-./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
-	-fpath $file  -field_name $fname -field_mode $mode
+#mode="Re(2,2)"
+#./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+#	-fpath $file  -field_name $fname -field_mode $mode
 
 
-fname="betax"
+#fname="betax"
 #mode="Re(0,0)"
 #./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 #	-fpath $file  -field_name $fname -field_mode $mode
 
-mode="Re(2,2)"
-./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
-	-fpath $file  -field_name $fname -field_mode $mode
+#mode="Re(2,2)"
+#./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+#	-fpath $file  -field_name $fname -field_mode $mode
+fi
 

From a919c1cb03f401e3750951321bd4a071e6f02ded Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 27 Jan 2025 11:48:46 -0500
Subject: [PATCH 21/72] minor.

---
 scripts/cce/pre_run.sh | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index 72e4d8b4f..5daa3406c 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # bin
-if true
+if false
 then
 # create
 dout='./debug/dev'
@@ -57,5 +57,23 @@ mode="Re(2,2)"
 #mode="Re(2,2)"
 #./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 #	-fpath $file  -field_name $fname -field_mode $mode
+
+# h5 bbh, q=2, fourier time derivatives
+elif true
+then
+file="./debug/q2_t_fourier/CceR0100.00.h5"
+dout='./debug/q2_t_fourier'
+mkdir -p ${dout}
+
+./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 \
+-ftype h5 -d_out "$dout" -radius 100 -t_deriv "Fourier"
+
+
+# plot fig:
+fname="gxx"
+mode="Re(2,2)"
+./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+	-fpath $file  -field_name $fname -field_mode $mode
+
 fi
 

From a560a82076daf833a98410f2eb779facad63e051 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 27 Jan 2025 11:55:12 -0500
Subject: [PATCH 22/72] fix: fourier time derives.

---
 scripts/cce/athk_to_spectre.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 70efaa27b..30ecb6345 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -185,7 +185,12 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
     attrs["r_out"] = np.array([rout])
     attrs["time"] = np.linspace(t.min(), t.max(), t.shape[0])
 
-    shape = (len([g_re, g_im]), attrs["lev_t"], attrs["max_n"], attrs["max_lm"])
+    shape = (
+        len([g_re, g_im]),
+        attrs["lev_t"],
+        attrs["max_n"],
+        attrs["max_lm"],
+    )
     ret = np.empty(shape=shape, dtype=float)
     ## NOTE: is it fine that we do interpolation?
     ret[g_re, :] = f_real(attrs["time"])
@@ -279,6 +284,9 @@ def get_attribute(fpath: str,
     with h5py.File(fpath, "r") as h5f:
       # find attribute about num. of time level, and n,l,m in C_nlm
       attrs["lev_t"] = len(h5f.keys()) - 1
+      if attrs["lev_t"] % 2 == 0: # for fourier transformation
+        attrs["lev_t"] -= 1
+
       attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
       attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1
       attrs["r_in"] = h5f["metadata"].attrs["Rin"]
@@ -349,10 +357,11 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
 
       dfft_coeff = np.empty_like(fft_coeff)
       dfft_coeff[0] = 0
+
       dfft_coeff[1:half] = (-np.imag(fft_coeff[1:half]) +
                             1j * np.real(fft_coeff[1:half])) * omega[1:]
       dfft_coeff[half:] = (np.imag(fft_coeff[half:]) -
-                           1j * np.real(fft_coeff[half:])) * omega[::-1][1:]
+                           1j * np.real(fft_coeff[half:])) * (omega[::-1][1:])
 
       # not optimized version
       """

From ab602d7b0cbcf28c700f77029a6d0654080b98a3 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 27 Jan 2025 12:03:38 -0500
Subject: [PATCH 23/72] add note.

---
 scripts/cce/athk_to_spectre.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 30ecb6345..783c2ef1d 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -334,7 +334,8 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
     return the time derivative of the given field using Fourier method
     field(t,rel/img,n,lm)
     """
-
+  
+  ## TODO: Fourier time derives not tested!
   print(f"Fourier time derivative: {field_name}", flush=True)
   _, len_t, len_n, len_lm = field.shape
   dt = attrs["time"][2] - attrs["time"][1]

From 1ff1ea02defb116fc1ddb5fe477513a7100b23d5 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 29 Jan 2025 10:12:28 -0500
Subject: [PATCH 24/72] feat: add chebu expansion.

---
 scripts/cce/athk_to_spectre.py | 113 ++++++++++++++++++++++++++++++---
 1 file changed, 105 insertions(+), 8 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 783c2ef1d..25fca9a5d 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -89,10 +89,12 @@ def parse_cli():
                  type=str,
                  required=True,
                  help="/path/to/cce/dir/or/file/dumps")
-  p.add_argument("-ftype",
-                 type=str,
-                 required=True,
-                 help="input file type:{h5,bin}")
+  p.add_argument(
+      "-ftype",
+      type=str,
+      required=True,
+      help="input file type:{h5:for pitnull,bin: for athenak}",
+  )
   p.add_argument("-d_out", type=str, required=True, help="/path/to/output/dir")
   p.add_argument("-debug", type=str, default="n", help="debug=[y,n]")
   p.add_argument(
@@ -158,6 +160,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
     dat_imag = []
     t = []
     for f in flist:
+      ##TODO: it reads all field and only use one of them. it's inefficient
       (
           nr,
           num_l_modes,
@@ -311,7 +314,7 @@ def time_derivative_findiff(field: np.array, field_name: str, attrs: dict,
                             args) -> np.array:
   """
     return the time derivative of the given field using finite diff. 2nd order
-    field(t,rel/img,n,lm)
+    field(rel/img,t,n,lm)
     """
 
   print(f"finite difference time derivative: {field_name}", flush=True)
@@ -332,9 +335,9 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
                             args) -> np.array:
   """
     return the time derivative of the given field using Fourier method
-    field(t,rel/img,n,lm)
+    field(rel/img,t, n,lm)
     """
-  
+
   ## TODO: Fourier time derives not tested!
   print(f"Fourier time derivative: {field_name}", flush=True)
   _, len_t, len_n, len_lm = field.shape
@@ -425,7 +428,7 @@ def radial_derivative_at_r_chebu(field: np.array,
     collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
     x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
 
-    field(t,rel/img,n,lm)
+    field(rel/img,t,n,lm)
     """
 
   print(f"ChebyU radial derivative: {field_name}", flush=True)
@@ -489,6 +492,97 @@ def time_derivative(field: np.array, field_name: str, attrs: dict, args):
     raise ValueError("no such option")
 
 
+class ChebUExpansion:
+  """
+    f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
+    collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
+    """
+
+  def __init__(self, attrs: dict, args: dict):
+    self.N = N = attrs["max_n"] # num. of coeffs or num of collocation pnts
+
+    # roots:
+    x_i = np.empty(shape=(N), dtype=float)
+    for i in range(N):
+      x_i[i] = cos(pi * (i + 1) / (N + 1))
+    self.x_i = x_i
+
+    # weights:
+    w_i = np.empty(shape=(N), dtype=float)
+    for i in range(N):
+      w_i[i] = sin(pi * (i + 1) / (N + 1))
+      w_i = np.square(w_i)
+      w_i *= pi / (N + 1)
+    self.w_i = w_i
+
+    # ChebU_j(x_i):
+    Uj_xi = np.empty(shape=(N, N), dtype=float)
+    for j in range(N):
+      for i in range(N):
+        x_i = cos(pi * (i + 1) / (N + 1))
+        Uj_xi[j, i] = special.chebyu(j)(x_i)
+    self.Uj_xi = Uj_xi
+
+  def Coefficients(self, field: np.array) -> np.array:
+    """
+        use Gauss Quadrature to expand the given field in ChebU
+        f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
+        collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
+        x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+
+        """
+
+    coeffs = np.zeros(shape=(N), dtype=float)
+
+    w_i = self.w_i
+    x_i = self.w_i
+    Uji = self.Uj_xi
+
+    for j in range(self.N):
+      tmp = 0.0
+      for i in range(self.N):
+        tmp += w_i * field[i] * Uji[j, i]
+      coeffs[j] = tmp
+
+    return coeffs
+
+
+def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict, args):
+  """
+    expands field in the radial direction using chebyshev of second kind
+    and returns the expansion coefficients. we have
+
+    f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
+    collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
+    x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+
+    field(rel/img,t,n,lm)
+
+    """
+
+  print(f"ChebU expansion {field_name}", flush=True)
+  expand = ChebUExpansion(attrs, args)
+  _, len_t, _, len_lm = field.shape
+
+  for t in range(len_t):
+    for lm in range(len_lm):
+      field[g_re, t, :, lm] = expand.Coefficients(field[g_re, t, :, lm])
+      field[g_im, t, :, lm] = expand.Coefficients(field[g_im, t, :, lm])
+
+
+def radial_expansion(field: np.array, field_name: str, attrs: dict, args):
+  """
+    expands field in the radial direction(if needed) and returns
+    """
+
+  if args["ftype"] == "bin":
+    return radial_expansion_chebu(field, field_name, attrs, args)
+  elif args["ftype"] == "h5":
+    return field
+  else:
+    raise ValueError("no such option")
+
+
 def radial_derivative_at_r(field: np.array, field_name: str, attrs: dict, args):
   """
     return the radial derivative of the given field at R=r
@@ -557,6 +651,9 @@ def process_field(field_name: str) -> dict:
   field = load(args["fpath"], field_name, attrs)
   # db[f"{field_name}"] = field
 
+  # radial expansion
+  field = radial_expansion(field, field_name, attrs, args)
+
   # time derivative
   dfield_dt = time_derivative(field, field_name, attrs, args)
 

From c30296344082db86e968cf829a12bd95980f3651 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 29 Jan 2025 10:12:45 -0500
Subject: [PATCH 25/72] minor.

---
 scripts/cce/pre_run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index 5daa3406c..1941f639b 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # bin
-if false
+if true
 then
 # create
 dout='./debug/dev'
@@ -59,7 +59,7 @@ mode="Re(2,2)"
 #	-fpath $file  -field_name $fname -field_mode $mode
 
 # h5 bbh, q=2, fourier time derivatives
-elif true
+elif false
 then
 file="./debug/q2_t_fourier/CceR0100.00.h5"
 dout='./debug/q2_t_fourier'

From 997c08ce624fdb73dbe59577823ef52b67585cef Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 29 Jan 2025 10:18:17 -0500
Subject: [PATCH 26/72] fix: bugs.

---
 scripts/cce/athk_to_spectre.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 25fca9a5d..6798690ca 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -500,17 +500,18 @@ class ChebUExpansion:
 
   def __init__(self, attrs: dict, args: dict):
     self.N = N = attrs["max_n"] # num. of coeffs or num of collocation pnts
+    pi = math.pi
 
     # roots:
     x_i = np.empty(shape=(N), dtype=float)
     for i in range(N):
-      x_i[i] = cos(pi * (i + 1) / (N + 1))
+      x_i[i] = math.cos(pi * (i + 1) / (N + 1))
     self.x_i = x_i
 
     # weights:
     w_i = np.empty(shape=(N), dtype=float)
     for i in range(N):
-      w_i[i] = sin(pi * (i + 1) / (N + 1))
+      w_i[i] = math.sin(pi * (i + 1) / (N + 1))
       w_i = np.square(w_i)
       w_i *= pi / (N + 1)
     self.w_i = w_i
@@ -519,7 +520,7 @@ def __init__(self, attrs: dict, args: dict):
     Uj_xi = np.empty(shape=(N, N), dtype=float)
     for j in range(N):
       for i in range(N):
-        x_i = cos(pi * (i + 1) / (N + 1))
+        x_i = math.cos(pi * (i + 1) / (N + 1))
         Uj_xi[j, i] = special.chebyu(j)(x_i)
     self.Uj_xi = Uj_xi
 
@@ -532,7 +533,7 @@ def Coefficients(self, field: np.array) -> np.array:
 
         """
 
-    coeffs = np.zeros(shape=(N), dtype=float)
+    coeffs = np.zeros(shape=(self.N), dtype=float)
 
     w_i = self.w_i
     x_i = self.w_i
@@ -541,13 +542,14 @@ def Coefficients(self, field: np.array) -> np.array:
     for j in range(self.N):
       tmp = 0.0
       for i in range(self.N):
-        tmp += w_i * field[i] * Uji[j, i]
+        tmp += w_i[i] * field[i] * Uji[j, i]
       coeffs[j] = tmp
 
-    return coeffs
+    return coeffs * (2.0 / math.pi)
 
 
-def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict, args):
+def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,
+                           args) -> np.array:
   """
     expands field in the radial direction using chebyshev of second kind
     and returns the expansion coefficients. we have
@@ -569,6 +571,8 @@ def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict, args):
       field[g_re, t, :, lm] = expand.Coefficients(field[g_re, t, :, lm])
       field[g_im, t, :, lm] = expand.Coefficients(field[g_im, t, :, lm])
 
+  return field
+
 
 def radial_expansion(field: np.array, field_name: str, attrs: dict, args):
   """

From 4f88e0563462bef5e080b5ec717f8731fdcc9701 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 29 Jan 2025 11:03:07 -0500
Subject: [PATCH 27/72] chores.

---
 scripts/cce/athk_to_spectre.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 6798690ca..16ba56386 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -431,11 +431,17 @@ def radial_derivative_at_r_chebu(field: np.array,
     field(rel/img,t,n,lm)
     """
 
-  print(f"ChebyU radial derivative: {field_name}", flush=True)
-  _, len_t, len_n, len_lm = field.shape
-
   r_1 = attrs["r_in"][0]
   r_2 = attrs["r_out"][0]
+  r = args["radius"]
+
+  print(
+      f"ChebyU radial derivative: {field_name}, at r={r} in [{r_1},{r_2}]",
+      flush=True,
+  )
+
+  _, len_t, len_n, len_lm = field.shape
+
   assert r_1 != r_2
   dx_dr = 2 / (r_2 - r_1)
 

From 60fd8d3e5f3dc4167259c79e0daddd5cd12cb3a9 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 08:46:04 -0500
Subject: [PATCH 28/72] fix: bugs add: tests.

---
 scripts/cce/athk_to_spectre.py | 45 +++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 16ba56386..ea62b563c 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -526,10 +526,12 @@ def __init__(self, attrs: dict, args: dict):
     Uj_xi = np.empty(shape=(N, N), dtype=float)
     for j in range(N):
       for i in range(N):
-        x_i = math.cos(pi * (i + 1) / (N + 1))
-        Uj_xi[j, i] = special.chebyu(j)(x_i)
+        Uj_xi[j, i] = special.chebyu(j)(x_i[i])
     self.Uj_xi = Uj_xi
 
+    if True:
+      self.__Debug()
+
   def Coefficients(self, field: np.array) -> np.array:
     """
         use Gauss Quadrature to expand the given field in ChebU
@@ -539,19 +541,40 @@ def Coefficients(self, field: np.array) -> np.array:
 
         """
 
-    coeffs = np.zeros(shape=(self.N), dtype=float)
-
     w_i = self.w_i
-    x_i = self.w_i
+    x_i = self.x_i
     Uji = self.Uj_xi
 
-    for j in range(self.N):
-      tmp = 0.0
-      for i in range(self.N):
-        tmp += w_i[i] * field[i] * Uji[j, i]
-      coeffs[j] = tmp
+    coeffs = Uji @ (w_i * field)
+    coeffs *= 2.0 / math.pi
+
+    return coeffs
+
+  def __Debug(self):
+    """
+        test if the expansion works for different known functions
+        """
+
+    N = self.N
+    x_i = self.x_i
+
+    # populate funcs:
+    fs = []
+    for n in range(N):
+      f = special.chebyu(n)(x_i)
+      fs.append((f, n))
+
+    # find coeffs
+    cs = []
+    for n in range(N):
+      c = self.Coefficients(fs[n][0])
+      cs.append((c, fs[n][1]))
+
+    for n in range(N):
+      order = cs[n][1]
+      print(f"chebyu_{order}, coeffs = {cs[n][0]}\n", flush=True)
 
-    return coeffs * (2.0 / math.pi)
+    exit(1)
 
 
 def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,

From 1b09aac405c9af2124add7dd47a2791ecf5ded49 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 10:16:04 -0500
Subject: [PATCH 29/72] fix: nasty bugs in cheb. exp.

---
 scripts/cce/athk_to_spectre.py | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index ea62b563c..f39534484 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -508,28 +508,19 @@ def __init__(self, attrs: dict, args: dict):
     self.N = N = attrs["max_n"] # num. of coeffs or num of collocation pnts
     pi = math.pi
 
-    # roots:
-    x_i = np.empty(shape=(N), dtype=float)
-    for i in range(N):
-      x_i[i] = math.cos(pi * (i + 1) / (N + 1))
-    self.x_i = x_i
-
-    # weights:
-    w_i = np.empty(shape=(N), dtype=float)
-    for i in range(N):
-      w_i[i] = math.sin(pi * (i + 1) / (N + 1))
-      w_i = np.square(w_i)
-      w_i *= pi / (N + 1)
-    self.w_i = w_i
+    # U_N roots:
+    a_i = np.arange(0, N, dtype=int) + 1
+    self.x_i = x_i = np.cos(math.pi * a_i / (N + 1))
+    # quadrature weights
+    self.w_i = (1 - np.square(x_i)) * pi / (N + 1)
 
     # ChebU_j(x_i):
     Uj_xi = np.empty(shape=(N, N), dtype=float)
     for j in range(N):
-      for i in range(N):
-        Uj_xi[j, i] = special.chebyu(j)(x_i[i])
+      Uj_xi[j, :] = special.chebyu(j)(x_i)
     self.Uj_xi = Uj_xi
 
-    if True:
+    if args["debug"] == "y":
       self.__Debug()
 
   def Coefficients(self, field: np.array) -> np.array:
@@ -558,7 +549,7 @@ def __Debug(self):
     N = self.N
     x_i = self.x_i
 
-    # populate funcs:
+    # populate funcs using bases themselves
     fs = []
     for n in range(N):
       f = special.chebyu(n)(x_i)
@@ -570,11 +561,10 @@ def __Debug(self):
       c = self.Coefficients(fs[n][0])
       cs.append((c, fs[n][1]))
 
+    # expect to see only the n-th entry is 1 for the chebyu(n) of order n
     for n in range(N):
       order = cs[n][1]
-      print(f"chebyu_{order}, coeffs = {cs[n][0]}\n", flush=True)
-
-    exit(1)
+      print(f"chebyu{order}, coeffs = {cs[n][0]}\n", flush=True)
 
 
 def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,

From 72a0e3d3adeac9fe014a5d0b155ccc0279ad6ba4 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 10:18:21 -0500
Subject: [PATCH 30/72] minor.

---
 scripts/cce/athk_to_spectre.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index f39534484..59ae79399 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -546,7 +546,7 @@ def __Debug(self):
         test if the expansion works for different known functions
         """
 
-    N = self.N
+    N = self.N+1 # +1 to see the roots of U_N
     x_i = self.x_i
 
     # populate funcs using bases themselves

From ea74e4af64fc01e276321f0b1c7840c0af738ff6 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 10:37:24 -0500
Subject: [PATCH 31/72] fix: collocation points for interpolation.

---
 src/utils/chebyshev.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils/chebyshev.hpp b/src/utils/chebyshev.hpp
index 4b7426879..ecc4c1f27 100644
--- a/src/utils/chebyshev.hpp
+++ b/src/utils/chebyshev.hpp
@@ -15,7 +15,7 @@
 KOKKOS_INLINE_FUNCTION
 Real ChebyshevSecondKindCollocationPoints(Real xmin, Real xmax, int N, int k) {
   // over the interval of -1 to 1
-  Real x_k = std::cos( M_PI * (k + 1) / (N + 2));
+  Real x_k = std::cos( M_PI * (k + 1) / (N + 1));
   return  0.5 * ( (xmin - xmax) * x_k + (xmin + xmax));
 }
 

From 30385b688b8cc9dbcea02287ac99e796953f2645 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 11:59:56 -0500
Subject: [PATCH 32/72] fix: sign bug due to x=[-1,1] convention.

---
 scripts/cce/athk_to_spectre.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 59ae79399..9fab2f395 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -75,6 +75,9 @@
 ## various attrs
 g_attrs = None
 
+## sign convention
+g_sign = None
+
 ## debug
 g_debug_max_l = 2
 
@@ -443,7 +446,7 @@ def radial_derivative_at_r_chebu(field: np.array,
   _, len_t, len_n, len_lm = field.shape
 
   assert r_1 != r_2
-  dx_dr = 2 / (r_2 - r_1)
+  dx_dr = g_sign * 2 / (r_2 - r_1)
 
   if args["debug"] == "y":
     # populate collocation points, roots of U_i
@@ -478,7 +481,7 @@ def radial_derivative_at_r_chebu(field: np.array,
 
   dfield = np.zeros(shape=(len([g_re, g_im]), len_t, len_lm))
   r = args["radius"]
-  x = (2 * r - r_1 - r_2) / (r_2 - r_1)
+  x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
   for k in range(len_n):
     dfield[:, :, :] += field[:, :, k, :] * dUk_dx(k, x)
 
@@ -536,7 +539,7 @@ def Coefficients(self, field: np.array) -> np.array:
     x_i = self.x_i
     Uji = self.Uj_xi
 
-    coeffs = Uji @ (w_i * field)
+    coeffs = Uji @ (w_i * field[::-g_sign])
     coeffs *= 2.0 / math.pi
 
     return coeffs
@@ -546,7 +549,7 @@ def __Debug(self):
         test if the expansion works for different known functions
         """
 
-    N = self.N+1 # +1 to see the roots of U_N
+    N = self.N + 1 # +1 to see the roots of U_N
     x_i = self.x_i
 
     # populate funcs using bases themselves
@@ -631,7 +634,7 @@ def __init__(self, attrs: dict, args: dict):
     r_1 = attrs["r_in"][0]
     r_2 = attrs["r_out"][0]
     self.r = r = args["radius"]
-    self.x = (2 * r - r_1 - r_2) / (r_2 - r_1)
+    self.x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
 
     if args["interpolation"] == "ChebU":
       self.Uk = np.empty(shape=self.len_n)
@@ -801,8 +804,23 @@ def main(args):
 
   global g_attrs
   global g_args
+  global g_sign
 
   g_args = args
+  """
+    note:
+    in Pitnull:
+    r = 0.5[(r_2 - r_1)*x + (r_2 + r_1)], r_2 > r_1 & x = [-1,1]
+    => x = -1 -> r = r_1
+    => x = +1 -> r = r_2
+
+    in AthenaK:
+    r = 0.5[(r_1 - r_2)*x + (r_2 + r_1)], r_2 > r_1 & x = [1,-1]
+    => x = -1 -> r = r_2
+    => x = +1 -> r = r_1
+
+  """
+  g_sign = int(-1) if args["ftype"] == "bin" else int(1)
 
   # check if output dir exist, if not, mkdir
   if not os.path.exists(g_args["d_out"]):

From 3afd56e90ac1256fd5cccf1607d04562f3fe59f2 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 12:09:36 -0500
Subject: [PATCH 33/72] minors.

---
 scripts/cce/athk_to_spectre.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 9fab2f395..3d3baa84c 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -429,7 +429,7 @@ def radial_derivative_at_r_chebu(field: np.array,
 
     f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
     collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
-    x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+    x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
 
     field(rel/img,t,n,lm)
     """
@@ -524,14 +524,14 @@ def __init__(self, attrs: dict, args: dict):
     self.Uj_xi = Uj_xi
 
     if args["debug"] == "y":
-      self.__Debug()
+      self.__debug()
 
-  def Coefficients(self, field: np.array) -> np.array:
+  def coefficients(self, field: np.array) -> np.array:
     """
         use Gauss Quadrature to expand the given field in ChebU
         f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
         collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
-        x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+        x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
 
         """
 
@@ -544,7 +544,7 @@ def Coefficients(self, field: np.array) -> np.array:
 
     return coeffs
 
-  def __Debug(self):
+  def __debug(self):
     """
         test if the expansion works for different known functions
         """
@@ -561,7 +561,7 @@ def __Debug(self):
     # find coeffs
     cs = []
     for n in range(N):
-      c = self.Coefficients(fs[n][0])
+      c = self.coefficients(fs[n][0])
       cs.append((c, fs[n][1]))
 
     # expect to see only the n-th entry is 1 for the chebyu(n) of order n
@@ -578,7 +578,7 @@ def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,
 
     f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
     collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
-    x = (2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+    x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
 
     field(rel/img,t,n,lm)
 
@@ -590,8 +590,8 @@ def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,
 
   for t in range(len_t):
     for lm in range(len_lm):
-      field[g_re, t, :, lm] = expand.Coefficients(field[g_re, t, :, lm])
-      field[g_im, t, :, lm] = expand.Coefficients(field[g_im, t, :, lm])
+      field[g_re, t, :, lm] = expand.coefficients(field[g_re, t, :, lm])
+      field[g_im, t, :, lm] = expand.coefficients(field[g_im, t, :, lm])
 
   return field
 

From 70343794e2b7946d5379128743aacb7c95e6e93f Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 30 Jan 2025 12:56:40 -0500
Subject: [PATCH 34/72] minor.

---
 scripts/cce/athk_to_spectre.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 3d3baa84c..fe409114a 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -636,10 +636,13 @@ def __init__(self, attrs: dict, args: dict):
     self.r = r = args["radius"]
     self.x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
 
+    assert -1 < self.x < 1
+
     if args["interpolation"] == "ChebU":
       self.Uk = np.empty(shape=self.len_n)
       for k in range(self.len_n):
         self.Uk[k] = special.chebyu(k)(self.x)
+      # print("x,Uk",self.x,self.Uk)
       self.interp = self.interpolate_at_r_chebu
     else:
       raise ValueError("no such option")

From 63c47d2ba0ce20a53237bde45bc97dbb92b954f2 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Sat, 1 Feb 2025 16:00:53 -0500
Subject: [PATCH 35/72] update legendre roots calculation and include unit test
 for spherical harmonics and integration

---
 .../unit_tests/gauss_legendre_test.athinput   |  48 +++++
 src/pgen/unit_tests/gauss_legendre_test.cpp   | 118 +++++++++++
 src/utils/legendre_roots.hpp                  | 196 ++++++++++--------
 3 files changed, 280 insertions(+), 82 deletions(-)
 create mode 100644 inputs/unit_tests/gauss_legendre_test.athinput
 create mode 100644 src/pgen/unit_tests/gauss_legendre_test.cpp

diff --git a/inputs/unit_tests/gauss_legendre_test.athinput b/inputs/unit_tests/gauss_legendre_test.athinput
new file mode 100644
index 000000000..9f279a001
--- /dev/null
+++ b/inputs/unit_tests/gauss_legendre_test.athinput
@@ -0,0 +1,48 @@
+# Athena++ (Kokkos version) input file for spherical blast problem
+
+<comment>
+problem   = gauss legendre test
+
+<job>
+basename  = gauss_legendre_test      # problem ID: basename of output filenames
+
+<mesh>
+nghost    = 2          # Number of ghost cells
+nx1       = 20        # Number of zones in X1-direction
+x1min     = -20       # minimum value of X1
+x1max     = 20        # maximum value of X1
+ix1_bc    = outflow   # inner-X1 boundary flag
+ox1_bc    = outflow   # outer-X1 boundary flag
+
+nx2       = 20        # Number of zones in X2-direction
+x2min     = -20      # minimum value of X2
+x2max     = 20       # maximum value of X2
+ix2_bc    = outflow   # inner-X2 boundary flag
+ox2_bc    = outflow   # outer-X2 boundary flag
+
+nx3       = 20        # Number of zones in X3-direction
+x3min     = -20       # minimum value of X3
+x3max     = 20        # maximum value of X3
+ix3_bc    = outflow   # inner-X3 boundary flag
+ox3_bc    = outflow   # outer-X3 boundary flag
+
+<meshblock>
+nx1       = 20         # Number of cells in each MeshBlock, X1-dir
+nx2       = 20         # Number of cells in each MeshBlock, X2-dir
+nx3       = 20           # Number of cells in each MeshBlock, X3-dir
+
+<time>
+<time>
+evolution  = dynamic    # dynamic/kinematic/static
+integrator = rk3        # time integration algorithm
+cfl_number = 0.1        # The Courant, Friedrichs, & Lewy (CFL) Number
+nlim       = 0         # cycle limit
+tlim       = 0        # time limit
+ndiag      = 1          # cycles between diagostic output
+
+<z4c>
+
+<problem>
+pgen_name = gauss_legendre_test
+ntheta = 25
+
diff --git a/src/pgen/unit_tests/gauss_legendre_test.cpp b/src/pgen/unit_tests/gauss_legendre_test.cpp
new file mode 100644
index 000000000..27c8008e8
--- /dev/null
+++ b/src/pgen/unit_tests/gauss_legendre_test.cpp
@@ -0,0 +1,118 @@
+//========================================================================================
+// AthenaXXX astrophysical plasma code
+// Copyright(C) 2020 James M. Stone <jmstone@ias.edu> and the Athena code team
+// Licensed under the 3-clause BSD License (the "LICENSE")
+//========================================================================================
+//! \file z4c_one_puncture.cpp
+//  \brief Problem generator for a single puncture placed at the origin of the domain
+
+#include <algorithm>
+#include <cmath>
+#include <sstream>
+#include <iomanip>
+#include <iostream>   // endl
+#include <limits>     // numeric_limits::max()
+#include <memory>
+#include <string>     // c_str(), string
+#include <vector>
+#include <cctype>
+#include <random>
+
+#include "athena.hpp"
+#include "parameter_input.hpp"
+#include "globals.hpp"
+#include "mesh/mesh.hpp"
+#include "coordinates/cell_locations.hpp"
+#include "geodesic-grid/gauss_legendre.hpp"
+#include "utils/spherical_harm.hpp"
+
+using u32    = uint_least32_t;
+using engine = std::mt19937;
+
+// void ADMOnePuncture(MeshBlockPack *pmbp, ParameterInput *pin);
+
+//----------------------------------------------------------------------------------------
+//! \fn ProblemGenerator::UserProblem_()
+//! \brief Problem Generator for single puncture
+void ProblemGenerator::UserProblem(ParameterInput *pin, const bool restart) {
+  MeshBlockPack *pmbp = pmy_mesh_->pmb_pack;
+  auto &indcs = pmy_mesh_->mb_indcs;
+
+  // ADMOnePuncture(pmbp, pin);
+  int ntheta = pin->GetOrAddInteger("problem", "ntheta", 16);
+
+  GaussLegendreGrid grid(pmbp, ntheta, 1);
+
+  // test that the cross integral of spherical harmonics are delta functions.
+  // First initialize 10 random pairs of l and m, with 0 <= l <=ntheta.
+
+  std::random_device os_seed;
+  const u32 seed = os_seed();
+
+  engine generator( seed );
+  std::uniform_int_distribution< u32 > distribute_l( 1, ntheta-1);
+
+  std::vector<int> ls;
+  std::vector<int> ms;
+
+  for (int repetition = 0; repetition < 10; ++repetition) {
+    int l = distribute_l(generator);
+    std::uniform_int_distribution< u32 > distribute_m( -l, l);
+    int m = distribute_m(generator);
+    ls.push_back(l);
+    ms.push_back(m);
+  }
+
+  double ylmR1,ylmI1,ylmR2,ylmI2;
+  double int_r, int_i;
+  bool failed = false;
+  double max_err = 0;
+
+  // outer loop over pairs of spherical harmonics
+  for (int n1 = 0; n1 < 10; ++n1)
+  for (int n2 = n1; n2 < 10; ++n2) {
+    // reset doubles to store integration value
+    int_r = 0;
+    int_i = 0;
+
+    // iterate over the angles
+    for (int ip = 0; ip < grid.nangles; ++ip) {
+      Real theta = grid.polar_pos.h_view(ip,0);
+      Real phi = grid.polar_pos.h_view(ip,1);
+      Real weight = grid.int_weights.h_view(ip);
+      SWSphericalHarm(&ylmR1,&ylmI1, ls[n1], ms[n1], 0, theta, phi);
+      SWSphericalHarm(&ylmR2,&ylmI2, ls[n2], ms[n2], 0, theta, phi);
+      // complex conjugate
+      ylmI2 *= -1;
+      int_r += weight*(ylmR1*ylmR2 - ylmI1*ylmI2);
+      int_i += weight*(ylmR1*ylmI2 + ylmR2*ylmI1);
+    }
+
+    if (ls[n1] == ls[n2] && ms[n1] == ms[n2]) {
+      max_err = (abs(int_r-1)> max_err) ? abs(int_r-1) : max_err;
+      max_err = (abs(int_i)> max_err) ? abs(int_i) : max_err;
+
+      if (abs(int_r-1) >= 1e-10 || abs(int_i) >= 1e-10) {
+        failed = true;
+      }
+    } else {
+      max_err = (abs(int_r)> max_err) ? abs(int_r) : max_err;
+      max_err = (abs(int_i)> max_err) ? abs(int_i) : max_err;
+
+      if (abs(int_r) >= 1e-10 || abs(int_i) >= 1e-10) {
+        failed = true;
+      }
+    }
+    if (failed == true) {
+      std::cout << "Gauss Legendre Integral Test Failed"<< std::endl;
+      std::cout << "l1=" << ls[n1] << '\t' << "m1=" << ms[n1]<< std::endl;
+      std::cout << "l2=" << ls[n2] << '\t' << "m2=" << ms[n2]<< std::endl;
+      std::cout << "value is " << int_r << "+1j*" << int_i << std::endl;
+      std::cout << std::endl;
+      exit(EXIT_FAILURE);
+    }
+  }
+  std::cout << "Test Passed with Maximum Error = " << max_err << std::endl;
+
+  return;
+}
diff --git a/src/utils/legendre_roots.hpp b/src/utils/legendre_roots.hpp
index cc8a89fbf..0b363bd4f 100644
--- a/src/utils/legendre_roots.hpp
+++ b/src/utils/legendre_roots.hpp
@@ -1,127 +1,159 @@
 #ifndef UTILS_LEGENDRE_ROOTS_HPP_
 #define UTILS_LEGENDRE_ROOTS_HPP_
+
 //========================================================================================
 // AthenaXXX astrophysical plasma code
-// Copyright(C) 2020 James M. Stone <jmstone@ias.edu> and the Athena code team
+// Copyright(C) 2020 James M. Stone <jmstone@ias.edu>
 // Licensed under the 3-clause BSD License (the "LICENSE")
 //========================================================================================
 //! \file legendre_roots.hpp
-//! \brief find roots of legendre polynomial
+//! \brief find roots of legendre polynomial using Newton–Raphson with Chebyshev guesses
 
 #include <iostream>
 #include <cmath>
-#include <list>
 #include <vector>
 
 #include "athena.hpp"
 
-/*Legendre Polynomial P0(x)*/
-double P0(double x) {
-  return 1;
+//-------------------------------------------------------------------------
+// 1. Basic Legendre polynomials
+//-------------------------------------------------------------------------
+/* P0(x) */
+inline double P0(double x) {
+  return 1.0;
 }
 
-/*Legendre Polynomial P1(x)*/
-double P1(double x) {
+/* P1(x) */
+inline double P1(double x) {
   return x;
 }
 
-/*Nth Legendre Polynomial Pn(x)*/
+/* Nth Legendre Polynomial, Pn(x) */
 double Pn(int n, double x) {
-  if(n==0) {
+  if (n == 0) {
     return P0(x);
-  } else if(n==1) {
+  } else if (n == 1) {
     return P1(x);
   } else {
-    //Use the recurrence relation
-    return (double )((2*n-1)*x*Pn(n-1,x)-(n-1)*Pn(n-2,x))/n;
+    //Use the recurrence relation:
+    //   P_n(x) = ( (2n-1)*x*P_{n-1}(x) - (n-1)*P_{n-2}(x) ) / n
+    return ((2.0*n - 1.0)*x*Pn(n - 1, x) - (n - 1.0)*Pn(n - 2, x))/n;
+  }
+}
+
+//-------------------------------------------------------------------------
+// 2. Derivative of the Legendre polynomial, Pn'(x)
+//
+// A known useful identity is:
+//   (1 - x^2) * Pn'(x) = n * [ P_{n-1}(x) - x * Pn(x) ]
+//   => Pn'(x) = n/(1-x^2) * [P_{n-1}(x) - x*Pn(x)]
+//-------------------------------------------------------------------------
+double PnPrime(int n, double x) {
+  if (n == 0) {
+    return 0.0;
+  } else {
+    return (n/(1.0 - x*x)) * (Pn(n - 1, x) - x*Pn(n, x));
   }
 }
 
-/*Lagrange terms*/
-double Li(int n, std::vector<double> x, int i, double X) {
-  int j;
-  double prod=1;
-  for(j=0; j<=n; j++) {
-    if (j!=i) {
-      prod=prod*(X-x[j])/(x[i]-x[j]);
+//-------------------------------------------------------------------------
+// 3. Newton–Raphson iteration for root of Pn(x)
+//
+//    x_{k+1} = x_k - Pn(n,x_k) / Pn'(n,x_k)
+//-------------------------------------------------------------------------
+double NewtonLegendreRoot(int n, double x0, double tol, int max_iter) {
+  for (int i = 0; i < max_iter; ++i) {
+    double f  = Pn(n, x0);
+    double df = PnPrime(n, x0);
+
+    // Avoid division by very small derivative
+    if (std::fabs(df) < 1.0e-30) {
+      break;  // or return x0 as is
+    }
+
+    double dx = f/df;
+    x0 -= dx;
+
+    if (std::fabs(dx) < tol) {
+      break;
     }
   }
-  return prod;
+  return x0;
 }
 
-/*Function definition to perform integration by Simpson's 1/3rd Rule */
-double Ci(int i, int n, std::vector<double> x, double a, double b, int N) {
-  double h,integral,X,sum=0;
-  int j,k;
-  h=(b-a)/N;
-  for(j=1; j<N; j++) {
-    X=a+j*h;
-    if(j%2==0) {
-      sum=sum+2*Li(n-1,x,i,X);
-    } else {
-      sum=sum+4*Li(n-1,x,i,X);
+//-------------------------------------------------------------------------
+// 4. Lagrange terms
+//-------------------------------------------------------------------------
+double Li(int n, const std::vector<double> &x, int i, double X) {
+  double prod = 1.0;
+  for (int j = 0; j <= n; j++) {
+    if (j != i) {
+      prod *= (X - x[j])/(x[i] - x[j]);
     }
   }
-  double Fa=Li(n-1,x,i,a);
-  double Fb=Li(n-1,x,i,b);
-  integral=(h/3.0)*(Fa+Fb+sum);
-  return integral;
+  return prod;
 }
 
-/*Function definition for bisection procedure[Returns the root if found
-or 999 for failure]*/
-double Bisection(int n,double f(int n,double x),double a, double b,
-                double eps, int maxSteps) {
-  double c;
-  if(f(n,a)*f(n,b)<=0) {
-    int iter=1;
-    /*Bisection Method begins that tabulates the various values at each iteration*/
-    do {
-      c=(a+b)/2;
-      if(f(n,a)*f(n,c)>0) {
-        a=c;
-      } else if(f(n,a)*f(n,c)<0) {
-        b=c;
-      } else if(f(n,c)==0) {
-          return c;
-      }
-        iter++;
-    } while(fabs(a-b)>=eps&&iter<=maxSteps);
-    return c;
-  } else {
-    return 999;
+//-------------------------------------------------------------------------
+// 5. Simpson's rule to compute the integral for weights via Lagrange
+//    polynomials
+//-------------------------------------------------------------------------
+double Ci(int i, int n, const std::vector<double> &x,
+          double a, double b, int N) {
+  double h = (b - a) / N;
+  double sum = 0.0;
+  // Evaluate interior points
+  for (int j = 1; j < N; j++) {
+    double X = a + j * h;
+    if (j % 2 == 0) {
+      sum += 2.0 * Li(n - 1, x, i, X);
+    } else {
+      sum += 4.0 * Li(n - 1, x, i, X);
+    }
   }
+  // Endpoints
+  double Fa = Li(n - 1, x, i, a);
+  double Fb = Li(n - 1, x, i, b);
+
+  return (h / 3.0) * (Fa + Fb + sum);
 }
 
+//-------------------------------------------------------------------------
+// 6. Main routine: find roots & weights of the Legendre polynomial
+//    using Newton–Raphson with Chebyshev initial guesses
+//-------------------------------------------------------------------------
 std::vector<std::vector<Real>> RootsAndWeights(int n) {
-  //Array to store the roots and weights of Legendre polynomials
+  // Storage: [0] = roots, [1] = weights
   std::vector<std::vector<Real>> roots_weights(2, std::vector<Real>(n));
-  std::vector<double> xi;
-
-  //window(Step-size) for bisection method
-  double h=0.01;
-  //dummy variable for bisection method
-  double x;
-  //dummy variable where the root is returned after bisection routine
-  double root;
-  int i=0;
-
-  for(x=-1.0; x<=1.0; x=x+h) {
-    //set the accuracy to approx. 10^-15 but there is also a limit on maxSteps.
-    // (Modify these acc. to your needs)
-    root=Bisection(n,Pn,x,x+h,1e-14,1000000);
-
-    if(root!=999) {
-      xi.push_back(root);
-      roots_weights[0][i]=root;
-      i++;
-    }
+  std::vector<double> xi(n);
+
+  // Parameters for Newton iteration
+  double tol      = 1e-14;
+  int    max_iter = 100000;
+
+  // Chebyshev-type initial guesses for the i-th root
+  // Common choice:
+  //    x0_i = cos( (pi*(i + 0.75)) / (n + 0.5) )
+  // for i = 0..n-1
+  for (int i = 0; i < n; ++i) {
+    double guess = std::cos( M_PI * ( i + 0.75 ) / (n + 0.5) );
+    double root  = NewtonLegendreRoot(n, guess, tol, max_iter);
+
+    xi[i] = root;
+    roots_weights[0][i] = root;
   }
 
-  for(i=0; i<n; i++) {
-      //(Modify the number of sub-intervals according to your needs)
-      roots_weights[1][i]=Ci(i,n,xi,-1,1,1000000);
+  // Compute weights (using the original Lagrange-based Ci or direct formula).
+  // Here we keep your original Lagrange-based approach:
+  for (int i = 0; i < n; i++) {
+    // Adjust N for integration as you see fit.
+    // For a large n, you might not need extremely large N
+    // if the function is well-behaved.
+    // But here we keep your original 60000 intervals.
+    roots_weights[1][i] = Ci(i, n, xi, -1.0, 1.0, 60000);
   }
+
   return roots_weights;
 }
+
 #endif // UTILS_LEGENDRE_ROOTS_HPP_

From 3a0d6da94e00622bf88591a7ccb3b4038e972031 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Sat, 1 Feb 2025 16:11:06 -0500
Subject: [PATCH 36/72] clean up

---
 src/pgen/unit_tests/gauss_legendre_test.cpp | 2 --
 src/utils/legendre_roots.hpp                | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/pgen/unit_tests/gauss_legendre_test.cpp b/src/pgen/unit_tests/gauss_legendre_test.cpp
index 27c8008e8..61eeabf83 100644
--- a/src/pgen/unit_tests/gauss_legendre_test.cpp
+++ b/src/pgen/unit_tests/gauss_legendre_test.cpp
@@ -29,8 +29,6 @@
 using u32    = uint_least32_t;
 using engine = std::mt19937;
 
-// void ADMOnePuncture(MeshBlockPack *pmbp, ParameterInput *pin);
-
 //----------------------------------------------------------------------------------------
 //! \fn ProblemGenerator::UserProblem_()
 //! \brief Problem Generator for single puncture
diff --git a/src/utils/legendre_roots.hpp b/src/utils/legendre_roots.hpp
index 0b363bd4f..dc0874718 100644
--- a/src/utils/legendre_roots.hpp
+++ b/src/utils/legendre_roots.hpp
@@ -3,7 +3,7 @@
 
 //========================================================================================
 // AthenaXXX astrophysical plasma code
-// Copyright(C) 2020 James M. Stone <jmstone@ias.edu>
+// Copyright(C) 2020 James M. Stone <jmstone@ias.edu> and the Athena code team
 // Licensed under the 3-clause BSD License (the "LICENSE")
 //========================================================================================
 //! \file legendre_roots.hpp

From 45c88c26c7ff2c8fbd2b230ad55d6b806c96f879 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Mon, 3 Feb 2025 18:55:24 -0500
Subject: [PATCH 37/72] addressing comments and improving accuracy

---
 src/utils/legendre_roots.hpp | 187 ++++++++++++++---------------------
 1 file changed, 75 insertions(+), 112 deletions(-)

diff --git a/src/utils/legendre_roots.hpp b/src/utils/legendre_roots.hpp
index dc0874718..b225891d5 100644
--- a/src/utils/legendre_roots.hpp
+++ b/src/utils/legendre_roots.hpp
@@ -9,151 +9,114 @@
 //! \file legendre_roots.hpp
 //! \brief find roots of legendre polynomial using Newton–Raphson with Chebyshev guesses
 
-#include <iostream>
+#include <algorithm>
 #include <cmath>
 #include <vector>
+#include <iostream>
+#include <limits> // for std::numeric_limits
+#include <utility>
 
 #include "athena.hpp"
 
-//-------------------------------------------------------------------------
-// 1. Basic Legendre polynomials
-//-------------------------------------------------------------------------
-/* P0(x) */
-inline double P0(double x) {
-  return 1.0;
-}
 
-/* P1(x) */
-inline double P1(double x) {
-  return x;
-}
+//----------------------------------------------------------------------------
+// 1. Legendre polynomial Pn(x) via recurrence
+//----------------------------------------------------------------------------
+inline double Pn(int n, double x) {
+  if (n == 0) return 1.0;
+  if (n == 1) return x;
+
+  double pnm2 = 1.0; // P0
+  double pnm1 = x;   // P1
+  double p     = 0.0;
 
-/* Nth Legendre Polynomial, Pn(x) */
-double Pn(int n, double x) {
-  if (n == 0) {
-    return P0(x);
-  } else if (n == 1) {
-    return P1(x);
-  } else {
-    //Use the recurrence relation:
-    //   P_n(x) = ( (2n-1)*x*P_{n-1}(x) - (n-1)*P_{n-2}(x) ) / n
-    return ((2.0*n - 1.0)*x*Pn(n - 1, x) - (n - 1.0)*Pn(n - 2, x))/n;
+  for(int k = 2; k <= n; ++k) {
+    p = ((2.0 * k - 1.0) * x * pnm1 - (k - 1.0) * pnm2) / static_cast<double>(k);
+    pnm2 = pnm1;
+    pnm1 = p;
   }
+  return p;
 }
 
-//-------------------------------------------------------------------------
-// 2. Derivative of the Legendre polynomial, Pn'(x)
-//
-// A known useful identity is:
-//   (1 - x^2) * Pn'(x) = n * [ P_{n-1}(x) - x * Pn(x) ]
-//   => Pn'(x) = n/(1-x^2) * [P_{n-1}(x) - x*Pn(x)]
-//-------------------------------------------------------------------------
-double PnPrime(int n, double x) {
-  if (n == 0) {
+//----------------------------------------------------------------------------
+// 2. Derivative Pn'(x):
+//    (1 - x^2) Pn'(x) = n [P_{n-1}(x) - x Pn(x)]
+//----------------------------------------------------------------------------
+inline double PnPrime(int n, double x) {
+  if (n == 0) return 0.0;
+  double denom = 1.0 - x*x;
+  if (std::fabs(denom) < 1.0e-30) {
     return 0.0;
-  } else {
-    return (n/(1.0 - x*x)) * (Pn(n - 1, x) - x*Pn(n, x));
   }
+  double numerator = n * (Pn(n - 1, x) - x * Pn(n, x));
+  return numerator / denom;
 }
 
-//-------------------------------------------------------------------------
-// 3. Newton–Raphson iteration for root of Pn(x)
-//
-//    x_{k+1} = x_k - Pn(n,x_k) / Pn'(n,x_k)
-//-------------------------------------------------------------------------
-double NewtonLegendreRoot(int n, double x0, double tol, int max_iter) {
-  for (int i = 0; i < max_iter; ++i) {
+//----------------------------------------------------------------------------
+// 3. Newton–Raphson iteration
+//    x_{k+1} = x_k - Pn(x_k)/Pn'(x_k)
+//    - Tolerance set to machine epsilon by default
+//    - Max iteration ~20 is typically plenty (typically uses 2 or 3 iter)
+//----------------------------------------------------------------------------
+inline double NewtonLegendreRoot(
+    int n,
+    double x0,
+    double tol = std::numeric_limits<double>::epsilon(),
+    int max_iter = 20
+) {
+  for(int i = 0; i < max_iter; ++i) {
     double f  = Pn(n, x0);
     double df = PnPrime(n, x0);
 
-    // Avoid division by very small derivative
-    if (std::fabs(df) < 1.0e-30) {
-      break;  // or return x0 as is
-    }
+    // avoid division by nearly zero
+    if(std::fabs(df) < 1.0e-30) break;
 
-    double dx = f/df;
+    double dx = f / df;
     x0 -= dx;
 
-    if (std::fabs(dx) < tol) {
+    if(std::fabs(dx) < tol) {
       break;
     }
   }
   return x0;
 }
 
-//-------------------------------------------------------------------------
-// 4. Lagrange terms
-//-------------------------------------------------------------------------
-double Li(int n, const std::vector<double> &x, int i, double X) {
-  double prod = 1.0;
-  for (int j = 0; j <= n; j++) {
-    if (j != i) {
-      prod *= (X - x[j])/(x[i] - x[j]);
-    }
-  }
-  return prod;
-}
+//----------------------------------------------------------------------------
+// 4. Main routine: compute n roots & weights for the Legendre polynomial
+//
+//    * Chebyshev initial guesses: x0_i = cos( (π*(i + 0.75)) / (n + 0.5) )
+//    * Newton–Raphson to refine each guess
+//    * Gauss–Legendre weight formula:  w_i = 2 / ( (1 - x_i^2) [Pn'(x_i)]^2 )
+//    * Returns a 2D array [2][n]:  row 0 => roots, row 1 => weights
+//----------------------------------------------------------------------------
+std::vector<std::vector<Real>> RootsAndWeights(int n) {
+  // temporary pair array for sorting by root
+  std::vector<std::pair<double,double>> rw_pairs(n);
 
-//-------------------------------------------------------------------------
-// 5. Simpson's rule to compute the integral for weights via Lagrange
-//    polynomials
-//-------------------------------------------------------------------------
-double Ci(int i, int n, const std::vector<double> &x,
-          double a, double b, int N) {
-  double h = (b - a) / N;
-  double sum = 0.0;
-  // Evaluate interior points
-  for (int j = 1; j < N; j++) {
-    double X = a + j * h;
-    if (j % 2 == 0) {
-      sum += 2.0 * Li(n - 1, x, i, X);
-    } else {
-      sum += 4.0 * Li(n - 1, x, i, X);
-    }
-  }
-  // Endpoints
-  double Fa = Li(n - 1, x, i, a);
-  double Fb = Li(n - 1, x, i, b);
+  for(int i = 0; i < n; ++i) {
+    // Chebyshev guess
+    double guess = std::cos(M_PI * (i + 0.75) / (n + 0.5));
+    // Newton iteration
+    double root = NewtonLegendreRoot(n, guess);
 
-  return (h / 3.0) * (Fa + Fb + sum);
-}
+    // weight
+    double dp = PnPrime(n, root);
+    double weight = 2.0 / ((1.0 - root*root)*(dp*dp));
 
-//-------------------------------------------------------------------------
-// 6. Main routine: find roots & weights of the Legendre polynomial
-//    using Newton–Raphson with Chebyshev initial guesses
-//-------------------------------------------------------------------------
-std::vector<std::vector<Real>> RootsAndWeights(int n) {
-  // Storage: [0] = roots, [1] = weights
-  std::vector<std::vector<Real>> roots_weights(2, std::vector<Real>(n));
-  std::vector<double> xi(n);
-
-  // Parameters for Newton iteration
-  double tol      = 1e-14;
-  int    max_iter = 100000;
-
-  // Chebyshev-type initial guesses for the i-th root
-  // Common choice:
-  //    x0_i = cos( (pi*(i + 0.75)) / (n + 0.5) )
-  // for i = 0..n-1
-  for (int i = 0; i < n; ++i) {
-    double guess = std::cos( M_PI * ( i + 0.75 ) / (n + 0.5) );
-    double root  = NewtonLegendreRoot(n, guess, tol, max_iter);
-
-    xi[i] = root;
-    roots_weights[0][i] = root;
+    rw_pairs[i] = std::make_pair(root, weight);
   }
 
-  // Compute weights (using the original Lagrange-based Ci or direct formula).
-  // Here we keep your original Lagrange-based approach:
-  for (int i = 0; i < n; i++) {
-    // Adjust N for integration as you see fit.
-    // For a large n, you might not need extremely large N
-    // if the function is well-behaved.
-    // But here we keep your original 60000 intervals.
-    roots_weights[1][i] = Ci(i, n, xi, -1.0, 1.0, 60000);
+  // Sort by ascending root
+  std::sort(rw_pairs.begin(), rw_pairs.end(),
+            [](auto &a, auto &b){ return a.first < b.first; });
+
+  // Prepare return structure
+  std::vector<std::vector<Real>> roots_weights(2, std::vector<Real>(n));
+  for(int i = 0; i < n; ++i) {
+    roots_weights[0][i] = rw_pairs[i].first;
+    roots_weights[1][i] = rw_pairs[i].second;
   }
 
   return roots_weights;
 }
-
 #endif // UTILS_LEGENDRE_ROOTS_HPP_

From eeb7dfbe235fa15a321807123c0612fbf3b5f757 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Mon, 3 Feb 2025 19:03:50 -0500
Subject: [PATCH 38/72] fix minor diagnostic things

---
 src/pgen/unit_tests/gauss_legendre_test.cpp |  4 ++--
 src/utils/legendre_roots.hpp                | 13 +++++++++----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/pgen/unit_tests/gauss_legendre_test.cpp b/src/pgen/unit_tests/gauss_legendre_test.cpp
index 61eeabf83..d89895bdd 100644
--- a/src/pgen/unit_tests/gauss_legendre_test.cpp
+++ b/src/pgen/unit_tests/gauss_legendre_test.cpp
@@ -105,12 +105,12 @@ void ProblemGenerator::UserProblem(ParameterInput *pin, const bool restart) {
       std::cout << "Gauss Legendre Integral Test Failed"<< std::endl;
       std::cout << "l1=" << ls[n1] << '\t' << "m1=" << ms[n1]<< std::endl;
       std::cout << "l2=" << ls[n2] << '\t' << "m2=" << ms[n2]<< std::endl;
-      std::cout << "value is " << int_r << "+1j*" << int_i << std::endl;
+      std::cout << "Maximum Error is " << max_err << std::endl;
       std::cout << std::endl;
       exit(EXIT_FAILURE);
     }
   }
-  std::cout << "Test Passed with Maximum Error = " << max_err << std::endl;
+  std::cout << "Test Passed with Maximum Error is " << max_err << std::endl;
 
   return;
 }
diff --git a/src/utils/legendre_roots.hpp b/src/utils/legendre_roots.hpp
index b225891d5..a70b034f5 100644
--- a/src/utils/legendre_roots.hpp
+++ b/src/utils/legendre_roots.hpp
@@ -30,7 +30,7 @@ inline double Pn(int n, double x) {
   double pnm1 = x;   // P1
   double p     = 0.0;
 
-  for(int k = 2; k <= n; ++k) {
+  for (int k = 2; k <= n; ++k) {
     p = ((2.0 * k - 1.0) * x * pnm1 - (k - 1.0) * pnm2) / static_cast<double>(k);
     pnm2 = pnm1;
     pnm1 = p;
@@ -64,19 +64,24 @@ inline double NewtonLegendreRoot(
     double tol = std::numeric_limits<double>::epsilon(),
     int max_iter = 20
 ) {
-  for(int i = 0; i < max_iter; ++i) {
+  for (int i = 0; i < max_iter; ++i) {
     double f  = Pn(n, x0);
     double df = PnPrime(n, x0);
 
     // avoid division by nearly zero
-    if(std::fabs(df) < 1.0e-30) break;
+    if (std::fabs(df) < 1.0e-30) break;
 
     double dx = f / df;
     x0 -= dx;
 
-    if(std::fabs(dx) < tol) {
+    if (std::fabs(dx) < tol) {
       break;
     }
+
+    if (i == max_iter-1) {
+      std::cout << "Failed to Initialize the Gauss-Legendre Grid"<< std::endl;
+      exit(EXIT_FAILURE);
+    }
   }
   return x0;
 }

From 01350f122eadf4270ef1087ac502e10db17c1420 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@della-vis1.princeton.edu>
Date: Wed, 5 Feb 2025 09:08:54 -0500
Subject: [PATCH 39/72] adding debug

---
 src/mesh/meshblock_pack.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/mesh/meshblock_pack.cpp b/src/mesh/meshblock_pack.cpp
index a2ab67fa7..fbe3ef304 100644
--- a/src/mesh/meshblock_pack.cpp
+++ b/src/mesh/meshblock_pack.cpp
@@ -199,9 +199,11 @@ void MeshBlockPack::AddPhysics(ParameterInput *pin) {
     int ncce = pin->GetOrAddInteger("cce", "num_radii", 0);
     pz4c_cce.reserve(ncce);// 10 different components for each radius
     for(int n = 0; n < ncce; ++n) {
+      std::cout << "Initializing CCE " << n << std::endl;
       // NOTE: these names are used for pittnull code, so DON'T change the convention
       pz4c_cce.push_back(new z4c::CCE(pmesh, pin,n));
     }
+    std::cout << "CCE Initialized" << std::endl;
     nphysics++;
   } else {
     pz4c = nullptr;

From 3d81529ebde2133d8fcba896239ed04164105b48 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Thu, 6 Feb 2025 10:33:00 -0500
Subject: [PATCH 40/72] add: plan for pittnull to cce spectre

---
 scripts/cce/plan.txt | 66 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 scripts/cce/plan.txt

diff --git a/scripts/cce/plan.txt b/scripts/cce/plan.txt
new file mode 100644
index 000000000..01dc1a484
--- /dev/null
+++ b/scripts/cce/plan.txt
@@ -0,0 +1,66 @@
+#define Max(a_,b_) ((a_)>(b_)? (a_):(b_))
+#define Min(a_,b_) ((a_)<(b_)? (a_):(b_))
+#define FourPi 12.566370614359172953850573533118
+#define ABS(x_) ((x_)>0 ? (x_) : (-(x_)))
+
+static long double fact(int i)
+{
+  int j;
+  long double temp = 1;
+
+  myassert(i>-1);
+
+  for (j=2; j <=i; j++)
+  {
+    temp *=j;
+  }
+  return temp;
+}
+
+double sPlm(int s, int l, int m, double theta)
+{
+  int k;
+  double temp = 0;
+  s= -s;  /* oops below is really the definition for
+            -s not s */
+
+  myassert(l>=ABS(s));
+  myassert(l>=ABS(m));
+
+  const double sc = (1-2*(ABS(s)%2))*
+        sqrt((2*l+1)/(FourPi)*fact(l+m)*fact(l-m)*fact(l+s)*fact(l-s));
+
+  for (k= Max(0, m-s); k <= Min(l+m, l-s); k++)
+  {
+    double t = sc;
+
+    t/= fact(l+m-k);
+    t/= fact(l-s-k);
+    t/= fact(k);
+    t/= fact(k+s-m);
+
+    temp += (1-2*(ABS(k)%2))*t*
+       pow(cos(0.5*theta),(double)(2*l+m-s-2*k))*
+       pow(sin(0.5*theta),(double)(2*k+s-m));
+  }
+
+  return temp;
+}
+
+complex <double> sYlm (int s, int l, int m, double theta, double phi)
+{
+  const complex<double> I(0,1);
+  return sPlm(s,l,m,theta)*exp(I*(complex<double>)(m*phi));
+}
+
+
+
+## notes:
+. cheb recons: for n 0...nmax-1
+  tmp[time][angle] += reall[]*U[n];
+
+. reconstrict g(th,ph) at Gauss-Legendre in the and equally spaced in ph:
+SHRE_gij(th,phi) |_r=r_0 += (RC(lm) * reYN(th,ph,lm) - IC(lm) * imYN(th,ph,lm))
+
+. don't forget about factor in your paper
+see also: pittnullcode/SphericalHarmonicReconGen/src/get.cc, line 51

From 86fa233f8b2c428c3c684bf4390a14baf0318e9e Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 7 Feb 2025 11:24:26 -0500
Subject: [PATCH 41/72] add: pit to spec transformation, not finished.

---
 scripts/cce/athk_to_spectre.py | 56 ++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index fe409114a..f6d4b81c7 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -131,6 +131,56 @@ def parse_cli():
   return args
 
 
+class AngularTransform:
+  """
+    angular coordinate transformation from pittnull to spectre
+    """
+
+  def __init__(self, attrs: dict):
+    self.attrs = attrs
+    self.th = self._theta_guass_legendre()
+    self.ph = self._phi_equispace()
+    self.ylm = self._ylm()
+
+  def reconstruct_pit_on_gl(self, coeff: np.array):
+    """
+        reconstruct field on gauss lengendre collocation pnts from coeffs of pittnull
+        """
+    rylm = self.ylm[g_re]
+    iylm = self.ylm[g_im]
+
+    shape = (self.attrs["lev_t"], self.attrs["max_n"], self.attrs["max_lm"])
+    field = np.empty(shape=shape, dtype=float)
+    for lm in range(self.attrs["max_lm"]):
+      # TODO: broadcast
+      field[:, :, lm] = (coeff[g_im, :, :, lm] * rylm[lm] -
+                         coeff[g_re, :, :, lm] * iylm[lm])
+
+    return field
+
+  def transform_field_to_coeff_gl(self, field: np.array):
+    """
+        transformation of the given field on guass legendre points to
+        spherical harmonics basis on gauss legendre points.
+        """
+
+    coeff = np.emptylike(field)
+    ...
+
+    return coeff
+
+  def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
+    """
+        transform pit coeffs which are on equispace theta and phi to spectre coeffs
+        which are equispace on phi and guass legendre on theta.
+        """
+
+    # first reconstruct field on gl collocation points
+    field = self.reconstruct_pit_on_gl(coeffs)
+
+    return self.transform_field_to_coeff_gl(field)
+
+
 def load(fpath: str, field_name: str, attrs: dict) -> list:
   """
     read the field accroding to attrs.
@@ -146,6 +196,8 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
     max_lm = attrs["max_lm"]
     shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
     ret = np.empty(shape=shape, dtype=float)
+
+    coords = AngularTransform(attrs)
     with h5py.File(fpath, "r") as h5f:
       # read & save
       for i in range(0, lev_t):
@@ -154,6 +206,10 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         h5_im = h5f[f"{key}/{field_name}/im"]
         ret[g_re, i, :] = h5_re
         ret[g_im, i, :] = h5_im
+
+      # transform from PITTNull coordinates to Spectre coordinates
+      ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
+
   elif attrs["file_type"] == "bin":
     # Load the list of files
     ##TODO: this depends on file name

From 0a1ef09194bdea8086635fda39f85d714eb262a8 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 7 Feb 2025 11:34:13 -0500
Subject: [PATCH 42/72] add: phi colloc.

---
 scripts/cce/athk_to_spectre.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index f6d4b81c7..1374ebb6a 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -138,10 +138,23 @@ class AngularTransform:
 
   def __init__(self, attrs: dict):
     self.attrs = attrs
-    self.th = self._theta_guass_legendre()
+    self.th = self._theta_gauss_legendre()
     self.ph = self._phi_equispace()
     self.ylm = self._ylm()
 
+  def _phi_equispace(self):
+    """
+        create equispace collocation pnts on phi
+        """
+    max_l = attrs["max_l"]
+    nphi = 2 * max_l + 1
+    phi = np.empty(shape=(nphi), dtype=float)
+
+    for i in range(nphi):
+      phi[i] = 2 * i * m.pi / nphi
+
+    return phi
+
   def reconstruct_pit_on_gl(self, coeff: np.array):
     """
         reconstruct field on gauss lengendre collocation pnts from coeffs of pittnull
@@ -160,7 +173,7 @@ def reconstruct_pit_on_gl(self, coeff: np.array):
 
   def transform_field_to_coeff_gl(self, field: np.array):
     """
-        transformation of the given field on guass legendre points to
+        transformation of the given field on gauss legendre points to
         spherical harmonics basis on gauss legendre points.
         """
 
@@ -172,7 +185,7 @@ def transform_field_to_coeff_gl(self, field: np.array):
   def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
     """
         transform pit coeffs which are on equispace theta and phi to spectre coeffs
-        which are equispace on phi and guass legendre on theta.
+        which are equispace on phi and gauss legendre on theta.
         """
 
     # first reconstruct field on gl collocation points

From cdf851beaf41d1691349679a46ac74720a3d9047 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 7 Feb 2025 11:49:06 -0500
Subject: [PATCH 43/72] add: theta colloc pnts using sympy.

---
 scripts/cce/athk_to_spectre.py | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 1374ebb6a..3e5aea08b 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -140,13 +140,45 @@ def __init__(self, attrs: dict):
     self.attrs = attrs
     self.th = self._theta_gauss_legendre()
     self.ph = self._phi_equispace()
+    self.l_root = self._legendre_root()
     self.ylm = self._ylm()
 
+  def _theta_gauss_legendre(self):
+    """
+        creating gl collocation pnts for theta
+        """
+    max_l = self.attrs["max_l"]
+    nth = 2 * max_l + 1
+    th = np.empty(shape=(nth), dtype=float)
+
+    for i in range(nth):
+      th[i] = m.acos(-self.l_root[i])
+
+    return th
+
+  def _legendre_root(self, i_th_root: int, l_order: int):
+    from sympy.solvers import solve
+    from sympy import legendre, re, N
+
+    max_l = self.attrs["max_l"]
+    nth = 2 * max_l + 1
+    precision = 18 # numerical precision
+    x = symbols("x")
+
+    root = solve(legendre(nth, x), x, minimal=True, quick=True, warn=True)
+    roots = []
+    nroot = len(root)
+    for j in range(nroot): # make it real, some of the roots has 10^-17 I part
+      roots.append(re(N(root[j], precision)))
+    roots.sort()
+
+    return self._legendre_root_tab[l_order][i_th_root]
+
   def _phi_equispace(self):
     """
         create equispace collocation pnts on phi
         """
-    max_l = attrs["max_l"]
+    max_l = self.attrs["max_l"]
     nphi = 2 * max_l + 1
     phi = np.empty(shape=(nphi), dtype=float)
 

From fc2319b01c152d1f64968fc543dcc94fc0ad2961 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 7 Feb 2025 14:45:30 -0500
Subject: [PATCH 44/72] add: ylm used in pittnull.

---
 scripts/cce/athk_to_spectre.py | 95 ++++++++++++++++++++++++++++++----
 1 file changed, 84 insertions(+), 11 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 3e5aea08b..35b4222e7 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -12,6 +12,7 @@
 import numpy as np
 from scipy import special
 import math
+import cmath
 import argparse
 import h5py
 import struct
@@ -138,19 +139,22 @@ class AngularTransform:
 
   def __init__(self, attrs: dict):
     self.attrs = attrs
+    # no. of collocation pnts
+    self.npnts = 2 * self.attrs["max_l"] + 1
+    # theta collocation coords
     self.th = self._theta_gauss_legendre()
+    # phi collocation coords
     self.ph = self._phi_equispace()
+    # legendre roots
     self.l_root = self._legendre_root()
-    self.ylm = self._ylm()
+    # ylm(real/imag,th,ph,lm) on collocation coords
+    self.ylm_pit = self._ylm_pit()
 
   def _theta_gauss_legendre(self):
     """
         creating gl collocation pnts for theta
         """
-    max_l = self.attrs["max_l"]
-    nth = 2 * max_l + 1
-    th = np.empty(shape=(nth), dtype=float)
-
+    th = np.empty(shape=(self.npnts), dtype=float)
     for i in range(nth):
       th[i] = m.acos(-self.l_root[i])
 
@@ -160,8 +164,7 @@ def _legendre_root(self, i_th_root: int, l_order: int):
     from sympy.solvers import solve
     from sympy import legendre, re, N
 
-    max_l = self.attrs["max_l"]
-    nth = 2 * max_l + 1
+    nth = self.npnts
     precision = 18 # numerical precision
     x = symbols("x")
 
@@ -178,15 +181,85 @@ def _phi_equispace(self):
     """
         create equispace collocation pnts on phi
         """
-    max_l = self.attrs["max_l"]
-    nphi = 2 * max_l + 1
-    phi = np.empty(shape=(nphi), dtype=float)
+    phi = np.empty(shape=(self.npnts), dtype=float)
 
     for i in range(nphi):
-      phi[i] = 2 * i * m.pi / nphi
+      phi[i] = 2 * i * m.pi / self.npnts
 
     return phi
 
+  def _ylm_pit(self):
+    """
+        compute ylm on gauss legnedre for theta, and equispace for phi
+        we use the ylm defined in pittnull code
+        """
+
+    ylms = np.empty(
+        shape=(
+            len[g_re, g_im],
+            self.npnts,
+            self.npnts,
+            self.attrs["max_lm"],
+        ),
+        dtype=float,
+    )
+    for i in range(self.npnts):
+      th = self._th[i]
+      for j in range(self.npnts):
+        ph = self._ph[j]
+        for l in range(0, attrs["max_l"]):
+          for m in range(l, -l - 1, -1):
+            lm = lm_mode(l, m)
+            y = self.__sYlm_pit(0, l, m, th, ph)
+            ylms[g_re, i, j, lm] = y.real()
+            ylms[g_re, i, j, lm] = y.imag()
+
+    return ylms
+
+  def __fact(i):
+    """
+        factorial in pittnull
+        """
+    assert i > -1
+    temp = 1
+    for j in range(2, i + 1):
+      temp *= j
+    return temp
+
+  def __sPlm(s, l, m, theta):
+    """
+        sPlm in pitnull
+        """
+    temp = 0
+    s = -s # oops below is really the definition for -s not s
+
+    assert l >= abs(s)
+    assert l >= abs(m)
+
+    sc = (1 - 2 * (abs(s) % 2)) * math.sqrt(
+        (2 * l + 1) / (4 * math.pi) * self.__fact(l + m) * self.__fact(l - m) *
+        self.__fact(l + s) * self.__fact(l - s))
+
+    for k in range(max(0, m - s), min(l + m, l - s) + 1):
+      t = sc
+      t /= self.__fact(l + m - k)
+      t /= self.__fact(l - s - k)
+      t /= self.__fact(k)
+      t /= self.__fact(k + s - m)
+
+      temp += ((1 - 2 * (abs(k) % 2)) * t * math.pow(math.cos(0.5 * theta),
+                                                     (2 * l + m - s - 2 * k)) *
+               math.pow(math.sin(0.5 * theta), (2 * k + s - m)))
+
+    return temp
+
+  def __sYlm_pit(s, l, m, theta, phi):
+    """
+        sYlm in pittnull
+        """
+    I = complex(0, 1)
+    return self.__sPlm(s, l, m, theta) * cmath.exp(I * (m * phi))
+
   def reconstruct_pit_on_gl(self, coeff: np.array):
     """
         reconstruct field on gauss lengendre collocation pnts from coeffs of pittnull

From 069d330450d60b1051259e2c21a8e7350d3b51b0 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 7 Feb 2025 14:58:37 -0500
Subject: [PATCH 45/72] fix: bugs.

---
 scripts/cce/athk_to_spectre.py | 35 +++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 35b4222e7..472873f46 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -141,12 +141,12 @@ def __init__(self, attrs: dict):
     self.attrs = attrs
     # no. of collocation pnts
     self.npnts = 2 * self.attrs["max_l"] + 1
+    self.l_root = self._legendre_root()
     # theta collocation coords
     self.th = self._theta_gauss_legendre()
     # phi collocation coords
     self.ph = self._phi_equispace()
     # legendre roots
-    self.l_root = self._legendre_root()
     # ylm(real/imag,th,ph,lm) on collocation coords
     self.ylm_pit = self._ylm_pit()
 
@@ -155,14 +155,14 @@ def _theta_gauss_legendre(self):
         creating gl collocation pnts for theta
         """
     th = np.empty(shape=(self.npnts), dtype=float)
-    for i in range(nth):
-      th[i] = m.acos(-self.l_root[i])
+    for i in range(self.npnts):
+      th[i] = math.acos(-self.l_root[i])
 
     return th
 
-  def _legendre_root(self, i_th_root: int, l_order: int):
+  def _legendre_root(self):
     from sympy.solvers import solve
-    from sympy import legendre, re, N
+    from sympy import legendre, re, N, symbols
 
     nth = self.npnts
     precision = 18 # numerical precision
@@ -175,7 +175,7 @@ def _legendre_root(self, i_th_root: int, l_order: int):
       roots.append(re(N(root[j], precision)))
     roots.sort()
 
-    return self._legendre_root_tab[l_order][i_th_root]
+    return roots
 
   def _phi_equispace(self):
     """
@@ -183,8 +183,8 @@ def _phi_equispace(self):
         """
     phi = np.empty(shape=(self.npnts), dtype=float)
 
-    for i in range(nphi):
-      phi[i] = 2 * i * m.pi / self.npnts
+    for i in range(self.npnts):
+      phi[i] = 2 * i * math.pi / self.npnts
 
     return phi
 
@@ -192,11 +192,12 @@ def _ylm_pit(self):
     """
         compute ylm on gauss legnedre for theta, and equispace for phi
         we use the ylm defined in pittnull code
+        ylm[real/imag, theta, phi, lm]
         """
 
     ylms = np.empty(
         shape=(
-            len[g_re, g_im],
+            len([g_re, g_im]),
             self.npnts,
             self.npnts,
             self.attrs["max_lm"],
@@ -204,19 +205,19 @@ def _ylm_pit(self):
         dtype=float,
     )
     for i in range(self.npnts):
-      th = self._th[i]
+      th = self.th[i]
       for j in range(self.npnts):
-        ph = self._ph[j]
-        for l in range(0, attrs["max_l"]):
+        ph = self.ph[j]
+        for l in range(0, self.attrs["max_l"]):
           for m in range(l, -l - 1, -1):
             lm = lm_mode(l, m)
             y = self.__sYlm_pit(0, l, m, th, ph)
-            ylms[g_re, i, j, lm] = y.real()
-            ylms[g_re, i, j, lm] = y.imag()
+            ylms[g_re, i, j, lm] = y.real
+            ylms[g_im, i, j, lm] = y.imag
 
     return ylms
 
-  def __fact(i):
+  def __fact(self, i):
     """
         factorial in pittnull
         """
@@ -226,7 +227,7 @@ def __fact(i):
       temp *= j
     return temp
 
-  def __sPlm(s, l, m, theta):
+  def __sPlm(self, s, l, m, theta):
     """
         sPlm in pitnull
         """
@@ -253,7 +254,7 @@ def __sPlm(s, l, m, theta):
 
     return temp
 
-  def __sYlm_pit(s, l, m, theta, phi):
+  def __sYlm_pit(self, s, l, m, theta, phi):
     """
         sYlm in pittnull
         """

From de453eaae555d062a7c0191af366ab95a7f96f28 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Fri, 7 Feb 2025 16:31:23 -0500
Subject: [PATCH 46/72] add: angular coordinate transformation between pittnull
 to spectre.

---
 scripts/cce/athk_to_spectre.py | 91 +++++++++++++++++++++++++++++-----
 1 file changed, 79 insertions(+), 12 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 472873f46..cf66c3b95 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -265,27 +265,94 @@ def reconstruct_pit_on_gl(self, coeff: np.array):
     """
         reconstruct field on gauss lengendre collocation pnts from coeffs of pittnull
         """
-    rylm = self.ylm[g_re]
-    iylm = self.ylm[g_im]
+    rylm = self.ylm_pit[g_re]
+    iylm = self.ylm_pit[g_im]
 
-    shape = (self.attrs["lev_t"], self.attrs["max_n"], self.attrs["max_lm"])
-    field = np.empty(shape=shape, dtype=float)
-    for lm in range(self.attrs["max_lm"]):
-      # TODO: broadcast
-      field[:, :, lm] = (coeff[g_im, :, :, lm] * rylm[lm] -
-                         coeff[g_re, :, :, lm] * iylm[lm])
+    shape = (
+        self.attrs["lev_t"],
+        self.attrs["max_n"],
+        self.npnts,
+        self.npnts,
+    )
+    field = np.zeros(shape=shape, dtype=float)
+    for i in range(self.npnts):
+      for j in range(self.npnts):
+        for lm in range(self.attrs["max_lm"]):
+          # TODO: broadcast
+          field[:, :, i, j] += (coeff[g_im, :, :, lm] * rylm[i, j, lm] -
+                                coeff[g_re, :, :, lm] * iylm[i, j, lm])
 
     return field
 
+  def _sp_expansion(self, f, L_max, N_theta, N_phi):
+    """
+        Expands function f(theta, phi) in spherical harmonics up to degree L_max
+        using Gauss-Legendre quadrature in theta and uniform quadrature in phi.
+
+        Parameters:
+            f : function [theta, phi] -> float
+                Function to expand.
+            L_max : int
+                Maximum degree of spherical harmonics.
+            N_theta : int
+                Number of Gauss-Legendre points in theta.
+            N_phi : int
+                Number of equispaced points in phi.
+
+        Returns:
+            coeff :Expansion coefficients.
+        """
+
+    # Gauss-Legendre quadrature points and weights for theta
+    x, w_theta = np.polynomial.legendre.leggauss(N_theta)
+    theta = np.arccos(-x) # Convert from [-1,1] to [0,pi]
+
+    # Uniform quadrature points for phi
+    phi = np.linspace(0, 2 * np.pi, N_phi, endpoint=False)
+
+    # Initialize coefficients
+    coeff = np.zeros(shape=(self.attrs["max_lm"]), dtype=complex)
+    dphi = 2 * np.pi / N_phi
+
+    # Compute expansion coefficients
+    for l in range(L_max + 1):
+      for m in range(-l, l + 1):
+        integral = 0.0
+        for i, t in enumerate(theta):
+          for j, p in enumerate(phi):
+            # NOTE: there are some convention difference for theta and phi in scipy
+            Ylm = special.sph_harm(m, l, p, t)
+            integral += f[i, j] * np.conj(Ylm) * w_theta[i] * dphi
+        coeff[lm_mode(l, m)] = integral # Store coefficient
+
+    return coeff
+
   def transform_field_to_coeff_gl(self, field: np.array):
     """
         transformation of the given field on gauss legendre points to
         spherical harmonics basis on gauss legendre points.
+        field[time,radius,theta,phi]
         """
 
-    coeff = np.emptylike(field)
-    ...
-
+    shape = (
+        len([g_re, g_im]),
+        self.attrs["lev_t"],
+        self.attrs["max_n"],
+        self.attrs["max_lm"],
+    )
+    coeff = np.empty(shape=shape, dtype=float)
+    # TODO: broadcast
+    for t in range(self.attrs["lev_t"]):
+      for r in range(self.attrs["max_n"]):
+        c = self._sp_expansion(field[t, r],
+                               self.attrs["max_l"],
+                               self.npnts,
+                               self.npnts)
+        
+        #print(c,c.real,c.imag)
+        coeff[g_re, t, r, :] = c.real
+        coeff[g_im, t, r, :] = c.imag
+        
     return coeff
 
   def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
@@ -295,7 +362,7 @@ def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
         """
 
     # first reconstruct field on gl collocation points
-    field = self.reconstruct_pit_on_gl(coeffs)
+    field = self.reconstruct_pit_on_gl(coeff)
 
     return self.transform_field_to_coeff_gl(field)
 

From b887bb5fb42701f8fa52ebcf5536671929467a62 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 09:06:51 -0500
Subject: [PATCH 47/72] feat: optimize.

---
 scripts/cce/athk_to_spectre.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index cf66c3b95..6ce5e9b7a 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -161,21 +161,13 @@ def _theta_gauss_legendre(self):
     return th
 
   def _legendre_root(self):
-    from sympy.solvers import solve
-    from sympy import legendre, re, N, symbols
-
+    """
+      legendre roots
+    """
     nth = self.npnts
-    precision = 18 # numerical precision
-    x = symbols("x")
-
-    root = solve(legendre(nth, x), x, minimal=True, quick=True, warn=True)
-    roots = []
-    nroot = len(root)
-    for j in range(nroot): # make it real, some of the roots has 10^-17 I part
-      roots.append(re(N(root[j], precision)))
-    roots.sort()
-
-    return roots
+    x, _ = np.polynomial.legendre.leggauss(nth)
+    x.sort()
+    return x
 
   def _phi_equispace(self):
     """

From a1ba6e6bccf96733537128e2857762529d2ac27f Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 09:09:57 -0500
Subject: [PATCH 48/72] feat: optimize.

---
 scripts/cce/athk_to_spectre.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 6ce5e9b7a..d1d063224 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -173,11 +173,8 @@ def _phi_equispace(self):
     """
         create equispace collocation pnts on phi
         """
-    phi = np.empty(shape=(self.npnts), dtype=float)
-
-    for i in range(self.npnts):
-      phi[i] = 2 * i * math.pi / self.npnts
-
+    phi = np.linspace(0, 2 * np.pi, self.npnts, endpoint=False)
+    
     return phi
 
   def _ylm_pit(self):

From cd98e7fcfcc246bddcc28e56c2577bccf5712f22 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 09:23:09 -0500
Subject: [PATCH 49/72] feat: optimize and test.

---
 scripts/cce/athk_to_spectre.py | 51 ++--------------------------------
 1 file changed, 3 insertions(+), 48 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index d1d063224..faabb3ae3 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -148,7 +148,7 @@ def __init__(self, attrs: dict):
     self.ph = self._phi_equispace()
     # legendre roots
     # ylm(real/imag,th,ph,lm) on collocation coords
-    self.ylm_pit = self._ylm_pit()
+    self.ylm_pit = self._ylm_on_gl()
 
   def _theta_gauss_legendre(self):
     """
@@ -177,10 +177,9 @@ def _phi_equispace(self):
     
     return phi
 
-  def _ylm_pit(self):
+  def _ylm_on_gl(self):
     """
         compute ylm on gauss legnedre for theta, and equispace for phi
-        we use the ylm defined in pittnull code
         ylm[real/imag, theta, phi, lm]
         """
 
@@ -200,56 +199,12 @@ def _ylm_pit(self):
         for l in range(0, self.attrs["max_l"]):
           for m in range(l, -l - 1, -1):
             lm = lm_mode(l, m)
-            y = self.__sYlm_pit(0, l, m, th, ph)
+            y = special.sph_harm(m, l, ph, th)
             ylms[g_re, i, j, lm] = y.real
             ylms[g_im, i, j, lm] = y.imag
 
     return ylms
 
-  def __fact(self, i):
-    """
-        factorial in pittnull
-        """
-    assert i > -1
-    temp = 1
-    for j in range(2, i + 1):
-      temp *= j
-    return temp
-
-  def __sPlm(self, s, l, m, theta):
-    """
-        sPlm in pitnull
-        """
-    temp = 0
-    s = -s # oops below is really the definition for -s not s
-
-    assert l >= abs(s)
-    assert l >= abs(m)
-
-    sc = (1 - 2 * (abs(s) % 2)) * math.sqrt(
-        (2 * l + 1) / (4 * math.pi) * self.__fact(l + m) * self.__fact(l - m) *
-        self.__fact(l + s) * self.__fact(l - s))
-
-    for k in range(max(0, m - s), min(l + m, l - s) + 1):
-      t = sc
-      t /= self.__fact(l + m - k)
-      t /= self.__fact(l - s - k)
-      t /= self.__fact(k)
-      t /= self.__fact(k + s - m)
-
-      temp += ((1 - 2 * (abs(k) % 2)) * t * math.pow(math.cos(0.5 * theta),
-                                                     (2 * l + m - s - 2 * k)) *
-               math.pow(math.sin(0.5 * theta), (2 * k + s - m)))
-
-    return temp
-
-  def __sYlm_pit(self, s, l, m, theta, phi):
-    """
-        sYlm in pittnull
-        """
-    I = complex(0, 1)
-    return self.__sPlm(s, l, m, theta) * cmath.exp(I * (m * phi))
-
   def reconstruct_pit_on_gl(self, coeff: np.array):
     """
         reconstruct field on gauss lengendre collocation pnts from coeffs of pittnull

From 3e316381db0f0c2e6bd282dbaa080ddd98bbe6d3 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 09:26:52 -0500
Subject: [PATCH 50/72] fix: bug on reconstruction

---
 scripts/cce/athk_to_spectre.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index faabb3ae3..729b19936 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -223,8 +223,8 @@ def reconstruct_pit_on_gl(self, coeff: np.array):
       for j in range(self.npnts):
         for lm in range(self.attrs["max_lm"]):
           # TODO: broadcast
-          field[:, :, i, j] += (coeff[g_im, :, :, lm] * rylm[i, j, lm] -
-                                coeff[g_re, :, :, lm] * iylm[i, j, lm])
+          field[:, :, i, j] += (coeff[g_re, :, :, lm] * rylm[i, j, lm] -
+                                coeff[g_im, :, :, lm] * iylm[i, j, lm])
 
     return field
 

From c77303d19f2492e386f28f4031fdb5958d36eaaa Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 11:08:10 -0500
Subject: [PATCH 51/72] feat: broadcast and debug coeff to field.

---
 scripts/cce/athk_to_spectre.py | 164 ++++++++++++++++-----------------
 1 file changed, 82 insertions(+), 82 deletions(-)
 mode change 100755 => 100644 scripts/cce/athk_to_spectre.py

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
old mode 100755
new mode 100644
index 729b19936..d4da2a10c
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/ usr / bin / env python3
 ## Alireza Rashti - Oct 2024 (C)
 ## usage:
 ## $ ./me -h
@@ -19,11 +19,11 @@
 import glob
 from scipy.interpolate import interp1d
 
-# from itertools import product
-# import matplotlib.pyplot as plt
-# import glob
-# import sympy
-# import re
+#from itertools import product
+#import matplotlib.pyplot as plt
+#import glob
+#import sympy
+#import re
 
 ## ---------------------------------------------------------------------- ##
 
@@ -139,15 +139,15 @@ class AngularTransform:
 
   def __init__(self, attrs: dict):
     self.attrs = attrs
-    # no. of collocation pnts
+#no.of collocation pnts
     self.npnts = 2 * self.attrs["max_l"] + 1
     self.l_root = self._legendre_root()
-    # theta collocation coords
+#theta collocation coords
     self.th = self._theta_gauss_legendre()
-    # phi collocation coords
+#phi collocation coords
     self.ph = self._phi_equispace()
-    # legendre roots
-    # ylm(real/imag,th,ph,lm) on collocation coords
+#legendre roots
+#ylm(real / imag, th, ph, lm) on collocation coords
     self.ylm_pit = self._ylm_on_gl()
 
   def _theta_gauss_legendre(self):
@@ -218,15 +218,10 @@ def reconstruct_pit_on_gl(self, coeff: np.array):
         self.npnts,
         self.npnts,
     )
-    field = np.zeros(shape=shape, dtype=float)
-    for i in range(self.npnts):
-      for j in range(self.npnts):
-        for lm in range(self.attrs["max_lm"]):
-          # TODO: broadcast
-          field[:, :, i, j] += (coeff[g_re, :, :, lm] * rylm[i, j, lm] -
-                                coeff[g_im, :, :, lm] * iylm[i, j, lm])
-
-    return field
+    field = (np.dot(coeff[g_re, :, :, :],rylm[:, :, np.newaxis, :, np.newaxis]) -
+             np.dot(coeff[g_im, :, :, :],iylm[:, :, np.newaxis, :, np.newaxis]))
+            
+    return field[:,:,:,:,0,0]
 
   def _sp_expansion(self, f, L_max, N_theta, N_phi):
     """
@@ -247,24 +242,24 @@ def _sp_expansion(self, f, L_max, N_theta, N_phi):
             coeff :Expansion coefficients.
         """
 
-    # Gauss-Legendre quadrature points and weights for theta
+#Gauss - Legendre quadrature points and weights for theta
     x, w_theta = np.polynomial.legendre.leggauss(N_theta)
     theta = np.arccos(-x) # Convert from [-1,1] to [0,pi]
 
-    # Uniform quadrature points for phi
+#Uniform quadrature points for phi
     phi = np.linspace(0, 2 * np.pi, N_phi, endpoint=False)
 
-    # Initialize coefficients
+#Initialize coefficients
     coeff = np.zeros(shape=(self.attrs["max_lm"]), dtype=complex)
     dphi = 2 * np.pi / N_phi
 
-    # Compute expansion coefficients
+#Compute expansion coefficients
     for l in range(L_max + 1):
       for m in range(-l, l + 1):
         integral = 0.0
         for i, t in enumerate(theta):
           for j, p in enumerate(phi):
-            # NOTE: there are some convention difference for theta and phi in scipy
+#NOTE : there are some convention difference for theta and phi in scipy
             Ylm = special.sph_harm(m, l, p, t)
             integral += f[i, j] * np.conj(Ylm) * w_theta[i] * dphi
         coeff[lm_mode(l, m)] = integral # Store coefficient
@@ -285,15 +280,15 @@ def transform_field_to_coeff_gl(self, field: np.array):
         self.attrs["max_lm"],
     )
     coeff = np.empty(shape=shape, dtype=float)
-    # TODO: broadcast
+#TODO : broadcast
     for t in range(self.attrs["lev_t"]):
       for r in range(self.attrs["max_n"]):
         c = self._sp_expansion(field[t, r],
                                self.attrs["max_l"],
                                self.npnts,
                                self.npnts)
-        
-        #print(c,c.real,c.imag)
+
+#print(c, c.real, c.imag)
         coeff[g_re, t, r, :] = c.real
         coeff[g_im, t, r, :] = c.imag
         
@@ -305,7 +300,7 @@ def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
         which are equispace on phi and gauss legendre on theta.
         """
 
-    # first reconstruct field on gl collocation points
+#first reconstruct field on gl collocation points
     field = self.reconstruct_pit_on_gl(coeff)
 
     return self.transform_field_to_coeff_gl(field)
@@ -329,7 +324,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
 
     coords = AngularTransform(attrs)
     with h5py.File(fpath, "r") as h5f:
-      # read & save
+#read &save
       for i in range(0, lev_t):
         key = f"{i}"
         h5_re = h5f[f"{key}/{field_name}/re"]
@@ -337,11 +332,11 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         ret[g_re, i, :] = h5_re
         ret[g_im, i, :] = h5_im
 
-      # transform from PITTNull coordinates to Spectre coordinates
+#transform from PITTNull coordinates to Spectre coordinates
       ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
 
   elif attrs["file_type"] == "bin":
-    # Load the list of files
+#Load the list of files
     ##TODO: this depends on file name
     flist = sorted(glob.glob(fpath + "/cce_*.bin"))
     data = []
@@ -390,7 +385,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
   else:
     raise ValueError("no such option")
 
-  # print(ret)
+#print(ret)
   return ret
 
 
@@ -419,41 +414,42 @@ def read_cce_file(filename):
         index_to_lm (dict): Mapping from data index to (l, m) values.
     """
   with open(filename, "rb") as f:
-    # Read number of radial points (nr)
+#Read number of radial points(nr)
     nr_bytes = f.read(4)
     nr = struct.unpack("<i", nr_bytes)[0] # little-endian integer
 
-    # Read number of l modes (num_l_modes)
+#Read number of l modes(num_l_modes)
     num_l_modes_bytes = f.read(4)
     num_l_modes = struct.unpack("<i", num_l_modes_bytes)[0]
 
-    # Read time
+#Read time
     time_bytes = f.read(8)
     time = struct.unpack("<d", time_bytes)[0] # little-endian double
 
-    # Read inner and outer radial boundaries (rin and rout)
+#Read inner and outer radial boundaries(rin and rout)
     rin_bytes = f.read(8)
     rin = struct.unpack("<d", rin_bytes)[0]
 
     rout_bytes = f.read(8)
     rout = struct.unpack("<d", rout_bytes)[0]
 
-    # Calculate the number of angular modes
+#Calculate the number of angular modes
     num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
 
-    # Total number of data points
+#Total number of data points
     count = 10 * nr * num_angular_modes
 
-    # Read data_real array
+#Read data_real array
     data_real = np.fromfile(f, dtype="<d", count=count) # little-endian double
     data_real = data_real.reshape((nr, 10, num_angular_modes))
 
-    # Read data_imag array
+#Read data_imag array
     data_imag = np.fromfile(f, dtype="<d", count=count)
     data_imag = data_imag.reshape((nr, 10, num_angular_modes))
 
-    # Create a mapping from index to (l, m)
-    index_to_lm = {}
+#Create a mapping from index to(l, m)
+    index_to_lm =
+{}
     for l in range(1, num_l_modes + 1):
       for m in range(-l, l + 1):
         index = l * l + l + m
@@ -474,7 +470,7 @@ def get_attribute(fpath: str,
   if type == "h5":
     attrs["file_type"] = "h5"
     with h5py.File(fpath, "r") as h5f:
-      # find attribute about num. of time level, and n,l,m in C_nlm
+#find attribute about num.of time level, and n, l, m in C_nlm
       attrs["lev_t"] = len(h5f.keys()) - 1
       if attrs["lev_t"] % 2 == 0: # for fourier transformation
         attrs["lev_t"] -= 1
@@ -483,7 +479,7 @@ def get_attribute(fpath: str,
       attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1
       attrs["r_in"] = h5f["metadata"].attrs["Rin"]
       attrs["r_out"] = h5f["metadata"].attrs["Rout"]
-      # read & save time
+#read &save time
       time = []
       for i in range(0, attrs["lev_t"]):
         key = f"{i}"
@@ -495,7 +491,7 @@ def get_attribute(fpath: str,
   else:
     raise ValueError("no such option")
 
-  # print(attrs)
+#print(attrs)
   return attrs
 
 
@@ -537,12 +533,14 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
   for n in range(len_n):
     for lm in range(len_lm):
       coeff = field[g_re, :, n, lm] + 1j * field[g_im, :, n, lm]
-      # F. transform
+#F.transform
       fft_coeff = np.fft.fft(coeff)
-      # if args["debug"] == 'y':
-      #  print("debug: normalization?",round(coeff[1],6) == round(np.fft.ifft(fft_coeff)[1],6))
+#if args["debug"] == 'y':
+#print(                                                                        \
+  "debug: normalization?",                                                     \
+  round(coeff[1], 6) == round(np.fft.ifft(fft_coeff)[1], 6))
 
-      # time derivative
+#time derivative
       half = len_t // 2 + 1
       omega = np.empty(shape=half)
       for i in range(0, half):
@@ -556,7 +554,7 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
       dfft_coeff[half:] = (np.imag(fft_coeff[half:]) -
                            1j * np.real(fft_coeff[half:])) * (omega[::-1][1:])
 
-      # not optimized version
+#not optimized version
       """
       dfft_coeff[0] = 0
       for i in range(1, half):
@@ -570,7 +568,7 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
         dfft_coeff[-i] = omega*complex(im2, -re2)
 
       """
-      # F. inverse
+#F.inverse
       coeff = np.fft.ifft(dfft_coeff)
       dfield[g_re, :, n, lm] = np.real(coeff)
       dfield[g_im, :, n, lm] = np.imag(coeff)
@@ -635,12 +633,12 @@ def radial_derivative_at_r_chebu(field: np.array,
   dx_dr = g_sign * 2 / (r_2 - r_1)
 
   if args["debug"] == "y":
-    # populate collocation points, roots of U_i
+#populate collocation points, roots of U_i
     x_i = np.empty(shape=len_n, dtype=float)
     for i in range(len_n):
       x_i[i] = math.cos(math.pi * (i + 1) / (len_n + 1))
 
-    # dU_k/dx|x=x_i
+#dU_k / dx | x = x_i
     duk_dx = np.empty(shape=(len_n, len_n), dtype=float)
     for k in range(len_n):
       for i in range(len_n):
@@ -697,13 +695,13 @@ def __init__(self, attrs: dict, args: dict):
     self.N = N = attrs["max_n"] # num. of coeffs or num of collocation pnts
     pi = math.pi
 
-    # U_N roots:
+#U_N roots:
     a_i = np.arange(0, N, dtype=int) + 1
     self.x_i = x_i = np.cos(math.pi * a_i / (N + 1))
-    # quadrature weights
+#quadrature weights
     self.w_i = (1 - np.square(x_i)) * pi / (N + 1)
 
-    # ChebU_j(x_i):
+#ChebU_j(x_i):
     Uj_xi = np.empty(shape=(N, N), dtype=float)
     for j in range(N):
       Uj_xi[j, :] = special.chebyu(j)(x_i)
@@ -717,7 +715,9 @@ def coefficients(self, field: np.array) -> np.array:
         use Gauss Quadrature to expand the given field in ChebU
         f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
         collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
-        x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
+        x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x !=
+{
+    1 or -1}
 
         """
 
@@ -738,19 +738,19 @@ def __debug(self):
     N = self.N + 1 # +1 to see the roots of U_N
     x_i = self.x_i
 
-    # populate funcs using bases themselves
+#populate funcs using bases themselves
     fs = []
     for n in range(N):
       f = special.chebyu(n)(x_i)
       fs.append((f, n))
 
-    # find coeffs
+#find coeffs
     cs = []
     for n in range(N):
       c = self.coefficients(fs[n][0])
       cs.append((c, fs[n][1]))
 
-    # expect to see only the n-th entry is 1 for the chebyu(n) of order n
+#expect to see only the n - th entry is 1 for the chebyu(n) of order n
     for n in range(N):
       order = cs[n][1]
       print(f"chebyu{order}, coeffs = {cs[n][0]}\n", flush=True)
@@ -828,7 +828,7 @@ def __init__(self, attrs: dict, args: dict):
       self.Uk = np.empty(shape=self.len_n)
       for k in range(self.len_n):
         self.Uk[k] = special.chebyu(k)(self.x)
-      # print("x,Uk",self.x,self.Uk)
+#print("x,Uk", self.x, self.Uk)
       self.interp = self.interpolate_at_r_chebu
     else:
       raise ValueError("no such option")
@@ -857,22 +857,22 @@ def process_field(field_name: str) -> dict:
     - interpolate at R=r
     """
 
-  # return
+#return
   attrs = g_attrs
   args = g_args
   db = {}
 
-  # load data
+#load data
   field = load(args["fpath"], field_name, attrs)
-  # db[f"{field_name}"] = field
+#db[f "{field_name}"] = field
 
-  # radial expansion
+#radial expansion
   field = radial_expansion(field, field_name, attrs, args)
 
-  # time derivative
+#time derivative
   dfield_dt = time_derivative(field, field_name, attrs, args)
 
-  # interpolate at a specific radii
+#interpolate at a specific radii
   interpolate = Interpolate_at_r(attrs, args)
   field_at_r = interpolate.interpolate(field, field_name)
   db[f"{field_name}|r"] = field_at_r
@@ -880,7 +880,7 @@ def process_field(field_name: str) -> dict:
   dfield_dt_at_r = interpolate.interpolate(dfield_dt, f"d{field_name}/dt")
   db[f"d{field_name}/dt|r"] = dfield_dt_at_r
 
-  # radial derivative at R=r
+#radial derivative at R = r
   dfield_dr_at_r = radial_derivative_at_r(field, field_name, attrs, args)
   db[f"d{field_name}/dr|r"] = dfield_dr_at_r
 
@@ -893,7 +893,7 @@ def h5_create_group(h5file, group_name: str):
     """
   h5group = None
 
-  # create group if not exists
+#create group if not exists
   if h5file.get(group_name, default=None) == None:
     h5group = h5file.create_group(group_name)
   else:
@@ -919,8 +919,8 @@ def h5_write_data(h5file,
     h5["gxx.dat"].attrs['Legend'] = the associated column =
       array(['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)', 'gxx_Re(1,1)', 'gxx_Im(1,1)', ...])
 
-    # => h5["gxx.dat"][3,0] = value of time at the dump level 3
-    # => h5["gxx.dat"][4,1] = value of gxx_Re(0,0) at the dump level 4
+#=> h5["gxx.dat"][3, 0] = value of time at the dump level 3
+#=> h5["gxx.dat"][4, 1] = value of gxx_Re(0, 0) at the dump level 4
 
     """
 
@@ -928,8 +928,8 @@ def h5_write_data(h5file,
       name=f"{data_name}",
       shape=(attrs["lev_t"], len([g_re, g_im]) * (attrs["max_l"]**2) + 1),
       dtype=float, # chunks=True,
-      # compression="gzip",
-      # shuffle=True,
+#compression = "gzip",
+#shuffle     = True,
   )
 
   data_attrs = ["time"]
@@ -1011,21 +1011,21 @@ def main(args):
   """
   g_sign = int(-1) if args["ftype"] == "bin" else int(1)
 
-  # check if output dir exist, if not, mkdir
+#check if output dir exist, if not, mkdir
   if not os.path.exists(g_args["d_out"]):
     os.makedirs(g_args["d_out"])
-  # find attribute for an arbitrary field
+#find attribute for an arbitrary field
   g_attrs = get_attribute(args["fpath"], type=args["ftype"])
 
-  # for each field
-  # I'm afraid this method takes too much memory
-  # from multiprocessing import Pool
-  # with Pool(processes=len(g_field_names)) as p:
-  #  db = p.map(process_field, g_field_names)
+#for each field
+#I'm afraid this method takes too much memory
+#from multiprocessing import Pool
+#with Pool(processes = len(g_field_names)) as p:
+#db = p.map(process_field, g_field_names)
 
   for f in g_field_names:
     db = process_field(f)
-    # write on disk
+#write on disk
     write(f, db, g_attrs, g_args)
 
 

From 3398bee1ef2c4bb652e216aac1b269cb7d0499b6 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 11:14:05 -0500
Subject: [PATCH 52/72] feat: linting.

---
 scripts/cce/athk_to_spectre.py | 172 ++++++++++++++++-----------------
 1 file changed, 81 insertions(+), 91 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index d4da2a10c..2de5ec93c 100644
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -1,4 +1,4 @@
-#!/ usr / bin / env python3
+#!/usr/bin/env python3
 ## Alireza Rashti - Oct 2024 (C)
 ## usage:
 ## $ ./me -h
@@ -19,11 +19,11 @@
 import glob
 from scipy.interpolate import interp1d
 
-#from itertools import product
-#import matplotlib.pyplot as plt
-#import glob
-#import sympy
-#import re
+# from itertools import product
+# import matplotlib.pyplot as plt
+# import glob
+# import sympy
+# import re
 
 ## ---------------------------------------------------------------------- ##
 
@@ -139,15 +139,15 @@ class AngularTransform:
 
   def __init__(self, attrs: dict):
     self.attrs = attrs
-#no.of collocation pnts
+    # no. of collocation pnts
     self.npnts = 2 * self.attrs["max_l"] + 1
     self.l_root = self._legendre_root()
-#theta collocation coords
+    # theta collocation coords
     self.th = self._theta_gauss_legendre()
-#phi collocation coords
+    # phi collocation coords
     self.ph = self._phi_equispace()
-#legendre roots
-#ylm(real / imag, th, ph, lm) on collocation coords
+    # legendre roots
+    # ylm(real/imag,th,ph,lm) on collocation coords
     self.ylm_pit = self._ylm_on_gl()
 
   def _theta_gauss_legendre(self):
@@ -162,8 +162,8 @@ def _theta_gauss_legendre(self):
 
   def _legendre_root(self):
     """
-      legendre roots
-    """
+        legendre roots
+        """
     nth = self.npnts
     x, _ = np.polynomial.legendre.leggauss(nth)
     x.sort()
@@ -174,7 +174,7 @@ def _phi_equispace(self):
         create equispace collocation pnts on phi
         """
     phi = np.linspace(0, 2 * np.pi, self.npnts, endpoint=False)
-    
+
     return phi
 
   def _ylm_on_gl(self):
@@ -212,16 +212,11 @@ def reconstruct_pit_on_gl(self, coeff: np.array):
     rylm = self.ylm_pit[g_re]
     iylm = self.ylm_pit[g_im]
 
-    shape = (
-        self.attrs["lev_t"],
-        self.attrs["max_n"],
-        self.npnts,
-        self.npnts,
-    )
-    field = (np.dot(coeff[g_re, :, :, :],rylm[:, :, np.newaxis, :, np.newaxis]) -
-             np.dot(coeff[g_im, :, :, :],iylm[:, :, np.newaxis, :, np.newaxis]))
-            
-    return field[:,:,:,:,0,0]
+    field = np.dot(
+        coeff[g_re, :, :, :], rylm[:, :, np.newaxis, :, np.newaxis]) - np.dot(
+            coeff[g_im, :, :, :], iylm[:, :, np.newaxis, :, np.newaxis])
+
+    return field[:, :, :, :, 0, 0]
 
   def _sp_expansion(self, f, L_max, N_theta, N_phi):
     """
@@ -242,24 +237,24 @@ def _sp_expansion(self, f, L_max, N_theta, N_phi):
             coeff :Expansion coefficients.
         """
 
-#Gauss - Legendre quadrature points and weights for theta
+    # Gauss-Legendre quadrature points and weights for theta
     x, w_theta = np.polynomial.legendre.leggauss(N_theta)
     theta = np.arccos(-x) # Convert from [-1,1] to [0,pi]
 
-#Uniform quadrature points for phi
+    # Uniform quadrature points for phi
     phi = np.linspace(0, 2 * np.pi, N_phi, endpoint=False)
 
-#Initialize coefficients
+    # Initialize coefficients
     coeff = np.zeros(shape=(self.attrs["max_lm"]), dtype=complex)
     dphi = 2 * np.pi / N_phi
 
-#Compute expansion coefficients
+    # Compute expansion coefficients
     for l in range(L_max + 1):
       for m in range(-l, l + 1):
         integral = 0.0
         for i, t in enumerate(theta):
           for j, p in enumerate(phi):
-#NOTE : there are some convention difference for theta and phi in scipy
+            # NOTE: there are some convention difference for theta and phi in scipy
             Ylm = special.sph_harm(m, l, p, t)
             integral += f[i, j] * np.conj(Ylm) * w_theta[i] * dphi
         coeff[lm_mode(l, m)] = integral # Store coefficient
@@ -280,7 +275,7 @@ def transform_field_to_coeff_gl(self, field: np.array):
         self.attrs["max_lm"],
     )
     coeff = np.empty(shape=shape, dtype=float)
-#TODO : broadcast
+    # TODO: broadcast
     for t in range(self.attrs["lev_t"]):
       for r in range(self.attrs["max_n"]):
         c = self._sp_expansion(field[t, r],
@@ -288,10 +283,10 @@ def transform_field_to_coeff_gl(self, field: np.array):
                                self.npnts,
                                self.npnts)
 
-#print(c, c.real, c.imag)
+        # print(c,c.real,c.imag)
         coeff[g_re, t, r, :] = c.real
         coeff[g_im, t, r, :] = c.imag
-        
+
     return coeff
 
   def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
@@ -300,7 +295,7 @@ def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
         which are equispace on phi and gauss legendre on theta.
         """
 
-#first reconstruct field on gl collocation points
+    # first reconstruct field on gl collocation points
     field = self.reconstruct_pit_on_gl(coeff)
 
     return self.transform_field_to_coeff_gl(field)
@@ -324,7 +319,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
 
     coords = AngularTransform(attrs)
     with h5py.File(fpath, "r") as h5f:
-#read &save
+      # read & save
       for i in range(0, lev_t):
         key = f"{i}"
         h5_re = h5f[f"{key}/{field_name}/re"]
@@ -332,11 +327,11 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         ret[g_re, i, :] = h5_re
         ret[g_im, i, :] = h5_im
 
-#transform from PITTNull coordinates to Spectre coordinates
+      # transform from PITTNull coordinates to Spectre coordinates
       ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
 
   elif attrs["file_type"] == "bin":
-#Load the list of files
+    # Load the list of files
     ##TODO: this depends on file name
     flist = sorted(glob.glob(fpath + "/cce_*.bin"))
     data = []
@@ -385,7 +380,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
   else:
     raise ValueError("no such option")
 
-#print(ret)
+  # print(ret)
   return ret
 
 
@@ -414,42 +409,41 @@ def read_cce_file(filename):
         index_to_lm (dict): Mapping from data index to (l, m) values.
     """
   with open(filename, "rb") as f:
-#Read number of radial points(nr)
+    # Read number of radial points (nr)
     nr_bytes = f.read(4)
     nr = struct.unpack("<i", nr_bytes)[0] # little-endian integer
 
-#Read number of l modes(num_l_modes)
+    # Read number of l modes (num_l_modes)
     num_l_modes_bytes = f.read(4)
     num_l_modes = struct.unpack("<i", num_l_modes_bytes)[0]
 
-#Read time
+    # Read time
     time_bytes = f.read(8)
     time = struct.unpack("<d", time_bytes)[0] # little-endian double
 
-#Read inner and outer radial boundaries(rin and rout)
+    # Read inner and outer radial boundaries (rin and rout)
     rin_bytes = f.read(8)
     rin = struct.unpack("<d", rin_bytes)[0]
 
     rout_bytes = f.read(8)
     rout = struct.unpack("<d", rout_bytes)[0]
 
-#Calculate the number of angular modes
+    # Calculate the number of angular modes
     num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
 
-#Total number of data points
+    # Total number of data points
     count = 10 * nr * num_angular_modes
 
-#Read data_real array
+    # Read data_real array
     data_real = np.fromfile(f, dtype="<d", count=count) # little-endian double
     data_real = data_real.reshape((nr, 10, num_angular_modes))
 
-#Read data_imag array
+    # Read data_imag array
     data_imag = np.fromfile(f, dtype="<d", count=count)
     data_imag = data_imag.reshape((nr, 10, num_angular_modes))
 
-#Create a mapping from index to(l, m)
-    index_to_lm =
-{}
+    # Create a mapping from index to (l, m)
+    index_to_lm = {}
     for l in range(1, num_l_modes + 1):
       for m in range(-l, l + 1):
         index = l * l + l + m
@@ -470,7 +464,7 @@ def get_attribute(fpath: str,
   if type == "h5":
     attrs["file_type"] = "h5"
     with h5py.File(fpath, "r") as h5f:
-#find attribute about num.of time level, and n, l, m in C_nlm
+      # find attribute about num. of time level, and n,l,m in C_nlm
       attrs["lev_t"] = len(h5f.keys()) - 1
       if attrs["lev_t"] % 2 == 0: # for fourier transformation
         attrs["lev_t"] -= 1
@@ -479,7 +473,7 @@ def get_attribute(fpath: str,
       attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1
       attrs["r_in"] = h5f["metadata"].attrs["Rin"]
       attrs["r_out"] = h5f["metadata"].attrs["Rout"]
-#read &save time
+      # read & save time
       time = []
       for i in range(0, attrs["lev_t"]):
         key = f"{i}"
@@ -491,7 +485,7 @@ def get_attribute(fpath: str,
   else:
     raise ValueError("no such option")
 
-#print(attrs)
+  # print(attrs)
   return attrs
 
 
@@ -533,14 +527,12 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
   for n in range(len_n):
     for lm in range(len_lm):
       coeff = field[g_re, :, n, lm] + 1j * field[g_im, :, n, lm]
-#F.transform
+      # F. transform
       fft_coeff = np.fft.fft(coeff)
-#if args["debug"] == 'y':
-#print(                                                                        \
-  "debug: normalization?",                                                     \
-  round(coeff[1], 6) == round(np.fft.ifft(fft_coeff)[1], 6))
+      # if args["debug"] == 'y':
+      #  print("debug: normalization?",round(coeff[1],6) == round(np.fft.ifft(fft_coeff)[1],6))
 
-#time derivative
+      # time derivative
       half = len_t // 2 + 1
       omega = np.empty(shape=half)
       for i in range(0, half):
@@ -554,7 +546,7 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
       dfft_coeff[half:] = (np.imag(fft_coeff[half:]) -
                            1j * np.real(fft_coeff[half:])) * (omega[::-1][1:])
 
-#not optimized version
+      # not optimized version
       """
       dfft_coeff[0] = 0
       for i in range(1, half):
@@ -568,7 +560,7 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
         dfft_coeff[-i] = omega*complex(im2, -re2)
 
       """
-#F.inverse
+      # F. inverse
       coeff = np.fft.ifft(dfft_coeff)
       dfield[g_re, :, n, lm] = np.real(coeff)
       dfield[g_im, :, n, lm] = np.imag(coeff)
@@ -633,12 +625,12 @@ def radial_derivative_at_r_chebu(field: np.array,
   dx_dr = g_sign * 2 / (r_2 - r_1)
 
   if args["debug"] == "y":
-#populate collocation points, roots of U_i
+    # populate collocation points, roots of U_i
     x_i = np.empty(shape=len_n, dtype=float)
     for i in range(len_n):
       x_i[i] = math.cos(math.pi * (i + 1) / (len_n + 1))
 
-#dU_k / dx | x = x_i
+    # dU_k/dx|x=x_i
     duk_dx = np.empty(shape=(len_n, len_n), dtype=float)
     for k in range(len_n):
       for i in range(len_n):
@@ -695,13 +687,13 @@ def __init__(self, attrs: dict, args: dict):
     self.N = N = attrs["max_n"] # num. of coeffs or num of collocation pnts
     pi = math.pi
 
-#U_N roots:
+    # U_N roots:
     a_i = np.arange(0, N, dtype=int) + 1
     self.x_i = x_i = np.cos(math.pi * a_i / (N + 1))
-#quadrature weights
+    # quadrature weights
     self.w_i = (1 - np.square(x_i)) * pi / (N + 1)
 
-#ChebU_j(x_i):
+    # ChebU_j(x_i):
     Uj_xi = np.empty(shape=(N, N), dtype=float)
     for j in range(N):
       Uj_xi[j, :] = special.chebyu(j)(x_i)
@@ -715,9 +707,7 @@ def coefficients(self, field: np.array) -> np.array:
         use Gauss Quadrature to expand the given field in ChebU
         f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
         collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
-        x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x !=
-{
-    1 or -1}
+        x = g_sign*(2*r - r_1 - r_2)/(r_2 - r_1), notes: x != {1 or -1}
 
         """
 
@@ -738,19 +728,19 @@ def __debug(self):
     N = self.N + 1 # +1 to see the roots of U_N
     x_i = self.x_i
 
-#populate funcs using bases themselves
+    # populate funcs using bases themselves
     fs = []
     for n in range(N):
       f = special.chebyu(n)(x_i)
       fs.append((f, n))
 
-#find coeffs
+    # find coeffs
     cs = []
     for n in range(N):
       c = self.coefficients(fs[n][0])
       cs.append((c, fs[n][1]))
 
-#expect to see only the n - th entry is 1 for the chebyu(n) of order n
+    # expect to see only the n-th entry is 1 for the chebyu(n) of order n
     for n in range(N):
       order = cs[n][1]
       print(f"chebyu{order}, coeffs = {cs[n][0]}\n", flush=True)
@@ -828,7 +818,7 @@ def __init__(self, attrs: dict, args: dict):
       self.Uk = np.empty(shape=self.len_n)
       for k in range(self.len_n):
         self.Uk[k] = special.chebyu(k)(self.x)
-#print("x,Uk", self.x, self.Uk)
+      # print("x,Uk",self.x,self.Uk)
       self.interp = self.interpolate_at_r_chebu
     else:
       raise ValueError("no such option")
@@ -857,22 +847,22 @@ def process_field(field_name: str) -> dict:
     - interpolate at R=r
     """
 
-#return
+  # return
   attrs = g_attrs
   args = g_args
   db = {}
 
-#load data
+  # load data
   field = load(args["fpath"], field_name, attrs)
-#db[f "{field_name}"] = field
+  # db[f"{field_name}"] = field
 
-#radial expansion
+  # radial expansion
   field = radial_expansion(field, field_name, attrs, args)
 
-#time derivative
+  # time derivative
   dfield_dt = time_derivative(field, field_name, attrs, args)
 
-#interpolate at a specific radii
+  # interpolate at a specific radii
   interpolate = Interpolate_at_r(attrs, args)
   field_at_r = interpolate.interpolate(field, field_name)
   db[f"{field_name}|r"] = field_at_r
@@ -880,7 +870,7 @@ def process_field(field_name: str) -> dict:
   dfield_dt_at_r = interpolate.interpolate(dfield_dt, f"d{field_name}/dt")
   db[f"d{field_name}/dt|r"] = dfield_dt_at_r
 
-#radial derivative at R = r
+  # radial derivative at R=r
   dfield_dr_at_r = radial_derivative_at_r(field, field_name, attrs, args)
   db[f"d{field_name}/dr|r"] = dfield_dr_at_r
 
@@ -893,7 +883,7 @@ def h5_create_group(h5file, group_name: str):
     """
   h5group = None
 
-#create group if not exists
+  # create group if not exists
   if h5file.get(group_name, default=None) == None:
     h5group = h5file.create_group(group_name)
   else:
@@ -919,8 +909,8 @@ def h5_write_data(h5file,
     h5["gxx.dat"].attrs['Legend'] = the associated column =
       array(['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)', 'gxx_Re(1,1)', 'gxx_Im(1,1)', ...])
 
-#=> h5["gxx.dat"][3, 0] = value of time at the dump level 3
-#=> h5["gxx.dat"][4, 1] = value of gxx_Re(0, 0) at the dump level 4
+    # => h5["gxx.dat"][3,0] = value of time at the dump level 3
+    # => h5["gxx.dat"][4,1] = value of gxx_Re(0,0) at the dump level 4
 
     """
 
@@ -928,8 +918,8 @@ def h5_write_data(h5file,
       name=f"{data_name}",
       shape=(attrs["lev_t"], len([g_re, g_im]) * (attrs["max_l"]**2) + 1),
       dtype=float, # chunks=True,
-#compression = "gzip",
-#shuffle     = True,
+      # compression="gzip",
+      # shuffle=True,
   )
 
   data_attrs = ["time"]
@@ -1011,21 +1001,21 @@ def main(args):
   """
   g_sign = int(-1) if args["ftype"] == "bin" else int(1)
 
-#check if output dir exist, if not, mkdir
+  # check if output dir exist, if not, mkdir
   if not os.path.exists(g_args["d_out"]):
     os.makedirs(g_args["d_out"])
-#find attribute for an arbitrary field
+  # find attribute for an arbitrary field
   g_attrs = get_attribute(args["fpath"], type=args["ftype"])
 
-#for each field
-#I'm afraid this method takes too much memory
-#from multiprocessing import Pool
-#with Pool(processes = len(g_field_names)) as p:
-#db = p.map(process_field, g_field_names)
+  # for each field
+  # I'm afraid this method takes too much memory
+  # from multiprocessing import Pool
+  # with Pool(processes=len(g_field_names)) as p:
+  #  db = p.map(process_field, g_field_names)
 
   for f in g_field_names:
     db = process_field(f)
-#write on disk
+    # write on disk
     write(f, db, g_attrs, g_args)
 
 

From acee1cbf997682b8a45bcd70051923c82df7f384 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 12:38:15 -0500
Subject: [PATCH 53/72] feat: broadcast.

---
 scripts/cce/athk_to_spectre.py | 37 ++++++++--------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)
 mode change 100644 => 100755 scripts/cce/athk_to_spectre.py

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
old mode 100644
new mode 100755
index 2de5ec93c..66e4fce8b
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -218,7 +218,7 @@ def reconstruct_pit_on_gl(self, coeff: np.array):
 
     return field[:, :, :, :, 0, 0]
 
-  def _sp_expansion(self, f, L_max, N_theta, N_phi):
+  def _sp_expansion(self, f):
     """
         Expands function f(theta, phi) in spherical harmonics up to degree L_max
         using Gauss-Legendre quadrature in theta and uniform quadrature in phi.
@@ -226,39 +226,18 @@ def _sp_expansion(self, f, L_max, N_theta, N_phi):
         Parameters:
             f : function [theta, phi] -> float
                 Function to expand.
-            L_max : int
-                Maximum degree of spherical harmonics.
-            N_theta : int
-                Number of Gauss-Legendre points in theta.
-            N_phi : int
-                Number of equispaced points in phi.
-
         Returns:
             coeff :Expansion coefficients.
         """
 
     # Gauss-Legendre quadrature points and weights for theta
-    x, w_theta = np.polynomial.legendre.leggauss(N_theta)
-    theta = np.arccos(-x) # Convert from [-1,1] to [0,pi]
-
-    # Uniform quadrature points for phi
-    phi = np.linspace(0, 2 * np.pi, N_phi, endpoint=False)
-
-    # Initialize coefficients
-    coeff = np.zeros(shape=(self.attrs["max_lm"]), dtype=complex)
+    _, w_theta = np.polynomial.legendre.leggauss(N_theta)
     dphi = 2 * np.pi / N_phi
-
-    # Compute expansion coefficients
-    for l in range(L_max + 1):
-      for m in range(-l, l + 1):
-        integral = 0.0
-        for i, t in enumerate(theta):
-          for j, p in enumerate(phi):
-            # NOTE: there are some convention difference for theta and phi in scipy
-            Ylm = special.sph_harm(m, l, p, t)
-            integral += f[i, j] * np.conj(Ylm) * w_theta[i] * dphi
-        coeff[lm_mode(l, m)] = integral # Store coefficient
-
+    Ylm = self.ylm_pit[g_re,...] + 1j*self.ylm_pit[g_im,...]
+    
+    coeff = np.einsum('ij,ijk,i',f,np.conj(Ylm),w_theta)
+    coeff *= dphi
+    
     return coeff
 
   def transform_field_to_coeff_gl(self, field: np.array):
@@ -278,7 +257,7 @@ def transform_field_to_coeff_gl(self, field: np.array):
     # TODO: broadcast
     for t in range(self.attrs["lev_t"]):
       for r in range(self.attrs["max_n"]):
-        c = self._sp_expansion(field[t, r],
+        c = self._sp_expansion(field[t, r])
                                self.attrs["max_l"],
                                self.npnts,
                                self.npnts)

From 29c8e908a8c37430d9597cc6a1a3ac1d2c69a606 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 12:49:02 -0500
Subject: [PATCH 54/72] fix: many bugs feat: broadcasting

---
 scripts/cce/athk_to_spectre.py | 35 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 66e4fce8b..cf53c2fd1 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -231,13 +231,15 @@ def _sp_expansion(self, f):
         """
 
     # Gauss-Legendre quadrature points and weights for theta
-    _, w_theta = np.polynomial.legendre.leggauss(N_theta)
-    dphi = 2 * np.pi / N_phi
-    Ylm = self.ylm_pit[g_re,...] + 1j*self.ylm_pit[g_im,...]
-    
-    coeff = np.einsum('ij,ijk,i',f,np.conj(Ylm),w_theta)
+    _, w_theta = np.polynomial.legendre.leggauss(self.npnts)
+    dphi = 2 * np.pi / self.npnts
+    Ylm = self.ylm_pit[g_re, ...] + 1j * self.ylm_pit[g_im, ...]
+
+    coeff = np.einsum("trij,ijk,i", f, np.conj(Ylm), w_theta)
     coeff *= dphi
-    
+
+    coeff = np.transpose(coeff, (2, 1, 0))
+
     return coeff
 
   def transform_field_to_coeff_gl(self, field: np.array):
@@ -254,17 +256,14 @@ def transform_field_to_coeff_gl(self, field: np.array):
         self.attrs["max_lm"],
     )
     coeff = np.empty(shape=shape, dtype=float)
-    # TODO: broadcast
-    for t in range(self.attrs["lev_t"]):
-      for r in range(self.attrs["max_n"]):
-        c = self._sp_expansion(field[t, r])
-                               self.attrs["max_l"],
-                               self.npnts,
-                               self.npnts)
-
-        # print(c,c.real,c.imag)
-        coeff[g_re, t, r, :] = c.real
-        coeff[g_im, t, r, :] = c.imag
+    c = self._sp_expansion(field)
+
+    # print(c,c.real,c.imag)
+    print("c.shape = ", c.shape)
+    print("coeff.shape = ", coeff.shape)
+
+    coeff[g_re, ...] = c.real
+    coeff[g_im, ...] = c.imag
 
     return coeff
 
@@ -791,7 +790,7 @@ def __init__(self, attrs: dict, args: dict):
     self.r = r = args["radius"]
     self.x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
 
-    assert -1 < self.x < 1
+    assert -1 < self.x < 1, f"x = {self.x}"
 
     if args["interpolation"] == "ChebU":
       self.Uk = np.empty(shape=self.len_n)

From ea9495c31ba256950b6236c8d1987f4aa23e9e43 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 17:14:28 -0500
Subject: [PATCH 55/72] add: more test scripts.

---
 scripts/cce/cce_run.sh               | 10 +++++++
 scripts/cce/debug_athk_to_spectre.py |  2 +-
 scripts/cce/pre_run.sh               | 41 ++++++++++++++++++++++++----
 3 files changed, 47 insertions(+), 6 deletions(-)

diff --git a/scripts/cce/cce_run.sh b/scripts/cce/cce_run.sh
index 919bf90e7..313f63467 100755
--- a/scripts/cce/cce_run.sh
+++ b/scripts/cce/cce_run.sh
@@ -29,5 +29,15 @@ echo "running preproc...\n"
 echo "-------------------------"
 echo "running CCE..."
 ./CharacteristicExtract --input-file cce_par_q2_h5.yaml
+
+# q2 dev
+elif false
+then
+cd runs
+echo "running preproc...\n"
+./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_q2_dev_h5.yaml
+echo "-------------------------"
+echo "running CCE..."
+./CharacteristicExtract --input-file cce_par_q2_dev_h5.yaml
 fi
 
diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index b56cf1800..c007e3c1b 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -69,7 +69,7 @@ def parse_cli():
   p.add_argument(
       "-time_dump",
       type=int,
-      default=100,
+      default=1000,
       help="how often dump for mode convergence",
   )
 
diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index 1941f639b..e646c89a0 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -1,13 +1,17 @@
 #!/usr/bin/env bash
 
+set -x
+
 # bin
-if true
+if false
 then
 # create
 dout='./debug/dev'
 file='./dat/bin/cce_bbh/'
 fdebug='./debug/dev/CceR0030.00.h5'
 
+rm -vf ${dout}/*
+
 ./athk_to_spectre.py -ftype bin -d_out "$dout" -debug y \
 -t_deriv fin_diff -r_deriv ChebU -interpolation ChebU \
 -radius 30 \
@@ -20,13 +24,15 @@ mode="Re(2,2)"
 	-fpath $fdebug  -field_name $fname -field_mode $mode
 
 # h5 bbh, q=2
-elif false
+elif true
 then
-./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 \
--ftype h5 -d_out ./debug/q2/ -radius 100
-
 file="./debug/q2/CceR0100.00.h5"
 dout='./debug/q2'
+mkdir -p ${dout}
+rm -rvf ${dout}/*
+
+./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 \
+-ftype h5 -d_out "${dout}" -radius 100
 
 # plot fig:
 fname="gxx"
@@ -75,5 +81,30 @@ mode="Re(2,2)"
 ./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
 	-fpath $file  -field_name $fname -field_mode $mode
 
+elif false
+then
+file="./debug/q2_dev/CceR0100.00.h5"
+dout='./debug/q2_dev/'
+mkdir -p ${dout}
+
+./athk_to_spectre.py -fpath dat/bbh_q2.0_chizp0.0_chizm0.0_d10.0_lev13_n128_fixed_cce_decomp_shell_3.h5 \
+-ftype h5 -d_out "$dout" -radius 100
+
+
+# plot fig:
+fname="gxx"
+mode="Re(2,2)"
+./debug_athk_to_spectre.py -debug plot_simple -dout "${dout}" \
+	-fpath $file  -field_name $fname -field_mode $mode
+
+# h5 quick development
+elif false
+then
+dout='./debug/quick_dev/'
+mkdir -p ${dout}
+
+./athk_to_spectre.py -fpath dat/backup_dilute_d32_fixed_cce_decomp_shell_4.h5 \
+-ftype h5 -d_out "$dout" -radius 70
+
 fi
 

From 3433986d90a1dce0fa80292d4087e5204a240f65 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 10 Feb 2025 17:39:37 -0500
Subject: [PATCH 56/72] feat: more optimized.

---
 scripts/cce/athk_to_spectre.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index cf53c2fd1..cf979bac5 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -235,10 +235,10 @@ def _sp_expansion(self, f):
     dphi = 2 * np.pi / self.npnts
     Ylm = self.ylm_pit[g_re, ...] + 1j * self.ylm_pit[g_im, ...]
 
-    coeff = np.einsum("trij,ijk,i", f, np.conj(Ylm), w_theta)
+    coeff = np.einsum("trij,ijk,i->trk", f, np.conj(Ylm), w_theta, optimize=True)
     coeff *= dphi
 
-    coeff = np.transpose(coeff, (2, 1, 0))
+    #coeff = np.transpose(coeff, (2, 1, 0))
 
     return coeff
 
@@ -258,10 +258,6 @@ def transform_field_to_coeff_gl(self, field: np.array):
     coeff = np.empty(shape=shape, dtype=float)
     c = self._sp_expansion(field)
 
-    # print(c,c.real,c.imag)
-    print("c.shape = ", c.shape)
-    print("coeff.shape = ", coeff.shape)
-
     coeff[g_re, ...] = c.real
     coeff[g_im, ...] = c.imag
 
@@ -306,6 +302,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         ret[g_im, i, :] = h5_im
 
       # transform from PITTNull coordinates to Spectre coordinates
+      print(f"transforming {field_name} from pitt to spectre",flush=True)
       ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
 
   elif attrs["file_type"] == "bin":

From 1cec746932b2e488d399eb846c00b8260aafd26e Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Tue, 11 Feb 2025 07:50:34 -0500
Subject: [PATCH 57/72] feat: add a nan checker.

---
 scripts/cce/athk_to_spectre.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index cf979bac5..fad787b42 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -304,6 +304,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
       # transform from PITTNull coordinates to Spectre coordinates
       print(f"transforming {field_name} from pitt to spectre",flush=True)
       ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
+      assert not np.any(np.isnan(ret)), f"{field_name} got nan!"
 
   elif attrs["file_type"] == "bin":
     # Load the list of files

From 1e23dbb6bead3836f2d950ea9df56806d4806041 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Tue, 11 Feb 2025 07:50:59 -0500
Subject: [PATCH 58/72] chore: update scripts.

---
 scripts/cce/cce_run.sh | 6 +++---
 scripts/cce/pre_run.sh | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/scripts/cce/cce_run.sh b/scripts/cce/cce_run.sh
index 313f63467..de7b668bb 100755
--- a/scripts/cce/cce_run.sh
+++ b/scripts/cce/cce_run.sh
@@ -10,8 +10,8 @@ echo "-------------------------"
 echo "running CCE..."
 ./CharacteristicExtract --input-file cce_par_bin.yaml
 
-# h5 test
-elif false
+# h5 quick dev
+elif true
 then
 cd runs
 echo "running preproc...\n"
@@ -21,7 +21,7 @@ echo "running CCE..."
 ./CharacteristicExtract --input-file cce_par_h5.yaml
 
 # q2
-elif true
+elif false
 then
 cd runs
 echo "running preproc...\n"
diff --git a/scripts/cce/pre_run.sh b/scripts/cce/pre_run.sh
index e646c89a0..07af380ee 100755
--- a/scripts/cce/pre_run.sh
+++ b/scripts/cce/pre_run.sh
@@ -24,7 +24,7 @@ mode="Re(2,2)"
 	-fpath $fdebug  -field_name $fname -field_mode $mode
 
 # h5 bbh, q=2
-elif true
+elif false
 then
 file="./debug/q2/CceR0100.00.h5"
 dout='./debug/q2'
@@ -98,9 +98,10 @@ mode="Re(2,2)"
 	-fpath $file  -field_name $fname -field_mode $mode
 
 # h5 quick development
-elif false
+elif true
 then
-dout='./debug/quick_dev/'
+dout='./debug/quick_dev'
+rm -rfv "${dout}"
 mkdir -p ${dout}
 
 ./athk_to_spectre.py -fpath dat/backup_dilute_d32_fixed_cce_decomp_shell_4.h5 \

From 7d7b0cb6cc04609d26da1a0eaa371ceafe69a678 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Tue, 11 Feb 2025 09:15:47 -0500
Subject: [PATCH 59/72] chore: rm plan.txt

---
 scripts/cce/plan.txt | 66 --------------------------------------------
 1 file changed, 66 deletions(-)
 delete mode 100644 scripts/cce/plan.txt

diff --git a/scripts/cce/plan.txt b/scripts/cce/plan.txt
deleted file mode 100644
index 01dc1a484..000000000
--- a/scripts/cce/plan.txt
+++ /dev/null
@@ -1,66 +0,0 @@
-#define Max(a_,b_) ((a_)>(b_)? (a_):(b_))
-#define Min(a_,b_) ((a_)<(b_)? (a_):(b_))
-#define FourPi 12.566370614359172953850573533118
-#define ABS(x_) ((x_)>0 ? (x_) : (-(x_)))
-
-static long double fact(int i)
-{
-  int j;
-  long double temp = 1;
-
-  myassert(i>-1);
-
-  for (j=2; j <=i; j++)
-  {
-    temp *=j;
-  }
-  return temp;
-}
-
-double sPlm(int s, int l, int m, double theta)
-{
-  int k;
-  double temp = 0;
-  s= -s;  /* oops below is really the definition for
-            -s not s */
-
-  myassert(l>=ABS(s));
-  myassert(l>=ABS(m));
-
-  const double sc = (1-2*(ABS(s)%2))*
-        sqrt((2*l+1)/(FourPi)*fact(l+m)*fact(l-m)*fact(l+s)*fact(l-s));
-
-  for (k= Max(0, m-s); k <= Min(l+m, l-s); k++)
-  {
-    double t = sc;
-
-    t/= fact(l+m-k);
-    t/= fact(l-s-k);
-    t/= fact(k);
-    t/= fact(k+s-m);
-
-    temp += (1-2*(ABS(k)%2))*t*
-       pow(cos(0.5*theta),(double)(2*l+m-s-2*k))*
-       pow(sin(0.5*theta),(double)(2*k+s-m));
-  }
-
-  return temp;
-}
-
-complex <double> sYlm (int s, int l, int m, double theta, double phi)
-{
-  const complex<double> I(0,1);
-  return sPlm(s,l,m,theta)*exp(I*(complex<double>)(m*phi));
-}
-
-
-
-## notes:
-. cheb recons: for n 0...nmax-1
-  tmp[time][angle] += reall[]*U[n];
-
-. reconstrict g(th,ph) at Gauss-Legendre in the and equally spaced in ph:
-SHRE_gij(th,phi) |_r=r_0 += (RC(lm) * reYN(th,ph,lm) - IC(lm) * imYN(th,ph,lm))
-
-. don't forget about factor in your paper
-see also: pittnullcode/SphericalHarmonicReconGen/src/get.cc, line 51

From d2bd99266918d86e29e9e5c6dc40567628dc45df Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Tue, 11 Feb 2025 12:45:04 -0500
Subject: [PATCH 60/72] fix: fix max_l inclusive bug.

---
 scripts/cce/athk_to_spectre.py | 36 ++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index fad787b42..a200c6b93 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -149,6 +149,10 @@ def __init__(self, attrs: dict):
     # legendre roots
     # ylm(real/imag,th,ph,lm) on collocation coords
     self.ylm_pit = self._ylm_on_gl()
+    # assert not np.any(np.isnan(self.th))
+    # assert not np.any(np.isnan(self.ph))
+    # assert not np.any(np.isnan(self.ylm_pit))
+    # assert not np.any(np.isnan(self.l_root))
 
   def _theta_gauss_legendre(self):
     """
@@ -196,10 +200,11 @@ def _ylm_on_gl(self):
       th = self.th[i]
       for j in range(self.npnts):
         ph = self.ph[j]
-        for l in range(0, self.attrs["max_l"]):
+        for l in range(0, self.attrs["max_l"] + 1):
           for m in range(l, -l - 1, -1):
             lm = lm_mode(l, m)
             y = special.sph_harm(m, l, ph, th)
+            # assert not np.isnan(y)
             ylms[g_re, i, j, lm] = y.real
             ylms[g_im, i, j, lm] = y.imag
 
@@ -235,10 +240,14 @@ def _sp_expansion(self, f):
     dphi = 2 * np.pi / self.npnts
     Ylm = self.ylm_pit[g_re, ...] + 1j * self.ylm_pit[g_im, ...]
 
-    coeff = np.einsum("trij,ijk,i->trk", f, np.conj(Ylm), w_theta, optimize=True)
+    coeff = np.einsum("trij,ijk,i->trk",
+                      f,
+                      np.conj(Ylm),
+                      w_theta,
+                      optimize=True)
     coeff *= dphi
 
-    #coeff = np.transpose(coeff, (2, 1, 0))
+    # coeff = np.transpose(coeff, (2, 1, 0))
 
     return coeff
 
@@ -291,8 +300,8 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
     shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
     ret = np.empty(shape=shape, dtype=float)
 
-    coords = AngularTransform(attrs)
     with h5py.File(fpath, "r") as h5f:
+      coords = AngularTransform(attrs)
       # read & save
       for i in range(0, lev_t):
         key = f"{i}"
@@ -302,9 +311,10 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         ret[g_im, i, :] = h5_im
 
       # transform from PITTNull coordinates to Spectre coordinates
-      print(f"transforming {field_name} from pitt to spectre",flush=True)
+      # assert not np.any(np.isnan(ret)), f"{field_name} has nans before transf.!"
+      print(f"transforming {field_name} from pitt to spectre", flush=True)
       ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
-      assert not np.any(np.isnan(ret)), f"{field_name} got nan!"
+      # assert not np.any(np.isnan(ret)), f"{field_name} got nans after transf.!"
 
   elif attrs["file_type"] == "bin":
     # Load the list of files
@@ -446,7 +456,7 @@ def get_attribute(fpath: str,
         attrs["lev_t"] -= 1
 
       attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
-      attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1
+      attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1 # inclusive l
       attrs["r_in"] = h5f["metadata"].attrs["Rin"]
       attrs["r_out"] = h5f["metadata"].attrs["Rout"]
       # read & save time
@@ -892,7 +902,10 @@ def h5_write_data(h5file,
 
   dataset_conf = dict(
       name=f"{data_name}",
-      shape=(attrs["lev_t"], len([g_re, g_im]) * (attrs["max_l"]**2) + 1),
+      shape=(
+          attrs["lev_t"],
+          len([g_re, g_im]) * (attrs["max_l"] + 1)**2 + 1,
+      ),
       dtype=float, # chunks=True,
       # compression="gzip",
       # shuffle=True,
@@ -908,11 +921,10 @@ def h5_write_data(h5file,
   flat = 0
   h5file[f"{data_name}"][:, flat] = attrs["time"]
   flat += 1
-  for l in range(0, attrs["max_l"]):
+  for l in range(0, attrs["max_l"] + 1):
     for m in range(l, -l - 1, -1):
-      if args["debug"] == "y":
-        assert all(data[g_re, :, lm_mode(l, m)] != np.nan)
-        assert all(data[g_im, :, lm_mode(l, m)] != np.nan)
+      assert not np.any(np.isnan(data[g_re, :, lm_mode(l, m)]))
+      assert not np.any(np.isnan(data[g_im, :, lm_mode(l, m)]))
 
       data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
       data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")

From 51873ed45407e3161d002bfffc8bdabe7c6a46f5 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Tue, 11 Feb 2025 12:45:42 -0500
Subject: [PATCH 61/72] chore: comments about max_l.

---
 scripts/cce/athk_to_spectre.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index a200c6b93..0a52d3cf5 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -456,7 +456,7 @@ def get_attribute(fpath: str,
         attrs["lev_t"] -= 1
 
       attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
-      attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1 # inclusive l
+      attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1 # NOTE:l must be inclusive in loops
       attrs["r_in"] = h5f["metadata"].attrs["Rin"]
       attrs["r_out"] = h5f["metadata"].attrs["Rout"]
       # read & save time
@@ -904,7 +904,7 @@ def h5_write_data(h5file,
       name=f"{data_name}",
       shape=(
           attrs["lev_t"],
-          len([g_re, g_im]) * (attrs["max_l"] + 1)**2 + 1,
+          len([g_re, g_im]) * (attrs["max_l"] + 1)**2 + 1, ## the last +1 for time
       ),
       dtype=float, # chunks=True,
       # compression="gzip",

From 9fd8b6d5888f1a9b74a50f225c0012acb93297e3 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 12 Feb 2025 16:02:47 -0500
Subject: [PATCH 62/72] fix: minor bugs.

---
 scripts/cce/athk_to_spectre.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 0a52d3cf5..6497f8ead 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -307,8 +307,8 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
         key = f"{i}"
         h5_re = h5f[f"{key}/{field_name}/re"]
         h5_im = h5f[f"{key}/{field_name}/im"]
-        ret[g_re, i, :] = h5_re
-        ret[g_im, i, :] = h5_im
+        ret[g_re, i, ...] = h5_re[:, 0:max_lm]
+        ret[g_im, i, ...] = h5_im[:, 0:max_lm]
 
       # transform from PITTNull coordinates to Spectre coordinates
       # assert not np.any(np.isnan(ret)), f"{field_name} has nans before transf.!"
@@ -348,7 +348,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
     attrs["lev_t"] = len(flist)
     attrs["max_n"] = nr
     attrs["max_l"] = num_l_modes
-    attrs["max_lm"] = (num_l_modes + 1)**2
+    attrs["max_lm"] = int((num_l_modes + 1)**2)
     attrs["r_in"] = np.array([rin])
     attrs["r_out"] = np.array([rout])
     attrs["time"] = np.linspace(t.min(), t.max(), t.shape[0])
@@ -456,7 +456,9 @@ def get_attribute(fpath: str,
         attrs["lev_t"] -= 1
 
       attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
-      attrs["max_l"] = int(math.sqrt(attrs["max_lm"])) - 1 # NOTE:l must be inclusive in loops
+      attrs["max_l"] = (int(math.sqrt(attrs["max_lm"])) - 1
+                        ) # NOTE:l must be inclusive in loops
+      attrs["max_lm"] = int((attrs["max_l"] + 1)**2)
       attrs["r_in"] = h5f["metadata"].attrs["Rin"]
       attrs["r_out"] = h5f["metadata"].attrs["Rout"]
       # read & save time
@@ -904,7 +906,8 @@ def h5_write_data(h5file,
       name=f"{data_name}",
       shape=(
           attrs["lev_t"],
-          len([g_re, g_im]) * (attrs["max_l"] + 1)**2 + 1, ## the last +1 for time
+          len([g_re, g_im]) * (attrs["max_l"] + 1)**2 +
+          1, ## the last +1 for time
       ),
       dtype=float, # chunks=True,
       # compression="gzip",

From ccb6e8a0e15e485e67f506c71ac861f2f66aeefd Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 12 Feb 2025 17:52:31 -0500
Subject: [PATCH 63/72] add: bms transformation.

---
 scripts/cce/BMS_tranformation.ipynb | 103 ++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 scripts/cce/BMS_tranformation.ipynb

diff --git a/scripts/cce/BMS_tranformation.ipynb b/scripts/cce/BMS_tranformation.ipynb
new file mode 100644
index 000000000..96350b4ba
--- /dev/null
+++ b/scripts/cce/BMS_tranformation.ipynb
@@ -0,0 +1,103 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0491f9aa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import scri\n",
+    "import matplotlib.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d45f67e0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "w = scri.WaveformModes()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fbbd0dea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "abd = scri.create_abd_from_h5(\n",
+    "      file_name=\"./CharacteristicExtractReduction_10_8.h5\",\n",
+    "      file_format=\"spectrecce_v1\",\n",
+    "      # ch_mass=1.0,  # Optional; helpful if known\n",
+    "      # t_interpolate=t_worldtube,  # Optional; for some specified values of `t_worldtube`\n",
+    "      t_0_superrest=1600,\n",
+    "      padding_time=200\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d82ee9c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "h = abd.h"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe2397e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.plot(h.t, h.data[:, h.index(2,2)])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a8fef4fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69149c1c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "help(scri.create_abd_from_h5)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "nr_misc",
+   "language": "python",
+   "name": "nr_misc"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

From 8ecdcb791a4c5be4614d5ee87406a1472da869f1 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 17 Feb 2025 06:52:45 -0500
Subject: [PATCH 64/72] fix: minor comment.

---
 scripts/cce/cce_extract_txt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cce/cce_extract_txt.py b/scripts/cce/cce_extract_txt.py
index 93c13fdc2..3904f913f 100755
--- a/scripts/cce/cce_extract_txt.py
+++ b/scripts/cce/cce_extract_txt.py
@@ -42,7 +42,7 @@ def parse_cli():
       "-mode",
       type=str,
       default="Y_2,2",
-      help='field modes:["Y_2,-2","Y_2,-2", ...]',
+      help='field modes:["Y_2,2","Y_2,-2", ...]',
   )
   args = p.parse_args()
   return args

From f3936aff0b8da1011cc8fc41c35ac36c4f32788b Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Mon, 17 Feb 2025 09:24:39 -0500
Subject: [PATCH 65/72] fix: bms transformation script.

---
 scripts/cce/BMS_tranformation.ipynb | 31 +++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/scripts/cce/BMS_tranformation.ipynb b/scripts/cce/BMS_tranformation.ipynb
index 96350b4ba..6a11268a0 100644
--- a/scripts/cce/BMS_tranformation.ipynb
+++ b/scripts/cce/BMS_tranformation.ipynb
@@ -7,8 +7,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# https://scri.readthedocs.io/en/latest/tutorial_abd.html#loading-cce-data-and-adjusting-the-bms-frame\n",
     "import scri\n",
-    "import matplotlib.pyplot as plt"
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np"
    ]
   },
   {
@@ -29,11 +31,11 @@
    "outputs": [],
    "source": [
     "abd = scri.create_abd_from_h5(\n",
-    "      file_name=\"./CharacteristicExtractReduction_10_8.h5\",\n",
+    "      file_name=\"debug/q2/CharacteristicExtractReduction_10_8.h5\",\n",
     "      file_format=\"spectrecce_v1\",\n",
-    "      # ch_mass=1.0,  # Optional; helpful if known\n",
+    "      ch_mass=1.0,  # Optional; helpful if known\n",
     "      # t_interpolate=t_worldtube,  # Optional; for some specified values of `t_worldtube`\n",
-    "      t_0_superrest=1600,\n",
+    "      t_0_superrest=500,\n",
     "      padding_time=200\n",
     "    )"
    ]
@@ -61,21 +63,34 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "a8fef4fa",
+   "id": "6bf07394",
    "metadata": {},
    "outputs": [],
    "source": [
-    "plt.show()"
+    "re = h.data[:,h.index(2,2)].real\n",
+    "im = h.data[:,h.index(2,2)].imag\n",
+    "am = np.sqrt(np.square(re)+np.square(im))"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "69149c1c",
+   "id": "dafbd8a1",
    "metadata": {},
    "outputs": [],
    "source": [
-    "help(scri.create_abd_from_h5)"
+    "plt.plot(h.t,am)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "66a3fd9e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dout = np.column_stack((h.t,re,im,am))\n",
+    "np.savetxt(\"./bms_h22.txt\",dout,header=\"# t re im am\")"
    ]
   }
  ],

From ce8f1ac494a176a04a402997cc4062e1c151a700 Mon Sep 17 00:00:00 2001
From: Alireza <rashti.alireza@gmail.com>
Date: Wed, 19 Feb 2025 16:52:45 -0500
Subject: [PATCH 66/72] minors.

---
 scripts/cce/Untitled.ipynb | 33 +++++++++++++++++++++++++++++++++
 scripts/cce/cce_run.sh     |  4 ++--
 2 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 scripts/cce/Untitled.ipynb

diff --git a/scripts/cce/Untitled.ipynb b/scripts/cce/Untitled.ipynb
new file mode 100644
index 000000000..b599cfb6f
--- /dev/null
+++ b/scripts/cce/Untitled.ipynb
@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0e042754",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "nr_misc",
+   "language": "python",
+   "name": "nr_misc"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/scripts/cce/cce_run.sh b/scripts/cce/cce_run.sh
index de7b668bb..9f4189fb7 100755
--- a/scripts/cce/cce_run.sh
+++ b/scripts/cce/cce_run.sh
@@ -11,14 +11,14 @@ echo "running CCE..."
 ./CharacteristicExtract --input-file cce_par_bin.yaml
 
 # h5 quick dev
-elif true
+elif false
 then
 cd runs
 echo "running preproc...\n"
 ./PreprocessCceWorldtube --input-file PreprocessCceWorldtube_h5.yaml
 echo "-------------------------"
 echo "running CCE..."
-./CharacteristicExtract --input-file cce_par_h5.yaml
+#./CharacteristicExtract --input-file cce_par_h5.yaml
 
 # q2
 elif false

From af39f69fcb0568d88ab82b133ebce778da1db3bf Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Fri, 21 Feb 2025 10:46:01 -0500
Subject: [PATCH 67/72] add restart time step for CCE

---
 src/outputs/restart.cpp | 1 +
 src/pgen/pgen.cpp       | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/src/outputs/restart.cpp b/src/outputs/restart.cpp
index 3ca3a6be1..901aedd75 100644
--- a/src/outputs/restart.cpp
+++ b/src/outputs/restart.cpp
@@ -221,6 +221,7 @@ void RestartOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin) {
     // store z4c information
     if (pz4c != nullptr) {
       resfile.Write_any_type(&(pz4c->last_output_time), sizeof(Real), "byte");
+      resfile.Write_any_type(&(pz4c->cce_dump_last_output_time), sizeof(Real), "byte");
     }
     // output puncture tracker data
     if (nco > 0) {
diff --git a/src/pgen/pgen.cpp b/src/pgen/pgen.cpp
index 868e3e7cf..9e11c0e00 100644
--- a/src/pgen/pgen.cpp
+++ b/src/pgen/pgen.cpp
@@ -176,6 +176,7 @@ ProblemGenerator::ProblemGenerator(ParameterInput *pin, Mesh *pm, IOWrapper resf
   // root process reads z4c last_output_time and tracker data
   if (pz4c != nullptr) {
     Real last_output_time;
+    Real cce_dump_last_output_time;
     if (global_variable::my_rank == 0) {
       if (resfile.Read_Reals(&last_output_time, 1) != 1) {
         std::cout << "### FATAL ERROR in " << __FILE__ << " at line " << __LINE__
@@ -183,11 +184,19 @@ ProblemGenerator::ProblemGenerator(ParameterInput *pin, Mesh *pm, IOWrapper resf
                   << "file is incorrect, restart file is broken." << std::endl;
         exit(EXIT_FAILURE);
       }
+      if (resfile.Read_Reals(&cce_dump_last_output_time, 1) != 1) {
+        std::cout << "### FATAL ERROR in " << __FILE__ << " at line " << __LINE__
+                  << std::endl << "z4c::cce_dump_last_output_time data size read from restart "
+                  << "file is incorrect, restart file is broken." << std::endl;
+        exit(EXIT_FAILURE);
+      }
     }
 #if MPI_PARALLEL_ENABLED
     MPI_Bcast(&last_output_time, sizeof(Real), MPI_CHAR, 0, MPI_COMM_WORLD);
+    MPI_Bcast(&cce_dump_last_output_time, sizeof(Real), MPI_CHAR, 0, MPI_COMM_WORLD);
 #endif
     pz4c->last_output_time = last_output_time;
+    pz4c->cce_dump_last_output_time = cce_dump_last_output_time;
 
     for (auto &pt : pz4c->ptracker) {
       Real pos[3];

From 7f58741ec0c8e7c2d357c7d4430b6dd8b6b1a047 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Wed, 26 Feb 2025 15:48:34 -0500
Subject: [PATCH 68/72] lint python

---
 scripts/cce/Untitled.ipynb           |   33 -
 scripts/cce/athk_to_spectre.py       | 1239 +++++++++++++-------------
 scripts/cce/cce_extract_txt.py       |  118 ++-
 scripts/cce/debug_athk_to_spectre.py |  425 +++++----
 4 files changed, 880 insertions(+), 935 deletions(-)
 delete mode 100644 scripts/cce/Untitled.ipynb

diff --git a/scripts/cce/Untitled.ipynb b/scripts/cce/Untitled.ipynb
deleted file mode 100644
index b599cfb6f..000000000
--- a/scripts/cce/Untitled.ipynb
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0e042754",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "nr_misc",
-   "language": "python",
-   "name": "nr_misc"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 6497f8ead..8d42f3cb6 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -84,147 +84,143 @@
 
 
 def parse_cli():
-  """
+    """
     arg parser
     """
-  p = argparse.ArgumentParser(
-      description="convert Athenak CCE dumps to Spectre CCE")
-  p.add_argument("-fpath",
-                 type=str,
-                 required=True,
-                 help="/path/to/cce/dir/or/file/dumps")
-  p.add_argument(
-      "-ftype",
-      type=str,
-      required=True,
-      help="input file type:{h5:for pitnull,bin: for athenak}",
-  )
-  p.add_argument("-d_out", type=str, required=True, help="/path/to/output/dir")
-  p.add_argument("-debug", type=str, default="n", help="debug=[y,n]")
-  p.add_argument(
-      "-radius",
-      type=float,
-      required=True,
-      help="interpolate all fields and their derivatives at this radius.",
-  )
-  p.add_argument(
-      "-t_deriv",
-      type=str,
-      default="fin_diff",
-      help="method to take the time derivative of fields:{Fourier,fin_diff}",
-  )
-  p.add_argument(
-      "-r_deriv",
-      type=str,
-      default="ChebU",
-      help=
-      "method to take the radial derivative of fields:{ChebU:Chebyshev of second kind}",
-  )
-  p.add_argument(
-      "-interpolation",
-      type=str,
-      default="ChebU",
-      help=
-      "method to interpolate fields at a given r:{ChebU:Chebyshev of second kind}",
-  )
-
-  args = p.parse_args()
-  return args
+    p = argparse.ArgumentParser(description="convert Athenak CCE dumps to Spectre CCE")
+    p.add_argument(
+        "-fpath", type=str, required=True, help="/path/to/cce/dir/or/file/dumps"
+    )
+    p.add_argument(
+        "-ftype",
+        type=str,
+        required=True,
+        help="input file type:{h5:for pitnull,bin: for athenak}",
+    )
+    p.add_argument("-d_out", type=str, required=True, help="/path/to/output/dir")
+    p.add_argument("-debug", type=str, default="n", help="debug=[y,n]")
+    p.add_argument(
+        "-radius",
+        type=float,
+        required=True,
+        help="interpolate all fields and their derivatives at this radius.",
+    )
+    p.add_argument(
+        "-t_deriv",
+        type=str,
+        default="fin_diff",
+        help="method to take the time derivative of fields:{Fourier,fin_diff}",
+    )
+    p.add_argument(
+        "-r_deriv",
+        type=str,
+        default="ChebU",
+        help="method to take the radial derivative of fields:{ChebU:Chebyshev of second kind}",
+    )
+    p.add_argument(
+        "-interpolation",
+        type=str,
+        default="ChebU",
+        help="method to interpolate fields at a given r:{ChebU:Chebyshev of second kind}",
+    )
+
+    args = p.parse_args()
+    return args
 
 
 class AngularTransform:
-  """
+    """
     angular coordinate transformation from pittnull to spectre
     """
 
-  def __init__(self, attrs: dict):
-    self.attrs = attrs
-    # no. of collocation pnts
-    self.npnts = 2 * self.attrs["max_l"] + 1
-    self.l_root = self._legendre_root()
-    # theta collocation coords
-    self.th = self._theta_gauss_legendre()
-    # phi collocation coords
-    self.ph = self._phi_equispace()
-    # legendre roots
-    # ylm(real/imag,th,ph,lm) on collocation coords
-    self.ylm_pit = self._ylm_on_gl()
-    # assert not np.any(np.isnan(self.th))
-    # assert not np.any(np.isnan(self.ph))
-    # assert not np.any(np.isnan(self.ylm_pit))
-    # assert not np.any(np.isnan(self.l_root))
-
-  def _theta_gauss_legendre(self):
-    """
+    def __init__(self, attrs: dict):
+        self.attrs = attrs
+        # no. of collocation pnts
+        self.npnts = 2 * self.attrs["max_l"] + 1
+        self.l_root = self._legendre_root()
+        # theta collocation coords
+        self.th = self._theta_gauss_legendre()
+        # phi collocation coords
+        self.ph = self._phi_equispace()
+        # legendre roots
+        # ylm(real/imag,th,ph,lm) on collocation coords
+        self.ylm_pit = self._ylm_on_gl()
+        # assert not np.any(np.isnan(self.th))
+        # assert not np.any(np.isnan(self.ph))
+        # assert not np.any(np.isnan(self.ylm_pit))
+        # assert not np.any(np.isnan(self.l_root))
+
+    def _theta_gauss_legendre(self):
+        """
         creating gl collocation pnts for theta
         """
-    th = np.empty(shape=(self.npnts), dtype=float)
-    for i in range(self.npnts):
-      th[i] = math.acos(-self.l_root[i])
+        th = np.empty(shape=(self.npnts), dtype=float)
+        for i in range(self.npnts):
+            th[i] = math.acos(-self.l_root[i])
 
-    return th
+        return th
 
-  def _legendre_root(self):
-    """
+    def _legendre_root(self):
+        """
         legendre roots
         """
-    nth = self.npnts
-    x, _ = np.polynomial.legendre.leggauss(nth)
-    x.sort()
-    return x
+        nth = self.npnts
+        x, _ = np.polynomial.legendre.leggauss(nth)
+        x.sort()
+        return x
 
-  def _phi_equispace(self):
-    """
+    def _phi_equispace(self):
+        """
         create equispace collocation pnts on phi
         """
-    phi = np.linspace(0, 2 * np.pi, self.npnts, endpoint=False)
+        phi = np.linspace(0, 2 * np.pi, self.npnts, endpoint=False)
 
-    return phi
+        return phi
 
-  def _ylm_on_gl(self):
-    """
+    def _ylm_on_gl(self):
+        """
         compute ylm on gauss legnedre for theta, and equispace for phi
         ylm[real/imag, theta, phi, lm]
         """
 
-    ylms = np.empty(
-        shape=(
-            len([g_re, g_im]),
-            self.npnts,
-            self.npnts,
-            self.attrs["max_lm"],
-        ),
-        dtype=float,
-    )
-    for i in range(self.npnts):
-      th = self.th[i]
-      for j in range(self.npnts):
-        ph = self.ph[j]
-        for l in range(0, self.attrs["max_l"] + 1):
-          for m in range(l, -l - 1, -1):
-            lm = lm_mode(l, m)
-            y = special.sph_harm(m, l, ph, th)
-            # assert not np.isnan(y)
-            ylms[g_re, i, j, lm] = y.real
-            ylms[g_im, i, j, lm] = y.imag
-
-    return ylms
-
-  def reconstruct_pit_on_gl(self, coeff: np.array):
-    """
+        ylms = np.empty(
+            shape=(
+                len([g_re, g_im]),
+                self.npnts,
+                self.npnts,
+                self.attrs["max_lm"],
+            ),
+            dtype=float,
+        )
+        for i in range(self.npnts):
+            th = self.th[i]
+            for j in range(self.npnts):
+                ph = self.ph[j]
+                for l in range(0, self.attrs["max_l"] + 1):
+                    for m in range(l, -l - 1, -1):
+                        lm = lm_mode(l, m)
+                        y = special.sph_harm(m, l, ph, th)
+                        # assert not np.isnan(y)
+                        ylms[g_re, i, j, lm] = y.real
+                        ylms[g_im, i, j, lm] = y.imag
+
+        return ylms
+
+    def reconstruct_pit_on_gl(self, coeff: np.array):
+        """
         reconstruct field on gauss lengendre collocation pnts from coeffs of pittnull
         """
-    rylm = self.ylm_pit[g_re]
-    iylm = self.ylm_pit[g_im]
+        rylm = self.ylm_pit[g_re]
+        iylm = self.ylm_pit[g_im]
 
-    field = np.dot(
-        coeff[g_re, :, :, :], rylm[:, :, np.newaxis, :, np.newaxis]) - np.dot(
-            coeff[g_im, :, :, :], iylm[:, :, np.newaxis, :, np.newaxis])
+        field = np.dot(
+            coeff[g_re, :, :, :], rylm[:, :, np.newaxis, :, np.newaxis]
+        ) - np.dot(coeff[g_im, :, :, :], iylm[:, :, np.newaxis, :, np.newaxis])
 
-    return field[:, :, :, :, 0, 0]
+        return field[:, :, :, :, 0, 0]
 
-  def _sp_expansion(self, f):
-    """
+    def _sp_expansion(self, f):
+        """
         Expands function f(theta, phi) in spherical harmonics up to degree L_max
         using Gauss-Legendre quadrature in theta and uniform quadrature in phi.
 
@@ -235,57 +231,53 @@ def _sp_expansion(self, f):
             coeff :Expansion coefficients.
         """
 
-    # Gauss-Legendre quadrature points and weights for theta
-    _, w_theta = np.polynomial.legendre.leggauss(self.npnts)
-    dphi = 2 * np.pi / self.npnts
-    Ylm = self.ylm_pit[g_re, ...] + 1j * self.ylm_pit[g_im, ...]
+        # Gauss-Legendre quadrature points and weights for theta
+        _, w_theta = np.polynomial.legendre.leggauss(self.npnts)
+        dphi = 2 * np.pi / self.npnts
+        Ylm = self.ylm_pit[g_re, ...] + 1j * self.ylm_pit[g_im, ...]
 
-    coeff = np.einsum("trij,ijk,i->trk",
-                      f,
-                      np.conj(Ylm),
-                      w_theta,
-                      optimize=True)
-    coeff *= dphi
+        coeff = np.einsum("trij,ijk,i->trk", f, np.conj(Ylm), w_theta, optimize=True)
+        coeff *= dphi
 
-    # coeff = np.transpose(coeff, (2, 1, 0))
+        # coeff = np.transpose(coeff, (2, 1, 0))
 
-    return coeff
+        return coeff
 
-  def transform_field_to_coeff_gl(self, field: np.array):
-    """
+    def transform_field_to_coeff_gl(self, field: np.array):
+        """
         transformation of the given field on gauss legendre points to
         spherical harmonics basis on gauss legendre points.
         field[time,radius,theta,phi]
         """
 
-    shape = (
-        len([g_re, g_im]),
-        self.attrs["lev_t"],
-        self.attrs["max_n"],
-        self.attrs["max_lm"],
-    )
-    coeff = np.empty(shape=shape, dtype=float)
-    c = self._sp_expansion(field)
+        shape = (
+            len([g_re, g_im]),
+            self.attrs["lev_t"],
+            self.attrs["max_n"],
+            self.attrs["max_lm"],
+        )
+        coeff = np.empty(shape=shape, dtype=float)
+        c = self._sp_expansion(field)
 
-    coeff[g_re, ...] = c.real
-    coeff[g_im, ...] = c.imag
+        coeff[g_re, ...] = c.real
+        coeff[g_im, ...] = c.imag
 
-    return coeff
+        return coeff
 
-  def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
-    """
+    def transform_pit_coeffs_to_spec_coeffs(self, coeff: np.array):
+        """
         transform pit coeffs which are on equispace theta and phi to spectre coeffs
         which are equispace on phi and gauss legendre on theta.
         """
 
-    # first reconstruct field on gl collocation points
-    field = self.reconstruct_pit_on_gl(coeff)
+        # first reconstruct field on gl collocation points
+        field = self.reconstruct_pit_on_gl(coeff)
 
-    return self.transform_field_to_coeff_gl(field)
+        return self.transform_field_to_coeff_gl(field)
 
 
 def load(fpath: str, field_name: str, attrs: dict) -> list:
-  """
+    """
     read the field accroding to attrs.
     return convention:
       ret[real/imag, time_level, n, lm], eg:
@@ -293,92 +285,92 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
       ret[g_im,3,2,:] = Im(C_2lm(t=3)) for all lm
     """
 
-  if attrs["file_type"] == "h5":
-    lev_t = attrs["lev_t"]
-    max_n = attrs["max_n"]
-    max_lm = attrs["max_lm"]
-    shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
-    ret = np.empty(shape=shape, dtype=float)
-
-    with h5py.File(fpath, "r") as h5f:
-      coords = AngularTransform(attrs)
-      # read & save
-      for i in range(0, lev_t):
-        key = f"{i}"
-        h5_re = h5f[f"{key}/{field_name}/re"]
-        h5_im = h5f[f"{key}/{field_name}/im"]
-        ret[g_re, i, ...] = h5_re[:, 0:max_lm]
-        ret[g_im, i, ...] = h5_im[:, 0:max_lm]
-
-      # transform from PITTNull coordinates to Spectre coordinates
-      # assert not np.any(np.isnan(ret)), f"{field_name} has nans before transf.!"
-      print(f"transforming {field_name} from pitt to spectre", flush=True)
-      ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
-      # assert not np.any(np.isnan(ret)), f"{field_name} got nans after transf.!"
-
-  elif attrs["file_type"] == "bin":
-    # Load the list of files
-    ##TODO: this depends on file name
-    flist = sorted(glob.glob(fpath + "/cce_*.bin"))
-    data = []
-    dat_real = []
-    dat_imag = []
-    t = []
-    for f in flist:
-      ##TODO: it reads all field and only use one of them. it's inefficient
-      (
-          nr,
-          num_l_modes,
-          time,
-          rin,
-          rout,
-          data_real,
-          data_imag,
-          index_to_lm,
-      ) = read_cce_file(f)
-      t.append(time)
-      dat_real.append(data_real[:, g_name_index[field_name], :])
-      dat_imag.append(data_imag[:, g_name_index[field_name], :])
-    dat_real = np.array(dat_real)
-    dat_imag = np.array(dat_imag)
-    t = np.array(t)
-    f_real = interp1d(t, dat_real, axis=0)
-    f_imag = interp1d(t, dat_imag, axis=0)
-
-    attrs["lev_t"] = len(flist)
-    attrs["max_n"] = nr
-    attrs["max_l"] = num_l_modes
-    attrs["max_lm"] = int((num_l_modes + 1)**2)
-    attrs["r_in"] = np.array([rin])
-    attrs["r_out"] = np.array([rout])
-    attrs["time"] = np.linspace(t.min(), t.max(), t.shape[0])
-
-    shape = (
-        len([g_re, g_im]),
-        attrs["lev_t"],
-        attrs["max_n"],
-        attrs["max_lm"],
-    )
-    ret = np.empty(shape=shape, dtype=float)
-    ## NOTE: is it fine that we do interpolation?
-    ret[g_re, :] = f_real(attrs["time"])
-    ret[g_im, :] = f_imag(attrs["time"])
-  else:
-    raise ValueError("no such option")
+    if attrs["file_type"] == "h5":
+        lev_t = attrs["lev_t"]
+        max_n = attrs["max_n"]
+        max_lm = attrs["max_lm"]
+        shape = (len([g_re, g_im]), lev_t, max_n, max_lm)
+        ret = np.empty(shape=shape, dtype=float)
+
+        with h5py.File(fpath, "r") as h5f:
+            coords = AngularTransform(attrs)
+            # read & save
+            for i in range(0, lev_t):
+                key = f"{i}"
+                h5_re = h5f[f"{key}/{field_name}/re"]
+                h5_im = h5f[f"{key}/{field_name}/im"]
+                ret[g_re, i, ...] = h5_re[:, 0:max_lm]
+                ret[g_im, i, ...] = h5_im[:, 0:max_lm]
+
+            # transform from PITTNull coordinates to Spectre coordinates
+            # assert not np.any(np.isnan(ret)), f"{field_name} has nans before transf.!"
+            print(f"transforming {field_name} from pitt to spectre", flush=True)
+            ret = coords.transform_pit_coeffs_to_spec_coeffs(ret)
+            # assert not np.any(np.isnan(ret)), f"{field_name} got nans after transf.!"
+
+    elif attrs["file_type"] == "bin":
+        # Load the list of files
+        ##TODO: this depends on file name
+        flist = sorted(glob.glob(fpath + "/cce_*.bin"))
+        data = []
+        dat_real = []
+        dat_imag = []
+        t = []
+        for f in flist:
+            ##TODO: it reads all field and only use one of them. it's inefficient
+            (
+                nr,
+                num_l_modes,
+                time,
+                rin,
+                rout,
+                data_real,
+                data_imag,
+                index_to_lm,
+            ) = read_cce_file(f)
+            t.append(time)
+            dat_real.append(data_real[:, g_name_index[field_name], :])
+            dat_imag.append(data_imag[:, g_name_index[field_name], :])
+        dat_real = np.array(dat_real)
+        dat_imag = np.array(dat_imag)
+        t = np.array(t)
+        f_real = interp1d(t, dat_real, axis=0)
+        f_imag = interp1d(t, dat_imag, axis=0)
+
+        attrs["lev_t"] = len(flist)
+        attrs["max_n"] = nr
+        attrs["max_l"] = num_l_modes
+        attrs["max_lm"] = int((num_l_modes + 1) ** 2)
+        attrs["r_in"] = np.array([rin])
+        attrs["r_out"] = np.array([rout])
+        attrs["time"] = np.linspace(t.min(), t.max(), t.shape[0])
+
+        shape = (
+            len([g_re, g_im]),
+            attrs["lev_t"],
+            attrs["max_n"],
+            attrs["max_lm"],
+        )
+        ret = np.empty(shape=shape, dtype=float)
+        ## NOTE: is it fine that we do interpolation?
+        ret[g_re, :] = f_real(attrs["time"])
+        ret[g_im, :] = f_imag(attrs["time"])
+    else:
+        raise ValueError("no such option")
 
-  # print(ret)
-  return ret
+    # print(ret)
+    return ret
 
 
 def lm_mode(l, m):
-  """
+    """
     l and m mode convention
     """
-  return l * l + l + m
+    return l * l + l + m
 
 
 def read_cce_file(filename):
-  """
+    """
     Reads binary data from a CCE output file generated by the provided C++ function.
 
     Parameters:
@@ -394,148 +386,152 @@ def read_cce_file(filename):
         data_imag (numpy.ndarray): Imaginary part of the data, shape (nr, 10, num_angular_modes).
         index_to_lm (dict): Mapping from data index to (l, m) values.
     """
-  with open(filename, "rb") as f:
-    # Read number of radial points (nr)
-    nr_bytes = f.read(4)
-    nr = struct.unpack("<i", nr_bytes)[0] # little-endian integer
+    with open(filename, "rb") as f:
+        # Read number of radial points (nr)
+        nr_bytes = f.read(4)
+        nr = struct.unpack("<i", nr_bytes)[0]  # little-endian integer
 
-    # Read number of l modes (num_l_modes)
-    num_l_modes_bytes = f.read(4)
-    num_l_modes = struct.unpack("<i", num_l_modes_bytes)[0]
+        # Read number of l modes (num_l_modes)
+        num_l_modes_bytes = f.read(4)
+        num_l_modes = struct.unpack("<i", num_l_modes_bytes)[0]
 
-    # Read time
-    time_bytes = f.read(8)
-    time = struct.unpack("<d", time_bytes)[0] # little-endian double
+        # Read time
+        time_bytes = f.read(8)
+        time = struct.unpack("<d", time_bytes)[0]  # little-endian double
 
-    # Read inner and outer radial boundaries (rin and rout)
-    rin_bytes = f.read(8)
-    rin = struct.unpack("<d", rin_bytes)[0]
+        # Read inner and outer radial boundaries (rin and rout)
+        rin_bytes = f.read(8)
+        rin = struct.unpack("<d", rin_bytes)[0]
 
-    rout_bytes = f.read(8)
-    rout = struct.unpack("<d", rout_bytes)[0]
+        rout_bytes = f.read(8)
+        rout = struct.unpack("<d", rout_bytes)[0]
 
-    # Calculate the number of angular modes
-    num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
+        # Calculate the number of angular modes
+        num_angular_modes = (num_l_modes + 1) * (num_l_modes + 1)
 
-    # Total number of data points
-    count = 10 * nr * num_angular_modes
+        # Total number of data points
+        count = 10 * nr * num_angular_modes
 
-    # Read data_real array
-    data_real = np.fromfile(f, dtype="<d", count=count) # little-endian double
-    data_real = data_real.reshape((nr, 10, num_angular_modes))
+        # Read data_real array
+        data_real = np.fromfile(f, dtype="<d", count=count)  # little-endian double
+        data_real = data_real.reshape((nr, 10, num_angular_modes))
 
-    # Read data_imag array
-    data_imag = np.fromfile(f, dtype="<d", count=count)
-    data_imag = data_imag.reshape((nr, 10, num_angular_modes))
+        # Read data_imag array
+        data_imag = np.fromfile(f, dtype="<d", count=count)
+        data_imag = data_imag.reshape((nr, 10, num_angular_modes))
 
-    # Create a mapping from index to (l, m)
-    index_to_lm = {}
-    for l in range(1, num_l_modes + 1):
-      for m in range(-l, l + 1):
-        index = l * l + l + m
-        index_to_lm[index] = (l, m)
+        # Create a mapping from index to (l, m)
+        index_to_lm = {}
+        for l in range(1, num_l_modes + 1):
+            for m in range(-l, l + 1):
+                index = l * l + l + m
+                index_to_lm[index] = (l, m)
 
-  return nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm
+    return nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm
 
 
-def get_attribute(fpath: str,
-                  field_name: str = "gxx",
-                  type: str = "h5",
-                  args=None) -> dict:
-  """
+def get_attribute(
+    fpath: str, field_name: str = "gxx", type: str = "h5", args=None
+) -> dict:
+    """
     find attributes such as num. of time level, and n, lm in C_nlm
     also saves the time value at each slice.
     """
-  attrs = {}
-  if type == "h5":
-    attrs["file_type"] = "h5"
-    with h5py.File(fpath, "r") as h5f:
-      # find attribute about num. of time level, and n,l,m in C_nlm
-      attrs["lev_t"] = len(h5f.keys()) - 1
-      if attrs["lev_t"] % 2 == 0: # for fourier transformation
-        attrs["lev_t"] -= 1
-
-      attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
-      attrs["max_l"] = (int(math.sqrt(attrs["max_lm"])) - 1
-                        ) # NOTE:l must be inclusive in loops
-      attrs["max_lm"] = int((attrs["max_l"] + 1)**2)
-      attrs["r_in"] = h5f["metadata"].attrs["Rin"]
-      attrs["r_out"] = h5f["metadata"].attrs["Rout"]
-      # read & save time
-      time = []
-      for i in range(0, attrs["lev_t"]):
-        key = f"{i}"
-        t = h5f[key].attrs["Time"][0]
-        time.append(t)
-      attrs["time"] = np.array(time)
-  elif type == "bin":
-    attrs["file_type"] = "bin"
-  else:
-    raise ValueError("no such option")
-
-  # print(attrs)
-  return attrs
-
-
-def time_derivative_findiff(field: np.array, field_name: str, attrs: dict,
-                            args) -> np.array:
-  """
+    attrs = {}
+    if type == "h5":
+        attrs["file_type"] = "h5"
+        with h5py.File(fpath, "r") as h5f:
+            # find attribute about num. of time level, and n,l,m in C_nlm
+            attrs["lev_t"] = len(h5f.keys()) - 1
+            if attrs["lev_t"] % 2 == 0:  # for fourier transformation
+                attrs["lev_t"] -= 1
+
+            attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
+            attrs["max_l"] = (
+                int(math.sqrt(attrs["max_lm"])) - 1
+            )  # NOTE:l must be inclusive in loops
+            attrs["max_lm"] = int((attrs["max_l"] + 1) ** 2)
+            attrs["r_in"] = h5f["metadata"].attrs["Rin"]
+            attrs["r_out"] = h5f["metadata"].attrs["Rout"]
+            # read & save time
+            time = []
+            for i in range(0, attrs["lev_t"]):
+                key = f"{i}"
+                t = h5f[key].attrs["Time"][0]
+                time.append(t)
+            attrs["time"] = np.array(time)
+    elif type == "bin":
+        attrs["file_type"] = "bin"
+    else:
+        raise ValueError("no such option")
+
+    # print(attrs)
+    return attrs
+
+
+def time_derivative_findiff(
+    field: np.array, field_name: str, attrs: dict, args
+) -> np.array:
+    """
     return the time derivative of the given field using finite diff. 2nd order
     field(rel/img,t,n,lm)
     """
 
-  print(f"finite difference time derivative: {field_name}", flush=True)
-  _, len_t, len_n, len_lm = field.shape
-  time = attrs["time"]
-  dt = np.gradient(time, 2)
-  dfield = np.empty_like(field)
+    print(f"finite difference time derivative: {field_name}", flush=True)
+    _, len_t, len_n, len_lm = field.shape
+    time = attrs["time"]
+    dt = np.gradient(time, 2)
+    dfield = np.empty_like(field)
 
-  for n in range(len_n):
-    for lm in range(len_lm):
-      dfield[g_re, :, n, lm] = np.gradient(field[g_re, :, n, lm], 2) / dt
-      dfield[g_im, :, n, lm] = np.gradient(field[g_im, :, n, lm], 2) / dt
+    for n in range(len_n):
+        for lm in range(len_lm):
+            dfield[g_re, :, n, lm] = np.gradient(field[g_re, :, n, lm], 2) / dt
+            dfield[g_im, :, n, lm] = np.gradient(field[g_im, :, n, lm], 2) / dt
 
-  return dfield
+    return dfield
 
 
-def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
-                            args) -> np.array:
-  """
+def time_derivative_fourier(
+    field: np.array, field_name: str, attrs: dict, args
+) -> np.array:
+    """
     return the time derivative of the given field using Fourier method
     field(rel/img,t, n,lm)
     """
 
-  ## TODO: Fourier time derives not tested!
-  print(f"Fourier time derivative: {field_name}", flush=True)
-  _, len_t, len_n, len_lm = field.shape
-  dt = attrs["time"][2] - attrs["time"][1]
-  wm = math.pi * 2.0 / (len_t * dt)
-
-  dfield = np.empty_like(field)
-  for n in range(len_n):
-    for lm in range(len_lm):
-      coeff = field[g_re, :, n, lm] + 1j * field[g_im, :, n, lm]
-      # F. transform
-      fft_coeff = np.fft.fft(coeff)
-      # if args["debug"] == 'y':
-      #  print("debug: normalization?",round(coeff[1],6) == round(np.fft.ifft(fft_coeff)[1],6))
-
-      # time derivative
-      half = len_t // 2 + 1
-      omega = np.empty(shape=half)
-      for i in range(0, half):
-        omega[i] = i * wm
-
-      dfft_coeff = np.empty_like(fft_coeff)
-      dfft_coeff[0] = 0
+    ## TODO: Fourier time derives not tested!
+    print(f"Fourier time derivative: {field_name}", flush=True)
+    _, len_t, len_n, len_lm = field.shape
+    dt = attrs["time"][2] - attrs["time"][1]
+    wm = math.pi * 2.0 / (len_t * dt)
 
-      dfft_coeff[1:half] = (-np.imag(fft_coeff[1:half]) +
-                            1j * np.real(fft_coeff[1:half])) * omega[1:]
-      dfft_coeff[half:] = (np.imag(fft_coeff[half:]) -
-                           1j * np.real(fft_coeff[half:])) * (omega[::-1][1:])
-
-      # not optimized version
-      """
+    dfield = np.empty_like(field)
+    for n in range(len_n):
+        for lm in range(len_lm):
+            coeff = field[g_re, :, n, lm] + 1j * field[g_im, :, n, lm]
+            # F. transform
+            fft_coeff = np.fft.fft(coeff)
+            # if args["debug"] == 'y':
+            #  print("debug: normalization?",round(coeff[1],6) == round(np.fft.ifft(fft_coeff)[1],6))
+
+            # time derivative
+            half = len_t // 2 + 1
+            omega = np.empty(shape=half)
+            for i in range(0, half):
+                omega[i] = i * wm
+
+            dfft_coeff = np.empty_like(fft_coeff)
+            dfft_coeff[0] = 0
+
+            dfft_coeff[1:half] = (
+                -np.imag(fft_coeff[1:half]) + 1j * np.real(fft_coeff[1:half])
+            ) * omega[1:]
+            dfft_coeff[half:] = (
+                np.imag(fft_coeff[half:]) - 1j * np.real(fft_coeff[half:])
+            ) * (omega[::-1][1:])
+
+            # not optimized version
+            """
       dfft_coeff[0] = 0
       for i in range(1, half):
         omega = i * wm
@@ -548,46 +544,47 @@ def time_derivative_fourier(field: np.array, field_name: str, attrs: dict,
         dfft_coeff[-i] = omega*complex(im2, -re2)
 
       """
-      # F. inverse
-      coeff = np.fft.ifft(dfft_coeff)
-      dfield[g_re, :, n, lm] = np.real(coeff)
-      dfield[g_im, :, n, lm] = np.imag(coeff)
+            # F. inverse
+            coeff = np.fft.ifft(dfft_coeff)
+            dfield[g_re, :, n, lm] = np.real(coeff)
+            dfield[g_im, :, n, lm] = np.imag(coeff)
 
-  if args["debug"] == "y":
-    for n in range(len_n):
-      for l in range(2, g_debug_max_l + 1):
-        for m in range(-l, l + 1):
-          hfile = (f"{args['d_out']}/debug_{field_name}_n{n}l{l}m{m}.txt")
-          write_data = np.column_stack((
-              attrs["time"],
-              dfield[g_re, :, n, lm_mode(l, m)],
-              dfield[g_im, :, n, lm_mode(l, m)],
-              field[g_re, :, n, lm_mode(l, m)],
-              field[g_im, :, n, lm_mode(l, m)],
-          ))
-          np.savetxt(hfile, write_data, header="t dre/dt dim/dt re im")
-
-  return dfield
+    if args["debug"] == "y":
+        for n in range(len_n):
+            for l in range(2, g_debug_max_l + 1):
+                for m in range(-l, l + 1):
+                    hfile = f"{args['d_out']}/debug_{field_name}_n{n}l{l}m{m}.txt"
+                    write_data = np.column_stack(
+                        (
+                            attrs["time"],
+                            dfield[g_re, :, n, lm_mode(l, m)],
+                            dfield[g_im, :, n, lm_mode(l, m)],
+                            field[g_re, :, n, lm_mode(l, m)],
+                            field[g_im, :, n, lm_mode(l, m)],
+                        )
+                    )
+                    np.savetxt(hfile, write_data, header="t dre/dt dim/dt re im")
+
+    return dfield
 
 
 def dUk_dx(order: int, x: float) -> float:
-  """
+    """
     d(Chebyshev of second kind)/dx
     """
-  assert x != 1 and x != -1
-  t = special.chebyt(order + 1)(x)
-  u = special.chebyu(order)(x)
-  duk_dx = (order + 1) * t - x * u
-  duk_dx /= x**2 - 1
+    assert x != 1 and x != -1
+    t = special.chebyt(order + 1)(x)
+    u = special.chebyu(order)(x)
+    duk_dx = (order + 1) * t - x * u
+    duk_dx /= x**2 - 1
 
-  return duk_dx
+    return duk_dx
 
 
-def radial_derivative_at_r_chebu(field: np.array,
-                                 field_name: str,
-                                 attrs: dict,
-                                 args) -> np.array:
-  """
+def radial_derivative_at_r_chebu(
+    field: np.array, field_name: str, attrs: dict, args
+) -> np.array:
+    """
     return the radial derivative of the given field using Chebyshev of
     2nd kind method at the radius of interest.
 
@@ -598,100 +595,100 @@ def radial_derivative_at_r_chebu(field: np.array,
     field(rel/img,t,n,lm)
     """
 
-  r_1 = attrs["r_in"][0]
-  r_2 = attrs["r_out"][0]
-  r = args["radius"]
-
-  print(
-      f"ChebyU radial derivative: {field_name}, at r={r} in [{r_1},{r_2}]",
-      flush=True,
-  )
-
-  _, len_t, len_n, len_lm = field.shape
-
-  assert r_1 != r_2
-  dx_dr = g_sign * 2 / (r_2 - r_1)
+    r_1 = attrs["r_in"][0]
+    r_2 = attrs["r_out"][0]
+    r = args["radius"]
 
-  if args["debug"] == "y":
-    # populate collocation points, roots of U_i
-    x_i = np.empty(shape=len_n, dtype=float)
-    for i in range(len_n):
-      x_i[i] = math.cos(math.pi * (i + 1) / (len_n + 1))
+    print(
+        f"ChebyU radial derivative: {field_name}, at r={r} in [{r_1},{r_2}]",
+        flush=True,
+    )
 
-    # dU_k/dx|x=x_i
-    duk_dx = np.empty(shape=(len_n, len_n), dtype=float)
-    for k in range(len_n):
-      for i in range(len_n):
-        t = special.chebyt(k + 1)(x_i[i])
-        u = special.chebyu(k)(x_i[i])
-        duk_dx[k, i] = (k + 1) * t - x_i[i] * u
+    _, len_t, len_n, len_lm = field.shape
 
-    duk_dx /= np.square(x_i) - 1
+    assert r_1 != r_2
+    dx_dr = g_sign * 2 / (r_2 - r_1)
 
-    uk = np.empty(shape=len_n, dtype=float)
-    tk = np.empty(shape=len_n, dtype=float)
+    if args["debug"] == "y":
+        # populate collocation points, roots of U_i
+        x_i = np.empty(shape=len_n, dtype=float)
+        for i in range(len_n):
+            x_i[i] = math.cos(math.pi * (i + 1) / (len_n + 1))
+
+        # dU_k/dx|x=x_i
+        duk_dx = np.empty(shape=(len_n, len_n), dtype=float)
+        for k in range(len_n):
+            for i in range(len_n):
+                t = special.chebyt(k + 1)(x_i[i])
+                u = special.chebyu(k)(x_i[i])
+                duk_dx[k, i] = (k + 1) * t - x_i[i] * u
+
+        duk_dx /= np.square(x_i) - 1
+
+        uk = np.empty(shape=len_n, dtype=float)
+        tk = np.empty(shape=len_n, dtype=float)
+        for k in range(len_n):
+            hfile = f"{args['d_out']}/cheb_k{k}.txt"
+            for i in range(len_n):
+                tk[i] = special.chebyt(k)(x_i[i])
+                uk[i] = special.chebyu(k)(x_i[i])
+
+            write_data = np.column_stack((x_i, uk, tk, duk_dx[k, :]))
+            np.savetxt(
+                hfile,
+                write_data,
+                header=f"x_i uk{k}(x_i) tk{k}(x_i) duk{k}(x_i)/dx",
+            )
+
+    dfield = np.zeros(shape=(len([g_re, g_im]), len_t, len_lm))
+    r = args["radius"]
+    x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
     for k in range(len_n):
-      hfile = f"{args['d_out']}/cheb_k{k}.txt"
-      for i in range(len_n):
-        tk[i] = special.chebyt(k)(x_i[i])
-        uk[i] = special.chebyu(k)(x_i[i])
-
-      write_data = np.column_stack((x_i, uk, tk, duk_dx[k, :]))
-      np.savetxt(
-          hfile,
-          write_data,
-          header=f"x_i uk{k}(x_i) tk{k}(x_i) duk{k}(x_i)/dx",
-      )
+        dfield[:, :, :] += field[:, :, k, :] * dUk_dx(k, x)
 
-  dfield = np.zeros(shape=(len([g_re, g_im]), len_t, len_lm))
-  r = args["radius"]
-  x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
-  for k in range(len_n):
-    dfield[:, :, :] += field[:, :, k, :] * dUk_dx(k, x)
-
-  return dfield * dx_dr
+    return dfield * dx_dr
 
 
 def time_derivative(field: np.array, field_name: str, attrs: dict, args):
-  """
+    """
     return the time derivative of the given field
     """
 
-  if args["t_deriv"] == "Fourier":
-    return time_derivative_fourier(field, field_name, attrs, args)
-  elif args["t_deriv"] == "fin_diff":
-    return time_derivative_findiff(field, field_name, attrs, args)
-  else:
-    raise ValueError("no such option")
+    if args["t_deriv"] == "Fourier":
+        return time_derivative_fourier(field, field_name, attrs, args)
+    elif args["t_deriv"] == "fin_diff":
+        return time_derivative_findiff(field, field_name, attrs, args)
+    else:
+        raise ValueError("no such option")
 
 
 class ChebUExpansion:
-  """
+    """
     f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
     collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
     """
 
-  def __init__(self, attrs: dict, args: dict):
-    self.N = N = attrs["max_n"] # num. of coeffs or num of collocation pnts
-    pi = math.pi
+    def __init__(self, attrs: dict, args: dict):
+        self.N = N = attrs["max_n"]  # num. of coeffs or num of collocation pnts
+        pi = math.pi
 
-    # U_N roots:
-    a_i = np.arange(0, N, dtype=int) + 1
-    self.x_i = x_i = np.cos(math.pi * a_i / (N + 1))
-    # quadrature weights
-    self.w_i = (1 - np.square(x_i)) * pi / (N + 1)
+        # U_N roots:
+        a_i = np.arange(0, N, dtype=int) + 1
+        self.x_i = x_i = np.cos(math.pi * a_i / (N + 1))
+        # quadrature weights
+        self.w_i = (1 - np.square(x_i)) * pi / (N + 1)
 
-    # ChebU_j(x_i):
-    Uj_xi = np.empty(shape=(N, N), dtype=float)
-    for j in range(N):
-      Uj_xi[j, :] = special.chebyu(j)(x_i)
-    self.Uj_xi = Uj_xi
+        # ChebU_j(x_i):
+        Uj_xi = np.empty(shape=(N, N), dtype=float)
+        for j in range(N):
+            Uj_xi[j, :] = special.chebyu(j)(x_i)
+        self.Uj_xi = Uj_xi
 
-    if args["debug"] == "y":
-      self.__debug()
+        if args["debug"] == "y":
+            self.__debug()
 
-  def coefficients(self, field: np.array) -> np.array:
-    """
+    def coefficients(self, field: np.array) -> np.array:
+        """
         use Gauss Quadrature to expand the given field in ChebU
         f(x) = sum_{i=0}^{N-1} C_i U_i(x), U_i(x) Chebyshev of 2nd kind
         collocation points (roots of U_i): x_i = cos(pi*(i+1)/(N+1))
@@ -699,44 +696,45 @@ def coefficients(self, field: np.array) -> np.array:
 
         """
 
-    w_i = self.w_i
-    x_i = self.x_i
-    Uji = self.Uj_xi
+        w_i = self.w_i
+        x_i = self.x_i
+        Uji = self.Uj_xi
 
-    coeffs = Uji @ (w_i * field[::-g_sign])
-    coeffs *= 2.0 / math.pi
+        coeffs = Uji @ (w_i * field[::-g_sign])
+        coeffs *= 2.0 / math.pi
 
-    return coeffs
+        return coeffs
 
-  def __debug(self):
-    """
+    def __debug(self):
+        """
         test if the expansion works for different known functions
         """
 
-    N = self.N + 1 # +1 to see the roots of U_N
-    x_i = self.x_i
+        N = self.N + 1  # +1 to see the roots of U_N
+        x_i = self.x_i
 
-    # populate funcs using bases themselves
-    fs = []
-    for n in range(N):
-      f = special.chebyu(n)(x_i)
-      fs.append((f, n))
+        # populate funcs using bases themselves
+        fs = []
+        for n in range(N):
+            f = special.chebyu(n)(x_i)
+            fs.append((f, n))
 
-    # find coeffs
-    cs = []
-    for n in range(N):
-      c = self.coefficients(fs[n][0])
-      cs.append((c, fs[n][1]))
+        # find coeffs
+        cs = []
+        for n in range(N):
+            c = self.coefficients(fs[n][0])
+            cs.append((c, fs[n][1]))
 
-    # expect to see only the n-th entry is 1 for the chebyu(n) of order n
-    for n in range(N):
-      order = cs[n][1]
-      print(f"chebyu{order}, coeffs = {cs[n][0]}\n", flush=True)
+        # expect to see only the n-th entry is 1 for the chebyu(n) of order n
+        for n in range(N):
+            order = cs[n][1]
+            print(f"chebyu{order}, coeffs = {cs[n][0]}\n", flush=True)
 
 
-def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,
-                           args) -> np.array:
-  """
+def radial_expansion_chebu(
+    field: np.array, field_name: str, attrs: dict, args
+) -> np.array:
+    """
     expands field in the radial direction using chebyshev of second kind
     and returns the expansion coefficients. we have
 
@@ -748,144 +746,139 @@ def radial_expansion_chebu(field: np.array, field_name: str, attrs: dict,
 
     """
 
-  print(f"ChebU expansion {field_name}", flush=True)
-  expand = ChebUExpansion(attrs, args)
-  _, len_t, _, len_lm = field.shape
+    print(f"ChebU expansion {field_name}", flush=True)
+    expand = ChebUExpansion(attrs, args)
+    _, len_t, _, len_lm = field.shape
 
-  for t in range(len_t):
-    for lm in range(len_lm):
-      field[g_re, t, :, lm] = expand.coefficients(field[g_re, t, :, lm])
-      field[g_im, t, :, lm] = expand.coefficients(field[g_im, t, :, lm])
+    for t in range(len_t):
+        for lm in range(len_lm):
+            field[g_re, t, :, lm] = expand.coefficients(field[g_re, t, :, lm])
+            field[g_im, t, :, lm] = expand.coefficients(field[g_im, t, :, lm])
 
-  return field
+    return field
 
 
 def radial_expansion(field: np.array, field_name: str, attrs: dict, args):
-  """
+    """
     expands field in the radial direction(if needed) and returns
     """
 
-  if args["ftype"] == "bin":
-    return radial_expansion_chebu(field, field_name, attrs, args)
-  elif args["ftype"] == "h5":
-    return field
-  else:
-    raise ValueError("no such option")
+    if args["ftype"] == "bin":
+        return radial_expansion_chebu(field, field_name, attrs, args)
+    elif args["ftype"] == "h5":
+        return field
+    else:
+        raise ValueError("no such option")
 
 
 def radial_derivative_at_r(field: np.array, field_name: str, attrs: dict, args):
-  """
+    """
     return the radial derivative of the given field at R=r
     """
 
-  if args["r_deriv"] == "ChebU":
-    return radial_derivative_at_r_chebu(field, field_name, attrs, args)
-  else:
-    raise ValueError("no such option")
+    if args["r_deriv"] == "ChebU":
+        return radial_derivative_at_r_chebu(field, field_name, attrs, args)
+    else:
+        raise ValueError("no such option")
 
 
 class Interpolate_at_r:
-
-  def __init__(self, attrs: dict, args: dict):
-    """
+    def __init__(self, attrs: dict, args: dict):
+        """
         interpolate the given field at R=r
         """
-    self.attrs = attrs
-    self.args = args
-    self.len_t = attrs["lev_t"]
-    self.len_n = attrs["max_n"]
-    self.len_lm = attrs["max_lm"]
-    r_1 = attrs["r_in"][0]
-    r_2 = attrs["r_out"][0]
-    self.r = r = args["radius"]
-    self.x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
-
-    assert -1 < self.x < 1, f"x = {self.x}"
-
-    if args["interpolation"] == "ChebU":
-      self.Uk = np.empty(shape=self.len_n)
-      for k in range(self.len_n):
-        self.Uk[k] = special.chebyu(k)(self.x)
-      # print("x,Uk",self.x,self.Uk)
-      self.interp = self.interpolate_at_r_chebu
-    else:
-      raise ValueError("no such option")
-
-  def interpolate_at_r_chebu(self, field: np.array, field_name: str):
-    """
+        self.attrs = attrs
+        self.args = args
+        self.len_t = attrs["lev_t"]
+        self.len_n = attrs["max_n"]
+        self.len_lm = attrs["max_lm"]
+        r_1 = attrs["r_in"][0]
+        r_2 = attrs["r_out"][0]
+        self.r = r = args["radius"]
+        self.x = g_sign * (2 * r - r_1 - r_2) / (r_2 - r_1)
+
+        assert -1 < self.x < 1, f"x = {self.x}"
+
+        if args["interpolation"] == "ChebU":
+            self.Uk = np.empty(shape=self.len_n)
+            for k in range(self.len_n):
+                self.Uk[k] = special.chebyu(k)(self.x)
+            # print("x,Uk",self.x,self.Uk)
+            self.interp = self.interpolate_at_r_chebu
+        else:
+            raise ValueError("no such option")
+
+    def interpolate_at_r_chebu(self, field: np.array, field_name: str):
+        """
         interpolate at R=r using Cheb U.
         """
-    print(f"Interpolating at R={self.r}: {field_name}", flush=True)
+        print(f"Interpolating at R={self.r}: {field_name}", flush=True)
 
-    field_r = np.zeros(shape=(len([g_re, g_im]), self.len_t, self.len_lm))
-    for k in range(self.len_n):
-      field_r[:, :, :] += field[:, :, k, :] * self.Uk[k]
+        field_r = np.zeros(shape=(len([g_re, g_im]), self.len_t, self.len_lm))
+        for k in range(self.len_n):
+            field_r[:, :, :] += field[:, :, k, :] * self.Uk[k]
 
-    return field_r
+        return field_r
 
-  def interpolate(self, field: np.array, field_name: str):
-    return self.interp(field, field_name)
+    def interpolate(self, field: np.array, field_name: str):
+        return self.interp(field, field_name)
 
 
 def process_field(field_name: str) -> dict:
-  """
+    """
     - read data
     - find time derives
     - find radial derives
     - interpolate at R=r
     """
 
-  # return
-  attrs = g_attrs
-  args = g_args
-  db = {}
+    # return
+    attrs = g_attrs
+    args = g_args
+    db = {}
 
-  # load data
-  field = load(args["fpath"], field_name, attrs)
-  # db[f"{field_name}"] = field
+    # load data
+    field = load(args["fpath"], field_name, attrs)
+    # db[f"{field_name}"] = field
 
-  # radial expansion
-  field = radial_expansion(field, field_name, attrs, args)
+    # radial expansion
+    field = radial_expansion(field, field_name, attrs, args)
 
-  # time derivative
-  dfield_dt = time_derivative(field, field_name, attrs, args)
+    # time derivative
+    dfield_dt = time_derivative(field, field_name, attrs, args)
 
-  # interpolate at a specific radii
-  interpolate = Interpolate_at_r(attrs, args)
-  field_at_r = interpolate.interpolate(field, field_name)
-  db[f"{field_name}|r"] = field_at_r
+    # interpolate at a specific radii
+    interpolate = Interpolate_at_r(attrs, args)
+    field_at_r = interpolate.interpolate(field, field_name)
+    db[f"{field_name}|r"] = field_at_r
 
-  dfield_dt_at_r = interpolate.interpolate(dfield_dt, f"d{field_name}/dt")
-  db[f"d{field_name}/dt|r"] = dfield_dt_at_r
+    dfield_dt_at_r = interpolate.interpolate(dfield_dt, f"d{field_name}/dt")
+    db[f"d{field_name}/dt|r"] = dfield_dt_at_r
 
-  # radial derivative at R=r
-  dfield_dr_at_r = radial_derivative_at_r(field, field_name, attrs, args)
-  db[f"d{field_name}/dr|r"] = dfield_dr_at_r
+    # radial derivative at R=r
+    dfield_dr_at_r = radial_derivative_at_r(field, field_name, attrs, args)
+    db[f"d{field_name}/dr|r"] = dfield_dr_at_r
 
-  return db
+    return db
 
 
 def h5_create_group(h5file, group_name: str):
-  """
+    """
     create a group for h5
     """
-  h5group = None
+    h5group = None
 
-  # create group if not exists
-  if h5file.get(group_name, default=None) == None:
-    h5group = h5file.create_group(group_name)
-  else:
-    raise ValueError("this group {group_name} is already exists.")
+    # create group if not exists
+    if h5file.get(group_name, default=None) == None:
+        h5group = h5file.create_group(group_name)
+    else:
+        raise ValueError("this group {group_name} is already exists.")
 
-  return h5group
+    return h5group
 
 
-def h5_write_data(h5file,
-                  data: np.array,
-                  data_name: str,
-                  attrs: dict,
-                  args: dict):
-  """
+def h5_write_data(h5file, data: np.array, data_name: str, attrs: dict, args: dict):
+    """
     reminder:
       data[real/imag, time_level, lm]
 
@@ -902,82 +895,80 @@ def h5_write_data(h5file,
 
     """
 
-  dataset_conf = dict(
-      name=f"{data_name}",
-      shape=(
-          attrs["lev_t"],
-          len([g_re, g_im]) * (attrs["max_l"] + 1)**2 +
-          1, ## the last +1 for time
-      ),
-      dtype=float, # chunks=True,
-      # compression="gzip",
-      # shuffle=True,
-  )
+    dataset_conf = dict(
+        name=f"{data_name}",
+        shape=(
+            attrs["lev_t"],
+            len([g_re, g_im]) * (attrs["max_l"] + 1) ** 2 + 1,  ## the last +1 for time
+        ),
+        dtype=float,  # chunks=True,
+        # compression="gzip",
+        # shuffle=True,
+    )
 
-  data_attrs = ["time"]
+    data_attrs = ["time"]
 
-  if args["debug"] == "y":
-    print(dataset_conf, flush=True)
+    if args["debug"] == "y":
+        print(dataset_conf, flush=True)
 
-  h5file.create_dataset(**dataset_conf)
+    h5file.create_dataset(**dataset_conf)
 
-  flat = 0
-  h5file[f"{data_name}"][:, flat] = attrs["time"]
-  flat += 1
-  for l in range(0, attrs["max_l"] + 1):
-    for m in range(l, -l - 1, -1):
-      assert not np.any(np.isnan(data[g_re, :, lm_mode(l, m)]))
-      assert not np.any(np.isnan(data[g_im, :, lm_mode(l, m)]))
+    flat = 0
+    h5file[f"{data_name}"][:, flat] = attrs["time"]
+    flat += 1
+    for l in range(0, attrs["max_l"] + 1):
+        for m in range(l, -l - 1, -1):
+            assert not np.any(np.isnan(data[g_re, :, lm_mode(l, m)]))
+            assert not np.any(np.isnan(data[g_im, :, lm_mode(l, m)]))
 
-      data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
-      data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")
-      h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(l, m)]
-      h5file[f"{data_name}"][:, flat + 1] = data[g_im, :, lm_mode(l, m)]
-      flat += 2
-  Legend = [s.encode("ascii", "ignore") for s in data_attrs]
-  h5file[f"{data_name}"].attrs["Legend"] = Legend
+            data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
+            data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")
+            h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(l, m)]
+            h5file[f"{data_name}"][:, flat + 1] = data[g_im, :, lm_mode(l, m)]
+            flat += 2
+    Legend = [s.encode("ascii", "ignore") for s in data_attrs]
+    h5file[f"{data_name}"].attrs["Legend"] = Legend
 
 
 def write(f: str, db: dict, attrs: dict, args: dict):
-  """
+    """
     write data on disk
     """
-  print(f"writing: {f}", flush=True)
+    print(f"writing: {f}", flush=True)
 
-  field_name = g_name_map[f"{f}"]
-  field_name_key = f"{field_name}.dat"
-  dfield_name_dr_key = f"Dr{field_name}.dat"
-  dfield_name_dt_key = f"Dt{field_name}.dat"
+    field_name = g_name_map[f"{f}"]
+    field_name_key = f"{field_name}.dat"
+    dfield_name_dr_key = f"Dr{field_name}.dat"
+    dfield_name_dt_key = f"Dt{field_name}.dat"
 
-  r = args["radius"]
-  file_name = os.path.join(args["d_out"], f"CceR{r:07.2f}.h5")
-  with h5py.File(file_name, "a") as h5file:
+    r = args["radius"]
+    file_name = os.path.join(args["d_out"], f"CceR{r:07.2f}.h5")
+    with h5py.File(file_name, "a") as h5file:
+        name = field_name_key
+        data = db[f"{f}|r"]
+        h5_write_data(h5file, data, name, attrs, args)
 
-    name = field_name_key
-    data = db[f"{f}|r"]
-    h5_write_data(h5file, data, name, attrs, args)
+        name = dfield_name_dr_key
+        data = db[f"d{f}/dr|r"]
+        h5_write_data(h5file, data, name, attrs, args)
 
-    name = dfield_name_dr_key
-    data = db[f"d{f}/dr|r"]
-    h5_write_data(h5file, data, name, attrs, args)
-
-    name = dfield_name_dt_key
-    data = db[f"d{f}/dt|r"]
-    h5_write_data(h5file, data, name, attrs, args)
+        name = dfield_name_dt_key
+        data = db[f"d{f}/dt|r"]
+        h5_write_data(h5file, data, name, attrs, args)
 
 
 def main(args):
-  """
+    """
     create output required by Specter code
     ref: https://spectre-code.org/tutorial_cce.html
     """
 
-  global g_attrs
-  global g_args
-  global g_sign
+    global g_attrs
+    global g_args
+    global g_sign
 
-  g_args = args
-  """
+    g_args = args
+    """
     note:
     in Pitnull:
     r = 0.5[(r_2 - r_1)*x + (r_2 + r_1)], r_2 > r_1 & x = [-1,1]
@@ -990,26 +981,26 @@ def main(args):
     => x = +1 -> r = r_1
 
   """
-  g_sign = int(-1) if args["ftype"] == "bin" else int(1)
+    g_sign = int(-1) if args["ftype"] == "bin" else int(1)
 
-  # check if output dir exist, if not, mkdir
-  if not os.path.exists(g_args["d_out"]):
-    os.makedirs(g_args["d_out"])
-  # find attribute for an arbitrary field
-  g_attrs = get_attribute(args["fpath"], type=args["ftype"])
+    # check if output dir exist, if not, mkdir
+    if not os.path.exists(g_args["d_out"]):
+        os.makedirs(g_args["d_out"])
+    # find attribute for an arbitrary field
+    g_attrs = get_attribute(args["fpath"], type=args["ftype"])
 
-  # for each field
-  # I'm afraid this method takes too much memory
-  # from multiprocessing import Pool
-  # with Pool(processes=len(g_field_names)) as p:
-  #  db = p.map(process_field, g_field_names)
+    # for each field
+    # I'm afraid this method takes too much memory
+    # from multiprocessing import Pool
+    # with Pool(processes=len(g_field_names)) as p:
+    #  db = p.map(process_field, g_field_names)
 
-  for f in g_field_names:
-    db = process_field(f)
-    # write on disk
-    write(f, db, g_attrs, g_args)
+    for f in g_field_names:
+        db = process_field(f)
+        # write on disk
+        write(f, db, g_attrs, g_args)
 
 
 if __name__ == "__main__":
-  args = parse_cli()
-  main(args.__dict__)
+    args = parse_cli()
+    main(args.__dict__)
diff --git a/scripts/cce/cce_extract_txt.py b/scripts/cce/cce_extract_txt.py
index 3904f913f..5352966a2 100755
--- a/scripts/cce/cce_extract_txt.py
+++ b/scripts/cce/cce_extract_txt.py
@@ -19,84 +19,76 @@
 
 
 def parse_cli():
-  """
+    """
     arg parser
     """
-  p = argparse.ArgumentParser(
-      description="extract cce spectre output in a txt file.")
-  p.add_argument("-f",
-                 type=str,
-                 required=True,
-                 help="/path/to/specter/cce/output.h5")
-  p.add_argument("-dout",
-                 type=str,
-                 required=True,
-                 help="/path/to/output/file.txt")
-  p.add_argument(
-      "-field",
-      type=str,
-      default="Psi4",
-      help="field names:[News,Psi0,Psi1,Psi2,Psi3,Psi4,Strain]",
-  )
-  p.add_argument(
-      "-mode",
-      type=str,
-      default="Y_2,2",
-      help='field modes:["Y_2,2","Y_2,-2", ...]',
-  )
-  args = p.parse_args()
-  return args
+    p = argparse.ArgumentParser(description="extract cce spectre output in a txt file.")
+    p.add_argument("-f", type=str, required=True, help="/path/to/specter/cce/output.h5")
+    p.add_argument("-dout", type=str, required=True, help="/path/to/output/file.txt")
+    p.add_argument(
+        "-field",
+        type=str,
+        default="Psi4",
+        help="field names:[News,Psi0,Psi1,Psi2,Psi3,Psi4,Strain]",
+    )
+    p.add_argument(
+        "-mode",
+        type=str,
+        default="Y_2,2",
+        help='field modes:["Y_2,2","Y_2,-2", ...]',
+    )
+    args = p.parse_args()
+    return args
 
 
 def find_h5_1mode(h5f, field_name, mode_name, args):
-  mode = 0
-  flag = False
-  for m in h5f[field_name].attrs["Legend"]:
-    if m.find(mode_name) != -1:
-      print("found mode for", field_name, m, mode_name)
-      flag = True
-      break
-    mode += 1
+    mode = 0
+    flag = False
+    for m in h5f[field_name].attrs["Legend"]:
+        if m.find(mode_name) != -1:
+            print("found mode for", field_name, m, mode_name)
+            flag = True
+            break
+        mode += 1
 
-  assert flag == True
-  return mode
+    assert flag == True
+    return mode
 
 
 def read_save_to_txt(args):
-
-  with h5py.File(args["f"], "r") as h5f:
-    # find group
-    keys = h5f.keys()
-    group = [k for k in keys if k.find("Spectre") != -1][0]
-    group = "/" + group + "/"
-    dataset = group + args["field"]
-    mode_re = find_h5_1mode(h5f, dataset, "Real " + args["mode"], args)
-    mode_im = find_h5_1mode(h5f, dataset, "Imag " + args["mode"], args)
-    t = h5f[dataset][:, 0]
-    re = h5f[dataset][:, mode_re]
-    im = h5f[dataset][:, mode_im]
-
-  mode_txt = args["mode"].replace(",", "_")
-  mode_txt = mode_txt.replace(" ", "")
-  out = os.path.join(args["dout"], args["field"] + "_" + mode_txt + ".txt")
-  # print(out)
-  stack = np.column_stack((t, re, im))
-  np.savetxt(
-      out,
-      stack,
-      header=f'# t re{args["mode"]} im{args["mode"]}',
-      fmt="%.16e %.16e %.16e",
-  )
+    with h5py.File(args["f"], "r") as h5f:
+        # find group
+        keys = h5f.keys()
+        group = [k for k in keys if k.find("Spectre") != -1][0]
+        group = "/" + group + "/"
+        dataset = group + args["field"]
+        mode_re = find_h5_1mode(h5f, dataset, "Real " + args["mode"], args)
+        mode_im = find_h5_1mode(h5f, dataset, "Imag " + args["mode"], args)
+        t = h5f[dataset][:, 0]
+        re = h5f[dataset][:, mode_re]
+        im = h5f[dataset][:, mode_im]
+
+    mode_txt = args["mode"].replace(",", "_")
+    mode_txt = mode_txt.replace(" ", "")
+    out = os.path.join(args["dout"], args["field"] + "_" + mode_txt + ".txt")
+    # print(out)
+    stack = np.column_stack((t, re, im))
+    np.savetxt(
+        out,
+        stack,
+        header=f'# t re{args["mode"]} im{args["mode"]}',
+        fmt="%.16e %.16e %.16e",
+    )
 
 
 def main(args):
-  """
+    """
     read cce spectre file
     """
 
-  read_save_to_txt(args)
+    read_save_to_txt(args)
 
 
 if __name__ == "__main__":
-  args = parse_cli()
-  main(args.__dict__)
+    args = parse_cli()
+    main(args.__dict__)
diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index c007e3c1b..3ed3ad4d7 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -31,298 +31,293 @@
 
 
 def parse_cli():
-  """
+    """
     arg parser
     """
-  p = argparse.ArgumentParser(description="debugging athk_to_spectre.py")
-  p.add_argument(
-      "-debug",
-      type=str,
-      default="plot_simple",
-      help="debug type=[plot_simple]",
-  )
-  p.add_argument(
-      "-fpath",
-      type=str,
-      required=True,
-      help="path/to/output/athk_to_spectre.py/h5",
-  )
-  p.add_argument(
-      "-dout",
-      type=str,
-      default="./",
-      help="path/to/output/dir",
-  )
-  p.add_argument(
-      "-field_name",
-      type=str,
-      default="gxx",
-      help="plot for this field [gxx,gxy,...]",
-  )
-  p.add_argument(
-      "-field_mode",
-      type=str,
-      default="Re(2,2)",
-      help="plot this mode[Re(l,m),Im(l,m)]",
-  )
-
-  p.add_argument(
-      "-time_dump",
-      type=int,
-      default=1000,
-      help="how often dump for mode convergence",
-  )
-
-  args = p.parse_args()
-  return args
+    p = argparse.ArgumentParser(description="debugging athk_to_spectre.py")
+    p.add_argument(
+        "-debug",
+        type=str,
+        default="plot_simple",
+        help="debug type=[plot_simple]",
+    )
+    p.add_argument(
+        "-fpath",
+        type=str,
+        required=True,
+        help="path/to/output/athk_to_spectre.py/h5",
+    )
+    p.add_argument(
+        "-dout",
+        type=str,
+        default="./",
+        help="path/to/output/dir",
+    )
+    p.add_argument(
+        "-field_name",
+        type=str,
+        default="gxx",
+        help="plot for this field [gxx,gxy,...]",
+    )
+    p.add_argument(
+        "-field_mode",
+        type=str,
+        default="Re(2,2)",
+        help="plot this mode[Re(l,m),Im(l,m)]",
+    )
+
+    p.add_argument(
+        "-time_dump",
+        type=int,
+        default=1000,
+        help="how often dump for mode convergence",
+    )
+
+    args = p.parse_args()
+    return args
 
 
 def find_h5_1mode(h5f, field_name, mode_name, args):
-  mode = 0
-  flag = False
-  for m in h5f[field_name].attrs["Legend"]:
-    if m.find(mode_name) != -1:
-      print("found mode for", field_name, m, mode_name)
-      flag = True
-      break
-    mode += 1
+    mode = 0
+    flag = False
+    for m in h5f[field_name].attrs["Legend"]:
+        if m.find(mode_name) != -1:
+            print("found mode for", field_name, m, mode_name)
+            flag = True
+            break
+        mode += 1
 
-  assert flag == True
-  return mode
+    assert flag == True
+    return mode
 
 
 def find_h5_all_modes(h5f, field_name, mode_name, args):
-  modes = []
-  # names = []
-  re_or_im = mode_name[0:2] ## Re(...)
-  # print(re_or_im)
-  assert re_or_im == "Re" or re_or_im == "Im"
+    modes = []
+    # names = []
+    re_or_im = mode_name[0:2]  ## Re(...)
+    # print(re_or_im)
+    assert re_or_im == "Re" or re_or_im == "Im"
 
-  legends = h5f[field_name].attrs["Legend"]
-  for i in range(len(legends)):
-    m = legends[i]
-    if m.find(re_or_im) != -1:
-      # names.append(m)
-      modes.append(i)
+    legends = h5f[field_name].attrs["Legend"]
+    for i in range(len(legends)):
+        m = legends[i]
+        if m.find(re_or_im) != -1:
+            # names.append(m)
+            modes.append(i)
 
-  assert len(modes)
+    assert len(modes)
 
-  # print(modes)
-  # print(names)
+    # print(modes)
+    # print(names)
 
-  return modes
+    return modes
 
 
 def read_h5_1mode(args):
-  """
+    """
     return t, f(t)|mode, df(t)/dr|mod, df(t)/dt|mode
     """
 
-  field_name = g_name_map[args["field_name"]]
-  field_name_key = f"{field_name}.dat"
-  dfield_name_dr_key = f"Dr{field_name}.dat"
-  dfield_name_dt_key = f"Dt{field_name}.dat"
+    field_name = g_name_map[args["field_name"]]
+    field_name_key = f"{field_name}.dat"
+    dfield_name_dr_key = f"Dr{field_name}.dat"
+    dfield_name_dt_key = f"Dt{field_name}.dat"
 
-  with h5py.File(args["fpath"], "r") as h5f:
-    mode = find_h5_1mode(h5f, f"{field_name_key}", args["field_mode"], args)
-    t = h5f[f"{field_name_key}"][:, 0]
-    f = h5f[f"{field_name_key}"][:, mode]
+    with h5py.File(args["fpath"], "r") as h5f:
+        mode = find_h5_1mode(h5f, f"{field_name_key}", args["field_mode"], args)
+        t = h5f[f"{field_name_key}"][:, 0]
+        f = h5f[f"{field_name_key}"][:, mode]
 
-    mode = find_h5_1mode(h5f, f"{dfield_name_dr_key}", args["field_mode"], args)
-    drf = h5f[f"{dfield_name_dr_key}"][:, mode]
+        mode = find_h5_1mode(h5f, f"{dfield_name_dr_key}", args["field_mode"], args)
+        drf = h5f[f"{dfield_name_dr_key}"][:, mode]
 
-    mode = find_h5_1mode(h5f, f"{dfield_name_dt_key}", args["field_mode"], args)
-    dtf = h5f[f"{dfield_name_dt_key}"][:, mode]
+        mode = find_h5_1mode(h5f, f"{dfield_name_dt_key}", args["field_mode"], args)
+        dtf = h5f[f"{dfield_name_dt_key}"][:, mode]
 
-  return (f, drf, dtf, t)
+    return (f, drf, dtf, t)
 
 
 def read_h5_all_modes(args) -> dict:
-  """
+    """
     return t, f(mode)|t, df(mode)/dr|t, df(mode)/dt|t
 
     f[t_i,modes] = at t_i what are the modes
     """
 
-  dat = {}
-  field_name = g_name_map[args["field_name"]]
-  field_name_key = f"{field_name}.dat"
-  dfield_name_dr_key = f"Dr{field_name}.dat"
-  dfield_name_dt_key = f"Dt{field_name}.dat"
-
-  with h5py.File(args["fpath"], "r") as h5f:
-    t = h5f[f"{field_name_key}"][:, 0]
-    n_dumps = len(t) // args["time_dump"]
-    n_dumps = n_dumps + 1 if len(t) % args["time_dump"] else n_dumps
-
-    modes = find_h5_all_modes(h5f,
-                              f"{field_name_key}",
-                              args["field_mode"],
-                              args)
-    f = np.empty(shape=(n_dumps, len(modes)))
-    k = 0 # note: f[k,.] is 0,1,... but i jumps
-    for i in range(0, len(t), args["time_dump"]):
-      for j in range(len(modes)):
-        f[k, j] = h5f[f"{field_name_key}"][i, modes[j]]
-      k += 1
-
-    modes = find_h5_all_modes(h5f,
-                              f"{dfield_name_dr_key}",
-                              args["field_mode"],
-                              args)
-    drf = np.empty(shape=(n_dumps, len(modes)))
-    k = 0
-    for i in range(0, len(t), args["time_dump"]):
-      for j in range(len(modes)):
-        drf[k, j] = h5f[f"{dfield_name_dr_key}"][i, modes[j]]
-      k += 1
-
-    modes = find_h5_all_modes(h5f,
-                              f"{dfield_name_dt_key}",
-                              args["field_mode"],
-                              args)
-    dtf = np.empty(shape=(n_dumps, len(modes)))
-    k = 0
-    for i in range(0, len(t), args["time_dump"]):
-      for j in range(len(modes)):
-        dtf[k, j] = h5f[f"{dfield_name_dt_key}"][i, modes[j]]
-      k += 1
-
-  dat["t"] = t
-  dat["f"] = f
-  dat["drf"] = drf
-  dat["dtf"] = dtf
-  dat["n_dumps"] = n_dumps
-
-  return dat
+    dat = {}
+    field_name = g_name_map[args["field_name"]]
+    field_name_key = f"{field_name}.dat"
+    dfield_name_dr_key = f"Dr{field_name}.dat"
+    dfield_name_dt_key = f"Dt{field_name}.dat"
+
+    with h5py.File(args["fpath"], "r") as h5f:
+        t = h5f[f"{field_name_key}"][:, 0]
+        n_dumps = len(t) // args["time_dump"]
+        n_dumps = n_dumps + 1 if len(t) % args["time_dump"] else n_dumps
+
+        modes = find_h5_all_modes(h5f, f"{field_name_key}", args["field_mode"], args)
+        f = np.empty(shape=(n_dumps, len(modes)))
+        k = 0  # note: f[k,.] is 0,1,... but i jumps
+        for i in range(0, len(t), args["time_dump"]):
+            for j in range(len(modes)):
+                f[k, j] = h5f[f"{field_name_key}"][i, modes[j]]
+            k += 1
+
+        modes = find_h5_all_modes(
+            h5f, f"{dfield_name_dr_key}", args["field_mode"], args
+        )
+        drf = np.empty(shape=(n_dumps, len(modes)))
+        k = 0
+        for i in range(0, len(t), args["time_dump"]):
+            for j in range(len(modes)):
+                drf[k, j] = h5f[f"{dfield_name_dr_key}"][i, modes[j]]
+            k += 1
+
+        modes = find_h5_all_modes(
+            h5f, f"{dfield_name_dt_key}", args["field_mode"], args
+        )
+        dtf = np.empty(shape=(n_dumps, len(modes)))
+        k = 0
+        for i in range(0, len(t), args["time_dump"]):
+            for j in range(len(modes)):
+                dtf[k, j] = h5f[f"{dfield_name_dt_key}"][i, modes[j]]
+            k += 1
+
+    dat["t"] = t
+    dat["f"] = f
+    dat["drf"] = drf
+    dat["dtf"] = dtf
+    dat["n_dumps"] = n_dumps
+
+    return dat
 
 
 def plot_simple_v_t(dat, args):
-  """
-    plot value vs time
     """
-
-  print("plot value vs time ...")
-  fig, axes = plt.subplots(3, 1, sharex=True)
-
-  # f
-  ax = axes[0]
-  label = args["field_name"]
-  conf = dict(ls="-", label=label, color="k")
-  ax.plot(dat[-1], dat[0], **conf)
-  ax.set_ylabel(label)
-  ax.grid(True)
-
-  # drf
-  ax = axes[1]
-  label = "d" + args["field_name"] + "/dr"
-  conf = dict(ls="-", label=label, color="k")
-  ax.plot(dat[-1], dat[1], **conf)
-  ax.set_ylabel(label)
-  ax.grid(True)
-
-  # dtf
-  ax = axes[2]
-  label = "d" + args["field_name"] + "/dt"
-  conf = dict(ls="-", label=label, color="k")
-  ax.plot(dat[-1], dat[2], **conf)
-  ax.grid(True)
-  ax.set_ylabel(label)
-  ax.set_xlabel("t/M")
-
-  plt.tight_layout()
-  # plt.show()
-
-  mode = args["field_mode"][3:-1]
-  lm = mode.split(",")
-  file_out = os.path.join(
-      args["dout"],
-      args["field_name"] + "_" + f"l{lm[0]}m{lm[1]}" + "_vs_time.png",
-  )
-  plt.savefig(file_out, dpi=200)
-
-
-def plot_simple_modes(dat, args):
-  """
-    plot modes for different times
+    plot value vs time
     """
 
-  print("plot modes for different times ...")
-
-  t = dat["t"]
-  f = dat["f"]
-  drf = dat["drf"]
-  dtf = dat["dtf"]
-  n_dumps = dat["n_dumps"]
-  x = np.arange(0, len(f[0,:]), dtype=int)
-  p = 0
-  for i in range(0, len(t), args["time_dump"]):
-    title = f"t_{t[i]}"
-
+    print("plot value vs time ...")
     fig, axes = plt.subplots(3, 1, sharex=True)
 
     # f
     ax = axes[0]
-    ax.set_title(title)
     label = args["field_name"]
     conf = dict(ls="-", label=label, color="k")
-    ax.plot(x, np.abs(f[p, :]), **conf)
+    ax.plot(dat[-1], dat[0], **conf)
     ax.set_ylabel(label)
-    ax.set_yscale('log')
     ax.grid(True)
 
     # drf
     ax = axes[1]
     label = "d" + args["field_name"] + "/dr"
     conf = dict(ls="-", label=label, color="k")
-    ax.plot(x, np.abs(drf[p,:]), **conf)
+    ax.plot(dat[-1], dat[1], **conf)
     ax.set_ylabel(label)
-    ax.set_yscale('log')
     ax.grid(True)
 
     # dtf
     ax = axes[2]
     label = "d" + args["field_name"] + "/dt"
     conf = dict(ls="-", label=label, color="k")
-    ax.plot(x, np.abs(dtf[p,:]), **conf)
+    ax.plot(dat[-1], dat[2], **conf)
     ax.grid(True)
     ax.set_ylabel(label)
-    ax.set_yscale('log')
-    ax.set_xlabel("modes")
+    ax.set_xlabel("t/M")
 
     plt.tight_layout()
     # plt.show()
 
+    mode = args["field_mode"][3:-1]
+    lm = mode.split(",")
     file_out = os.path.join(
         args["dout"],
-        args["field_name"] + "_" + "modes_" + f"{title}.png",
+        args["field_name"] + "_" + f"l{lm[0]}m{lm[1]}" + "_vs_time.png",
     )
     plt.savefig(file_out, dpi=200)
-    plt.close()
-    p += 1
+
+
+def plot_simple_modes(dat, args):
+    """
+    plot modes for different times
+    """
+
+    print("plot modes for different times ...")
+
+    t = dat["t"]
+    f = dat["f"]
+    drf = dat["drf"]
+    dtf = dat["dtf"]
+    n_dumps = dat["n_dumps"]
+    x = np.arange(0, len(f[0, :]), dtype=int)
+    p = 0
+    for i in range(0, len(t), args["time_dump"]):
+        title = f"t_{t[i]}"
+
+        fig, axes = plt.subplots(3, 1, sharex=True)
+
+        # f
+        ax = axes[0]
+        ax.set_title(title)
+        label = args["field_name"]
+        conf = dict(ls="-", label=label, color="k")
+        ax.plot(x, np.abs(f[p, :]), **conf)
+        ax.set_ylabel(label)
+        ax.set_yscale("log")
+        ax.grid(True)
+
+        # drf
+        ax = axes[1]
+        label = "d" + args["field_name"] + "/dr"
+        conf = dict(ls="-", label=label, color="k")
+        ax.plot(x, np.abs(drf[p, :]), **conf)
+        ax.set_ylabel(label)
+        ax.set_yscale("log")
+        ax.grid(True)
+
+        # dtf
+        ax = axes[2]
+        label = "d" + args["field_name"] + "/dt"
+        conf = dict(ls="-", label=label, color="k")
+        ax.plot(x, np.abs(dtf[p, :]), **conf)
+        ax.grid(True)
+        ax.set_ylabel(label)
+        ax.set_yscale("log")
+        ax.set_xlabel("modes")
+
+        plt.tight_layout()
+        # plt.show()
+
+        file_out = os.path.join(
+            args["dout"],
+            args["field_name"] + "_" + "modes_" + f"{title}.png",
+        )
+        plt.savefig(file_out, dpi=200)
+        plt.close()
+        p += 1
 
 
 def debug_plot_simple(args):
-  # value vs time
-  dat = read_h5_1mode(args)
-  plot_simple_v_t(dat, args)
+    # value vs time
+    dat = read_h5_1mode(args)
+    plot_simple_v_t(dat, args)
 
-  # conv test
-  dat = read_h5_all_modes(args)
-  plot_simple_modes(dat, args)
+    # conv test
+    dat = read_h5_all_modes(args)
+    plot_simple_modes(dat, args)
 
 
 def main(args):
-  """
+    """
     debug
     """
 
-  if args["debug"] == "plot_simple":
-    debug_plot_simple(args)
-  else:
-    raise ValueError("no such option")
+    if args["debug"] == "plot_simple":
+        debug_plot_simple(args)
+    else:
+        raise ValueError("no such option")
 
 
 if __name__ == "__main__":
-  args = parse_cli()
-  main(args.__dict__)
+    args = parse_cli()
+    main(args.__dict__)

From 89a7c3b23ff8b79993e0fd98cbef7195ec509d5a Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Wed, 26 Feb 2025 15:50:58 -0500
Subject: [PATCH 69/72] fix c++ linting error

---
 src/pgen/pgen.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pgen/pgen.cpp b/src/pgen/pgen.cpp
index 39fa648e9..b50ce804a 100644
--- a/src/pgen/pgen.cpp
+++ b/src/pgen/pgen.cpp
@@ -187,8 +187,8 @@ ProblemGenerator::ProblemGenerator(ParameterInput *pin, Mesh *pm, IOWrapper resf
       }
       if (resfile.Read_Reals(&cce_dump_last_output_time, 1,single_file_per_rank) != 1) {
         std::cout << "### FATAL ERROR in " << __FILE__ << " at line " << __LINE__
-                  << std::endl << "z4c::cce_dump_last_output_time data size read from restart "
-                  << "file is incorrect, restart file is broken." << std::endl;
+          << std::endl << "z4c::cce_dump_last_output_time data size read from restart "
+          << "file is incorrect, restart file is broken." << std::endl;
         exit(EXIT_FAILURE);
       }
     }

From 1dc087227e9c98a4ff462344016470b26ca287f3 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Wed, 26 Feb 2025 16:07:33 -0500
Subject: [PATCH 70/72] fix python linting

---
 scripts/cce/athk_to_spectre.py       | 126 +++++++++++++--------------
 scripts/cce/cce_extract_txt.py       |  15 ++--
 scripts/cce/debug_athk_to_spectre.py |  17 ++--
 src/outputs/restart.cpp              |   4 +-
 4 files changed, 77 insertions(+), 85 deletions(-)

diff --git a/scripts/cce/athk_to_spectre.py b/scripts/cce/athk_to_spectre.py
index 8d42f3cb6..13792f8b0 100755
--- a/scripts/cce/athk_to_spectre.py
+++ b/scripts/cce/athk_to_spectre.py
@@ -1,18 +1,16 @@
 #!/usr/bin/env python3
-## Alireza Rashti - Oct 2024 (C)
-## usage:
-## $ ./me -h
-##
-## BUG:
-## -[] see TODOs
-## -[] see NOTEs
-
-import sys
+# Alireza Rashti - Oct 2024 (C)
+# usage:
+# $ ./me -h
+#
+# BUG:
+# -[] see TODOs
+# -[] see NOTEs
+
 import os
 import numpy as np
 from scipy import special
 import math
-import cmath
 import argparse
 import h5py
 import struct
@@ -25,9 +23,9 @@
 # import sympy
 # import re
 
-## ---------------------------------------------------------------------- ##
+# ---------------------------------------------------------------------- #
 
-## field names
+# field names
 g_field_names = [
     "gxx",
     "gxy",
@@ -67,19 +65,19 @@
     "alp": 0,
 }
 
-## real/imag
+# real/imag
 g_re = 0
 g_im = 1
 
-## args
+# args
 g_args = None
-## various attrs
+# various attrs
 g_attrs = None
 
-## sign convention
+# sign convention
 g_sign = None
 
-## debug
+# debug
 g_debug_max_l = 2
 
 
@@ -115,7 +113,8 @@ def parse_cli():
         "-r_deriv",
         type=str,
         default="ChebU",
-        help="method to take the radial derivative of fields:{ChebU:Chebyshev of second kind}",
+        help="method to take the radial derivative of \
+            fields:{ChebU:Chebyshev of second kind}",
     )
     p.add_argument(
         "-interpolation",
@@ -196,10 +195,10 @@ def _ylm_on_gl(self):
             th = self.th[i]
             for j in range(self.npnts):
                 ph = self.ph[j]
-                for l in range(0, self.attrs["max_l"] + 1):
-                    for m in range(l, -l - 1, -1):
-                        lm = lm_mode(l, m)
-                        y = special.sph_harm(m, l, ph, th)
+                for ll in range(0, self.attrs["max_l"] + 1):
+                    for m in range(ll, -ll - 1, -1):
+                        lm = lm_mode(ll, m)
+                        y = special.sph_harm(m, ll, ph, th)
                         # assert not np.isnan(y)
                         ylms[g_re, i, j, lm] = y.real
                         ylms[g_im, i, j, lm] = y.imag
@@ -310,14 +309,13 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
 
     elif attrs["file_type"] == "bin":
         # Load the list of files
-        ##TODO: this depends on file name
+        # TODO: this depends on file name
         flist = sorted(glob.glob(fpath + "/cce_*.bin"))
-        data = []
         dat_real = []
         dat_imag = []
         t = []
         for f in flist:
-            ##TODO: it reads all field and only use one of them. it's inefficient
+            # TODO: it reads all field and only use one of them. it's inefficient
             (
                 nr,
                 num_l_modes,
@@ -352,7 +350,7 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
             attrs["max_lm"],
         )
         ret = np.empty(shape=shape, dtype=float)
-        ## NOTE: is it fine that we do interpolation?
+        # NOTE: is it fine that we do interpolation?
         ret[g_re, :] = f_real(attrs["time"])
         ret[g_im, :] = f_imag(attrs["time"])
     else:
@@ -362,11 +360,11 @@ def load(fpath: str, field_name: str, attrs: dict) -> list:
     return ret
 
 
-def lm_mode(l, m):
+def lm_mode(ll, m):
     """
-    l and m mode convention
+    ll and m mode convention
     """
-    return l * l + l + m
+    return ll * ll + ll + m
 
 
 def read_cce_file(filename):
@@ -378,20 +376,22 @@ def read_cce_file(filename):
 
     Returns:
         nr (int): Number of radial points.
-        num_l_modes (int): Number of l modes.
+        num_l_modes (int): Number of ll modes.
         time (float): Time value from the simulation.
         rin (float): Inner radial boundary.
         rout (float): Outer radial boundary.
-        data_real (numpy.ndarray): Real part of the data, shape (nr, 10, num_angular_modes).
-        data_imag (numpy.ndarray): Imaginary part of the data, shape (nr, 10, num_angular_modes).
-        index_to_lm (dict): Mapping from data index to (l, m) values.
+        data_real (numpy.ndarray): Real part of the data, shape
+                (nr, 10, num_angular_modes).
+        data_imag (numpy.ndarray): Imaginary part of the data, shape
+                (nr, 10, num_angular_modes).
+        index_to_lm (dict): Mapping from data index to (ll, m) values.
     """
     with open(filename, "rb") as f:
         # Read number of radial points (nr)
         nr_bytes = f.read(4)
         nr = struct.unpack("<i", nr_bytes)[0]  # little-endian integer
 
-        # Read number of l modes (num_l_modes)
+        # Read number of ll modes (num_l_modes)
         num_l_modes_bytes = f.read(4)
         num_l_modes = struct.unpack("<i", num_l_modes_bytes)[0]
 
@@ -420,12 +420,12 @@ def read_cce_file(filename):
         data_imag = np.fromfile(f, dtype="<d", count=count)
         data_imag = data_imag.reshape((nr, 10, num_angular_modes))
 
-        # Create a mapping from index to (l, m)
+        # Create a mapping from index to (ll, m)
         index_to_lm = {}
-        for l in range(1, num_l_modes + 1):
-            for m in range(-l, l + 1):
-                index = l * l + l + m
-                index_to_lm[index] = (l, m)
+        for ll in range(1, num_l_modes + 1):
+            for m in range(-ll, ll + 1):
+                index = ll * ll + ll + m
+                index_to_lm[index] = (ll, m)
 
     return nr, num_l_modes, time, rin, rout, data_real, data_imag, index_to_lm
 
@@ -441,7 +441,7 @@ def get_attribute(
     if type == "h5":
         attrs["file_type"] = "h5"
         with h5py.File(fpath, "r") as h5f:
-            # find attribute about num. of time level, and n,l,m in C_nlm
+            # find attribute about num. of time level, and n,ll,m in C_nlm
             attrs["lev_t"] = len(h5f.keys()) - 1
             if attrs["lev_t"] % 2 == 0:  # for fourier transformation
                 attrs["lev_t"] -= 1
@@ -449,7 +449,7 @@ def get_attribute(
             attrs["max_n"], attrs["max_lm"] = h5f[f"1/{field_name}/re"].shape
             attrs["max_l"] = (
                 int(math.sqrt(attrs["max_lm"])) - 1
-            )  # NOTE:l must be inclusive in loops
+            )  # NOTE:ll must be inclusive in loops
             attrs["max_lm"] = int((attrs["max_l"] + 1) ** 2)
             attrs["r_in"] = h5f["metadata"].attrs["Rin"]
             attrs["r_out"] = h5f["metadata"].attrs["Rout"]
@@ -499,7 +499,7 @@ def time_derivative_fourier(
     field(rel/img,t, n,lm)
     """
 
-    ## TODO: Fourier time derives not tested!
+    # TODO: Fourier time derives not tested!
     print(f"Fourier time derivative: {field_name}", flush=True)
     _, len_t, len_n, len_lm = field.shape
     dt = attrs["time"][2] - attrs["time"][1]
@@ -511,8 +511,6 @@ def time_derivative_fourier(
             coeff = field[g_re, :, n, lm] + 1j * field[g_im, :, n, lm]
             # F. transform
             fft_coeff = np.fft.fft(coeff)
-            # if args["debug"] == 'y':
-            #  print("debug: normalization?",round(coeff[1],6) == round(np.fft.ifft(fft_coeff)[1],6))
 
             # time derivative
             half = len_t // 2 + 1
@@ -551,16 +549,16 @@ def time_derivative_fourier(
 
     if args["debug"] == "y":
         for n in range(len_n):
-            for l in range(2, g_debug_max_l + 1):
-                for m in range(-l, l + 1):
-                    hfile = f"{args['d_out']}/debug_{field_name}_n{n}l{l}m{m}.txt"
+            for ll in range(2, g_debug_max_l + 1):
+                for m in range(-ll, ll + 1):
+                    hfile = f"{args['d_out']}/debug_{field_name}_n{n}ll{ll}m{m}.txt"
                     write_data = np.column_stack(
                         (
                             attrs["time"],
-                            dfield[g_re, :, n, lm_mode(l, m)],
-                            dfield[g_im, :, n, lm_mode(l, m)],
-                            field[g_re, :, n, lm_mode(l, m)],
-                            field[g_im, :, n, lm_mode(l, m)],
+                            dfield[g_re, :, n, lm_mode(ll, m)],
+                            dfield[g_im, :, n, lm_mode(ll, m)],
+                            field[g_re, :, n, lm_mode(ll, m)],
+                            field[g_im, :, n, lm_mode(ll, m)],
                         )
                     )
                     np.savetxt(hfile, write_data, header="t dre/dt dim/dt re im")
@@ -697,7 +695,6 @@ def coefficients(self, field: np.array) -> np.array:
         """
 
         w_i = self.w_i
-        x_i = self.x_i
         Uji = self.Uj_xi
 
         coeffs = Uji @ (w_i * field[::-g_sign])
@@ -869,7 +866,7 @@ def h5_create_group(h5file, group_name: str):
     h5group = None
 
     # create group if not exists
-    if h5file.get(group_name, default=None) == None:
+    if h5file.get(group_name, default=None) is None:
         h5group = h5file.create_group(group_name)
     else:
         raise ValueError("this group {group_name} is already exists.")
@@ -885,7 +882,8 @@ def h5_write_data(h5file, data: np.array, data_name: str, attrs: dict, args: dic
     write syntax, eg:
 
     h5["gxx.dat"] =
-      [time_level, ['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)', 'gxx_Re(1,1)', 'gxx_Im(1,1)', ...] ]
+      [time_level, ['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)',
+      'gxx_Re(1,1)', 'gxx_Im(1,1)', ...] ]
 
     h5["gxx.dat"].attrs['Legend'] = the associated column =
       array(['time', 'gxx_Re(0,0)', 'gxx_Im(0,0)', 'gxx_Re(1,1)', 'gxx_Im(1,1)', ...])
@@ -899,7 +897,7 @@ def h5_write_data(h5file, data: np.array, data_name: str, attrs: dict, args: dic
         name=f"{data_name}",
         shape=(
             attrs["lev_t"],
-            len([g_re, g_im]) * (attrs["max_l"] + 1) ** 2 + 1,  ## the last +1 for time
+            len([g_re, g_im]) * (attrs["max_l"] + 1) ** 2 + 1,  # the last +1 for time
         ),
         dtype=float,  # chunks=True,
         # compression="gzip",
@@ -916,15 +914,15 @@ def h5_write_data(h5file, data: np.array, data_name: str, attrs: dict, args: dic
     flat = 0
     h5file[f"{data_name}"][:, flat] = attrs["time"]
     flat += 1
-    for l in range(0, attrs["max_l"] + 1):
-        for m in range(l, -l - 1, -1):
-            assert not np.any(np.isnan(data[g_re, :, lm_mode(l, m)]))
-            assert not np.any(np.isnan(data[g_im, :, lm_mode(l, m)]))
-
-            data_attrs.append(f"{data_name[:-4]}_Re({l},{m})")
-            data_attrs.append(f"{data_name[:-4]}_Im({l},{m})")
-            h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(l, m)]
-            h5file[f"{data_name}"][:, flat + 1] = data[g_im, :, lm_mode(l, m)]
+    for ll in range(0, attrs["max_l"] + 1):
+        for m in range(ll, -ll - 1, -1):
+            assert not np.any(np.isnan(data[g_re, :, lm_mode(ll, m)]))
+            assert not np.any(np.isnan(data[g_im, :, lm_mode(ll, m)]))
+
+            data_attrs.append(f"{data_name[:-4]}_Re({ll},{m})")
+            data_attrs.append(f"{data_name[:-4]}_Im({ll},{m})")
+            h5file[f"{data_name}"][:, flat] = data[g_re, :, lm_mode(ll, m)]
+            h5file[f"{data_name}"][:, flat + 1] = data[g_im, :, lm_mode(ll, m)]
             flat += 2
     Legend = [s.encode("ascii", "ignore") for s in data_attrs]
     h5file[f"{data_name}"].attrs["Legend"] = Legend
diff --git a/scripts/cce/cce_extract_txt.py b/scripts/cce/cce_extract_txt.py
index 5352966a2..4a78ea70e 100755
--- a/scripts/cce/cce_extract_txt.py
+++ b/scripts/cce/cce_extract_txt.py
@@ -1,21 +1,18 @@
 #!/usr/bin/env python3
-## Alireza Rashti - Jan 2025 (C)
-## usage:
-## $ ./me -h
-##
+# Alireza Rashti - Jan 2025 (C)
+# usage:
+# $ ./me -h
+#
 
-import sys
 import os
 import numpy as np
-import math as m
 import argparse
-import re
 import h5py
 
 # import matplotlib.pyplot as plt
 # import glob
 # import sympy
-## ---------------------------------------------------------------------- ##
+# ---------------------------------------------------------------------- #
 
 
 def parse_cli():
@@ -51,7 +48,7 @@ def find_h5_1mode(h5f, field_name, mode_name, args):
             break
         mode += 1
 
-    assert flag == True
+    assert flag is True
     return mode
 
 
diff --git a/scripts/cce/debug_athk_to_spectre.py b/scripts/cce/debug_athk_to_spectre.py
index 3ed3ad4d7..ea00bf122 100755
--- a/scripts/cce/debug_athk_to_spectre.py
+++ b/scripts/cce/debug_athk_to_spectre.py
@@ -1,20 +1,18 @@
 #!/usr/bin/env python3
-## Alireza Rashti - Jan 2025 (C)
-## usage:
-## $ ./me -h
-##
+# Alireza Rashti - Jan 2025 (C)
+# usage:
+# $ ./me -h
+#
 
-import sys
 import os
 import numpy as np
-import math as m
 import argparse
 import h5py
 import matplotlib.pyplot as plt
 
 # import glob
 # import sympy
-## ---------------------------------------------------------------------- ##
+# ---------------------------------------------------------------------- #
 
 g_name_map = {
     "gxx": "gxx",
@@ -87,14 +85,14 @@ def find_h5_1mode(h5f, field_name, mode_name, args):
             break
         mode += 1
 
-    assert flag == True
+    assert flag is True
     return mode
 
 
 def find_h5_all_modes(h5f, field_name, mode_name, args):
     modes = []
     # names = []
-    re_or_im = mode_name[0:2]  ## Re(...)
+    re_or_im = mode_name[0:2]  # Re(...)
     # print(re_or_im)
     assert re_or_im == "Re" or re_or_im == "Im"
 
@@ -248,7 +246,6 @@ def plot_simple_modes(dat, args):
     f = dat["f"]
     drf = dat["drf"]
     dtf = dat["dtf"]
-    n_dumps = dat["n_dumps"]
     x = np.arange(0, len(f[0, :]), dtype=int)
     p = 0
     for i in range(0, len(t), args["time_dump"]):
diff --git a/src/outputs/restart.cpp b/src/outputs/restart.cpp
index 8c7a3bd93..49c5a138a 100644
--- a/src/outputs/restart.cpp
+++ b/src/outputs/restart.cpp
@@ -247,9 +247,9 @@ void RestartOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin) {
   if (global_variable::my_rank == 0 || single_file_per_rank) {
     // store z4c information
     if (pz4c != nullptr) {
-      resfile.Write_any_type(&(pz4c->last_output_time), sizeof(Real), "byte"),
+      resfile.Write_any_type(&(pz4c->last_output_time), sizeof(Real), "byte",
                              single_file_per_rank);
-      resfile.Write_any_type(&(pz4c->cce_dump_last_output_time), sizeof(Real), "byte"),
+      resfile.Write_any_type(&(pz4c->cce_dump_last_output_time), sizeof(Real), "byte",
                              single_file_per_rank);
     }
     // output puncture tracker data

From 65351d8e360d1aa9424203dcceea0e75b85cda20 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Wed, 5 Mar 2025 11:27:34 -0500
Subject: [PATCH 71/72] fix compilation

---
 src/pgen/pgen.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pgen/pgen.cpp b/src/pgen/pgen.cpp
index b50ce804a..e39fe0e6f 100644
--- a/src/pgen/pgen.cpp
+++ b/src/pgen/pgen.cpp
@@ -196,6 +196,7 @@ ProblemGenerator::ProblemGenerator(ParameterInput *pin, Mesh *pm, IOWrapper resf
     if (!single_file_per_rank) {
       MPI_Bcast(&last_output_time, sizeof(Real), MPI_CHAR, 0, MPI_COMM_WORLD);
       MPI_Bcast(&cce_dump_last_output_time, sizeof(Real), MPI_CHAR, 0, MPI_COMM_WORLD);
+    }
 #endif
     pz4c->last_output_time = last_output_time;
     pz4c->cce_dump_last_output_time = cce_dump_last_output_time;

From 5ef04103a4e685a69c1d2163a6368840abc1e935 Mon Sep 17 00:00:00 2001
From: Hengrui Zhu <hz0693@princeton.edu>
Date: Fri, 7 Mar 2025 23:37:36 -0500
Subject: [PATCH 72/72] fix z4c restart

---
 src/outputs/restart.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/outputs/restart.cpp b/src/outputs/restart.cpp
index 49c5a138a..b9724f5c1 100644
--- a/src/outputs/restart.cpp
+++ b/src/outputs/restart.cpp
@@ -303,7 +303,7 @@ void RestartOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin) {
   IOWrapperSizeT step2size = (pm->nmb_total)*(sizeof(LogicalLocation) + sizeof(float));
 
   IOWrapperSizeT step3size = 3*nco*sizeof(Real);
-  if (pz4c != nullptr) step3size += sizeof(Real);
+  if (pz4c != nullptr) step3size += 2*sizeof(Real);
   if (pturb != nullptr) step3size += sizeof(RNG_State);
 
   // write cell-centered variables in parallel