add g(r) fitting into kafka

scripts/_data_export.py

@@ -200,6 +200,9 @@ def read_qepro_by_stream(uid, stream_name='primary', data_agent='tiled', beamlin
 
 
 
+## TODO: add reading scattering data
+# def read_scatter_by_stream(uid, stream_name='scattering', data_agent='tiled', beamline_acronym='xpd-ldrd20-31'):
+#     return scatter_dic, metadata_dic
 
 ## Get uids from data folders through daily experiments for export database for ML
 ## Read uid from **fluorescence** folder

scripts/_pdf_calculator.py

@@ -0,0 +1,136 @@
+import pymatgen as pm
+from diffpy.pdffit2 import PdfFit
+from diffpy.Structure import loadStructure
+from pymatgen.io.cif import CifWriter
+from pymatgen.io.cif import CifParser
+import os
+
+
+
+def _no_oxidation_cif(cif_file):
+    parser = CifParser(cif_file)
+    structure = parser.get_structures()[0] 
+    structure.remove_oxidation_states()
+    print(structure, '\n\n')
+    cif_pym = os.path.dirname(cif) + '/' + os.path.basename(cif)[:-4] + '_pym.cif'
+    w = CifWriter(structure,symprec=0.1)
+    w.write_file(cif_pym)
+
+    return cif_pym
+
+
+
+def _pdffit2_CsPbX3(gr_data, cif_list, qmax=20, qdamp=0.031, qbroad=0.032):
+
+    # Initialize the CifParser with the path to your .cif file
+    # Parse the .cif file
+    pym_cif = []
+    for cif in cif_list:
+        cif_new = _no_oxidation_cif(cif)
+        pym_cif.append(cif_new)
+
+    # Create new PDF calculator object.
+    pf = PdfFit()
+
+    # Load experimental x-ray PDF data
+    # qmax = 20.0  # Q-cutoff used in PDF calculation in 1/A
+    # qdamp = 0.03 # instrument Q-resolution factor, responsible for PDF decay
+    pf.read_data(gr_data, 'X', qmax, qdamp)
+
+    # Load and add structure ------------------------------------------------------------------
+    for pym in pym_cif:
+        stru = loadStructure(pym)
+        stru.Uisoequiv = 0.04
+        stru.title = os.path.basename(pym)[:-4]
+        #Add loaded .cif 
+        pf.add_structure(stru)
+
+    # set contrains for lattice parameter, ADP
+    _set_CsPbBr3_constrain(pf)
+
+    # set constrain for data scale
+    pf.constrain(pf.dscale, '@902')
+    pf.setpar(902, 0.65)
+
+    # Set value for Qdamp, Qbraod
+    pf.setvar(pf.qdamp, qdamp)
+    pf.setvar(pf.qbroad, qbroad)
+
+    phase_fraction = pf.phase_fractions()['mass']
+
+    particel_size = []
+    for i in range(pf.num_phases()):
+        pf.setphase(i+1)
+        particel_size.append(pf.getvar(pf.spdiameter))
+
+    return phase_fraction, particel_size
+
+
+
+'''
+lat_dic = {'a':11, 'b':12, 'c':13}
+'''
+def _set_CsPbBr3_constrain(PDF_calculator_object, fix_APD=True):
+    pf = PDF_calculator_object
+    pf.setphase(phase_idx)
+
+    # Refine lattice parameters a, b, c.
+    pf.constrain(pf.lat(1), "@11")
+    pf.constrain(pf.lat(2), "@12")
+    pf.constrain(pf.lat(3), "@13")
+    # set initial value of parameter @1, @2, @3
+    pf.setpar(11, pf.lat(1))
+    pf.setpar(12, pf.lat(2))
+    pf.setpar(13, pf.lat(3))
+
+    # Refine phase scale factor.  Right side can have formulas.
+    pf.constrain('pscale', '@111')
+    pf.setpar(111, 0.5)
+    # pf.setpar(20, pf.getvar(pf.pscale) / 2.0)
+
+    # Refine sharpening factor for correlated motion of close atoms.
+    pf.constrain(pf.delta2, '@122')
+    pf.setpar(122, 6.87)
+
+    # Refine diameter for the spherical particle
+    pf.constrain(pf.spdiameter, '@133')
+    pf.setpar(133, 46)
+
+    # Set temperature factors isotropic to each atom
+    # idx starts from 1 not 0
+    # 1-4: Cs
+    for idx in range(1, 5):
+        pf.constrain(pf.u11(idx), '@101')
+        pf.constrain(pf.u22(idx), '@101')
+        pf.constrain(pf.u33(idx), '@101')
+    pf.setpar(101, 0.029385)
+
+    # 5-8: Pb
+    for idx in range(5, 9):
+        pf.constrain(pf.u11(idx), '@102')
+        pf.constrain(pf.u22(idx), '@102')
+        pf.constrain(pf.u33(idx), '@102')
+    pf.setpar(102, 0.027296)
+
+    # 9-16: Br
+    for idx in range(9, 17):
+        pf.constrain(pf.u11(idx), '@103')
+        pf.constrain(pf.u22(idx), '@103')
+        pf.constrain(pf.u33(idx), '@103')
+    pf.setpar(103, 0.041577)
+
+    # 16-20: Br
+    for idx in range(17, 21):
+        pf.constrain(pf.u11(idx), '@104')
+        pf.constrain(pf.u22(idx), '@104')
+        pf.constrain(pf.u33(idx), '@104')
+    pf.setpar(104, 0.028164)
+
+
+    if fix_APD:
+        for par in [101, 102, 103, 104]:
+            pf.fixpar(101)
+
+
+
+

scripts/kafka_consumer_iterate_RM.py

@@ -20,6 +20,7 @@
 import _data_export as de
 from _plot_helper import plot_uvvis
 import _data_analysis as da
+import _pdf_calculator as pc
 
 # from bluesky_queueserver.manager.comms import zmq_single_request
 from bluesky_queueserver_api.zmq import REManagerAPI
@@ -241,8 +242,11 @@ def print_message(consumer, doctype, doc,
                 hei = height[0]
                 dis = distance[0]
 
-            ## remove 'scattering' from stream_list to avoid redundant work in next for loop
+            ## obtain phase fraction & particle size from g(r)
             if 'scattering' in stream_list:
+                phase_fraction, particel_size = pc._pdffit2_CsPbX3(gr_data, cif_list, qmax=20, qdamp=0.031, qbroad=0.032)
+
+                ## remove 'scattering' from stream_list to avoid redundant work in next for loop
                 stream_list.remove('scattering')
 
             ## Export, plotting, fitting, calculate # of good/bad data, add queue item
@@ -364,6 +368,9 @@ def print_message(consumer, doctype, doc,
 
                             optical_property = {'Peak': peak_emission, 'FWHM':fwhm, 'PLQY':plqy}
 
+
+                            ## Save data for ML agent
+                            ## TODO: add phase ratio & particle size from scattering
                             if write_agent_data:
                                 agent_data = {}
 
@@ -377,30 +384,18 @@ def print_message(consumer, doctype, doc,
 
                                 print(f"\nwrote to {agent_data_path}")
 
-
-                            ### Three parameters for ML: peak_emission, fwhm, plqy
-                            # TODO: add ML agent code here
-
 
                             if USE_AGENT_iterate:
 
                                 # print(f"\ntelling agent {agent_data}")
-                                # agent.tell(data=agent_data, append=True)
-
-                                # agent = build_agen_Cl(peak_target=peak_target)
                                 agent = build_agen2(peak_target=peak_target)
 
                                 if len(agent.table) < 2:
                                     acq_func = "qr"
                                 else:
                                     acq_func = "qei"
 
-                                # agent._construct_all_models()
-                                # agent._train_all_models()
-                                # new_points, _ = agent.ask(acq_func, n=1)
                                 new_points = agent.ask(acq_func, n=1)
-                                # new_points = agent.ask("qem", n=1)
-                                # agent.save_data(filepath="/home/xf28id2/data_halide/init_240122_01.h5")
 
                                 # peak_diff = peak_emission - peak_target
                                 peak_diff = False
@@ -422,15 +417,18 @@ def print_message(consumer, doctype, doc,
                                 else:
                                     agent_iteration.append(True)
 
-                        else:
-                            plqy_dic = None
-                            optical_property = None
+                        # TODO: remove the fllowing 3 lines if no error reported
+                        # else:
+                        #     plqy_dic = None
+                        #     optical_property = None
 
+                        ## Save fitting data
                         print(f'\nFitting function: {f_fit}\n')
                         ff={'fit_function': f_fit, 'curve_fit': popt}
                         de.dic_to_csv_for_stream(saving_path, qepro_dic, metadata_dic, stream_name=stream_name, fitting=ff, plqy_dic=plqy_dic)
                         print(f'\n** export fitting results complete**\n')
 
+                        ## Plot fitting data
                         u.plot_peak_fit(x, y, f_fit, popt, peak=p, fill_between=True)
                         print(f'\n** plot fitting results complete**\n')
                         if stream_name == 'primary':
@@ -492,6 +490,7 @@ def print_message(consumer, doctype, doc,
                 print('*** Add new points from agent to the fron of qsever ***\n')
                 print(f'*** New points from agent: {new_points} ***\n')
 
+                ## TODO: add PF oil
                 if post_dilute:
                     set_target_list = [0 for i in range(len(pump_list))]
                     # rate_list = new_points['points'].tolist()[0][:-1] + [new_points['points'].sum()]