Updates for CLMM version 1.14.1 and redshift info handling

examples/cosmodc2/Cluster_pipelines/1deg2-nersc.sub

@@ -6,5 +6,6 @@
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=32
 
-source $CFS/lsst/groups/WL/users/zuntz/setup-txpipe
-tx ceci examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-nersc.yml
+module load python
+conda activate ./conda
+ceci examples/cosmodc2/Cluster_pipelines/pipeline-1deg2-CL-nersc.yml

examples/cosmodc2/Cluster_pipelines/20deg2-in2p3.sub

@@ -5,5 +5,6 @@
 #SBATCH --cpus-per-task=1
 #SBATCH --mem=128000
 
-source /pbs/throng/lsst/users/jzuntz/txpipe-environments/setup-txpipe
+source ./conda/bin/activate
+export HDF5_DO_MPI_FILE_SYNC=0
 ceci examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml

examples/cosmodc2/Cluster_pipelines/20deg2-nersc.sub

@@ -6,5 +6,7 @@
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=32
 
-source $CFS/lsst/groups/WL/users/zuntz/setup-txpipe
-tx ceci examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml
+
+module load python
+conda activate ./conda
+ceci examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml

examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml

@@ -10,7 +10,7 @@ TXSourceSelectorMetadetect:
     true_z: false
     shear_prefix: ''
 
-Inform_BPZ_lite:
+BPZliteInformer:
     zmin: 0.0
     zmax: 3.0
     nzbins: 301
@@ -32,7 +32,7 @@ Inform_BPZ_lite:
 
 
 
-BPZ_lite:
+BPZliteEstimator:
     zmin: 0.0
     zmax: 3.0
     dz: 0.01
@@ -64,7 +64,9 @@ CLClusterShearCatalogs:
     chunk_rows: 100_000  # rows to read at once from source cat
     max_radius: 5   # Mpc
     delta_z: 0.2   # redshift buffer
-    redshift_criterion: mean     # might also need PDF
+    redshift_cut_criterion: pdf
+    redshift_weight_criterion: pdf
+    redshift_cut_criterion_pdf_fraction: 0.9
     subtract_mean_shear: true
 
 

examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml

@@ -23,12 +23,12 @@ python_paths: []
 stages:
   - name: TXSourceSelectorMetadetect
     nprocess: 30
-  - name: Inform_BPZ_lite
+  - name: BPZliteInformer
     nprocess: 1
     aliases:
         input: spectroscopic_catalog
         model: photoz_model
-  - name: BPZ_lite
+  - name: BPZliteEstimator
     nprocess: 30
     aliases:
         model: photoz_model
@@ -50,12 +50,12 @@ config: ./examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml
 
 inputs:
     # See README for paths to download these files
-    shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5
-    #photometry_catalog:  ./data/cosmodc2/20deg2/photometry_catalog.hdf5
+    shear_catalog: /sps/lsst/users/mricci/TXPipe_data/data_link/cosmodc2/20deg2/shear_catalog.hdf5
+    #photometry_catalog:  /sps/lsst/users/mricci/TXPipe_data/data_link/cosmodc2/20deg2/photometry_catalog.hdf5
     fiducial_cosmology: ./data/fiducial_cosmology.yml
-    calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat
-    spectroscopic_catalog: ./data/cosmodc2/20deg2/spectroscopic_catalog.hdf5
-    cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5
+    calibration_table: /sps/lsst/users/mricci/TXPipe_data/data_link/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat
+    spectroscopic_catalog: /sps/lsst/users/mricci/TXPipe_data/data_link/cosmodc2/20deg2/spectroscopic_catalog.hdf5
+    cluster_catalog: /sps/lsst/users/mricci/TXPipe_data/data_link/cosmodc2/20deg2/cluster_catalog.hdf5
     #shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5
     #source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5
 resume: true

examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml

@@ -6,8 +6,7 @@
 
 #for NERSC
 site:
-    name: cori-batch
-    image: ghcr.io/lsstdesc/txpipe-dev
+    name: nersc-interactive
 
 
 #all the following steps should not depend on where you run 
@@ -23,12 +22,12 @@ python_paths: []
 stages:
   - name: TXSourceSelectorMetadetect
     nprocess: 30
-  - name: Inform_BPZ_lite
+  - name: BPZliteInformer
     nprocess: 1
     aliases:
         input: spectroscopic_catalog
         model: photoz_model
-  - name: BPZ_lite
+  - name: BPZliteEstimator
     nprocess: 30
     aliases:
         model: photoz_model
@@ -37,7 +36,7 @@ stages:
   - name: CLClusterBinningRedshiftRichness
     nprocess: 1
   - name: CLClusterShearCatalogs
-    nprocess: 1   #>1 does not work with mpi
+    nprocess: 32   #>1 does not work with mpi
   - name: CLClusterEnsembleProfiles
     nprocess: 1
 #    - name: CLClusterDataVector

examples/cosmodc2/pipeline-20deg2-laptop.yml

@@ -0,0 +1,116 @@
+launcher:
+    name: mini
+    interval: 3.0
+
+site:
+    name: local
+
+
+modules: txpipe
+
+python_paths:
+  - submodules/WLMassMap/python/desc/
+  - submodules/FlexZPipe
+
+stages:
+  - name: TXRandomCat
+    nprocess: 12
+  - name: TXSourceSelectorMetadetect
+    nprocess: 12
+  - name: TXLSSWeightsUnit
+  - name: TXRandomForestLensSelector
+    nprocess: 12
+  - name: TXLensTrueNumberDensity
+    classname: TXTruePhotozStack
+    aliases:
+        tomography_catalog: lens_tomography_catalog
+        catalog: photometry_catalog
+        weights_catalog: none
+        photoz_stack: lens_photoz_stack
+  - name: TXSourceTrueNumberDensity
+    classname: TXTruePhotozStack
+    aliases:
+        tomography_catalog: shear_tomography_catalog
+        catalog: shear_catalog
+        weights_catalog: shear_catalog
+        photoz_stack: shear_photoz_stack
+  - name: TXPhotozPlotSource
+    classname: TXPhotozPlot
+    aliases:
+        photoz_stack: shear_photoz_stack
+        nz_plot: source_nz
+  - name: TXPhotozPlotLens
+    classname: TXPhotozPlot
+    aliases:
+        photoz_stack: lens_photoz_stack
+        nz_plot: lens_nz
+  - name: TXShearCalibration
+    nprocess: 12
+  - name: TXTruthLensCatalogSplitter
+    nprocess: 12
+  - name: TXTwoPointFourier
+    nprocess: 2
+    nodes: 1
+    threads_per_process: 6
+  - name: TXTwoPointTheoryFourier
+  - name: TXTwoPointPlotsFourier
+  - name: TXJackknifeCenters
+  - name: TXSourceMaps
+    nprocess: 8
+  - name: TXLensMaps
+    nprocess: 8
+  - name: TXAuxiliarySourceMaps
+    nprocess: 8
+  - name: TXAuxiliaryLensMaps
+    nprocess: 8
+  - name: TXDensityMaps
+  - name: TXSourceNoiseMaps
+    nprocess: 4
+    nodes: 1
+    threads_per_process: 1
+  - name: TXLensNoiseMaps
+    nprocess: 4
+    nodes: 1
+    threads_per_process: 1
+  - name: TXSimpleMask
+  - name: TXMapPlots
+  - name: TXTracerMetadata
+  - name: TXNullBlinding
+  - name: TXTwoPoint
+    threads_per_process: 12
+    nprocess: 1
+    nodes: 1
+  - name: TXTwoPointPlotsTheory
+  - name: TXDiagnosticQuantiles
+    nodes: 1
+    nprocess: 12
+  - name: TXLensDiagnosticPlots
+    nprocess: 12
+    nodes: 1
+  - name: TXSourceDiagnosticPlots
+    nprocess: 12
+    nodes: 1
+  - name: TXFourierGaussianCovariance
+    threads_per_process: 12
+  - name: TXTwoPointTheoryReal
+  - name: TXRealGaussianCovariance
+    threads_per_process: 12
+  - name: TXConvergenceMaps
+    threads_per_process: 12
+  - name: TXConvergenceMapPlots
+
+output_dir: data/cosmodc2/outputs-20deg2
+config: examples/cosmodc2/config-20deg2.yml
+
+inputs:
+    # See README for paths to download these files
+    shear_catalog: ./data/cosmodc2/20deg2/shear_catalog.hdf5
+    photometry_catalog: ./data/cosmodc2/20deg2/photometry_catalog.hdf5
+    fiducial_cosmology: data/fiducial_cosmology.yml
+    calibration_table: ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat
+    none: none
+
+resume: true
+log_dir: data/cosmodc2/logs
+pipeline_log: data/cosmodc2/log.txt
+