Merge pull request #481 from bryanmiller/GSCHED-708

GSCHED-716: initial plans using GPP

scheduler/core/components/optimizer/greedymax.py

@@ -315,6 +315,12 @@ def _find_max_group(self, plans: Plans) -> Optional[MaxGroup]:
                             max_score = np.max(group_data.group_info.scores[plans.night_idx]
                                                [interval[group_interval[0]:group_interval[1]]])
 
+                            # Add a penalty if the length of the group is slightly more than the interval
+                            # (within n_slots_min_visit), to discourage leaving small pieces behind
+                            n_slots_min_visit = time2slots(self.time_slot_length, self.min_visit_len)
+                            if 0 < num_time_slots_remaining - grp_interval_length < n_slots_min_visit:
+                                max_score *= 0.5
+
                             # Find the max_score in the group intervals with non-zero scores
                             # The length of the non-zero interval must be at least as large as
                             # the minimum length

scheduler/core/components/ranker/default.py

@@ -231,7 +231,7 @@ def score_observation(self, program: Program, obs: Observation):
         cplt = (program.total_used() + remaining) / program.total_awarded()
 
         metric, metric_s = self._metric_slope(np.array([cplt]),
-                                              np.array([program.band.value]),
+                                              np.array([obs.band.value]),
                                               np.array([0.8]),
                                               program.thesis)
 

scheduler/core/components/selector/selector.py

@@ -444,11 +444,11 @@ def _calculate_observation_group(self,
             # night_filtering[night_idx] = night_filter.group_filter(group)
 
         if obs.obs_class in [ObservationClass.SCIENCE, ObservationClass.PROGCAL]:
-            # If we are science or progcal, then the neg HA value for a night is if the first HA for the night
-            # is negative.
-            neg_ha = {night_idx: target_info[night_idx].hourangle[0].value < 0 for night_idx in night_indices}
+            # If we are science or progcal, then the check if the first HA for the night is negative,
+            # indicating that the target is rising
+            rising = {night_idx: target_info[night_idx].hourangle[0].value < 0 for night_idx in night_indices}
         else:
-            neg_ha = {night_idx: False for night_idx in night_indices}
+            rising = {night_idx: True for night_idx in night_indices}
         too_type = obs.too_type
 
         # Calculate when the conditions are met and an adjustment array if the conditions are better than needed.
@@ -470,9 +470,14 @@ def _calculate_observation_group(self,
             # has already been covered and should be ineligible for scheduling.
             # Zero out the part of the night that was already done.
             variant = self._variant_snapshot_per_site[obs.site].make_variant(total_timeslots_in_night)
-            conditions_score[night_idx] = Selector.match_conditions(mrc, variant, neg_ha[night_idx], too_type)
+            # print(f'Selector: Night {night_idx} for obs {obs.id.id} ({obs.internal_id}) @ {obs.site.name}')
+            # print(f'Current conditions: {max(variant.iq)} {max(variant.cc)} {max(variant.wind_dir)} {max(variant.wind_spd)}')
+            # print(f'Conditions req: IQ {mrc.iq}, CC {mrc.cc}')
+            # print(f'rising: {rising[night_idx]}, Too: {too_type}')
+            conditions_score[night_idx] = Selector.match_conditions(mrc, variant, rising[night_idx], too_type)
             conditions_score[night_idx][:starting_timeslot_in_night] = 0
             wind_score[night_idx] = Selector._wind_conditions(variant, target_info[night_idx].az)
+            # print(f'conditions score: {max(conditions_score[night_idx])}, wind_score: {max(wind_score[night_idx])}')
 
         # Calculate the schedulable slot indices.
         # These are the indices where the observation has:
@@ -482,12 +487,15 @@ def _calculate_observation_group(self,
         schedulable_slot_indices = {}
         for night_idx in night_indices:
             vis_idx = target_info[night_idx].visibility_slot_idx
+            # print(f'len(vis_idx) = {len(vis_idx)}')
             if night_filtering[night_idx]:
                 schedulable_slot_indices[night_idx] = np.where(conditions_score[night_idx][vis_idx] > 0)[0]
             else:
                 schedulable_slot_indices[night_idx] = np.array([])
+        # print(f'number schedulable slots night: {len(schedulable_slot_indices[night_idx])}')
 
         obs_scores = ranker.score_observation(program, obs)
+        # print(f'obs_scores: {max(obs_scores[night_idx])}')
 
         # Calculate the scores for the observation across all night indices across all timeslots.
         scores = {night_idx: np.multiply(
@@ -505,6 +513,7 @@ def _calculate_observation_group(self,
         for night_idx, time_slot_idx in starting_time_slots_for_site.items():
             if night_idx in night_indices:
                 scores[night_idx][:time_slot_idx] = 0.0
+        # print(f'scores: {max(scores[night_idx])}\n')
 
         # These scores might differ from the observation score in the ranker since they have been adjusted for
         # conditions and wind.
@@ -659,13 +668,13 @@ def _wind_conditions(variant: Variant,
     @staticmethod
     def match_conditions(required_conditions: Conditions,
                          actual_conditions: Variant,
-                         neg_ha: bool,
+                         rising: bool,
                          too_status: Optional[TooType]) -> npt.NDArray[float]:
         """
         Determine if the required conditions are satisfied by the actual conditions variant.
         * required_conditions: the conditions required by an observation
         * actual_conditions: the actual conditions variant, which can hold scalars or numpy arrays
-        * neg_ha: a numpy array indexed by night that indicates if the first angle hour is negative
+        * rising: a numpy array indexed by night that indicates if the first angle hour is negative => rising
         * too_status: the TOO status of the observation, if any
 
         We return a numpy array with entries in [0,1] indicating how well the actual conditions match
@@ -715,9 +724,10 @@ def match_conditions(required_conditions: Conditions,
         # does not set soon and is not a rapid ToO.
         # This should work as we are adjusting structures that are passed by reference.
         def adjuster(array, value):
-            better_idx = np.where(array < value)[0] if neg_ha else np.array([])
+            better_idx = np.where(array < value)[0] if rising else np.array([])
             if len(better_idx) > 0 and (too_status is None or too_status not in {TooType.RAPID, TooType.INTERRUPT}):
-                cond_match[better_idx] = cond_match[better_idx] * array[better_idx] / value
+                # cond_match[better_idx] = cond_match[better_idx] * array[better_idx] / value
+                cond_match[better_idx] = cond_match[better_idx] * (1.0 - (value - array[better_idx]))
 
         adjuster(actual_iq, required_conditions.iq)
         adjuster(actual_cc, required_conditions.cc)

scheduler/core/programprovider/gpp/gppprogramprovider.py

@@ -291,6 +291,7 @@ class _ConstraintKeys:
         TIMING_WINDOWS = 'timing_windows'
 
     class _AtomKeys:
+        ATOM = 'atom'
         OBS_CLASS = 'class'
         # INSTRUMENT = ''
         # INST_NAME = ''
@@ -782,15 +783,16 @@ def determine_mode(inst: str) -> ObservationMode:
 
         atoms = []
         classes = []
+        all_classes = []
         guiding = []
         qa_states = []
         prev_atom_id = -1
         n_atom = 0
         instrument_resources = frozenset([self._sources.origin.resource.lookup_resource(instrument)])
         for step in sequence:
-            if step['class'] != 'ACQUISITION':
+            if step[GppProgramProvider._AtomKeys.OBS_CLASS] != 'ACQUISITION':
                 next_atom = False
-                atom_id = step['atom']
+                atom_id = step[GppProgramProvider._AtomKeys.ATOM]
                 observe_class = step[GppProgramProvider._AtomKeys.OBS_CLASS]
                 step_time = step[GppProgramProvider._AtomKeys.TOTAL_TIME]
 
@@ -863,6 +865,7 @@ def determine_mode(inst: str) -> ObservationMode:
 
                 # Update atom
                 classes.append(observe_class)
+                all_classes.append(observe_class)
                 guiding.append(guide_state(step))
 
                 atoms[-1].exec_time += timedelta(seconds=step_time)
@@ -897,7 +900,17 @@ def determine_mode(inst: str) -> ObservationMode:
                 previous_atom.wavelengths = frozenset(wavelengths)
                 previous_atom.obs_mode = mode
 
-        return atoms
+        obs_class = ObservationClass.NONE
+        if 'SCIENCE' in all_classes:
+            obs_class = ObservationClass.SCIENCE
+        elif 'PROGRAM_CAL' in all_classes:
+            obs_class = ObservationClass.PROGCAL
+        elif 'PARTNER_CAL' in all_classes:
+            obs_class = ObservationClass.PARTNERCAL
+        elif 'DAY_CAL' in all_classes:
+            obs_class = ObservationClass.DAYCAL
+
+        return atoms, obs_class
 
     def parse_target(self, data: dict, targ_type: str) -> Target:
         """
@@ -933,7 +946,7 @@ def parse_observation(self,
             else f"{program_id.id}-{internal_id.replace('-', '')}"
 
         order = None
-        obs_class = None
+        obs_class = ObservationClass.NONE
         belongs_to = program_id
 
         try:
@@ -943,7 +956,7 @@ def parse_observation(self,
                 print(f"Observation {obs_id} is inactive (skipping).")
                 return None
 
-            # ToDo: there is no longer an observation-leveel obs_class, maybe check later from atom classes
+            # ToDo: there is no longer an observation-level obs_class, maybe check later from atom classes
             # obs_class = ObservationClass[data[GppProgramProvider._ObsKeys.OBS_CLASS].upper()]
             # if obs_class not in self._obs_classes or not active:
             #     logger.warning(f'Observation {obs_id} not in a specified class (skipping): {obs_class.name}.')
@@ -956,7 +969,7 @@ def parse_observation(self,
             # site = Site[data[GppProgramProvider._ObsKeys.ID].split('-')[0]]
             site = self._site_for_inst[data[GppProgramProvider._ObsKeys.INSTRUMENT]]
             # priority = Priority[data[GppProgramProvider._ObsKeys.PRIORITY].upper()]
-            priority = Priority.LOW
+            priority = Priority.MEDIUM
 
             # If the status is not legal, terminate parsing.
             status = ObservationStatus[data[GppProgramProvider._ObsKeys.STATUS].upper()]
@@ -971,7 +984,7 @@ def parse_observation(self,
             acq_overhead = timedelta(seconds=data['execution']['digest']['setup']['full']['seconds'])
 
             # Science band
-            band = data[GppProgramProvider._ObsKeys.BAND]
+            band = Band[data[GppProgramProvider._ObsKeys.BAND]]
 
             # Constraints
             find_constraints = {
@@ -988,7 +1001,7 @@ def parse_observation(self,
             # Atoms
             # ToDo: Perhaps add the sequence query to the original observation query
             sequence = explore.sequence(internal_id, include_acquisition=True)
-            atoms = self.parse_atoms(site, sequence)
+            atoms, obs_class = self.parse_atoms(site, sequence)
 
             # Pre-imaging
             preimaging = False
@@ -1137,7 +1150,7 @@ def parse_group(self, data: dict, program_id: ProgramID, group_id: GroupID,
             # AND cadence - delays not None, number_to_observe None, ordered should be forced to True
             # AND conseq - delays None,  number_to_observe None
             group_option = AndOption.ANYORDER
-            if delay_min is not None:
+            if delay_max is not None:
                 group_option = AndOption.CUSTOM
                 ordered = True
                 number_to_observe = len(data[GppProgramProvider._GroupKeys.ELEMENTS])

scheduler/core/programprovider/ocs/ocsprogramprovider.py

@@ -1004,7 +1004,8 @@ def parse_observation(self,
                           num: Tuple[Optional[int], int],
                           program_id: ProgramID,
                           split: bool,
-                          split_by_iterator: bool) -> Optional[Observation]:
+                          split_by_iterator: bool,
+                          band: Band) -> Optional[Observation]:
         """
         Right now, obs_num contains an optional organizational folder number and
         a mandatory observation number, which may overlap with others in organizational folders.
@@ -1170,7 +1171,8 @@ def parse_observation(self,
                 constraints=constraints,
                 belongs_to=program_id,
                 too_type=too_type,
-                preimaging=preimaging
+                preimaging=preimaging,
+                band=band
             )
 
         except KeyError as ex:
@@ -1225,7 +1227,7 @@ def parse_time_used(data: dict) -> TimeUsed:
     #     raise NotImplementedError('OCS does not support OR groups.')
 
     def parse_group(self, data: dict, program_id: ProgramID, group_id: GroupID,
-                        split: bool, split_by_iterator: bool) -> Optional[Group]:
+                        split: bool, split_by_iterator: bool, band: Band) -> Optional[Group]:
         """
         In the OCS, a SchedulingFolder or a program are AND groups.
         We do not allow nested groups in OCS, so this is relatively easy.
@@ -1263,7 +1265,7 @@ def parse_group(self, data: dict, program_id: ProgramID, group_id: GroupID,
         for key in scheduling_group_keys:
             subgroup_id = GroupID(key.split('-')[-1])
             subgroup = self.parse_group(data[key], program_id, subgroup_id, split=split,
-                                            split_by_iterator=split_by_iterator)
+                                            split_by_iterator=split_by_iterator, band=band)
             if subgroup is not None:
                 children.append(subgroup)
 
@@ -1305,7 +1307,7 @@ def parse_unique_obs_id(of_key: Optional[str], obs_key: str) -> Tuple[Optional[i
         observations = []
         for *keys, obs_data in obs_data_blocks:
             obs_id = parse_unique_obs_id(*keys)
-            obs = self.parse_observation(obs_data, obs_id, program_id,
+            obs = self.parse_observation(obs_data, obs_id, program_id, band=band,
                                          split=split, split_by_iterator=split_by_iterator)
             if obs is not None:
                 observations.append(obs)
@@ -1370,13 +1372,23 @@ def parse_program(self, data: dict) -> Optional[Program]:
         if not split_by_iterator:
             split_by_iterator = self.parse_notes(notes, OcsProgramProvider._SPLIT_BY_ITER_STRINGS)
 
+        program_mode = ProgramMode[data[OcsProgramProvider._ProgramKeys.MODE].upper()]
+        try:
+            band = Band(int(data[OcsProgramProvider._ProgramKeys.BAND]))
+        except ValueError:
+            # Treat classical as Band 1, other types as Band 2
+            if program_mode == ProgramMode.CLASSICAL:
+                band = Band(1)
+            else:
+                band = Band(2)
+
         # Now we parse the groups. For this, we need:
         # 1. A list of Observations at the root level.
         # 2. A list of Observations for each Scheduling Group.
         # 3. A list of Observations for each Organizational Folder.
         # We can treat (1) the same as (2) and (3) by simply passing all the JSON
         # data to the parse_and_group method.
-        root_group = self.parse_group(data, program_id, ROOT_GROUP_ID,
+        root_group = self.parse_group(data, program_id, ROOT_GROUP_ID, band=band,
                                           split=split, split_by_iterator=split_by_iterator)
         if root_group is None:
             logger.warning(f'Program {program_id} has empty root group. Skipping.')
@@ -1403,15 +1415,6 @@ def parse_program(self, data: dict) -> Optional[Program]:
             logger.warning(f'Could not determine program type for program {program_id}. Skipping.')
             return None
 
-        program_mode = ProgramMode[data[OcsProgramProvider._ProgramKeys.MODE].upper()]
-        try:
-            band = Band(int(data[OcsProgramProvider._ProgramKeys.BAND]))
-        except ValueError:
-            # Treat classical as Band 1, other types as Band 2
-            if program_mode == ProgramMode.CLASSICAL:
-                band = Band(1)
-            else:
-                band = Band(2)
         thesis = data[OcsProgramProvider._ProgramKeys.THESIS]
         # print(f'\t program_mode = {program_mode}, band = {band}')
 

scheduler/data/gpp_program_ids.txt

@@ -1,2 +1,2 @@
 p-147
-# p-11d
+p-11d

scheduler/services/environment/data/simulation/gn_weather_data.csv

@@ -2,4 +2,5 @@ Local_Night,Local_Time,raw_iq,raw_cc,WindDir,WindSpeed
 2024-09-30 12:00:00-10:00, 2024-09-30 23:30:00-10:00, 0.20, 0.50, 180, 5.0
 2024-09-30 12:00:00-10:00, 2024-10-01 03:30:00-10:00, 0.70, 0.50, 190, 6.0
 2024-10-31 12:00:00-10:00, 2024-11-01 01:30:00-10:00, 0.70, 0.70, 200, 9.0
-2024-10-31 12:00:00-10:00, 2024-11-01 04:00:00-10:00, 0.85, 0.70, 210, 12.0
+2024-10-31 12:00:00-10:00, 2024-11-01 04:00:00-10:00, 0.85, 0.70, 210, 12.0
+2024-12-01 12:00:00-10:00, 2024-12-01 21:00:00-10:00, 0.85, 0.80, 140, 6.0

scheduler/services/environment/data/simulation/gs_weather_data.csv

@@ -1,4 +1,4 @@
 Local_Night,Local_Time,raw_iq,raw_cc,WindDir,WindSpeed
 2024-09-30 12:00:00-03:00, 2024-10-01 02:30:00-03:00, 0.7, 0.5, 270, 5.0
-2024-10-31 12:00:00-03:00, 2024-10-31 22:30:00-03:00, 0.85, 0.70, 290, 7.0
-2024-10-31 12:00:00-03:00, 2024-11-01 04:00:00-03:00, 1.0, 0.8, 295, 8.0
+2024-10-31 12:00:00-03:00, 2024-10-31 22:30:00-03:00, 1.0, 0.7, 290, 7.0
+2024-10-31 12:00:00-03:00, 2024-11-01 02:00:00-03:00, 0.7, 0.5, 295, 8.0

scheduler/services/resource/data/simulation/GMOSN_GRAT202407.txt

@@ -5,13 +5,13 @@
  2024-09-13 , B480+         , B1200+ , R400+        
  2024-10-01 , B480+         ,         R831+ , R400+        
  2024-10-17 , B480+         , B1200+ , R400+        
- 2024-10-31 , B480+         ,         R150+ , R400+        
+ 2024-10-31 , B480+         ,         R150+ , R831+
  2024-11-07 , B1200+ , R150+         , R400+        
  2024-11-09 , B480+ , R150+         , R400+        
  2024-11-16 , B480+         , B1200+ , R400+        
  2024-11-23 , B480+         ,         R831+ , R400+        
  2024-11-29 , B480+         , R150+  , R400+        
- 2024-11-30 , B480+         , B1200+ , R400+        
+ 2024-11-30 , B480+         , B1200+ , R831+
  2024-12-03 , B480+         ,         R150+ , R400+        
  2024-12-17 , B480+         , B1200+ , R400+        
  2024-12-21 , B480+         ,         R831+ , R400+        

scheduler/services/resource/data/simulation/GMOSS_GRAT202407.txt

@@ -5,8 +5,9 @@
  2024-10-03 , R150+  , R400+         , B480+        
  2024-10-17 , B1200+ , R400+         , B600+        
  2024-10-26 , R150+ , R400+         , B480+        
- 2024-11-07 , R831+  , R400+         , B600+        
- 2024-12-17 , B1200+ , R400+         , B600+        
+ 2024-11-03 , R831+  , R400+         , B600+
+ 2024-12-01 , B1200+ , R400+         , B480+
+ 2024-12-17 , B1200+ , R400+         , B600+
  2024-12-26 ,         R831+ , R400+         , B480+        
  2024-12-30 , R150+  , R400+         , B480+        
  2025-01-04 , B1200+ , R400+         , B480+