Handle Merge on EventId for EventNotes and Speed up NarrativeCache population for iterative updates

api-src/org/labkey/api/snd/SNDSequencer.java

@@ -24,10 +24,10 @@ public enum SNDSequencer
     PKGID ("org.labkey.snd.api.Package", 10000),
     SUPERPKGID ("org.labkey.snd.api.SuperPackage", 10000),
     CATEGORYID ("org.labkey.snd.api.Categories", 100),
-    PROJECTID ("org.labkey.snd.api.Project", 1000),
+    PROJECTID ("org.labkey.snd.api.Project", 10000),
     PROJECTITEMID ("org.labkey.snd.api.ProjectItem", 30000),
     EVENTID ("org.labkey.snd.api.Event", 2000000),
-    EVENTDATAID ("org.labkey.snd.api.EventData", 3500000);
+    EVENTDATAID ("org.labkey.snd.api.EventData", 5000000);
 
     private String sequenceName;
     private int minId;

api-src/org/labkey/api/snd/SNDService.java

@@ -67,7 +67,7 @@ static SNDService get()
     void fillInNarrativeCache(Container c, User u, Logger logger);
     void clearNarrativeCache(Container c, User u);
     void deleteNarrativeCacheRows(Container c, User u, List<Map<String, Object>> eventIds);
-    void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger);
+    void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger, boolean isFullReload);
     Map<Integer, Category> getAllCategories(Container c, User u);
     Integer getQCStateId(Container c, User u, QCStateEnum qcState);
     QCStateEnum getQCState(Container c, User u, int qcStateId);

src/org/labkey/snd/SNDManager.java

@@ -3113,10 +3113,10 @@ public int updateNarrativeCache(Container container, User user, Set<Integer>cach
                 log.info("Deleting affected narrative cache rows.");
                 deleteNarrativeCacheRows(container, user, rows, errors);
                 //repopulate
-                if (isUpdate)
+                if (!errors.hasErrors() && isUpdate)
                 {
                     log.info("Repopulate affected rows in narrative cache.");
-                    populateNarrativeCache(container, user, eventIds, errors, log);
+                    populateNarrativeCache(container, user, eventIds, errors, log, false);
                 }
             }
 
@@ -3142,7 +3142,7 @@ public void deleteNarrativeCacheRows(Container c, User u, List<Map<String, Objec
         }
         catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e)
         {
-            e.printStackTrace();
+            errors.addRowError(new ValidationException(e.getMessage()));
         }
     }
 
@@ -3182,13 +3182,13 @@ public void fillInNarrativeCache(Container c, User u, BatchValidationException e
 
         List<Integer> eventIds = selector.getArrayList(Integer.class);
 
-        populateNarrativeCache(c, u, eventIds, errors, logger);
+        populateNarrativeCache(c, u, eventIds, errors, logger, true);
     }
 
     /**
      * Populate specific event narratives in narrative cache.
      */
-    public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, BatchValidationException errors, @Nullable Logger logger)
+    public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, BatchValidationException errors, @Nullable Logger logger, boolean isFullReload)
     {
         UserSchema sndSchema = getSndUserSchemaAdminRole(c, u);
         QueryUpdateService eventsCacheQus = getNewQueryUpdateService(sndSchema, SNDSchema.EVENTSCACHE_TABLE_NAME);
@@ -3208,7 +3208,7 @@ public void populateNarrativeCache(Container c, User u, List<Integer> eventIds,
             logger.info("Generating narratives.");
         }
 
-        Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, null, lookups, errors);
+        Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, isFullReload ? null : eventIds, lookups, errors, isFullReload);
 
         AtomicInteger count = new AtomicInteger(0);
 
@@ -3801,7 +3801,7 @@ public Event getEvent(Container c, User u, int eventId, Set<EventNarrativeOption
         List<GWTPropertyDescriptor> packageExtraFields = getExtraFields(c, u, SNDSchema.PKGS_TABLE_NAME);
 
         // Retrieve all SuperPackages associated with a single event
-        Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, eventId, lookups, errors);
+        Map<Integer, SuperPackage> superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, Collections.singletonList(eventId), lookups, errors, false);
 
         // Map top-level EventDataIds to associated SuperPackage objects and group by EventId
         Map<Integer, Map<Integer, SuperPackage>> topLevelEventDataSuperPkgs = getBulkTopLevelEventDataSuperPkgs(c, u, Collections.singletonList(eventId), superPackages);
@@ -3911,14 +3911,14 @@ public Map<Integer, Event> getBulkEvents(Container c, User u, List<Integer> even
      * @param u User object representing the current user.
      * @param packageExtraFields Extra fields related to the SuperPackage that need to be included in the response.
      * @param pkgQus QueryUpdateService for handling SuperPackage updates.
-     * @param eventId The ID of the event for which SuperPackages should be retrieved. Can be null to retrieve all SuperPackages.
+     * @param eventIds The ID of the event for which SuperPackages should be retrieved. Can be null to retrieve all SuperPackages.
      * @param lookups Map for additional lookup criteria or filtering, used to retrieve SuperPackages.
      * @param errors BatchValidationException object used to accumulate any errors encountered during the process.
      *
      * @return A map of SuperPackage objects, keyed by SuperPkgId.
      */
-    private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GWTPropertyDescriptor> packageExtraFields, QueryUpdateService pkgQus, @Nullable Integer eventId,
-                                                        Map<String, String> lookups, BatchValidationException errors) {
+    private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GWTPropertyDescriptor> packageExtraFields, QueryUpdateService pkgQus, @Nullable List<Integer> eventIds,
+                                                        Map<String, String> lookups, BatchValidationException errors, boolean isFullReload) {
 
         UserSchema schema = getSndUserSchema(c, u);
 
@@ -3929,10 +3929,15 @@ private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GW
         sql.append(schema.getTable(SNDSchema.SUPERPKGS_TABLE_NAME), "sp");
         sql.append(" JOIN " + SNDSchema.NAME + "." + SNDSchema.PKGS_TABLE_NAME + " pkg");
         sql.append(" ON sp.PkgId = pkg.PkgId ");
-        if (eventId != null) {
+        if (eventIds != null) {
             sql.append(" INNER JOIN " + SNDSchema.NAME + "." + SNDSchema.EVENTDATA_TABLE_NAME + " ed");
             sql.append(" ON ed.SuperPkgId = sp.SuperPkgId ");
-            sql.append(" WHERE ed.EventId = ? ").add(eventId);
+            sql.append(" WHERE ed.EventId IN ( ");
+            ArrayDeque<Integer> eventIdsQueue = new ArrayDeque<>(eventIds);
+            while (eventIdsQueue.size() > 1) {
+                sql.append("?, ").add(eventIdsQueue.pop());
+            }
+            sql.append("?) ").add(eventIdsQueue.pop());
         }
         sql.append(" ORDER BY sp.SuperPkgId ");
 
@@ -3943,7 +3948,7 @@ private Map<Integer, SuperPackage> getBulkSuperPkgs(Container c, User u, List<GW
 
         List<SuperPackage> fullTreeSuperPkgs;
 
-        if (eventId != null) {
+        if (!isFullReload) {
             fullTreeSuperPkgs = new ArrayList<SuperPackage>();
             pkgIds.forEach(pkgId -> {
                 SQLFragment packageSql = new SQLFragment("SELECT * FROM ");
@@ -4328,6 +4333,7 @@ private Map<Integer, String> getBulkEventNotes(Container c, User u, List<Integer
         return eventNotesById;
     }
 
+
     /**
      * Query the Projects table and retrieve the ID concatenated with RevisionNum for a set of objectIds
      *

src/org/labkey/snd/SNDServiceImpl.java

@@ -558,11 +558,12 @@ public void deleteNarrativeCacheRows(Container c, User u, List<Map<String, Objec
     }
 
     @Override
-    public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger)
+
+    public void populateNarrativeCache(Container c, User u, List<Integer> eventIds, Logger logger, boolean isFullReload)
     {
         BatchValidationException errors = new BatchValidationException();
 
-        SNDManager.get().populateNarrativeCache(c, u, eventIds, errors, logger);
+        SNDManager.get().populateNarrativeCache(c, u, eventIds, errors, logger, isFullReload);
 
         if (errors.hasErrors())
             throw new ApiUsageException(errors);

src/org/labkey/snd/query/EventNotesTable.java

@@ -36,8 +36,11 @@
 import org.labkey.snd.security.permissions.SNDViewerPermission;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class EventNotesTable extends SimpleUserSchema.SimpleTable<SNDUserSchema>
 {
@@ -67,6 +70,7 @@ public UpdateService(SimpleUserSchema.SimpleTable ti)
         }
 
         private final SNDService _sndService = SNDService.get();
+        private final SNDManager _sndManager = SNDManager.get();
 
         private int getRowCount(DataIteratorBuilder rows, @Nullable Map<Enum,Object> configParameters, BatchValidationException errors)
         {
@@ -103,11 +107,29 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B
             else
             {
                 log.info("Merging rows.");
+
+                DataIteratorContext context = getDataIteratorContext(errors, QueryUpdateService.InsertOption.MERGE, configParameters);
+
+                Set<Integer> eventIds = rows.getDataIterator(context).stream()
+                        .filter(row -> row.containsKey("eventId"))
+                        .map(row -> (Integer) row.get("eventId"))
+                        .collect(Collectors.toSet());
+
                 result = super.mergeRows(user, container, rows, errors, configParameters, extraScriptContext);
+
+                _sndManager.updateNarrativeCache(container, user, eventIds, log);
             }
             return result;
         }
 
+        @Override
+        public void configureDataIteratorContext(DataIteratorContext context)
+        {
+            if (context.getInsertOption() == QueryUpdateService.InsertOption.MERGE)
+            {
+                context.addAlternateKeys(Collections.singleton("EventId"));
+            }
+        }
     }
 
     @Override