From c7708e45a56dbd288144cbdcf4d8ab094832a062 Mon Sep 17 00:00:00 2001 From: Jay Allen <107942890+jallentxbiomed@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:30:00 -0500 Subject: [PATCH] Handle Merge on EventId for EventNotes and Speed up NarrativeCache population for iterative updates (#243) * added logic for separating rows containing eventNoteIds * added isUpdate logic to getBulkSuperPkgs * added isUpdate logic to getBulkSuperPkgs * Merge alt key * error handling * added logic for separating rows containing eventNoteIds * added isUpdate logic to getBulkSuperPkgs * changed order of method calls * rebased with commits for alternate key * removed space * removed getEventNote method * EventNotesDataIterator * Remove unused variable --------- Co-authored-by: Marty Pradere --- api-src/org/labkey/api/snd/SNDSequencer.java | 4 +- api-src/org/labkey/api/snd/SNDService.java | 2 +- src/org/labkey/snd/SNDManager.java | 32 ++++--- src/org/labkey/snd/SNDServiceImpl.java | 5 +- .../snd/query/EventNotesDataIterator.java | 91 +++++++++++++++++++ .../query/EventNotesDataIteratorBuilder.java | 33 +++++++ src/org/labkey/snd/query/EventNotesTable.java | 16 +++- 7 files changed, 163 insertions(+), 20 deletions(-) create mode 100644 src/org/labkey/snd/query/EventNotesDataIterator.java create mode 100644 src/org/labkey/snd/query/EventNotesDataIteratorBuilder.java diff --git a/api-src/org/labkey/api/snd/SNDSequencer.java b/api-src/org/labkey/api/snd/SNDSequencer.java index eb9fa6a6..94d1a1e9 100644 --- a/api-src/org/labkey/api/snd/SNDSequencer.java +++ b/api-src/org/labkey/api/snd/SNDSequencer.java @@ -24,10 +24,10 @@ public enum SNDSequencer PKGID ("org.labkey.snd.api.Package", 10000), SUPERPKGID ("org.labkey.snd.api.SuperPackage", 10000), CATEGORYID ("org.labkey.snd.api.Categories", 100), - PROJECTID ("org.labkey.snd.api.Project", 1000), + PROJECTID ("org.labkey.snd.api.Project", 10000), PROJECTITEMID ("org.labkey.snd.api.ProjectItem", 30000), EVENTID ("org.labkey.snd.api.Event", 2000000), - EVENTDATAID ("org.labkey.snd.api.EventData", 3500000); + EVENTDATAID ("org.labkey.snd.api.EventData", 5000000); private String sequenceName; private int minId; diff --git a/api-src/org/labkey/api/snd/SNDService.java b/api-src/org/labkey/api/snd/SNDService.java index e6081dba..c601108f 100644 --- a/api-src/org/labkey/api/snd/SNDService.java +++ b/api-src/org/labkey/api/snd/SNDService.java @@ -67,7 +67,7 @@ static SNDService get() void fillInNarrativeCache(Container c, User u, Logger logger); void clearNarrativeCache(Container c, User u); void deleteNarrativeCacheRows(Container c, User u, List> eventIds); - void populateNarrativeCache(Container c, User u, List eventIds, Logger logger); + void populateNarrativeCache(Container c, User u, List eventIds, Logger logger, boolean isFullReload); Map getAllCategories(Container c, User u); Integer getQCStateId(Container c, User u, QCStateEnum qcState); QCStateEnum getQCState(Container c, User u, int qcStateId); diff --git a/src/org/labkey/snd/SNDManager.java b/src/org/labkey/snd/SNDManager.java index 2a84eaed..a88df60b 100644 --- a/src/org/labkey/snd/SNDManager.java +++ b/src/org/labkey/snd/SNDManager.java @@ -3113,10 +3113,10 @@ public int updateNarrativeCache(Container container, User user, Setcach log.info("Deleting affected narrative cache rows."); deleteNarrativeCacheRows(container, user, rows, errors); //repopulate - if (isUpdate) + if (!errors.hasErrors() && isUpdate) { log.info("Repopulate affected rows in narrative cache."); - populateNarrativeCache(container, user, eventIds, errors, log); + populateNarrativeCache(container, user, eventIds, errors, log, false); } } @@ -3142,7 +3142,7 @@ public void deleteNarrativeCacheRows(Container c, User u, List eventIds = selector.getArrayList(Integer.class); - populateNarrativeCache(c, u, eventIds, errors, logger); + populateNarrativeCache(c, u, eventIds, errors, logger, true); } /** * Populate specific event narratives in narrative cache. */ - public void populateNarrativeCache(Container c, User u, List eventIds, BatchValidationException errors, @Nullable Logger logger) + public void populateNarrativeCache(Container c, User u, List eventIds, BatchValidationException errors, @Nullable Logger logger, boolean isFullReload) { UserSchema sndSchema = getSndUserSchemaAdminRole(c, u); QueryUpdateService eventsCacheQus = getNewQueryUpdateService(sndSchema, SNDSchema.EVENTSCACHE_TABLE_NAME); @@ -3208,7 +3208,7 @@ public void populateNarrativeCache(Container c, User u, List eventIds, logger.info("Generating narratives."); } - Map superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, null, lookups, errors); + Map superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, isFullReload ? null : eventIds, lookups, errors, isFullReload); AtomicInteger count = new AtomicInteger(0); @@ -3801,7 +3801,7 @@ public Event getEvent(Container c, User u, int eventId, Set packageExtraFields = getExtraFields(c, u, SNDSchema.PKGS_TABLE_NAME); // Retrieve all SuperPackages associated with a single event - Map superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, eventId, lookups, errors); + Map superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, Collections.singletonList(eventId), lookups, errors, false); // Map top-level EventDataIds to associated SuperPackage objects and group by EventId Map> topLevelEventDataSuperPkgs = getBulkTopLevelEventDataSuperPkgs(c, u, Collections.singletonList(eventId), superPackages); @@ -3911,14 +3911,14 @@ public Map getBulkEvents(Container c, User u, List even * @param u User object representing the current user. * @param packageExtraFields Extra fields related to the SuperPackage that need to be included in the response. * @param pkgQus QueryUpdateService for handling SuperPackage updates. - * @param eventId The ID of the event for which SuperPackages should be retrieved. Can be null to retrieve all SuperPackages. + * @param eventIds The ID of the event for which SuperPackages should be retrieved. Can be null to retrieve all SuperPackages. * @param lookups Map for additional lookup criteria or filtering, used to retrieve SuperPackages. * @param errors BatchValidationException object used to accumulate any errors encountered during the process. * * @return A map of SuperPackage objects, keyed by SuperPkgId. */ - private Map getBulkSuperPkgs(Container c, User u, List packageExtraFields, QueryUpdateService pkgQus, @Nullable Integer eventId, - Map lookups, BatchValidationException errors) { + private Map getBulkSuperPkgs(Container c, User u, List packageExtraFields, QueryUpdateService pkgQus, @Nullable List eventIds, + Map lookups, BatchValidationException errors, boolean isFullReload) { UserSchema schema = getSndUserSchema(c, u); @@ -3929,10 +3929,15 @@ private Map getBulkSuperPkgs(Container c, User u, List eventIdsQueue = new ArrayDeque<>(eventIds); + while (eventIdsQueue.size() > 1) { + sql.append("?, ").add(eventIdsQueue.pop()); + } + sql.append("?) ").add(eventIdsQueue.pop()); } sql.append(" ORDER BY sp.SuperPkgId "); @@ -3943,7 +3948,7 @@ private Map getBulkSuperPkgs(Container c, User u, List fullTreeSuperPkgs; - if (eventId != null) { + if (!isFullReload) { fullTreeSuperPkgs = new ArrayList(); pkgIds.forEach(pkgId -> { SQLFragment packageSql = new SQLFragment("SELECT * FROM "); @@ -4328,6 +4333,7 @@ private Map getBulkEventNotes(Container c, User u, List eventIds, Logger logger) + + public void populateNarrativeCache(Container c, User u, List eventIds, Logger logger, boolean isFullReload) { BatchValidationException errors = new BatchValidationException(); - SNDManager.get().populateNarrativeCache(c, u, eventIds, errors, logger); + SNDManager.get().populateNarrativeCache(c, u, eventIds, errors, logger, isFullReload); if (errors.hasErrors()) throw new ApiUsageException(errors); diff --git a/src/org/labkey/snd/query/EventNotesDataIterator.java b/src/org/labkey/snd/query/EventNotesDataIterator.java new file mode 100644 index 00000000..e1c3803f --- /dev/null +++ b/src/org/labkey/snd/query/EventNotesDataIterator.java @@ -0,0 +1,91 @@ +package org.labkey.snd.query; + +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbScope; +import org.labkey.api.dataiterator.AbstractDataIterator; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.snd.SNDManager; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +public class EventNotesDataIterator extends AbstractDataIterator +{ + private static final SNDManager _sndManager = SNDManager.get(); + private static final String EVENT_ID_COL = "eventId"; + private User _user; + private Container _container; + private int _eventIdColIndex; + private Set _eventIds = new HashSet<>(); + private DataIterator _in; + private Logger log = LogHelper.getLogger(EventNotesDataIterator.class, "Fill out event notes"); + + public static DataIterator wrap(DataIterator in, DataIteratorContext context, Container c, User u) + { + return new EventNotesDataIterator(in, context, c, u); + } + + private EventNotesDataIterator(DataIterator in, DataIteratorContext context, Container c, User u) + { + super(context); + _user = u; + _container = c; + _in = in; + + _eventIdColIndex = DataIteratorUtil.createColumnNameMap(in).get(EVENT_ID_COL); + } + + @Override + public int getColumnCount() + { + return _in.getColumnCount(); + } + + @Override + public ColumnInfo getColumnInfo(int i) + { + return _in.getColumnInfo(i); + } + + @Override + public boolean next() throws BatchValidationException + { + boolean hasNext = _in.next(); + if (hasNext) + { + Integer eventId = (Integer)_in.get(_eventIdColIndex); + _eventIds.add(eventId); + } + else + { + UserSchema schema = SNDManager.getSndUserSchema(_container, _user); + + // Add a post commit task to update the narrative cache after the transaction updating the notes is committed. + SNDManager.get().getTableInfo(schema, "EventNotes").getSchema().getScope().addCommitTask(() -> { + _sndManager.updateNarrativeCache(_container, _user, _eventIds, log); + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + return hasNext; + } + + @Override + public Object get(int i) + { + return _in.get(i); + } + + @Override + public void close() throws IOException + { + _in.close(); + } +} diff --git a/src/org/labkey/snd/query/EventNotesDataIteratorBuilder.java b/src/org/labkey/snd/query/EventNotesDataIteratorBuilder.java new file mode 100644 index 00000000..a3190fb6 --- /dev/null +++ b/src/org/labkey/snd/query/EventNotesDataIteratorBuilder.java @@ -0,0 +1,33 @@ +package org.labkey.snd.query; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.security.User; + +public class EventNotesDataIteratorBuilder implements DataIteratorBuilder +{ + private final DataIteratorBuilder in; + private final User user; + private final Container container; + + public EventNotesDataIteratorBuilder(@NotNull DataIteratorBuilder in, User user, Container container) + { + this.in = in; + this.user = user; + this.container = container; + } + + @Override + public DataIterator getDataIterator(DataIteratorContext context) + { + DataIterator it = in.getDataIterator(context); + DataIterator in = DataIteratorUtil.wrapMap(it, false); + return EventNotesDataIterator.wrap(in, context, container, user); + } +} + + diff --git a/src/org/labkey/snd/query/EventNotesTable.java b/src/org/labkey/snd/query/EventNotesTable.java index 8e097a45..a0bd74d3 100644 --- a/src/org/labkey/snd/query/EventNotesTable.java +++ b/src/org/labkey/snd/query/EventNotesTable.java @@ -36,6 +36,7 @@ import org.labkey.snd.security.permissions.SNDViewerPermission; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -67,6 +68,7 @@ public UpdateService(SimpleUserSchema.SimpleTable ti) } private final SNDService _sndService = SNDService.get(); + private final SNDManager _sndManager = SNDManager.get(); private int getRowCount(DataIteratorBuilder rows, @Nullable Map configParameters, BatchValidationException errors) { @@ -94,7 +96,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B { Logger log = SNDManager.getLogger(configParameters, EventNotesTable.class); // Large merge triggers importRows path - int result = 0; + int result; if (getRowCount(rows, configParameters, errors) > SNDManager.MAX_MERGE_ROWS) { log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); @@ -103,11 +105,21 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B else { log.info("Merging rows."); - result = super.mergeRows(user, container, rows, errors, configParameters, extraScriptContext); + DataIteratorBuilder dib = new EventNotesDataIteratorBuilder(rows, user, container); + + result = super.mergeRows(user, container, dib, errors, configParameters, extraScriptContext); } return result; } + @Override + public void configureDataIteratorContext(DataIteratorContext context) + { + if (context.getInsertOption() == QueryUpdateService.InsertOption.MERGE) + { + context.addAlternateKeys(Collections.singleton("EventId")); + } + } } @Override