From b277243987b19a0026512ff2deec38ae402d22c9 Mon Sep 17 00:00:00 2001 From: Brian Tate Date: Thu, 18 May 2023 08:07:56 -0400 Subject: [PATCH 1/9] Create project-scoped id indices for faster sort operations. --- api/main/models.py | 4 ++++ api/main/search.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api/main/models.py b/api/main/models.py index e0453ede5..d9318edaf 100644 --- a/api/main/models.py +++ b/api/main/models.py @@ -1794,6 +1794,7 @@ class Meta: # e.g. Not relaying solely on `db_index=True` in django. BUILT_IN_INDICES = { MediaType: [ + {'name': '$id', 'dtype': 'native'}, {'name': '$name', 'dtype': 'native_string'}, {'name': '$created_datetime', 'dtype': 'native'}, {'name': '$modified_datetime', 'dtype': 'native'}, @@ -1803,14 +1804,17 @@ class Meta: {'name': '$archive_state', 'dtype': 'native_string'} ], LocalizationType: [ + {'name': '$id', 'dtype': 'native'}, {'name': '$created_datetime', 'dtype': 'native'}, {'name': '$modified_datetime', 'dtype': 'native'} ], StateType: [ + {'name': '$id', 'dtype': 'native'}, {'name': '$created_datetime', 'dtype': 'native'}, {'name': '$modified_datetime', 'dtype': 'native'}, ], LeafType: [ + {'name': '$id', 'dtype': 'native'}, {'name': '$name', 'dtype': 'string'}, {'name': '$path', 'dtype': 'string'}, {'name': '$name', 'dtype': 'upper_string'}, diff --git a/api/main/search.py b/api/main/search.py index bb351bbe1..5c2efbef2 100644 --- a/api/main/search.py +++ b/api/main/search.py @@ -42,7 +42,10 @@ def _get_unique_index_name(entity_type, attribute): entity_name_sanitized=re.sub(r"[^a-zA-Z0-9]","_",entity_type.name).lower() attribute_name_sanitized=re.sub(r"[^a-zA-Z0-9]","_",attribute['name']).lower() if attribute['name'].startswith('$'): - index_name=f"tator_proj_{entity_type.project.id}_{type_name_sanitized}_internal_{attribute_name_sanitized}" + if attribute['dtype'] == 'native': + index_name=f"tator_proj_{entity_type.project.id}_internal_{attribute_name_sanitized}" + else: + index_name=f"tator_proj_{entity_type.project.id}_{type_name_sanitized}_internal_{attribute_name_sanitized}" else: index_name=f"tator_proj_{entity_type.project.id}_{type_name_sanitized}_{entity_name_sanitized}_{attribute_name_sanitized}" return index_name From 355a831a4fc3a1f367a8d1b4e804a4214a6f9486 Mon Sep 17 00:00:00 2001 From: Brian Tate Date: Mon, 22 May 2023 00:43:59 +0000 Subject: [PATCH 2/9] Add a 'after_name' for media-based queries. --- api/main/rest/_media_query.py | 4 ++++ api/main/schema/_media_query.py | 9 +++++++++ ui/src/js/annotation/annotation-page.js | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/main/rest/_media_query.py b/api/main/rest/_media_query.py index e29dddc81..429fcf72c 100644 --- a/api/main/rest/_media_query.py +++ b/api/main/rest/_media_query.py @@ -57,6 +57,7 @@ def _get_media_psql_queryset(project, filter_ops, params): gid = params.get('gid') uid = params.get('uid') after = params.get('after') + after_name = params.get('after_name') start = params.get('start') stop = params.get('stop') section_id = params.get('section') @@ -106,6 +107,9 @@ def _get_media_psql_queryset(project, filter_ops, params): if after is not None: qs = qs.filter(pk__gt=after) + if after_name is not None: + qs = qs.filter(name__gt=after_name) + if archive_states is not None: qs = qs.filter(archive_state__in=archive_states) diff --git a/api/main/schema/_media_query.py b/api/main/schema/_media_query.py index 24cdda780..cd837b9d2 100644 --- a/api/main/schema/_media_query.py +++ b/api/main/schema/_media_query.py @@ -71,6 +71,15 @@ 'parameters are relative to this modified range.', 'schema': {'type': 'integer'}, }, + { + 'name': 'after_name', + 'in': 'query', + 'required': False, + 'description': 'If given, all results returned will be after the ' + 'media with this name. The `start` and `stop` ' + 'parameters are relative to this modified range.', + 'schema': {'type': 'string'}, + }, { "name": "archive_lifecycle", "in": "query", diff --git a/ui/src/js/annotation/annotation-page.js b/ui/src/js/annotation/annotation-page.js index 860cb8c6e..52774601d 100644 --- a/ui/src/js/annotation/annotation-page.js +++ b/ui/src/js/annotation/annotation-page.js @@ -457,7 +457,7 @@ export class AnnotationPage extends TatorPage { }); const countUrl = `/rest/MediaCount/${data.project}?${searchParams.toString()}`; - searchParams.set("after", data.id); + searchParams.set("after_name", data.name); const afterUrl = `/rest/MediaCount/${data.project}?${searchParams.toString()}`; const countPromise = fetchRetry(countUrl, { method: "GET", From bebec87b400e5bbe4e335a0e567b64a860ecfd9e Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Sun, 21 May 2023 22:47:58 -0400 Subject: [PATCH 3/9] Remove new algorithm option in media card menu --- ui/src/js/components/filter-interface.js | 62 -------------------- ui/src/js/project-detail/algorithm-button.js | 4 -- 2 files changed, 66 deletions(-) diff --git a/ui/src/js/components/filter-interface.js b/ui/src/js/components/filter-interface.js index bc27655e8..29e1a1958 100644 --- a/ui/src/js/components/filter-interface.js +++ b/ui/src/js/components/filter-interface.js @@ -29,9 +29,6 @@ export class FilterInterface extends TatorElement { const filterButton = document.createElement("filter-data-button"); this._filterNavDiv.appendChild(filterButton); - this._algoButton = document.createElement("algorithm-button"); - this._filterNavDiv.appendChild(this._algoButton); - /** * Filter condition interface */ @@ -61,8 +58,6 @@ export class FilterInterface extends TatorElement { cancel.textContent = "Cancel"; footerDiv.appendChild(cancel); - - /** * Filter condition pill boxes */ @@ -81,9 +76,6 @@ export class FilterInterface extends TatorElement { this._moreNavDiv.setAttribute("class", "analysis__more_nav px-1"); this._topNav.appendChild(this._moreNavDiv); - this._confirmRunAlgorithm = document.createElement("confirm-run-algorithm"); - this._shadow.appendChild(this._confirmRunAlgorithm); - this._modalNotify = document.createElement("modal-notify"); this._shadow.appendChild(this._modalNotify); @@ -111,10 +103,6 @@ export class FilterInterface extends TatorElement { this._filterStringDiv.style.display = "flex"; this._moreNavDiv.style.display = "block"; }); - - // Respond to user requesting to run an algorithm - this._algoButton.addEventListener("runAlgorithm", this._openConfirmRunAlgoModal.bind(this)); - this._confirmRunAlgorithm.addEventListener("close", this._closeConfirmRunAlgoModal.bind(this)); } _notify(title, message, error_or_ok) { @@ -123,51 +111,6 @@ export class FilterInterface extends TatorElement { this.setAttribute("has-open-modal", ""); } - /** - * Callback when user clicks on an algorithm button. - * This launches the confirm run algorithm modal window. - */ - _openConfirmRunAlgoModal(evt) { - - var extraParameters = [ - { - name: "encoded_filters", - value: encodeURIComponent(encodeURIComponent(JSON.stringify(this._filterConditionGroup.getConditions()))), - } - ] - console.log(extraParameters); - this._confirmRunAlgorithm.init(evt.detail.algorithmName, evt.detail.projectId, [], null, extraParameters); - this._confirmRunAlgorithm.setAttribute("is-open",""); - this.setAttribute("has-open-modal", ""); - document.body.classList.add("shortcuts-disabled"); - } - - /** - * Callback from confirm run algorithm modal choice - */ - _closeConfirmRunAlgoModal(evt) { - - this._confirmRunAlgorithm.removeAttribute("is-open"); - this.removeAttribute("has-open-modal"); - document.body.classList.remove("shortcuts-disabled"); - - var that = this; - if (evt.detail.confirm) { - this._dataView.launchAlgorithm(evt.detail.algorithmName, evt.detail.extraParameters).then(launched => { - if (launched) { - that._notify("Algorithm launched!", - `Successfully launched ${evt.detail.algorithmName}!`, - "ok"); - } - else { - that._notify("Error launching algorithm!", - `Failed to launch ${evt.detail.algorithmName}`, - "error"); - } - }); - } - } - /** * Applies the filter conditions based on the dialog, updates the GUI appropriately and * dispatches the filterParameters event @@ -202,11 +145,6 @@ export class FilterInterface extends TatorElement { this._filterConditionGroup.data = this._dataView.getAllTypes(); this._filterConditionGroup._div.style.marginTop = "10px"; this._conditionsDiv.appendChild(this._filterConditionGroup); - - // Get the algorithm information. If there are no registered algorithms, disable the button - this._algoButton.setAttribute("project-id", this._dataView.getProjectId()); - this._algoButton.algorithms = this._dataView.getAlgorithms(); - this._algoButton.hideNewAlgorithm(); } /** diff --git a/ui/src/js/project-detail/algorithm-button.js b/ui/src/js/project-detail/algorithm-button.js index a49b90e2c..66ead2e44 100644 --- a/ui/src/js/project-detail/algorithm-button.js +++ b/ui/src/js/project-detail/algorithm-button.js @@ -51,10 +51,6 @@ export class AlgorithmButton extends TatorElement { set algorithms(val) { this._algorithmMenu.algorithms = val; } - - hideNewAlgorithm() { - this._algorithmMenu.hideNewAlgorithm(); - } } if (!customElements.get("algorithm-button")) { From c5a01a12b844c7933d36bda0debdf091dcd9df6e Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Sun, 21 May 2023 22:48:30 -0400 Subject: [PATCH 4/9] Remove algorithm menu option + force left-align in algo menu (especially for long algo names) --- ui/src/js/project-detail/algorithm-menu.js | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/ui/src/js/project-detail/algorithm-menu.js b/ui/src/js/project-detail/algorithm-menu.js index 8e4908456..619c48ac2 100644 --- a/ui/src/js/project-detail/algorithm-menu.js +++ b/ui/src/js/project-detail/algorithm-menu.js @@ -7,20 +7,15 @@ export class AlgorithmMenu extends TatorElement { this._algorithmButtons = document.createElement("div"); this._algorithmButtons.setAttribute("class", "d-flex flex-column px-4 py-3 lh-condensed"); this._shadow.appendChild(this._algorithmButtons); - - this._newAlgorithm = document.createElement("new-algorithm-button"); - this._algorithmButtons.appendChild(this._newAlgorithm); - - this._newAlgorithm.addEventListener("click", evt => { - this.dispatchEvent(new Event("newAlgorithm", {composed: true})); - }); } set algorithms(val) { + for (let algorithm of val) { const button = document.createElement("button"); button.setAttribute("class", "btn-clear py-2 px-0 text-gray hover-text-white d-flex flex-items-center"); - this._algorithmButtons.insertBefore(button, this._newAlgorithm); + button.style.textAlign = "left"; + this._algorithmButtons.appendChild(button); const span = document.createElement("span"); span.setAttribute("class", "px-2"); @@ -35,10 +30,6 @@ export class AlgorithmMenu extends TatorElement { }); } } - - hideNewAlgorithm() { - this._newAlgorithm.hidden = true; - } } if (!customElements.get("algorithm-menu")) { From de57b98387fa0f27e17a570904b66db0cc389ef9 Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Sun, 21 May 2023 22:49:43 -0400 Subject: [PATCH 5/9] Query media IDs of selected section prior to launching algo Update confirm launch workflow dialog UI --- .../project-detail/confirm-run-algorithm.js | 115 +++++++++++++-- ui/src/js/project-detail/media-section.js | 47 +++--- ui/src/js/project-detail/project-detail.js | 135 +++++++++++------- 3 files changed, 209 insertions(+), 88 deletions(-) diff --git a/ui/src/js/project-detail/confirm-run-algorithm.js b/ui/src/js/project-detail/confirm-run-algorithm.js index 19db6652a..23870955c 100644 --- a/ui/src/js/project-detail/confirm-run-algorithm.js +++ b/ui/src/js/project-detail/confirm-run-algorithm.js @@ -1,21 +1,46 @@ import { ModalDialog } from "../components/modal-dialog.js"; +import { fetchRetry } from "../util/fetch-retry.js"; +import { svgNamespace } from "../components/tator-element.js"; export class ConfirmRunAlgorithm extends ModalDialog { constructor() { super(); - const icon = document.createElement("modal-warning"); - this._header.insertBefore(icon, this._titleDiv); + const iconWrapper = document.createElement("div"); + this._header.insertBefore(iconWrapper, this._titleDiv); + + const svg = document.createElementNS(svgNamespace, "svg"); + svg.setAttribute("width", "24"); + svg.setAttribute("height", "24"); + svg.setAttribute("viewBox", "0 0 24 24"); + svg.setAttribute("fill", "none"); + svg.style.fill = "none"; + svg.setAttribute("stroke", "currentColor"); + svg.setAttribute("stroke-width", "2"); + svg.setAttribute("stroke-linecap", "round"); + svg.setAttribute("stroke-linejoin", "round"); + iconWrapper.appendChild(svg); + + const poly = document.createElementNS(svgNamespace, "polyline"); + poly.setAttribute("points", "22 12 18 12 15 21 9 3 6 12 2 12"); + svg.appendChild(poly); this._message = document.createElement("p"); - this._message.setAttribute("class", "text-semibold py-3 text-center"); + this._message.setAttribute("class", "py-3 text-center"); this._main.appendChild(this._message); + this._icon = document.createElement("div"); + this._icon.setAttribute("class", "text-center py-3") + this._icon.style.margin = "auto"; + this._icon.innerHTML = ''; + this._icon.style.display = "none"; + //this._main.appendChild(this._icon); + this._accept = document.createElement("button"); this._accept.setAttribute("class", "btn btn-clear btn-purple"); this._accept.textContent = "Yes"; this._footer.appendChild(this._accept); - + this._cancel = document.createElement("button"); this._cancel.setAttribute("class", "btn btn-clear btn-charcoal"); this._cancel.textContent = "No"; @@ -37,7 +62,7 @@ export class ConfirmRunAlgorithm extends ModalDialog { confirm: true, projectId: this._projectId, mediaIds: this._mediaIds, - mediaQuery: this._mediaQuery, + section: this._section, algorithmName: this._algorithmName, extraParameters: this._extraParameters}})); }); @@ -46,21 +71,83 @@ export class ConfirmRunAlgorithm extends ModalDialog { /** * Initialize the dialog window with the algorithm information prior * to displaying it - * + * * @param {integer} projectId Project ID associate with algorithm - * @param {string} algorithmName Name of algorithm to run - * @param {array} mediaIds List of media IDs to process - * @param {string} mediaQuery Media query string when launching algorithm - * @param {array} extraParameters #TODO add useful info + * @param {string} algorithmName Name of workflow to run + * @param {array} mediaIds List of media IDs to process. Can be null, if so section is checked. + * @param {Tator.Section} section Section to process. If this and mediaIds is null, assume that all the media in the project will be processed. + * @param {array} extraParameters key/value pairs of parameters to pass along when launching the workflow */ - init(algorithmName, projectId, mediaIds, mediaQuery, extraParameters) + async init(algorithmName, projectId, mediaIds, section, extraParameters) { - this._title.nodeValue = "Run Algorithm"; - this._message.textContent = "Do you want to run " + algorithmName + "?"; + this._title.nodeValue = `Launch Workflow`; + this._accept.style.display = "none"; + this._cancel.style.display = "none"; + this._message.textContent = `Retrieving data...`; + this._message.classList.add("text-gray"); + + if (mediaIds == null) { + + if (section == null) { + var response = await fetchRetry(`/rest/MediaCount/${projectId}`, { + method: "GET", + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + "Accept": "application/json", + "Content-Type": "application/json" + } + }); + var count = await response.json(); + var msgText = `Do you want to run ` + algorithmName + ` on
the ALL media (media count: ${count})?`; + } + else { + var response = await fetchRetry(`/rest/MediaCount/${projectId}?section=${section.id}`, { + method: "GET", + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + "Accept": "application/json", + "Content-Type": "application/json" + } + }); + var count = await response.json(); + var msgText = `Do you want to run ` + algorithmName + ` on
media in ${section.name} (media count: ${count})?`; + } + + // Small delay so that the user can acknowledge something was happening. + // This isn't something we want users to rush through anyway. + await new Promise(resolve => setTimeout(resolve, 2000)); + } + else { + var count = mediaIds.length; + if (count == 1) { + + var response = await fetchRetry(`/rest/Media/${mediaIds[0]}`, { + method: "GET", + credentials: "same-origin", + headers: { + "X-CSRFToken": getCookie("csrftoken"), + "Accept": "application/json", + "Content-Type": "application/json" + } + }); + var media = await response.json(); + var msgText = `Do you want to run ` + algorithmName + ` on
${media.name} (ID: ${media.id})?`; + } + else { + var msgText = `Do you want to run ` + algorithmName + ` on
${count} media?`; + } + } + + this._message.classList.remove("text-gray"); + this._accept.style.display = "flex"; + this._cancel.style.display = "flex"; + this._message.innerHTML = msgText; this._algorithmName = algorithmName; this._projectId = projectId; this._mediaIds = mediaIds; - this._mediaQuery = mediaQuery; + this._section = section; this._extraParameters = extraParameters; } diff --git a/ui/src/js/project-detail/media-section.js b/ui/src/js/project-detail/media-section.js index 4a3afe170..e432d7a3a 100644 --- a/ui/src/js/project-detail/media-section.js +++ b/ui/src/js/project-detail/media-section.js @@ -23,7 +23,10 @@ export class MediaSection extends TatorElement { header.appendChild(this._name); this._nameText = document.createTextNode(""); + this._nameIdText = document.createElement("span"); + this._nameIdText.setAttribute("class", "text-dark-gray px-1"); this._name.appendChild(this._nameText); + this._name.appendChild(this._nameIdText); const numFiles = document.createElement("span"); numFiles.setAttribute("class", "text-gray px-2"); @@ -31,7 +34,7 @@ export class MediaSection extends TatorElement { this._numFiles = document.createTextNode(""); numFiles.appendChild(this._numFiles); - + const pagePosition = document.createElement("div"); pagePosition.setAttribute("class", "py-3 f1 text-normal text-gray"); this._name.appendChild(pagePosition); @@ -75,7 +78,7 @@ export class MediaSection extends TatorElement { this._files = document.createElement("section-files"); this._files.setAttribute("class", "col-12"); this._files.mediaParams = this._sectionParams.bind(this); - + div.appendChild(this._files); this._paginator_bottom = document.createElement("entity-gallery-paginator"); @@ -87,7 +90,7 @@ export class MediaSection extends TatorElement { this._searchParams = new URLSearchParams(); this._numFilesCount = 0; this._searchString = ""; - + this._setCallbacks(); } @@ -95,17 +98,19 @@ export class MediaSection extends TatorElement { if (section === null) { this._sectionName = "All Media"; this._upload.setAttribute("section", ""); + this._nameText.nodeValue = "All Media"; + this._nameIdText.textContent = ""; } else { this._sectionName = section.name; this._upload.setAttribute("section", section.name); + this._nameText.nodeValue = `${section.name}`; + this._nameIdText.textContent = `(ID: ${section.id})`; } this._project = project; this._section = section; this._sectionName = this._sectionName; this._files.setAttribute("project-id", project); - - - this._nameText.nodeValue = this._sectionName; + this._upload.setAttribute("project-id", project); this._more.section = section; @@ -113,7 +118,7 @@ export class MediaSection extends TatorElement { this._stop = this._paginator_top._pageSize; this._after = new Map(); - + return this.reload(); } @@ -193,7 +198,7 @@ export class MediaSection extends TatorElement { removeMedia(mediaId) { const single = !(mediaId.indexOf(",") > -1); - if (!single) mediaId = mediaId.split(","); + if (!single) mediaId = mediaId.split(","); console.log("MEDIA ID (list or single? ... "+single); console.log(mediaId); @@ -205,13 +210,13 @@ export class MediaSection extends TatorElement { mediaCard.parentNode.removeChild(mediaCard); const numFiles = Number(this._numFiles.textContent.split(' ')[0]) - 1; this._updateNumFiles(numFiles); // do this at the end - } + } } - + // clear any selected cards & reload this.reload(); this._bulkEdit.clearAllCheckboxes(); - + } _updateNumFiles(numFiles) { @@ -221,7 +226,7 @@ export class MediaSection extends TatorElement { } this._numFiles.nodeValue = `${numFiles} ${fileText}`; this._numFilesCount = Number(numFiles); - + if (numFiles != this._paginator_top._numFiles) { this._start = 0; this._stop = this._paginator_top._pageSize; @@ -356,7 +361,7 @@ export class MediaSection extends TatorElement { composed: true, detail: { algorithmName: evt.detail.algorithmName, - mediaQuery: `?${this._sectionParams().toString()}`, + section: this._section, projectId: this._project, } })); @@ -709,7 +714,7 @@ export class MediaSection extends TatorElement { setTimeout(() => { this._name.classList.remove("text-green"); }, 800) - + this.dispatchEvent(new CustomEvent("newName", { detail: { id: this._section.id, @@ -779,11 +784,11 @@ export class MediaSection extends TatorElement { }); this._paginator_top.addEventListener("selectPage", (evt) => { - this._setPage(evt, this._paginator_bottom); + this._setPage(evt, this._paginator_bottom); }); this._paginator_bottom.addEventListener("selectPage", (evt) => { - this._setPage(evt, this._paginator_top); + this._setPage(evt, this._paginator_top); }); this._reload.addEventListener("click", this.reload.bind(this)); @@ -797,8 +802,8 @@ export class MediaSection extends TatorElement { // clear any selected cards this._bulkEdit.clearAllCheckboxes(); - otherPaginator.init(otherPaginator._numFiles, this._paginationState); - + otherPaginator.init(otherPaginator._numFiles, this._paginationState); + this._updatePageArgs(); await this._loadMedia(); } @@ -822,7 +827,7 @@ export class MediaSection extends TatorElement { newUrl += `${key}=${value}`; firstParm = false; } - + } // Only add back params if we're not on default page 1, and pageSize 10 @@ -832,7 +837,7 @@ export class MediaSection extends TatorElement { } else { newUrl += "?"; } - newUrl += `page=${Number(this._paginator_top._page) + 1}&pagesize=${this._paginator_top._pageSize}` + newUrl += `page=${Number(this._paginator_top._page) + 1}&pagesize=${this._paginator_top._pageSize}` } window.history.pushState({}, "", newUrl); @@ -856,7 +861,7 @@ export class MediaSection extends TatorElement { { finalMetadataFilters.push(this._modelData._convertFilterForTator(filter)); } - + } if (finalMediaFilters.length > 0) diff --git a/ui/src/js/project-detail/project-detail.js b/ui/src/js/project-detail/project-detail.js index e1822edd2..223e9d3cf 100644 --- a/ui/src/js/project-detail/project-detail.js +++ b/ui/src/js/project-detail/project-detail.js @@ -8,6 +8,8 @@ import { FilterData } from "../components/filter-data.js"; import { v1 as uuidv1 } from "uuid"; import { store } from "./store.js"; import { api } from "./store.js"; +import { FilterConditionData } from "../util/filter-utilities.js" +import { fetchRetry } from "../util/fetch-retry.js"; import Gear from "../../images/svg/gear.svg"; export class ProjectDetail extends TatorPage { @@ -44,7 +46,7 @@ export class ProjectDetail extends TatorPage { /* LEFT*** Navigation Pane - Project Detail Viewer */ this.aside = document.createElement("aside"); - this.aside.setAttribute("class", "entity-panel--container-left col-3"); //slide-close + this.aside.setAttribute("class", "entity-panel--container-left col-3"); //slide-close this.aside.hidden = true; this.mainWrapper.appendChild(this.aside); @@ -54,7 +56,7 @@ export class ProjectDetail extends TatorPage { // const section = document.createElement("section"); - section.setAttribute("class", "sections-wrap py-6 col-3 px-5 text-gray"); // + section.setAttribute("class", "sections-wrap py-6 col-3 px-5 text-gray"); // const folderHeader = document.createElement("div"); folderHeader.setAttribute("class", "d-flex flex-justify-between flex-items-center py-4"); @@ -136,7 +138,7 @@ export class ProjectDetail extends TatorPage { section.appendChild(this._archivedFolders); this._mainSection = document.createElement("section"); - this._mainSection.setAttribute("class", "py-3 px-6 flex-grow"); //project__main + this._mainSection.setAttribute("class", "py-3 px-6 flex-grow"); //project__main this.main.appendChild(this._mainSection); this.gallery = {}; @@ -202,7 +204,6 @@ export class ProjectDetail extends TatorPage { this._mainSection.appendChild(filterdiv); this._filterView = document.createElement("filter-interface"); - this._filterView._algoButton.hidden = true; filterdiv.appendChild(this._filterView); this._collaborators = document.createElement("project-collaborators"); @@ -565,17 +566,6 @@ export class ProjectDetail extends TatorPage { this.removeAttribute("has-open-modal", ""); }); - this._newAlgorithmCallback = evt => { - const newAlgorithm = document.createElement("new-algorithm-form"); - this._projects.appendChild(newAlgorithm); - newAlgorithm.setAttribute("is-open", ""); - this.setAttribute("has-open-modal", ""); - newAlgorithm.addEventListener("close", evt => { - this.removeAttribute("has-open-modal", ""); - this._projects.removeChild(evt.target); - }); - }; - cancelJob.addEventListener("confirmGroupCancel", () => { cancelJob.removeAttribute("is-open"); }); @@ -830,11 +820,11 @@ export class ProjectDetail extends TatorPage { projectId, additionalTools: true, permission: project.permission - }); + }); - const parsedAlgos = algos.filter(function (alg) { + var parsedAlgos = algos.filter(function (alg) { if (Array.isArray(alg.categories)) { for (const category of alg.categories) { if (hiddenAlgoCategories.includes(category)) { @@ -844,6 +834,8 @@ export class ProjectDetail extends TatorPage { } return !hiddenAlgos.includes(alg.name); }); + parsedAlgos.sort((a, b) => a.name.localeCompare(b.name)); + if (!hasPermission(project.permission, "Full Control")) { this._settingsButton.style.display = "none"; } @@ -1003,7 +995,6 @@ export class ProjectDetail extends TatorPage { this._mediaSection.addEventListener("remove", this._removeCallback); this._mediaSection.addEventListener("moveFile", this._moveFileCallback); this._mediaSection.addEventListener("deleteFile", this._deleteFileCallback); - this._mediaSection.addEventListener("newAlgorithm", this._newAlgorithmCallback); params.delete("section"); if (section !== null) { @@ -1059,15 +1050,8 @@ export class ProjectDetail extends TatorPage { */ _openConfirmRunAlgoModal(evt) { - if ('mediaIds' in evt.detail) { - this._confirmRunAlgorithm.init( - evt.detail.algorithmName, evt.detail.projectId, evt.detail.mediaIds, null); - } - else { - this._confirmRunAlgorithm.init( - evt.detail.algorithmName, evt.detail.projectId, null, evt.detail.mediaQuery); - } - + this._confirmRunAlgorithm.init( + evt.detail.algorithmName, evt.detail.projectId, evt.detail.mediaIds, evt.detail.section); this._confirmRunAlgorithm.setAttribute("is-open", ""); this.setAttribute("has-open-modal", ""); document.body.classList.add("shortcuts-disabled"); @@ -1075,29 +1059,77 @@ export class ProjectDetail extends TatorPage { /** * Callback from confirm run algorithm modal choice + * @async */ - _closeConfirmRunAlgoModal(evt) { + async _closeConfirmRunAlgoModal(evt) { + + console.log(evt); this._confirmRunAlgorithm.removeAttribute("is-open"); this.removeAttribute("has-open-modal"); document.body.classList.remove("shortcuts-disabled"); + if (evt.detail == null) { return; } + var that = this; + var jobMediaIds = []; if (evt.detail.confirm) { - if (evt.detail.mediaIds != null) { - var body = JSON.stringify({ - "algorithm_name": evt.detail.algorithmName, - "media_ids": evt.detail.mediaIds - }); + + // Retrieve media IDs first (if needed) + if (evt.detail.mediaIds == null) { + + this.showDimmer(); + this.loading.showSpinner(); + + var filterConditions = []; + var mediaTypes = this._modelData.getStoredMediaTypes(); + if (evt.detail.section != null) { + filterConditions.push(new FilterConditionData(mediaTypes[0].name, "$section", "==", `${evt.detail.section.id}`, "")) + } + + var totalCounts = await this._modelData.getFilteredMedias("count", filterConditions); + console.log(`mediaCounts: ${totalCounts}`); + var afterMap = new Map(); + var pageSize = 5000; + var pageStart = 0; + var pageEnd = pageStart + pageSize; + var allMedia = []; + var numPages = Math.floor(totalCounts / pageSize) + 1; + var pageCount = 1; + while (allMedia.length < totalCounts) { + + console.log(`Processing media page ${pageCount} of ${numPages}`); + var pageMedia = await this._modelData.getFilteredMedias( + "objects", + filterConditions, + 0, + Math.min(totalCounts - allMedia.length, 5000), + afterMap, + true); + allMedia.push(...pageMedia); + pageStart = pageEnd; + pageEnd = pageStart + pageSize; + pageCount += 1; + } + + for (const media of allMedia) { + jobMediaIds.push(media.id); + } + + this.loading.hideSpinner(); + this.hideDimmer(); } else { - var body = JSON.stringify({ - "algorithm_name": evt.detail.algorithmName, - "media_query": evt.detail.mediaQuery - }); + jobMediaIds = evt.detail.mediaIds; } - fetch("/rest/Jobs/" + evt.detail.projectId, { + var body = JSON.stringify({ + "algorithm_name": evt.detail.algorithmName, + "media_ids": jobMediaIds + }); + console.log(`${jobMediaIds.length} | ${evt.detail.algorithmName}`); + + var response = await fetchRetry("/rest/Jobs/" + evt.detail.projectId, { method: "POST", credentials: "same-origin", headers: { @@ -1106,21 +1138,18 @@ export class ProjectDetail extends TatorPage { "Content-Type": "application/json" }, body: body, - }) - .then(response => { - if (response.status == 201) { - that._notify("Algorithm launched!", - `Successfully launched ${evt.detail.algorithmName}! Monitor progress by clicking the "Activity" button.`, - "ok"); - } - else { - that._notify("Error launching algorithm!", - `Failed to launch ${evt.detail.algorithmName}: ${response.statusText}.`, - "error"); - } - return response.json(); - }) - .then(data => console.log(data)); + }); + var data = await response.json(); + if (data.status == 201) { + that._notify("Workflow launched!", + `Successfully launched ${evt.detail.algorithmName}! Monitor progress by clicking the "Activity" button.`, + "ok"); + } + else { + that._notify("Error launching workflow!", + `Failed to launch ${evt.detail.algorithmName}: ${data.statusText}.`, + "error"); + } } } From 27622357775c8946270ed690a14c19e52e9fae30 Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Sun, 21 May 2023 22:50:01 -0400 Subject: [PATCH 6/9] Fix getFilteredStates + add ignorePresign option in tator-data --- ui/src/js/util/tator-data.js | 100 ++++++++++------------------------- 1 file changed, 28 insertions(+), 72 deletions(-) diff --git a/ui/src/js/util/tator-data.js b/ui/src/js/util/tator-data.js index dd8eb3cfc..772e45fc2 100644 --- a/ui/src/js/util/tator-data.js +++ b/ui/src/js/util/tator-data.js @@ -649,7 +649,8 @@ export class TatorData { listStart, listStop, afterMap, - mediaIds) { + mediaIds, + ignorePresign) { var finalAnnotationFilters = []; for (const filter of annotationFilterData) { @@ -727,7 +728,7 @@ export class TatorData { } } - if (annotationType == "Medias") { + if (annotationType == "Medias" && !ignorePresign) { url += "&presigned=28800"; } @@ -838,6 +839,7 @@ export class TatorData { listStop, afterMap, mediaIds, + null ); return outData; @@ -872,36 +874,23 @@ export class TatorData { */ async getFilteredStates(outputType, filters, listStart, listStop, afterMap) { - throw new Error("#TODO getFilteredStates requires a v1 related update!"); - // Loop through the filters, if there are any media specific ones var mediaFilters = []; var stateFilters = []; - var typeIds = []; - var versionIds = []; - var typePromises = []; var mediaIds = []; // Separate out the filter conditions into their groups if (Array.isArray(filters)) { filters.forEach(filter => { if (this._mediaTypeNames.indexOf(filter.category) >= 0) { - if (filter.field == "$section") { - var newFilter = Object.assign({}, filter); - newFilter.field = "_section"; - newFilter.value = filter.value; - mediaFilters.push(newFilter); + if (filter.field == "$id") { + mediaIds.push(Number(filter.value)); } - else if (filter.field == "_dtype") { + else if (filter.field.includes("$") && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); - newFilter.field = "_meta"; - newFilter.value = filter.value.split('(ID:')[1].replace(")",""); + newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); mediaFilters.push(newFilter); } - else if (filter.field == "Modified By") { - filter.field = "_modified_by"; - mediaFilters.push(filter); - } else { mediaFilters.push(filter); } @@ -910,11 +899,10 @@ export class TatorData { stateFilters.push(filter); } else if (this._stateTypeNames.indexOf(filter.category) >= 0) { - if (filter.field == "_version") { - versionIds.push(Number(filter.value.split('(ID:')[1].replace(")",""))); - } - else if (filter.field == "_type") { - typeIds.push(Number(filter.value.split('(ID:')[1].replace(")",""))); + if (filter.field.includes("$") && filter.value.includes("(ID:")) { + var newFilter = Object.assign({}, filter); + newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); + stateFilters.push(newFilter); } else { stateFilters.push(filter); @@ -923,54 +911,18 @@ export class TatorData { }); } - if (typeIds.length > 0) { - typeIds.forEach(dtypeId => { - typePromises.push(this._getAnnotationData( - outputType, - "States", - stateFilters, - mediaFilters, - listStart, - listStop, - afterMap, - mediaIds, - versionIds, - dtypeId - )); - }); - } - else { - typePromises.push(this._getAnnotationData( - outputType, - "States", - stateFilters, - mediaFilters, - listStart, - listStop, - afterMap, - mediaIds, - versionIds - )); - } - - // Wait for all the data requests to complete. Once complete, return the appropriate data. - var typeResults = await Promise.all(typePromises); - var outData; - if (outputType == "count") { - outData = 0; - } - else { - outData = []; - } + var outData = await this._getAnnotationData( + outputType, + "States", + stateFilters, + mediaFilters, + listStart, + listStop, + afterMap, + mediaIds, + null + ); - for (let idx = 0; idx < typeResults.length; idx++) { - if (outputType == "count") { - outData += Number(typeResults[idx]); - } - else { - outData.push(...typeResults[idx]); - } - } return outData; } @@ -996,10 +948,13 @@ export class TatorData { * Used in conjunction with the other pagination. Modified here. * If null, pagination is ignored. * + * @param {bool} ignorePresign + * Used to ignored presigning media files + * * @returns {array of integers} * List of localization IDs matching the filter criteria */ - async getFilteredMedias(outputType, filters, listStart, listStop, afterMap) { + async getFilteredMedias(outputType, filters, listStart, listStop, afterMap, ignorePresign) { // Loop through the filters, if there are any media specific ones var mediaFilters = []; @@ -1047,6 +1002,7 @@ export class TatorData { listStop, afterMap, mediaIds, + ignorePresign ); return outData; From caaf2586fbea7a23fc9985f1239d2514356c3300 Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Mon, 22 May 2023 14:53:11 -0400 Subject: [PATCH 7/9] Fix cases where provided filter condition may not be a string --- ui/src/js/util/tator-data.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ui/src/js/util/tator-data.js b/ui/src/js/util/tator-data.js index 772e45fc2..8b09120f4 100644 --- a/ui/src/js/util/tator-data.js +++ b/ui/src/js/util/tator-data.js @@ -618,7 +618,7 @@ export class TatorData { var value = filter.value; var field = filter.field; - if (field.includes("$") && value.includes("(ID:")) { + if (field.includes("$") && typeof(value) === "string" && value.includes("(ID:")) { value = Number(value.split('(ID:')[1].replace(")","")); } @@ -808,7 +808,7 @@ export class TatorData { if (filter.field == "$id") { mediaIds.push(Number(filter.value)); } - else if (filter.field.includes("$") && filter.value.includes("(ID:")) { + else if (filter.field.includes("$") && typeof(filter.value) === "string" && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); mediaFilters.push(newFilter); @@ -818,7 +818,7 @@ export class TatorData { } } else if (this._localizationTypeNames.indexOf(filter.category) >= 0) { - if (filter.field.includes("$") && filter.value.includes("(ID:")) { + if (filter.field.includes("$") && typeof(filter.value) === "string" && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); localizationFilters.push(newFilter); @@ -886,7 +886,7 @@ export class TatorData { if (filter.field == "$id") { mediaIds.push(Number(filter.value)); } - else if (filter.field.includes("$") && filter.value.includes("(ID:")) { + else if (filter.field.includes("$") && typeof(filter.value) === "string" && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); mediaFilters.push(newFilter); @@ -899,7 +899,7 @@ export class TatorData { stateFilters.push(filter); } else if (this._stateTypeNames.indexOf(filter.category) >= 0) { - if (filter.field.includes("$") && filter.value.includes("(ID:")) { + if (filter.field.includes("$") && typeof(filter.value) === "string" && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); stateFilters.push(newFilter); @@ -971,7 +971,7 @@ export class TatorData { if (filter.field.includes("$id")) { mediaIds.push(Number(filter.value)); } - else if (filter.field.includes("$") && filter.value.includes("(ID:")) { + else if (filter.field.includes("$") && typeof(filter.value) === "string" && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); mediaFilters.push(newFilter); @@ -981,7 +981,7 @@ export class TatorData { } } else if (this._localizationTypeNames.indexOf(filter.category) >= 0) { - if (filter.field.includes("$") && filter.value.includes("(ID:")) { + if (filter.field.includes("$") && typeof(filter.value) === "string" && filter.value.includes("(ID:")) { var newFilter = Object.assign({}, filter); newFilter.value = Number(filter.value.split('(ID:')[1].replace(")","")); mediaFilters.push(newFilter); From aca670472cecf59dc3271c182a04538bcac9fc12 Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Mon, 22 May 2023 16:35:28 -0400 Subject: [PATCH 8/9] Fixed bug with algorithm launch status --- ui/src/js/project-detail/project-detail.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/js/project-detail/project-detail.js b/ui/src/js/project-detail/project-detail.js index 223e9d3cf..d624622ab 100644 --- a/ui/src/js/project-detail/project-detail.js +++ b/ui/src/js/project-detail/project-detail.js @@ -1140,14 +1140,14 @@ export class ProjectDetail extends TatorPage { body: body, }); var data = await response.json(); - if (data.status == 201) { + if (response.status == 201) { that._notify("Workflow launched!", `Successfully launched ${evt.detail.algorithmName}! Monitor progress by clicking the "Activity" button.`, "ok"); } else { that._notify("Error launching workflow!", - `Failed to launch ${evt.detail.algorithmName}: ${data.statusText}.`, + `Failed to launch ${evt.detail.algorithmName}: ${response.statusText}.`, "error"); } } From 70e3d5d1460f626a724951a04a1a4dd731078de2 Mon Sep 17 00:00:00 2001 From: Mark Taipan Date: Mon, 22 May 2023 17:14:39 -0400 Subject: [PATCH 9/9] Fix typo in dialog --- ui/src/js/project-detail/confirm-run-algorithm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/js/project-detail/confirm-run-algorithm.js b/ui/src/js/project-detail/confirm-run-algorithm.js index 23870955c..c744b797c 100644 --- a/ui/src/js/project-detail/confirm-run-algorithm.js +++ b/ui/src/js/project-detail/confirm-run-algorithm.js @@ -99,7 +99,7 @@ export class ConfirmRunAlgorithm extends ModalDialog { } }); var count = await response.json(); - var msgText = `Do you want to run ` + algorithmName + ` on
the ALL media (media count: ${count})?`; + var msgText = `Do you want to run ` + algorithmName + ` on
ALL media (media count: ${count})?`; } else { var response = await fetchRetry(`/rest/MediaCount/${projectId}?section=${section.id}`, {