From 56858e1db50cf7dcf01a601ed60eec3f6f5b5693 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Tue, 7 Jan 2025 14:05:04 -0500 Subject: [PATCH 01/64] Make FeatureUtils and FeatureCache accessible in client API --- js/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/index.js b/js/index.js index 580e769e3..e16c9120c 100644 --- a/js/index.js +++ b/js/index.js @@ -7,7 +7,7 @@ import {createBrowser, createTrack, removeAllBrowsers, removeBrowser, visibility import embedCss from "./embedCss.js" import version from "./version.js" import * as TrackUtils from "./util/trackUtils.js" -import {igvxhr} from "../node_modules/igv-utils/src/index.js" +import {igvxhr, FeatureUtils, FeatureCache} from "../node_modules/igv-utils/src/index.js" import {registerTrackClass, registerTrackCreatorFunction} from "./trackFactory.js" import TrackBase from "./trackBase.js" import Hub from "./ucsc/ucscHub.js" @@ -30,6 +30,8 @@ const oauth = igvxhr.oauth export default { TrackUtils, + FeatureUtils, + FeatureCache, IGVGraphics, MenuUtils, DataRangeDialog, From 033aef22d79dca97c2e6a4dbeecd483e47d9443f Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Tue, 7 Jan 2025 15:14:31 -0500 Subject: [PATCH 02/64] Add proof-of-concept for dynamic filtering --- examples/dynamic-filtering.html | 167 ++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 examples/dynamic-filtering.html diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html new file mode 100644 index 000000000..d5e568fec --- /dev/null +++ b/examples/dynamic-filtering.html @@ -0,0 +1,167 @@ + + + + + + + igv.js + + + + + + +

Ad-hoc filtering in IGV

+ + +
+ + + + + + From 5472a2dd9b98282ceb88138b667c42848c4a764b Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Tue, 7 Jan 2025 15:19:25 -0500 Subject: [PATCH 03/64] Parameterize dimension to filter --- examples/dynamic-filtering.html | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index d5e568fec..85ad9e423 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -68,7 +68,7 @@ return isFeatureInFrame } - function filterVariantType() { + function filterFeatures(dimension) { const trackIndex = 3 // Track index const igvBrowser = window.igvBrowser @@ -88,7 +88,7 @@ } const filteredFeatures = originalChrFeatures.filter( - feature => scoreSelection.has(feature.type) && getIsFeatureInFrame(feature, igvBrowser) + feature => scoreSelection.has(feature[dimension]) && getIsFeatureInFrame(feature, igvBrowser) ) console.log('filteredFeatures', filteredFeatures) @@ -102,11 +102,8 @@

Ad-hoc filtering in IGV

    Type -
    • -
    • -
    • -
    • -
    • +
    • +
From 7f2447a03e9340a0dbfacfa1f4a645e64d894e12 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Tue, 7 Jan 2025 15:28:29 -0500 Subject: [PATCH 04/64] Parameterize track selector for dynamic filtering --- examples/dynamic-filtering.html | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index 85ad9e423..a89f5dfa4 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -68,8 +68,19 @@ return isFeatureInFrame } - function filterFeatures(dimension) { - const trackIndex = 3 // Track index + function filterFeatures(trackSelector, dimension) { + let trackIndex + console.log('typeof trackSelector', typeof trackSelector) + if (typeof trackSelector === 'string') { + window.igvBrowser.trackViews.find((trackView, i) => { + trackIndex = i + return trackView.track.config?.name === trackSelector + }) + } else { + trackIndex = trackSelector + } + + // const trackIndex = 3 // Track index const igvBrowser = window.igvBrowser const scoreSelection = new Set([]) @@ -102,8 +113,8 @@

Ad-hoc filtering in IGV

    Type -
    • -
    • +
    • +
From cec7cbacc2a29cb789da661edd8fd4f5213b636d Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Wed, 8 Jan 2025 10:28:26 -0500 Subject: [PATCH 05/64] Scaffold initFacets, to generalize dynamic filtering --- examples/dynamic-filtering.html | 79 ++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index a89f5dfa4..14d24ac87 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -68,20 +68,8 @@ return isFeatureInFrame } - function filterFeatures(trackSelector, dimension) { - let trackIndex - console.log('typeof trackSelector', typeof trackSelector) - if (typeof trackSelector === 'string') { - window.igvBrowser.trackViews.find((trackView, i) => { - trackIndex = i - return trackView.track.config?.name === trackSelector - }) - } else { - trackIndex = trackSelector - } - - // const trackIndex = 3 // Track index - const igvBrowser = window.igvBrowser + function filterFeatures(trackSelector, dimension, igvBrowser) { + const trackIndex = getTrackIndex(trackSelector, igvBrowser); const scoreSelection = new Set([]) @@ -105,6 +93,67 @@ console.log('filteredFeatures', filteredFeatures) updateTrack(trackIndex, filteredFeatures, igv, igvBrowser) } + + function getFeaturesInFrame(trackSelector, igvBrowser) { + const trackIndex = getTrackIndex(trackSelector, igvBrowser); + const originalChrFeatures = getOriginalChrFeatures(trackIndex, igvBrowser) + const featuresInFrame = originalChrFeatures.filter( + feature => getIsFeatureInFrame(feature, igvBrowser) + ) + return featuresInFrame + } + + /** Get index position of track in IGV browser, given its name or index */ + function getTrackIndex(trackSelector, igvBrowser) { + let trackIndex = null + if (typeof trackSelector === 'string') { + igvBrowser.trackViews.find((trackView, i) => { + trackIndex = i + return trackView.track.config?.name === trackSelector + }) + } else { + trackIndex = trackSelector + } + return trackIndex + } + + /** + * Initialize facets and filters for the current genomic frame + * + * Output: + * { + * "features": [ + * // Each "feature" (e.g. variant, read, gene) is represented as an + * // array of numbers. For categorical facets, each number is a + * // compact representation (e.g. 0) of the filter / value (e.g. + * // "SNP") of a particular facet / dimension (e.g. "Variant type"). + * [] + * ], + * "facets": [ + * { + * "name": , // Name of facet, e.g. "Variant type" + * + * // Names of filters in facet, for categorical facets. The index + * // of these names correspond to the integer value at the + * // appropriate position in the `features` array. + * "filters": [], + * "type": either "Categorical" or "Numeric" + * } + * ] + * } + */ + function initFacets(trackSelector, igvBrowser) { + const trackIndex = getTrackIndex(trackSelector, igvBrowser) + const featuresInFrame = getFeaturesInFrame(trackIndex, igvBrowser) + const dimensions = Object.keys(featuresInFrame[0].info); + const facets = [] + dimensions.forEach(dimension => { + // const facet = {"name": dimension, }facets[dimension] = 0 + }) + featuresInFrame = + console.log('featuresInFrame', featuresInFrame) + } + @@ -113,7 +162,7 @@

Ad-hoc filtering in IGV

    Type -
    • +
From ff8583f608e71f279fa7fbc9dfb23d41b17e8522 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Wed, 8 Jan 2025 15:20:05 -0500 Subject: [PATCH 06/64] Populate features, and (for each facet) filterNames --- examples/dynamic-filtering.html | 66 +++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index 14d24ac87..ee49dd5bc 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -133,25 +133,75 @@ * { * "name": , // Name of facet, e.g. "Variant type" * - * // Names of filters in facet, for categorical facets. The index - * // of these names correspond to the integer value at the + * "type": // "Categorical" or "Numeric" + * + * // Names of filters in facet, only for categorical facets. + * // The index of these names correspond to the integer value at the * // appropriate position in the `features` array. - * "filters": [], - * "type": either "Categorical" or "Numeric" + * "filterNames": [], * } * ] * } */ function initFacets(trackSelector, igvBrowser) { + const t0 = Date.now() const trackIndex = getTrackIndex(trackSelector, igvBrowser) const featuresInFrame = getFeaturesInFrame(trackIndex, igvBrowser) - const dimensions = Object.keys(featuresInFrame[0].info); + + console.log('featuresInFrame', featuresInFrame) + + isCategoricalByFacetName = {} + + // Populate facets model, except for filterNames array values const facets = [] + const basicInfo = featuresInFrame[0].info + const dimensions = Object.keys(basicInfo); dimensions.forEach(dimension => { - // const facet = {"name": dimension, }facets[dimension] = 0 + const type = isNaN(parseFloat(basicInfo[dimension])) ? 'categorical' : 'numeric' + const facet = {'name': dimension, type} + if (type === 'categorical') { + facet.filterNames = [] + isCategoricalByFacetName[facet.name] = true + } else { + isCategoricalByFacetName[facet.name] = false + } + facets.push(facet) }) - featuresInFrame = - console.log('featuresInFrame', featuresInFrame) + + // Populate features, and (for each facet) filterNames + const features = [] + for (let i = 0; i < featuresInFrame.length; i++) { + const feature = [] + const igvFeature = featuresInFrame[i] + const info = igvFeature.info + let j = 0 + for (facetName in info) { + const isCategorical = isCategoricalByFacetName[facetName] + const rawValue = info[facetName] + let value + if (isCategorical) { + if (!facets[j].filterNames.includes(rawValue)) { + // Populate filterNames + facets[j].filterNames.push(rawValue); + } + + // Using index offsets as values, rather than raw strings, helps + // make categorical filtering fast. + value = facets[j].filterNames.indexOf(rawValue) + } else { + value = rawValue + } + feature.push(value) + j++ + } + + // Populate features + features.push(feature) + } + + const t1 = Date.now() + console.log('Duration (ms):', (t1 - t0)) + return {facets, features} } From a8117ccfca59df912398381153a9fa3835c6577f Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Wed, 8 Jan 2025 16:06:59 -0500 Subject: [PATCH 07/64] Add API for base categorical filter counts --- examples/dynamic-filtering.html | 50 ++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index ee49dd5bc..9b0463670 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -133,12 +133,18 @@ * { * "name": , // Name of facet, e.g. "Variant type" * - * "type": // "Categorical" or "Numeric" + * "type": // "categorical" or "numeric" * * // Names of filters in facet, only for categorical facets. * // The index of these names correspond to the integer value at the * // appropriate position in the `features` array. * "filterNames": [], + * + * + * // Number of features satisfying this filter (for categorical) + * "countsByFilterName": { + * : + * } * } * ] * } @@ -148,19 +154,18 @@ const trackIndex = getTrackIndex(trackSelector, igvBrowser) const featuresInFrame = getFeaturesInFrame(trackIndex, igvBrowser) - console.log('featuresInFrame', featuresInFrame) - isCategoricalByFacetName = {} // Populate facets model, except for filterNames array values const facets = [] - const basicInfo = featuresInFrame[0].info - const dimensions = Object.keys(basicInfo); + const headInfo = featuresInFrame[0].header.INFO + const dimensions = Object.keys(headInfo); dimensions.forEach(dimension => { - const type = isNaN(parseFloat(basicInfo[dimension])) ? 'categorical' : 'numeric' + const type = headInfo[dimension].Type === 'String' ? 'categorical' : 'numeric' const facet = {'name': dimension, type} if (type === 'categorical') { facet.filterNames = [] + facet.countsByFilterName = {} isCategoricalByFacetName[facet.name] = true } else { isCategoricalByFacetName[facet.name] = false @@ -175,21 +180,27 @@ const igvFeature = featuresInFrame[i] const info = igvFeature.info let j = 0 - for (facetName in info) { - const isCategorical = isCategoricalByFacetName[facetName] - const rawValue = info[facetName] + for (facetName in headInfo) { let value - if (isCategorical) { - if (!facets[j].filterNames.includes(rawValue)) { - // Populate filterNames - facets[j].filterNames.push(rawValue); - } - - // Using index offsets as values, rather than raw strings, helps - // make categorical filtering fast. - value = facets[j].filterNames.indexOf(rawValue) + const isCategorical = isCategoricalByFacetName[facetName] + if (facetName in info === false) { + value = undefined // Use an "empty" value } else { - value = rawValue + const rawValue = info[facetName] + if (isCategorical) { + if (!facets[j].filterNames.includes(rawValue)) { + // Populate filterNames + facets[j].filterNames.push(rawValue); + facets[j].countsByFilterName[rawValue] = 0 + } + + // Using index offsets as values, rather than raw strings, helps + // make categorical filtering fast. + value = facets[j].filterNames.indexOf(rawValue) + facets[j].countsByFilterName[rawValue] += 1 + } else { + value = rawValue + } } feature.push(value) j++ @@ -204,6 +215,7 @@ return {facets, features} } + From 71f38bd87ddd6caf31815744da066e5984d90053 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Wed, 8 Jan 2025 20:11:17 -0500 Subject: [PATCH 08/64] Refine type handling in dynamic filters --- examples/dynamic-filtering.html | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index 9b0463670..190e5f5f0 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -159,10 +159,15 @@ // Populate facets model, except for filterNames array values const facets = [] const headInfo = featuresInFrame[0].header.INFO + console.log('headInfo', headInfo) const dimensions = Object.keys(headInfo); dimensions.forEach(dimension => { - const type = headInfo[dimension].Type === 'String' ? 'categorical' : 'numeric' - const facet = {'name': dimension, type} + const igvFacet = headInfo[dimension] + // string, flag, integer, or float + const igvType = igvFacet.Type.toLowerCase() + const type = ['string', 'flag'].includes(igvType) ? 'categorical' : igvType + const description = igvFacet.Description + const facet = {'name': dimension, type, description} if (type === 'categorical') { facet.filterNames = [] facet.countsByFilterName = {} @@ -199,7 +204,8 @@ value = facets[j].filterNames.indexOf(rawValue) facets[j].countsByFilterName[rawValue] += 1 } else { - value = rawValue + const isFloat = facets[j].type === 'float' + value = isFloat ? parseFloat(rawValue) : parseInt(rawValue) } } feature.push(value) @@ -211,11 +217,9 @@ } const t1 = Date.now() - console.log('Duration (ms):', (t1 - t0)) + console.log('initFacets duration (ms):', (t1 - t0)) return {facets, features} } - - @@ -251,7 +255,7 @@

Ad-hoc filtering in IGV

name: "Phase 3 WGS variants", type: "variant", format: "vcf", - visibilityWindow: 300000, + visibilityWindow: 3000000, url: "https://s3.amazonaws.com/1000genomes/release/20130502/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5b.20130502.sites.vcf.gz", indexURL: "https://s3.amazonaws.com/1000genomes/release/20130502/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5b.20130502.sites.vcf.gz.tbi" }, @@ -273,11 +277,13 @@

Ad-hoc filtering in IGV

window.igv = igv - igv.createBrowser(igvDiv, options) - .then(function (browser) { - console.log("Created IGV browser") - window.igvBrowser = browser - }) + const browser = await igv.createBrowser(igvDiv, options) + window.igvBrowser = browser + + browser.on('locuschange', referenceFrameList => { + const facets = initFacets('Phase 3 WGS variants', browser) + console.log('facets', facets) + }) From a1d0e8110812353496b4a37aeadfe9c9a4b0f729 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Wed, 8 Jan 2025 21:01:31 -0500 Subject: [PATCH 09/64] Add rudimentary rendering for categorical facets --- examples/dynamic-filtering.html | 67 +++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index 190e5f5f0..8df2f2ef0 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -126,12 +126,14 @@ * // Each "feature" (e.g. variant, read, gene) is represented as an * // array of numbers. For categorical facets, each number is a * // compact representation (e.g. 0) of the filter / value (e.g. - * // "SNP") of a particular facet / dimension (e.g. "Variant type"). + * // "SNP") of a particular facet / dimension (e.g. "VT"). * [] * ], * "facets": [ * { - * "name": , // Name of facet, e.g. "Variant type" + * "name": , // Internal name of facet, e.g. "VT" + * + * "displayName": , // e.g. "Variant type" * * "type": // "categorical" or "numeric" * @@ -140,7 +142,6 @@ * // appropriate position in the `features` array. * "filterNames": [], * - * * // Number of features satisfying this filter (for categorical) * "countsByFilterName": { * : @@ -189,7 +190,9 @@ let value const isCategorical = isCategoricalByFacetName[facetName] if (facetName in info === false) { - value = undefined // Use an "empty" value + // Use an "empty" value if the feature lacks a value + // for this dimension, which is pretty common. + value = undefined } else { const rawValue = info[facetName] if (isCategorical) { @@ -220,18 +223,67 @@ console.log('initFacets duration (ms):', (t1 - t0)) return {facets, features} } + + function getCategoricalFacetHtml(facet) { + const filtersHtml = facet.filterNames.map(filterName => { + const fullName = `${facet.name}:${filterName}` + const attrs = `value="${filterName}" name="${fullName}"` + const count = facet.countsByFilterName[filterName] + const filterHtml = ` + ` + console.log('filterHtml', filterHtml) + return filterHtml + }).join('') + const facetHtml = ` +
+ ${facet.name} + ${filtersHtml} +
` + console.log('facetHtml', facetHtml) + return facetHtml + } + + function getFacetsHtml(trackSelector, browser) { + let facetsHtml = '' + const facetContainers = initFacets(trackSelector, browser) + console.log('facetContainers', facetContainers) + facetContainers.facets.forEach(facetContainer => { + let facetHtml + if (facetContainer.type === 'categorical') { + facetHtml = getCategoricalFacetHtml(facetContainer) + } else { + facetHtml = '' // TODO: Support numeric facets + } + facetsHtml += facetHtml + }) + return facetsHtml + } +

Ad-hoc filtering in IGV

-
    + + + +
    +
    From 8936e99012821c6113d020b42a14320497d9f7e8 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Wed, 8 Jan 2025 22:59:10 -0500 Subject: [PATCH 10/64] Enable partial filter list collapse, toggling; filter on click --- examples/dynamic-filtering.html | 75 +++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index 8df2f2ef0..5da8cf6cc 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -23,7 +23,6 @@ function getOriginalChrFeatures(trackIndex, igvBrowser) { const chr = Array.from(igvBrowser.tracks[0].trackView.viewports[0].featureCache.features.keys())[0] - console.log('igvBrowser.tracks[0]', igvBrowser.tracks[0]) console.log('igvBrowser.tracks[0]', igvBrowser.tracks[0]) console.log('chr', chr) @@ -68,26 +67,28 @@ return isFeatureInFrame } - function filterFeatures(trackSelector, dimension, igvBrowser) { + function filterFeatures(trackSelector, dimension) { const trackIndex = getTrackIndex(trackSelector, igvBrowser); - const scoreSelection = new Set([]) + const selection = new Set([]) const originalChrFeatures = getOriginalChrFeatures(trackIndex, igvBrowser) - const inputs = document.querySelectorAll('.filters input') + const facetSel = `.igv-facet-${dimension}` + const inputs = document.querySelectorAll(`.filters ${facetSel} input`) + console.log('in filterFeatures, inputs', inputs) inputs.forEach(input => { if (input.checked) { - scoreSelection.add(input.value) + selection.add(input.value) } }) - if (scoreSelection.size === 0) { - return scoreSelection.originalChrFeatures + if (selection.size === 0) { + return selection.originalChrFeatures } const filteredFeatures = originalChrFeatures.filter( - feature => scoreSelection.has(feature[dimension]) && getIsFeatureInFrame(feature, igvBrowser) + feature => selection.has(feature.info[dimension]) && getIsFeatureInFrame(feature, igvBrowser) ) console.log('filteredFeatures', filteredFeatures) @@ -224,46 +225,75 @@ return {facets, features} } - function getCategoricalFacetHtml(facet) { - const filtersHtml = facet.filterNames.map(filterName => { + function getCategoricalFacetHtml(facet, isPartlyCollapsed=true) { + // Categorical facets need > 1 filter to be useful + const numFilters = facet.filterNames.length + if (numFilters < 2) return '' + + let filters = facet.filterNames; + if (isPartlyCollapsed) filters = filters.slice(0, 5) + + // Get list of filter checkboxes + const filtersHtml = filters.map(filterName => { const fullName = `${facet.name}:${filterName}` - const attrs = `value="${filterName}" name="${fullName}"` + const attrs = ` + value="${filterName}" + name="${fullName}" + onChange="filterFeatures('Phase 3 WGS variants', '${facet.name}')" + ` const count = facet.countsByFilterName[filterName] const filterHtml = ` ` - console.log('filterHtml', filterHtml) return filterHtml }).join('') + + + let moreOrLess = '' + if (numFilters > 5) { + moreOrLess = ` + + ${isPartlyCollapsed ? 'More...' : 'Less...'} + ` + } + + const facetSel = `igv-facet-${facet.name}` + // Add a facet header to the filters const facetHtml = ` -
    +
    ${facet.name} ${filtersHtml} + ${moreOrLess}
    ` - console.log('facetHtml', facetHtml) + + // console.log('facetHtml', facetHtml) return facetHtml } function getFacetsHtml(trackSelector, browser) { - let facetsHtml = '' const facetContainers = initFacets(trackSelector, browser) - console.log('facetContainers', facetContainers) - facetContainers.facets.forEach(facetContainer => { + // console.log('facetContainers', facetContainers) + const facetList = facetContainers.facets.map(facetContainer => { let facetHtml if (facetContainer.type === 'categorical') { facetHtml = getCategoricalFacetHtml(facetContainer) } else { facetHtml = '' // TODO: Support numeric facets } - facetsHtml += facetHtml - }) + return facetHtml + }).join('') + + const facetsHtml = `
    ${facetList}
    ` return facetsHtml } @@ -280,11 +310,11 @@

    Ad-hoc filtering in IGV

--> -
    +
    -
    +
    From 6a791c6cf3f6d36ba9cc9fbaab81051c3aec8557 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Fri, 10 Jan 2025 15:40:57 -0500 Subject: [PATCH 15/64] Trim empty facets and filter indexes, reducing dimensionality --- examples/dynamic-filtering.html | 55 +++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index 284109645..b0fe923d5 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -217,7 +217,10 @@ /** Get crossfilter-initialized features by facet */ function getFeaturesByFacet(features, facetNames) { + console.log('in getFeaturesByFacet, features', features) const featureCrossfilter = crossfilter(features) + window.featureCrossfilter = featureCrossfilter + console.log('in getFeaturesByFacet, featureCrossfilter', featureCrossfilter) const featuresByFacet = {} for (let i = 0; i < facetNames.length; i++) { const facetName = facetNames[i] @@ -232,10 +235,9 @@ selection, featuresByFacet, initFacets, filterableFeatures, rawFacets ) { const t0 = Date.now() - const facets = - initFacets - .filter(facet => facet.isLoaded) - .map(facet => facet.name) + const facets = initFacets + + console.log('in filterFeaturesNew, featuresByFacet', featuresByFacet) let fn; let facet; let results @@ -357,7 +359,7 @@ // For categorical facets, we have an array of integers, e.g. [6, 0, 7, 0, 0]. // Each element in the array is the index-offset of the feature's category assignment // for the facet at that index. - const facetIndex = features[i] + const facetIndex = rawFeatures[i] feature.facetIndex = facetIndex features.push(feature) } @@ -388,7 +390,7 @@ * { * "name": , // Internal name of facet, e.g. "VT" * - * "displayName": , // e.g. "Variant type" + * "displayName": , // e.g. "Variant type" TODO * * "type": // "categorical" or "integer" or "float" * @@ -444,7 +446,7 @@ }) // Populate features, and (for each facet) filterNames - const features = [] + let features = [] for (let i = 0; i < featuresInFrame.length; i++) { const feature = [] const igvFeature = featuresInFrame[i] @@ -485,6 +487,37 @@ features.push(feature) } + const undefinedIndexes = new Set() + + // Trim unused facets + facets = facets.filter((facet, i) => { + if ( + (facet.type === 'categorical' && facet.filterNames.length < 2) || + (facet.type !== 'categorical' && facet.filterNumbers.length < 2) + ) { + undefinedIndexes.add(i) + return false + } else { + return true + } + }) + + // Trim undefined indexes from features + if (undefinedIndexes.length > 0) { + const trimmedFeatures = [] + for (let i = 0; i < features.length; i++) { + const feature = features[i]; + const trimmedFeature = [] + for (let j = 0; j < features.length; j++) { + if (!undefinedIndexes.has(j)) { + trimmedFeature.push(feature[j]) + } + } + trimmedFeatures.push(trimmedFeature) + } + features = trimmedFeatures + } + // Populate statistics for numeric facets facets = facets.map(facet => { if (facet.type === 'categorical') return facet @@ -497,6 +530,8 @@ const t1 = Date.now() console.log('initFacets duration (ms):', (t1 - t0)) + console.log('in initFacets, facets', facets) + console.log('in initFacets, features', features) return {facets, features} } @@ -570,7 +605,7 @@ return ` Ad-hoc filtering in IGV const filtersDom = document.querySelectorAll('.igv-filter') filtersDom.forEach(filterDom => { + const selection = getSelection() + console.log('filterDom', filterDom) filterDom.addEventListener('change', (event) => { + console.log('in filterDom change') const facetName = filterDom.getAttribute('data-facet-name'); - const selection = getSelection() filterFeaturesNew(selection, featuresByFacet, facets, features, facets) // filterFeatures(trackSelector, facetName) }) From 319d843fdd9344c562ee4a59af0fba2d71095b2e Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Fri, 10 Jan 2025 19:33:35 -0500 Subject: [PATCH 16/64] Robustify null handling in categorical facets --- examples/dynamic-filtering.html | 101 ++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index b0fe923d5..bf3ee9ff5 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -73,7 +73,7 @@ const selection = {} const categoricalFilters = document.querySelectorAll('.igv-filter-categorical') - console.log('categoricalFilters', categoricalFilters) + // console.log('categoricalFilters', categoricalFilters) categoricalFilters.forEach(input => { if (input.checked) { const facetName = input.getAttribute('data-facet-name') @@ -101,7 +101,7 @@ // } }) - console.log('in getSelection, selection', selection) + // console.log('in getSelection, selection', selection) return selection } @@ -232,12 +232,15 @@ /** Get filtered feature results */ function filterFeaturesNew( - selection, featuresByFacet, initFacets, filterableFeatures, rawFacets + selection, featuresByFacet, facets, filterableFeatures ) { const t0 = Date.now() - const facets = initFacets + // facets = structuredClone(facets) + console.log('in filterFeaturesNew, facets', facets) + console.log('in filterFeaturesNew, selection', selection) console.log('in filterFeaturesNew, featuresByFacet', featuresByFacet) + console.log('in filterFeaturesNew, featuresByFacet[facets[0].name].top(Infinity)', featuresByFacet[facets[0].name].top(Infinity)) let fn; let facet; let results @@ -246,46 +249,57 @@ } else { for (let i = 0; i < facets.length; i++) { facet = facets[i] - if (facet in selection) { + const facetName = facet.name + console.log('facet.name', facet.name) + if (facetName in selection) { if (facet.type === 'categorical') { // e.g. 'infant_sick_YN' - const friendlyFilters = selection[facet] // e.g. ['yes', 'NA'] + const friendlyFilters = selection[facetName] // e.g. ['yes', 'NA'] const filter = new Set() friendlyFilters.forEach(friendlyFilter => { - // find the original index of the filter in the source facet as the list here may be trimmed already - const sourceFacet = rawFacets.find(f => f.name === facet) - const filterIndex = sourceFacet.groups.indexOf(friendlyFilter) + const filterIndex = facet.filterNames.indexOf(friendlyFilter) filter.add(filterIndex) }) - fn = function(d) { + console.log('in filterFeaturesNew, friendlyFilters', friendlyFilters) + console.log('in filterFeaturesNew, filter', filter) + + fn = function(d, i) { + console.log('i, d, filter, filter.has(d)', i, d, filter, filter.has(d)) return filter.has(d) + // return true } // Apply the actual crossfilter method - featuresByFacet[facet].filterFunction(fn) + featuresByFacet[facetName].filterFunction(fn) } else { - const numericFilters = selection[facet] // e.g. [0, 20] + const numericFilters = selection[facetName] // e.g. [0, 20] fn = function(d) { - return applyNumericFilters(d, numericFilters) + return true // applyNumericFilters(d, numericFilters) } - featuresByFacet[facet].filterFunction(fn) + featuresByFacet[facetName].filterFunction(fn) } } else { fn = null + console.log('fn = null') // Apply the actual crossfilter method - featuresByFacet[facet].filter(fn) + featuresByFacet[facetName].filter(fn) } } - results = featuresByFacet[facet].top(Infinity) + console.log('before results, featuresByFacet', featuresByFacet) + console.log('before results, facet.name', facet.name) + // results = featuresByFacet[facet.name].top(Infinity) + results = featuresByFacet['VT'].top(Infinity) } - const facetNames = initFacets.map(facet => facet.name) + const facetNames = facets.map(facet => facet.name) const t0Counts = Date.now() - const counts = getFilterCounts(facetNames, featuresByFacet, initFacets, selection) + const counts = {} //getFilterCounts(facetNames, featuresByFacet, initFacets, selection) + + console.log('in filterFeaturesNew, results', results) return [results, counts] } @@ -300,12 +314,12 @@ const rawFilterCounts = facetCrossfilter.group().top(Infinity) let countsByFilter - if (facet.includes('--group--')) { + if (facet.type === 'categorical') { countsByFilter = {} - facets[i].groups?.forEach((group, j) => { + facets[i].filterNames?.forEach((filterName, j) => { let count = null // check for originalGroups array first, if present - const originalGroups = facets[i].originalGroups || facets[i].groups + const originalGroups = facets[i].originalGroups || facets[i].filterNames const groupIdx = originalGroups.indexOf(group) const rawFilterKeyAndValue = rawFilterCounts.find(rfc => rfc.key === groupIdx) if (rawFilterKeyAndValue) { @@ -366,10 +380,10 @@ const featuresByFacet = getFeaturesByFacet(features, facetNames) - const filterCounts = getFilterCounts(facetNames, featuresByFacet, facets, null) + const filterCounts = {} // getFilterCounts(facetNames, featuresByFacet, facets, null) return { - features, featuresByFacet, loadedFacets: facets, + features, featuresByFacet, facets, filterCounts } } @@ -502,15 +516,32 @@ } }) + console.log('undefinedIndexes', undefinedIndexes) // Trim undefined indexes from features - if (undefinedIndexes.length > 0) { + if (undefinedIndexes.size > 0) { const trimmedFeatures = [] for (let i = 0; i < features.length; i++) { const feature = features[i]; const trimmedFeature = [] - for (let j = 0; j < features.length; j++) { + let k = -2 + for (let j = 0; j < feature.length; j++) { if (!undefinedIndexes.has(j)) { - trimmedFeature.push(feature[j]) + let featureValue = feature[j] + if (featureValue === undefined) { + featureValue = null + console.log('k, facets', k, facets) + if (facets[k].type === 'categorical') { + if (!facets[k-1].filterNames.includes(null)) { + facets[k-1].filterNames.push(null) + facets[k-1].countsByFilterName[null] = 1 + } else { + facets[k-1].countsByFilterName[null] += 1 + } + } + } + trimmedFeature.push(featureValue) + } else { + k++ } } trimmedFeatures.push(trimmedFeature) @@ -541,7 +572,7 @@ if (numFilters < 2) return '' let filters = facet.filterNames; - if (isPartlyCollapsed) filters = filters.slice(0, 5) + // if (isPartlyCollapsed) filters = filters.slice(0, 5) // Get list of filter checkboxes const filtersHtml = filters.map(filterName => { @@ -805,18 +836,24 @@

    Ad-hoc filtering in IGV

    filterContainerDom.insertAdjacentHTML('beforeend', facetsHtml) const { - features, featuresByFacet, loadedFacets: facets, + features, featuresByFacet, facets, filterCounts } = initCrossfilter(facetsContainer) + window.facets = facets + + window.features = features + window.featuresByFacet = featuresByFacet + const filtersDom = document.querySelectorAll('.igv-filter') filtersDom.forEach(filterDom => { - const selection = getSelection() - console.log('filterDom', filterDom) + // console.log('filterDom', filterDom) filterDom.addEventListener('change', (event) => { console.log('in filterDom change') - const facetName = filterDom.getAttribute('data-facet-name'); - filterFeaturesNew(selection, featuresByFacet, facets, features, facets) + + const selection = getSelection() + const facetName = filterDom.getAttribute('data-facet-name') + filterFeaturesNew(selection, featuresByFacet, facets, features) // filterFeatures(trackSelector, facetName) }) }) From 4631e722a67032456a56217f671ac537acdadc04 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Fri, 10 Jan 2025 20:40:43 -0500 Subject: [PATCH 17/64] Refine handling of partly-null facets --- examples/dynamic-filtering.html | 46 +++++---------------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index bf3ee9ff5..daa9fe6ae 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -24,9 +24,6 @@ function getOriginalChrFeatures(trackIndex, igvBrowser) { const chr = Array.from(igvBrowser.tracks[0].trackView.viewports[0].featureCache.features.keys())[0] - console.log('igvBrowser.tracks[0]', igvBrowser.tracks[0]) - console.log('chr', chr) - if ( typeof window.originalFeatures === 'undefined' || chr in window.originalFeatures === false @@ -73,7 +70,6 @@ const selection = {} const categoricalFilters = document.querySelectorAll('.igv-filter-categorical') - // console.log('categoricalFilters', categoricalFilters) categoricalFilters.forEach(input => { if (input.checked) { const facetName = input.getAttribute('data-facet-name') @@ -101,7 +97,6 @@ // } }) - // console.log('in getSelection, selection', selection) return selection } @@ -114,7 +109,6 @@ const facetSel = `.igv-facet-${facetName}` const inputs = document.querySelectorAll(`.filters ${facetSel} input`) - console.log('in filterFeatures, inputs', inputs) const facets = {} @@ -133,8 +127,6 @@ } }) - console.log('in filterFeatures, facets', facets) - if (selection.size === 0) { return selection.originalChrFeatures } @@ -148,7 +140,6 @@ feature => selection.has(feature.info[dimension]) && getIsFeatureInFrame(feature, igvBrowser) ) - console.log('filteredFeatures', filteredFeatures) updateTrack(trackIndex, filteredFeatures, igv, igvBrowser) } @@ -217,10 +208,8 @@ /** Get crossfilter-initialized features by facet */ function getFeaturesByFacet(features, facetNames) { - console.log('in getFeaturesByFacet, features', features) const featureCrossfilter = crossfilter(features) - window.featureCrossfilter = featureCrossfilter - console.log('in getFeaturesByFacet, featureCrossfilter', featureCrossfilter) + // window.featureCrossfilter = featureCrossfilter const featuresByFacet = {} for (let i = 0; i < facetNames.length; i++) { const facetName = facetNames[i] @@ -236,12 +225,6 @@ ) { const t0 = Date.now() - // facets = structuredClone(facets) - console.log('in filterFeaturesNew, facets', facets) - console.log('in filterFeaturesNew, selection', selection) - console.log('in filterFeaturesNew, featuresByFacet', featuresByFacet) - console.log('in filterFeaturesNew, featuresByFacet[facets[0].name].top(Infinity)', featuresByFacet[facets[0].name].top(Infinity)) - let fn; let facet; let results if (Object.keys(selection).length === 0) { @@ -250,7 +233,6 @@ for (let i = 0; i < facets.length; i++) { facet = facets[i] const facetName = facet.name - console.log('facet.name', facet.name) if (facetName in selection) { if (facet.type === 'categorical') { // e.g. 'infant_sick_YN' @@ -262,13 +244,8 @@ filter.add(filterIndex) }) - console.log('in filterFeaturesNew, friendlyFilters', friendlyFilters) - console.log('in filterFeaturesNew, filter', filter) - fn = function(d, i) { - console.log('i, d, filter, filter.has(d)', i, d, filter, filter.has(d)) return filter.has(d) - // return true } // Apply the actual crossfilter method @@ -283,22 +260,18 @@ } } else { fn = null - console.log('fn = null') // Apply the actual crossfilter method featuresByFacet[facetName].filter(fn) } } - console.log('before results, featuresByFacet', featuresByFacet) - console.log('before results, facet.name', facet.name) - // results = featuresByFacet[facet.name].top(Infinity) - results = featuresByFacet['VT'].top(Infinity) + + results = featuresByFacet[facet.name].top(Infinity) } const facetNames = facets.map(facet => facet.name) const t0Counts = Date.now() const counts = {} //getFilterCounts(facetNames, featuresByFacet, initFacets, selection) - console.log('in filterFeaturesNew, results', results) return [results, counts] } @@ -527,9 +500,10 @@ for (let j = 0; j < feature.length; j++) { if (!undefinedIndexes.has(j)) { let featureValue = feature[j] + + // Handle facets where _some but not all_ values are not available if (featureValue === undefined) { - featureValue = null - console.log('k, facets', k, facets) + featureValue = k-1 if (facets[k].type === 'categorical') { if (!facets[k-1].filterNames.includes(null)) { facets[k-1].filterNames.push(null) @@ -619,15 +593,11 @@ ${moreOrLess} ` - // console.log('facetHtml', facetHtml) return facetHtml } function updateInputValue(event) { - console.log('updateInputValue, event', event) const target = event.target - console.log('target', target) - console.log('target.value', target.value) const facetName = target.name.split(':')[0] filterFeatures('Phase 3 WGS variants', facetName) } @@ -753,7 +723,6 @@ } function getFacetsHtml(facetsContainer) { - // console.log('facetsContainer', facetsContainer) const facetList = facetsContainer.facets.map(facetContainer => { let facetHtml if (facetContainer.type === 'categorical') { @@ -847,10 +816,7 @@

    Ad-hoc filtering in IGV

    const filtersDom = document.querySelectorAll('.igv-filter') filtersDom.forEach(filterDom => { - // console.log('filterDom', filterDom) filterDom.addEventListener('change', (event) => { - console.log('in filterDom change') - const selection = getSelection() const facetName = filterDom.getAttribute('data-facet-name') filterFeaturesNew(selection, featuresByFacet, facets, features) From 12e462729ea018bb18356bb48a739bfd568c36d6 Mon Sep 17 00:00:00 2001 From: Eric Weitz Date: Fri, 10 Jan 2025 21:37:39 -0500 Subject: [PATCH 18/64] Make dynamic filtering multidimensional --- examples/dynamic-filtering.html | 74 ++++++++++++--------------------- 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/examples/dynamic-filtering.html b/examples/dynamic-filtering.html index daa9fe6ae..dadae1f4e 100644 --- a/examples/dynamic-filtering.html +++ b/examples/dynamic-filtering.html @@ -24,6 +24,9 @@ function getOriginalChrFeatures(trackIndex, igvBrowser) { const chr = Array.from(igvBrowser.tracks[0].trackView.viewports[0].featureCache.features.keys())[0] + console.log('igvBrowser.tracks[0]', igvBrowser.tracks[0]) + console.log('chr', chr) + if ( typeof window.originalFeatures === 'undefined' || chr in window.originalFeatures === false @@ -100,47 +103,18 @@ return selection } - function filterFeatures(trackSelector, facetName) { + function filterFeatures(trackSelector, filteredFeatures, counts, igvBrowser) { const trackIndex = getTrackIndex(trackSelector, igvBrowser); + const featuresInFrame = getFeaturesInFrame(trackSelector, igvBrowser) - const selection = {} - - const originalChrFeatures = getOriginalChrFeatures(trackIndex, igvBrowser) - - const facetSel = `.igv-facet-${facetName}` - const inputs = document.querySelectorAll(`.filters ${facetSel} input`) - - const facets = {} - - inputs.forEach(input => { - if (input.checked) { - const filter = input.value - if (!facets[facetName]?.includes(filter)) { - if (facet in selection) { - selection[facetName].push(filter) - } else { - selection[facetName] = [filter] - } - } - } else if (input.type === 'text') { - - } - }) - - if (selection.size === 0) { - return selection.originalChrFeatures + const featuresToPlot = [] + for (let i = 0; i < filteredFeatures.length; i++) { + const rawIndex = filteredFeatures[i].rawIndex + const featureToPlot = featuresInFrame[rawIndex] + featuresToPlot.push(featureToPlot) } - // Filter features by selection (i.e., selected facets and filters) - const [newFilteredFeatures, newFilterCounts] = filterFeaturesNew( - selection, featuresByFacet, initFacets, originalChrFeatures, rawFacets - ) - - const filteredFeatures = originalChrFeatures.filter( - feature => selection.has(feature.info[dimension]) && getIsFeatureInFrame(feature, igvBrowser) - ) - - updateTrack(trackIndex, filteredFeatures, igv, igvBrowser) + updateTrack(trackIndex, featuresToPlot, igv, igvBrowser) } function getFeaturesInFrame(trackSelector, igvBrowser) { @@ -209,7 +183,7 @@ /** Get crossfilter-initialized features by facet */ function getFeaturesByFacet(features, facetNames) { const featureCrossfilter = crossfilter(features) - // window.featureCrossfilter = featureCrossfilter + window.featureCrossfilter = featureCrossfilter const featuresByFacet = {} for (let i = 0; i < facetNames.length; i++) { const facetName = facetNames[i] @@ -219,8 +193,8 @@ return featuresByFacet } - /** Get filtered feature results */ - function filterFeaturesNew( + /** Get filtered feature results, and counts */ + function getFilteredFeatures( selection, featuresByFacet, facets, filterableFeatures ) { const t0 = Date.now() @@ -264,7 +238,6 @@ featuresByFacet[facetName].filter(fn) } } - results = featuresByFacet[facet.name].top(Infinity) } @@ -272,6 +245,7 @@ const t0Counts = Date.now() const counts = {} //getFilterCounts(facetNames, featuresByFacet, initFacets, selection) + console.log('in filterFeaturesNew, results', results) return [results, counts] } @@ -341,7 +315,7 @@ const features = [] for (let i = 0; i < rawFeatures.length; i++) { - const feature = {} + const feature = { 'rawIndex': i } // For categorical facets, we have an array of integers, e.g. [6, 0, 7, 0, 0]. // Each element in the array is the index-offset of the feature's category assignment @@ -409,6 +383,7 @@ const t0 = Date.now() const trackIndex = getTrackIndex(trackSelector, igvBrowser) const featuresInFrame = getFeaturesInFrame(trackIndex, igvBrowser) + console.log('featuresInFrame[0]', featuresInFrame[0]) // Populate facets model, except for filterNames array values let facets = [] @@ -535,8 +510,6 @@ const t1 = Date.now() console.log('initFacets duration (ms):', (t1 - t0)) - console.log('in initFacets, facets', facets) - console.log('in initFacets, features', features) return {facets, features} } @@ -549,7 +522,10 @@ // if (isPartlyCollapsed) filters = filters.slice(0, 5) // Get list of filter checkboxes - const filtersHtml = filters.map(filterName => { + const filtersHtml = filters.map((filterName, i) => { + let style = '' + if (isPartlyCollapsed && i >= 5) style = 'display: none;' + const fullName = `${facet.name}:${filterName}` const attrs = ` value="${filterName}" @@ -560,7 +536,7 @@ ` const count = facet.countsByFilterName[filterName] const filterHtml = ` -