Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Data Explorer: Compute column profiles in smaller batches for large data frames, avoid timeouts #5148

Merged
merged 4 commits into from
Oct 24, 2024
Merged

Data Explorer: Compute column profiles in smaller batches for large data frames, avoid timeouts #5148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fa1a591
For very large data frames, compute column profiles one by one rather…
wesm Oct 23, 2024
0aff1ca
Send requests one at a time
wesm Oct 23, 2024
5bfdc36
Simpler code
wesm Oct 23, 2024
1fdf953
Compute profiles in batches of 4 as a compromise
wesm Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/vs/workbench/services/languageRuntime/common/languageRuntimeDataExplorerClient.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,9 +402,9 @@ export class DataExplorerClientInstance extends Disposable {
this._asyncTasks.set(callbackId, promise);
await this._backendClient.getColumnProfiles(callbackId, profiles, this._profileFormatOptions);

const timeout = 10000;
const timeout = 60000;

// Don't leave unfulfilled promise indefinitely; reject after 10 seconds pass
// Don't leave unfulfilled promise indefinitely; reject after one minute
// for now just in case
setTimeout(() => {
// If the promise has already been resolved, do nothing.
Expand Down
187 changes: 106 additions & 81 deletions src/vs/workbench/services/positronDataExplorer/common/tableSummaryCache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,109 +319,134 @@ export class TableSummaryCache extends Disposable {
const histogramSupported = this.isHistogramSupported();
const frequencyTableSupported = this.isFrequencyTableSupported();

// Load the column profiles.
const columnProfileResults = await this._dataExplorerClientInstance.getColumnProfiles(
columnIndices.map((column_index): ColumnProfileRequest => {
// Get the column schema.
const columnSchema = this._columnSchemaCache.get(column_index);

// Build the array of column profiles to load. Always load the null count.
const profiles: ColumnProfileSpec[] = [{
profile_type: ColumnProfileType.NullCount
}];

// Determine whether the column is expanded.
const columnExpanded = this._expandedColumns.has(column_index);

// If the column is expanded, load the summary stats.
if (columnExpanded) {
profiles.push({ profile_type: ColumnProfileType.SummaryStats });
}
const columnRequests = columnIndices.map((column_index): ColumnProfileRequest => {
// Get the column schema.
const columnSchema = this._columnSchemaCache.get(column_index);

// Build the array of column profiles to load. Always load the null count.
const profiles: ColumnProfileSpec[] = [{
profile_type: ColumnProfileType.NullCount
}];

// Determine whether the column is expanded.
const columnExpanded = this._expandedColumns.has(column_index);

// If the column is expanded, load the summary stats.
if (columnExpanded) {
profiles.push({ profile_type: ColumnProfileType.SummaryStats });
}

// Determine whether to load the histogram or the frequency table for the column.
switch (columnSchema?.type_display) {
// Number.
case ColumnDisplayType.Number: {
// If histograms are supported, load them.
if (histogramSupported) {
// Load the small histogram.
// Determine whether to load the histogram or the frequency table for the column.
switch (columnSchema?.type_display) {
// Number.
case ColumnDisplayType.Number: {
// If histograms are supported, load them.
if (histogramSupported) {
// Load the small histogram.
profiles.push({
profile_type: ColumnProfileType.SmallHistogram,
params: {
method: ColumnHistogramParamsMethod.FreedmanDiaconis,
num_bins: SMALL_HISTOGRAM_NUM_BINS,
}
});

// If the column is expanded, load the large histogram.
if (columnExpanded) {
profiles.push({
profile_type: ColumnProfileType.SmallHistogram,
profile_type: ColumnProfileType.LargeHistogram,
params: {
method: ColumnHistogramParamsMethod.FreedmanDiaconis,
num_bins: SMALL_HISTOGRAM_NUM_BINS,
num_bins: LARGE_HISTOGRAM_NUM_BINS,
}
});

// If the column is expanded, load the large histogram.
if (columnExpanded) {
profiles.push({
profile_type: ColumnProfileType.LargeHistogram,
params: {
method: ColumnHistogramParamsMethod.FreedmanDiaconis,
num_bins: LARGE_HISTOGRAM_NUM_BINS,
}
});
}
}
break;
}
break;
}

// Boolean.
case ColumnDisplayType.Boolean: {
// If frequency tables are supported, load them.
if (frequencyTableSupported) {
// Load the small frequency table. Note that we do not load the large
// frequency table because there are only two possible values.
profiles.push({
profile_type: ColumnProfileType.SmallFrequencyTable,
params: {
limit: 2
}
});
// Boolean.
case ColumnDisplayType.Boolean: {
// If frequency tables are supported, load them.
if (frequencyTableSupported) {
// Load the small frequency table. Note that we do not load the large
// frequency table because there are only two possible values.
profiles.push({
profile_type: ColumnProfileType.SmallFrequencyTable,
params: {
limit: 2
}
});

}
break;
}
break;
}

// String.
case ColumnDisplayType.String: {
// If frequency tables are supported, load them.
if (frequencyTableSupported) {
// Load the small frequency table.
// String.
case ColumnDisplayType.String: {
// If frequency tables are supported, load them.
if (frequencyTableSupported) {
// Load the small frequency table.
profiles.push({
profile_type: ColumnProfileType.SmallFrequencyTable,
params: {
limit: SMALL_FREQUENCY_TABLE_LIMIT
}
});

// If the column is expanded, load the large frequency table.
if (columnExpanded) {
profiles.push({
profile_type: ColumnProfileType.SmallFrequencyTable,
profile_type: ColumnProfileType.LargeFrequencyTable,
params: {
limit: SMALL_FREQUENCY_TABLE_LIMIT
limit: LARGE_FREQUENCY_TABLE_LIMIT
}
});

// If the column is expanded, load the large frequency table.
if (columnExpanded) {
profiles.push({
profile_type: ColumnProfileType.LargeFrequencyTable,
params: {
limit: LARGE_FREQUENCY_TABLE_LIMIT
}
});
}
}
break;
}
break;
}
}

// Return the column profile request.
return { column_index, profiles };
})
);
// Return the column profile request.
return { column_index, profiles };
});

// Cache the column profiles that were returned.
for (let i = 0; i < columnProfileResults.length; i++) {
this._columnProfileCache.set(columnIndices[i], columnProfileResults[i]);
}
const tableState = await this._dataExplorerClientInstance.getBackendState();

// Fire the onDidUpdate event.
this._onDidUpdateEmitter.fire();
// For more than 10 million rows, we request profiles one by one rather than as a batch for
// better responsiveness
const BATCHING_THRESHOLD = 5_000_000;
if (tableState.table_shape.num_rows > BATCHING_THRESHOLD) {
const BATCH_SIZE = 4;
for (let i = 0; i < columnIndices.length; i += BATCH_SIZE) {
// Get the next batch of up to 4 requests
const batchColumnRequests = columnRequests.slice(i, i + BATCH_SIZE);
const batchColumnIndices = columnIndices.slice(i, i + BATCH_SIZE);

// Send the batch of requests to getColumnProfiles
const results = await this._dataExplorerClientInstance.getColumnProfiles(batchColumnRequests);

// Cache the returned column profiles for each index in the batch
for (let j = 0; j < results.length; j++) {
this._columnProfileCache.set(batchColumnIndices[j], results[j]);
}

// Fire the onDidUpdate event so things update as soon as they are returned
this._onDidUpdateEmitter.fire();
}
} else {
// Load the column profiles as a batch
const columnProfileResults = await this._dataExplorerClientInstance.getColumnProfiles(
columnRequests
);
// Cache the column profiles that were returned.
for (let i = 0; i < columnProfileResults.length; i++) {
this._columnProfileCache.set(columnIndices[i], columnProfileResults[i]);
}
// Fire the onDidUpdate event.
this._onDidUpdateEmitter.fire();
}
}

/**
Expand Down