From e5af55817dd27c180f604e5c9ef6d79c93462af7 Mon Sep 17 00:00:00 2001 From: Ilya Matiach Date: Wed, 23 Aug 2023 15:53:13 -0400 Subject: [PATCH] add e2e UI notebook tests to DBPedia text classification notebook (#2265) --- ...n.yml => CI-e2e-notebooks-text-vision.yml} | 33 ++++++++++-- .../datasetExplorer.spec.ts | 13 +++++ .../errorAnalysis.spec.ts | 11 ++++ .../modelOverview.spec.ts | 10 ++++ .../modelAssessment/IModelAssessmentData.ts | 4 +- ...DBPediaTextClassificationModelDebugging.ts | 54 +++++++++++++++++++ .../datasets/modelAssessmentDatasets.ts | 2 + .../modelOverview/describeModelOverview.ts | 25 ++++++--- ...tasetCohortsViewBasicElementsArePresent.ts | 4 +- ...rtsViewElementsAfterSelectionArePresent.ts | 4 +- .../ensureNewCohortsShowUpInCharts.ts | 6 +-- ...-text-classification-model-debugging.ipynb | 11 ++-- scripts/e2e-widget.js | 35 ++++++++---- 13 files changed, 176 insertions(+), 36 deletions(-) rename .github/workflows/{CI-e2e-notebooks-vision.yml => CI-e2e-notebooks-text-vision.yml} (78%) create mode 100644 apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/datasetExplorer.spec.ts create mode 100644 apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/errorAnalysis.spec.ts create mode 100644 apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/modelOverview.spec.ts create mode 100644 libs/e2e/src/lib/describer/modelAssessment/datasets/DBPediaTextClassificationModelDebugging.ts diff --git a/.github/workflows/CI-e2e-notebooks-vision.yml b/.github/workflows/CI-e2e-notebooks-text-vision.yml similarity index 78% rename from .github/workflows/CI-e2e-notebooks-vision.yml rename to .github/workflows/CI-e2e-notebooks-text-vision.yml index aa6f634bc8..a4ce84c573 100644 --- a/.github/workflows/CI-e2e-notebooks-vision.yml +++ b/.github/workflows/CI-e2e-notebooks-text-vision.yml @@ -1,4 +1,4 @@ -name: CI e2e notebooks vision +name: CI e2e notebooks text vision on: push: @@ -8,9 +8,11 @@ on: paths: - "raiwidgets/**" - "responsibleai_vision/**" + - "responsibleai_text/**" - ".github/workflows/CI-e2e-notebooks-vision.yml" - "libs/e2e/src/lib/describer/modelAssessment/**" - "libs/interpret-vision/**" + - "libs/interpret-text/**" - "notebooks/**" jobs: @@ -43,7 +45,7 @@ jobs: name: raiwidgets-js path: raiwidgets/raiwidgets/widget - ci-e2e-notebook-vision: + ci-e2e-notebook-text-vision: needs: ui-build env: @@ -56,7 +58,7 @@ jobs: operatingSystem: [ubuntu-latest, windows-latest] pythonVersion: [3.7, 3.8, 3.9, "3.10"] flights: [""] - notebookGroup: ["nb_group_1"] + notebookGroup: ["vis_nb_group_1", "text_nb_group_1"] runs-on: ${{ matrix.operatingSystem }} @@ -104,13 +106,28 @@ jobs: pip install -v -e . working-directory: raiwidgets - - name: Install vision dependencies + - if: ${{ matrix.notebookGroup == 'vis_nb_group_1'}} + name: Install vision dependencies shell: bash -l {0} run: | pip install -r requirements-dev.txt pip install . working-directory: responsibleai_vision + - if: ${{ matrix.notebookGroup == 'text_nb_group_1'}} + name: Install text dependencies + shell: bash -l {0} + run: | + pip install -r requirements-dev.txt + pip install . + working-directory: responsibleai_text + + - if: ${{ matrix.notebookGroup == 'text_nb_group_1'}} + name: Setup spacy + shell: bash -l {0} + run: | + python -m spacy download en_core_web_sm + - name: Pip freeze shell: bash -l {0} run: | @@ -125,7 +142,7 @@ jobs: path: raiwidgets/installed-requirements-dev.txt # keep list of notebooks in sync with scripts/e2e-widget.js, create new notebook group if necessary - - if: ${{ matrix.notebookGroup == 'nb_group_1'}} + - if: ${{ matrix.notebookGroup == 'vis_nb_group_1'}} name: Run widget tests shell: bash -l {0} run: | @@ -133,6 +150,12 @@ jobs: yarn e2e-widget -n "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging" -f ${{ matrix.flights }} yarn e2e-widget -n "responsibleaidashboard-fridge-object-detection-model-debugging" -f ${{ matrix.flights }} + - if: ${{ matrix.notebookGroup == 'text_nb_group_1'}} + name: Run widget tests + shell: bash -l {0} + run: | + yarn e2e-widget -n "responsibleaidashboard-DBPedia-text-classification-model-debugging" -f ${{ matrix.flights }} + - name: Upload e2e test screen shot if: always() uses: actions/upload-artifact@v3 diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/datasetExplorer.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/datasetExplorer.spec.ts new file mode 100644 index 0000000000..7d8085f2d5 --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/datasetExplorer.spec.ts @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeDatasetExplorer, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; +const datasetShape = + modelAssessmentDatasets.DBPediaTextClassificationModelDebugging; +describeDatasetExplorer( + datasetShape, + "DBPediaTextClassificationModelDebugging" +); diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/errorAnalysis.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/errorAnalysis.spec.ts new file mode 100644 index 0000000000..ccb08e3a0a --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/errorAnalysis.spec.ts @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeErrorAnalysis, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; + +const datasetShape = + modelAssessmentDatasets.DBPediaTextClassificationModelDebugging; +describeErrorAnalysis(datasetShape, "DBPediaTextClassificationModelDebugging"); diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/modelOverview.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/modelOverview.spec.ts new file mode 100644 index 0000000000..3324312858 --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxDBPediaTextClassificationModelDebugging/modelOverview.spec.ts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeModelOverview, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; +const datasetShape = + modelAssessmentDatasets.DBPediaTextClassificationModelDebugging; +describeModelOverview(datasetShape, "DBPediaTextClassificationModelDebugging"); diff --git a/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts b/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts index e21c493e6e..1d44a07143 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts @@ -21,6 +21,7 @@ export interface IModelAssessmentData { isObjectDetection?: boolean; isMultiLabel?: boolean; isImageClassification?: boolean; + isTextClassification?: boolean; } export interface IErrorAnalysisData { @@ -204,5 +205,6 @@ export enum RAINotebookNames { "OrangeJuiceForecastingDataBalanceExperience" = "responsibleaidashboard-orange-juice-forecasting.py", "FridgeImageClassificationModelDebugging" = "responsibleaidashboard-fridge-image-classification-model-debugging.py", "FridgeMultilabelModelDebugging" = "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.py", - "FridgeObjectDetectionModelDebugging" = "responsibleaidashboard-fridge-object-detection-model-debugging.py" + "FridgeObjectDetectionModelDebugging" = "responsibleaidashboard-fridge-object-detection-model-debugging.py", + "DBPediaTextClassificationModelDebugging" = "responsibleaidashboard-DBPedia-text-classification-model-debugging.py" } diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/DBPediaTextClassificationModelDebugging.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/DBPediaTextClassificationModelDebugging.ts new file mode 100644 index 0000000000..f1116a7a7c --- /dev/null +++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/DBPediaTextClassificationModelDebugging.ts @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const DBPediaTextClassificationModelDebugging = { + causalAnalysisData: { + hasCausalAnalysisComponent: false + }, + checkDupCohort: true, + cohortDefaultName: "All data", + dataBalanceData: { + aggregateBalanceMeasuresComputed: false, + distributionBalanceMeasuresComputed: false, + featureBalanceMeasuresComputed: false + }, + errorAnalysisData: { + hasErrorAnalysisComponent: true + }, + featureImportanceData: { + hasFeatureImportanceComponent: false + }, + featureNames: ["text"], + isTextClassification: true, + modelOverviewData: { + featureCohortView: { + firstFeatureToSelect: "positive_words", + multiFeatureCohorts: 7, + secondFeatureToSelect: "negative_words", + singleFeatureCohorts: 3 + }, + hasModelOverviewComponent: true, + initialCohorts: [ + { + metrics: { + accuracy: "0.6", + macroF1: "0.649", + macroPrecision: "0.625", + macroRecall: "0.675" + }, + name: "All data", + sampleSize: "134" + } + ], + newCohort: { + metrics: { + accuracy: "0.9", + macroF1: "0.9", + macroPrecision: "0.9", + macroRecall: "0.9" + }, + name: "CohortCreateE2E-text-classification", + sampleSize: "5" + } + } +}; diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts index b0171a7f51..54d67c4256 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts @@ -6,6 +6,7 @@ import _ from "lodash"; import { IModelAssessmentData } from "../IModelAssessmentData"; import { CensusClassificationModelDebugging } from "./CensusClassificationModelDebugging"; +import { DBPediaTextClassificationModelDebugging } from "./DBPediaTextClassificationModelDebugging"; import { DiabetesDecisionMaking } from "./DiabetesDecisionMaking"; import { DiabetesRegressionModelDebugging } from "./DiabetesRegressionModelDebugging"; import { FridgeImageClassificationModelDebugging } from "./FridgeImageClassificationModelDebugging"; @@ -21,6 +22,7 @@ export const regExForNumbersWithBrackets = /^\((\d+)\)$/; // Ex: (60) const modelAssessmentDatasets: { [name: string]: IModelAssessmentData } = { CensusClassificationModelDebugging, + DBPediaTextClassificationModelDebugging, DiabetesDecisionMaking, DiabetesRegressionModelDebugging, FridgeImageClassificationModelDebugging, diff --git a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/describeModelOverview.ts b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/describeModelOverview.ts index ec171853be..94b7272664 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/describeModelOverview.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/describeModelOverview.ts @@ -16,6 +16,15 @@ import { ensureNewCohortsShowUpInCharts } from "./ensureNewCohortsShowUpInCharts const testName = "Model Overview v2"; +function getDashboardName(isVision: boolean, isText: boolean): string { + if (isVision) { + return "modelAssessmentVision"; + } else if (isText) { + return "modelAssessmentText"; + } + return "modelAssessment"; +} + export function describeModelOverview( datasetShape: IModelAssessmentData, name?: keyof typeof modelAssessmentDatasetsIncludingFlights, @@ -28,15 +37,15 @@ export function describeModelOverview( datasetShape.isImageClassification ? true : false; + const isText = datasetShape.isTextClassification ? true : false; + const isTabular = !isVision && !isText; if (isNotebookTest) { before(() => { visit(name); }); } else { before(() => { - const dashboardName = isVision - ? "modelAssessmentVision" - : "modelAssessment"; + const dashboardName = getDashboardName(isVision, isText); cy.visit(`#/${dashboardName}/${name}/light/english/Version-2`); }); } @@ -48,7 +57,7 @@ export function describeModelOverview( datasetShape, false, isNotebookTest, - isVision + isTabular ); }); @@ -68,7 +77,7 @@ export function describeModelOverview( ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent( datasetShape, 1, - isVision + isTabular ); }); @@ -81,16 +90,16 @@ export function describeModelOverview( ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent( datasetShape, 2, - isVision + isTabular ); }); it("should show new cohorts in charts", () => { - ensureNewCohortsShowUpInCharts(datasetShape, isNotebookTest, isVision); + ensureNewCohortsShowUpInCharts(datasetShape, isNotebookTest, isTabular); }); it("should pivot between charts when clicking", () => { - if (!isVision) { + if (isTabular) { ensureChartsPivot(datasetShape, isNotebookTest, true); } }); diff --git a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent.ts b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent.ts index 6f00727607..9f9881efe4 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent.ts @@ -15,7 +15,7 @@ export function ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent( datasetShape: IModelAssessmentData, includeNewCohort: boolean, isNotebookTest: boolean, - isVision: boolean + isTabular: boolean ): void { const data = datasetShape.modelOverviewData; const initialCohorts = data?.initialCohorts; @@ -104,7 +104,7 @@ export function ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent( "not.exist" ); - if (!isVision) { + if (isTabular) { if (isNotebookTest) { cy.get(Locators.ModelOverviewHeatmapCells) .should( diff --git a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts index 5f03d3d399..de2d1e8d99 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts @@ -13,7 +13,7 @@ import { export function ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent( datasetShape: IModelAssessmentData, selectedFeatures: number, - isVision: boolean + isTabular: boolean ): void { cy.get(Locators.ModelOverviewFeatureSelection).should("exist"); cy.get(Locators.ModelOverviewFeatureConfigurationActionButton).should( @@ -21,7 +21,7 @@ export function ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionAr ); cy.get(Locators.ModelOverviewDatasetCohortStatsTable).should("not.exist"); - if (!isVision) { + if (isTabular) { cy.get(Locators.ModelOverviewHeatmapVisualDisplayToggle).should("exist"); // TODO: check! cy.get(Locators.ModelOverviewDisaggregatedAnalysisTable).should("exist"); diff --git a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureNewCohortsShowUpInCharts.ts b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureNewCohortsShowUpInCharts.ts index fe3db8f1dd..557523ea08 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureNewCohortsShowUpInCharts.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureNewCohortsShowUpInCharts.ts @@ -10,20 +10,20 @@ import { ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent } from export function ensureNewCohortsShowUpInCharts( datasetShape: IModelAssessmentData, isNotebookTest: boolean, - isVision: boolean + isTabular: boolean ): void { cy.get(Locators.ModelOverviewCohortViewDatasetCohortViewButton).click(); ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent( datasetShape, false, isNotebookTest, - isVision + isTabular ); createCohort(datasetShape.modelOverviewData?.newCohort?.name); ensureAllModelOverviewDatasetCohortsViewBasicElementsArePresent( datasetShape, true, isNotebookTest, - isVision + isTabular ); } diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-DBPedia-text-classification-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-DBPedia-text-classification-model-debugging.ipynb index 2d067a5f0b..1c6828e137 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-DBPedia-text-classification-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-DBPedia-text-classification-model-debugging.ipynb @@ -85,7 +85,7 @@ "id": "b0804d4d", "metadata": {}, "source": [ - "Next we load the DBPedia dataset from huggingface datasets" + "Next we load the DBPedia dataset from huggingface datasets. Note we use only 6 examples and 8 additional error instances here since it can take some time to compute explanations, especially on CPU. You can increase the NUM_TEST_SAMPLES to 100 or more to get a more interesting dashboard." ] }, { @@ -95,13 +95,14 @@ "metadata": {}, "outputs": [], "source": [ - "NUM_TEST_SAMPLES = 100\n", + "# Bump up the number of examples to 100 or greater to view more\n", + "# information, but it may take longer to compute\n", + "NUM_TEST_SAMPLES = 6\n", "\n", "def load_dataset(split):\n", " dataset = datasets.load_dataset(\"DeveloperOats/DBPedia_Classes\", split=split)\n", " return pd.DataFrame({\"text\": dataset[\"text\"], \"l1\": dataset[\"l1\"]})\n", "\n", - "pd_data = load_dataset(\"train\")\n", "pd_valid_data = load_dataset(\"test\")\n", "\n", "def rename_label_column(dataset):\n", @@ -109,11 +110,9 @@ " dataset = dataset.drop(columns=\"l1\")\n", " return dataset\n", "\n", - "pd_data = rename_label_column(pd_data)\n", "pd_valid_data = rename_label_column(pd_valid_data)\n", "\n", "START_INDEX = 0\n", - "train_data = pd_data[NUM_TEST_SAMPLES:]\n", "test_data = pd_valid_data[:NUM_TEST_SAMPLES]" ] }, @@ -353,7 +352,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.8.17" } }, "nbformat": 4, diff --git a/scripts/e2e-widget.js b/scripts/e2e-widget.js index e0f72f5996..ccaee71cff 100644 --- a/scripts/e2e-widget.js +++ b/scripts/e2e-widget.js @@ -9,6 +9,7 @@ const nxPath = path.join(__dirname, "../node_modules/@nrwl/cli/bin/nx.js"); const baseDir = path.join(__dirname, "../notebooks/responsibleaidashboard"); const tabularDir = path.join(baseDir, "tabular"); const visionDir = path.join(baseDir, "vision"); +const textDir = path.join(baseDir, "text"); const filePrefix = "responsibleaidashboard-"; // Please add notebook name into the appropriate 'fileNames' array only when you are adding e2e tests to that notebook. // Keep this list in sync with .github/workflows/CI-e2e-notebooks.yml and/or .github/workflows/CI-e2e-notebooks-vision.yml @@ -26,7 +27,16 @@ const visionFileNames = [ "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging", "responsibleaidashboard-fridge-object-detection-model-debugging" ]; -const fileNames = tabularFileNames.concat(visionFileNames); +const textFileNames = [ + "responsibleaidashboard-DBPedia-text-classification-model-debugging" +]; +const ignoredFiles = [ + "responsibleaidashboard-covid19-event-multilabel-text-classification-model-debugging", + "responsibleaidashboard-blbooksgenre-binary-text-classification-model-debugging" +]; +const fileNames = tabularFileNames + .concat(visionFileNames) + .concat(textFileNames); const notebookHostReg = /^ResponsibleAI started at (http:\/\/localhost:\d+)$/m; const serveHostReg = /Web Development Server is listening at\s+(.*)$/m; const timeout = 3600; @@ -42,6 +52,8 @@ function getDirForNotebook(notebook) { } if (visionFileNames.includes(notebook)) { return visionDir; + } else if (textFileNames.includes(notebook)) { + return textDir; } else if (tabularFileNames.includes(notebook)) { return tabularDir; } else { @@ -49,6 +61,13 @@ function getDirForNotebook(notebook) { } } +function getFilesFromNotebookDirs() { + return fs + .readdirSync(tabularDir) + .concat(fs.readdirSync(visionDir)) + .concat(fs.readdirSync(textDir)); +} + /** * * @param {string} host @@ -155,11 +174,10 @@ function addFlightsInFile(path, flights) { function checkIfAllNotebooksHaveTests() { console.log(`Checking if all notebooks under ${baseDir} have tests`); - const files = fs - .readdirSync(tabularDir) - .concat(fs.readdirSync(visionDir)) + const files = getFilesFromNotebookDirs() .filter((f) => f.startsWith(filePrefix) && f.endsWith(".ipynb")) - .map((f) => f.replace(".ipynb", "")); + .map((f) => f.replace(".ipynb", "")) + .filter((f) => !ignoredFiles.includes(f)); const allNotebooksHaveTests = _.isEqual(_.sortBy(files), _.sortBy(fileNames)); if (!allNotebooksHaveTests) { throw new Error( @@ -211,10 +229,9 @@ function convertNotebooks(notebook, flights) { * @returns {Host[]} */ async function runNotebooks(selectedNotebook, host) { - let files = fs - .readdirSync(tabularDir) - .concat(fs.readdirSync(visionDir)) - .filter((f) => f.startsWith(filePrefix) && f.endsWith(".py")); + let files = getFilesFromNotebookDirs().filter( + (f) => f.startsWith(filePrefix) && f.endsWith(".py") + ); console.log("Available notebooks:"); files.forEach((file) => { console.log(` ${file}`);