Merge pull request #1401 from ral-facilities/feature/free-text-search…

…-#1152 Lucene search UI changes
ral-facilities · Jul 24, 2024 · 4b8a1c3 · 4b8a1c3
2 parents 1e141a4 + bdb9c6c
commit 4b8a1c3
Show file tree

Hide file tree

Showing 180 changed files with 14,761 additions and 6,333 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -15,3 +15,6 @@ CODEOWNERS
 LICENSE.md
 README.md
 Dockerfile
+**/*.test.*
+**/setupTests*
+**/testData*
diff --git a/.github/add_icat_rules.py b/.github/add_icat_rules.py
@@ -0,0 +1,50 @@
+from icat.client import Client
+
+client = Client(
+    "https://localhost:8181",
+    checkCert=False,
+)
+client.login("simple", {"username": "root", "password": "pw"})
+
+public_tables = [
+    "Instrument",
+    "ParameterType",
+    "InvestigationType",
+    "DatasetType",
+    "SampleType",
+    "DatafileFormat",
+    "Facility",
+    "FacilityCycle",
+]
+
+client.createRules("R", public_tables)
+
+public_steps = [
+    ("Datafile", "dataset"),
+    ("Dataset", "investigation"),
+    ("Sample", "investigation"),
+    ("Instrument", "instrumentScientists"),
+    ("Investigation", "investigationFacilityCycles"),
+    ("InstrumentScientist", "user"),
+    ("Investigation", "publications"),
+    ("Sample", "type"),
+    ("InvestigationUser", "user"),
+    ("Investigation", "investigationUsers"),
+    ("Investigation", "investigationInstruments"),
+    ("Dataset", "sample"),
+    ("Dataset", "datafiles"),
+    ("Investigation", "datasets"),
+    ("Investigation", "samples"),
+    ("Sample", "parameters"),
+    ("Investigation", "parameters"),
+    ("Dataset", "parameters"),
+    ("Datafile", "parameters"),
+]
+
+public_step_objects = []
+
+for step in public_steps:
+    ps = client.new("PublicStep", origin=step[0], field=step[1])
+    public_step_objects.append(ps)
+
+client.createMany(public_step_objects)
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
@@ -143,7 +143,7 @@ jobs:
         with:
           repository: ral-facilities/datagateway-api
           path: datagateway-api
-          ref: v9.0.1
+          ref: v9.0.2
 
       # DataGateway API file setup
       - name: Create search_api_mapping.json
@@ -301,7 +301,7 @@ jobs:
         with:
           repository: ral-facilities/datagateway-api
           path: datagateway-api
-          ref: v9.0.1
+          ref: v9.0.2
 
       # DataGateway API file setup
       - name: Create search_api_mapping.json
@@ -396,6 +396,12 @@ jobs:
       - name: Replace default payara user with Actions user
         run: |
           sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
+      - name: Change icat.server version 
+        run: |
+          echo "icat_server_version: '6.1.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml
+      - name: Change icat.lucene version 
+        run: |
+          echo "icat_lucene_version: '3.0.0-SNAPSHOT'" >> icat-ansible/group_vars/all/vars.yml
       - name: Amending roles
         run: |
           sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml
@@ -430,7 +436,7 @@ jobs:
         with:
           repository: ral-facilities/datagateway-api
           path: datagateway-api
-          ref: v9.0.1
+          ref: v9.0.2
 
       # DataGateway API file setup
       - name: Create search_api_mapping.json
@@ -449,24 +455,14 @@ jobs:
       - name: Add dummy data to icatdb
         run: |
           cd datagateway-api/; poetry run python -m util.icat_db_generator
+      - name: Add ICAT rules & public steps
+        run: cd datagateway-api/; poetry run python ../.github/add_icat_rules.py
 
       # Recreate Lucene indexes
-      - name: Login to ICAT
-        run: |
-          curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output
-      - name: Recreate Datafile index
-        run: |
-          curl -k --request POST 'https://localhost:8181/icat/lucene/db/Datafile/0' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output`
-      - name: Recreate Dataset index
-        run: |
-          curl -k --request POST 'https://localhost:8181/icat/lucene/db/Dataset/1' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output`
-      - name: Recreate Investigation index
-        run: |
-          curl -k --request POST 'https://localhost:8181/icat/lucene/db/Investigation/1' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output`
+      - name: Use icatadmin to reindex all indexes
+        run: ~/bin/icatadmin https://localhost:8181 simple username root password pw -- populate --delete
       - name: Let reindexes complete
         run: sleep 30
-      - name: Remove session ID data
-        run: rm -f login_output
 
       - name: Start API
         run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

diff --git a/package.json b/package.json
@@ -15,7 +15,7 @@
   },
   "scripts": {
     "build": "yarn workspaces foreach --interlaced --verbose --parallel --jobs 3 --exclude datagateway-common run build",
-    "lint": "yarn workspaces foreach --interlaced --verbose --parallel --jobs 3 run lint:js --max-warnings=0",
+    "lint": "yarn workspaces foreach --interlaced --verbose --parallel --jobs 4 run lint:js",
     "tsc": "yarn workspace datagateway-common tsc",
     "test": "yarn test:unit && yarn test:e2e",
     "test:unit": "yarn workspaces foreach --interlaced --verbose --parallel --jobs 4 --exclude datagateway run test --runInBand",

diff --git a/packages/datagateway-common/src/api/datafiles.test.tsx b/packages/datagateway-common/src/api/datafiles.test.tsx
@@ -301,52 +301,6 @@ describe('datafile api functions', () => {
       expect(result.current.data).toEqual(mockData.length);
     });
 
-    it('sends axios request to fetch datafile count and returns successful response using the stored filters', async () => {
-      (axios.get as jest.Mock).mockResolvedValue({
-        data: mockData.length,
-      });
-
-      const { result, waitFor } = renderHook(
-        () =>
-          useDatafileCount(
-            [
-              {
-                filterType: 'distinct',
-                filterValue: JSON.stringify(['name', 'title']),
-              },
-            ],
-            {
-              name: { value: 'test2', type: 'include' },
-            },
-            'investigation'
-          ),
-        {
-          wrapper: createReactQueryWrapper(history),
-        }
-      );
-
-      await waitFor(() => result.current.isSuccess);
-
-      params.append(
-        'where',
-        JSON.stringify({
-          name: { ilike: 'test2' },
-        })
-      );
-      params.append('distinct', JSON.stringify(['name', 'title']));
-
-      expect(axios.get).toHaveBeenCalledWith(
-        'https://example.com/api/datafiles/count',
-        expect.objectContaining({
-          params,
-        })
-      );
-      expect((axios.get as jest.Mock).mock.calls[0][1].params.toString()).toBe(
-        params.toString()
-      );
-      expect(result.current.data).toEqual(mockData.length);
-    });
-
     it('sends axios request to fetch datafile count and calls handleICATError on failure', async () => {
       (axios.get as jest.Mock).mockRejectedValue({
         message: 'Test error',

diff --git a/packages/datagateway-common/src/api/datafiles.tsx b/packages/datagateway-common/src/api/datafiles.tsx
@@ -161,17 +161,12 @@ export const fetchDatafileCountQuery = (
 };
 
 export const useDatafileCount = (
-  additionalFilters?: AdditionalFilters,
-  storedFilters?: FiltersType,
-  currentTab?: string
+  additionalFilters?: AdditionalFilters
 ): UseQueryResult<number, AxiosError> => {
   const apiUrl = useSelector((state: StateType) => state.dgcommon.urls.apiUrl);
   const location = useLocation();
 
-  const filters =
-    currentTab === 'datafile' || !storedFilters
-      ? parseSearchToQuery(location.search).filters
-      : storedFilters;
+  const filters = parseSearchToQuery(location.search).filters;
 
   return useQuery<
     number,

diff --git a/packages/datagateway-common/src/api/datasets.test.tsx b/packages/datagateway-common/src/api/datasets.test.tsx
@@ -366,52 +366,6 @@ describe('dataset api functions', () => {
       expect(result.current.data).toEqual(mockData.length);
     });
 
-    it('sends axios request to fetch dataset count and returns successful response using the stored filters', async () => {
-      (axios.get as jest.Mock).mockResolvedValue({
-        data: mockData.length,
-      });
-
-      const { result, waitFor } = renderHook(
-        () =>
-          useDatasetCount(
-            [
-              {
-                filterType: 'distinct',
-                filterValue: JSON.stringify(['name', 'title']),
-              },
-            ],
-            {
-              name: { value: 'test2', type: 'include' },
-            },
-            'datafile'
-          ),
-        {
-          wrapper: createReactQueryWrapper(history),
-        }
-      );
-
-      await waitFor(() => result.current.isSuccess);
-
-      params.append(
-        'where',
-        JSON.stringify({
-          name: { ilike: 'test2' },
-        })
-      );
-      params.append('distinct', JSON.stringify(['name', 'title']));
-
-      expect(axios.get).toHaveBeenCalledWith(
-        'https://example.com/api/datasets/count',
-        expect.objectContaining({
-          params,
-        })
-      );
-      expect((axios.get as jest.Mock).mock.calls[0][1].params.toString()).toBe(
-        params.toString()
-      );
-      expect(result.current.data).toEqual(mockData.length);
-    });
-
     it('sends axios request to fetch dataset count and calls handleICATError on failure', async () => {
       (axios.get as jest.Mock).mockRejectedValue({
         message: 'Test error',

diff --git a/packages/datagateway-common/src/api/datasets.tsx b/packages/datagateway-common/src/api/datasets.tsx
@@ -68,7 +68,7 @@ export const useDataset = (
     [string, number, AdditionalFilters?]
   >(
     ['dataset', datasetId, additionalFilters],
-    (params) => {
+    (_) => {
       return fetchDatasets(apiUrl, { sort: {}, filters: {} }, [
         {
           filterType: 'where',
@@ -194,16 +194,11 @@ export const fetchDatasetCountQuery = (
 };
 
 export const useDatasetCount = (
-  additionalFilters?: AdditionalFilters,
-  storedFilters?: FiltersType,
-  currentTab?: string
+  additionalFilters?: AdditionalFilters
 ): UseQueryResult<number, AxiosError> => {
   const apiUrl = useSelector((state: StateType) => state.dgcommon.urls.apiUrl);
   const location = useLocation();
-  const filters =
-    currentTab === 'dataset' || !storedFilters
-      ? parseSearchToQuery(location.search).filters
-      : storedFilters;
+  const filters = parseSearchToQuery(location.search).filters;
 
   return useQuery<
     number,