diff --git a/arches/app/models/migrations/11477_permitted_graphs_policy.py b/arches/app/models/migrations/11477_permitted_graphs_policy.py new file mode 100644 index 00000000000..1a4ced23f7e --- /dev/null +++ b/arches/app/models/migrations/11477_permitted_graphs_policy.py @@ -0,0 +1,38 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("models", "11042_update__arches_staging_to_tile"), + ] + sql = """ + INSERT INTO search_component + VALUES ( + '56b3feba-ffd4-445e-bd82-377fca29ff92', + 'Permited Graphs Policy', + '', + 'permitted_graphs_policy.py', + 'PermittedGraphsPolicy', + 'search-policy', + NULL, + 'permitted-graphs-policy', + '{}' + ); + UPDATE search_component + SET config = '{"default":true,"linkedSearchFilters":[{"componentname":"map-filter","searchcomponentid":"09d97fc6-8c83-4319-9cef-3aaa08c3fbec","layoutSortorder":1},{"componentname":"advanced-search","searchcomponentid":"f0e56205-acb5-475b-9c98-f5e44f1dbd2c","layoutSortorder":2},{"componentname":"related-resources-filter","searchcomponentid":"59f28272-d1f1-4805-af51-227771739aed","layoutSortorder":3},{"componentname":"provisional-filter","searchcomponentid":"073406ed-93e5-4b5b-9418-b61c26b3640f","layoutSortorder":4},{"componentname":"resource-type-filter","searchcomponentid":"f1c46b7d-0132-421b-b1f3-95d67f9b3980","layoutSortorder":5},{"componentname":"saved-searches","searchcomponentid":"6dc29637-43a1-4fba-adae-8d9956dcd3b9","layoutSortorder":6},{"componentname":"search-export","searchcomponentid":"9c6a5a9c-a7ec-48d2-8a25-501b55b8eff6","layoutSortorder":7},{"componentname":"search-result-details","searchcomponentid":"f5986dae-8b01-11ea-b65a-77903936669c","layoutSortorder":8},{"componentname":"sort-results","searchcomponentid":"6a2fe122-de54-4e44-8e93-b6a0cda7955c","layoutSortorder":9},{"componentname":"term-filter","searchcomponentid":"1f42f501-ed70-48c5-bae1-6ff7d0d187da","layoutSortorder":10},{"componentname":"time-filter","searchcomponentid":"7497ed4f-2085-40da-bee5-52076a48bcb1","layoutSortorder":11},{"componentname":"paging-filter","searchcomponentid":"7aff5819-651c-4390-9b9a-a61221ba52c6","required":true,"layoutSortorder":12,"executionSortorder":2},{"componentname":"search-results","searchcomponentid":"00673743-8c1c-4cc0-bd85-c073a52e03ec","required":true,"layoutSortorder":13,"executionSortorder":1},{"componentname":"permitted-graphs-policy","searchcomponentid":"56b3feba-ffd4-445e-bd82-377fca29ff92","required":true,"layoutSortorder":14,"executionSortorder":3}]}' + WHERE searchcomponentid = '69695d63-6f03-4536-8da9-841b07116381'; + """ + reverse_sql = """ + UPDATE search_component + SET config = '{"default":true,"linkedSearchFilters":[{"componentname":"map-filter","searchcomponentid":"09d97fc6-8c83-4319-9cef-3aaa08c3fbec","layoutSortorder":1},{"componentname":"advanced-search","searchcomponentid":"f0e56205-acb5-475b-9c98-f5e44f1dbd2c","layoutSortorder":2},{"componentname":"related-resources-filter","searchcomponentid":"59f28272-d1f1-4805-af51-227771739aed","layoutSortorder":3},{"componentname":"provisional-filter","searchcomponentid":"073406ed-93e5-4b5b-9418-b61c26b3640f","layoutSortorder":4},{"componentname":"resource-type-filter","searchcomponentid":"f1c46b7d-0132-421b-b1f3-95d67f9b3980","layoutSortorder":5},{"componentname":"saved-searches","searchcomponentid":"6dc29637-43a1-4fba-adae-8d9956dcd3b9","layoutSortorder":6},{"componentname":"search-export","searchcomponentid":"9c6a5a9c-a7ec-48d2-8a25-501b55b8eff6","layoutSortorder":7},{"componentname":"search-result-details","searchcomponentid":"f5986dae-8b01-11ea-b65a-77903936669c","layoutSortorder":8},{"componentname":"sort-results","searchcomponentid":"6a2fe122-de54-4e44-8e93-b6a0cda7955c","layoutSortorder":9},{"componentname":"term-filter","searchcomponentid":"1f42f501-ed70-48c5-bae1-6ff7d0d187da","layoutSortorder":10},{"componentname":"time-filter","searchcomponentid":"7497ed4f-2085-40da-bee5-52076a48bcb1","layoutSortorder":11},{"componentname":"paging-filter","searchcomponentid":"7aff5819-651c-4390-9b9a-a61221ba52c6","required":true,"layoutSortorder":12,"executionSortorder":2},{"componentname":"search-results","searchcomponentid":"00673743-8c1c-4cc0-bd85-c073a52e03ec","required":true,"layoutSortorder":13,"executionSortorder":1}]}' + WHERE searchcomponentid = '69695d63-6f03-4536-8da9-841b07116381'; + DELETE FROM search_component WHERE searchcomponentid = '56b3feba-ffd4-445e-bd82-377fca29ff92'; + """ + + operations = [ + migrations.RunSQL( + sql=sql, + reverse_sql=reverse_sql, + ), + ] diff --git a/arches/app/search/components/advanced_search.py b/arches/app/search/components/advanced_search.py index bf8b256205f..24b49e50596 100644 --- a/arches/app/search/components/advanced_search.py +++ b/arches/app/search/components/advanced_search.py @@ -25,9 +25,10 @@ class AdvancedSearch(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): - querystring_params = kwargs.get("querystring", "") - advanced_filters = JSONDeserializer().deserialize(querystring_params) + + def generate_dsl(self, search_query_object, **kwargs): + querysting_params = kwargs.get("querystring", "[]") + advanced_filters = JSONDeserializer().deserialize(querysting_params) datatype_factory = DataTypeFactory() search_query = Bool() advanced_query = Bool() @@ -70,7 +71,11 @@ def append_dsl(self, search_query_object, **kwargs): for grouped_query in grouped_queries: advanced_query.should(grouped_query) search_query.must(advanced_query) - search_query_object["query"].add_query(search_query) + return search_query + + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) def view_data(self): ret = {} diff --git a/arches/app/search/components/base.py b/arches/app/search/components/base.py index b0b6e06cf46..04bcbbe35f7 100644 --- a/arches/app/search/components/base.py +++ b/arches/app/search/components/base.py @@ -14,9 +14,19 @@ def __init__(self, request=None, user=None, componentname=None): self.user = user self.componentname = componentname + def generate_dsl(self, search_query_object, **kwargs): + """ + used to generate ES query dsl + + """ + + pass + def append_dsl(self, search_query_object, **kwargs): """ - used to append ES query dsl to the search request + used to append ES query dsl to the search request, + also used to modify the search query object + if custom index hooks are present """ diff --git a/arches/app/search/components/lifecycle_state_filter.py b/arches/app/search/components/lifecycle_state_filter.py index a4ba370bd57..1d48ddeb7d5 100644 --- a/arches/app/search/components/lifecycle_state_filter.py +++ b/arches/app/search/components/lifecycle_state_filter.py @@ -18,13 +18,11 @@ class LifecycleStateFilter(BaseSearchFilter): - def append_dsl( - self, search_results_object, permitted_nodegroups, include_provisional - ): + def generate_dsl(self, search_query_object, **kwargs): search_query = Bool() resource_instance_lifecycle_state_filter = Bool() - querystring_params = self.request.GET.get(details["componentname"], "") + querystring_params = self.request.GET.get(self.componentname, "") resource_instance_lifecycle_state_filter_term = JSONDeserializer().deserialize( querystring_params )[0] @@ -54,4 +52,8 @@ def append_dsl( ) search_query.must(resource_instance_lifecycle_state_filter) - search_results_object["query"].add_query(search_query) + return search_query + + def append_dsl(self, search_results_object, **kwargs): + dsl = self.generate_dsl(search_results_object, kwargs) + search_results_object["query"].add_query(dsl) diff --git a/arches/app/search/components/map_filter.py b/arches/app/search/components/map_filter.py index 32bd6cfbcf1..d5820829521 100644 --- a/arches/app/search/components/map_filter.py +++ b/arches/app/search/components/map_filter.py @@ -23,7 +23,8 @@ class MapFilter(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): + + def generate_dsl(self, search_query_object, **kwargs): permitted_nodegroups = kwargs.get("permitted_nodegroups") include_provisional = kwargs.get("include_provisional") search_query = Bool() @@ -72,17 +73,21 @@ def append_dsl(self, search_query_object, **kwargs): ) search_query.filter(Nested(path="geometries", query=spatial_query)) + try: + search_query_object[self.componentname][ + "search_buffer" + ] = feature_geom + except NameError: + logger.info(_("Feature geometry is not defined")) + return search_query - search_query_object["query"].add_query(search_query) + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) if self.componentname not in search_query_object: search_query_object[self.componentname] = {} - try: - search_query_object[self.componentname]["search_buffer"] = feature_geom - except NameError: - logger.info(_("Feature geometry is not defined")) - def _buffer(geojson, width=0, unit="ft"): geojson = JSONSerializer().serialize(geojson) diff --git a/arches/app/search/components/permitted_graphs_policy.py b/arches/app/search/components/permitted_graphs_policy.py new file mode 100644 index 00000000000..0119b1ba976 --- /dev/null +++ b/arches/app/search/components/permitted_graphs_policy.py @@ -0,0 +1,37 @@ +from arches.app.search.elasticsearch_dsl_builder import Bool, Terms +from arches.app.search.components.base import BaseSearchFilter +from arches.app.search.components.resource_type_filter import get_permitted_graphids + +details = { + "searchcomponentid": "", + "name": "Permitted Graphs Policy", + "icon": "", + "modulename": "permitted_graphs_policy.py", + "classname": "PermittedGraphsPolicy", + "type": "search-policy", + "componentpath": None, + "componentname": "permitted-graphs-policy", + "config": {}, +} + + +class PermittedGraphsPolicy(BaseSearchFilter): + def generate_dsl(self, search_query_object, **kwargs): + resource_model_filter = None + permitted_nodegroups = kwargs.get("permitted_nodegroups") + try: + search_query_object["query"].dsl["query"]["bool"]["filter"][0]["terms"][ + "graph_id" + ] # check if resource_type filter is already applied + except (KeyError, IndexError): + resource_model_filter = Bool() + permitted_graphids = get_permitted_graphids(permitted_nodegroups) + terms = Terms(field="graph_id", terms=list(permitted_graphids)) + resource_model_filter.filter(terms) + search_query_object["query"].add_query(resource_model_filter) + return resource_model_filter + + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + if dsl: + search_query_object["query"].add_query(dsl) diff --git a/arches/app/search/components/provisional_filter.py b/arches/app/search/components/provisional_filter.py index 9f83de6dd54..7324b80330e 100644 --- a/arches/app/search/components/provisional_filter.py +++ b/arches/app/search/components/provisional_filter.py @@ -15,7 +15,8 @@ class ProvisionalFilter(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): + + def generate_dsl(self, search_query_object, **kwargs): include_provisional = kwargs.get("include_provisional") search_query = Bool() @@ -33,4 +34,8 @@ def append_dsl(self, search_query_object, **kwargs): ) search_query.must(provisional_resource_filter) - search_query_object["query"].add_query(search_query) + return search_query + + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) diff --git a/arches/app/search/components/resource_type_filter.py b/arches/app/search/components/resource_type_filter.py index 1b575a2901b..34678330d1d 100644 --- a/arches/app/search/components/resource_type_filter.py +++ b/arches/app/search/components/resource_type_filter.py @@ -25,7 +25,8 @@ def get_permitted_graphids(permitted_nodegroups): class ResourceTypeFilter(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): + + def generate_dsl(self, search_query_object, **kwargs): permitted_nodegroups = kwargs.get("permitted_nodegroups") search_query = Bool() querystring_params = kwargs.get("querystring", "") @@ -49,8 +50,11 @@ def append_dsl(self, search_query_object, **kwargs): terms = Terms(field="graph_id", terms=graph_ids) search_query.filter(terms) + return search_query - search_query_object["query"].add_query(search_query) + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) def view_data(self): return { diff --git a/arches/app/search/components/search_results.py b/arches/app/search/components/search_results.py index 94d72fecadf..39b718ba43c 100644 --- a/arches/app/search/components/search_results.py +++ b/arches/app/search/components/search_results.py @@ -30,24 +30,14 @@ class SearchResultsFilter(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): + + def generate_dsl(self, search_query_object, **kwargs): permitted_nodegroups = kwargs.get("permitted_nodegroups") include_provisional = kwargs.get("include_provisional") nested_agg = NestedAgg(path="points", name="geo_aggs") nested_agg_filter = FiltersAgg(name="inner") geo_agg_filter = Bool() - try: - search_query_object["query"].dsl["query"]["bool"]["filter"][0]["terms"][ - "graph_id" - ] # check if resource_type filter is already applied - except (KeyError, IndexError): - resource_model_filter = Bool() - permitted_graphids = get_permitted_graphids(permitted_nodegroups) - terms = Terms(field="graph_id", terms=list(permitted_graphids)) - resource_model_filter.filter(terms) - search_query_object["query"].add_query(resource_model_filter) - if include_provisional is True: geo_agg_filter.filter( Terms(field="points.provisional", terms=["false", "true"]) @@ -89,9 +79,12 @@ def append_dsl(self, search_query_object, **kwargs): ) ) search_query.must(subsearch_query) - search_query_object["query"].add_query(search_query) + return search_query, nested_agg - search_query_object["query"].add_aggregation(nested_agg) + def append_dsl(self, search_query_object, **kwargs): + dsl, aggregation = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) + search_query_object["query"].add_aggregation(aggregation) def post_search_hook(self, search_query_object, response_object, **kwargs): permitted_nodegroups = kwargs.get("permitted_nodegroups") diff --git a/arches/app/search/components/sort_results.py b/arches/app/search/components/sort_results.py index 4682998560f..9c4d3561f14 100644 --- a/arches/app/search/components/sort_results.py +++ b/arches/app/search/components/sort_results.py @@ -15,17 +15,26 @@ class SortResults(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): - sort_param = self.request.GET.get(self.componentname, None) - if sort_param is not None and sort_param != "": - search_query_object["query"].sort( - field="displayname.value", - dsl={ - "order": sort_param, - "nested": { - "path": "displayname", - "filter": {"term": {"displayname.language": get_language()}}, - }, + def generate_dsl(self, search_query_object, **kwargs): + sort_param = self.request.GET.get(self.componentname, "") + field = None + dsl = None + if sort_param != "": + field = "displayname.value" + dsl = { + "order": sort_param, + "nested": { + "path": "displayname", + "filter": {"term": {"displayname.language": get_language()}}, }, + } + return field, dsl + + def append_dsl(self, search_query_object, **kwargs): + sort_field, sort_dsl = self.generate_dsl(search_query_object, **kwargs) + if sort_field and sort_dsl: + search_query_object["query"].sort( + field=sort_field, + dsl=sort_dsl, ) diff --git a/arches/app/search/components/term_filter.py b/arches/app/search/components/term_filter.py index bd429d6db75..d9c4200c38b 100644 --- a/arches/app/search/components/term_filter.py +++ b/arches/app/search/components/term_filter.py @@ -28,7 +28,8 @@ class TermFilter(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): + + def generate_dsl(self, search_query_object, **kwargs): permitted_nodegroups = kwargs.get("permitted_nodegroups") include_provisional = kwargs.get("include_provisional") search_query = Bool() @@ -148,8 +149,11 @@ def append_dsl(self, search_query_object, **kwargs): search_query.must_not(nested_conceptid_filter) else: search_query.filter(nested_conceptid_filter) + return search_query - search_query_object["query"].add_query(search_query) + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) def _get_child_concepts(conceptid): diff --git a/arches/app/search/components/time_filter.py b/arches/app/search/components/time_filter.py index 6413d125ad8..c074458e324 100644 --- a/arches/app/search/components/time_filter.py +++ b/arches/app/search/components/time_filter.py @@ -18,7 +18,7 @@ class TimeFilter(BaseSearchFilter): - def append_dsl(self, search_query_object, **kwargs): + def generate_dsl(self, search_query_object, **kwargs): permitted_nodegroups = kwargs.get("permitted_nodegroups") include_provisional = kwargs.get("include_provisional") search_query = Bool() @@ -148,8 +148,11 @@ def append_dsl(self, search_query_object, **kwargs): temporal_query.should(Nested(path="dates", query=date_query)) search_query.filter(temporal_query) + return search_query - search_query_object["query"].add_query(search_query) + def append_dsl(self, search_query_object, **kwargs): + dsl = self.generate_dsl(search_query_object, **kwargs) + search_query_object["query"].add_query(dsl) def view_data(self): ret = {}