From 8ec19b117f163779f8530dc229c5871090ea50fd Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Thu, 3 Aug 2023 18:32:56 +0100 Subject: [PATCH] enhance smoosh to cleanup search indexes when ddocs change --- src/dreyfus/src/dreyfus_fabric_cleanup.erl | 25 +----------------- src/dreyfus/src/dreyfus_util.erl | 30 +++++++++++++++++++++- src/fabric/src/fabric.erl | 27 ++++++++++++++----- src/nouveau/src/nouveau_fabric_cleanup.erl | 9 +------ src/nouveau/src/nouveau_util.erl | 18 +++++++++++++ src/smoosh/src/smoosh_server.erl | 5 ++++ 6 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/dreyfus/src/dreyfus_fabric_cleanup.erl b/src/dreyfus/src/dreyfus_fabric_cleanup.erl index e2710744d98..f2a36d43943 100644 --- a/src/dreyfus/src/dreyfus_fabric_cleanup.erl +++ b/src/dreyfus/src/dreyfus_fabric_cleanup.erl @@ -14,40 +14,17 @@ -module(dreyfus_fabric_cleanup). --include("dreyfus.hrl"). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). -export([go/1]). go(DbName) -> - {ok, DesignDocs} = fabric:design_docs(DbName), - ActiveSigs = lists:usort( - lists:flatmap( - fun active_sigs/1, - [couch_doc:from_json_obj(DD) || DD <- DesignDocs] - ) - ), + ActiveSigs = dreyfus_util:active_sigs(DbName), cleanup_local_purge_doc(DbName, ActiveSigs), clouseau_rpc:cleanup(DbName, ActiveSigs), ok. -active_sigs(#doc{body = {Fields}} = Doc) -> - try - {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), - {IndexNames, _} = lists:unzip(RawIndexes), - [ - begin - {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), - Index#index.sig - end - || IndexName <- IndexNames - ] - catch - error:{badmatch, _Error} -> - [] - end. - cleanup_local_purge_doc(DbName, ActiveSigs) -> {ok, BaseDir} = clouseau_rpc:get_root_dir(), DbNamePattern = <>, diff --git a/src/dreyfus/src/dreyfus_util.erl b/src/dreyfus/src/dreyfus_util.erl index 301d3887acd..328f997c750 100644 --- a/src/dreyfus/src/dreyfus_util.erl +++ b/src/dreyfus/src/dreyfus_util.erl @@ -29,7 +29,8 @@ maybe_create_local_purge_doc/2, maybe_create_local_purge_doc/3, get_signature_from_idxdir/1, - verify_index_exists/2 + verify_index_exists/2, + active_sigs/1 ]). get_shards(DbName, #index_query_args{partition = nil} = Args) -> @@ -428,6 +429,33 @@ verify_index_exists(DbName, Props) -> false end. +active_sigs(DbName) when is_binary(DbName) -> + couch_util:with_db(DbName, fun active_sigs/1); +active_sigs(Db) -> + {ok, DesignDocs} = couch_db:get_design_docs(Db), + lists:usort( + lists:flatmap( + fun sigs_from_ddoc/1, + [couch_doc:from_json_obj(DD) || DD <- DesignDocs] + ) + ). + +sigs_from_ddoc(#doc{body = {Fields}} = Doc) -> + try + {RawIndexes} = couch_util:get_value(<<"indexes">>, Fields, {[]}), + {IndexNames, _} = lists:unzip(RawIndexes), + [ + begin + {ok, Index} = dreyfus_index:design_doc_to_index(Doc, IndexName), + Index#index.sig + end + || IndexName <- IndexNames + ] + catch + error:{badmatch, _Error} -> + [] + end. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index dfd433d3875..53087c6a6ad 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -590,25 +590,38 @@ cleanup_index_files(DbName) -> cleanup_local_indices_and_purge_checkpoints([]) -> ok; cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) -> - AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs), - AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs), - Sigs = couch_mrview_util:get_signatures(hd(Dbs)), - ok = cleanup_purges(Sigs, AllPurges, Dbs), - ok = cleanup_indices(Sigs, AllIndices). + MrViewIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs), + MrViewPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs), + MrViewSigs = couch_mrview_util:get_signatures(hd(Dbs)), + ok = cleanup_mrview_purges(MrViewSigs, MrViewPurges, Dbs), + ok = cleanup_mrview_indices(MrViewSigs, MrViewIndices), -cleanup_purges(Sigs, AllPurges, Dbs) -> + ClouseauSigs = dreyfus_util:active_sigs(hd(Dbs)), + ok = cleanup_clouseau_indices(Dbs, ClouseauSigs), + + NouveauSigs = nouveau_util:active_sigs(hd(Dbs)), + ok = cleanup_nouveau_indices(Dbs, NouveauSigs). + +cleanup_mrview_purges(Sigs, AllPurges, Dbs) -> Fun = fun(DbPurges, Db) -> couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges) end, lists:zipwith(Fun, AllPurges, Dbs), ok. -cleanup_indices(Sigs, AllIndices) -> +cleanup_mrview_indices(Sigs, AllIndices) -> Fun = fun(DbIndices) -> couch_mrview_cleanup:cleanup_indices(Sigs, DbIndices) end, lists:foreach(Fun, AllIndices). +cleanup_clouseau_indices(Dbs, ActiveSigs) -> + Fun = fun(Db) -> clouseau_rpc:cleanup(Db, ActiveSigs) end, + lists:foreach(Fun, Dbs). +cleanup_nouveau_indices(Dbs, ActiveSigs) -> + Fun = fun(Db) -> nouveau_api:delete_path(nouveau_util:index_name(Db), ActiveSigs) end, + lists:foreach(Fun, Dbs). + %% @doc clean up index files for a specific db on all nodes -spec cleanup_index_files_all_nodes(dbname()) -> [reference()]. cleanup_index_files_all_nodes(DbName) -> diff --git a/src/nouveau/src/nouveau_fabric_cleanup.erl b/src/nouveau/src/nouveau_fabric_cleanup.erl index cd4128fb1d8..e1567ef8d1e 100644 --- a/src/nouveau/src/nouveau_fabric_cleanup.erl +++ b/src/nouveau/src/nouveau_fabric_cleanup.erl @@ -14,9 +14,6 @@ -module(nouveau_fabric_cleanup). --include_lib("couch/include/couch_db.hrl"). - --include("nouveau.hrl"). -include_lib("mem3/include/mem3.hrl"). -export([go/1]). @@ -26,7 +23,7 @@ go(DbName) -> ActiveSigs = lists:usort( lists:flatmap( - fun(Doc) -> active_sigs(DbName, Doc) end, + fun(Doc) -> nouveau_util:active_sigs(DbName, Doc) end, [couch_doc:from_json_obj(DD) || DD <- DesignDocs] ) ), @@ -37,7 +34,3 @@ go(DbName) -> end, Shards ). - -active_sigs(DbName, #doc{} = Doc) -> - Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc), - lists:map(fun(Index) -> Index#index.sig end, Indexes). diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl index b7da7e802e4..6e37477abad 100644 --- a/src/nouveau/src/nouveau_util.erl +++ b/src/nouveau/src/nouveau_util.erl @@ -22,6 +22,8 @@ index_name/1, design_doc_to_indexes/2, design_doc_to_index/3, + active_sigs/1, + active_sigs/2, nouveau_url/0 ]). @@ -93,5 +95,21 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> {error, InvalidDDocError} end. +active_sigs(DbName) when is_binary(DbName) -> + couch_util:with_db(DbName, fun active_sigs/1); +active_sigs(Db) -> + DbName = couch_db:name(Db), + {ok, DesignDocs} = couch_db:get_design_docs(Db), + lists:usort( + lists:flatmap( + fun(Doc) -> active_sigs(DbName, Doc) end, + [couch_doc:from_json_obj(DD) || DD <- DesignDocs] + ) + ). + +active_sigs(DbName, #doc{} = Doc) -> + Indexes = design_doc_to_indexes(DbName, Doc), + lists:map(fun(Index) -> Index#index.sig end, Indexes). + nouveau_url() -> config:get("nouveau", "url", "http://127.0.0.1:8080"). diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl index 10368a5494d..5e7314dff89 100644 --- a/src/smoosh/src/smoosh_server.erl +++ b/src/smoosh/src/smoosh_server.erl @@ -123,6 +123,9 @@ handle_db_event(DbName, local_updated, St) -> handle_db_event(DbName, updated, St) -> enqueue(DbName), {ok, St}; +handle_db_event(DbName, ddoc_updated, St) -> + enqueue({?INDEX_CLEANUP, mem3:dbname(DbName)}), + {ok, St}; handle_db_event(DbName, {index_commit, IdxName}, St) -> enqueue({DbName, IdxName}), {ok, St}; @@ -342,6 +345,8 @@ find_channel(#state{} = State, [Channel | Rest], Object) -> find_channel(State, Rest, Object) end. +stale_enough({?INDEX_CLEANUP, _}) -> + true; stale_enough(Object) -> LastUpdatedSec = last_updated(Object), Staleness = erlang:monotonic_time(second) - LastUpdatedSec,