From 0a8e7d7d84368a42c5bb95ea79d90593b0f8e938 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 12 Sep 2023 14:10:15 +0200
Subject: [PATCH 01/32] Stub API setup for tag usage stats

---
 javascript/_utils.js               | 23 +++++++++++++++++++++++
 scripts/tag_autocomplete_helper.py | 11 +++++++++++
 2 files changed, 34 insertions(+)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 6ef46c0..2619ad5 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -81,6 +81,17 @@ async function fetchAPI(url, json = true, cache = false) {
         return await response.text();
 }
 
+async function postAPI(url, body) {
+    let response = await fetch(url, { method: "POST", body: body });
+
+    if (response.status != 200) {
+        console.error(`Error posting to API endpoint "${url}": ` + response.status, response.statusText);
+        return null;
+    }
+
+    return await response.json();
+}
+
 // Extra network preview thumbnails
 async function getExtraNetworkPreviewURL(filename, type) {
     const previewJSON = await fetchAPI(`tacapi/v1/thumb-preview/${filename}?type=${type}`, true, true);
@@ -147,6 +158,18 @@ function flatten(obj, roots = [], sep = ".") {
   );
 }
 
+// Calculate biased tag score based on post count and frequent usage
+function tagBias(count, uses) {
+    return Math.log(count) + Math.log(uses);
+}
+// Call API endpoint to increase bias of tag in the database
+function increaseUseCount(tagName) {
+    postAPI(`tacapi/v1/increase-use-count/${tagName}`, null);
+}
+// Get use count of tag from the database
+async function getUseCount(tagName) {
+    return (await fetchAPI(`tacapi/v1/get-use-count/${tagName}`, true, false))["count"];
+}
 
 // Sliding window function to get possible combination groups of an array
 function toNgrams(inputArray, size) {
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 1a05f01..cbc7de7 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -529,5 +529,16 @@ async def get_wildcard_contents(basepath: str, filename: str):
         except Exception as e:
             return JSONResponse({"error": e}, status_code=500)
 
+    @app.post("/tacapi/v1/increase-use-count/{tagname}")
+    async def increase_use_count(tagname: str):
+        pass
+
+    @app.get("/tacapi/v1/get-use-count/{tagname}")
+    async def get_use_count(tagname: str):
+        return JSONResponse({"count": 0})
+    
+    @app.put("/tacapi/v1/reset-use-count/{tagname}")
+    async def reset_use_count(tagname: str):
+        pass
 
 script_callbacks.on_app_started(api_tac)

From 0f487a5c5cb2aadb8d15dcc11713a99a197bf200 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 24 Sep 2023 16:28:32 +0200
Subject: [PATCH 02/32] WIP database setup inspired by ImageBrowser

---
 scripts/tag_autocomplete_helper.py |  21 ++++-
 scripts/tag_frequency_db.py        | 134 +++++++++++++++++++++++++++++
 2 files changed, 152 insertions(+), 3 deletions(-)
 create mode 100644 scripts/tag_frequency_db.py

diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 1271673..5af3b34 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -18,6 +18,15 @@
                                            write_model_keyword_path)
 from scripts.shared_paths import *
 
+try:
+    from scripts.tag_frequency_db import TagFrequencyDb, version
+    db = TagFrequencyDb()
+    if db.version != version:
+        raise ValueError("Tag Autocomplete: Tag frequency database version mismatch, disabling tag frequency sorting")
+except (ImportError, ValueError):
+    print("Tag Autocomplete: Tag frequency database could not be loaded, disabling tag frequency sorting")
+    db = None
+
 # Attempt to get embedding load function, using the same call as api.
 try:
     load_textual_inversion_embeddings = sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings
@@ -569,14 +578,20 @@ async def get_wildcard_contents(basepath: str, filename: str):
 
     @app.post("/tacapi/v1/increase-use-count/{tagname}")
     async def increase_use_count(tagname: str):
-        pass
+        db.increase_tag_count(tagname)
 
     @app.get("/tacapi/v1/get-use-count/{tagname}")
     async def get_use_count(tagname: str):
-        return JSONResponse({"count": 0})
+        db_count = db.get_tag_count(tagname)
+        return JSONResponse({"count": db_count})
     
     @app.put("/tacapi/v1/reset-use-count/{tagname}")
     async def reset_use_count(tagname: str):
-        pass
+        db.reset_tag_count(tagname)
+
+    @app.get("/tacapi/v1/get-all-tag-counts")
+    async def get_all_tag_counts():
+        db_counts = db.get_all_tags()
+        return JSONResponse({"counts": db_counts})
 
 script_callbacks.on_app_started(api_tac)
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
new file mode 100644
index 0000000..d215b7e
--- /dev/null
+++ b/scripts/tag_frequency_db.py
@@ -0,0 +1,134 @@
+import sqlite3
+from contextlib import contextmanager
+
+from scripts.shared_paths import TAGS_PATH
+
+db_file = TAGS_PATH.joinpath("tag_frequency.db")
+timeout = 30
+version = 1
+
+
+@contextmanager
+def transaction(db=db_file):
+    """Context manager for database transactions.
+    Ensures that the connection is properly closed after the transaction.
+    """
+    conn = sqlite3.connect(db, timeout=timeout)
+    try:
+        conn.isolation_level = None
+        cursor = conn.cursor()
+        cursor.execute("BEGIN")
+        yield cursor
+        cursor.execute("COMMIT")
+    finally:
+        conn.close()
+
+
+class TagFrequencyDb:
+    """Class containing creation and interaction methods for the tag frequency database"""
+
+    def __init__(self) -> None:
+        self.version = self.__check()
+
+    def __check(self):
+        if not db_file.exists():
+            print("Tag Autocomplete: Creating frequency database")
+            with transaction() as cursor:
+                self.__create_db(cursor)
+                self.__update_db_data(cursor, "version", version)
+            print("Tag Autocomplete: Database successfully created")
+
+        return self.__get_version()
+
+    def __create_db(self, cursor: sqlite3.Cursor):
+        cursor.execute(
+            """
+        CREATE TABLE IF NOT EXISTS db_data (
+            key TEXT PRIMARY KEY,
+            value TEXT
+        )
+        """
+        )
+
+        cursor.execute(
+            """
+        CREATE TABLE IF NOT EXISTS tag_frequency (
+            name TEXT PRIMARY KEY,
+            count INT,
+            last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """
+        )
+
+    def __update_db_data(cursor: sqlite3.Cursor, key, value):
+        cursor.execute(
+            """
+        INSERT OR REPLACE
+        INTO db_data (key, value)
+        VALUES (?, ?)
+        """,
+            (key, value),
+        )
+
+    def __get_version(self):
+        with transaction() as cursor:
+            cursor.execute(
+                """
+            SELECT value
+            FROM db_data
+            WHERE key = 'version'
+            """
+            )
+            db_version = cursor.fetchone()
+
+        return db_version
+
+    def get_all_tags(self):
+        with transaction() as cursor:
+            cursor.execute(
+                """
+            SELECT name
+            FROM tag_frequency
+            ORDER BY count DESC
+            """
+            )
+            tags = cursor.fetchall()
+
+        return tags
+
+    def get_tag_count(self, tag):
+        with transaction() as cursor:
+            cursor.execute(
+                """
+            SELECT count
+            FROM tag_frequency
+            WHERE name = ?
+            """,
+                (tag,),
+            )
+            tag_count = cursor.fetchone()
+
+        return tag_count or 0
+
+    def increase_tag_count(self, tag):
+        current_count = self.get_tag_count(tag)
+        with transaction() as cursor:
+            cursor.execute(
+                """
+            INSERT OR REPLACE
+            INTO tag_frequency (name, count)
+            VALUES (?, ?)
+            """,
+                (tag, current_count + 1),
+            )
+
+    def reset_tag_count(self, tag):
+        with transaction() as cursor:
+            cursor.execute(
+                """
+            UPDATE tag_frequency
+            SET count = 0
+            WHERE name = ?
+            """,
+                (tag,),
+            )

From 1e81403180c8b4eeac4af4352bdf7e05ad319055 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 24 Sep 2023 16:50:03 +0200
Subject: [PATCH 03/32] Safety catches for DB API access

---
 scripts/tag_autocomplete_helper.py | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 5af3b34..bbb4034 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -576,22 +576,36 @@ async def get_wildcard_contents(basepath: str, filename: str):
         except Exception as e:
             return JSONResponse({"error": e}, status_code=500)
 
+    NO_DB = JSONResponse({"error": "Database not initialized"}, status_code=500)
+
     @app.post("/tacapi/v1/increase-use-count/{tagname}")
     async def increase_use_count(tagname: str):
-        db.increase_tag_count(tagname)
+        if db is not None:
+            db.increase_tag_count(tagname)
+        else:
+            return NO_DB
 
     @app.get("/tacapi/v1/get-use-count/{tagname}")
     async def get_use_count(tagname: str):
-        db_count = db.get_tag_count(tagname)
-        return JSONResponse({"count": db_count})
-    
+        if db is not None:
+            db_count = db.get_tag_count(tagname)
+            return JSONResponse({"count": db_count})
+        else:
+            return NO_DB
+
     @app.put("/tacapi/v1/reset-use-count/{tagname}")
     async def reset_use_count(tagname: str):
-        db.reset_tag_count(tagname)
+        if db is not None:
+            db.reset_tag_count(tagname)
+        else:
+            return NO_DB
 
     @app.get("/tacapi/v1/get-all-tag-counts")
     async def get_all_tag_counts():
-        db_counts = db.get_all_tags()
-        return JSONResponse({"counts": db_counts})
-
+        if db is not None:
+            db_tags = db.get_all_tags()
+            return JSONResponse({"tags": db_tags})
+        else:
+            return NO_DB
+        
 script_callbacks.on_app_started(api_tac)

From b44c36425a148a49899db0c93b8370ea2d80c46d Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 24 Sep 2023 17:59:14 +0200
Subject: [PATCH 04/32] Fix db load version comparison, add sort options

---
 javascript/_utils.js               | 11 +++++++++++
 scripts/tag_autocomplete_helper.py | 16 ++++++++++++----
 scripts/tag_frequency_db.py        | 10 +++++-----
 3 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 50ab101..d9c7759 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -92,6 +92,17 @@ async function postAPI(url, body) {
     return await response.json();
 }
 
+async function putAPI(url, body) {
+    let response = await fetch(url, { method: "PUT", body: body });
+    
+    if (response.status != 200) {
+        console.error(`Error putting to API endpoint "${url}": ` + response.status, response.statusText);
+        return null;
+    }
+
+    return await response.json();
+}
+
 // Extra network preview thumbnails
 async function getExtraNetworkPreviewURL(filename, type) {
     const previewJSON = await fetchAPI(`tacapi/v1/thumb-preview/${filename}?type=${type}`, true, true);
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index bbb4034..97fb756 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -1,7 +1,6 @@
 # This helper script scans folders for wildcards and embeddings and writes them
 # to a temporary file to expose it to the javascript side
 
-import os
 import glob
 import json
 import urllib.parse
@@ -19,11 +18,12 @@
 from scripts.shared_paths import *
 
 try:
-    from scripts.tag_frequency_db import TagFrequencyDb, version
+    from scripts.tag_frequency_db import TagFrequencyDb, db_ver
     db = TagFrequencyDb()
-    if db.version != version:
+    if int(db.version) != int(db_ver):
         raise ValueError("Tag Autocomplete: Tag frequency database version mismatch, disabling tag frequency sorting")
-except (ImportError, ValueError):
+except (ImportError, ValueError) as e:
+    print(e)
     print("Tag Autocomplete: Tag frequency database could not be loaded, disabling tag frequency sorting")
     db = None
 
@@ -391,6 +391,12 @@ def needs_restart(self):
             return self
         shared.OptionInfo.needs_restart = needs_restart
 
+    # Dictionary of function options and their explanations
+    frequency_sort_functions = {
+        "Logarithmic": "Will respect the base order and slightly prefer more frequent tags",
+        "Usage first": "Will list used tags by frequency before all others",
+    }
+
     tac_options = {
         # Main tag file
         "tac_tagFile": shared.OptionInfo("danbooru.csv", "Tag filename", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files),
@@ -418,6 +424,8 @@ def needs_restart(self):
         "tac_showWikiLinks": shared.OptionInfo(False, "Show '?' next to tags, linking to its Danbooru or e621 wiki page").info("Warning: This is an external site and very likely contains NSFW examples!"),
         "tac_showExtraNetworkPreviews": shared.OptionInfo(True, "Show preview thumbnails for extra networks if available"),
         "tac_modelSortOrder": shared.OptionInfo("Name", "Model sort order", gr.Dropdown, lambda: {"choices": list(sort_criteria.keys())}).info("Order for extra network models and wildcards in dropdown"),
+        "tac_frequencySort": shared.OptionInfo(True, "Locally record tag usage and sort frequent tags higher").info("Will also work for extra networks, keeping the specified base order"),
+        "tac_frequencyFunction": shared.OptionInfo("Logarithmic", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
         # Insertion related settings
         "tac_replaceUnderscores": shared.OptionInfo(True, "Replace underscores with spaces on insertion"),
         "tac_escapeParentheses": shared.OptionInfo(True, "Escape parentheses on insertion"),
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index d215b7e..d186b0d 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -5,7 +5,7 @@
 
 db_file = TAGS_PATH.joinpath("tag_frequency.db")
 timeout = 30
-version = 1
+db_ver = 1
 
 
 @contextmanager
@@ -35,7 +35,7 @@ def __check(self):
             print("Tag Autocomplete: Creating frequency database")
             with transaction() as cursor:
                 self.__create_db(cursor)
-                self.__update_db_data(cursor, "version", version)
+                self.__update_db_data(cursor, "version", db_ver)
             print("Tag Autocomplete: Database successfully created")
 
         return self.__get_version()
@@ -60,7 +60,7 @@ def __create_db(self, cursor: sqlite3.Cursor):
         """
         )
 
-    def __update_db_data(cursor: sqlite3.Cursor, key, value):
+    def __update_db_data(self, cursor: sqlite3.Cursor, key, value):
         cursor.execute(
             """
         INSERT OR REPLACE
@@ -81,7 +81,7 @@ def __get_version(self):
             )
             db_version = cursor.fetchone()
 
-        return db_version
+        return db_version[0] if db_version else 0
 
     def get_all_tags(self):
         with transaction() as cursor:
@@ -108,7 +108,7 @@ def get_tag_count(self, tag):
             )
             tag_count = cursor.fetchone()
 
-        return tag_count or 0
+        return tag_count[0] if tag_count else 0
 
     def increase_tag_count(self, tag):
         current_count = self.get_tag_count(tag)

From 3caa1b51edc03855632a0dc2b0ee1d1c10cf62d8 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 24 Sep 2023 17:59:39 +0200
Subject: [PATCH 05/32] Add db to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index e9e3707..d324b1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 tags/temp/
 __pycache__/
+tags/tag_frequency.db

From 109a8a155e2a7163a1e1d218074366427571fa76 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 24 Sep 2023 18:00:41 +0200
Subject: [PATCH 06/32] Change endpoint name for consistency

---
 scripts/tag_autocomplete_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 97fb756..9b35232 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -608,7 +608,7 @@ async def reset_use_count(tagname: str):
         else:
             return NO_DB
 
-    @app.get("/tacapi/v1/get-all-tag-counts")
+    @app.get("/tacapi/v1/get-all-use-counts")
     async def get_all_tag_counts():
         if db is not None:
             db_tags = db.get_all_tags()

From 6cf9acd6abd2ef0a53b691eca167216631d93678 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 24 Sep 2023 20:06:40 +0200
Subject: [PATCH 07/32] Catch sqlite exceptions, add tag list endpoint

---
 javascript/_utils.js               |  5 ++-
 scripts/tag_autocomplete_helper.py | 49 +++++++++++++++---------------
 scripts/tag_frequency_db.py        | 14 +++++++++
 3 files changed, 43 insertions(+), 25 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index d9c7759..ab67f0f 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -179,7 +179,10 @@ function increaseUseCount(tagName) {
 }
 // Get use count of tag from the database
 async function getUseCount(tagName) {
-    return (await fetchAPI(`tacapi/v1/get-use-count/${tagName}`, true, false))["count"];
+    return (await fetchAPI(`tacapi/v1/get-use-count/${tagName}`, true, false))["result"];
+}
+async function getUseCounts(tagNames) {
+    return (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}`))["result"];
 }
 
 // Sliding window function to get possible combination groups of an array
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 9b35232..9b37dfa 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -3,12 +3,13 @@
 
 import glob
 import json
+import sqlite3
 import urllib.parse
 from pathlib import Path
 
 import gradio as gr
 import yaml
-from fastapi import FastAPI
+from fastapi import FastAPI, Query
 from fastapi.responses import FileResponse, JSONResponse
 from modules import script_callbacks, sd_hijack, shared
 
@@ -21,10 +22,9 @@
     from scripts.tag_frequency_db import TagFrequencyDb, db_ver
     db = TagFrequencyDb()
     if int(db.version) != int(db_ver):
-        raise ValueError("Tag Autocomplete: Tag frequency database version mismatch, disabling tag frequency sorting")
-except (ImportError, ValueError) as e:
-    print(e)
-    print("Tag Autocomplete: Tag frequency database could not be loaded, disabling tag frequency sorting")
+        raise ValueError("Database version mismatch")
+except (ImportError, ValueError, sqlite3.Error) as e:
+    print(f"Tag Autocomplete: Tag frequency database error - \"{e}\"")
     db = None
 
 # Attempt to get embedding load function, using the same call as api.
@@ -584,36 +584,37 @@ async def get_wildcard_contents(basepath: str, filename: str):
         except Exception as e:
             return JSONResponse({"error": e}, status_code=500)
 
-    NO_DB = JSONResponse({"error": "Database not initialized"}, status_code=500)
+    def db_request(func, get = False):
+        if db is not None:
+            try:
+                if get:
+                    ret = func()
+                    return JSONResponse({"result": ret})
+                else:
+                    func()
+            except sqlite3.Error as e:
+                return JSONResponse({"error": e}, status_code=500)
+        else:
+            return JSONResponse({"error": "Database not initialized"}, status_code=500)
 
     @app.post("/tacapi/v1/increase-use-count/{tagname}")
     async def increase_use_count(tagname: str):
-        if db is not None:
-            db.increase_tag_count(tagname)
-        else:
-            return NO_DB
+        db_request(lambda: db.increase_tag_count(tagname))
 
     @app.get("/tacapi/v1/get-use-count/{tagname}")
     async def get_use_count(tagname: str):
-        if db is not None:
-            db_count = db.get_tag_count(tagname)
-            return JSONResponse({"count": db_count})
-        else:
-            return NO_DB
+        return db_request(lambda: db.get_tag_count(tagname), get=True)
+        
+    @app.get("/tacapi/v1/get-use-count-list")
+    async def get_use_count_list(tags: list[str] | None = Query(default=None)):
+        return db_request(lambda: list(db.get_tag_counts(tags)), get=True)
 
     @app.put("/tacapi/v1/reset-use-count/{tagname}")
     async def reset_use_count(tagname: str):
-        if db is not None:
-            db.reset_tag_count(tagname)
-        else:
-            return NO_DB
+        db_request(lambda: db.reset_tag_count(tagname))
 
     @app.get("/tacapi/v1/get-all-use-counts")
     async def get_all_tag_counts():
-        if db is not None:
-            db_tags = db.get_all_tags()
-            return JSONResponse({"tags": db_tags})
-        else:
-            return NO_DB
+        return db_request(lambda: db.get_all_tags(), get=True)
         
 script_callbacks.on_app_started(api_tac)
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index d186b0d..77945bb 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -110,6 +110,20 @@ def get_tag_count(self, tag):
 
         return tag_count[0] if tag_count else 0
 
+    def get_tag_counts(self, tags: list[str]):
+        with transaction() as cursor:
+            for tag in tags:
+                cursor.execute(
+                    """
+                SELECT count
+                FROM tag_frequency
+                WHERE name = ?
+                """,
+                    (tag,),
+                )
+                tag_count = cursor.fetchone()
+                yield (tag, tag_count[0]) if tag_count else (tag, 0)
+
     def increase_tag_count(self, tag):
         current_count = self.get_tag_count(tag)
         with transaction() as cursor:

From 74ea5493e5600b8a8e38991889bcacedc4089cd0 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 10:58:46 +0200
Subject: [PATCH 08/32] Add rest of utils functions

---
 javascript/_utils.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index ab67f0f..22b5c54 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -184,6 +184,12 @@ async function getUseCount(tagName) {
 async function getUseCounts(tagNames) {
     return (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}`))["result"];
 }
+async function getAllUseCounts() {
+    return (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
+}
+async function resetUseCount(tagName) {
+    putAPI(`tacapi/v1/reset-use-count/${tagName}`, null);
+}
 
 // Sliding window function to get possible combination groups of an array
 function toNgrams(inputArray, size) {

From 581bf1e6a401542ab12e9db93539efb88b81e945 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 11:35:24 +0200
Subject: [PATCH 09/32] Use composite key with name & type to prevent
 collisions

---
 javascript/_utils.js               | 16 ++++++-------
 scripts/tag_autocomplete_helper.py | 16 ++++++-------
 scripts/tag_frequency_db.py        | 38 ++++++++++++++++--------------
 3 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 22b5c54..c1ccb60 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -174,21 +174,21 @@ function tagBias(count, uses) {
     return Math.log(count) + Math.log(uses);
 }
 // Call API endpoint to increase bias of tag in the database
-function increaseUseCount(tagName) {
-    postAPI(`tacapi/v1/increase-use-count/${tagName}`, null);
+function increaseUseCount(tagName, type) {
+    postAPI(`tacapi/v1/increase-use-count/${tagName}?ttype=${type}`, null);
 }
 // Get use count of tag from the database
-async function getUseCount(tagName) {
-    return (await fetchAPI(`tacapi/v1/get-use-count/${tagName}`, true, false))["result"];
+async function getUseCount(tagName, type) {
+    return (await fetchAPI(`tacapi/v1/get-use-count/${tagName}?ttype=${type}`, true, false))["result"];
 }
-async function getUseCounts(tagNames) {
-    return (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}`))["result"];
+async function getUseCounts(tagNames, types) {
+    return (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}&ttypes=${types.join("&ttypes=")}`))["result"];
 }
 async function getAllUseCounts() {
     return (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
 }
-async function resetUseCount(tagName) {
-    putAPI(`tacapi/v1/reset-use-count/${tagName}`, null);
+async function resetUseCount(tagName, type) {
+    putAPI(`tacapi/v1/reset-use-count/${tagName}?ttype=${type}`, null);
 }
 
 // Sliding window function to get possible combination groups of an array
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 9b37dfa..daaf6a3 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -598,20 +598,20 @@ def db_request(func, get = False):
             return JSONResponse({"error": "Database not initialized"}, status_code=500)
 
     @app.post("/tacapi/v1/increase-use-count/{tagname}")
-    async def increase_use_count(tagname: str):
-        db_request(lambda: db.increase_tag_count(tagname))
+    async def increase_use_count(tagname: str, ttype: int):
+        db_request(lambda: db.increase_tag_count(tagname, ttype))
 
     @app.get("/tacapi/v1/get-use-count/{tagname}")
-    async def get_use_count(tagname: str):
-        return db_request(lambda: db.get_tag_count(tagname), get=True)
+    async def get_use_count(tagname: str, ttype: int):
+        return db_request(lambda: db.get_tag_count(tagname, ttype), get=True)
         
     @app.get("/tacapi/v1/get-use-count-list")
-    async def get_use_count_list(tags: list[str] | None = Query(default=None)):
-        return db_request(lambda: list(db.get_tag_counts(tags)), get=True)
+    async def get_use_count_list(tags: list[str] | None = Query(default=None), ttypes: list[int] | None = Query(default=None)):
+        return db_request(lambda: list(db.get_tag_counts(tags, ttypes)), get=True)
 
     @app.put("/tacapi/v1/reset-use-count/{tagname}")
-    async def reset_use_count(tagname: str):
-        db_request(lambda: db.reset_tag_count(tagname))
+    async def reset_use_count(tagname: str, ttype: int):
+        db_request(lambda: db.reset_tag_count(tagname, ttype))
 
     @app.get("/tacapi/v1/get-all-use-counts")
     async def get_all_tag_counts():
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index 77945bb..bed8355 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -53,9 +53,11 @@ def __create_db(self, cursor: sqlite3.Cursor):
         cursor.execute(
             """
         CREATE TABLE IF NOT EXISTS tag_frequency (
-            name TEXT PRIMARY KEY,
+            name TEXT NOT NULL,
+            type INT NOT NULL,
             count INT,
-            last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            PRIMARY KEY (name, type)
         )
         """
         )
@@ -87,7 +89,7 @@ def get_all_tags(self):
         with transaction() as cursor:
             cursor.execute(
                 """
-            SELECT name
+            SELECT name, type, count
             FROM tag_frequency
             ORDER BY count DESC
             """
@@ -96,53 +98,53 @@ def get_all_tags(self):
 
         return tags
 
-    def get_tag_count(self, tag):
+    def get_tag_count(self, tag, ttype):
         with transaction() as cursor:
             cursor.execute(
                 """
             SELECT count
             FROM tag_frequency
-            WHERE name = ?
+            WHERE name = ? AND type = ?
             """,
-                (tag,),
+                (tag,ttype),
             )
             tag_count = cursor.fetchone()
 
         return tag_count[0] if tag_count else 0
 
-    def get_tag_counts(self, tags: list[str]):
+    def get_tag_counts(self, tags: list[str], ttypes: list[str]):
         with transaction() as cursor:
-            for tag in tags:
+            for tag, ttype in zip(tags, ttypes):
                 cursor.execute(
                     """
                 SELECT count
                 FROM tag_frequency
-                WHERE name = ?
+                WHERE name = ? AND type = ?
                 """,
-                    (tag,),
+                    (tag,ttype),
                 )
                 tag_count = cursor.fetchone()
                 yield (tag, tag_count[0]) if tag_count else (tag, 0)
 
-    def increase_tag_count(self, tag):
-        current_count = self.get_tag_count(tag)
+    def increase_tag_count(self, tag, ttype):
+        current_count = self.get_tag_count(tag, ttype)
         with transaction() as cursor:
             cursor.execute(
                 """
             INSERT OR REPLACE
-            INTO tag_frequency (name, count)
-            VALUES (?, ?)
+            INTO tag_frequency (name, type, count)
+            VALUES (?, ?, ?)
             """,
-                (tag, current_count + 1),
+                (tag, ttype, current_count + 1),
             )
 
-    def reset_tag_count(self, tag):
+    def reset_tag_count(self, tag, ttype):
         with transaction() as cursor:
             cursor.execute(
                 """
             UPDATE tag_frequency
             SET count = 0
-            WHERE name = ?
+            WHERE name = ? AND type = ?
             """,
-                (tag,),
+                (tag,ttype),
             )

From 460d32a4edba479885cd59b4547af120d4767d77 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 11:45:42 +0200
Subject: [PATCH 10/32] Ensure proper reload, fix error message

---
 scripts/tag_autocomplete_helper.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index daaf6a3..42abfb5 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -2,6 +2,7 @@
 # to a temporary file to expose it to the javascript side
 
 import glob
+import importlib
 import json
 import sqlite3
 import urllib.parse
@@ -19,9 +20,13 @@
 from scripts.shared_paths import *
 
 try:
-    from scripts.tag_frequency_db import TagFrequencyDb, db_ver
-    db = TagFrequencyDb()
-    if int(db.version) != int(db_ver):
+    import scripts.tag_frequency_db as tdb
+
+    # Ensure the db dependency is reloaded on script reload
+    importlib.reload(tdb)
+
+    db = tdb.TagFrequencyDb()
+    if int(db.version) != int(tdb.db_ver):
         raise ValueError("Database version mismatch")
 except (ImportError, ValueError, sqlite3.Error) as e:
     print(f"Tag Autocomplete: Tag frequency database error - \"{e}\"")
@@ -593,7 +598,7 @@ def db_request(func, get = False):
                 else:
                     func()
             except sqlite3.Error as e:
-                return JSONResponse({"error": e}, status_code=500)
+                return JSONResponse({"error": e.__cause__}, status_code=500)
         else:
             return JSONResponse({"error": "Database not initialized"}, status_code=500)
 

From 030a83aa4d78eeb2f39acaa24d1aa74863f3a0a2 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 11:55:12 +0200
Subject: [PATCH 11/32] Use query parameter instead of path to fix wildcard
 subfolder issues

---
 javascript/_utils.js               | 6 +++---
 scripts/tag_autocomplete_helper.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index c1ccb60..554e4f0 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -175,11 +175,11 @@ function tagBias(count, uses) {
 }
 // Call API endpoint to increase bias of tag in the database
 function increaseUseCount(tagName, type) {
-    postAPI(`tacapi/v1/increase-use-count/${tagName}?ttype=${type}`, null);
+    postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`, null);
 }
 // Get use count of tag from the database
 async function getUseCount(tagName, type) {
-    return (await fetchAPI(`tacapi/v1/get-use-count/${tagName}?ttype=${type}`, true, false))["result"];
+    return (await fetchAPI(`tacapi/v1/get-use-count?tagname=${tagName}&ttype=${type}`, true, false))["result"];
 }
 async function getUseCounts(tagNames, types) {
     return (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}&ttypes=${types.join("&ttypes=")}`))["result"];
@@ -188,7 +188,7 @@ async function getAllUseCounts() {
     return (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
 }
 async function resetUseCount(tagName, type) {
-    putAPI(`tacapi/v1/reset-use-count/${tagName}?ttype=${type}`, null);
+    putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}`, null);
 }
 
 // Sliding window function to get possible combination groups of an array
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 42abfb5..1a28bb3 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -602,11 +602,11 @@ def db_request(func, get = False):
         else:
             return JSONResponse({"error": "Database not initialized"}, status_code=500)
 
-    @app.post("/tacapi/v1/increase-use-count/{tagname}")
+    @app.post("/tacapi/v1/increase-use-count")
     async def increase_use_count(tagname: str, ttype: int):
         db_request(lambda: db.increase_tag_count(tagname, ttype))
 
-    @app.get("/tacapi/v1/get-use-count/{tagname}")
+    @app.get("/tacapi/v1/get-use-count")
     async def get_use_count(tagname: str, ttype: int):
         return db_request(lambda: db.get_tag_count(tagname, ttype), get=True)
         
@@ -614,7 +614,7 @@ async def get_use_count(tagname: str, ttype: int):
     async def get_use_count_list(tags: list[str] | None = Query(default=None), ttypes: list[int] | None = Query(default=None)):
         return db_request(lambda: list(db.get_tag_counts(tags, ttypes)), get=True)
 
-    @app.put("/tacapi/v1/reset-use-count/{tagname}")
+    @app.put("/tacapi/v1/reset-use-count")
     async def reset_use_count(tagname: str, ttype: int):
         db_request(lambda: db.reset_tag_count(tagname, ttype))
 

From 22365ec8d678404f8aae861acf2ff85e36133771 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 12:02:36 +0200
Subject: [PATCH 12/32] Add missing type return to list request

---
 scripts/tag_frequency_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index bed8355..71862e6 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -124,7 +124,7 @@ def get_tag_counts(self, tags: list[str], ttypes: list[str]):
                     (tag,ttype),
                 )
                 tag_count = cursor.fetchone()
-                yield (tag, tag_count[0]) if tag_count else (tag, 0)
+                yield (tag, ttype, tag_count[0]) if tag_count else (tag, ttype, 0)
 
     def increase_tag_count(self, tag, ttype):
         current_count = self.get_tag_count(tag, ttype)

From ac790c8edefd246046f6fb6a2f75269b83114c58 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 12:12:46 +0200
Subject: [PATCH 13/32] Return dict instead of array for clarity

---
 scripts/tag_autocomplete_helper.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 1a28bb3..46b9081 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -594,6 +594,8 @@ def db_request(func, get = False):
             try:
                 if get:
                     ret = func()
+                    if ret is list:
+                        ret = [{"name": t[0], "type": t[1], "count": t[2]} for t in ret]
                     return JSONResponse({"result": ret})
                 else:
                     func()

From d7e98200a85f69913acfb9ea903623cf85d606fa Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Tue, 26 Sep 2023 12:20:15 +0200
Subject: [PATCH 14/32] Use count increase logic

---
 javascript/tagAutocomplete.js | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 87b09b5..c101831 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -218,6 +218,7 @@ async function syncOptions() {
         showWikiLinks: opts["tac_showWikiLinks"],
         showExtraNetworkPreviews: opts["tac_showExtraNetworkPreviews"],
         modelSortOrder: opts["tac_modelSortOrder"],
+        frequencySort: opts["tac_frequencySort"],
         // Insertion related settings
         replaceUnderscores: opts["tac_replaceUnderscores"],
         escapeParentheses: opts["tac_escapeParentheses"],
@@ -461,6 +462,34 @@ async function insertTextAtCursor(textArea, result, tagword, tabCompletedWithout
         }
     }
 
+    // Frequency db update
+    if (TAC_CFG.frequencySort) {
+        let name = null;
+
+        switch (tagType) {
+            case ResultType.wildcardFile:
+            case ResultType.yamlWildcard:
+                // We only want to update the frequency for a full wildcard, not partial paths
+                if (sanitizedText.endsWith("__"))
+                    name = text
+                break;
+            case ResultType.chant:
+                // Chants use a slightly different format
+                name = result.aliases;
+                break;
+            default:
+                name = text;
+                break;
+        }
+
+        if (name && name.length > 0) {
+            // Sanitize name for API call
+            name = encodeURIComponent(name)
+            // Call API & update db
+            increaseUseCount(name, tagType)
+        }
+    }
+
     var prompt = textArea.value;
 
     // Edit prompt text

From 80fb247dbe9bc94371a28c8a5791004984c0dce6 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 1 Oct 2023 21:44:24 +0200
Subject: [PATCH 15/32] Sort results by usage count

---
 javascript/_result.js         |  1 +
 javascript/_utils.js          | 20 +++++++----
 javascript/tagAutocomplete.js | 63 +++++++++++++++++++++++++++++++----
 3 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/javascript/_result.js b/javascript/_result.js
index 823f26d..7c32f08 100644
--- a/javascript/_result.js
+++ b/javascript/_result.js
@@ -24,6 +24,7 @@ class AutocompleteResult {
     // Additional info, only used in some cases
     category = null;
     count = null;
+    usageBias = null;
     aliases = null;
     meta = null;
     hash = null;
diff --git a/javascript/_utils.js b/javascript/_utils.js
index 554e4f0..09f7220 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -170,25 +170,31 @@ function flatten(obj, roots = [], sep = ".") {
 }
 
 // Calculate biased tag score based on post count and frequent usage
-function tagBias(count, uses) {
-    return Math.log(count) + Math.log(uses);
+function calculateUsageBias(count, uses) {
+    return Math.log(1 + count) + Math.log(1 + uses);
+}
+// Beautify return type for easier parsing
+function mapUseCountArray(useCounts) {
+    return useCounts.map(useCount => {return {"name": useCount[0], "type": useCount[1], "count": useCount[2]}});
 }
 // Call API endpoint to increase bias of tag in the database
-function increaseUseCount(tagName, type) {
-    postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`, null);
+async function increaseUseCount(tagName, type) {
+    await postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`, null);
 }
 // Get use count of tag from the database
 async function getUseCount(tagName, type) {
     return (await fetchAPI(`tacapi/v1/get-use-count?tagname=${tagName}&ttype=${type}`, true, false))["result"];
 }
 async function getUseCounts(tagNames, types) {
-    return (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}&ttypes=${types.join("&ttypes=")}`))["result"];
+    const rawArray = (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}&ttypes=${types.join("&ttypes=")}`))["result"]
+    return mapUseCountArray(rawArray);
 }
 async function getAllUseCounts() {
-    return (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
+    const rawArray = (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
+    return mapUseCountArray(rawArray);
 }
 async function resetUseCount(tagName, type) {
-    putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}`, null);
+    await putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}`, null);
 }
 
 // Sliding window function to get possible combination groups of an array
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index c101831..eacbdcd 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -84,6 +84,10 @@ const autocompleteCSS = `
         white-space: nowrap;
         color: var(--meta-text-color);
     }
+    .acMetaText.biased::before {
+        content: "✨";
+        margin-right: 2px;
+    }
     .acWikiLink {
         padding: 0.5rem;
         margin: -0.5rem 0 -0.5rem -0.5rem;
@@ -486,7 +490,7 @@ async function insertTextAtCursor(textArea, result, tagword, tabCompletedWithout
             // Sanitize name for API call
             name = encodeURIComponent(name)
             // Call API & update db
-            increaseUseCount(name, tagType)
+            await increaseUseCount(name, tagType)
         }
     }
 
@@ -773,6 +777,11 @@ function addResultsToList(textArea, results, tagword, resetList) {
             flexDiv.appendChild(metaDiv);
         }
 
+        // Add small ✨ marker to indicate usage sorting
+        if (result.usageBias) {
+            flexDiv.querySelector(".acMetaText").classList.add("biased");
+        }
+
         // Add listener
         li.addEventListener("click", function () { insertTextAtCursor(textArea, result, tagword); });
         // Add element to list
@@ -1042,6 +1051,9 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
     resultCountBeforeNormalTags = 0;
     tagword = tagword.toLowerCase().replace(/[\n\r]/g, "");
 
+    // Needed for slicing check later
+    let normalTags = false;
+
     // Process all parsers
     let resultCandidates = (await processParsers(textArea, prompt))?.filter(x => x.length > 0);
     // If one ore more result candidates match, use their results
@@ -1077,6 +1089,7 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
     if (!resultCandidates || resultCandidates.length === 0
         || (TAC_CFG.includeEmbeddingsInNormalResults && !(tagword.startsWith("<") || tagword.startsWith("*<")))
     ) {
+        normalTags = true;
         resultCountBeforeNormalTags = results.length;
 
         // Create escaped search regex with support for * as a start placeholder
@@ -1131,11 +1144,6 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
                 results = results.concat(extraResults);
             }
         }
-
-        // Slice if the user has set a max result count
-        if (!TAC_CFG.showAllResults) {
-            results = results.slice(0, TAC_CFG.maxResults + resultCountBeforeNormalTags);
-        }
     }
 
     // Guard for empty results
@@ -1145,6 +1153,49 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
         return;
     }
 
+    // Sort again with frequency / usage count if enabled
+    if (TAC_CFG.frequencySort) {
+        // Split our results into a list of names and types
+        let names = [];
+        let types = [];
+        // We need to limit size for the request url
+        results.slice(0, 100).forEach(r => {
+            const name = r.type === ResultType.chant ? r.aliases : r.text;
+            names.push(name);
+            types.push(r.type);
+        });
+
+        // Request use counts from the DB
+        const counts = await getUseCounts(names, types);
+        const usedResults = counts.filter(c => c.count > 0).map(c => c.name);
+
+        // Sort all
+        results = results.sort((a, b) => {
+            const aName = a.type === ResultType.chant ? a.aliases : a.text;
+            const bName = b.type === ResultType.chant ? b.aliases : b.text;
+
+            const aUseStats = counts.find(c => c.name === aName && c.type === a.type);
+            const bUseStats = counts.find(c => c.name === bName && c.type === b.type);
+
+            const aWeight = calculateUsageBias(a.count || 0, aUseStats ? aUseStats.count : 0);
+            const bWeight = calculateUsageBias(b.count || 0, bUseStats ? bUseStats.count : 0);
+
+            return bWeight - aWeight;
+        });
+
+        // Mark results
+        results.forEach(r => {
+            const name = r.type === ResultType.chant ? r.aliases : r.text;
+            if (usedResults.includes(name))
+                r.usageBias = true;
+        });
+    }
+
+    // Slice if the user has set a max result count and we are not in a extra networks / wildcard list
+    if (!TAC_CFG.showAllResults && normalTags) {
+        results = results.slice(0, TAC_CFG.maxResults + resultCountBeforeNormalTags);
+    }
+
     addResultsToList(textArea, results, tagword, true);
     showResults(textArea);
 }

From 440f109f1fc7283297a3cb8e0c9d5cc6ba251493 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 1 Oct 2023 22:30:47 +0200
Subject: [PATCH 16/32] Use POST + body to get around URL length limit

---
 javascript/_utils.js               | 18 ++++++++++++------
 javascript/tagAutocomplete.js      |  5 ++---
 scripts/tag_autocomplete_helper.py | 17 ++++++++++++-----
 3 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 09f7220..10917a9 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -81,8 +81,12 @@ async function fetchAPI(url, json = true, cache = false) {
         return await response.text();
 }
 
-async function postAPI(url, body) {
-    let response = await fetch(url, { method: "POST", body: body });
+async function postAPI(url, body = null) {
+    let response = await fetch(url, {
+        method: "POST",
+        headers: {'Content-Type': 'application/json'},
+        body: body
+    });
 
     if (response.status != 200) {
         console.error(`Error posting to API endpoint "${url}": ` + response.status, response.statusText);
@@ -92,7 +96,7 @@ async function postAPI(url, body) {
     return await response.json();
 }
 
-async function putAPI(url, body) {
+async function putAPI(url, body = null) {
     let response = await fetch(url, { method: "PUT", body: body });
     
     if (response.status != 200) {
@@ -179,14 +183,16 @@ function mapUseCountArray(useCounts) {
 }
 // Call API endpoint to increase bias of tag in the database
 async function increaseUseCount(tagName, type) {
-    await postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`, null);
+    await postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`);
 }
 // Get use count of tag from the database
 async function getUseCount(tagName, type) {
     return (await fetchAPI(`tacapi/v1/get-use-count?tagname=${tagName}&ttype=${type}`, true, false))["result"];
 }
 async function getUseCounts(tagNames, types) {
-    const rawArray = (await fetchAPI(`tacapi/v1/get-use-count-list?tags=${tagNames.join("&tags=")}&ttypes=${types.join("&ttypes=")}`))["result"]
+    // While semantically weird, we have to use POST here for the body, as urls are limited in length
+    const body = JSON.stringify({"tagNames": tagNames, "tagTypes": types});
+    const rawArray = (await postAPI(`tacapi/v1/get-use-count-list`, body))["result"]
     return mapUseCountArray(rawArray);
 }
 async function getAllUseCounts() {
@@ -194,7 +200,7 @@ async function getAllUseCounts() {
     return mapUseCountArray(rawArray);
 }
 async function resetUseCount(tagName, type) {
-    await putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}`, null);
+    await putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}`);
 }
 
 // Sliding window function to get possible combination groups of an array
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index eacbdcd..44b8616 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -1158,8 +1158,7 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
         // Split our results into a list of names and types
         let names = [];
         let types = [];
-        // We need to limit size for the request url
-        results.slice(0, 100).forEach(r => {
+        results.forEach(r => {
             const name = r.type === ResultType.chant ? r.aliases : r.text;
             names.push(name);
             types.push(r.type);
@@ -1324,7 +1323,7 @@ async function refreshTacTempFiles(api = false) {
     }
     
     if (api) {
-        await postAPI("tacapi/v1/refresh-temp-files", null);
+        await postAPI("tacapi/v1/refresh-temp-files");
         await reload();
     } else {
         setTimeout(async () => {
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 46b9081..e9c2701 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -10,9 +10,10 @@
 
 import gradio as gr
 import yaml
-from fastapi import FastAPI, Query
+from fastapi import FastAPI
 from fastapi.responses import FileResponse, JSONResponse
 from modules import script_callbacks, sd_hijack, shared
+from pydantic import BaseModel
 
 from scripts.model_keyword_support import (get_lora_simple_hash,
                                            load_hash_cache, update_hash_cache,
@@ -611,10 +612,16 @@ async def increase_use_count(tagname: str, ttype: int):
     @app.get("/tacapi/v1/get-use-count")
     async def get_use_count(tagname: str, ttype: int):
         return db_request(lambda: db.get_tag_count(tagname, ttype), get=True)
-        
-    @app.get("/tacapi/v1/get-use-count-list")
-    async def get_use_count_list(tags: list[str] | None = Query(default=None), ttypes: list[int] | None = Query(default=None)):
-        return db_request(lambda: list(db.get_tag_counts(tags, ttypes)), get=True)
+    
+    # Small dataholder class
+    class UseCountListRequest(BaseModel):
+        tagNames: list[str]
+        tagTypes: list[int]
+
+    # Semantically weird to use post here, but it's required for the body on js side
+    @app.post("/tacapi/v1/get-use-count-list")
+    async def get_use_count_list(body: UseCountListRequest):
+        return db_request(lambda: list(db.get_tag_counts(body.tagNames, body.tagTypes)), get=True)
 
     @app.put("/tacapi/v1/reset-use-count")
     async def reset_use_count(tagname: str, ttype: int):

From ffc0e378d3fd6092c737abd1c3e346c300859393 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 1 Oct 2023 22:44:35 +0200
Subject: [PATCH 17/32] Add different sorting functions

---
 javascript/_utils.js               | 11 ++++++++++-
 javascript/tagAutocomplete.js      |  1 +
 scripts/tag_autocomplete_helper.py |  5 +++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 10917a9..9aae894 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -175,7 +175,16 @@ function flatten(obj, roots = [], sep = ".") {
 
 // Calculate biased tag score based on post count and frequent usage
 function calculateUsageBias(count, uses) {
-    return Math.log(1 + count) + Math.log(1 + uses);
+    switch (TAC_CFG.frequencyFunction) {
+        case "Logarithmic (weak)":
+            return Math.log(1 + count) + Math.log(1 + uses);
+        case "Logarithmic (strong)":
+            return Math.log(1 + count) + 2 * Math.log(1 + uses);
+        case "Usage first":
+            return uses;
+        default:
+            return count;
+    }
 }
 // Beautify return type for easier parsing
 function mapUseCountArray(useCounts) {
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 44b8616..d918711 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -223,6 +223,7 @@ async function syncOptions() {
         showExtraNetworkPreviews: opts["tac_showExtraNetworkPreviews"],
         modelSortOrder: opts["tac_modelSortOrder"],
         frequencySort: opts["tac_frequencySort"],
+        frequencyFunction: opts["tac_frequencyFunction"],
         // Insertion related settings
         replaceUnderscores: opts["tac_replaceUnderscores"],
         escapeParentheses: opts["tac_escapeParentheses"],
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index e9c2701..25baa9c 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -399,7 +399,8 @@ def needs_restart(self):
 
     # Dictionary of function options and their explanations
     frequency_sort_functions = {
-        "Logarithmic": "Will respect the base order and slightly prefer more frequent tags",
+        "Logarithmic (weak)": "Will respect the base order and slightly prefer often used tags",
+        "Logarithmic (strong)": "Same as Logarithmic (weak), but with a stronger bias",
         "Usage first": "Will list used tags by frequency before all others",
     }
 
@@ -431,7 +432,7 @@ def needs_restart(self):
         "tac_showExtraNetworkPreviews": shared.OptionInfo(True, "Show preview thumbnails for extra networks if available"),
         "tac_modelSortOrder": shared.OptionInfo("Name", "Model sort order", gr.Dropdown, lambda: {"choices": list(sort_criteria.keys())}).info("Order for extra network models and wildcards in dropdown"),
         "tac_frequencySort": shared.OptionInfo(True, "Locally record tag usage and sort frequent tags higher").info("Will also work for extra networks, keeping the specified base order"),
-        "tac_frequencyFunction": shared.OptionInfo("Logarithmic", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
+        "tac_frequencyFunction": shared.OptionInfo("Logarithmic (weak)", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
         # Insertion related settings
         "tac_replaceUnderscores": shared.OptionInfo(True, "Replace underscores with spaces on insertion"),
         "tac_escapeParentheses": shared.OptionInfo(True, "Escape parentheses on insertion"),

From 04551a8132220168a868caace7b61480f3418556 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 1 Oct 2023 22:59:28 +0200
Subject: [PATCH 18/32] Don't await increase, limit to 2k for performance

---
 javascript/tagAutocomplete.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index d918711..2e5b2d0 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -491,7 +491,7 @@ async function insertTextAtCursor(textArea, result, tagword, tabCompletedWithout
             // Sanitize name for API call
             name = encodeURIComponent(name)
             // Call API & update db
-            await increaseUseCount(name, tagType)
+            increaseUseCount(name, tagType)
         }
     }
 
@@ -1159,7 +1159,8 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
         // Split our results into a list of names and types
         let names = [];
         let types = [];
-        results.forEach(r => {
+        // Limit to 2k for performance reasons
+        results.slice(0,2000).forEach(r => {
             const name = r.type === ResultType.chant ? r.aliases : r.text;
             names.push(name);
             types.push(r.type);

From 363895494b6fd9baf1d966144008e7384b249ee1 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 1 Oct 2023 23:17:12 +0200
Subject: [PATCH 19/32] Fix hide after insert race condition

---
 javascript/_utils.js          | 4 ++--
 javascript/tagAutocomplete.js | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 9aae894..04321bb 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -191,8 +191,8 @@ function mapUseCountArray(useCounts) {
     return useCounts.map(useCount => {return {"name": useCount[0], "type": useCount[1], "count": useCount[2]}});
 }
 // Call API endpoint to increase bias of tag in the database
-async function increaseUseCount(tagName, type) {
-    await postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`);
+function increaseUseCount(tagName, type) {
+    postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`);
 }
 // Get use count of tag from the database
 async function getUseCount(tagName, type) {
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 2e5b2d0..bd46c95 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -588,7 +588,8 @@ async function insertTextAtCursor(textArea, result, tagword, tabCompletedWithout
 
     // Since we've modified a Gradio Textbox component manually, we need to simulate an `input` DOM event to ensure it's propagated back to python.
     // Uses a built-in method from the webui's ui.js which also already accounts for event target
-    tacSelfTrigger = true;
+    if (tagType === ResultType.wildcardTag || tagType === ResultType.wildcardFile || tagType === ResultType.yamlWildcard)
+        tacSelfTrigger = true;
     updateInput(textArea);
 
     // Update previous tags with the edited prompt to prevent re-searching the same term

From 3108daf0e8fd307f49bedc7e35f08d546ba64259 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sun, 1 Oct 2023 23:51:16 +0200
Subject: [PATCH 20/32] Remove kaomoji inclusion in < search

because it interfered with use count searching and is not commonly needed
---
 javascript/tagAutocomplete.js | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index bd46c95..1d0048d 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -1065,27 +1065,6 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
         // Sort results, but not if it's umi tags since they are sorted by count
         if (!(resultCandidates.length === 1 && results[0].type === ResultType.umiWildcard))
             results = results.sort(getSortFunction());
-
-        // Since some tags are kaomoji, we have to add the normal results in some cases
-        if (tagword.startsWith("<") || tagword.startsWith("*<")) {
-            // Create escaped search regex with support for * as a start placeholder
-            let searchRegex;
-            if (tagword.startsWith("*")) {
-                tagword = tagword.slice(1);
-                searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i');
-            } else {
-                searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i');
-            }
-            let genericResults = allTags.filter(x => x[0].toLowerCase().search(searchRegex) > -1).slice(0, TAC_CFG.maxResults);
-
-            genericResults.forEach(g => {
-                let result = new AutocompleteResult(g[0].trim(), ResultType.tag)
-                result.category = g[1];
-                result.count = g[2];
-                result.aliases = g[3];
-                results.push(result);
-            });
-        }
     }
     // Else search the normal tag list
     if (!resultCandidates || resultCandidates.length === 0

From bd0ddfbb24705995c84d7dd9ff7e1ada9b8f4d74 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Mon, 2 Oct 2023 00:16:58 +0200
Subject: [PATCH 21/32] Fix embeddings not at top (only affecting the "include
 embeddings in normal results" option)

---
 javascript/tagAutocomplete.js | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 1d0048d..19714f0 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -1158,8 +1158,14 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
             const aUseStats = counts.find(c => c.name === aName && c.type === a.type);
             const bUseStats = counts.find(c => c.name === bName && c.type === b.type);
 
-            const aWeight = calculateUsageBias(a.count || 0, aUseStats ? aUseStats.count : 0);
-            const bWeight = calculateUsageBias(b.count || 0, bUseStats ? bUseStats.count : 0);
+            let aNoCountFallback = 0;
+            let bNoCountFallback = 0;
+            if (TAC_CFG.includeEmbeddingsInNormalResults) {
+                aNoCountFallback = a.type === ResultType.embedding ? Infinity : 0;
+                bNoCountFallback = b.type === ResultType.embedding ? Infinity : 0;
+            }
+            const aWeight = calculateUsageBias(a.count || aNoCountFallback, aUseStats ? aUseStats.count : 0);
+            const bWeight = calculateUsageBias(b.count || bNoCountFallback, bUseStats ? bUseStats.count : 0);
 
             return bWeight - aWeight;
         });

From 7128efc4f4f4df8863ca9dc95cae40a19d87adaa Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Mon, 2 Oct 2023 00:41:38 +0200
Subject: [PATCH 22/32] Apply same fix to extra tags Count now defaults to max
 safe integer, which simplifies the sort function Before, it resulted in
 really bad performance

---
 javascript/_result.js         |  2 +-
 javascript/tagAutocomplete.js | 12 +++---------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/javascript/_result.js b/javascript/_result.js
index 7c32f08..45c8f0a 100644
--- a/javascript/_result.js
+++ b/javascript/_result.js
@@ -23,7 +23,7 @@ class AutocompleteResult {
 
     // Additional info, only used in some cases
     category = null;
-    count = null;
+    count = Number.MAX_SAFE_INTEGER;
     usageBias = null;
     aliases = null;
     meta = null;
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 19714f0..d197fb7 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -747,7 +747,7 @@ function addResultsToList(textArea, results, tagword, resetList) {
         }
 
         // Post count
-        if (result.count && !isNaN(result.count)) {
+        if (result.count && !isNaN(result.count) && result.count !== Number.MAX_SAFE_INTEGER) {
             let postCount = result.count;
             let formatter;
 
@@ -1158,14 +1158,8 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
             const aUseStats = counts.find(c => c.name === aName && c.type === a.type);
             const bUseStats = counts.find(c => c.name === bName && c.type === b.type);
 
-            let aNoCountFallback = 0;
-            let bNoCountFallback = 0;
-            if (TAC_CFG.includeEmbeddingsInNormalResults) {
-                aNoCountFallback = a.type === ResultType.embedding ? Infinity : 0;
-                bNoCountFallback = b.type === ResultType.embedding ? Infinity : 0;
-            }
-            const aWeight = calculateUsageBias(a.count || aNoCountFallback, aUseStats ? aUseStats.count : 0);
-            const bWeight = calculateUsageBias(b.count || bNoCountFallback, bUseStats ? bUseStats.count : 0);
+            const aWeight = calculateUsageBias(a.count, aUseStats ? aUseStats.count : 0);
+            const bWeight = calculateUsageBias(b.count, bUseStats ? bUseStats.count : 0);
 
             return bWeight - aWeight;
         });

From 15478e73b57046e4b77bc14910452aca9f7b7cf5 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Wed, 29 Nov 2023 15:20:15 +0100
Subject: [PATCH 23/32] Count positive / negative prompt usage separately

---
 javascript/_utils.js               | 20 +++++------
 javascript/tagAutocomplete.js      | 13 +++++--
 scripts/tag_autocomplete_helper.py | 19 +++++-----
 scripts/tag_frequency_db.py        | 58 +++++++++++++++++++-----------
 4 files changed, 68 insertions(+), 42 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 1c6d978..45ff5e8 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -190,25 +190,25 @@ function mapUseCountArray(useCounts) {
     return useCounts.map(useCount => {return {"name": useCount[0], "type": useCount[1], "count": useCount[2]}});
 }
 // Call API endpoint to increase bias of tag in the database
-function increaseUseCount(tagName, type) {
-    postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}`);
+function increaseUseCount(tagName, type, negative = false) {
+    postAPI(`tacapi/v1/increase-use-count?tagname=${tagName}&ttype=${type}&neg=${negative}`);
 }
 // Get use count of tag from the database
-async function getUseCount(tagName, type) {
-    return (await fetchAPI(`tacapi/v1/get-use-count?tagname=${tagName}&ttype=${type}`, true, false))["result"];
+async function getUseCount(tagName, type, negative = false) {
+    return (await fetchAPI(`tacapi/v1/get-use-count?tagname=${tagName}&ttype=${type}&neg=${negative}`, true, false))["result"];
 }
-async function getUseCounts(tagNames, types) {
+async function getUseCounts(tagNames, types, negative = false) {
     // While semantically weird, we have to use POST here for the body, as urls are limited in length
-    const body = JSON.stringify({"tagNames": tagNames, "tagTypes": types});
+    const body = JSON.stringify({"tagNames": tagNames, "tagTypes": types, "neg": negative});
     const rawArray = (await postAPI(`tacapi/v1/get-use-count-list`, body))["result"]
     return mapUseCountArray(rawArray);
 }
-async function getAllUseCounts() {
-    const rawArray = (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
+async function getAllUseCounts(negative = false) {
+    const rawArray = (await fetchAPI(`tacapi/v1/get-all-use-counts?neg=${negative}`))["result"];
     return mapUseCountArray(rawArray);
 }
-async function resetUseCount(tagName, type) {
-    await putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}`);
+async function resetUseCount(tagName, type, resetPosCount, resetNegCount) {
+    await putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}&pos=${resetPosCount}&neg=${resetNegCount}`);
 }
 
 // Sliding window function to get possible combination groups of an array
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 75a51de..b28f3f0 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -1,4 +1,4 @@
-const styleColors = {
+const styleColors = {
     "--results-bg": ["#0b0f19", "#ffffff"],
     "--results-border-color": ["#4b5563", "#e5e7eb"],
     "--results-border-width": ["1px", "1.5px"],
@@ -488,10 +488,13 @@ async function insertTextAtCursor(textArea, result, tagword, tabCompletedWithout
         }
 
         if (name && name.length > 0) {
+            // Check if it's a negative prompt
+            let textAreaId = getTextAreaIdentifier(textArea);
+            let isNegative = textAreaId.includes("n");
             // Sanitize name for API call
             name = encodeURIComponent(name)
             // Call API & update db
-            increaseUseCount(name, tagType)
+            increaseUseCount(name, tagType, isNegative)
         }
     }
 
@@ -1160,8 +1163,12 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
             types.push(r.type);
         });
 
+        // Check if it's a negative prompt
+        let textAreaId = getTextAreaIdentifier(textArea);
+        let isNegative = textAreaId.includes("n");
+
         // Request use counts from the DB
-        const counts = await getUseCounts(names, types);
+        const counts = await getUseCounts(names, types, isNegative);
         const usedResults = counts.filter(c => c.count > 0).map(c => c.name);
 
         // Sort all
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index d6b044e..43f6c39 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -607,29 +607,30 @@ def db_request(func, get = False):
             return JSONResponse({"error": "Database not initialized"}, status_code=500)
 
     @app.post("/tacapi/v1/increase-use-count")
-    async def increase_use_count(tagname: str, ttype: int):
-        db_request(lambda: db.increase_tag_count(tagname, ttype))
+    async def increase_use_count(tagname: str, ttype: int, neg: bool):
+        db_request(lambda: db.increase_tag_count(tagname, ttype, neg))
 
     @app.get("/tacapi/v1/get-use-count")
-    async def get_use_count(tagname: str, ttype: int):
-        return db_request(lambda: db.get_tag_count(tagname, ttype), get=True)
+    async def get_use_count(tagname: str, ttype: int, neg: bool):
+        return db_request(lambda: db.get_tag_count(tagname, ttype, neg), get=True)
     
     # Small dataholder class
     class UseCountListRequest(BaseModel):
         tagNames: list[str]
         tagTypes: list[int]
+        neg: bool = False
 
     # Semantically weird to use post here, but it's required for the body on js side
     @app.post("/tacapi/v1/get-use-count-list")
     async def get_use_count_list(body: UseCountListRequest):
-        return db_request(lambda: list(db.get_tag_counts(body.tagNames, body.tagTypes)), get=True)
+        return db_request(lambda: list(db.get_tag_counts(body.tagNames, body.tagTypes, body.neg)), get=True)
 
     @app.put("/tacapi/v1/reset-use-count")
-    async def reset_use_count(tagname: str, ttype: int):
-        db_request(lambda: db.reset_tag_count(tagname, ttype))
+    async def reset_use_count(tagname: str, ttype: int, pos: bool, neg: bool):
+        db_request(lambda: db.reset_tag_count(tagname, ttype, pos, neg))
 
     @app.get("/tacapi/v1/get-all-use-counts")
-    async def get_all_tag_counts():
-        return db_request(lambda: db.get_all_tags(), get=True)
+    async def get_all_tag_counts(neg: bool = False):
+        return db_request(lambda: db.get_all_tags(neg), get=True)
         
 script_callbacks.on_app_started(api_tac)
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index 71862e6..4f4f26b 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -55,7 +55,8 @@ def __create_db(self, cursor: sqlite3.Cursor):
         CREATE TABLE IF NOT EXISTS tag_frequency (
             name TEXT NOT NULL,
             type INT NOT NULL,
-            count INT,
+            count_pos INT,
+            count_neg INT,
             last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
             PRIMARY KEY (name, type)
         )
@@ -85,24 +86,26 @@ def __get_version(self):
 
         return db_version[0] if db_version else 0
 
-    def get_all_tags(self):
+    def get_all_tags(self, negative=False):
+        count_str = "count_neg" if negative else "count_pos"
         with transaction() as cursor:
             cursor.execute(
-                """
-            SELECT name, type, count
+                f"""
+            SELECT name, type, {count_str}
             FROM tag_frequency
-            ORDER BY count DESC
+            ORDER BY {count_str} DESC
             """
             )
             tags = cursor.fetchall()
 
         return tags
 
-    def get_tag_count(self, tag, ttype):
+    def get_tag_count(self, tag, ttype, negative=False):
+        count_str = "count_neg" if negative else "count_pos"
         with transaction() as cursor:
             cursor.execute(
-                """
-            SELECT count
+                f"""
+            SELECT {count_str}
             FROM tag_frequency
             WHERE name = ? AND type = ?
             """,
@@ -112,12 +115,13 @@ def get_tag_count(self, tag, ttype):
 
         return tag_count[0] if tag_count else 0
 
-    def get_tag_counts(self, tags: list[str], ttypes: list[str]):
+    def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False):
+        count_str = "count_neg" if negative else "count_pos"
         with transaction() as cursor:
             for tag, ttype in zip(tags, ttypes):
                 cursor.execute(
-                    """
-                SELECT count
+                    f"""
+                SELECT {count_str}
                 FROM tag_frequency
                 WHERE name = ? AND type = ?
                 """,
@@ -126,24 +130,38 @@ def get_tag_counts(self, tags: list[str], ttypes: list[str]):
                 tag_count = cursor.fetchone()
                 yield (tag, ttype, tag_count[0]) if tag_count else (tag, ttype, 0)
 
-    def increase_tag_count(self, tag, ttype):
-        current_count = self.get_tag_count(tag, ttype)
+    def increase_tag_count(self, tag, ttype, negative=False):
+        pos_count = self.get_tag_count(tag, ttype, False)
+        neg_count = self.get_tag_count(tag, ttype, True)
+
+        if negative:
+            neg_count += 1
+        else:
+            pos_count += 1
+
         with transaction() as cursor:
             cursor.execute(
-                """
+                f"""
             INSERT OR REPLACE
-            INTO tag_frequency (name, type, count)
-            VALUES (?, ?, ?)
+            INTO tag_frequency (name, type, count_pos, count_neg)
+            VALUES (?, ?, ?, ?)
             """,
-                (tag, ttype, current_count + 1),
+                (tag, ttype, pos_count, neg_count),
             )
 
-    def reset_tag_count(self, tag, ttype):
+    def reset_tag_count(self, tag, ttype, positive=True, negative=False):
+        if positive and negative:
+            set_str = "count_pos = 0, count_neg = 0"
+        elif positive:
+            set_str = "count_pos = 0"
+        elif negative:
+            set_str = "count_neg = 0"
+        
         with transaction() as cursor:
             cursor.execute(
-                """
+                f"""
             UPDATE tag_frequency
-            SET count = 0
+            SET {set_str}
             WHERE name = ? AND type = ?
             """,
                 (tag,ttype),

From a156214a48e194c3690f04298124106b7df8f53f Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Wed, 29 Nov 2023 17:45:51 +0100
Subject: [PATCH 24/32] Last used & min count settings Also some performance
 improvements

---
 javascript/_utils.js               | 20 ++++++++++++++++++--
 javascript/tagAutocomplete.js      | 20 +++++++++-----------
 scripts/tag_autocomplete_helper.py |  4 +++-
 scripts/tag_frequency_db.py        | 22 +++++++++++-----------
 4 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 45ff5e8..fc823a6 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -173,7 +173,16 @@ function flatten(obj, roots = [], sep = ".") {
 }
 
 // Calculate biased tag score based on post count and frequent usage
-function calculateUsageBias(count, uses) {
+function calculateUsageBias(result, count, uses, lastUseDate) {
+    // Guard for minimum usage count & last usage date
+    const diffTime = Math.abs(Date.now() - (lastUseDate || Date.now()));
+    const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
+    if (uses < TAC_CFG.frequencyMinCount || diffDays > TAC_CFG.frequencyMaxAge) {
+        uses = 0;
+    } else if (uses != 0) {
+        result.usageBias = true;
+    }
+
     switch (TAC_CFG.frequencyFunction) {
         case "Logarithmic (weak)":
             return Math.log(1 + count) + Math.log(1 + uses);
@@ -187,7 +196,14 @@ function calculateUsageBias(count, uses) {
 }
 // Beautify return type for easier parsing
 function mapUseCountArray(useCounts) {
-    return useCounts.map(useCount => {return {"name": useCount[0], "type": useCount[1], "count": useCount[2]}});
+    return useCounts.map(useCount => {
+        return {
+            "name": useCount[0],
+            "type": useCount[1],
+            "count": useCount[2],
+            "lastUseDate": useCount[3]
+        }
+    });
 }
 // Call API endpoint to increase bias of tag in the database
 function increaseUseCount(tagName, type, negative = false) {
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index b28f3f0..5b29ddb 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -1,4 +1,4 @@
-const styleColors = {
+const styleColors = {
     "--results-bg": ["#0b0f19", "#ffffff"],
     "--results-border-color": ["#4b5563", "#e5e7eb"],
     "--results-border-width": ["1px", "1.5px"],
@@ -224,6 +224,8 @@ async function syncOptions() {
         modelSortOrder: opts["tac_modelSortOrder"],
         frequencySort: opts["tac_frequencySort"],
         frequencyFunction: opts["tac_frequencyFunction"],
+        frequencyMinCount: opts["tac_frequencyMinCount"],
+        frequencyMaxAge: opts["tac_frequencyMaxAge"],
         // Insertion related settings
         replaceUnderscores: opts["tac_replaceUnderscores"],
         escapeParentheses: opts["tac_escapeParentheses"],
@@ -1169,7 +1171,6 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
 
         // Request use counts from the DB
         const counts = await getUseCounts(names, types, isNegative);
-        const usedResults = counts.filter(c => c.count > 0).map(c => c.name);
 
         // Sort all
         results = results.sort((a, b) => {
@@ -1178,19 +1179,16 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
 
             const aUseStats = counts.find(c => c.name === aName && c.type === a.type);
             const bUseStats = counts.find(c => c.name === bName && c.type === b.type);
+            const aUses = aUseStats?.count || 0;
+            const bUses = bUseStats?.count || 0;
+            const aLastUseDate = Date.parse(aUseStats?.lastUseDate);
+            const bLastUseDate = Date.parse(bUseStats?.lastUseDate);
 
-            const aWeight = calculateUsageBias(a.count, aUseStats ? aUseStats.count : 0);
-            const bWeight = calculateUsageBias(b.count, bUseStats ? bUseStats.count : 0);
+            const aWeight = calculateUsageBias(a, a.count, aUses, aLastUseDate);
+            const bWeight = calculateUsageBias(b, b.count, bUses, bLastUseDate);
 
             return bWeight - aWeight;
         });
-
-        // Mark results
-        results.forEach(r => {
-            const name = r.type === ResultType.chant ? r.aliases : r.text;
-            if (usedResults.includes(name))
-                r.usageBias = true;
-        });
     }
 
     // Slice if the user has set a max result count and we are not in a extra networks / wildcard list
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 43f6c39..1ef64e8 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -421,6 +421,8 @@ def needs_restart(self):
         "tac_modelSortOrder": shared.OptionInfo("Name", "Model sort order", gr.Dropdown, lambda: {"choices": list(sort_criteria.keys())}).info("Order for extra network models and wildcards in dropdown"),
         "tac_frequencySort": shared.OptionInfo(True, "Locally record tag usage and sort frequent tags higher").info("Will also work for extra networks, keeping the specified base order"),
         "tac_frequencyFunction": shared.OptionInfo("Logarithmic (weak)", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
+        "tac_frequencyMinCount": shared.OptionInfo(3, "Minimum number of uses for a tag to be considered frequent").info("Tags with less uses than this will not be sorted higher, even if the sorting function would normally result in a higher position."),
+        "tac_frequencyMaxAge": shared.OptionInfo(30, "Maximum days since last use for a tag to be considered frequent").info("Similar to the above, tags that haven't been used in this many days will not be sorted higher."),
         # Insertion related settings
         "tac_replaceUnderscores": shared.OptionInfo(True, "Replace underscores with spaces on insertion"),
         "tac_escapeParentheses": shared.OptionInfo(True, "Escape parentheses on insertion"),
@@ -597,7 +599,7 @@ def db_request(func, get = False):
                 if get:
                     ret = func()
                     if ret is list:
-                        ret = [{"name": t[0], "type": t[1], "count": t[2]} for t in ret]
+                        ret = [{"name": t[0], "type": t[1], "count": t[2], "lastUseDate": t[3]} for t in ret]
                     return JSONResponse({"result": ret})
                 else:
                     func()
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index 4f4f26b..c0fae9d 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -91,7 +91,7 @@ def get_all_tags(self, negative=False):
         with transaction() as cursor:
             cursor.execute(
                 f"""
-            SELECT name, type, {count_str}
+            SELECT name, type, {count_str}, last_used
             FROM tag_frequency
             ORDER BY {count_str} DESC
             """
@@ -105,15 +105,15 @@ def get_tag_count(self, tag, ttype, negative=False):
         with transaction() as cursor:
             cursor.execute(
                 f"""
-            SELECT {count_str}
+            SELECT {count_str}, last_used
             FROM tag_frequency
             WHERE name = ? AND type = ?
             """,
-                (tag,ttype),
+                (tag, ttype),
             )
             tag_count = cursor.fetchone()
 
-        return tag_count[0] if tag_count else 0
+        return tag_count[0], tag_count[1] if tag_count else 0
 
     def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False):
         count_str = "count_neg" if negative else "count_pos"
@@ -121,18 +121,18 @@ def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False):
             for tag, ttype in zip(tags, ttypes):
                 cursor.execute(
                     f"""
-                SELECT {count_str}
+                SELECT {count_str}, last_used
                 FROM tag_frequency
                 WHERE name = ? AND type = ?
                 """,
-                    (tag,ttype),
+                    (tag, ttype),
                 )
                 tag_count = cursor.fetchone()
-                yield (tag, ttype, tag_count[0]) if tag_count else (tag, ttype, 0)
+                yield (tag, ttype, tag_count[0], tag_count[1]) if tag_count else (tag, ttype, 0)
 
     def increase_tag_count(self, tag, ttype, negative=False):
-        pos_count = self.get_tag_count(tag, ttype, False)
-        neg_count = self.get_tag_count(tag, ttype, True)
+        pos_count = self.get_tag_count(tag, ttype, False)[0]
+        neg_count = self.get_tag_count(tag, ttype, True)[0]
 
         if negative:
             neg_count += 1
@@ -156,7 +156,7 @@ def reset_tag_count(self, tag, ttype, positive=True, negative=False):
             set_str = "count_pos = 0"
         elif negative:
             set_str = "count_neg = 0"
-        
+
         with transaction() as cursor:
             cursor.execute(
                 f"""
@@ -164,5 +164,5 @@ def reset_tag_count(self, tag, ttype, positive=True, negative=False):
             SET {set_str}
             WHERE name = ? AND type = ?
             """,
-                (tag,ttype),
+                (tag, ttype),
             )

From 4df90f5c95d8727197692dfb1c7689d7d443222a Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Wed, 29 Nov 2023 18:04:50 +0100
Subject: [PATCH 25/32] Don't frequency sort alias results by default with an
 option to enable it if desired

---
 javascript/_utils.js               | 9 +++++++--
 javascript/tagAutocomplete.js      | 1 +
 scripts/tag_autocomplete_helper.py | 1 +
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index fc823a6..3cd07a5 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -174,10 +174,15 @@ function flatten(obj, roots = [], sep = ".") {
 
 // Calculate biased tag score based on post count and frequent usage
 function calculateUsageBias(result, count, uses, lastUseDate) {
-    // Guard for minimum usage count & last usage date
+    // Calculate days since last use
     const diffTime = Math.abs(Date.now() - (lastUseDate || Date.now()));
     const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
-    if (uses < TAC_CFG.frequencyMinCount || diffDays > TAC_CFG.frequencyMaxAge) {
+    // Check setting conditions
+    if (
+        uses < TAC_CFG.frequencyMinCount ||
+        diffDays > TAC_CFG.frequencyMaxAge ||
+        (!TAC_CFG.frequencyIncludeAlias && !result.text.includes(tagword))
+    ) {
         uses = 0;
     } else if (uses != 0) {
         result.usageBias = true;
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 5b29ddb..49ba940 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -226,6 +226,7 @@ async function syncOptions() {
         frequencyFunction: opts["tac_frequencyFunction"],
         frequencyMinCount: opts["tac_frequencyMinCount"],
         frequencyMaxAge: opts["tac_frequencyMaxAge"],
+        frequencyIncludeAlias: opts["tac_frequencyIncludeAlias"],
         // Insertion related settings
         replaceUnderscores: opts["tac_replaceUnderscores"],
         escapeParentheses: opts["tac_escapeParentheses"],
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 1ef64e8..6350b80 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -423,6 +423,7 @@ def needs_restart(self):
         "tac_frequencyFunction": shared.OptionInfo("Logarithmic (weak)", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
         "tac_frequencyMinCount": shared.OptionInfo(3, "Minimum number of uses for a tag to be considered frequent").info("Tags with less uses than this will not be sorted higher, even if the sorting function would normally result in a higher position."),
         "tac_frequencyMaxAge": shared.OptionInfo(30, "Maximum days since last use for a tag to be considered frequent").info("Similar to the above, tags that haven't been used in this many days will not be sorted higher."),
+        "tac_frequencyIncludeAlias": shared.OptionInfo(False, "Frequency sorting matches aliases for frequent tags").info("Tag frequency will be increased for the main tag even if an alias is used for completion. This option can be used to override the default behavior of alias results being ignored for frequency sorting."),
         # Insertion related settings
         "tac_replaceUnderscores": shared.OptionInfo(True, "Replace underscores with spaces on insertion"),
         "tac_escapeParentheses": shared.OptionInfo(True, "Escape parentheses on insertion"),

From 2dd48eab791f5b81d7cc816c60373d0bfa537b9c Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Wed, 29 Nov 2023 18:14:14 +0100
Subject: [PATCH 26/32] Fix error with db return value for no matches

---
 scripts/tag_autocomplete_helper.py |  1 +
 scripts/tag_frequency_db.py        | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 6350b80..3a805c8 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -419,6 +419,7 @@ def needs_restart(self):
         "tac_showWikiLinks": shared.OptionInfo(False, "Show '?' next to tags, linking to its Danbooru or e621 wiki page").info("Warning: This is an external site and very likely contains NSFW examples!"),
         "tac_showExtraNetworkPreviews": shared.OptionInfo(True, "Show preview thumbnails for extra networks if available"),
         "tac_modelSortOrder": shared.OptionInfo("Name", "Model sort order", gr.Dropdown, lambda: {"choices": list(sort_criteria.keys())}).info("Order for extra network models and wildcards in dropdown"),
+        # Frequency sorting settings
         "tac_frequencySort": shared.OptionInfo(True, "Locally record tag usage and sort frequent tags higher").info("Will also work for extra networks, keeping the specified base order"),
         "tac_frequencyFunction": shared.OptionInfo("Logarithmic (weak)", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
         "tac_frequencyMinCount": shared.OptionInfo(3, "Minimum number of uses for a tag to be considered frequent").info("Tags with less uses than this will not be sorted higher, even if the sorting function would normally result in a higher position."),
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index c0fae9d..cee0623 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -113,7 +113,10 @@ def get_tag_count(self, tag, ttype, negative=False):
             )
             tag_count = cursor.fetchone()
 
-        return tag_count[0], tag_count[1] if tag_count else 0
+        if tag_count:
+            return tag_count[0], tag_count[1]
+        else:
+            return 0, None
 
     def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False):
         count_str = "count_neg" if negative else "count_pos"
@@ -128,7 +131,10 @@ def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False):
                     (tag, ttype),
                 )
                 tag_count = cursor.fetchone()
-                yield (tag, ttype, tag_count[0], tag_count[1]) if tag_count else (tag, ttype, 0)
+                if tag_count:
+                    yield (tag, ttype, tag_count[0], tag_count[1]) 
+                else:
+                    yield (tag, ttype, 0, None)
 
     def increase_tag_count(self, tag, ttype, negative=False):
         pos_count = self.get_tag_count(tag, ttype, False)[0]

From e82e958c3e2e1971d0c0df786ecfd32adc7a0906 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Wed, 29 Nov 2023 18:15:59 +0100
Subject: [PATCH 27/32] Fix alias check for non-aliased tag types

---
 javascript/_utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 3cd07a5..8307a92 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -181,7 +181,7 @@ function calculateUsageBias(result, count, uses, lastUseDate) {
     if (
         uses < TAC_CFG.frequencyMinCount ||
         diffDays > TAC_CFG.frequencyMaxAge ||
-        (!TAC_CFG.frequencyIncludeAlias && !result.text.includes(tagword))
+        (!TAC_CFG.frequencyIncludeAlias && result.aliases && !result.text.includes(tagword))
     ) {
         uses = 0;
     } else if (uses != 0) {

From 1fe8f26670be2e30b18ea1383b297112a8454b64 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Mon, 4 Dec 2023 13:56:15 +0100
Subject: [PATCH 28/32] Add explanatory tooltip and inline reset ability Also
 add tooltip for wiki links

---
 javascript/tagAutocomplete.js | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index 49ba940..4a9101e 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -722,6 +722,7 @@ function addResultsToList(textArea, results, tagword, resetList) {
             let wikiLink = document.createElement("a");
             wikiLink.classList.add("acWikiLink");
             wikiLink.innerText = "?";
+            wikiLink.title = "Open external wiki page for this tag"
 
             let linkPart = displayText;
             // Only use alias result if it is one
@@ -802,10 +803,21 @@ function addResultsToList(textArea, results, tagword, resetList) {
         // Add small ✨ marker to indicate usage sorting
         if (result.usageBias) {
             flexDiv.querySelector(".acMetaText").classList.add("biased");
+            flexDiv.title = "✨ Frequent tag. Ctrl/Cmd + click to reset usage count."
         }
 
+        // Check if it's a negative prompt
+        let isNegative = textAreaId.includes("n");
+
         // Add listener
-        li.addEventListener("click", function () { insertTextAtCursor(textArea, result, tagword); });
+        li.addEventListener("click", (e) => {
+            if (e.ctrlKey || e.metaKey) {
+                resetUseCount(result.text, result.type, !isNegative, isNegative);
+                flexDiv.querySelector(".acMetaText").classList.remove("biased");
+            } else {
+                insertTextAtCursor(textArea, result, tagword);
+            }
+        });
         // Add element to list
         resultsList.appendChild(li);
     }

From 20b6635a2a3e10d1f1f2c34239e2178b014e2440 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Mon, 4 Dec 2023 15:00:19 +0100
Subject: [PATCH 29/32] WIP usage info table Might get replaced with gradio
 depending on how well it works

---
 javascript/_utils.js               | 54 +++++++++++++++++++++++++++---
 scripts/tag_autocomplete_helper.py |  4 +--
 scripts/tag_frequency_db.py        |  8 ++---
 3 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 8307a92..6f454db 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -200,8 +200,17 @@ function calculateUsageBias(result, count, uses, lastUseDate) {
     }
 }
 // Beautify return type for easier parsing
-function mapUseCountArray(useCounts) {
+function mapUseCountArray(useCounts, posAndNeg = false) {
     return useCounts.map(useCount => {
+        if (posAndNeg) {
+            return {
+                "name": useCount[0],
+                "type": useCount[1],
+                "count": useCount[2],
+                "negCount": useCount[3],
+                "lastUseDate": useCount[4]
+            }
+        }
         return {
             "name": useCount[0],
             "type": useCount[1],
@@ -224,14 +233,51 @@ async function getUseCounts(tagNames, types, negative = false) {
     const rawArray = (await postAPI(`tacapi/v1/get-use-count-list`, body))["result"]
     return mapUseCountArray(rawArray);
 }
-async function getAllUseCounts(negative = false) {
-    const rawArray = (await fetchAPI(`tacapi/v1/get-all-use-counts?neg=${negative}`))["result"];
-    return mapUseCountArray(rawArray);
+async function getAllUseCounts() {
+    const rawArray = (await fetchAPI(`tacapi/v1/get-all-use-counts`))["result"];
+    return mapUseCountArray(rawArray, true);
 }
 async function resetUseCount(tagName, type, resetPosCount, resetNegCount) {
     await putAPI(`tacapi/v1/reset-use-count?tagname=${tagName}&ttype=${type}&pos=${resetPosCount}&neg=${resetNegCount}`);
 }
 
+function createTagUsageTable(tagCounts) {
+    // Create table
+    let tagTable = document.createElement("table");
+    tagTable.innerHTML =
+    `<thead>
+        <tr>
+            <td>Name</td>
+            <td>Type</td>
+            <td>Count(+)</td>
+            <td>Count(-)</td>
+            <td>Last used</td>
+        </tr>
+    </thead>`;
+    tagTable.id = "tac_tagUsageTable"
+
+    tagCounts.forEach(t => {
+        let tr = document.createElement("tr");
+        
+        // Fill values
+        let values = [t.name, t.type-1, t.count, t.negCount, t.lastUseDate]
+        values.forEach(v => {
+            let td = document.createElement("td");
+            td.innerText = v;
+            tr.append(td);
+        });
+        // Add delete/reset button
+        let delButton = document.createElement("button");
+        delButton.innerText = "🗑️";
+        delButton.title = "Reset count";
+        tr.append(delButton);
+        
+        tagTable.append(tr)
+    });
+
+    return tagTable;
+}
+
 // Sliding window function to get possible combination groups of an array
 function toNgrams(inputArray, size) {
     return Array.from(
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 3a805c8..eba0f7f 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -634,7 +634,7 @@ async def reset_use_count(tagname: str, ttype: int, pos: bool, neg: bool):
         db_request(lambda: db.reset_tag_count(tagname, ttype, pos, neg))
 
     @app.get("/tacapi/v1/get-all-use-counts")
-    async def get_all_tag_counts(neg: bool = False):
-        return db_request(lambda: db.get_all_tags(neg), get=True)
+    async def get_all_tag_counts():
+        return db_request(lambda: db.get_all_tags(), get=True)
         
 script_callbacks.on_app_started(api_tac)
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index cee0623..9a0cd10 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -86,14 +86,14 @@ def __get_version(self):
 
         return db_version[0] if db_version else 0
 
-    def get_all_tags(self, negative=False):
-        count_str = "count_neg" if negative else "count_pos"
+    def get_all_tags(self):
         with transaction() as cursor:
             cursor.execute(
                 f"""
-            SELECT name, type, {count_str}, last_used
+            SELECT name, type, count_pos, count_neg, last_used
             FROM tag_frequency
-            ORDER BY {count_str} DESC
+            WHERE count_pos > 0 OR count_neg > 0
+            ORDER BY count_pos + count_neg DESC
             """
             )
             tags = cursor.fetchall()

From d496569c9a86939d24bcaff4a891f8833b24f810 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Fri, 19 Jan 2024 20:17:14 +0100
Subject: [PATCH 30/32] Cache sort key for small performance increase

---
 javascript/_utils.js | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index 6f454db..c967d2b 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -335,12 +335,19 @@ function getSortFunction() {
     let criterion = TAC_CFG.modelSortOrder || "Name";
 
     const textSort = (a, b, reverse = false) => {
-        const textHolderA = a.type === ResultType.chant ? a.aliases : a.text;
-        const textHolderB = b.type === ResultType.chant ? b.aliases : b.text;
+        // Assign keys so next sort is faster
+        if (!a.sortKey) {
+            a.sortKey = a.type === ResultType.chant
+                ? a.aliases
+                : a.text;
+        }
+        if (!b.sortKey) {
+            b.sortKey = b.type === ResultType.chant
+                ? b.aliases
+                : b.text;
+        }
 
-        const aKey = a.sortKey || textHolderA;
-        const bKey = b.sortKey || textHolderB;
-        return reverse ? bKey.localeCompare(aKey) : aKey.localeCompare(bKey);
+        return reverse ? b.sortKey.localeCompare(a.sortKey) : a.sortKey.localeCompare(b.sortKey);
     }
     const numericSort = (a, b, reverse = false) => {
         const noKey = reverse ? "-1" : Number.MAX_SAFE_INTEGER;

From 342fbc904137c2bcb53ca780e1ba97bb4b47e4e5 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Fri, 19 Jan 2024 21:10:09 +0100
Subject: [PATCH 31/32] Pre-calculate usage bias for all results instead of in
 the sort function Roughly doubles the sort performance

---
 javascript/tagAutocomplete.js | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index ec1a985..afee57b 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -1188,21 +1188,23 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
         const counts = await getUseCounts(names, types, isNegative);
 
         // Sort all
-        results = results.sort((a, b) => {
-            const aName = a.type === ResultType.chant ? a.aliases : a.text;
-            const bName = b.type === ResultType.chant ? b.aliases : b.text;
-
-            const aUseStats = counts.find(c => c.name === aName && c.type === a.type);
-            const bUseStats = counts.find(c => c.name === bName && c.type === b.type);
-            const aUses = aUseStats?.count || 0;
-            const bUses = bUseStats?.count || 0;
-            const aLastUseDate = Date.parse(aUseStats?.lastUseDate);
-            const bLastUseDate = Date.parse(bUseStats?.lastUseDate);
-
-            const aWeight = calculateUsageBias(a, a.count, aUses, aLastUseDate);
-            const bWeight = calculateUsageBias(b, b.count, bUses, bLastUseDate);
 
-            return bWeight - aWeight;
+        // Pre-calculate weights to prevent duplicate work
+        const resultBiasMap = new Map();
+        results.forEach(result => {
+            const name = result.type === ResultType.chant ? result.aliases : result.text;
+            const type = result.type;
+            // Find matching pair from DB results
+            const useStats = counts.find(c => c.name === name && c.type === type);
+            const uses = useStats?.count || 0;
+            const lastUseDate = Date.parse(useStats?.lastUseDate);
+            // Calculate & set weight
+            const weight = calculateUsageBias(result, result.count, uses, lastUseDate)
+            resultBiasMap.set(result, weight);
+        });
+        // Actual sorting with the pre-calculated weights
+        results = results.sort((a, b) => {
+            return resultBiasMap.get(b) - resultBiasMap.get(a);
         });
     }
 

From ef59cff6516be9a9e4d82c5bb1a91376406601f8 Mon Sep 17 00:00:00 2001
From: DominikDoom <rehdominik@gmail.com>
Date: Sat, 16 Mar 2024 16:44:43 +0100
Subject: [PATCH 32/32] Move last used date check guard to SQL side, implement
 max cap - Server side date comparison and cap check further improve js sort
 performance - The alias check has also been moved out of calculateUsageBias
 to support the new cap system

---
 javascript/_utils.js               | 11 ++-------
 javascript/tagAutocomplete.js      | 19 ++++++++++-----
 scripts/tag_autocomplete_helper.py | 18 +++++++++++++--
 scripts/tag_frequency_db.py        | 37 +++++++++++++++++++++---------
 4 files changed, 57 insertions(+), 28 deletions(-)

diff --git a/javascript/_utils.js b/javascript/_utils.js
index a02cbc4..f44aff2 100644
--- a/javascript/_utils.js
+++ b/javascript/_utils.js
@@ -196,16 +196,9 @@ function flatten(obj, roots = [], sep = ".") {
 }
 
 // Calculate biased tag score based on post count and frequent usage
-function calculateUsageBias(result, count, uses, lastUseDate) {
-    // Calculate days since last use
-    const diffTime = Math.abs(Date.now() - (lastUseDate || Date.now()));
-    const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
+function calculateUsageBias(result, count, uses) {
     // Check setting conditions
-    if (
-        uses < TAC_CFG.frequencyMinCount ||
-        diffDays > TAC_CFG.frequencyMaxAge ||
-        (!TAC_CFG.frequencyIncludeAlias && result.aliases && !result.text.includes(tagword))
-    ) {
+    if (uses < TAC_CFG.frequencyMinCount) {
         uses = 0;
     } else if (uses != 0) {
         result.usageBias = true;
diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js
index e221831..e29a983 100644
--- a/javascript/tagAutocomplete.js
+++ b/javascript/tagAutocomplete.js
@@ -229,6 +229,7 @@ async function syncOptions() {
         frequencyFunction: opts["tac_frequencyFunction"],
         frequencyMinCount: opts["tac_frequencyMinCount"],
         frequencyMaxAge: opts["tac_frequencyMaxAge"],
+        frequencyRecommendCap: opts["tac_frequencyRecommendCap"],
         frequencyIncludeAlias: opts["tac_frequencyIncludeAlias"],
         useStyleVars: opts["tac_useStyleVars"],
         // Insertion related settings
@@ -1177,12 +1178,20 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
     // Sort again with frequency / usage count if enabled
     if (TAC_CFG.frequencySort) {
         // Split our results into a list of names and types
-        let names = [];
+        let tagNames = [];
+        let aliasNames = [];
         let types = [];
         // Limit to 2k for performance reasons
+        const aliasTypes = [ResultType.tag, ResultType.extra];
         results.slice(0,2000).forEach(r => {
             const name = r.type === ResultType.chant ? r.aliases : r.text;
-            names.push(name);
+            // Add to alias list or tag list depending on if the name includes the tagword
+            // (the same criteria is used in the filter in calculateUsageBias)
+            if (aliasTypes.includes(r.type) && !name.includes(tagword)) {
+                aliasNames.push(name);
+            } else {
+                tagNames.push(name);
+            }
             types.push(r.type);
         });
 
@@ -1191,10 +1200,9 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
         let isNegative = textAreaId.includes("n");
 
         // Request use counts from the DB
+        const names = TAC_CFG.frequencyIncludeAlias ? tagNames.concat(aliasNames) : tagNames;
         const counts = await getUseCounts(names, types, isNegative);
 
-        // Sort all
-
         // Pre-calculate weights to prevent duplicate work
         const resultBiasMap = new Map();
         results.forEach(result => {
@@ -1203,9 +1211,8 @@ async function autocomplete(textArea, prompt, fixedTag = null) {
             // Find matching pair from DB results
             const useStats = counts.find(c => c.name === name && c.type === type);
             const uses = useStats?.count || 0;
-            const lastUseDate = Date.parse(useStats?.lastUseDate);
             // Calculate & set weight
-            const weight = calculateUsageBias(result, result.count, uses, lastUseDate)
+            const weight = calculateUsageBias(result, result.count, uses)
             resultBiasMap.set(result, weight);
         });
         // Actual sorting with the pre-calculated weights
diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py
index 981bd5d..925f138 100644
--- a/scripts/tag_autocomplete_helper.py
+++ b/scripts/tag_autocomplete_helper.py
@@ -546,7 +546,8 @@ def needs_restart(self):
         "tac_frequencySort": shared.OptionInfo(True, "Locally record tag usage and sort frequent tags higher").info("Will also work for extra networks, keeping the specified base order"),
         "tac_frequencyFunction": shared.OptionInfo("Logarithmic (weak)", "Function to use for frequency sorting", gr.Dropdown, lambda: {"choices": list(frequency_sort_functions.keys())}).info("; ".join([f'<b>{key}</b>: {val}' for key, val in frequency_sort_functions.items()])),
         "tac_frequencyMinCount": shared.OptionInfo(3, "Minimum number of uses for a tag to be considered frequent").info("Tags with less uses than this will not be sorted higher, even if the sorting function would normally result in a higher position."),
-        "tac_frequencyMaxAge": shared.OptionInfo(30, "Maximum days since last use for a tag to be considered frequent").info("Similar to the above, tags that haven't been used in this many days will not be sorted higher."),
+        "tac_frequencyMaxAge": shared.OptionInfo(30, "Maximum days since last use for a tag to be considered frequent").info("Similar to the above, tags that haven't been used in this many days will not be sorted higher. Set to 0 to disable."),
+        "tac_frequencyRecommendCap": shared.OptionInfo(10, "Maximum number of recommended tags").info("Limits the maximum number of recommended tags to not drown out normal results. Set to 0 to disable."),
         "tac_frequencyIncludeAlias": shared.OptionInfo(False, "Frequency sorting matches aliases for frequent tags").info("Tag frequency will be increased for the main tag even if an alias is used for completion. This option can be used to override the default behavior of alias results being ignored for frequency sorting."),
         # Insertion related settings
         "tac_replaceUnderscores": shared.OptionInfo(True, "Replace underscores with spaces on insertion"),
@@ -783,7 +784,20 @@ class UseCountListRequest(BaseModel):
     # Semantically weird to use post here, but it's required for the body on js side
     @app.post("/tacapi/v1/get-use-count-list")
     async def get_use_count_list(body: UseCountListRequest):
-        return db_request(lambda: list(db.get_tag_counts(body.tagNames, body.tagTypes, body.neg)), get=True)
+        # If a date limit is set > 0, pass it to the db
+        date_limit = getattr(shared.opts, "tac_frequencyMaxAge", 30)
+        date_limit = date_limit if date_limit > 0 else None
+
+        count_list = list(db.get_tag_counts(body.tagNames, body.tagTypes, body.neg, date_limit))
+    
+        # If a limit is set, return at max the top n results by count
+        if count_list and len(count_list):
+            limit = int(min(getattr(shared.opts, "tac_frequencyRecommendCap", 10), len(count_list)))
+            # Sort by count and return the top n
+            if limit > 0:
+                count_list = sorted(count_list, key=lambda x: x[2], reverse=True)[:limit]
+
+        return db_request(lambda: count_list, get=True)
 
     @app.put("/tacapi/v1/reset-use-count")
     async def reset_use_count(tagname: str, ttype: int, pos: bool, neg: bool):
diff --git a/scripts/tag_frequency_db.py b/scripts/tag_frequency_db.py
index 9a0cd10..5b1b195 100644
--- a/scripts/tag_frequency_db.py
+++ b/scripts/tag_frequency_db.py
@@ -13,15 +13,19 @@ def transaction(db=db_file):
     """Context manager for database transactions.
     Ensures that the connection is properly closed after the transaction.
     """
-    conn = sqlite3.connect(db, timeout=timeout)
     try:
+        conn = sqlite3.connect(db, timeout=timeout)
+        
         conn.isolation_level = None
         cursor = conn.cursor()
         cursor.execute("BEGIN")
         yield cursor
         cursor.execute("COMMIT")
+    except sqlite3.Error as e:
+        print("Tag Autocomplete: Frequency database error:", e)
     finally:
-        conn.close()
+        if conn:
+            conn.close()
 
 
 class TagFrequencyDb:
@@ -118,18 +122,29 @@ def get_tag_count(self, tag, ttype, negative=False):
         else:
             return 0, None
 
-    def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False):
+    def get_tag_counts(self, tags: list[str], ttypes: list[str], negative=False, date_limit=None):
         count_str = "count_neg" if negative else "count_pos"
         with transaction() as cursor:
             for tag, ttype in zip(tags, ttypes):
-                cursor.execute(
-                    f"""
-                SELECT {count_str}, last_used
-                FROM tag_frequency
-                WHERE name = ? AND type = ?
-                """,
-                    (tag, ttype),
-                )
+                if date_limit is not None:
+                    cursor.execute(
+                        f"""
+                    SELECT {count_str}, last_used
+                    FROM tag_frequency
+                    WHERE name = ? AND type = ?
+                    AND last_used > datetime('now', '-' || ? || ' days')
+                    """,
+                        (tag, ttype, date_limit),
+                    )
+                else:
+                    cursor.execute(
+                        f"""
+                    SELECT {count_str}, last_used
+                    FROM tag_frequency
+                    WHERE name = ? AND type = ?
+                    """,
+                        (tag, ttype),
+                    )
                 tag_count = cursor.fetchone()
                 if tag_count:
                     yield (tag, ttype, tag_count[0], tag_count[1])