add feedback and feedback_task with cli (#102)

* add log10 feedback or log10 feedback-task create and cli

* update examples/feedback/simple_feedback.py

* update README

---------

Co-authored-by: Kim Tran <[email protected]>

README.md

@@ -143,6 +143,11 @@ Read more here for options for logging using library wrapper, langchain callback
 
 Optimizing prompts requires a lot of manual effort. Log10 provides a copilot that can help you with suggestions on how to [optimize your prompt](https://log10.io/docs/prompt_engineering/auto_prompt#how-to-use-auto-prompting-in-log10-python-library). 
 
+### 👷🔢 Feedback
+
+Add feedback to your completions. Checkout the Python [example](/examples/feedback/simple_feedback.py)
+or use CLI `log10 feedback-task create` and `log10 feedback create`. Please check our [doc](https://log10.io/docs/feedback) for more details.
+
 ### 🔍🐞 Prompt chain debugging
 
 Prompt chains such as those in [Langchain](https://github.com/hwchase17/langchain) can be difficult to debug. Log10 provides prompt provenance, session tracking and call stack functionality to help debug chains.

examples/feedback/simple_feedback.py

@@ -0,0 +1,51 @@
+import uuid
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from log10.feedback.feedback import Feedback
+from log10.feedback.feedback_task import FeedbackTask
+from log10.load import OpenAI
+
+
+#
+# use log10 to log an openai completion
+#
+
+# create a unique id
+unique_id = str(uuid.uuid4())
+print(f"Use tag: {unique_id}")
+client = OpenAI(tags=[unique_id])
+completion = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are the most knowledgable Star Wars guru on the planet",
+        },
+        {
+            "role": "user",
+            "content": "Write the time period of all the Star Wars movies and spinoffs?",
+        },
+    ],
+)
+print(completion.choices[0].message)
+
+#
+# add feedback to the completion
+#
+
+
+# define a feedback task
+class EmojiFeedback(BaseModel):
+    feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis")
+
+
+# create a feedback
+fb = EmojiFeedback(feedback="😀")
+
+task = FeedbackTask().create(name="emoji_task_test", task_schema=fb.model_json_schema())
+task_dump = task.json()
+
+print(fb.model_dump_json())
+Feedback().create(task_id=task_dump["id"], values=fb.model_dump(), completion_tags_selector=[unique_id])

log10/__main__.py

@@ -0,0 +1,28 @@
+import click
+
+from log10.feedback.feedback import create_feedback
+from log10.feedback.feedback_task import create_feedback_task
+
+
+@click.group()
+def cli():
+    pass
+
+
+@click.group()
+def feedback():
+    pass
+
+
+@click.group()
+def feedback_task():
+    pass
+
+
+cli.add_command(feedback)
+feedback.add_command(create_feedback, "create")
+cli.add_command(feedback_task)
+feedback_task.add_command(create_feedback_task, "create")
+
+if __name__ == "__main__":
+    cli()

log10/feedback/feedback.py

@@ -0,0 +1,65 @@
+import json
+import logging
+
+import click
+import httpx
+from dotenv import load_dotenv
+
+from log10.llm import Log10Config
+
+
+load_dotenv()
+
+logging.basicConfig(
+    format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger: logging.Logger = logging.getLogger("LOG10")
+logger.setLevel(logging.INFO)
+
+
+class Feedback:
+    feedback_create_url = "api/v1/feedback"
+
+    def __init__(self, log10_config: Log10Config = None):
+        self._log10_config = log10_config or Log10Config()
+        self._http_client = httpx.Client()
+
+    def _post_request(self, url: str, json_payload: dict) -> httpx.Response:
+        headers = {
+            "x-log10-token": self._log10_config.token,
+            "x-log10-organization-id": self._log10_config.org_id,
+            "Content-Type": "application/json",
+        }
+        json_payload["organization_id"] = self._log10_config.org_id
+        try:
+            res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload)
+            res.raise_for_status()
+            return res
+        except Exception as e:
+            logger.error(e)
+            logger.error(e.response.json()["error"])
+            raise
+
+    def create(
+        self, task_id: str, values: dict, completion_tags_selector: list[str], comment: str = None
+    ) -> httpx.Response:
+        json_payload = {
+            "task_id": task_id,
+            "json_values": values,
+            "completion_tags_selector": completion_tags_selector,
+        }
+        res = self._post_request(self.feedback_create_url, json_payload)
+        return res
+
+
+@click.command()
+@click.option("--task_id", prompt="Enter task id", help="Task ID")
+@click.option("--values", prompt="Enter task values", help="Feedback in JSON format")
+@click.option("--completion_tags_selector", prompt="Enter completion tags selector", help="Completion tags selector")
+def create_feedback(task_id, values, completion_tags_selector):
+    click.echo("Creating feedback")
+    tags = completion_tags_selector.split(",")
+    values = json.loads(values)
+    feedback = Feedback().create(task_id=task_id, values=values, completion_tags_selector=tags)
+    click.echo(feedback.json())

log10/feedback/feedback_task.py

@@ -0,0 +1,63 @@
+import json
+import logging
+
+import click
+import httpx
+from dotenv import load_dotenv
+
+from log10.llm import Log10Config
+
+
+load_dotenv()
+
+logging.basicConfig(
+    format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger: logging.Logger = logging.getLogger("LOG10")
+logger.setLevel(logging.INFO)
+
+
+class FeedbackTask:
+    feedback_task_create_url = "api/v1/feedback_task"
+
+    def __init__(self, log10_config: Log10Config = None):
+        self._log10_config = log10_config or Log10Config()
+        self._http_client = httpx.Client()
+
+    def _post_request(self, url: str, json_payload: dict) -> httpx.Response:
+        headers = {
+            "x-log10-token": self._log10_config.token,
+            "Content-Type": "application/json",
+            "x-log10-organization-id": self._log10_config.org_id,
+        }
+        json_payload["organization_id"] = self._log10_config.org_id
+        try:
+            res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload)
+            res.raise_for_status()
+            return res
+        except Exception as e:
+            logger.error(e)
+            logger.error(e.response.json()["error"])
+            raise
+
+    def create(self, task_schema: dict, name: str = None, instruction: str = None) -> httpx.Response:
+        json_payload = {"json_schema": task_schema}
+        if name:
+            json_payload["name"] = name
+        if instruction:
+            json_payload["instruction"] = instruction
+
+        res = self._post_request(self.feedback_task_create_url, json_payload)
+        return res
+
+
+# create a cli interface for FeebackTask.create function
+@click.command()
+@click.option("--name", prompt="Enter feedback task name", help="Name of the task")
+@click.option("--task_schema", prompt="Enter feedback task schema", help="Task schema")
+def create_feedback_task(name, task_schema):
+    click.echo("Creating feedback task")
+    task_schema = json.loads(task_schema)
+    task = FeedbackTask().create(name=name, task_schema=task_schema)
+    click.echo(f"Use this task_id to add feedback: {task.json()['id']}")

pyproject.toml

@@ -19,6 +19,9 @@ packages = [
     { include = "log10" },
 ]
 
+[tool.poetry.scripts]
+log10 = "log10.__main__:cli"
+
 [tool.poetry.group.dev.dependencies]
 build = "^0.10.0"
 pytest = "^8.0.0"
@@ -48,17 +51,17 @@ together = "^0.2.7"
 
 [tool.ruff]
 # Never enforce `E501` (line length violations).
-ignore = ["C901", "E501", "E741", "F402", "F823" ]
-select = ["C", "E", "F", "I", "W"]
+lint.ignore = ["C901", "E501", "E741", "F402", "F823" ]
+lint.select = ["C", "E", "F", "I", "W"]
 line-length = 119
 
 # Ignore import violations in all `__init__.py` files.
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["E402", "F401", "F403", "F811"]
 "log10/langchain.py" = ["E402"]
 "examples/logging/*.py" = ["E402", "F821"]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 lines-after-imports = 2
 known-first-party = ["log10"]