From 5fd0b1d7de8ea22c91a124b04e1ae681384eca62 Mon Sep 17 00:00:00 2001 From: kasunami Date: Mon, 5 Aug 2024 19:44:19 -0400 Subject: [PATCH 1/3] update scipy versions --- poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4d6d699d1..879cc4310 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5794,16 +5794,17 @@ tqdm = ">=4.27" [package.extras] accelerate = ["accelerate (>=0.21.0)"] agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (<=0.9.16)", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy + ", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (<=0.9.16)", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] benchmark = ["optimum-benchmark (>=0.2.0)"] codecarbon = ["codecarbon (==1.2.0)"] deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.4.4)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.4.4)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.4.4)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.4.4)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.19,<0.20)", "urllib3 (<2.0.0)"] dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.4.4)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] From 073fa62993562ec2a8e385c8e2f76e1366dd0f0b Mon Sep 17 00:00:00 2001 From: kasunami Date: Mon, 5 Aug 2024 20:15:40 -0400 Subject: [PATCH 2/3] update numpy versions --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 879cc4310..aaf23703d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -713,7 +713,7 @@ httpx = ">=0.27.0" importlib-resources = "*" kubernetes = ">=28.1.0" mmh3 = ">=4.0.1" -numpy = ">=1.22.5,<2.0.0" +numpy = ">=1.22.5" onnxruntime = ">=1.14.1" opentelemetry-api = ">=1.2.0" opentelemetry-exporter-otlp-proto-grpc = ">=1.2.0" @@ -1639,7 +1639,7 @@ importlib-resources = ">=1.3,<7.0" jinja2 = "<4.0" markupsafe = ">=2.0,<3.0" matplotlib = ">=3.0,<4.0" -numpy = ">=1.0,<3.0" +numpy = ">=1.0" orjson = ">=3.0,<4.0" packaging = "*" pandas = ">=1.0,<3.0" @@ -2413,7 +2413,7 @@ httpx = "*" nest-asyncio = ">=1.5.8,<2.0.0" networkx = ">=3.0" nltk = ">=3.8.1,<4.0.0" -numpy = "<2.0.0" +numpy = ">=1.20" openai = ">=1.1.0" pandas = "*" pillow = ">=9.0.0" @@ -3626,7 +3626,7 @@ files = [ [package.dependencies] coloredlogs = "*" flatbuffers = "*" -numpy = ">=1.21.6,<2.0" +numpy = ">=1.21.6" packaging = "*" protobuf = "*" sympy = "*" @@ -5250,7 +5250,7 @@ files = [ ] [package.dependencies] -numpy = ">=1.23.5,<2.3" +numpy = ">=1.23.5" [package.extras] dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] From 59c3dc390e77f8499bd8383580be583010879694 Mon Sep 17 00:00:00 2001 From: kasunami Date: Wed, 4 Sep 2024 08:57:23 -0400 Subject: [PATCH 3/3] Create a kafka_processor.py file to handle input/output via kafka using the completions processes. This uses the completions_router.py as a reference/baseline and routes all kafka data through there, as well as an attempt to structure all responses as json. --- .../server/completions/kafka_processor.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 private_gpt/server/completions/kafka_processor.py diff --git a/private_gpt/server/completions/kafka_processor.py b/private_gpt/server/completions/kafka_processor.py new file mode 100644 index 000000000..decca4d6e --- /dev/null +++ b/private_gpt/server/completions/kafka_processor.py @@ -0,0 +1,122 @@ +# kafka_processor.py +from confluent_kafka import Consumer, Producer +from pydantic import BaseModel, ValidationError +from typing import Optional, List +from private_gpt.open_ai.extensions.context_filter import ContextFilter +from private_gpt.open_ai.openai_models import ( + OpenAICompletion, + OpenAIMessage, +) +from private_gpt.server.chat.chat_router import ChatBody, chat_completion + +import json + +# Kafka configuration variables +KAFKA_ADDRESS = 'localhost' +KAFKA_PORT = 9002 + +class CompletionsBody(BaseModel): + prompt: str + system_prompt: Optional[str] = "Always format your response as a valid JSON object, even if the request doesn't explicitly ask for it." + use_context: bool = False + context_filter: Optional[ContextFilter] = None + include_sources: bool = True + stream: bool = False + + model_config = { + "json_schema_extra": { + "examples": [ + { + "prompt": "How do you fry an egg?", + "system_prompt": "You are a rapper. Always answer with a rap.", + "stream": False, + "use_context": False, + "include_sources": False, + } + ] + } + } + + +def convert_body_to_messages(body: CompletionsBody) -> List[OpenAIMessage]: + messages = [OpenAIMessage(content=body.prompt, role="user")] + if body.system_prompt: + messages.insert(0, OpenAIMessage(content=body.system_prompt, role="system")) + return messages + + +def process_message(message_value: str) -> str: # Return type is now str (JSON string) + try: + body = CompletionsBody.parse_raw(message_value) + chat_body = ChatBody( + messages=convert_body_to_messages(body), + use_context=body.use_context, + stream=body.stream, + include_sources=body.include_sources, + context_filter=body.context_filter, + + ) + completion_response = chat_completion(request=None, chat_body=chat_body) + # Wrap the successful response in a JSON structure with status + return json.dumps({ + "status": "success", + "data": completion_response.model_dump_json() # Assuming model_dump_json returns a dict + }) + except ValidationError as e: + # Return a JSON structure with error details and status + return json.dumps({ + "status": "error", + "exception": str(e), + "location": "process_message - Parsing input message" + }) + +def consume_messages(consumer: Consumer, producer: Producer, output_topic: str): + while True: + msg = consumer.poll(1.0) + if msg is None: + continue + if msg.error(): + print(f"Consumer error: {msg.error()}") + continue + print(f"Received message: {msg.value().decode('utf-8')}") + + + completion_response = process_message(msg.value().decode('utf-8')) + producer.produce(output_topic, completion_response) # Send the JSON string directly + producer.flush() + + consumer.commit(asynchronous=False) + + +def main(): + bootstrap_servers = f"{KAFKA_ADDRESS}:{KAFKA_PORT}" + + consumer_config = { + 'bootstrap.servers': bootstrap_servers, + 'group.id': 'completions-group', + 'auto.offset.reset': 'earliest', + 'enable.auto.commit': False + } + + producer_config = { + 'bootstrap.servers': bootstrap_servers + } + + input_topic = 'prompt_request' # Updated input topic + output_topic = 'prompt_response' # Updated output topic + + consumer = Consumer(consumer_config) + producer = Producer(producer_config) + + consumer.subscribe([input_topic]) + + try: + consume_messages(consumer, producer, output_topic) + except KeyboardInterrupt: + pass + finally: + consumer.close() + + +if __name__ == "__main__": + main() \ No newline at end of file