From beb433b7513fdd32985a67b8f4130f55e304244b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 2 Aug 2024 20:39:31 +0000 Subject: [PATCH 1/5] Move to Uvicorn, use few shots --- .vscode/launch.json | 3 +- src/backend/Dockerfile | 11 +++--- src/backend/entrypoint.sh | 3 +- src/backend/fastapi_app/__init__.py | 4 ++- .../fastapi_app/prompts/query_fewshots.json | 34 +++++++++++++++++++ src/backend/fastapi_app/rag_advanced.py | 13 +++++-- src/backend/fastapi_app/rag_base.py | 2 ++ src/backend/gunicorn.conf.py | 11 ------ src/backend/pyproject.toml | 4 +-- src/backend/requirements.txt | 3 +- 10 files changed, 61 insertions(+), 27 deletions(-) create mode 100644 src/backend/fastapi_app/prompts/query_fewshots.json delete mode 100644 src/backend/gunicorn.conf.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 50b295df..4c233e69 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,9 +17,10 @@ "name": "Backend", "type": "debugpy", "request": "launch", + "cwd": "${workspaceFolder}", "module": "uvicorn", "args": ["fastapi_app:create_app", "--factory", "--reload"], - "justMyCode": true + "justMyCode": false } ], "compounds": [ diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index f788e118..29a192f0 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/devcontainers/python:3.12-bullseye +FROM python:3.12-bullseye RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends postgresql-client \ @@ -12,8 +12,9 @@ WORKDIR /demo-code COPY requirements.txt . RUN python -m pip install -r requirements.txt -COPY entrypoint.sh . -RUN chmod +x entrypoint.sh - COPY . . -CMD bash -c ". entrypoint.sh" \ No newline at end of file +RUN python -m pip install . + +RUN chmod +x entrypoint.sh +EXPOSE 8000 +CMD ["bash", "-c", ". entrypoint.sh"] diff --git a/src/backend/entrypoint.sh b/src/backend/entrypoint.sh index 0a743968..91a81837 100644 --- a/src/backend/entrypoint.sh +++ b/src/backend/entrypoint.sh @@ -1,4 +1,3 @@ #!/bin/bash set -e -python3 -m pip install . -python3 -m gunicorn "fastapi_app:create_app()" \ No newline at end of file +python3 -m uvicorn "fastapi_app:create_app" --factory --port 8000 diff --git a/src/backend/fastapi_app/__init__.py b/src/backend/fastapi_app/__init__.py index 55a60334..318eab97 100644 --- a/src/backend/fastapi_app/__init__.py +++ b/src/backend/fastapi_app/__init__.py @@ -47,13 +47,15 @@ async def lifespan(app: fastapi.FastAPI) -> AsyncIterator[State]: def create_app(testing: bool = False): if os.getenv("RUNNING_IN_PRODUCTION"): - logging.basicConfig(level=logging.WARNING) + # You may choose to reduce this to logging.WARNING for production + logging.basicConfig(level=logging.INFO) else: if not testing: load_dotenv(override=True) logging.basicConfig(level=logging.INFO) # Turn off particularly noisy INFO level logs from Azure Core SDK: logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(logging.WARNING) + logging.getLogger("azure.identity").setLevel(logging.WARNING) if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): logger.info("Configuring Azure Monitor") diff --git a/src/backend/fastapi_app/prompts/query_fewshots.json b/src/backend/fastapi_app/prompts/query_fewshots.json new file mode 100644 index 00000000..d5a026f2 --- /dev/null +++ b/src/backend/fastapi_app/prompts/query_fewshots.json @@ -0,0 +1,34 @@ +[ + {"role": "user", "content": "good options for climbing gear that can be used outside?"}, + {"role": "assistant", "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "arguments": "{\"search_query\":\"climbing gear outside\"}", + "name": "search_database" + } + } + ]}, + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "Search results for climbing gear that can be used outside: ..." + }, + {"role": "user", "content": "are there any shoes less than $50?"}, + {"role": "assistant", "tool_calls": [ + { + "id": "call_abc456", + "type": "function", + "function": { + "arguments": "{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}", + "name": "search_database" + } + } + ]}, + { + "role": "tool", + "tool_call_id": "call_abc456", + "content": "Search results for shoes cheaper than 50: ..." + } +] diff --git a/src/backend/fastapi_app/rag_advanced.py b/src/backend/fastapi_app/rag_advanced.py index ddbd65ce..5dd1c7b4 100644 --- a/src/backend/fastapi_app/rag_advanced.py +++ b/src/backend/fastapi_app/rag_advanced.py @@ -38,12 +38,19 @@ async def generate_search_query( self, original_user_query: str, past_messages: list[ChatCompletionMessageParam], query_response_token_limit: int ) -> tuple[list[ChatCompletionMessageParam], Any | str | None, list]: """Generate an optimized keyword search query based on the chat history and the last question""" + + tools = build_search_function() + tool_choice = "auto" + query_messages: list[ChatCompletionMessageParam] = build_messages( model=self.chat_model, system_prompt=self.query_prompt_template, + few_shots=self.query_fewshots, new_user_content=original_user_query, past_messages=past_messages, - max_tokens=self.chat_token_limit - query_response_token_limit, # TODO: count functions + max_tokens=self.chat_token_limit - query_response_token_limit, + tools=tools, + tool_choice=tool_choice, fallback_to_default=True, ) @@ -54,8 +61,8 @@ async def generate_search_query( temperature=0.0, # Minimize creativity for search query generation max_tokens=query_response_token_limit, # Setting too low risks malformed JSON, too high risks performance n=1, - tools=build_search_function(), - tool_choice="auto", + tools=tools, + tool_choice=tool_choice, ) query_text, filters = extract_search_arguments(original_user_query, chat_completion) diff --git a/src/backend/fastapi_app/rag_base.py b/src/backend/fastapi_app/rag_base.py index f7f7bff4..183647e7 100644 --- a/src/backend/fastapi_app/rag_base.py +++ b/src/backend/fastapi_app/rag_base.py @@ -1,3 +1,4 @@ +import json import pathlib from abc import ABC, abstractmethod from collections.abc import AsyncGenerator @@ -17,6 +18,7 @@ class RAGChatBase(ABC): current_dir = pathlib.Path(__file__).parent query_prompt_template = open(current_dir / "prompts/query.txt").read() + query_fewshots = json.loads(open(current_dir / "prompts/query_fewshots.json").read()) answer_prompt_template = open(current_dir / "prompts/answer.txt").read() def get_params(self, messages: list[ChatCompletionMessageParam], overrides: ChatRequestOverrides) -> ChatParams: diff --git a/src/backend/gunicorn.conf.py b/src/backend/gunicorn.conf.py deleted file mode 100644 index 03df0f74..00000000 --- a/src/backend/gunicorn.conf.py +++ /dev/null @@ -1,11 +0,0 @@ -import multiprocessing - -max_requests = 1000 -max_requests_jitter = 50 -log_file = "-" -bind = "0.0.0.0:8000" -workers = (multiprocessing.cpu_count() * 2) + 1 - -worker_class = "uvicorn.workers.UvicornWorker" - -timeout = 600 diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 0e62fdc7..d02ad3cf 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "fastapi_app" version = "1.0.0" -description = "Create a application with fastapi and postgres-flexible" +description = "Create a RAG application with FastAPI and PostgreSQL" dependencies = [ "fastapi>=0.111.0,<1.0.0", "python-dotenv>=1.0.1,<2.0.0", @@ -13,7 +13,7 @@ dependencies = [ "pgvector>=0.2.5,<0.3.0", "openai>=1.34.0,<2.0.0", "tiktoken>=0.7.0,<0.8.0", - "openai-messages-token-helper>=0.1.5,<0.2.0", + "openai-messages-token-helper>=0.1.7,<0.2.0", "azure-monitor-opentelemetry>=1.6.0,<2.0.0", "opentelemetry-instrumentation-sqlalchemy>=0.46b0,<1.0.0", "opentelemetry-instrumentation-aiohttp-client>=0.46b0,<1.0.0", diff --git a/src/backend/requirements.txt b/src/backend/requirements.txt index cf57fdaa..5bfba051 100644 --- a/src/backend/requirements.txt +++ b/src/backend/requirements.txt @@ -1,2 +1 @@ -gunicorn>=22.0.0,<23.0.0 -uvicorn>=0.30.1,<1.0.0 \ No newline at end of file +uvicorn>=0.30.1,<1.0.0 From e67793e4cb7ae0164e8ea61df27f142238613171 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 2 Aug 2024 22:20:15 +0000 Subject: [PATCH 2/5] Make mypy happy --- src/backend/fastapi_app/rag_advanced.py | 4 ++-- src/backend/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/fastapi_app/rag_advanced.py b/src/backend/fastapi_app/rag_advanced.py index 5dd1c7b4..09f4f6c5 100644 --- a/src/backend/fastapi_app/rag_advanced.py +++ b/src/backend/fastapi_app/rag_advanced.py @@ -1,5 +1,5 @@ from collections.abc import AsyncGenerator -from typing import Any +from typing import Any, Final from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam @@ -40,7 +40,7 @@ async def generate_search_query( """Generate an optimized keyword search query based on the chat history and the last question""" tools = build_search_function() - tool_choice = "auto" + tool_choice: Final = "auto" query_messages: list[ChatCompletionMessageParam] = build_messages( model=self.chat_model, diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index d02ad3cf..d697f4d7 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pgvector>=0.2.5,<0.3.0", "openai>=1.34.0,<2.0.0", "tiktoken>=0.7.0,<0.8.0", - "openai-messages-token-helper>=0.1.7,<0.2.0", + "openai-messages-token-helper>=0.1.8,<0.2.0", "azure-monitor-opentelemetry>=1.6.0,<2.0.0", "opentelemetry-instrumentation-sqlalchemy>=0.46b0,<1.0.0", "opentelemetry-instrumentation-aiohttp-client>=0.46b0,<1.0.0", From 27af016f8310d58e3bc6c9218075c6521e781fef Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 2 Aug 2024 22:24:14 +0000 Subject: [PATCH 3/5] Update snapshots --- .../advanced_chat_flow_response.json | 8 +++++++- .../advanced_chat_streaming_flow_response.jsonlines | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index c7692bd1..2351da77 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -19,6 +19,12 @@ "title": "Prompt to generate search arguments", "description": [ "{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}", + "{'role': 'user', 'content': 'good options for climbing gear that can be used outside?'}", + "{'role': 'assistant', 'tool_calls': [{'id': 'call_abc123', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"climbing gear outside\"}', 'name': 'search_database'}}]}", + "{'role': 'tool', 'tool_call_id': 'call_abc123', 'content': 'Search results for climbing gear that can be used outside: ...'}", + "{'role': 'user', 'content': 'are there any shoes less than $50?'}", + "{'role': 'assistant', 'tool_calls': [{'id': 'call_abc456', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}', 'name': 'search_database'}}]}", + "{'role': 'tool', 'tool_call_id': 'call_abc456', 'content': 'Search results for shoes cheaper than 50: ...'}", "{'role': 'user', 'content': 'What is the capital of France?'}" ], "props": { @@ -65,4 +71,4 @@ "followup_questions": null }, "sessionState": null -} \ No newline at end of file +} diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index b7e4efa3..7df511cb 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'good options for climbing gear that can be used outside?'}","{'role': 'assistant', 'tool_calls': [{'id': 'call_abc123', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"climbing gear outside\"}', 'name': 'search_database'}}]}","{'role': 'tool', 'tool_call_id': 'call_abc123', 'content': 'Search results for climbing gear that can be used outside: ...'}","{'role': 'user', 'content': 'are there any shoes less than $50?'}","{'role': 'assistant', 'tool_calls': [{'id': 'call_abc456', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}', 'name': 'search_database'}}]}","{'role': 'tool', 'tool_call_id': 'call_abc456', 'content': 'Search results for shoes cheaper than 50: ...'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} {"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} From cdea114fbafe8ee6c85a4baee43065f43520953f Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 2 Aug 2024 22:33:22 +0000 Subject: [PATCH 4/5] Newline in snapshot --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48296dc8..46e7b88e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,7 @@ repos: hooks: - id: check-yaml - id: end-of-file-fixer + exclude: ^tests/snapshots - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.0 From 24092b19ee9d597380a1b50d44658f6dec71b040 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 2 Aug 2024 22:34:06 +0000 Subject: [PATCH 5/5] Newline in snapshot --- .../test_advanced_chat_flow/advanced_chat_flow_response.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 2351da77..1251792c 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -71,4 +71,4 @@ "followup_questions": null }, "sessionState": null -} +} \ No newline at end of file