Move to Uvicorn, use few shots

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
+        exclude: ^tests/snapshots
     -   id: trailing-whitespace
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.1.0

.vscode/launch.json

@@ -17,9 +17,10 @@
         "name": "Backend",
         "type": "debugpy",
         "request": "launch",
+        "cwd": "${workspaceFolder}",
         "module": "uvicorn",
         "args": ["fastapi_app:create_app", "--factory", "--reload"],
-        "justMyCode": true
+        "justMyCode": false
     }
   ],
   "compounds": [

src/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.12-bullseye
+FROM python:3.12-bullseye
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     && apt-get -y install --no-install-recommends postgresql-client \
@@ -12,8 +12,9 @@ WORKDIR /demo-code
 COPY requirements.txt .
 RUN python -m pip install -r requirements.txt
 
-COPY entrypoint.sh .
-RUN chmod +x entrypoint.sh
-
 COPY . .
-CMD bash -c ". entrypoint.sh"
+RUN python -m pip install .
+
+RUN chmod +x entrypoint.sh
+EXPOSE 8000
+CMD ["bash", "-c", ". entrypoint.sh"]

src/backend/entrypoint.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
 set -e
-python3 -m pip install .
-python3 -m gunicorn "fastapi_app:create_app()"
+python3 -m uvicorn "fastapi_app:create_app" --factory --port 8000

src/backend/fastapi_app/__init__.py

@@ -47,13 +47,15 @@ async def lifespan(app: fastapi.FastAPI) -> AsyncIterator[State]:
 
 def create_app(testing: bool = False):
     if os.getenv("RUNNING_IN_PRODUCTION"):
-        logging.basicConfig(level=logging.WARNING)
+        # You may choose to reduce this to logging.WARNING for production
+        logging.basicConfig(level=logging.INFO)
     else:
         if not testing:
             load_dotenv(override=True)
         logging.basicConfig(level=logging.INFO)
     # Turn off particularly noisy INFO level logs from Azure Core SDK:
     logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(logging.WARNING)
+    logging.getLogger("azure.identity").setLevel(logging.WARNING)
 
     if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"):
         logger.info("Configuring Azure Monitor")

src/backend/fastapi_app/prompts/query_fewshots.json

@@ -0,0 +1,34 @@
+[
+    {"role": "user", "content": "good options for climbing gear that can be used outside?"},
+    {"role": "assistant", "tool_calls": [
+        {
+            "id": "call_abc123",
+            "type": "function",
+            "function": {
+                "arguments": "{\"search_query\":\"climbing gear outside\"}",
+                "name": "search_database"
+            }
+        }
+    ]},
+    {
+        "role": "tool",
+        "tool_call_id": "call_abc123",
+        "content": "Search results for climbing gear that can be used outside: ..."
+    },
+    {"role": "user", "content": "are there any shoes less than $50?"},
+    {"role": "assistant", "tool_calls": [
+        {
+            "id": "call_abc456",
+            "type": "function",
+            "function": {
+                "arguments": "{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}",
+                "name": "search_database"
+            }
+        }
+    ]},
+    {
+        "role": "tool",
+        "tool_call_id": "call_abc456",
+        "content": "Search results for shoes cheaper than 50: ..."
+    }
+]

src/backend/fastapi_app/rag_advanced.py

@@ -1,5 +1,5 @@
 from collections.abc import AsyncGenerator
-from typing import Any
+from typing import Any, Final
 
 from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream
 from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam
@@ -38,12 +38,19 @@ async def generate_search_query(
         self, original_user_query: str, past_messages: list[ChatCompletionMessageParam], query_response_token_limit: int
     ) -> tuple[list[ChatCompletionMessageParam], Any | str | None, list]:
         """Generate an optimized keyword search query based on the chat history and the last question"""
+
+        tools = build_search_function()
+        tool_choice: Final = "auto"
+
         query_messages: list[ChatCompletionMessageParam] = build_messages(
             model=self.chat_model,
             system_prompt=self.query_prompt_template,
+            few_shots=self.query_fewshots,
             new_user_content=original_user_query,
             past_messages=past_messages,
-            max_tokens=self.chat_token_limit - query_response_token_limit,  # TODO: count functions
+            max_tokens=self.chat_token_limit - query_response_token_limit,
+            tools=tools,
+            tool_choice=tool_choice,
             fallback_to_default=True,
         )
 
@@ -54,8 +61,8 @@ async def generate_search_query(
             temperature=0.0,  # Minimize creativity for search query generation
             max_tokens=query_response_token_limit,  # Setting too low risks malformed JSON, too high risks performance
             n=1,
-            tools=build_search_function(),
-            tool_choice="auto",
+            tools=tools,
+            tool_choice=tool_choice,
         )
 
         query_text, filters = extract_search_arguments(original_user_query, chat_completion)

src/backend/fastapi_app/rag_base.py

@@ -1,3 +1,4 @@
+import json
 import pathlib
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator
@@ -17,6 +18,7 @@
 class RAGChatBase(ABC):
     current_dir = pathlib.Path(__file__).parent
     query_prompt_template = open(current_dir / "prompts/query.txt").read()
+    query_fewshots = json.loads(open(current_dir / "prompts/query_fewshots.json").read())
     answer_prompt_template = open(current_dir / "prompts/answer.txt").read()
 
     def get_params(self, messages: list[ChatCompletionMessageParam], overrides: ChatRequestOverrides) -> ChatParams:

src/backend/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "fastapi_app"
 version = "1.0.0"
-description = "Create a application with fastapi and postgres-flexible"
+description = "Create a RAG application with FastAPI and PostgreSQL"
 dependencies = [
     "fastapi>=0.111.0,<1.0.0",
     "python-dotenv>=1.0.1,<2.0.0",
@@ -13,7 +13,7 @@ dependencies = [
     "pgvector>=0.2.5,<0.3.0",
     "openai>=1.34.0,<2.0.0",
     "tiktoken>=0.7.0,<0.8.0",
-    "openai-messages-token-helper>=0.1.5,<0.2.0",
+    "openai-messages-token-helper>=0.1.8,<0.2.0",
     "azure-monitor-opentelemetry>=1.6.0,<2.0.0",
     "opentelemetry-instrumentation-sqlalchemy>=0.46b0,<1.0.0",
     "opentelemetry-instrumentation-aiohttp-client>=0.46b0,<1.0.0",

src/backend/requirements.txt

@@ -1,2 +1 @@
-gunicorn>=22.0.0,<23.0.0
-uvicorn>=0.30.1,<1.0.0
+uvicorn>=0.30.1,<1.0.0

tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json

@@ -19,6 +19,12 @@
                 "title": "Prompt to generate search arguments",
                 "description": [
                     "{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}",
+                    "{'role': 'user', 'content': 'good options for climbing gear that can be used outside?'}",
+                    "{'role': 'assistant', 'tool_calls': [{'id': 'call_abc123', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"climbing gear outside\"}', 'name': 'search_database'}}]}",
+                    "{'role': 'tool', 'tool_call_id': 'call_abc123', 'content': 'Search results for climbing gear that can be used outside: ...'}",
+                    "{'role': 'user', 'content': 'are there any shoes less than $50?'}",
+                    "{'role': 'assistant', 'tool_calls': [{'id': 'call_abc456', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}', 'name': 'search_database'}}]}",
+                    "{'role': 'tool', 'tool_call_id': 'call_abc456', 'content': 'Search results for shoes cheaper than 50: ...'}",
                     "{'role': 'user', 'content': 'What is the capital of France?'}"
                 ],
                 "props": {

tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines

@@ -1,2 +1,2 @@
-{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null}
+{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'good options for climbing gear that can be used outside?'}","{'role': 'assistant', 'tool_calls': [{'id': 'call_abc123', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"climbing gear outside\"}', 'name': 'search_database'}}]}","{'role': 'tool', 'tool_call_id': 'call_abc123', 'content': 'Search results for climbing gear that can be used outside: ...'}","{'role': 'user', 'content': 'are there any shoes less than $50?'}","{'role': 'assistant', 'tool_calls': [{'id': 'call_abc456', 'type': 'function', 'function': {'arguments': '{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}', 'name': 'search_database'}}]}","{'role': 'tool', 'tool_call_id': 'call_abc456', 'content': 'Search results for shoes cheaper than 50: ...'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null}
 {"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null}