Move to Uvicorn, use few shots

Author: pamelafox

.vscode/launch.json

@@ -17,9 +17,10 @@
         "name": "Backend",
         "type": "debugpy",
         "request": "launch",
+        "cwd": "${workspaceFolder}",
         "module": "uvicorn",
         "args": ["fastapi_app:create_app", "--factory", "--reload"],
-        "justMyCode": true
+        "justMyCode": false
     }
   ],
   "compounds": [

src/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.12-bullseye
+FROM python:3.12-bullseye
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     && apt-get -y install --no-install-recommends postgresql-client \
@@ -12,8 +12,9 @@ WORKDIR /demo-code
 COPY requirements.txt .
 RUN python -m pip install -r requirements.txt
 
-COPY entrypoint.sh .
-RUN chmod +x entrypoint.sh
-
 COPY . .
-CMD bash -c ". entrypoint.sh"
+RUN python -m pip install .
+
+RUN chmod +x entrypoint.sh
+EXPOSE 8000
+CMD ["bash", "-c", ". entrypoint.sh"]

src/backend/entrypoint.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
 set -e
-python3 -m pip install .
-python3 -m gunicorn "fastapi_app:create_app()"
+python3 -m uvicorn "fastapi_app:create_app" --factory --port 8000

src/backend/fastapi_app/__init__.py

@@ -47,13 +47,15 @@ async def lifespan(app: fastapi.FastAPI) -> AsyncIterator[State]:
 
 def create_app(testing: bool = False):
     if os.getenv("RUNNING_IN_PRODUCTION"):
-        logging.basicConfig(level=logging.WARNING)
+        # You may choose to reduce this to logging.WARNING for production
+        logging.basicConfig(level=logging.INFO)
     else:
         if not testing:
             load_dotenv(override=True)
         logging.basicConfig(level=logging.INFO)
     # Turn off particularly noisy INFO level logs from Azure Core SDK:
     logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(logging.WARNING)
+    logging.getLogger("azure.identity").setLevel(logging.WARNING)
 
     if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"):
         logger.info("Configuring Azure Monitor")

src/backend/fastapi_app/prompts/query_fewshots.json

@@ -0,0 +1,34 @@
+[
+    {"role": "user", "content": "good options for climbing gear that can be used outside?"},
+    {"role": "assistant", "tool_calls": [
+        {
+            "id": "call_abc123",
+            "type": "function",
+            "function": {
+                "arguments": "{\"search_query\":\"climbing gear outside\"}",
+                "name": "search_database"
+            }
+        }
+    ]},
+    {
+        "role": "tool",
+        "tool_call_id": "call_abc123",
+        "content": "Search results for climbing gear that can be used outside: ..."
+    },
+    {"role": "user", "content": "are there any shoes less than $50?"},
+    {"role": "assistant", "tool_calls": [
+        {
+            "id": "call_abc456",
+            "type": "function",
+            "function": {
+                "arguments": "{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}",
+                "name": "search_database"
+            }
+        }
+    ]},
+    {
+        "role": "tool",
+        "tool_call_id": "call_abc456",
+        "content": "Search results for shoes cheaper than 50: ..."
+    }
+]

src/backend/fastapi_app/rag_advanced.py

@@ -38,12 +38,19 @@ async def generate_search_query(
         self, original_user_query: str, past_messages: list[ChatCompletionMessageParam], query_response_token_limit: int
     ) -> tuple[list[ChatCompletionMessageParam], Any | str | None, list]:
         """Generate an optimized keyword search query based on the chat history and the last question"""
+
+        tools = build_search_function()
+        tool_choice = "auto"
+
         query_messages: list[ChatCompletionMessageParam] = build_messages(
             model=self.chat_model,
             system_prompt=self.query_prompt_template,
+            few_shots=self.query_fewshots,
             new_user_content=original_user_query,
             past_messages=past_messages,
-            max_tokens=self.chat_token_limit - query_response_token_limit,  # TODO: count functions
+            max_tokens=self.chat_token_limit - query_response_token_limit,
+            tools=tools,
+            tool_choice=tool_choice,
             fallback_to_default=True,
         )
 
@@ -54,8 +61,8 @@ async def generate_search_query(
             temperature=0.0,  # Minimize creativity for search query generation
             max_tokens=query_response_token_limit,  # Setting too low risks malformed JSON, too high risks performance
             n=1,
-            tools=build_search_function(),
-            tool_choice="auto",
+            tools=tools,
+            tool_choice=tool_choice,
         )
 
         query_text, filters = extract_search_arguments(original_user_query, chat_completion)

src/backend/fastapi_app/rag_base.py

@@ -1,3 +1,4 @@
+import json
 import pathlib
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator
@@ -17,6 +18,7 @@
 class RAGChatBase(ABC):
     current_dir = pathlib.Path(__file__).parent
     query_prompt_template = open(current_dir / "prompts/query.txt").read()
+    query_fewshots = json.loads(open(current_dir / "prompts/query_fewshots.json").read())
     answer_prompt_template = open(current_dir / "prompts/answer.txt").read()
 
     def get_params(self, messages: list[ChatCompletionMessageParam], overrides: ChatRequestOverrides) -> ChatParams:

src/backend/gunicorn.conf.py

@@ -1,7 +1,7 @@
 [project]
 name = "fastapi_app"
 version = "1.0.0"
-description = "Create a application with fastapi and postgres-flexible"
+description = "Create a RAG application with FastAPI and PostgreSQL"
 dependencies = [
     "fastapi>=0.111.0,<1.0.0",
     "python-dotenv>=1.0.1,<2.0.0",
@@ -13,7 +13,7 @@ dependencies = [
     "pgvector>=0.2.5,<0.3.0",
     "openai>=1.34.0,<2.0.0",
     "tiktoken>=0.7.0,<0.8.0",
-    "openai-messages-token-helper>=0.1.5,<0.2.0",
+    "openai-messages-token-helper>=0.1.7,<0.2.0",
     "azure-monitor-opentelemetry>=1.6.0,<2.0.0",
     "opentelemetry-instrumentation-sqlalchemy>=0.46b0,<1.0.0",
     "opentelemetry-instrumentation-aiohttp-client>=0.46b0,<1.0.0",

src/backend/pyproject.toml

@@ -1,2 +1 @@
-gunicorn>=22.0.0,<23.0.0
-uvicorn>=0.30.1,<1.0.0
+uvicorn>=0.30.1,<1.0.0