Skip to content

Commit 154c4d2

Browse files
pamelafoxpamelafox
committed
Start workflow
1 parent 66a443b commit 154c4d2

File tree

3 files changed

+130
-8
lines changed

3 files changed

+130
-8
lines changed

‎.github/workflows/evaluate.yaml

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
name: Evaluate
2+
3+
on:
4+
workflow_dispatch:
5+
6+
# Set up permissions for deploying with secretless Azure federated credentials
7+
# https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
8+
permissions:
9+
id-token: write
10+
contents: read
11+
12+
jobs:
13+
evaluate:
14+
runs-on: ubuntu-latest
15+
env:
16+
AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
17+
AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
18+
AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
19+
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
20+
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
21+
AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
22+
AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME }}
23+
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
24+
AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME }}
25+
AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
26+
AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
27+
AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
- name: Check for MacOS Runner
32+
if: matrix.os == 'macos-latest-xlarge'
33+
run: brew install postgresql@14
34+
- name: Install pgvector on Windows using install-pgvector.bat
35+
if: matrix.os == 'windows-latest'
36+
shell: cmd
37+
run: .github\workflows\install-pgvector.bat
38+
39+
- name: Install PostgreSQL development libraries
40+
if: matrix.os == 'ubuntu-latest'
41+
run: |
42+
sudo apt update
43+
sudo apt install postgresql-server-dev-14
44+
45+
- name: Setup postgres
46+
uses: ikalnytskyi/action-setup-postgres@v6
47+
with:
48+
username: admin
49+
password: postgres
50+
database: postgres
51+
52+
- name: Install pgvector on MacOS/Linux using install-pgvector.sh
53+
if: matrix.os != 'windows-latest'
54+
run: .github/workflows/install-pgvector.sh
55+
56+
- name: Install python
57+
uses: actions/setup-python@v5
58+
with:
59+
python-version: '3.12' # install the python version needed
60+
61+
- name: Install azd
62+
uses: Azure/setup-azd@v1.0.0
63+
64+
- name: Install dependencies
65+
run: |
66+
python -m pip install -r requirements-dev.txt
67+
68+
- name: Install app as editable app
69+
run: |
70+
python -m pip install -e src/backend
71+
72+
- name: Setup local database with seed data
73+
run: |
74+
cp .env.sample .env
75+
python ./src/backend/fastapi_app/setup_postgres_database.py
76+
python ./src/backend/fastapi_app/setup_postgres_seeddata.py
77+
78+
- name: Setup node
79+
uses: actions/setup-node@v4
80+
with:
81+
node-version: 18
82+
83+
- name: Build frontend
84+
run: |
85+
cd ./src/frontend
86+
npm install
87+
npm run build
88+
89+
- name: Install python packages
90+
run: |
91+
python -m pip install --upgrade pip
92+
pip install -r requirements-dev.txt
93+
94+
- name: Login to Azure
95+
uses: azure/login@v2
96+
with:
97+
client-id: ${{ env.AZURE_CLIENT_ID }}
98+
tenant-id: ${{ env.AZURE_TENANT_ID }}
99+
subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
100+
101+
- name: Set az account
102+
uses: azure/CLI@v2
103+
with:
104+
inlineScript: |
105+
az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}}
106+
107+
108+
- name: Run local server
109+
run: |
110+
python3 -m uvicorn fastapi_app:create_app --factory
111+
112+
- name: Evaluate local RAG flow
113+
run: |
114+
python evals/evaluate.py
115+
116+
- name: Upload eval results as build artifact
117+
uses: actions/upload-artifact@v4
118+
with:
119+
name: eval_result
120+
path: ./src/api/evaluate/eval_results.jsonl
121+
122+
- name: GitHub Summary Step
123+
if: ${{ success() }}
124+
working-directory: ./src/api
125+
run: |
126+
echo "" >> $GITHUB_STEP_SUMMARY
127+
128+
echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY
129+
cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ Further documentation is available in the `docs/` folder:
194194
* [Deploying with existing resources](docs/deploy_existing.md)
195195
* [Monitoring with Azure Monitor](docs/monitoring.md)
196196
* [Load testing](docs/loadtesting.md)
197+
* [Evaluation](docs/evaluation.md)
197198
198199
Please post in the issue tracker with any questions or issues.
199200

‎evals/README.md

Lines changed: 0 additions & 8 deletions
This file was deleted.

0 commit comments

Comments
 (0)