Skip to content

Commit 134652d

Browse files
alex000kimalex000kim
committed
Initialize Git repository
0 parents  commit 134652d

9 files changed

+448
-0
lines changed

‎.devcontainer.json

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"name": "example-cv",
3+
"image": "mcr.microsoft.com/devcontainers/python:3.10",
4+
"runArgs": ["--ipc=host"],
5+
"features": {
6+
"ghcr.io/devcontainers/features/nvidia-cuda:1": {
7+
"installCudnn": true
8+
},
9+
"ghcr.io/iterative/features/nvtop:1": {}
10+
},
11+
"extensions": [
12+
"Iterative.dvc",
13+
"ms-python.python",
14+
"redhat.vscode-yaml"
15+
],
16+
"postCreateCommand": "pip install --user -r requirements.txt"
17+
}

‎.gitattributes

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.dvc linguist-language=YAML
2+
dvc.lock linguist-language=YAML

‎.github/workflows/deploy-model-sagemaker.yml

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Deploy model (Sagemaker)
2+
3+
on:
4+
push:
5+
tags:
6+
- "*"
7+
8+
permissions:
9+
contents: write
10+
id-token: write
11+
12+
jobs:
13+
parse:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v3
17+
- name: "Parse GTO tag"
18+
id: gto
19+
uses: iterative/gto-action@v2
20+
outputs:
21+
event: ${{ steps.gto.outputs.event }}
22+
name: ${{ steps.gto.outputs.name }}
23+
stage: ${{ steps.gto.outputs.stage }}
24+
version: ${{ steps.gto.outputs.version }}
25+
26+
deploy-model:
27+
needs: parse
28+
if: "${{ needs.parse.outputs.event == 'assignment' }}"
29+
environment: cloud
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v3
33+
with:
34+
fetch-depth: 0
35+
36+
- uses: aws-actions/configure-aws-credentials@v4
37+
with:
38+
aws-region: us-east-2
39+
role-to-assume: ${{ vars.AWS_SANDBOX_ROLE }}
40+
role-duration-seconds: 43200
41+
42+
- name: Set up Python
43+
uses: actions/setup-python@v4
44+
with:
45+
python-version: '3.8'
46+
cache: 'pip'
47+
cache-dependency-path: requirements.txt
48+
49+
- run: pip install -r requirements.txt
50+
51+
- run: dvc remote add -d --local storage s3://dvc-public/remote/get-started-pools
52+
53+
- run: |
54+
MODEL_DATA=$(dvc get --show-url . model.tar.gz)
55+
python sagemaker/deploy_model.py \
56+
--name ${{ needs.parse.outputs.name }} \
57+
--stage ${{ needs.parse.outputs.stage }} \
58+
--version ${{ needs.parse.outputs.version }} \
59+
--model_data $MODEL_DATA \
60+
--role ${{ vars.AWS_SANDBOX_ROLE }}

‎.github/workflows/deploy-model-template.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Deploy Model (Template)
2+
3+
on:
4+
# the workflow is triggered whenever a tag is pushed to the repository
5+
push:
6+
tags:
7+
- "*"
8+
jobs:
9+
10+
# This job parses the git tag with the GTO GitHub Action to identify model registry actions
11+
parse:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v3
15+
- name: "Parse GTO tag"
16+
id: gto
17+
uses: iterative/gto-action@v2
18+
outputs:
19+
event: ${{ steps.gto.outputs.event }}
20+
name: ${{ steps.gto.outputs.name }}
21+
stage: ${{ steps.gto.outputs.stage }}
22+
version: ${{ steps.gto.outputs.version }}
23+
24+
deploy-model:
25+
needs: parse
26+
# using the outputs from the "parse" job, we run this job only for actions
27+
# in the model registry and only when the model was assigned to a stage called "prod"
28+
if: ${{ needs.parse.outputs.event == 'assignment' && needs.parse.outputs.stage == 'prod' }}
29+
runs-on: ubuntu-latest
30+
steps:
31+
- uses: iterative/setup-dvc@v1
32+
# this step uses DVC to download the model from our remote repository and deploys the model
33+
# Model deployment is mocked here as it is specific to each deployment environment
34+
# The DVC Studio token is used to avoid having to store specific remote storage credentials on GitHub
35+
- name: Get Model For Deployment
36+
run: |
37+
dvc config --global studio.token ${{ secrets.DVC_STUDIO_TOKEN }}
38+
dvc artifacts get ${{ github.server_url }}/${{ github.repository }} ${{ needs.parse.outputs.name }} --rev ${{ needs.parse.outputs.version }}
39+
echo "The right model is available and you can use the rest of this command to deploy it. Good job!"

‎.github/workflows/dvc-studio.yml

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
name: DVC Studio Experiment
2+
3+
on:
4+
5+
push:
6+
tags-ignore:
7+
- '**'
8+
9+
workflow_dispatch:
10+
inputs:
11+
exp-run-args:
12+
description: 'Args to be passed to dvc exp run call'
13+
required: false
14+
type: string
15+
default: ''
16+
parent-sha:
17+
description: 'SHA of the commit to start the experiment from'
18+
required: false
19+
type: string
20+
default: ''
21+
cloud:
22+
description: 'Cloud compute provider to host the runner'
23+
required: false
24+
default: 'aws'
25+
type: choice
26+
options:
27+
- aws
28+
- azure
29+
- gcp
30+
type:
31+
description: 'https://registry.terraform.io/providers/iterative/iterative/latest/docs/resources/task#machine-type'
32+
required: false
33+
default: 'g5.2xlarge'
34+
region:
35+
description: 'https://registry.terraform.io/providers/iterative/iterative/latest/docs/resources/task#cloud-region'
36+
required: false
37+
default: 'us-east'
38+
spot:
39+
description: 'Request a spot instance'
40+
required: false
41+
default: false
42+
type: boolean
43+
storage:
44+
description: 'Disk size in GB'
45+
required: false
46+
default: 40
47+
type: number
48+
timeout:
49+
description: 'Timeout in seconds'
50+
required: false
51+
default: 3600
52+
type: number
53+
54+
permissions:
55+
contents: write
56+
id-token: write
57+
pull-requests: write
58+
59+
jobs:
60+
61+
deploy-runner:
62+
if: ${{ (github.actor == 'iterative-studio[bot]') || (github.event_name == 'workflow_dispatch') }}
63+
environment: cloud
64+
runs-on: ubuntu-latest
65+
66+
steps:
67+
- uses: actions/checkout@v3
68+
with:
69+
ref: ${{ inputs.parent-sha || '' }}
70+
- uses: iterative/setup-cml@v2
71+
- uses: aws-actions/configure-aws-credentials@v4
72+
with:
73+
aws-region: us-east-2
74+
role-to-assume: ${{ vars.AWS_SANDBOX_ROLE }}
75+
role-duration-seconds: 43200
76+
- name: Create Runner
77+
env:
78+
REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
79+
run: |
80+
cml runner launch --single \
81+
--labels=cml \
82+
--cloud=${{ inputs.cloud || 'aws' }} \
83+
--cloud-region=${{ inputs.region || 'us-east' }} \
84+
--cloud-hdd-size=${{ inputs.storage || '40' }} \
85+
--cloud-type=${{ inputs.type || 'g5.2xlarge' }} \
86+
--idle-timeout=${{ inputs.timeout || '3600' }} \
87+
${{ (inputs.spot == 'true' && '--cloud-spot') || '' }}
88+
89+
runner-job:
90+
needs: deploy-runner
91+
runs-on: [ self-hosted, cml ]
92+
environment: cloud
93+
container:
94+
image: iterativeai/cml:latest-gpu
95+
options: --gpus all --ipc host
96+
97+
steps:
98+
- uses: actions/checkout@v3
99+
with:
100+
ref: ${{ inputs.parent-sha || '' }}
101+
- uses: aws-actions/configure-aws-credentials@v4
102+
with:
103+
aws-region: us-east-2
104+
role-to-assume: ${{ vars.AWS_SANDBOX_ROLE }}
105+
role-duration-seconds: 43200
106+
107+
- run: pip install -r requirements.txt
108+
109+
- name: Train
110+
env:
111+
REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
112+
DVC_STUDIO_TOKEN: ${{ secrets.DVC_STUDIO_TOKEN }}
113+
DVCLIVE_LOGLEVEL: DEBUG
114+
run: |
115+
cml ci --fetch-depth 0
116+
dvc exp run --pull --allow-missing ${{ github.event.inputs.exp-run-args }}
117+
dvc remote add --local push_remote s3://dvc-public/remote/get-started-pools
118+
119+
- name: Workflow Dispatch Sharing
120+
if: github.event_name == 'workflow_dispatch'
121+
env:
122+
DVC_STUDIO_TOKEN: ${{ secrets.DVC_STUDIO_TOKEN }}
123+
run: |
124+
dvc exp push origin -r push_remote
125+
126+
- name: Commit-based Sharing
127+
if: github.actor == 'iterative-studio[bot]'
128+
env:
129+
REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
130+
run: |
131+
dvc push -r push_remote
132+
cml pr --squash --skip-ci .
133+
echo "## Metrics" > report.md
134+
dvc metrics diff main --md >> report.md
135+
echo "## Params" >> report.md
136+
dvc params diff main --md >> report.md
137+
cml comment create --pr report.md

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.venv/

‎.gitlab-ci.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Deploy Model (Template)
2+
3+
workflow:
4+
rules:
5+
# Run the pipeline whenever a tag is pushed to the repository
6+
- if: $CI_COMMIT_TAG
7+
8+
parse:
9+
# This job parses the model tag to identify model registry actions
10+
image: python:3.11-slim
11+
script:
12+
# Install GTO to parse model tags
13+
- pip install gto
14+
# This job parses the model tags to identify model registry actions
15+
- echo "CI_COMMIT_TAG - ${CI_COMMIT_TAG}"
16+
- echo MODEL_NAME="$(gto check-ref ${CI_COMMIT_TAG} --name)" >> parse.env
17+
- echo MODEL_VERSION="$(gto check-ref ${CI_COMMIT_TAG} --version)" >> parse.env
18+
- echo MODEL_EVENT="$(gto check-ref ${CI_COMMIT_TAG} --event)" >> parse.env
19+
- echo MODEL_STAGE="$(gto check-ref ${CI_COMMIT_TAG} --stage)" >> parse.env
20+
# Print variables saved to parse.env
21+
- cat parse.env
22+
artifacts:
23+
reports:
24+
dotenv: parse.env
25+
26+
deploy-model:
27+
needs:
28+
- job: parse
29+
artifacts: true
30+
image: python:3.11-slim
31+
script:
32+
# Check if the model is assigned to prod (variables from parse.env are only available in the 'script' section)
33+
- if [[ $MODEL_EVENT == 'assignment' && $MODEL_STAGE == 'prod' ]]; then echo "Deploy model"; else exit 1; fi
34+
# Install DVC
35+
- pip install dvc
36+
# Build commands to download and deploy the model
37+
- dvc config --global studio.token ${DVC_STUDIO_TOKEN}
38+
- dvc artifacts get ${CI_REPOSITORY_URL} ${MODEL_NAME} --rev ${MODEL_VERSION}
39+
- echo "The right model is available and you can use the rest of this command to deploy it. Good job!"

0 commit comments

Comments
 (0)