Skip to content

Commit 027dd08

Browse files
kryanbeanekryanbeane
committed
feat(RHOAIENG-25241): disable ray usage metrics by default
1 parent 028d0d5 commit 027dd08

File tree

12 files changed

+110
-15
lines changed

12 files changed

+110
-15
lines changed

‎.github/workflows/e2e_tests.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ jobs:
117117
pip install poetry
118118
poetry install --with test,docs
119119
echo "Running e2e tests..."
120-
poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
120+
poetry run pytest -v -s ./tests/e2e/local_interactive_sdk_kind_test.py::TestRayLocalInteractiveOauth::test_local_interactives_nvidia_gpu > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
121121
env:
122122
GRPC_DNS_RESOLVER: "native"
123123

‎docs/sphinx/user-docs/cluster-configuration.rst

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,21 @@ requirements for creating the Ray Cluster.
4444
documentation on building a custom image
4545
`here <https://github.com/opendatahub-io/distributed-workloads/tree/main/images/runtime/examples>`__.
4646

47+
Ray Usage Statistics
48+
-------------------
49+
50+
By default, Ray usage statistics collection is disabled in CodeFlare SDK clusters. This stops statistics from being sent to AnyScale. If you want to enable usage statistics collection, you can set the ``RAY_USAGE_STATS_ENABLED`` environment variable to ``1`` in your cluster configuration:
51+
52+
.. code:: python
53+
54+
from codeflare_sdk import Cluster, ClusterConfiguration
55+
56+
cluster = Cluster(ClusterConfiguration(
57+
name='ray-example',
58+
namespace='default',
59+
envs={'RAY_USAGE_STATS_ENABLED': '1'} # Enable usage statistics
60+
))
61+
4762
The ``labels={"exampleLabel": "example"}`` parameter can be used to
4863
apply additional labels to the RayCluster resource.
4964

‎src/codeflare_sdk/ray/cluster/config.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,10 @@ def __post_init__(self):
161161
"Warning: TLS verification has been disabled - Endpoint checks will be bypassed"
162162
)
163163

164+
# Set default environment variable to disable Ray usage stats if not already set
165+
if "RAY_USAGE_STATS_ENABLED" not in self.envs:
166+
self.envs["RAY_USAGE_STATS_ENABLED"] = "0"
167+
164168
if self.enable_gcs_ft:
165169
if not self.redis_address:
166170
raise ValueError(

‎src/codeflare_sdk/ray/cluster/test_cluster.py

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -465,11 +465,10 @@ def test_get_cluster_no_appwrapper(mocker):
465465
return_value=expected_rc,
466466
)
467467
get_cluster("test-all-params", "ns", write_to_file=True)
468-
assert filecmp.cmp(
469-
f"{aw_dir}test-all-params.yaml",
470-
f"{expected_clusters_dir}/ray/unit-test-all-params.yaml",
471-
shallow=True,
472-
)
468+
469+
with open(f"{aw_dir}test-all-params.yaml") as f:
470+
generated_rc = yaml.load(f, Loader=yaml.FullLoader)
471+
assert generated_rc == expected_rc
473472

474473

475474
def test_get_cluster_with_appwrapper(mocker):
@@ -487,11 +486,10 @@ def test_get_cluster_with_appwrapper(mocker):
487486
return_value=expected_aw,
488487
)
489488
get_cluster("aw-all-params", "ns", write_to_file=True)
490-
assert filecmp.cmp(
491-
f"{aw_dir}aw-all-params.yaml",
492-
f"{expected_clusters_dir}/appwrapper/unit-test-all-params.yaml",
493-
shallow=True,
494-
)
489+
490+
with open(f"{aw_dir}aw-all-params.yaml") as f:
491+
generated_aw = yaml.load(f, Loader=yaml.FullLoader)
492+
assert generated_aw == expected_aw
495493

496494

497495
def test_wait_ready(mocker, capsys):

‎src/codeflare_sdk/ray/cluster/test_config.py

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import filecmp
2525
import pytest
2626
import os
27+
import yaml
2728

2829
parent = Path(__file__).resolve().parents[4] # project directory
2930
expected_clusters_dir = f"{parent}/tests/test_cluster_yamls"
@@ -85,7 +86,11 @@ def test_config_creation_all_parameters(mocker):
8586
assert cluster.config.worker_memory_requests == "12G"
8687
assert cluster.config.worker_memory_limits == "16G"
8788
assert cluster.config.appwrapper == False
88-
assert cluster.config.envs == {"key1": "value1", "key2": "value2"}
89+
assert cluster.config.envs == {
90+
"key1": "value1",
91+
"key2": "value2",
92+
"RAY_USAGE_STATS_ENABLED": "0",
93+
}
8994
assert cluster.config.image == "example/ray:tag"
9095
assert cluster.config.image_pull_secrets == ["secret1", "secret2"]
9196
assert cluster.config.write_to_file == True
@@ -206,6 +211,46 @@ def test_gcs_fault_tolerance_config_validation():
206211
)
207212

208213

214+
def test_ray_usage_stats_default(mocker):
215+
mocker.patch("kubernetes.client.ApisApi.get_api_versions")
216+
mocker.patch("kubernetes.client.CustomObjectsApi.list_namespaced_custom_object")
217+
218+
cluster = Cluster(
219+
ClusterConfiguration(name="default-usage-stats-cluster", namespace="ns")
220+
)
221+
222+
# Verify that usage stats are disabled by default
223+
assert cluster.config.envs["RAY_USAGE_STATS_ENABLED"] == "0"
224+
225+
# Check that the environment variable is set in the YAML
226+
head_container = cluster.resource_yaml["spec"]["headGroupSpec"]["template"]["spec"][
227+
"containers"
228+
][0]
229+
env_vars = {env["name"]: env["value"] for env in head_container["env"]}
230+
assert env_vars["RAY_USAGE_STATS_ENABLED"] == "0"
231+
232+
233+
def test_ray_usage_stats_enabled(mocker):
234+
mocker.patch("kubernetes.client.ApisApi.get_api_versions")
235+
mocker.patch("kubernetes.client.CustomObjectsApi.list_namespaced_custom_object")
236+
237+
cluster = Cluster(
238+
ClusterConfiguration(
239+
name="usage-stats-enabled-cluster",
240+
namespace="ns",
241+
envs={"RAY_USAGE_STATS_ENABLED": "1"},
242+
)
243+
)
244+
245+
assert cluster.config.envs["RAY_USAGE_STATS_ENABLED"] == "1"
246+
247+
head_container = cluster.resource_yaml["spec"]["headGroupSpec"]["template"]["spec"][
248+
"containers"
249+
][0]
250+
env_vars = {env["name"]: env["value"] for env in head_container["env"]}
251+
assert env_vars["RAY_USAGE_STATS_ENABLED"] == "1"
252+
253+
209254
# Make sure to always keep this function last
210255
def test_cleanup():
211256
os.remove(f"{aw_dir}test-all-params.yaml")

‎tests/e2e/local_interactive_sdk_kind_test.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ def run_local_interactives(
4848
head_cpu_limits="500m",
4949
head_memory_requests=2,
5050
head_memory_limits=2,
51+
head_extended_resource_requests={gpu_resource_name: 0},
5152
worker_cpu_requests="500m",
5253
worker_cpu_limits=1,
5354
worker_memory_requests=1,
@@ -68,16 +69,16 @@ def run_local_interactives(
6869
ray.shutdown()
6970
ray.init(address=cluster.local_client_url(), logging_level="DEBUG")
7071

71-
@ray.remote(num_gpus=number_of_gpus / 2)
72-
def heavy_calculation_part(num_iterations):
72+
@ray.remote(num_gpus=number_of_gpus)
73+
def heavy_calculation(num_iterations):
7374
result = 0.0
7475
for i in range(num_iterations):
7576
for j in range(num_iterations):
7677
for k in range(num_iterations):
7778
result += math.sin(i) * math.cos(j) * math.tan(k)
7879
return result
7980

80-
@ray.remote(num_gpus=number_of_gpus / 2)
81+
@ray.remote(num_gpus=number_of_gpus)
8182
def heavy_calculation(num_iterations):
8283
results = ray.get(
8384
[heavy_calculation_part.remote(num_iterations // 30) for _ in range(30)]

‎tests/test_cluster_yamls/appwrapper/unit-test-all-params.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ spec:
5454
value: value1
5555
- name: key2
5656
value: value2
57+
- name: RAY_USAGE_STATS_ENABLED
58+
value: '0'
5759
image: example/ray:tag
5860
imagePullPolicy: Always
5961
lifecycle:
@@ -159,6 +161,8 @@ spec:
159161
value: value1
160162
- name: key2
161163
value: value2
164+
- name: RAY_USAGE_STATS_ENABLED
165+
value: '0'
162166
image: example/ray:tag
163167
imagePullPolicy: Always
164168
lifecycle:

‎tests/test_cluster_yamls/kueue/aw_kueue.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
- mountPath: /etc/ssl/certs/odh-ca-bundle.crt
7676
name: odh-ca-cert
7777
subPath: odh-ca-bundle.crt
78+
env:
79+
- name: RAY_USAGE_STATS_ENABLED
80+
value: '0'
7881
volumes:
7982
- configMap:
8083
items:
@@ -133,6 +136,9 @@ spec:
133136
- mountPath: /etc/ssl/certs/odh-ca-bundle.crt
134137
name: odh-ca-cert
135138
subPath: odh-ca-bundle.crt
139+
env:
140+
- name: RAY_USAGE_STATS_ENABLED
141+
value: '0'
136142
volumes:
137143
- configMap:
138144
items:

‎tests/test_cluster_yamls/kueue/ray_cluster_kueue.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
- mountPath: /etc/ssl/certs/odh-ca-bundle.crt
7676
name: odh-ca-cert
7777
subPath: odh-ca-bundle.crt
78+
env:
79+
- name: RAY_USAGE_STATS_ENABLED
80+
value: '0'
7881
volumes:
7982
- configMap:
8083
items:
@@ -133,6 +136,9 @@ spec:
133136
- mountPath: /etc/ssl/certs/odh-ca-bundle.crt
134137
name: odh-ca-cert
135138
subPath: odh-ca-bundle.crt
139+
env:
140+
- name: RAY_USAGE_STATS_ENABLED
141+
value: '0'
136142
volumes:
137143
- configMap:
138144
items:

‎tests/test_cluster_yamls/ray/default-appwrapper.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ spec:
5353
name: dashboard
5454
- containerPort: 10001
5555
name: client
56+
env:
57+
- name: RAY_USAGE_STATS_ENABLED
58+
value: '0'
5659
resources:
5760
limits:
5861
cpu: 2
@@ -111,6 +114,9 @@ spec:
111114
- -c
112115
- ray stop
113116
name: machine-learning
117+
env:
118+
- name: RAY_USAGE_STATS_ENABLED
119+
value: '0'
114120
resources:
115121
limits:
116122
cpu: 1

‎tests/test_cluster_yamls/ray/default-ray-cluster.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ spec:
4545
name: dashboard
4646
- containerPort: 10001
4747
name: client
48+
env:
49+
- name: RAY_USAGE_STATS_ENABLED
50+
value: '0'
4851
resources:
4952
limits:
5053
cpu: 2
@@ -110,6 +113,9 @@ spec:
110113
requests:
111114
cpu: 1
112115
memory: 2G
116+
env:
117+
- name: RAY_USAGE_STATS_ENABLED
118+
value: '0'
113119
volumeMounts:
114120
- mountPath: /etc/pki/tls/certs/odh-trusted-ca-bundle.crt
115121
name: odh-trusted-ca-cert

‎tests/test_cluster_yamls/ray/unit-test-all-params.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ spec:
4545
value: value1
4646
- name: key2
4747
value: value2
48+
- name: RAY_USAGE_STATS_ENABLED
49+
value: '0'
4850
image: example/ray:tag
4951
imagePullPolicy: Always
5052
lifecycle:
@@ -150,6 +152,8 @@ spec:
150152
value: value1
151153
- name: key2
152154
value: value2
155+
- name: RAY_USAGE_STATS_ENABLED
156+
value: '0'
153157
image: example/ray:tag
154158
imagePullPolicy: Always
155159
lifecycle:

0 commit comments

Comments
 (0)