diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 5e4c9aa9..fa537d8e 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -164,24 +164,32 @@ jobs: strategy: matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] python-version: ["3.12"] arch: ["x64"] + docker-engine: ["docker"] unit-tesseract: ${{ fromJson(needs.get-e2e-matrix.outputs.matrix) }} include: # Test on arm to ensure compatibility with Apple M1 chips # (OSX runners don't have access to Docker so we use Linux ARM runners instead) - - os: "ubuntu-22.04" + - os: "ubuntu-24.04" python-version: "3.12" arch: "arm" + docker-engine: "docker" + unit-tesseract: "base" + # Run tests using Podman + - os: "ubuntu-24.04" + python-version: "3.12" + arch: "x64" + docker-engine: "podman" unit-tesseract: "base" fail-fast: false - runs-on: ${{ matrix.arch == 'x64' && matrix.os == 'ubuntu-22.04' && 'ubuntu-22.04' || matrix.arch == 'arm' && matrix.os == 'ubuntu-22.04' && 'linux-arm64-ubuntu2204' }} + runs-on: ${{ matrix.arch == 'x64' && matrix.os == 'ubuntu-24.04' && 'ubuntu-24.04' || matrix.arch == 'arm' && matrix.os == 'ubuntu-24.04' && 'linux-arm64-ubuntu2404'}} defaults: run: @@ -209,6 +217,17 @@ jobs: run: | uv sync --extra dev --frozen + - name: Set up Podman + if: matrix.docker-engine == 'podman' + run: | + systemctl start --user podman.socket + systemctl status --user podman.socket + + podman --version + + export DOCKER_HOST=unix:///run/user/1001/podman/podman.sock + echo "DOCKER_HOST=${DOCKER_HOST}" >> $GITHUB_ENV + - name: Run test suite run: | if [ "${{ matrix.unit-tesseract }}" == "base" ]; then @@ -221,12 +240,12 @@ jobs: -k "not test_examples" else uv run --no-sync pytest \ - --always-run-endtoend \ - --cov-report=term-missing:skip-covered \ - --cov-report=xml:coverage.xml \ - --cov=tesseract_core \ - tests/endtoend_tests/test_examples.py \ - -k "[${{ matrix.unit-tesseract }}]" + --always-run-endtoend \ + --cov-report=term-missing:skip-covered \ + --cov-report=xml:coverage.xml \ + --cov=tesseract_core \ + tests/endtoend_tests/test_examples.py \ + -k "[${{ matrix.unit-tesseract }}]" fi - name: Upload coverage reports to Codecov diff --git a/tesseract_core/sdk/docker_client.py b/tesseract_core/sdk/docker_client.py index 4bc86465..f3fc46fc 100644 --- a/tesseract_core/sdk/docker_client.py +++ b/tesseract_core/sdk/docker_client.py @@ -42,10 +42,48 @@ def from_dict(cls, json_dict: dict) -> "Image": class Images: """Namespace for functions to interface with Tesseract docker images.""" + @staticmethod + def _tag_exists(image_name: str, tags: list_) -> bool: + """Helper function to check if image name exists in the list of tags. + + Specially handling has to be done to achieve unfuzzy substring matching, i.e. + if image tag is foo/bar/image, we need to return true for foo/bar/image, bar/image, + and image, but not ar/image. + + There is no equivalent in docker-py. + + Params: + image_name: The image name to check. + tags: The list of tags to check against. + + Returns: + True if the image name exists in the list of tags, False otherwise. + """ + if ":" not in image_name: + image_name += ":latest" + + image_name_parts = image_name.strip("/").split("/") + for tag in tags: + tag_parts = tag.strip("/").split("/") + # Check if the last parts of the tag matches the subportion that image name specifies + if tag_parts[-len(image_name_parts) :] == image_name_parts: + return True + return False + @staticmethod def get(image_id_or_name: str | bytes, tesseract_only: bool = True) -> Image: """Returns the metadata for a specific image. + In docker-py, there is no substring matching and the image name is the + last tag in the list of tags, so if an image has multiple tags, only + one of the tags would be able to find the image. + + However, in podman, this is not the case. Podman has substring matching + by "/" segments to handle repository urls and returns images even if + partial name is specified, or if image has multiple tags. + + We chose to support podman's largest string matching functionality here. + Params: image_id_or_name: The image name or id to get. tesseract_only: If True, only retrieves Tesseract images. @@ -76,12 +114,7 @@ def is_image_id(s: str) -> bool: if ( image_obj.id == image_id_or_name or image_obj.short_id == image_id_or_name - or image_id_or_name in image_obj.tags - or ( - any( - tag.split("/")[-1] == image_id_or_name for tag in image_obj.tags - ) - ) + or Images._tag_exists(image_id_or_name, image_obj.tags) ): return image_obj @@ -711,7 +744,7 @@ class DockerException(Exception): class BuildError(DockerException): """Raised when a build fails.""" - def __init__(self, build_log: str) -> None: + def __init__(self, build_log: list_[str]) -> None: self.build_log = build_log diff --git a/tesseract_core/sdk/engine.py b/tesseract_core/sdk/engine.py index dafdd210..d113005f 100644 --- a/tesseract_core/sdk/engine.py +++ b/tesseract_core/sdk/engine.py @@ -10,10 +10,12 @@ import os import random import shlex +import socket import string import tempfile import threading from collections.abc import Callable, Sequence +from contextlib import closing from pathlib import Path from shutil import copy, copytree, rmtree from typing import Any @@ -116,14 +118,31 @@ def wrapper_needs_docker(*args: Any, **kwargs: Any) -> None: return wrapper_needs_docker -def get_free_port() -> int: - """Find a free port to use for HTTP.""" - import socket - from contextlib import closing +def get_free_port(within_range: tuple[int, int] | None = None) -> int: + """Find a random free port to use for HTTP.""" + if within_range is None: + # Let OS pick a random free port + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("localhost", 0)) + return s.getsockname()[1] - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.bind(("localhost", 0)) - return s.getsockname()[1] + start, end = within_range + if start < 0 or end > 65535 or start >= end: + raise ValueError("Invalid port range") + + # Try random ports in the given range + portlist = list(range(start, end)) + random.shuffle(portlist) + for port in portlist: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + try: + s.bind(("localhost", port)) + except OSError: + # Port is already in use + continue + else: + return port + raise RuntimeError(f"No free ports found in range {start}-{end}") def parse_requirements( @@ -560,6 +579,12 @@ def _create_docker_compose_template( if ports is None: ports = [str(get_free_port()) for _ in range(len(image_ids))] + # Convert port ranges to fixed ports + for i, port in enumerate(ports): + if "-" in port: + port_start, port_end = port.split("-") + ports[i] = str(get_free_port(within_range=(int(port_start), int(port_end)))) + gpu_settings = None if gpus: if (len(gpus) == 1) and (gpus[0] == "all"): diff --git a/tesseract_core/sdk/templates/pytorch/tesseract_requirements.txt b/tesseract_core/sdk/templates/pytorch/tesseract_requirements.txt index d0b29f2a..0133df7c 100644 --- a/tesseract_core/sdk/templates/pytorch/tesseract_requirements.txt +++ b/tesseract_core/sdk/templates/pytorch/tesseract_requirements.txt @@ -2,7 +2,8 @@ # Generated by tesseract {{version}} on {{timestamp}} # Add Python requirements like this: -torch --index-url https://download.pytorch.org/whl/cpu +--index-url https://download.pytorch.org/whl/cpu +torch # This may contain private dependencies via SSH URLs: # git+ssh://git@github.com/username/repo.git@branch diff --git a/tests/conftest.py b/tests/conftest.py index b7c2bdde..f7057a6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,8 @@ import string from pathlib import Path from shutil import copytree +from textwrap import indent +from traceback import format_exception from typing import Any import pytest @@ -188,53 +190,89 @@ def docker_client(): @pytest.fixture(scope="module") +def docker_cleanup_module(docker_client, request): + """Clean up all tesseracts created by the tests after the module exits.""" + return _docker_cleanup(docker_client, request) + + +@pytest.fixture def docker_cleanup(docker_client, request): - """Clean up all tesseracts created by the tests.""" - from tesseract_core.sdk.docker_client import ContainerError, ImageNotFound + """Clean up all tesseracts created by the tests after the test exits.""" + return _docker_cleanup(docker_client, request) + +def _docker_cleanup(docker_client, request): + """Clean up all tesseracts created by the tests.""" # Shared object to track what objects need to be cleaned up in each test context = {"images": [], "project_ids": [], "containers": []} + def pprint_exc(e: BaseException) -> str: + """Pretty print exception.""" + return "".join( + indent(line, " ") for line in format_exception(type(e), e, e.__traceback__) + ) + def cleanup_func(): + failures = [] + # Teardown projects first for project_id in context["project_ids"]: - if docker_client.compose.exists(project_id): + try: docker_client.compose.down(project_id) + except Exception as e: + failures.append( + f"Failed to tear down project {project_id}: {pprint_exc(e)}" + ) # Remove containers for container in context["containers"]: try: - container_obj = docker_client.containers.get(container) - except ContainerError: - continue - container_obj.remove(v=True, force=True) + if isinstance(container, str): + container_obj = docker_client.containers.get(container.id) + else: + container_obj = container + + container_obj.remove(v=True, force=True) + except Exception as e: + failures.append( + f"Failed to remove container {container}: {pprint_exc(e)}" + ) # Remove images for image in context["images"]: try: - docker_client.images.remove(image) - except ImageNotFound: - continue + if isinstance(image, str): + image_obj = docker_client.images.get(image) + else: + image_obj = image + + docker_client.images.remove(image_obj.id) + except Exception as e: + failures.append(f"Failed to remove image {image}: {pprint_exc(e)}") + + if failures: + raise RuntimeError( + "Failed to clean up some Docker objects during test teardown:\n" + + "\n".join(failures) + ) request.addfinalizer(cleanup_func) return context @pytest.fixture -def dummy_image_name(docker_cleanup): +def dummy_image_name(): """Create a dummy image name, and clean up after the test.""" image_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16)) image_name = f"tmp_tesseract_image_{image_id}" - docker_cleanup["images"].append(image_name) yield image_name @pytest.fixture(scope="module") -def shared_dummy_image_name(docker_cleanup): +def shared_dummy_image_name(): """Create a dummy image name, and clean up after all tests.""" image_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16)) image_name = f"tmp_tesseract_image_{image_id}" - docker_cleanup["images"].append(image_name) yield image_name diff --git a/tests/endtoend_tests/common.py b/tests/endtoend_tests/common.py index fecce6ce..6684ff7d 100644 --- a/tests/endtoend_tests/common.py +++ b/tests/endtoend_tests/common.py @@ -48,7 +48,7 @@ def print_debug_info(result): def build_tesseract( - sourcedir, image_name, config_override=None, tag=None, build_retries=3 + client, sourcedir, image_name, config_override=None, tag=None, build_retries=3 ): cli_runner = CliRunner(mix_stderr=False) @@ -87,6 +87,7 @@ def build_tesseract( assert result.exit_code == 0, result.exception image_tags = json.loads(result.stdout.strip()) - assert image_name in image_tags + + assert client.images._tag_exists(image_name, image_tags) return image_tags[0] diff --git a/tests/endtoend_tests/test_docker_client.py b/tests/endtoend_tests/test_docker_client.py index 424768ea..6a8f2396 100644 --- a/tests/endtoend_tests/test_docker_client.py +++ b/tests/endtoend_tests/test_docker_client.py @@ -3,6 +3,7 @@ """End to end tests for docker cli wrapper.""" +import os import subprocess import textwrap from contextlib import closing @@ -52,7 +53,7 @@ def test_get_image(docker_client, docker_client_built_image_name, docker_py_clie # Get the image image = docker_client.images.get(docker_client_built_image_name) assert image is not None - assert docker_client_built_image_name in image.tags + assert docker_client.images._tag_exists(docker_client_built_image_name, image.tags) docker_py_image = docker_py_client.images.get(docker_client_built_image_name) assert docker_py_image is not None @@ -100,7 +101,7 @@ def test_get_image(docker_client, docker_client_built_image_name, docker_py_clie assert docker_py_image_list is not None # Check that every image in image_list is also in docker_py_image_list for image in image_list: - assert image.id in [img.id for img in docker_py_image_list] + assert image.id in [img.id for img in docker_py_image_list if img] def test_create_image( @@ -120,7 +121,9 @@ def test_create_image( try: image = docker_client.images.get(docker_client_built_image_name) assert image is not None - assert docker_client_built_image_name in image.tags + assert docker_client.images._tag_exists( + docker_client_built_image_name, image.tags + ) image_id_obj = docker_client.images.get(image.id) image_short_id_obj = docker_client.images.get(image.short_id) @@ -138,7 +141,7 @@ def test_create_image( image1 = docker_client.images.get(image1_name) image1_name = image1_name + ":latest" assert image1 is not None - assert image1_name in image1.tags + assert docker_client.images._tag_exists(image1_name, image1.tags) assert image_exists(docker_client, image1_name) assert image_exists(docker_py_client, image1_name) @@ -155,14 +158,33 @@ def test_create_image( # Our docker client should be able to retrieve images with just the name image2 = docker_client.images.get(repo_url + image2_name) image2_no_url = docker_client.images.get(image2_name) + assert docker_client.images._tag_exists(image2_name, image2.tags) + assert image2 is not None assert image2_no_url is not None assert image2.id == image2_py.id assert image2_no_url.id == image2_py.id - assert image_exists(docker_client, image2_name) - assert image_exists(docker_client, repo_url + image2_name) - assert image_exists(docker_py_client, repo_url + image2_name) + # Check we are not overmatching but we are getting all possible cases + docker_host = os.environ.get("DOCKER_HOST", "") + + podman = False + if "podman" in docker_host: + podman = True + + for client in [docker_client, docker_py_client]: + assert not image_exists(client, "create_image") + + if (podman and client == docker_py_client) or client == docker_client: + # Docker-py does not support partial string matching + assert image_exists(client, image2_name) + assert image_exists(client, f"/{image2_name}") + assert image_exists(client, f"bar/{image2_name}") + assert image_exists(client, f"bar/{image2_name}:latest") + assert not image_exists(client, f"ar/{image2_name}") + assert image_exists(client, f"foo/bar/{image2_name}") + + assert image_exists(client, repo_url + image2_name) finally: # Clean up the images @@ -252,65 +274,60 @@ def test_create_container( def test_container_volume_mounts( - docker_client, docker_client_built_image_name, tmp_path + docker_client, docker_cleanup, docker_client_built_image_name, tmp_path ): """Test container volume mounts.""" container1 = None - try: - # Pytest creates the tmp_path fixture with drwx------ mode, we need others - # to be able to read and execute the path so the Docker volume is readable - # from within the container - tmp_path.chmod(0o0707) - - dest = Path("/foo/") - bar_file = dest / "hello.txt" - stdout = docker_client.containers.run( - docker_client_built_image_name, - [f"touch {bar_file} && chmod 777 {bar_file} && echo hello"], - detach=False, - volumes={tmp_path: {"bind": dest, "mode": "rw"}}, - remove=True, - ) + # Pytest creates the tmp_path fixture with drwx------ mode, we need others + # to be able to read and execute the path so the Docker volume is readable + # from within the container + tmp_path.chmod(0o0707) + + dest = Path("/foo/") + bar_file = dest / "hello.txt" + stdout = docker_client.containers.run( + docker_client_built_image_name, + [f"touch {bar_file} && chmod 777 {bar_file} && echo hello"], + detach=False, + volumes={tmp_path: {"bind": dest, "mode": "rw"}}, + remove=True, + ) - assert stdout == "hello\n" - # Check file exists in tmp path - assert (tmp_path / "hello.txt").exists() - - # Check container is removed and there are no running containers associated with the test image - result = subprocess.run( - [ - "docker", - "ps", - "--filter", - f"ancestor={docker_client_built_image_name}", - "-q", - ], - capture_output=True, - text=True, - check=True, - ) - assert result.stdout == "" - - # Open tmp_path/hello.txt in write mode - with open(tmp_path / "hello.txt", "w") as f: - f.write("hello tesseract\n") - - # Check we can read it in another container - container1 = docker_client.containers.run( - docker_client_built_image_name, - [f"cat {dest}/hello.txt"], - detach=True, - volumes={tmp_path: {"bind": dest, "mode": "rw"}}, - ) - status = container1.wait() - assert status["StatusCode"] == 0 - stdout = container1.logs(stdout=True, stderr=False) - assert stdout == b"hello tesseract\n" + assert stdout == "hello\n" + # Check file exists in tmp path + assert (tmp_path / "hello.txt").exists() - finally: - # Clean up the container - if container1: - container1.remove(v=True, force=True) + # Check container is removed and there are no running containers associated with the test image + result = subprocess.run( + [ + "docker", + "ps", + "--filter", + f"ancestor={docker_client_built_image_name}", + "-q", + ], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout == "" + + # Open tmp_path/hello.txt in write mode + with open(tmp_path / "hello.txt", "w") as f: + f.write("hello tesseract\n") + + # Check we can read it in another container + container1 = docker_client.containers.run( + docker_client_built_image_name, + [f"cat {dest}/hello.txt"], + detach=True, + volumes={tmp_path: {"bind": dest, "mode": "rw"}}, + ) + docker_cleanup["containers"].append(container1) + status = container1.wait() + assert status["StatusCode"] == 0 + stdout = container1.logs(stdout=True, stderr=False) + assert stdout == b"hello tesseract\n" def test_compose_up_down( diff --git a/tests/endtoend_tests/test_endtoend.py b/tests/endtoend_tests/test_endtoend.py index 471dffb0..36774b9a 100644 --- a/tests/endtoend_tests/test_endtoend.py +++ b/tests/endtoend_tests/test_endtoend.py @@ -17,12 +17,17 @@ @pytest.fixture(scope="module") def built_image_name( - docker_client, docker_cleanup, shared_dummy_image_name, dummy_tesseract_location + docker_client, + docker_cleanup_module, + shared_dummy_image_name, + dummy_tesseract_location, ): """Build the dummy Tesseract image for the tests.""" - image_name = build_tesseract(dummy_tesseract_location, shared_dummy_image_name) + image_name = build_tesseract( + docker_client, dummy_tesseract_location, shared_dummy_image_name + ) assert image_exists(docker_client, image_name) - docker_cleanup["images"].append(image_name) + docker_cleanup_module["images"].append(image_name) yield image_name @@ -59,10 +64,15 @@ def test_build_from_init_endtoend( config_override["build_config.base_image"] = base_image image_name = build_tesseract( - tmp_path, dummy_image_name, config_override=config_override, tag=img_tag + docker_client, + tmp_path, + dummy_image_name, + config_override=config_override, + tag=img_tag, ) - assert image_exists(docker_client, image_name) + docker_cleanup["images"].append(image_name) + assert image_exists(docker_client, image_name) # Test that the image can be run and that --help is forwarded correctly result = cli_runner.invoke( diff --git a/tests/endtoend_tests/test_examples.py b/tests/endtoend_tests/test_examples.py index ac636043..e53a9a11 100644 --- a/tests/endtoend_tests/test_examples.py +++ b/tests/endtoend_tests/test_examples.py @@ -21,7 +21,7 @@ import numpy.typing as npt import pytest import requests -from common import build_tesseract +from common import build_tesseract, image_exists from typer.testing import CliRunner @@ -739,15 +739,6 @@ def unit_tesseract_config(unit_tesseract_names, unit_tesseract_path): return TEST_CASES[unit_tesseract_path.name] -def image_exists(client, image_name): - # Docker images may be prefixed with the registry URL - return any( - tag.split("/")[-1] == image_name - for img in client.images.list() - for tag in img.tags - ) - - def print_debug_info(result): """Print debug info from result of a CLI command if it failed.""" if result.exit_code == 0: @@ -816,6 +807,7 @@ def test_unit_tesseract_endtoend( # Stage 1: Build img_name = build_tesseract( + docker_client, unit_tesseract_path, dummy_image_name, tag="sometag", diff --git a/tests/endtoend_tests/test_tesseract_sdk.py b/tests/endtoend_tests/test_tesseract_sdk.py index a6fd23aa..b05b2738 100644 --- a/tests/endtoend_tests/test_tesseract_sdk.py +++ b/tests/endtoend_tests/test_tesseract_sdk.py @@ -24,12 +24,17 @@ @pytest.fixture(scope="module") def built_image_name( - docker_client, shared_dummy_image_name, dummy_tesseract_location, docker_cleanup + docker_client, + shared_dummy_image_name, + dummy_tesseract_location, + docker_cleanup_module, ): """Build the dummy Tesseract image for the tests.""" - image_name = build_tesseract(dummy_tesseract_location, shared_dummy_image_name) + image_name = build_tesseract( + docker_client, dummy_tesseract_location, shared_dummy_image_name + ) assert image_exists(docker_client, image_name) - docker_cleanup["images"].append(image_name) + docker_cleanup_module["images"].append(image_name) yield image_name