Improve caching strategy for inventory cache (#49693)

* Improve caching strategy for inventory cache

Inventory cache is used to decrease the potential failures coming
from not-available (Tempoararily) external inventories of intersphinx.

This includes both - airflow package inventory and external intersphinx
docs as well.

We are more concerned about availability of third-party inventories
than ours, so we should use the cache mostly for the external packages
rather than for ours and we should refresh it every time any dependency
changes.

The caching strategy implemented originally is to keep inventories for
12 hours, but this might cause 12 hours of failures in CI since we are
using latest cached inventories for stability.

This change adds --refresh-airflow-inventories flag and uses it in
the CI to always refresh airflow inventories and leave the other
cached inventories.

* fixup! Improve caching strategy for inventory cache

Author: potiuk

.github/workflows/ci-image-checks.yml

@@ -281,19 +281,18 @@ jobs:
         uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
         with:
           path: ./generated/_inventory_cache/
-          # TODO(potiuk): do better with determining the key
-          key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
+          key: cache-docs-inventory-v1-${{ hashFiles('**/pyproject.toml') }}
         id: restore-docs-inventory-cache
       - name: "Building docs with ${{ matrix.flag }} flag"
         env:
           DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
         run: >
-          breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }}
+          breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }} --refresh-airflow-inventories
       - name: "Save docs inventory cache"
         uses: apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
         with:
           path: ./generated/_inventory_cache/
-          key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
+          key: cache-docs-inventory-v1-${{ hashFiles('**/pyproject.toml') }}
           if-no-files-found: 'error'
           retention-days: '2'
         # If we upload from multiple matrix jobs we could end up with a race condition. so just pick one job

dev/breeze/doc/images/output_build-docs.svg

@@ -1 +1 @@
-8ae2b68e8981f5fc57c32b7e064211bd
+0da87f68c7956d7611372a00164b6c5c

dev/breeze/doc/images/output_build-docs.txt

@@ -667,9 +667,15 @@ def start_airflow(
 @option_builder
 @click.option(
     "--clean-build",
-    help="Clean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts "
-    "before the build - useful for a clean build.",
     is_flag=True,
+    help="Cleans the build directory before building the documentation and removes all inventory "
+    "cache (including external inventories).",
+)
+@click.option(
+    "--refresh-airflow-inventories",
+    is_flag=True,
+    help="When set, only airflow package inventories will be refreshed, regardless "
+    "if they are already downloaded. With `--clean-build` - everything is cleaned..",
 )
 @click.option("-d", "--docs-only", help="Only build documentation.", is_flag=True)
 @option_dry_run
@@ -704,6 +710,7 @@ def start_airflow(
 def build_docs(
     builder: str,
     clean_build: bool,
+    refresh_airflow_inventories: bool,
     docs_only: bool,
     github_repository: str,
     include_not_ready_providers: bool,
@@ -730,9 +737,17 @@ def build_docs(
         directories_to_clean = ["apis"]
     generated_path = AIRFLOW_ROOT_PATH / "generated"
     for dir_name in directories_to_clean:
+        get_console().print("Removing all generated dirs.")
         for directory in generated_path.rglob(dir_name):
             get_console().print(f"[info]Removing {directory}")
             shutil.rmtree(directory, ignore_errors=True)
+    if refresh_airflow_inventories and not clean_build:
+        get_console().print("Removing airflow inventories.")
+        package_globs = ["helm-chart", "docker-stack", "apache-airflow*"]
+        for package_glob in package_globs:
+            for directory in (generated_path / "_inventory_cache").rglob(package_glob):
+                get_console().print(f"[info]Removing {directory}")
+                shutil.rmtree(directory, ignore_errors=True)
 
     docs_list_as_tuple: tuple[str, ...] = ()
     if distributions_list and len(distributions_list):

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -304,15 +304,28 @@
     ],
     "breeze build-docs": [
         {
-            "name": "Doc flags",
+            "name": "Build scope (default is to build docs and spellcheck)",
+            "options": ["--docs-only", "--spellcheck-only"],
+        },
+        {
+            "name": "Type of build",
+            "options": ["--one-pass-only"],
+        },
+        {
+            "name": "Cleaning inventories",
+            "options": ["--clean-build", "--refresh-airflow-inventories"],
+        },
+        {
+            "name": "Filtering options",
             "options": [
-                "--docs-only",
-                "--spellcheck-only",
-                "--clean-build",
-                "--one-pass-only",
                 "--package-filter",
                 "--include-not-ready-providers",
                 "--include-removed-providers",
+            ],
+        },
+        {
+            "name": "Misc options",
+            "options": [
                 "--github-repository",
                 "--builder",
                 "--distributions-list",

dev/breeze/src/airflow_breeze/commands/developer_commands_config.py

@@ -132,12 +132,19 @@ def _promote_new_flags():
     console.print("You can also use other extra flags to iterate faster:")
     console.print("   [bright_blue]--docs-only       - Only build documentation[/]")
     console.print("   [bright_blue]--spellcheck-only - Only perform spellchecking[/]")
-    console.print("   [bright_blue]--clean-build     - Refresh inventories for inter-sphinx references[/]")
     console.print()
     console.print("You can list all packages you can build:")
     console.print()
     console.print("   [bright_blue]--list-packages   - Shows the list of packages you can build[/]")
     console.print()
+    console.print("You can run clean build - refreshing inter-sphinx inventories or refresh airflow ones.\n")
+    console.print(
+        "   [bright_blue]--clean-build                 - Refresh inventories and build files for all inter-sphinx references (including external ones)[/]"
+    )
+    console.print(
+        "   [bright_blue]--refresh-airflow-inventories - Force refresh only airflow inventories (without cleaning build files or external inventories).[/]"
+    )
+    console.print()
     console.print("For more info:")
     console.print("   [bright_blue]uv run build-docs --help[/]")
     console.print()
@@ -457,7 +464,11 @@ def is_command_available(command):
         },
         {
             "name": "Type of build",
-            "options": ["--autobuild", "--one-pass-only", "--clean-build"],
+            "options": ["--autobuild", "--one-pass-only"],
+        },
+        {
+            "name": "Cleaning inventories",
+            "options": ["--clean-build", "--refresh-airflow-inventories"],
         },
         {
             "name": "Filtering options",
@@ -497,9 +508,6 @@ def is_command_available(command):
     "only that package is selected to build. "
     "If the command is run in the root of the Airflow repo, all packages are selected to be built.",
 )
-@click.option(
-    "--clean-build", is_flag=True, help="Cleans the build directory before building the documentation."
-)
 @click.option("--docs-only", is_flag=True, help="Only build documentation")
 @click.option("--spellcheck-only", is_flag=True, help="Only perform spellchecking")
 @click.option(
@@ -516,6 +524,18 @@ def is_command_available(command):
     is_flag=True,
     help="Lists all available packages. You can use it to check the names of the packages you want to build.",
 )
+@click.option(
+    "--clean-build",
+    is_flag=True,
+    help="Cleans the build directory before building the documentation and removes all inventory "
+    "cache (including external inventories).",
+)
+@click.option(
+    "--refresh-airflow-inventories",
+    is_flag=True,
+    help="When set, only airflow package inventories will be refreshed, regardless "
+    "if they are already downloaded. With `--clean-build` - everything is cleaned..",
+)
 @click.option(
     "-v",
     "--verbose",
@@ -537,6 +557,7 @@ def build_docs(
     spellcheck_only,
     jobs,
     list_packages,
+    refresh_airflow_inventories,
     verbose,
     packages,
 ):
@@ -579,7 +600,9 @@ def build_docs(
     with with_group("Fetching inventories"):
         # Inventories that could not be retrieved should be built first. This may mean this is a
         # new package.
-        packages_without_inventories = fetch_inventories(clean_build=clean_build)
+        packages_without_inventories = fetch_inventories(
+            clean_build=clean_build, refresh_airflow_inventories=refresh_airflow_inventories
+        )
     normal_packages, priority_packages = partition(
         lambda d: d in packages_without_inventories, packages_to_build
     )

devel-common/src/docs/build_docs.py

@@ -93,7 +93,13 @@ def _is_outdated(path: str):
     return delta > datetime.timedelta(hours=12)
 
 
-def fetch_inventories(clean_build: bool) -> list[str]:
+def should_be_refreshed(pkg_name: str, refresh_airflow_inventories: bool) -> bool:
+    if pkg_name in ["helm-chart", "docker-stack"] or pkg_name.startswith("apache-airflow"):
+        return refresh_airflow_inventories
+    return False
+
+
+def fetch_inventories(clean_build: bool, refresh_airflow_inventories: bool = False) -> list[str]:
     """Fetch all inventories for Airflow documentation packages and store in cache."""
     if clean_build:
         shutil.rmtree(CACHE_PATH)
@@ -136,7 +142,11 @@ def fetch_inventories(clean_build: bool) -> list[str]:
         for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()
     )
 
-    to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]
+    to_download = [
+        (pkg_name, url, path)
+        for pkg_name, url, path in to_download
+        if _is_outdated(path) or should_be_refreshed(pkg_name, refresh_airflow_inventories)
+    ]
     if not to_download:
         print("Nothing to do")
         return []