Python 3.12 support (#642)

Author: benfred

.github/workflows/build.yml

@@ -35,6 +35,9 @@ jobs:
     - name: Install Dependencies
       run: sudo apt install libunwind-dev
       if: runner.os == 'Linux'
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
     - name: Build
       run: cargo build --release --verbose --examples
     - uses: actions/setup-python@v4
@@ -180,7 +183,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6.7, 3.6.15, 3.7.1, 3.7.17, 3.8.0, 3.8.18, 3.9.0, 3.9.20, 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13, 3.10.14, 3.10.15, 3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5, 3.11.6, 3.11.7, 3.11.8, 3.11.9, 3.11.10]
+        python-version: [3.6.7, 3.6.15, 3.7.1, 3.7.17, 3.8.0, 3.8.18, 3.9.0, 3.9.20, 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13, 3.10.14, 3.10.15, 3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5, 3.11.6, 3.11.7, 3.11.8, 3.11.9, 3.11.10, 3.12.0]
         # TODO: also test windows
         os: [ubuntu-20.04, macos-13]
         # some versions of python can't be tested on GHA with osx because of SIP:
@@ -207,6 +210,8 @@ jobs:
             python-version: 3.11.9
           - os: macos-13
             python-version: 3.11.10
+          - os: macos-13
+            python-version: 3.12.0
 
     steps:
       - uses: actions/checkout@v2

generate_bindings.py

@@ -17,7 +17,8 @@ def build_python(cpython_path, version):
     print("Compiling python %s from repo at %s" % (version, cpython_path))
     install_path = os.path.abspath(os.path.join(cpython_path, version))
 
-    ret = os.system(f"""
+    ret = os.system(
+        f"""
         cd {cpython_path}
         git checkout {version}
 
@@ -27,7 +28,8 @@ def build_python(cpython_path, version):
         ../configure prefix={install_path}
         make
         make install
-    """)
+    """
+    )
     if ret:
         return ret
 
@@ -57,8 +59,9 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):
             size_t interp_head = offsetof(_PyRuntimeState, interpreters.head);
             printf("pub static INTERP_HEAD_OFFSET: usize = %i;\n", interp_head);
 
-            size_t tstate_current = offsetof(_PyRuntimeState, gilstate.tstate_current);
-            printf("pub static TSTATE_CURRENT: usize = %i;\n", tstate_current);
+            // tstate_current has been replaced by a thread-local variable in python 3.12
+            // size_t tstate_current = offsetof(_PyRuntimeState, gilstate.tstate_current);
+            // printf("pub static TSTATE_CURRENT: usize = %i;\n", tstate_current);
         }
     """
 
@@ -88,7 +91,7 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):
         else:
             ret = os.system(f"""gcc {source_filename} -I {cpython_path} -I {cpython_path}/Include -o {exe}""")
         if ret:
-            print("Failed to compile""")
+            print("Failed to compile")
             return ret
 
         ret = os.system(exe)
@@ -100,19 +103,22 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):
 def extract_bindings(cpython_path, version, configure=False):
     print("Generating bindings for python %s from repo at %s" % (version, cpython_path))
 
-    ret = os.system(f"""
+    ret = os.system(
+        f"""
         cd {cpython_path}
         git checkout {version}
 
         # need to run configure on the current branch to generate pyconfig.h sometimes
         {("./configure prefix=" + os.path.abspath(os.path.join(cpython_path, version))) if configure else ""}
 
-        cat Include/Python.h > bindgen_input.h
-        cat Include/frameobject.h >> bindgen_input.h
+
+        echo "// autogenerated by generate_bindings.py " > bindgen_input.h
         echo '#define Py_BUILD_CORE 1\n' >> bindgen_input.h
-        cat Include/internal/pycore_pystate.h >> bindgen_input.h
+        cat Include/Python.h >> bindgen_input.h
+        echo '#undef HAVE_STD_ATOMIC' >> bindgen_input.h
+        cat Include/frameobject.h >> bindgen_input.h
         cat Include/internal/pycore_interp.h >> bindgen_input.h
-        cat Include/internal/pycore_frame.h >> bindgen_input.h
+        cat Include/internal/pycore_dict.h >> bindgen_input.h
 
         bindgen  bindgen_input.h -o bindgen_output.rs \
             --with-derive-default \
@@ -132,13 +138,12 @@ def extract_bindings(cpython_path, version, configure=False):
             --whitelist-type PyFloatObject \
             --whitelist-type PyDictObject \
             --whitelist-type PyDictKeysObject \
-            --whitelist-type PyDictKeyEntry \
-            --whitelist-type PyDictUnicodeEntry \
             --whitelist-type PyObject \
             --whitelist-type PyTypeObject \
             --whitelist-type PyHeapTypeObject \
              -- -I . -I ./Include -I ./Include/internal
-    """)
+    """
+    )
     if ret:
         return ret
 
@@ -152,36 +157,40 @@ def extract_bindings(cpython_path, version, configure=False):
         o.write("#![allow(clippy::useless_transmute)]\n")
         o.write("#![allow(clippy::default_trait_access)]\n")
         o.write("#![allow(clippy::cast_lossless)]\n")
-        o.write("#![allow(clippy::trivially_copy_pass_by_ref)]\n\n")
-        o.write("#![allow(clippy::upper_case_acronyms)]\n\n")
+        o.write("#![allow(clippy::trivially_copy_pass_by_ref)]\n")
+        o.write("#![allow(clippy::upper_case_acronyms)]\n")
+        o.write("#![allow(clippy::too_many_arguments)]\n\n")
+
         o.write(open(os.path.join(cpython_path, "bindgen_output.rs")).read())
 
 
 if __name__ == "__main__":
-
     if sys.platform.startswith("win"):
         default_cpython_path = os.path.join(os.getenv("userprofile"), "code", "cpython")
     else:
         default_cpython_path = os.path.join(os.getenv("HOME"), "code", "cpython")
 
-    parser = argparse.ArgumentParser(description="runs bindgen on cpython version",
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("--cpython", type=str, default=default_cpython_path,
-                        dest="cpython", help="path to cpython repo")
-    parser.add_argument("--configure",
-                        help="Run configure script prior to generating bindings",
-                        action="store_true")
-    parser.add_argument("--pyruntime",
-                        help="generate offsets for pyruntime",
-                        action="store_true")
-    parser.add_argument("--build",
-                        help="Build python for this version",
-                        action="store_true")
-    parser.add_argument("--all",
-                        help="Build all versions",
-                        action="store_true")
-
-    parser.add_argument("versions", type=str, nargs='*', help='versions to extract')
+    parser = argparse.ArgumentParser(
+        description="runs bindgen on cpython version",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--cpython",
+        type=str,
+        default=default_cpython_path,
+        dest="cpython",
+        help="path to cpython repo",
+    )
+    parser.add_argument(
+        "--configure",
+        help="Run configure script prior to generating bindings",
+        action="store_true",
+    )
+    parser.add_argument("--pyruntime", help="generate offsets for pyruntime", action="store_true")
+    parser.add_argument("--build", help="Build python for this version", action="store_true")
+    parser.add_argument("--all", help="Build all versions", action="store_true")
+
+    parser.add_argument("versions", type=str, nargs="*", help="versions to extract")
 
     args = parser.parse_args()
 
@@ -191,7 +200,16 @@ def extract_bindings(cpython_path, version, configure=False):
         sys.exit(1)
 
     if args.all:
-        versions = ['v3.8.0b4', 'v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
+        versions = [
+            "v3.8.0b4",
+            "v3.7.0",
+            "v3.6.6",
+            "v3.5.5",
+            "v3.4.8",
+            "v3.3.7",
+            "v3.2.6",
+            "v2.7.15",
+        ]
     else:
         versions = args.versions
         if not versions:

src/coredump.rs

@@ -15,7 +15,7 @@ use crate::binary_parser::{parse_binary, BinaryInfo};
 use crate::config::Config;
 use crate::dump::print_trace;
 use crate::python_bindings::{
-    v2_7_15, v3_10_0, v3_11_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5,
+    v2_7_15, v3_10_0, v3_11_0, v3_12_0, v3_3_7, v3_5_5, v3_6_6, v3_7_0, v3_8_0, v3_9_5,
 };
 use crate::python_data_access::format_variable;
 use crate::python_interpreters::InterpreterState;
@@ -303,6 +303,11 @@ impl PythonCoreDump {
                 minor: 11,
                 ..
             } => self._get_stack::<v3_11_0::_is>(config),
+            Version {
+                major: 3,
+                minor: 12,
+                ..
+            } => self._get_stack::<v3_12_0::_is>(config),
             _ => Err(format_err!(
                 "Unsupported version of Python: {}",
                 self.version

src/python_bindings/mod.rs

@@ -1,6 +1,7 @@
 pub mod v2_7_15;
 pub mod v3_10_0;
 pub mod v3_11_0;
+pub mod v3_12_0;
 pub mod v3_3_7;
 pub mod v3_5_5;
 pub mod v3_6_6;
@@ -71,7 +72,7 @@ pub mod pyruntime {
             } => 32,
             Version {
                 major: 3,
-                minor: 11,
+                minor: 11..=12,
                 ..
             } => 40,
             _ => 24,