From e0a0418170cfc3667eae2e96a17dc27f8d85727c Mon Sep 17 00:00:00 2001 From: Haoran Geng <71596067+geng-haoran@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:37:33 +0000 Subject: [PATCH] fix(newton): support newton 1.5/1.6 (removed attributes, target layout, tiled camera) The pinned newton (git main, 1.6.0.dev0) could not build a scene: Model.joint_target_pos was removed in 1.5, the tiled-camera API changed in 1.4, Model.num_worlds was renamed in 1.6, and position targets moved to the joint_q layout so every env after a free joint received its targets three joints off. Version-guarded shims and index tables fix all four; CameraState.depth is emitted as (N, H, W). Verified: 82 newton tests pass and the four newton get_started cases render. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_017i6VtKoovBNed815mWFqxw --- CHANGELOG.md | 1 + metasim/sim/newton/_newton_compat.py | 68 +++++++++++ metasim/sim/newton/newton.py | 133 ++++++++++++++------- metasim/test/test_newton_compat_general.py | 113 +++++++++++++++++ 4 files changed, 275 insertions(+), 40 deletions(-) create mode 100644 metasim/test/test_newton_compat_general.py diff --git a/CHANGELOG.md b/CHANGELOG.md index eb0177d..4f62ce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - IsaacGym and PyBullet reported `joint_pos_target` in native DoF order while `joint_pos` is in sorted-name order; both now use `get_joint_names(sort=True)` (completes #12). - `ParallelSimWrapper`: a worker that died during handler construction or `launch` surfaced as a bare `EOFError`/`ConnectionResetError` from the handshake and left the other workers running; the handshake now raises the worker's own traceback, `close()` tolerates dead workers, and a failed constructor tears the pool down. - `hf_util.check_and_download_single`: a `roboverse_data/...` path evaluated from a working directory that is not the parent of `ROBOVERSE_DATA_DIR` was reported as a path-traversal attempt; the error now names the CWD / `ROBOVERSE_DATA_DIR` mismatch and where the asset already is. `test_check_and_download_single_falls_back_to_private_roboverse_data` no longer depends on the caller's `ROBOVERSE_DATA_DIR`. +- Newton backend works again on the pinned newton (1.5/1.6): `joint_target_pos`/`joint_target_vel` and `num_worlds` are forwarded to their new names, position targets use `joint_target_q_start` (on newton >= 1.5 they follow the `joint_q` layout, so envs after a free-floating object were driven to the wrong joints), the tiled camera uses the 1.4+ `RenderConfig`/`utils`/`update` API with an untextured fallback, and `CameraState.depth` is `(N, H, W)`. - MuJoCo: `` is reserved by default; humanoid + mesh scenes no longer die with `mj_stackAlloc: out of memory` (get_started/10_mount_camera.py). - `hf_util`: a symlinked `roboverse_data` is no longer refused as path traversal; concurrent diff --git a/metasim/sim/newton/_newton_compat.py b/metasim/sim/newton/_newton_compat.py index ba931f3..91310a8 100644 --- a/metasim/sim/newton/_newton_compat.py +++ b/metasim/sim/newton/_newton_compat.py @@ -18,6 +18,9 @@ - Model attribute aliases body_key / joint_key / shape_key -> *_label - add_mjcf(builder, path, **kw): ModelBuilder.add_mjcf method removed in 1.2 but parse_mjcf in newton._src.utils.import_mjcf still works +- Model/Control joint_target_pos / joint_target_vel -> joint_target_q / + joint_target_qd (removed in 1.5; the new names are forwarded read/write) +- Model.num_worlds -> Model.world_count (renamed in 1.6) Backward-compat guarantee: - Anything that worked on newton < 1.2 still works. @@ -123,6 +126,71 @@ def _install_model_attr_aliases(): _install_model_attr_aliases() +def _install_removed_attr_aliases(): + """Route attributes newton 1.5 removed back to their replacements. + + newton 1.5 renamed ``joint_target_pos`` / ``joint_target_vel`` to ``joint_target_q`` / + ``joint_target_qd`` on both ``Model`` and ``Control`` and shadows the old names with a + ``RemovedAttribute`` descriptor that raises on read *and* write. The handler reads and + assigns both, so install read/write properties that forward to the new instance attributes. + Older newton (real attributes, no descriptor) is left untouched. + """ + try: + import newton + except ImportError: + return + renames = [("joint_target_pos", "joint_target_q"), ("joint_target_vel", "joint_target_qd")] + for cls_name in ("Model", "Control"): + cls = getattr(newton, cls_name, None) + if cls is None: + continue + for old, new in renames: + current = cls.__dict__.get(old) + if current is None or type(current).__name__ != "RemovedAttribute": + continue + + def _get(self, _n=new): + return getattr(self, _n) + + def _set(self, value, _n=new): + setattr(self, _n, value) + + setattr(cls, old, property(_get, _set)) + + +_install_removed_attr_aliases() + + +def _install_renamed_attr_aliases(): + """Forward attribute names newton renamed *without* leaving a descriptor behind. + + ``Model.num_worlds`` became ``Model.world_count`` in newton 1.6 and simply disappeared. The + alias reads the instance attribute when an older newton still sets it and falls back to the + new name otherwise, so both spellings work on every version. + """ + try: + import newton + except ImportError: + return + for cls_name, old, new in [("Model", "num_worlds", "world_count")]: + cls = getattr(newton, cls_name, None) + if cls is None or old in cls.__dict__: + continue + + def _get(self, _old=old, _new=new): + if _old in self.__dict__: + return self.__dict__[_old] + return getattr(self, _new) + + def _set(self, value, _old=old): + self.__dict__[_old] = value + + setattr(cls, old, property(_get, _set)) + + +_install_renamed_attr_aliases() + + def add_mjcf(builder, mjcf_path: str, **kwargs): """Add an MJCF file to ``ModelBuilder``, version-portable. diff --git a/metasim/sim/newton/newton.py b/metasim/sim/newton/newton.py index 06c066e..3b75c4e 100644 --- a/metasim/sim/newton/newton.py +++ b/metasim/sim/newton/newton.py @@ -48,6 +48,10 @@ # auto-populated by solver). See _newton_compat.py. from ._newton_compat import JointType, add_mjcf, populate_contacts, set_current_world +# newton < 1.4 exposes ``SensorTiledCamera.Options`` and a per-sensor resolution; newer releases +# render model-wide through ``sensor.utils`` / ``sensor.update``. +_TILED_CAMERA_LEGACY_API = hasattr(SensorTiledCamera, "Options") + def _physics_mode_name(obj) -> str | None: physics = getattr(obj, "physics", None) @@ -789,6 +793,13 @@ def _build_name_caches(self) -> None: # Access start indices for fast lookup self._joint_q_starts = self._model.joint_q_start.numpy() self._joint_qd_starts = self._model.joint_qd_start.numpy() + # newton >= 1.5 lays ``joint_target_q`` out like ``joint_q`` (a free joint takes 7 slots, + # not 6) and publishes ``joint_target_q_start``; older releases index position targets by + # dof. Using the dof layout on new newton shifts every env after a free joint — env 1's + # targets land three joints off. Velocity targets (``joint_target_qd``) stay dof-indexed. + target_starts = getattr(self._model, "joint_target_q_start", None) + self._joint_target_starts = target_starts.numpy() if target_starts is not None else self._joint_qd_starts + self._target_uses_q_layout = target_starts is not None self._joint_types = self._model.joint_type.numpy() # Build per-object joint name maps to disambiguate duplicate joint names @@ -829,6 +840,7 @@ def _build_state_index_caches(self) -> None: self._obj_sorted_joint_names = {} self._obj_joint_q_idx = {} self._obj_joint_qd_idx = {} + self._obj_joint_target_idx = {} self._obj_joint_valid = {} self._obj_root_joint_idx = {} self._obj_root_joint_type = {} @@ -932,6 +944,8 @@ def _build_state_index_caches(self) -> None: self._obj_joint_q_idx[name] = q_idx_tensor self._obj_joint_qd_idx[name] = qd_idx_tensor + # For 1-DoF joints the target slot is the q slot on newton >= 1.5, the qd slot before. + self._obj_joint_target_idx[name] = q_idx_tensor if self._target_uses_q_layout else qd_idx_tensor self._obj_joint_valid[name] = valid_mask if self._joint_q_starts is not None: @@ -1138,7 +1152,7 @@ def _apply_actuator_settings(self) -> None: # Initialize target position from the current joint state for 1-DoF joints if qd_end - qd_start == 1: q_start = self._joint_q_starts[joint_idx] - joint_target_pos[qd_start] = joint_q[q_start] + joint_target_pos[self._joint_target_starts[joint_idx]] = joint_q[q_start] joint_target_vel[qd_start] = 0.0 updated = True @@ -1432,26 +1446,49 @@ def _init_tiled_cameras(self) -> None: try: with wp.ScopedDevice(self._model.device): for (width, height), cam_indices in cam_groups.items(): - sensor = SensorTiledCamera( - model=self._model, - num_cameras=len(cam_indices), - width=width, - height=height, - options=SensorTiledCamera.Options( - default_light=True, - default_light_shadows=True, - ), - ) - if self._shape_color_overrides: - colors = np.ones((self._model.shape_count, 4), dtype=np.float32) - for shape_idx, color in self._shape_color_overrides.items(): - if 0 <= shape_idx < colors.shape[0]: - colors[shape_idx] = color - sensor.render_context.shape_colors = wp.array(colors, dtype=wp.vec4f) fovs = [math.radians(self.scenario.cameras[i].vertical_fov) for i in cam_indices] - camera_rays = sensor.compute_pinhole_camera_rays(fovs) - color_image = sensor.create_color_image_output() - depth_image = sensor.create_depth_image_output() + if _TILED_CAMERA_LEGACY_API: + sensor = SensorTiledCamera( + model=self._model, + num_cameras=len(cam_indices), + width=width, + height=height, + options=SensorTiledCamera.Options( + default_light=True, + default_light_shadows=True, + ), + ) + if self._shape_color_overrides: + colors = np.ones((self._model.shape_count, 4), dtype=np.float32) + for shape_idx, color in self._shape_color_overrides.items(): + if 0 <= shape_idx < colors.shape[0]: + colors[shape_idx] = color + sensor.render_context.shape_colors = wp.array(colors, dtype=wp.vec4f) + camera_rays = sensor.compute_pinhole_camera_rays(fovs) + color_image = sensor.create_color_image_output() + depth_image = sensor.create_depth_image_output() + else: + # newton >= 1.4: the sensor is model-wide; resolution, rays, outputs and + # lights come from ``sensor.utils`` and rendering is ``sensor.update``. + render_config = SensorTiledCamera.RenderConfig(enable_shadows=True) + try: + sensor = SensorTiledCamera(model=self._model, default_render_config=render_config) + except Exception as texture_err: # e.g. RGB textures where the sensor expects RGBA + log.warning( + f"SensorTiledCamera could not load textures ({texture_err}); rendering untextured." + ) + sensor = SensorTiledCamera( + model=self._model, default_render_config=render_config, load_textures=False + ) + sensor.utils.create_default_light(enable_shadows=True) + if self._shape_color_overrides: + log.warning( + "Per-shape color overrides are not supported by this newton version's " + "SensorTiledCamera; rendering with model colors." + ) + camera_rays = sensor.utils.compute_camera_rays_pinhole(width, height, camera_fovs=fovs) + color_image = sensor.utils.create_color_image_output(width, height, len(cam_indices)) + depth_image = sensor.utils.create_depth_image_output(width, height, len(cam_indices)) camera_transforms = self._build_camera_transforms(cam_indices) self._camera_groups.append({ @@ -1487,13 +1524,24 @@ def _render_tiled_cameras(self, env_ids: list[int]) -> dict: with wp.ScopedDevice(self._model.device): for group in self._camera_groups: sensor: SensorTiledCamera = group["sensor"] - sensor.render( - self._state_0, - group["camera_transforms"], - group["camera_rays"], - color_image=group["color_image"], - depth_image=group["depth_image"], - ) + if _TILED_CAMERA_LEGACY_API: + sensor.render( + self._state_0, + group["camera_transforms"], + group["camera_rays"], + color_image=group["color_image"], + depth_image=group["depth_image"], + ) + else: + # Shape BVHs are built for the initial state at finalize; refit for moved bodies. + self._model.bvh_refit_shapes(self._state_0) + sensor.update( + self._state_0, + group["camera_transforms"], + group["camera_rays"], + color_image=group["color_image"], + depth_image=group["depth_image"], + ) color_np = group["color_image"].numpy() if group["color_image"] is not None else None depth_np = group["depth_image"].numpy() if group["depth_image"] is not None else None @@ -1517,8 +1565,8 @@ def _render_tiled_cameras(self, env_ids: list[int]) -> dict: rgb_tensor = rgb_tensor[use_env_ids] if depth_np is not None: - depth_cam = depth_np[:, local_idx, :].reshape(num_worlds, height, width) - depth = depth_cam[..., None] + # CameraState.depth is (num_envs, H, W) — no trailing channel axis. + depth = depth_np[:, local_idx, :].reshape(num_worlds, height, width) depth_tensor = torch.from_numpy(depth).to(self._device) if len(use_env_ids) != num_worlds: depth_tensor = depth_tensor[use_env_ids] @@ -1687,13 +1735,14 @@ def _gather_dof(src: torch.Tensor, idx: torch.Tensor, valid_mask: torch.Tensor) valid_mask = self._obj_joint_valid[robot_name] q_idx = q_idx_all[env_ids_t] qd_idx = qd_idx_all[env_ids_t] + t_idx = self._obj_joint_target_idx[robot_name][env_ids_t] if joint_q is not None: joint_pos = _gather_dof(joint_q, q_idx, valid_mask) if joint_qd is not None: joint_vel = _gather_dof(joint_qd, qd_idx, valid_mask) if joint_target_pos is not None: - joint_pos_target = _gather_dof(joint_target_pos, qd_idx, valid_mask) + joint_pos_target = _gather_dof(joint_target_pos, t_idx, valid_mask) if joint_target_vel is not None: joint_vel_target = _gather_dof(joint_target_vel, qd_idx, valid_mask) if joint_f is not None: @@ -1716,7 +1765,7 @@ def _gather_dof(src: torch.Tensor, idx: torch.Tensor, valid_mask: torch.Tensor) if joint_qd is not None: joint_vel[row, col] = joint_qd[qd_start] if joint_target_pos is not None: - joint_pos_target[row, col] = joint_target_pos[qd_start] + joint_pos_target[row, col] = joint_target_pos[self._joint_target_starts[joint_idx]] if joint_target_vel is not None: joint_vel_target[row, col] = joint_target_vel[qd_start] if joint_f is not None: @@ -1990,7 +2039,7 @@ def to_np(x): dirty_joint_vels = True if control_joint_target_pos is not None: - control_joint_target_pos[qd_start] = values[0] + control_joint_target_pos[self._joint_target_starts[j_idx]] = values[0] if control_joint_target_vel is not None: control_joint_target_vel[qd_start] = 0.0 @@ -2231,7 +2280,8 @@ def _apply_root_state(name: str, root_state: torch.Tensor) -> None: _scatter_values(joint_qd_targets, qd_idx, torch.zeros_like(joint_pos), valid_mask) if control_joint_target_pos is not None: - _scatter_values([control_joint_target_pos], qd_idx, joint_pos, valid_mask) + t_idx = self._obj_joint_target_idx[robot.name][env_ids_t] + _scatter_values([control_joint_target_pos], t_idx, joint_pos, valid_mask) if control_joint_target_vel is not None: if joint_vel is None: joint_vel = torch.zeros_like(joint_pos) @@ -2321,8 +2371,10 @@ def _set_dof_targets(self, actions: CompatActionInput) -> None: continue qd_idx = qd_idx_all[:robot_env_count, :max_joints] + t_idx = self._obj_joint_target_idx[robot_name][:robot_env_count, :max_joints] if qd_idx.device != target_device: qd_idx = qd_idx.to(target_device) + t_idx = t_idx.to(target_device) action_slice = robot_actions[:robot_env_count, :max_joints] if isinstance(valid_mask, torch.Tensor): @@ -2330,18 +2382,19 @@ def _set_dof_targets(self, actions: CompatActionInput) -> None: mask = valid_mask[:max_joints] if mask.device != target_device: mask = mask.to(target_device) - qd_idx_sel = qd_idx[:, mask] - action_sel = action_slice[:, mask] - idx_flat = qd_idx_sel.reshape(-1) - val_flat = action_sel.reshape(-1) + idx_flat = qd_idx[:, mask].reshape(-1) + tidx_flat = t_idx[:, mask].reshape(-1) + val_flat = action_slice[:, mask].reshape(-1) else: mask = valid_mask[:robot_env_count, :max_joints] if mask.device != target_device: mask = mask.to(target_device) idx_flat = qd_idx[mask] + tidx_flat = t_idx[mask] val_flat = action_slice[mask] else: idx_flat = qd_idx.reshape(-1) + tidx_flat = t_idx.reshape(-1) val_flat = action_slice.reshape(-1) if idx_flat.numel() == 0: @@ -2351,7 +2404,7 @@ def _set_dof_targets(self, actions: CompatActionInput) -> None: joint_f[idx_flat] = val_flat else: if joint_target_pos is not None: - joint_target_pos[idx_flat] = val_flat + joint_target_pos[tidx_flat] = val_flat if joint_target_vel is not None: joint_target_vel[idx_flat] = 0.0 continue @@ -2373,7 +2426,7 @@ def _set_dof_targets(self, actions: CompatActionInput) -> None: joint_f[qd_start] = value else: if joint_target_pos is not None: - joint_target_pos[qd_start] = value + joint_target_pos[self._joint_target_starts[joint_idx]] = value if joint_target_vel is not None: joint_target_vel[qd_start] = 0.0 return @@ -2399,7 +2452,7 @@ def _set_dof_targets(self, actions: CompatActionInput) -> None: values = self._coerce_dof_values(target, qd_end - qd_start) if values is None: continue - joint_target_pos[qd_start] = values[0] + joint_target_pos[self._joint_target_starts[joint_idx]] = values[0] if joint_target_vel is not None and joint_name not in dof_vel_target: joint_target_vel[qd_start] = 0.0 diff --git a/metasim/test/test_newton_compat_general.py b/metasim/test/test_newton_compat_general.py new file mode 100644 index 0000000..8891aa6 --- /dev/null +++ b/metasim/test/test_newton_compat_general.py @@ -0,0 +1,113 @@ +"""The newton compat shims forward renamed / removed attributes on both old and new newton APIs. + +newton 1.5 replaced ``Model.joint_target_pos`` / ``joint_target_vel`` (and the same on ``Control``) +with ``joint_target_q`` / ``joint_target_qd`` behind a ``RemovedAttribute`` descriptor that raises on +read and write; 1.6 renamed ``Model.num_worlds`` to ``world_count`` outright. The shims are exercised +against a synthetic ``newton`` module so the test runs without the engine installed. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys +import types + +import pytest + + +class RemovedAttribute: + """Mirror of ``newton._src.utils.deprecation.RemovedAttribute`` (data descriptor that raises).""" + + def __init__(self, replacement: str): + self._message = f"removed; use {replacement} instead" + + def __get__(self, instance, owner=None): + raise AttributeError(self._message) + + def __set__(self, instance, value): + raise AttributeError(self._message) + + +def _fake_newton(*, removed: bool, renamed: bool) -> types.ModuleType: + mod = types.ModuleType("newton") + + class Model: + def __init__(self): + if renamed: + self.world_count = 4 + else: + self.num_worlds = 4 + if removed: + self.joint_target_q = "q" + self.joint_target_qd = "qd" + else: + self.joint_target_pos = "q" + self.joint_target_vel = "qd" + + class Control: + def __init__(self): + if removed: + self.joint_target_q = "cq" + else: + self.joint_target_pos = "cq" + + if removed: + Model.joint_target_pos = RemovedAttribute("joint_target_q") + Model.joint_target_vel = RemovedAttribute("joint_target_qd") + Control.joint_target_pos = RemovedAttribute("joint_target_q") + Control.joint_target_vel = RemovedAttribute("joint_target_qd") + + class ModelBuilder: + pass + + mod.Model, mod.Control, mod.ModelBuilder = Model, Control, ModelBuilder + mod.JointType = object + mod.sensors = types.ModuleType("newton.sensors") # no populate_contacts: exercises the >=1.2 path + return mod + + +def _load_compat(monkeypatch, fake): + """Load ``_newton_compat.py`` against ``fake`` as the ``newton`` package. + + Loaded from its file so ``metasim.sim.newton/__init__`` (which imports the real engine) stays out + of the picture. + """ + for name in list(sys.modules): + if name == "newton" or name.startswith("newton."): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "newton", fake) + monkeypatch.setitem(sys.modules, "newton.sensors", fake.sensors) + import metasim + + path = os.path.join(os.path.dirname(metasim.__file__), "sim", "newton", "_newton_compat.py") + spec = importlib.util.spec_from_file_location("_newton_compat_under_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.general +def test_removed_joint_target_attributes_forward_to_new_names(monkeypatch): + fake = _fake_newton(removed=True, renamed=True) + compat = _load_compat(monkeypatch, fake) + compat._install_removed_attr_aliases() + compat._install_renamed_attr_aliases() + model, control = fake.Model(), fake.Control() + assert model.joint_target_pos == "q" and model.joint_target_vel == "qd" + assert control.joint_target_pos == "cq" + model.joint_target_pos = "new" + assert model.joint_target_q == "new", "writes must land on the new attribute, not a dead shadow" + assert model.num_worlds == 4 + + +@pytest.mark.general +def test_old_newton_attributes_are_left_alone(monkeypatch): + fake = _fake_newton(removed=False, renamed=False) + compat = _load_compat(monkeypatch, fake) + compat._install_removed_attr_aliases() + compat._install_renamed_attr_aliases() + model = fake.Model() + assert model.joint_target_pos == "q" and model.num_worlds == 4 + model.num_worlds = 8 + assert model.num_worlds == 8