modosaic.depth.validators.impl.imagebind

modosaic.depth.validators.impl.imagebind

ImageBindValidator

ImageBindValidator(generation_model)

Bases: DepthValidator[float]

Validate depth-image consistency with ImageBind embeddings.

Initialize the validator.

Parameters:

Name	Type	Description	Default
generation_model	DepthGenerationModel	Depth model used to generate the depth map. The value informs metric-vs-relative depth normalization.	required

Source code in modosaic/depth/validators/impl/imagebind.py

def __init__(self, generation_model: DepthGenerationModel) -> None:
    """Initialize the validator.

    Args:
        generation_model: Depth model used to generate the depth map. The
            value informs metric-vs-relative depth normalization.
    """
    self.generation_model = generation_model
    self.device, self.dtype = DeviceService.get_device_type()
    self.model = self._load_model()

depth_np_to_imagebind_depth

depth_np_to_imagebind_depth(depth, assume_metric=None, prefer_disparity=True, use_log_for_metric=True, p_lo=1.0, p_hi=99.0, eps=1e-06)

Convert a depth map to ImageBind depth input format.

Parameters:

Name	Type	Description	Default
depth	ndarray	Generated depth map.	required
assume_metric	bool | None	Whether to treat values as metric depth. If omitted, a heuristic is used.	None
prefer_disparity	bool	Whether metric depth should be converted to inverse-depth style values.	True
use_log_for_metric	bool	Whether metric depth should be log-compressed when prefer_disparity is false.	True
p_lo	float	Lower percentile for robust normalization.	1.0
p_hi	float	Upper percentile for robust normalization.	99.0
eps	float	Numerical stability constant.	1e-06

Returns:

Type	Description
Tensor	Torch tensor ready for ImageBind depth input.

Raises:

Type	Description
ValueError	If the depth map has no finite values.

Source code in modosaic/depth/validators/impl/imagebind.py

def depth_np_to_imagebind_depth(
        self,
        depth: np.ndarray,
        assume_metric: bool | None = None,
        prefer_disparity: bool = True,
        use_log_for_metric: bool = True,
        p_lo: float = 1.0,
        p_hi: float = 99.0,
        eps: float = 1e-6,
) -> torch.Tensor:
    """Convert a depth map to ImageBind depth input format.

    Args:
        depth: Generated depth map.
        assume_metric: Whether to treat values as metric depth. If omitted,
            a heuristic is used.
        prefer_disparity: Whether metric depth should be converted to
            inverse-depth style values.
        use_log_for_metric: Whether metric depth should be log-compressed
            when `prefer_disparity` is false.
        p_lo: Lower percentile for robust normalization.
        p_hi: Upper percentile for robust normalization.
        eps: Numerical stability constant.

    Returns:
        Torch tensor ready for ImageBind depth input.

    Raises:
        ValueError: If the depth map has no finite values.
    """
    d = BoundaryService.coerce_hw_float(depth, convert_np=False)
    mask = np.isfinite(d)
    if not mask.any():
        raise ValueError("Depth has no finite values.")

    # Infer metric-like scale when caller-provided model metadata is unavailable.
    if assume_metric is None:
        # Metric depth maps are usually positive and bounded to a physical
        # range, while relative maps may use arbitrary or normalized units.
        d99 = np.nanpercentile(d, 99)
        d01 = np.nanpercentile(d, 1)
        # Treat the map as metric-like when the robust range is positive and bounded.
        assume_metric = (d01 >= 0.0 and 0.05 < d99 < 500.0)

    # --- canonicalize representation ---
    dc = d.copy()

    if assume_metric:
        # Metric depth should be positive
        dc = np.where(mask, np.maximum(dc, eps), np.nan)

        if prefer_disparity:
            # disparity-like (inverse depth)
            dc = 1.0 / (dc + eps)
        else:
            # keep depth, maybe log-compress
            if use_log_for_metric:
                dc = np.log(dc + eps)
    else:
        # Relative depth may encode either distance or inverse-distance
        # ordering. Preserve the original orientation and robust-normalize.
        pass

    # --- robust normalize to roughly standard scale ---
    lo = np.nanpercentile(dc, p_lo)
    hi = np.nanpercentile(dc, p_hi)
    if not np.isfinite(lo) or not np.isfinite(hi) or abs(hi - lo) < 1e-9:
        # fallback: simple mean/std over finite pixels
        mu = np.nanmean(dc)
        sigma = np.nanstd(dc) + eps
        dc = (dc - mu) / sigma
    else:
        dc = np.clip(dc, lo, hi)
        # map to [-1, 1] then (optionally) z-score-like scale
        dc = 2.0 * (dc - lo) / (hi - lo + eps) - 1.0

    # fill nans with 0 (center)
    dc = np.where(np.isfinite(dc), dc, 0.0).astype(np.float32)

    # to torch: 1x1xHxW -> 1x1x224x224
    x = torch.from_numpy(dc).unsqueeze(0).unsqueeze(0)  # 1,1,H,W
    x = ImageBindValidator._resize_center_crop_224(x)
    return x.to(self.device)

validate

validate(record, generated)

Score agreement between the source image and generated depth map.

Source code in modosaic/depth/validators/impl/imagebind.py

@override
def validate(self, record: ImageRecord, generated: np.ndarray) -> float:
    """Score agreement between the source image and generated depth map."""
    logger.debug(f"Validating depth-image embedding for record sample {record.sample_id}")
    inputs = self._build_inputs(record, generated)
    with torch.no_grad():
        embeddings = self.model(inputs)
        vision_embeddings = embeddings[ModalityType.VISION]
        depth_embeddings = embeddings[ModalityType.DEPTH]
        cos_sim = F.cosine_similarity(vision_embeddings, depth_embeddings).item()
        percent_like = max(0.0, min(1.0, (cos_sim + 1) / 2))
        logger.debug(f"Percent like {percent_like}")
        return percent_like