import torch import torch.nn as nn import torch.nn.functional as F import matplotlib.pyplot as plt from torch.nn.functional import grid_sample as GridSample def _gaussian_kernel1d(kernel_size, sigma, device, dtype): radius = kernel_size // 2 x = torch.arange(-radius, radius + 1, device=device, dtype=dtype) kernel = torch.exp(-0.5 * (x / sigma) ** 2) kernel = kernel / kernel.sum() return kernel def apply_smoothing(grid, kernel_size=5, sigma=0.2): """ Apply Gaussian smoothing to parameter grids to ensure smooth deformations. This function smooths the adaptive sampling parameters to prevent abrupt changes in the sampling grid, which could cause artifacts or instability during training. Args: grid (torch.Tensor): Input parameter grid of shape (batch, h, w) Contains the raw adaptive sampling parameters kernel_size (int): Size of the Gaussian blur kernel (default: 5) Larger values create smoother deformations Returns: torch.Tensor: Smoothed grid of same shape as input (batch, h, w) """ if kernel_size <= 1: return grid grid_4d = grid.unsqueeze(1) kernel_1d = _gaussian_kernel1d(kernel_size, sigma, device=grid.device, dtype=grid.dtype) radius = kernel_size // 2 kernel_h = kernel_1d.view(1, 1, 1, -1) kernel_v = kernel_1d.view(1, 1, -1, 1) smoothed = F.conv2d(grid_4d, kernel_h, padding=(0, radius)) smoothed = F.conv2d(smoothed, kernel_v, padding=(radius, 0)) return smoothed.squeeze(1) def normalize_cumsum(grid): """ Create monotonic cumulative sum along the last dimension for grid generation. This function converts probability-like parameters into monotonic coordinates by computing cumulative sums. This ensures that the resulting grid coordinates are monotonically increasing, which is required for valid spatial transformations. Args: grid (torch.Tensor): Input parameter grid of shape (batch, h, w) The w-axis represents the grid direction where cumsum is applied Values should typically be positive (post-softmax) Returns: torch.Tensor: Monotonic grid of same shape (batch, h, w) Values increase monotonically along the last dimension """ monotonic_grid = torch.cumsum(grid, dim=-1) monotonic_grid = monotonic_grid return monotonic_grid def get_coarse_adaptive_grid(params_x, params_y): """ Generate a coarse adaptive (locally scaled) sampling grid from learnable parameters to perfrom local scaling of an image. This is the core function that converts raw learnable parameters into a valid sampling grid. The grid defines how to spatially transform input images/features by specifying new coordinate locations for each pixel. Args: params_x (torch.Tensor): X-axis sampling parameters of shape (batch, h, w) Raw learnable parameters for horizontal deformation params_y (torch.Tensor): Y-axis sampling parameters of shape (batch, h, w) Raw learnable parameters for vertical deformation Returns: torch.Tensor: Adaptive sampling grid of shape (batch, 2, h+1, w+1) dim=1 contains [x_coordinates, y_coordinates] Grid values are in range [0, 1] representing normalized coordinates """ params_x_smooth = F.softmax(3 * apply_smoothing(params_x), dim=-1) params_y_smooth = F.softmax(3 * apply_smoothing(params_y), dim=-1) params_x_smooth = torch.cat([torch.zeros_like(params_x_smooth[:, :, 0]).unsqueeze(2), params_x_smooth], dim=2) params_y_smooth = torch.cat([torch.zeros_like(params_y_smooth[:, :, 0]).unsqueeze(2), params_y_smooth], dim=2) monotonic_params_x = normalize_cumsum(params_x_smooth) monotonic_params_y = normalize_cumsum(params_y_smooth) return torch.stack([monotonic_params_x, torch.transpose(monotonic_params_y, -1, -2)], dim=1) def deform(params_x, params_y, images, *, resolution=None, mode='bilinear'): """ Apply local scaling to input images using learned parameters. This is the main forward transformation function that applies the learned local scaling to the input images or feature maps. It first deforms the regular grid of the image accoring to the local scaling parameters, and then use the deformed grid to sample the image. Args: params_x (torch.Tensor): X-axis deformation parameters of shape (batch, h, w) Raw learnable parameters for horizontal deformation params_y (torch.Tensor): Y-axis deformation parameters of shape (batch, h, w) Raw learnable parameters for vertical deformation images (torch.Tensor): Input images/features of shape (batch, channels, height, width) The spatial dimensions will be transformed according to params resolution (tuple, optional): Target resolution (height, width) for processing If None, uses original image resolution mode (str): Interpolation mode for grid sampling ('bilinear' or 'nearest') Default: 'bilinear' for smooth transformations Returns: torch.Tensor: Deformed images of same shape as input (batch, channels, height, width) Spatial content is rearranged according to learned parameters """ if resolution is None: resolution = images.shape[-2:] original_resolution = None else: original_resolution = images.shape[-2:] images = F.interpolate(images, size=(resolution[0], resolution[1]), mode='bilinear', align_corners=True, antialias=True) if params_x.shape[-2] == 1: expand_size = (params_x.shape[0], params_x.shape[-1]+1, params_x.shape[-1]) params_x = params_x.expand(expand_size) if params_y.shape[-2] == 1: expand_size = (params_y.shape[0], params_y.shape[-1]+1, params_y.shape[-1]) params_y = params_y.expand(expand_size) coarse_grid = get_coarse_adaptive_grid(params_x, params_y).to(images.device) denser_grid = F.interpolate(coarse_grid, size=(resolution[0], resolution[1]), mode='bilinear', align_corners=True).to(images.device) denser_grid = denser_grid * 2 - 1 sampled_images = GridSample(images, denser_grid.permute(0, 2, 3, 1), mode=mode) if mode == 'nearest' else \ grid_sample(images, denser_grid.permute(0, 2, 3, 1), mode='bilinear', align_corners=True) if original_resolution is not None: sampled_images = F.interpolate(sampled_images, size=(original_resolution[0], original_resolution[1]), mode='bilinear', align_corners=True, antialias=True) return sampled_images def inv_deform(params_x, params_y, images, resolution=None, mode='bilinear'): """ Apply inverse local scaling to reverse the effect of deform(). This function computes the inverse transformation of the local scaling. This is required for dense prediction tasks or inverse transforming the hidden features of the deep nets. Args: params_x (torch.Tensor): X-axis deformation parameters of shape (batch, h, w) Same parameters used in forward deform() function params_y (torch.Tensor): Y-axis deformation parameters of shape (batch, h, w) Same parameters used in forward deform() function images (torch.Tensor): Input images/features of shape (batch, channels, height, width) These should be the OUTPUT of a previous deform() operation resolution (tuple, optional): Target resolution (height, width) for processing If None, uses original image resolution mode (str): Interpolation mode for grid sampling ('bilinear' or 'nearest') Default: 'bilinear' for smooth transformations Returns: torch.Tensor: Inverse deformed images of same shape as input (batch, channels, height, width) Should approximately recover the original input to deform() function """ if resolution is None: resolution = images.shape[-2:] original_resolution = None else: original_resolution = images.shape[-2:] images = F.interpolate(images, size=(resolution[0], resolution[1]), mode='bilinear', align_corners=True, antialias=True) if params_x.shape[-2] == 1: expand_size = (params_x.shape[0], params_x.shape[-1]+1, params_x.shape[-1]) params_x = params_x.expand(expand_size) if params_y.shape[-2] == 1: expand_size = (params_y.shape[0], params_y.shape[-1]+1, params_y.shape[-1]) params_y = params_y.expand(expand_size) denser_grid = F.interpolate(get_coarse_adaptive_grid(params_x, params_y).to(images.device), size=(resolution[0],resolution[1]), mode='bilinear', align_corners=True).to(images.device) x_uniform = torch.linspace(0, 1, resolution[1], device=params_x.device).to(images.device) y_uniform = torch.linspace(0, 1, resolution[0], device=params_x.device).to(images.device) inverse_mesh_x = interpolate_nonuniform_to_uniform(denser_grid[:, 0, :, :], x_uniform) inverse_mesh_y = interpolate_nonuniform_to_uniform(denser_grid[:, 1, :, :].transpose(-1, -2), y_uniform) inverse_grid = torch.concat([inverse_mesh_x, inverse_mesh_y.transpose(-1, -2)], dim=1) * 2 - 1 sampled_images = GridSample(images, inverse_grid.permute(0, 2, 3, 1), mode=mode) if mode == 'nearest' else \ grid_sample(images, inverse_grid.permute(0, 2, 3, 1), mode='bilinear', align_corners=True) if original_resolution is not None: sampled_images = F.interpolate(sampled_images, size=(original_resolution[0], original_resolution[1]), mode='bilinear', align_corners=True, antialias=True) return sampled_images def rerrange_and_scale_tokens(phi_x, phi_y, tokens, inv_transform=False, cls_token=None, num_prefix_tokens=None, mode='bilinear', defom_resolution=None): """ Apply local scaling to Vision Transformer tokens. It handles the conversion between token sequences and spatial feature maps, applies deformation, and converts back. Args: phi_x (torch.Tensor): X-axis deformation parameters of shape (batch, h, w) phi_y (torch.Tensor): Y-axis deformation parameters of shape (batch, h, w) tokens (torch.Tensor): ViT tokens of shape (batch, num_tokens, embed_dim) where num_tokens = num_prefix_tokens + h*w inv_transform (bool): If True, applies inverse deformation. Default: False cls_token (torch.Tensor, optional): Classification token(s) to preserve Will be extracted and reattached num_prefix_tokens (int, optional): Number of special tokens (CLS, etc.) at the beginning If None and cls_token is not None, assumes 1 token mode (str): Interpolation mode for deformation ('bilinear' or 'nearest') defom_resolution (tuple, optional): Target resolution for deformation processing Returns: torch.Tensor: Transformed tokens of same shape as input (batch, num_tokens, embed_dim) Spatial tokens are rearranged according to learned parameters """ if cls_token is not None: if num_prefix_tokens is None: cls_token = tokens[:, 0, :] hidden_states = tokens[:, 1:, :] else: cls_token = tokens[:, :num_prefix_tokens, :] hidden_states = tokens[:, num_prefix_tokens:, :] else: hidden_states = tokens resolution = hidden_states.shape[-2] h = int(resolution**0.5) hidden_states = hidden_states.transpose(1, 2).reshape(-1, tokens.shape[-1], h, h) if inv_transform: hidden_states = inv_deform(phi_x, phi_y, hidden_states, mode=mode, resolution=defom_resolution) else: hidden_states = deform(phi_x, phi_y, hidden_states, mode=mode, resolution=defom_resolution) hidden_states = hidden_states.reshape(-1, hidden_states.shape[1], h * h).transpose(1, 2) if cls_token is not None: if num_prefix_tokens is None: cls_token = cls_token.unsqueeze(1) return torch.cat([cls_token, hidden_states], dim=1) return hidden_states ########### ### General utility Functions for Grid Interpolation and Sampling ########### def interpolate_nonuniform_to_uniform(x_nonuniform, x_uniform): B, N = x_nonuniform.shape[:2] M = x_uniform.shape[0] x_nonuniform = x_nonuniform.unsqueeze(1).contiguous() x_uniform = x_uniform.unsqueeze(0).unsqueeze( 0).unsqueeze(0).expand(B, 1, N, M).contiguous() y_nonuniform = x_uniform.clone() # Find indices of intervals idxs = torch.searchsorted(x_nonuniform, x_uniform) # Clip indices to be within bounds idxs = torch.clamp(idxs, 0, M - 1) # Gather left and right values for interpolation idxs_l = torch.clamp(idxs - 1, 0, M - 1) x_left = torch.gather(x_nonuniform, -1, idxs_l) x_right = torch.gather(x_nonuniform, -1, idxs) y_left = torch.gather(y_nonuniform, -1, idxs_l) y_right = torch.gather(y_nonuniform, -1, idxs) # Linear interpolation denom = (x_right - x_left).clamp(min=1e-8) weight_right = (x_uniform - x_left) / denom weight_left = 1 - weight_right return weight_left * y_left + weight_right * y_right def grid_sample(image, grid, **kwargs): N, C, IH, IW = image.shape _, H, W, _ = grid.shape ix = grid[..., 0] iy = grid[..., 1] ix = ((ix + 1) / 2) * (IW - 1) iy = ((iy + 1) / 2) * (IH - 1) with torch.no_grad(): ix_nw = torch.floor(ix) iy_nw = torch.floor(iy) ix_ne = ix_nw + 1 iy_ne = iy_nw ix_sw = ix_nw iy_sw = iy_nw + 1 ix_se = ix_nw + 1 iy_se = iy_nw + 1 nw = (ix_se - ix) * (iy_se - iy) ne = (ix - ix_sw) * (iy_sw - iy) sw = (ix_ne - ix) * (iy - iy_ne) se = (ix - ix_nw) * (iy - iy_nw) with torch.no_grad(): torch.clamp(ix_nw, 0, IW - 1, out=ix_nw) torch.clamp(iy_nw, 0, IH - 1, out=iy_nw) torch.clamp(ix_ne, 0, IW - 1, out=ix_ne) torch.clamp(iy_ne, 0, IH - 1, out=iy_ne) torch.clamp(ix_sw, 0, IW - 1, out=ix_sw) torch.clamp(iy_sw, 0, IH - 1, out=iy_sw) torch.clamp(ix_se, 0, IW - 1, out=ix_se) torch.clamp(iy_se, 0, IH - 1, out=iy_se) image = image.reshape(N, C, IH * IW) nw_val = torch.gather( image, 2, (iy_nw * IW + ix_nw).long().view( N, 1, H * W).repeat( 1, C, 1)) ne_val = torch.gather( image, 2, (iy_ne * IW + ix_ne).long().view( N, 1, H * W).repeat( 1, C, 1)) sw_val = torch.gather( image, 2, (iy_sw * IW + ix_sw).long().view( N, 1, H * W).repeat( 1, C, 1)) se_val = torch.gather( image, 2, (iy_se * IW + ix_se).long().view( N, 1, H * W).repeat( 1, C, 1)) out_val = (nw_val.view(N, C, H, W) * nw.view(N, 1, H, W) + ne_val.view(N, C, H, W) * ne.view(N, 1, H, W) + sw_val.view(N, C, H, W) * sw.view(N, 1, H, W) + se_val.view(N, C, H, W) * se.view(N, 1, H, W)) return out_val